From 7e2a48e44db450ddfa804b2aafa6702ef509e92d Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 08:30:49 +0100 Subject: [PATCH 01/19] chore: run tests one at a time to avoid an NVIDIA driver deadlock --- .cargo/config.toml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..c2df13c5 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,6 @@ +[env] +# The device tests each build their own VkDevice. The NVIDIA driver (seen on +# 615.78.08) can deadlock when one thread destroys a device while another +# creates objects on a different one, so run tests one at a time. An explicit +# `--test-threads` still overrides this. +RUST_TEST_THREADS = "1" From da332ca2318188e484c21e6caea6a6a608e31c49 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 08:30:50 +0100 Subject: [PATCH 02/19] docs: add the encode input design --- docs/encode-input.md | 211 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 docs/encode-input.md diff --git a/docs/encode-input.md b/docs/encode-input.md new file mode 100644 index 00000000..4f17bf8f --- /dev/null +++ b/docs/encode-input.md @@ -0,0 +1,211 @@ +# Encode input: design + +This document describes how frames get into the encoder. It replaces the +`EncodeConfig::with_rgb_input` / `ColorConverterConfig::rgb_encode_input` +pair, where the caller chose the conversion path and called a different +method for each. + +## Goals + +1. The caller describes the source one time. Pixelforge selects the path. +2. One encode entry point for all paths. +3. No CPU wait in the hot path. The GPU orders all work. +4. The caller knows when it can reuse a source image. +5. A DMA-BUF source goes to the encoder with no pixelforge copy, when the + device can read it directly. +6. Colour changes go through one method, which updates the path that runs. + +## The source description + +`EncodeConfig` gets a `Source`: + +- `Source::Yuv` (the default): frames are already YUV, in the encoder's + pixel format and bit depth. This is the old behaviour. +- `Source::Rgb(RgbSource { format, color, reference_white_nits })`: frames are + RGB. `format` is the pixel layout (`InputFormat`) and `color` is what the + pixels mean (`ColorSpec`). + +The stream's colour stays `EncodeConfig::color_description`. For an RGB +source, pixelforge derives the conversion target from it: BT.709 primaries, +transfer and matrix are `ColorSpec::Srgb`; BT.2020 and PQ are +`ColorSpec::Bt2020Pq`. Any other description is an error for an RGB source, +because no conversion produces it. + +### Path selection + +`Encoder::new` selects one `Conversion`: + +| Conversion | When | What runs | +|---|---|---| +| `None` | YUV source | Nothing. | +| `Encoder` | RGB source, `VK_VALVE_video_encode_rgb_conversion`, source space equals target space, 4:2:0 at the input's bit depth, and the driver accepts the matrix, range and format | The encoder converts. The input image is RGB. | +| `Shader` | Any other RGB source the shader can convert | The colour converter writes into the encoder's YUV input image. | + +`EncodeConfig::with_conversion(ConversionPreference)` can force `Encoder` or +`Shader`. `Auto` (the default) prefers `Encoder`. A forced path that cannot +work is an error from `Encoder::new`. `Encoder::conversion()` tells which path +was selected. + +### Colour changes + +`Encoder::set_color_description` is the one way to change colour: + +- `None` and `Shader`: the session parameters are made again (as before), and + the shader gets the new target and range. +- `Encoder`: the matrix and range are fixed in the video session, so the codec + encoder is made again, and the path is selected again. Packet timestamps + continue from where they were. The next frame is an IDR in all cases. + +This removes the bug where the label changed and the hardware conversion did +not. + +## Submitting a frame + +```rust +let future = encoder.submit(Frame::image(image, layout).wait(&points))?; +let future = encoder.submit(Frame::external(&imported).wait_sync_file(fence))?; +``` + +`Encoder::encode(image)` and `Encoder::encode_after(image, layout, waits)` +stay, as short forms of `submit`. + +A `Frame` names its source: + +- `Frame::image(image, layout)`: a `vk::Image` the caller owns. If it is + `Encoder::input_image()`, the encoder reads it in place. Otherwise + pixelforge copies it (`None`, `Encoder`) or converts it (`Shader`) into the + input image. +- `Frame::external(&ExternalImage)`: a DMA-BUF that pixelforge imported or + allocated. See below. + +### No CPU waits + +The copy from a caller image into the input image signals a timeline +semaphore, and the encode waits on it on the GPU. The copy no longer waits on +a fence. Each pipeline slot has its own upload command buffer. A slot is used +again only after its encode is complete, and that encode waited for the copy, +so the reuse is safe. + +The only CPU wait left is backpressure: when every pipeline slot is in +flight, `submit` waits for the oldest one. + +### Release point + +`EncodeFuture::source_released()` returns the `TimelinePoint` after which the +source can be written again, or given back to its producer: + +| Source | Released by | +|---|---| +| `Encoder::input_image()` | the encode | +| caller image, `None` / `Encoder` | the copy | +| caller image, `Shader` | the conversion | +| `ExternalImage`, read directly | the encode | +| `ExternalImage`, copied | the copy | + +Before this change, `encode()` returned only after the copy was done. Now it +returns at once, so a caller that writes the source again must wait on this +point first. + +The semaphores behind these points are retired to the context when the +encoder is dropped, as the converter already does. A wait on a point that the +caller already submitted stays valid. + +## DMA-BUF input (feature `dmabuf`) + +### Which buffers the encoder reads directly + +`Encoder::input_formats()` returns the DRM format (fourcc) that the encoder +wants, and for each DRM modifier the device knows: + +- `zero_copy`: the encoder's path reads it with no pixelforge copy. For + `None` and `Encoder` that means `VIDEO_ENCODE_SRC` usage with the + encoder's video profile. For `Shader` it means `SAMPLED` usage, because the + shader reads the source directly. +- `importable` and `exportable`: whether a DMA-BUF with this modifier can be + imported, or allocated by pixelforge and exported. + +A caller negotiates with its producer: it offers the `zero_copy` modifiers +first, then the others. + +### Allocating and exporting (consumer-allocated buffers) + +`Encoder::allocate_input(modifiers)` makes an `ExternalImage` with one of the +given modifiers. It prefers one the encoder reads directly; if there is none, +it takes one it can copy from. +`ExternalImage::export()` gives the DMA-BUF (fd, modifier, plane offsets and +strides). This is for `ext-image-copy-capture`, where the client supplies the +buffer and the compositor writes into it. + +The protocol requires the buffer to be exactly the capture size. Thus an +exported image has the display size, not the aligned coded size. For H.264 +and H.265 the encoder then gives the display size as the source picture +extent. The spec allows this, and the stream headers already crop to the +display size. AV1 already uses the display size. + +### Importing (producer-allocated buffers) + +`Encoder::import(&DmaBuf)` makes an `ExternalImage` from a producer's buffer. +It is checked one time, at import: + +- If the modifier is `zero_copy`, the encoder reads it directly. +- Otherwise, if it can be imported for transfer, `submit` copies it. +- Otherwise the import fails. + +Only single-buffer DMA-BUFs are supported: all planes in one buffer, at +different offsets. Disjoint planes are an error. + +### Ownership + +A DMA-BUF is shared with another process. On each frame, pixelforge acquires +it from `VK_QUEUE_FAMILY_FOREIGN_EXT` (or `EXTERNAL` without +`VK_EXT_queue_family_foreign`) in the submission that reads it, and releases +it back in the same submission. The first acquire uses `UNDEFINED` as the old +layout, as the converter already does for a first-time import, and later ones +use `GENERAL`. + +### Lifetime + +An `ExternalImage` remembers the release point of its last frame. Its `Drop` +waits for that point before it destroys the image. + +## sync_file fences (feature `dmabuf`) + +- `Frame::wait_sync_file(fd)`: the frame's first submission waits for the + fence. Pixelforge imports it into a binary semaphore (temporary import), + which takes ownership of the fd. +- `Frame::signal_sync_file()`: the submission that releases the source also + signals a binary semaphore, and `EncodeFuture::take_release_sync_file()` + gives it as a sync_file. The caller can attach it to the DMA-BUF + (`DMA_BUF_IOCTL_IMPORT_SYNC_FILE`) or give it to the producer. + +Each pipeline slot has one acquire and one release semaphore. They are used +again only after the slot's encode is complete. + +### Driver quirks + +NVIDIA's driver (615.78.08) reports more DMA-BUF support than works, and +both failures give a wrong picture, not an error. `quirks_allow` in +`src/encoder/external.rs` holds the driver back: + +- An image made with a video profile reads a block-linear buffer as black. + Imported without the profile, the same buffer reads correctly. So the + encode never reads a DMA-BUF directly on NVIDIA; it is copied. +- A two-plane YUV buffer with a block-linear modifier loses its chroma plane + between two Vulkan images, with no pixelforge code involved. Only LINEAR is + offered for YUV on NVIDIA. Single-plane RGB is not affected. + +The same driver lays a LINEAR image out by its usage and ignores the plane +layout an importer gives. So an image pixelforge allocates for export also +gets `TRANSFER_DST`, the usage a producer needs to write into it. + +`PIXELFORGE_NO_DMABUF_QUIRKS=1` turns the quirks off, to try a newer driver. +The direct read path can only be checked on a device without them (RADV). + +## Not in this change + +- Selecting the GPU by DRM render node (PR #23). DMA-BUF import and export + need the encoder on the producer's GPU. Today a caller can make sure of + that with `build_from_existing_encode`. +- Implicit sync done by pixelforge itself (the DMA-BUF sync_file ioctls). The + caller does this with the fds above. +- Disjoint multi-plane DMA-BUFs. From 67884d92939d66065864cea244eb688d9d7b6e9b Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 08:30:50 +0100 Subject: [PATCH 03/19] feat(encoder)!: describe the source once and submit frames without CPU waits --- AGENTS.md | 36 +- CHANGELOG.md | 37 +- README.md | 163 +++--- src/converter/mod.rs | 159 ++---- src/encoder/api.rs | 460 ++++++++++++++++ src/encoder/av1/init.rs | 51 +- src/encoder/av1/record.rs | 5 +- src/encoder/codec.rs | 305 ++++++++-- src/encoder/external.rs | 1038 +++++++++++++++++++++++++++++++++++ src/encoder/frame.rs | 120 ++++ src/encoder/h264/init.rs | 51 +- src/encoder/h264/record.rs | 7 +- src/encoder/h265/init.rs | 49 +- src/encoder/h265/record.rs | 7 +- src/encoder/input.rs | 152 +++++ src/encoder/mod.rs | 254 ++------- src/encoder/pipeline.rs | 87 ++- src/encoder/profile.rs | 91 ++- src/encoder/resources.rs | 218 +++----- src/encoder/rgb.rs | 8 +- src/encoder/source.rs | 296 ++++++++++ src/encoder/support.rs | 2 +- src/lib.rs | 184 ++++--- src/video.rs | 2 +- src/vulkan.rs | 105 +++- src/vulkan/encode_device.rs | 8 +- tests/dmabuf.rs | 663 ++++++++++++++++++++++ tests/rgb_encode.rs | 179 ++++-- 28 files changed, 3861 insertions(+), 876 deletions(-) create mode 100644 src/encoder/api.rs create mode 100644 src/encoder/external.rs create mode 100644 src/encoder/frame.rs create mode 100644 src/encoder/input.rs create mode 100644 src/encoder/source.rs create mode 100644 tests/dmabuf.rs diff --git a/AGENTS.md b/AGENTS.md index 0fae7fd7..441b85ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,16 +117,32 @@ checks the output against a context pixelforge created itself. It needs no ffmpeg. Anything that reaches a core 1.2 or 1.3 entry point panics there, so run it after touching submission or barrier code. -The RGB input path (`EncodeConfig::with_rgb_input`) needs -`VK_VALVE_video_encode_rgb_conversion`, which today only RADV has. -`tests/rgb_encode.rs` compares it against the colour converter, checks that -`supports_rgb_encode_input` agrees with encoder creation, and skips what the -device cannot do, so a green run elsewhere says little about it. Under -validation it draws two messages that are the layer's, not pixelforge's: the -layer does not model that extension, so it rejects the RGB profile struct in a -query pool's profile chain (the spec requires that chain to match the -session's) and checks the RGB input view against the plain format features -rather than the video profile's. +An RGB source (`Source::Rgb`) is converted by the encoder itself +(`Conversion::Encoder`) only with `VK_VALVE_video_encode_rgb_conversion`, +which today only RADV has. `tests/rgb_encode.rs` checks the shader path +everywhere (byte-identical to converting by hand), and on RADV compares the +encoder's conversion against it and checks that `supports_rgb_encode_input` +agrees with encoder creation, so a green run elsewhere says little about the +encoder path. Under validation that path draws two messages that are the +layer's, not pixelforge's: the layer does not model that extension, so it +rejects the RGB profile struct in a query pool's profile chain (the spec +requires that chain to match the session's) and checks the RGB input view +against the plain format features rather than the video profile's. + +DMA-BUF input (`--features dmabuf`, `tests/dmabuf.rs`) has three ways to read a +buffer: the encode itself, a copy into the input image, and the colour +converter's shader. The tests play the producer by importing the same buffer +into an image of their own. On NVIDIA (615.78.08) the driver reports more than +works, and `quirks_allow` in `src/encoder/external.rs` holds it back: the encode +never reads a DMA-BUF directly (it reads black), and two-plane YUV buffers are +LINEAR only (block-linear ones lose their chroma plane). So on NVIDIA the +direct read and the unaligned-size test are skipped, and only RADV can check +them. `PIXELFORGE_NO_DMABUF_QUIRKS=1` turns the quirks off, which makes two of +those tests fail on that driver. + +Tests run one at a time (`RUST_TEST_THREADS=1` in `.cargo/config.toml`): the +NVIDIA driver can deadlock when one thread destroys a device while another +creates objects on a different one. Make sure there are no Vulkan validation layer errors during execution. Enable them with `PIXELFORGE_VALIDATION=1`; the layer's messages are routed through diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cbe27b2..2ad7701d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,20 +9,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Add RGB input converted by the encoder itself through - `VK_VALVE_video_encode_rgb_conversion`: `EncodeConfig::with_rgb_input`. -- Add `VideoContext::supports_rgb_encode_input` and - `ColorConverterConfig::rgb_encode_input`, which say when a conversion can be - left to the encoder. -- Add `VideoContext::has_video_encode_rgb_conversion` and +- Add RGB input: `EncodeConfig::with_source(Source::Rgb(..))`. The encoder + converts RGB frames to the YUV its colour description describes, through + `VK_VALVE_video_encode_rgb_conversion` where the device can and the colour + converter's shader otherwise (`Encoder::conversion`, + `EncodeConfig::with_conversion`). +- Add `Encoder::submit` and `Frame`: one way to hand over a frame, whatever + the conversion, with its GPU waits. +- Add `EncodeFuture::source_released`: the point after which a frame's source + may be written again. +- Add DMA-BUF input (feature `dmabuf`): `Encoder::input_formats`, + `Encoder::import`, `Encoder::allocate_input`, `ExternalImage` and + `Frame::external`. The encoder reads a buffer directly where the device + can, and copies it otherwise. +- Add sync_file fences (feature `dmabuf`): `Frame::wait_sync_file`, + `Frame::signal_sync_file` and `EncodeFuture::take_release_sync_file`. +- Add `VideoContext::supports_rgb_encode_input`, + `VideoContext::has_video_encode_rgb_conversion` and `DeviceFeatures::video_encode_rgb_conversion`. ### Changed +- The copy of a frame into the encoder's input image no longer waits on the + CPU: the encode waits for it on the GPU. `Encoder::encode_after` and + `Encoder::submit` return before the source is read; `Encoder::encode` still + returns once the source can be written again. - `Encoder::encode_after` takes the source image's layout, as `ColorConverter::convert_async` does: for `UNDEFINED`, a first-time external-memory import, the copy into the input image acquires it from `VK_QUEUE_FAMILY_EXTERNAL`. +- `Encoder::set_color_description` also updates the conversion of an RGB + source, and makes the encoder again where the encoder converts. Packet + timestamps carry on. +- The `dmabuf` feature enables its extensions only where the device has them, + rather than failing device creation where it does not. + +### Fixed + +- The source image barrier of the input copy now waits for earlier work on + the image, rather than starting at `TOP_OF_PIPE`. ## [0.10.0] - 19-10-2026 diff --git a/README.md b/README.md index 60a1dc71..67151764 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,10 @@ H.265 and AV1 encode, and H.264 decode. - **Asynchronous pipelines**: both directions submit without waiting. Encoding hands back an [`EncodeFuture`]; decoding delivers frames through a [`DecodeSource`] as the GPU finishes with them. -- **GPU color conversion**: RGB/BGR → YUV via Vulkan compute shaders (BT.709, BT.2020, sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ), - or in the encoder itself where `VK_VALVE_video_encode_rgb_conversion` is available. +- **RGB input**: describe the source once and hand the encoder RGB frames. + It converts them in the encoder itself where `VK_VALVE_video_encode_rgb_conversion` + allows, and with a compute shader otherwise (BT.709, BT.2020, + sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ). - **Shared devices**: encode and decode on a Vulkan device your application created, from Vulkan 1.1 up, on queues you choose, ordered against your own work with timeline semaphores. @@ -24,7 +26,9 @@ H.265 and AV1 encode, and H.264 decode. - **Flexible configuration**: Rate control (CBR, VBR, CQP), quality levels, GOP settings. - **Multiple input formats**: BGRx, RGBx, BGRA, RGBA, ABGR2101010 (10-bit packed), RGBA16F (FP16). - **Utility helpers**: [`InputImage`] for easy YUV data upload to GPU. -- **Optional DMA-BUF support**: Zero-copy image import from external processes (Linux only). +- **Optional DMA-BUF input**: import a producer's buffers, or allocate buffers + for it, read with no copy where the driver allows, with sync_file fences + (Linux only). > **Note**: B-frame support is not yet implemented. Setting `b_frame_count > 0` will panic. @@ -58,7 +62,7 @@ pixelforge = "0.1" | Feature | Description | |---------|-------------| -| `dmabuf` | Enable DMA-BUF support for zero-copy image import from external processes (Linux only). Adds Vulkan extensions: `VK_KHR_external_memory`, `VK_KHR_external_memory_fd`, `VK_EXT_external_memory_dma_buf`, `VK_EXT_image_drm_format_modifier`. | +| `dmabuf` | DMA-BUF input and sync_file fences (Linux only). Enables, where the device has them, `VK_KHR_external_memory_fd`, `VK_EXT_external_memory_dma_buf`, `VK_EXT_image_drm_format_modifier`, `VK_KHR_external_semaphore_fd` and `VK_EXT_queue_family_foreign`. | To enable DMA-BUF support: @@ -187,11 +191,48 @@ bounds how many can be outstanding before the decoder starts copying pictures out instead of handing over its own. Reading a frame back to the CPU is the consumer's job; `examples/common` shows one way. -### Color Conversion (RGB → YUV) +### RGB input -PixelForge includes a GPU compute shader for converting RGB input to YUV -output. The source color describes the input. The target color describes the stream. -Both are a [`ColorSpec`]. +An encoder takes RGB frames once it is told what they are, with a +[`Source`]. It then converts them to the YUV that the stream's +[`ColorDescription`] describes, and that description also picks the target: +BT.709 for SDR, BT.2020 with PQ for HDR10. The source and the stream are +described once each, so the conversion and the stream's label cannot +disagree, and a colour change through [`Encoder::set_color_description`] +changes both. + +```rust +use pixelforge::{ + Codec, ColorDescription, ColorSpec, EncodeConfig, Encoder, InputFormat, RgbSource, + Source, VideoContextBuilder, +}; + +let context = VideoContextBuilder::new() + .require_encode(Codec::H265) + .build()?; + +// SDR desktop content, encoded as HDR10 for an HDR streaming session. +let config = EncodeConfig::h265(1920, 1080) + .with_bit_depth(pixelforge::EncodeBitDepth::Ten) + .with_source(Source::Rgb(RgbSource::new(InputFormat::BGRx, ColorSpec::Srgb))) + .with_color_description(ColorDescription::bt2020_pq().with_full_range(true)); +let mut encoder = Encoder::new(context, config)?; +println!("converting with {:?}", encoder.conversion()); +// let packet = encoder.encode(rgb_image)?; +``` + +[`Encoder::new`] picks the [`Conversion`]: + +- [`Conversion::Encoder`] where the conversion is nothing but the YUV + matrix, as for ordinary SDR content, and the device has + `VK_VALVE_video_encode_rgb_conversion` with that matrix and range. No + shader runs at all. +- [`Conversion::Shader`] otherwise: the colour converter's compute shader + writes the encoder's YUV input. + +[`EncodeConfig::with_conversion`] can force one, for tests and comparisons. + +The shader's colour spaces are [`ColorSpec`]s: | `ColorSpec` | | Can be a target | |-------------|-|-----------------| @@ -204,71 +245,27 @@ The linear spaces cannot be a target, because a video file has no way to record that it holds linear light. Any source can be converted to `Bt2020Pq`. Only `Srgb` can be converted to `Srgb`; the others would need tone mapping or a gamma curve applied, which the shader does not do. -[`ColorConverter::new`] rejects the combinations it cannot do. +[`Encoder::new`] rejects the combinations nothing can do. Encoding to HDR needs to know how bright the source's white is, since HDR carries real brightness values and SDR does not. Each space has a sensible default, see [`ColorSpec::reference_white_nits`], overridable with -[`ColorConverterConfig::with_reference_white_nits`]. +[`RgbSource::with_reference_white_nits`]. Supported input formats: BGRx, RGBx, BGRA, RGBA, ABGR2101010 (10-bit packed), RGBA16F (FP16). -Supported output formats: NV12 (8-bit), I420 (8-bit), YUV444 (8-bit), P010 (10-bit), YUV444P10 (10-bit). - -Pass [`ColorConverter::color_description`] to the encoder rather than -describing the colours a second time by hand. The two have to agree: if the -converter writes full-range pixels and the stream says limited, players -stretch the range again and the picture comes out wrong. -```rust -use pixelforge::{ - Codec, ColorConverter, ColorConverterConfig, ColorRange, ColorSpec, EncodeConfig, - Encoder, InputFormat, OutputFormat, VideoContextBuilder, -}; - -let context = VideoContextBuilder::new() - .app_name("Color Converter") - .require_encode(Codec::H265) - .build()?; - -// SDR desktop content, encoded as HDR10 for an HDR streaming session. -let config = ColorConverterConfig::new( - 1920, - 1080, - InputFormat::BGRx, - OutputFormat::P010, - ColorSpec::Srgb, - ColorSpec::Bt2020Pq, - ColorRange::Full, -); - -let mut converter = ColorConverter::new(context.clone(), config)?; - -// The encoder declares exactly what the shader wrote. -let encode_config = EncodeConfig::h265(1920, 1080) - .with_color_description(converter.color_description()); - -let mut encoder = Encoder::new(context, encode_config)?; -// converter.convert(input_image, layout, encoder.input_image())?; -``` +[`ColorConverter`] is the same shader on its own, for converting without +encoding. -Where the conversion is nothing but the YUV matrix, as for ordinary SDR -content, some encoders can do it themselves and the shader need not run at -all. [`ColorConverterConfig::rgb_encode_input`] says whether that works for -a given conversion and codec on this device. When it does, build the -encoder with [`EncodeConfig::with_rgb_input`] and hand it the RGB frames, -passing `UNDEFINED` to [`Encoder::encode_after`] for first-time DMA-BUF -imports. +### Submitting frames -```rust -let encode_config = EncodeConfig::h264(1920, 1080) - .with_color_description(config.color_description().expect("SDR is encodable")); -let encoder = match config.rgb_encode_input(&context, Codec::H264) { - // The encoder converts: give it the RGB images as they are. - Some(format) => Encoder::new(context, encode_config.with_rgb_input(format))?, - // The shader converts into the encoder's input image, as above. - None => Encoder::new(context, encode_config)?, -}; -``` +[`Encoder::submit`] takes a [`Frame`]: where the pixels are, and what has +to happen on the GPU before they can be read. It does not wait on the CPU, +except when every pipeline slot is in flight. The source is read by a copy, +the shader, or the encode itself, after the call returns, so write it again +only after [`EncodeFuture::source_released`]. [`Encoder::encode`] is the +short form that also waits for that point, which keeps the simple loop of +writing one image and encoding it correct. ### Encoding on your own device @@ -284,25 +281,35 @@ may not be submitted to from two threads at once, so give pixelforge queues the application does not use, with [`VideoContextBuilder::with_encode_queue`] and friends. -[`ColorConverter::convert_async`] and [`Encoder::encode_after`] then order -each frame on the GPU: the application's submission signals a -[`TimelinePoint`], the conversion waits for it and signals its own, and the -encode waits for that. This stays off the CPU as long as the encoder's own -[`Encoder::input_image`] is what gets encoded, as below: any other image is -copied into it first, and that copy is waited for on the CPU. +Each frame is then ordered on the GPU: the application's submission signals +a [`TimelinePoint`], the encoder's first submission for the frame waits for +it, and the application waits for [`EncodeFuture::source_released`] before +it draws into the image again. ```rust // `rendered` reaches `frame_number` once the application has drawn `image`. -let converted = converter.convert_async( - image, - vk::ImageLayout::GENERAL, - encoder.input_image(), - &[TimelinePoint::new(rendered, frame_number)], -)?; -let packet = encoder.encode_after(encoder.input_image(), vk::ImageLayout::GENERAL, &[converted])?; +let ready = [TimelinePoint::new(rendered, frame_number)]; +let packet = encoder.submit(Frame::image(image, vk::ImageLayout::GENERAL).wait(&ready))?; +// Wait on this before drawing into `image` again. +let reusable = packet.source_released(); ``` -## Benchmarking +### DMA-BUF input + +With the `dmabuf` feature, an encoder reads DMA-BUFs from another process. +`Encoder::input_formats` gives the DRM format it wants and, per DRM +modifier, whether it reads such a buffer with no copy. Offer those +modifiers to the producer first. `Encoder::import` takes a producer's +buffer; `Encoder::allocate_input` makes one for a producer that writes +into buffers the consumer supplies, such as `ext-image-copy-capture`. +Either is checked once and gives an `ExternalImage`, which +`Frame::external` submits. A buffer the encoder cannot read directly is +copied on the GPU instead. + +The producer's write fence and the encoder's read fence can travel as +sync_files: `Frame::wait_sync_file` and `Frame::signal_sync_file`. + +//! # Benchmarking Run the encode latency benchmark with: diff --git a/src/converter/mod.rs b/src/converter/mod.rs index b543400d..91bd88c1 100644 --- a/src/converter/mod.rs +++ b/src/converter/mod.rs @@ -8,7 +8,8 @@ mod pipeline; -use crate::encoder::{Codec, ColorDescription}; +use crate::encoder::ColorDescription; +use crate::encoder::input::SourceOwnership; use crate::error::{PixelForgeError, Result}; use crate::sync::TimelinePoint; use crate::video::TimelineChain; @@ -325,43 +326,11 @@ impl ColorConverterConfig { ) } - /// The RGB input to give a `codec` encoder instead of running this - /// conversion, when the encoder can do it itself. - /// - /// The hardware applies only the YUV matrix, so this is `Some` only for a - /// conversion that is nothing more: source and target are the same space, - /// so no transfer function or gamut change is involved, and the output is - /// 4:2:0 at the input's bit depth, NV12 from 8-bit RGB or P010 from - /// `ABGR2101010`. The device is then asked, with - /// [`VideoContext::supports_rgb_encode_input`], whether it does that - /// matrix and range for `codec` and takes this format. - /// - /// When `Some`, build the encoder with - /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input) - /// and this format, and with [`Self::color_description`], then hand it the - /// RGB images directly and skip the converter. - pub fn rgb_encode_input(&self, context: &VideoContext, codec: Codec) -> Option { - if self.source != self.target { - return None; - } - let shape_matches = match self.output_format { - OutputFormat::NV12 => matches!( - self.input_format, - InputFormat::BGRx | InputFormat::BGRA | InputFormat::RGBx | InputFormat::RGBA - ), - OutputFormat::P010 => self.input_format == InputFormat::ABGR2101010, - _ => false, - }; - let desc = self.color_description()?; - (shape_matches && context.supports_rgb_encode_input(codec, self.input_format, &desc)) - .then_some(self.input_format) - } - /// Whether the shader can get from this source to this target. /// /// Anything that would need tone mapping or a forward gamma encode is not /// implemented, and is rejected rather than silently passed through. - fn conversion_supported(&self) -> bool { + pub(crate) fn conversion_supported(&self) -> bool { match self.target { // Anything can be converted to HDR. ColorSpec::Bt2020Pq => true, @@ -373,7 +342,7 @@ impl ColorConverterConfig { } /// Error explaining why [`Self::conversion_supported`] said no. - fn unsupported_conversion(&self) -> PixelForgeError { + pub(crate) fn unsupported_conversion(&self) -> PixelForgeError { let reason = if !self.target.is_encodable() { "video cannot be encoded in a linear space, so it can only be a source" } else { @@ -832,6 +801,27 @@ impl ColorConverter { src_layout: vk::ImageLayout, target_image: vk::Image, wait: &[TimelinePoint], + ) -> Result { + let waits: Vec> = + wait.iter().map(TimelinePoint::wait_info).collect(); + self.convert_with( + src_image, + SourceOwnership::caller_image(src_layout), + target_image, + &waits, + &[], + ) + } + + /// [`Self::convert_async`] with the source's ownership transfers spelled + /// out, and extra semaphores to wait on and signal. + pub(crate) fn convert_with( + &mut self, + src_image: vk::Image, + ownership: SourceOwnership, + target_image: vk::Image, + waits: &[vk::SemaphoreSubmitInfo<'static>], + extra_signals: &[vk::SemaphoreSubmitInfo<'static>], ) -> Result { // Before anything is reused, and before the previous view goes. self.wait_idle()?; @@ -854,47 +844,17 @@ impl ColorConverter { // --- Phase 1: Transition source image for shader read --- - // For external memory (DMA-BUF) imports with EXCLUSIVE sharing, - // the first use must include a queue family acquire operation. - // Set srcQueueFamilyIndex to EXTERNAL (ownership is foreign) - // and dstQueueFamilyIndex to the encoder's compute queue family. - let needs_acquire = src_layout == vk::ImageLayout::UNDEFINED; - let src_barrier = vk::ImageMemoryBarrier::default() - .old_layout(src_layout) - .new_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) - .src_queue_family_index(if needs_acquire { - vk::QUEUE_FAMILY_EXTERNAL - } else { - vk::QUEUE_FAMILY_IGNORED - }) - .dst_queue_family_index(if needs_acquire { - self.context.compute_queue_family() - } else { - vk::QUEUE_FAMILY_IGNORED - }) - .image(src_image) - .subresource_range(vk::ImageSubresourceRange { - aspect_mask: vk::ImageAspectFlags::COLOR, - base_mip_level: 0, - level_count: 1, - base_array_layer: 0, - layer_count: 1, - }) - .src_access_mask(if needs_acquire { - vk::AccessFlags::empty() - } else { - vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE - }) - .dst_access_mask(vk::AccessFlags::SHADER_READ); - - device.cmd_pipeline_barrier( + let compute_family = self.context.compute_queue_family(); + let src_barrier = [ownership.acquire_barrier( + src_image, + compute_family, + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + vk::PipelineStageFlags2::COMPUTE_SHADER, + vk::AccessFlags2::SHADER_READ, + )]; + self.context.sync2().cmd_pipeline_barrier2( self.command_buffer, - vk::PipelineStageFlags::ALL_COMMANDS, - vk::PipelineStageFlags::COMPUTE_SHADER, - vk::DependencyFlags::empty(), - &[], - &[], - &[src_barrier], + &vk::DependencyInfo::default().image_memory_barriers(&src_barrier), ); // --- Phase 2: Run compute shader (reads source image directly) --- @@ -1064,31 +1024,27 @@ impl ColorConverter { layer_count: 1, }); - // Transition source image back to GENERAL for reuse. - let src_barrier_back = vk::ImageMemoryBarrier::default() - .old_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) - .new_layout(vk::ImageLayout::GENERAL) - .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) - .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) - .image(src_image) - .subresource_range(vk::ImageSubresourceRange { - aspect_mask: vk::ImageAspectFlags::COLOR, - base_mip_level: 0, - level_count: 1, - base_array_layer: 0, - layer_count: 1, - }) - .src_access_mask(vk::AccessFlags::SHADER_READ) - .dst_access_mask(vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE); - device.cmd_pipeline_barrier( self.command_buffer, - vk::PipelineStageFlags::TRANSFER | vk::PipelineStageFlags::COMPUTE_SHADER, + vk::PipelineStageFlags::TRANSFER, vk::PipelineStageFlags::BOTTOM_OF_PIPE, vk::DependencyFlags::empty(), &[], &[], - &[target_barrier_to_encode, src_barrier_back], + &[target_barrier_to_encode], + ); + + // Transition the source back to GENERAL for reuse, handing it back + // to its owner if it has one. + let src_back = [ownership.release_barrier( + src_image, + compute_family, + vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL, + vk::PipelineStageFlags2::COMPUTE_SHADER, + )]; + self.context.sync2().cmd_pipeline_barrier2( + self.command_buffer, + &vk::DependencyInfo::default().image_memory_barriers(&src_back), ); device @@ -1096,17 +1052,18 @@ impl ColorConverter { .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; } - let waits: Vec = - wait.iter().map(TimelinePoint::wait_info).collect(); let (semaphore, value) = self.timeline.pending_signal(); - let signals = [vk::SemaphoreSubmitInfo::default() - .semaphore(semaphore) - .value(value) - .stage_mask(vk::PipelineStageFlags2::ALL_COMMANDS)]; + let mut signals = vec![ + vk::SemaphoreSubmitInfo::default() + .semaphore(semaphore) + .value(value) + .stage_mask(vk::PipelineStageFlags2::ALL_COMMANDS), + ]; + signals.extend_from_slice(extra_signals); let command_buffers = [vk::CommandBufferSubmitInfo::default().command_buffer(self.command_buffer)]; let submit_info = vk::SubmitInfo2::default() - .wait_semaphore_infos(&waits) + .wait_semaphore_infos(waits) .command_buffer_infos(&command_buffers) .signal_semaphore_infos(&signals); diff --git a/src/encoder/api.rs b/src/encoder/api.rs new file mode 100644 index 00000000..f4c61a02 --- /dev/null +++ b/src/encoder/api.rs @@ -0,0 +1,460 @@ +//! The public [`Encoder`]: the codec encoder, plus whatever turns the +//! configured [`Source`](super::Source) into its YUV. + +use ash::vk; + +use super::codec::{CodecEncoder, EncoderCommon, VideoCodec}; +use super::frame::{Frame, FrameSource}; +use super::input::{FrameInput, FrameSync, SourceOwnership}; +use super::source::{Conversion, ConversionPreference, plan, target_space}; +use super::{Codec, ColorDescription, EncodeConfig, EncodeFuture}; +use crate::converter::{ColorConverter, ColorRange}; +use crate::error::{PixelForgeError, Result}; +use crate::sync::TimelinePoint; +use crate::vulkan::VideoContext; + +static NEXT_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// The codec-erased operations every [`CodecEncoder`] exposes. +/// +/// One blanket impl covers all codecs, so [`Encoder`] can hold any of them +/// behind a single boxed pointer instead of an enum that re-dispatches by hand. +pub(crate) trait EncoderApi: Send { + fn common(&self) -> &EncoderCommon; + fn input_image(&self) -> vk::Image; + fn encode_frame(&mut self, input: FrameInput, sync: FrameSync) -> Result; + fn flush(&mut self) -> Result<()>; + fn request_idr(&mut self); + fn invalidate_reference_frames(&mut self, first_lost_display_order: u64); + fn set_color_description(&mut self, desc: ColorDescription) -> Result<()>; +} + +impl EncoderApi for CodecEncoder { + fn common(&self) -> &EncoderCommon { + &self.common + } + fn input_image(&self) -> vk::Image { + CodecEncoder::input_image(self) + } + fn encode_frame(&mut self, input: FrameInput, sync: FrameSync) -> Result { + CodecEncoder::encode_frame(self, input, sync) + } + fn flush(&mut self) -> Result<()> { + CodecEncoder::flush(self) + } + fn request_idr(&mut self) { + CodecEncoder::request_idr(self) + } + fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) { + CodecEncoder::invalidate_reference_frames(self, first_lost_display_order) + } + fn set_color_description(&mut self, desc: ColorDescription) -> Result<()> { + CodecEncoder::set_color_description(self, desc) + } +} + +/// Video encoder supporting multiple codecs. +/// +/// Constructed via [`Encoder::new`], which selects the codec from the config, +/// and for an RGB [`Source`](super::Source) the [`Conversion`] that turns +/// frames into the stream's YUV. Frames go in through [`Encoder::submit`], +/// whatever the conversion. +pub struct Encoder { + inner: Box, + context: VideoContext, + config: EncodeConfig, + conversion: Conversion, + converter: Option, + /// Added to every packet's timestamps: the frames encoded before the + /// codec encoder was last made again. + pts_offset: u64, + /// Tells encoders apart, so an image made for one is not read by another. + id: u64, + /// Counts the times the codec encoder was made again with a different + /// conversion. An [`ExternalImage`](super::external::ExternalImage) made + /// for one cannot be read by another. + generation: u64, + #[cfg(feature = "dmabuf")] + sync_files: Option, +} + +impl Encoder { + /// Create a new encoder for the codec named in `config`. + /// + /// For an RGB [`Source`](super::Source), this also picks the + /// [`Conversion`], see [`Self::conversion`]. It fails if no conversion + /// gets from the source to [`EncodeConfig::color_description`], or if + /// [`EncodeConfig::conversion`] asks for one that cannot. + pub fn new(context: VideoContext, config: EncodeConfig) -> Result { + let plan = plan(&context, &config)?; + if let Some(format) = plan.rgb_input { + super::rgb::check_rgb_input(&context, &config, format)?; + } + let converter = plan + .converter + .map(|c| ColorConverter::new(context.clone(), c)) + .transpose()?; + let inner = create_codec_encoder(&context, &config, plan.rgb_input)?; + tracing::debug!( + "{:?} encoder, {:?} source, conversion {:?}", + config.codec, + config.source, + plan.conversion + ); + Ok(Self { + inner, + context, + config, + conversion: plan.conversion, + converter, + pts_offset: 0, + id: NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed), + generation: 0, + #[cfg(feature = "dmabuf")] + sync_files: None, + }) + } + + /// The conversion this encoder picked for its source. + pub fn conversion(&self) -> Conversion { + self.conversion + } + + /// The configuration, with any colour description set since. + pub fn config(&self) -> &EncodeConfig { + &self.config + } + + /// Get the current slot's input image. + /// + /// For a YUV source, rendering or uploading into this image and passing it + /// to [`Frame::image`] encodes it in place, with no copy. It is in + /// `VIDEO_ENCODE_SRC_KHR` layout, and must be left there. This call waits + /// until the slot's previous encode has finished with it. + /// + /// For an RGB source the input image belongs to the conversion: hand the + /// RGB frames to [`Self::submit`] instead. + pub fn input_image(&self) -> vk::Image { + self.inner.input_image() + } + + /// Encode a frame from a GPU image in `GENERAL` layout, and return once + /// the image can be written again. + /// + /// Like `submit(Frame::image(src_image, vk::ImageLayout::GENERAL))`, but + /// for an image other than [`Self::input_image`] it also waits on the CPU + /// for [`EncodeFuture::source_released`]: the copy or conversion out of + /// it. That keeps the simple loop of writing one image and encoding it + /// correct. [`Self::submit`] does not wait, for callers that order their + /// writes on the GPU instead. + /// + /// ```no_run + /// use pixelforge::{InputImage, Encoder, EncodeConfig, EncodeBitDepth, PixelFormat, VideoContext, Codec}; + /// + /// # fn example(context: VideoContext) -> Result<(), Box> { + /// let config = EncodeConfig::h264(1920, 1080); + /// let mut encoder = Encoder::new(context.clone(), config)?; + /// let mut input = InputImage::new( + /// context, + /// Codec::H264, + /// 1920, + /// 1080, + /// EncodeBitDepth::Eight, + /// PixelFormat::Yuv420, + /// )?; + /// + /// // Upload YUV420 data to the input image + /// # let yuv_data = vec![0u8; 1920 * 1080 * 3 / 2]; + /// input.upload_yuv420(&yuv_data)?; + /// + /// // Submit the frame and await its packet. + /// let future = encoder.encode(input.image())?; + /// let packet = pollster::block_on(future)?; + /// // ... use packet.data ... + /// # Ok(()) + /// # } + /// ``` + pub fn encode(&mut self, src_image: vk::Image) -> Result { + // Read before submitting, since submitting moves to the next slot. + let in_place = + self.conversion != Conversion::Shader && src_image == self.inner.input_image(); + let future = self.submit(Frame::image(src_image, vk::ImageLayout::GENERAL))?; + if !in_place { + self.context.wait_point(future.source_released())?; + } + Ok(future) + } + + /// Encode a frame once GPU work the caller submitted has finished. + /// + /// Short for `submit(Frame::image(src_image, src_layout).wait(wait))`. It + /// does not wait for the source to be read: write it again only after + /// [`EncodeFuture::source_released`]. + pub fn encode_after( + &mut self, + src_image: vk::Image, + src_layout: vk::ImageLayout, + wait: &[TimelinePoint], + ) -> Result { + self.submit(Frame::image(src_image, src_layout).wait(wait)) + } + + /// Encode one frame. + /// + /// Nothing here waits on the CPU for the GPU, except backpressure: when + /// every pipeline slot is still in flight, this waits for the oldest. The + /// frame's first submission waits for the frame's + /// [`wait`](Frame::wait) points, and the source is read by a copy into + /// the encoder's input image, by the colour converter, or by the encode + /// itself, depending on the [`Conversion`] and the source. + /// + /// Because the call returns before the source is read, write the source + /// again only after [`EncodeFuture::source_released`]. + pub fn submit(&mut self, frame: Frame<'_>) -> Result { + #[cfg_attr(not(feature = "dmabuf"), allow(unused_mut))] + let mut sync = FrameSync { + waits: frame.waits.iter().map(TimelinePoint::wait_info).collect(), + release_signals: Vec::new(), + }; + + #[cfg(feature = "dmabuf")] + let slot = { + // The slot's semaphores are free once the slot is. + let _ = self.inner.input_image(); + let slot = self.inner.common().pipeline.current_index(); + if frame.wait_sync_file.is_some() || frame.signal_sync_file { + let files = match &mut self.sync_files { + Some(files) => files, + files @ None => files.insert(super::external::SyncFiles::new(&self.context)?), + }; + if let Some(fence) = frame.wait_sync_file { + sync.waits.push(files.import_wait(slot, fence)?); + } + if frame.signal_sync_file { + sync.release_signals.push(files.release_signal(slot)); + } + } + slot + }; + + let mut future = match frame.source { + FrameSource::Image { image, layout } => { + self.submit_image(image, SourceOwnership::caller_image(layout), sync)? + } + #[cfg(feature = "dmabuf")] + FrameSource::External(external) => self.submit_external(external, sync)?, + #[cfg(not(feature = "dmabuf"))] + FrameSource::Never(_) => unreachable!(), + }; + future.pts_offset = self.pts_offset; + + #[cfg(feature = "dmabuf")] + if frame.signal_sync_file + && let Some(files) = &self.sync_files + { + future.release_sync_file = files.export_release(slot)?; + } + Ok(future) + } + + /// Encode from an image the caller owns. + pub(crate) fn submit_image( + &mut self, + image: vk::Image, + ownership: SourceOwnership, + sync: FrameSync, + ) -> Result { + match self.conversion { + Conversion::None | Conversion::Encoder => self + .inner + .encode_frame(FrameInput::Copy { image, ownership }, sync), + Conversion::Shader => self.convert_and_encode(image, ownership, sync), + } + } + + /// Run the colour converter into the slot's input image, then encode it. + pub(crate) fn convert_and_encode( + &mut self, + image: vk::Image, + ownership: SourceOwnership, + sync: FrameSync, + ) -> Result { + let target = self.inner.input_image(); + if image == target { + return Err(PixelForgeError::InvalidInput( + "the encoder's input image holds converted YUV; hand it the RGB frame instead" + .to_string(), + )); + } + let converter = self + .converter + .as_mut() + .expect("a shader conversion has a converter"); + let converted = + converter.convert_with(image, ownership, target, &sync.waits, &sync.release_signals)?; + let mut future = self.inner.encode_frame( + FrameInput::InPlace, + FrameSync { + waits: vec![converted.wait_info()], + release_signals: Vec::new(), + }, + )?; + future.set_released(converted); + Ok(future) + } + + #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] + pub(crate) fn inner(&self) -> &dyn EncoderApi { + &*self.inner + } + + #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] + pub(crate) fn context(&self) -> &VideoContext { + &self.context + } + + #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] + pub(crate) fn generation(&self) -> u64 { + self.generation + } + + #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] + pub(crate) fn id(&self) -> u64 { + self.id + } + + /// Encode `input` with the codec encoder, as it is. + #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] + pub(crate) fn encode_input( + &mut self, + input: FrameInput, + sync: FrameSync, + ) -> Result { + self.inner.encode_frame(input, sync) + } + + /// Wait for all in-flight frames to finish encoding (end-of-stream barrier). + /// + /// Packets are delivered through the [`EncodeFuture`]s returned by + /// [`Encoder::submit`]; await those to obtain them. Once this returns, + /// every outstanding future has been resolved. + pub fn flush(&mut self) -> Result<()> { + self.inner.flush() + } + + /// Request that the next frame be an IDR frame. + pub fn request_idr(&mut self) { + self.inner.request_idr() + } + + /// Recover from packet loss without a full IDR (reference frame invalidation). + /// + /// When a client reports that it could not decode a range of frames, every + /// reference picture the encoder still holds from the earliest lost frame + /// onward is transitively undecodable on the client. This drops those + /// references so the next P-frame is predicted from the most recent + /// *surviving* reference instead — a much cheaper recovery than re-sending a + /// full keyframe. If no reference survives (the loss covers the encoder's + /// whole reference window), it transparently falls back to forcing an IDR. + /// + /// `first_lost_display_order` is the display order — the `pts` reported on + /// [`EncodedPacket`](super::EncodedPacket)s — of the earliest frame the + /// client lost. + /// + /// Effective recovery requires the encoder to keep more than one reference + /// (see [`EncodeConfig::with_max_reference_frames`]); with a single + /// reference this necessarily falls back to an IDR. This applies uniformly + /// across H.264, H.265, and AV1 — each keeps a multi-reference window and + /// re-anchors prediction to the most recent survivor, falling back to an + /// IDR (AV1: key frame) only when the loss covers the entire window. + pub fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) { + let local = first_lost_display_order.saturating_sub(self.pts_offset); + self.inner.invalidate_reference_frames(local) + } + + /// Change what the stream says about its colours, and what the encoder + /// makes its YUV to match. + /// + /// The next frame is an IDR/key frame carrying the new header. For an RGB + /// source the conversion follows: the colour converter gets the new target + /// and range, and where the encoder converts, it is made again with the + /// new matrix and range, since those are fixed in its video session. + /// Packet timestamps carry on across that. If the new description needs a + /// different [`Conversion`], images imported or allocated for the old one + /// have to be made again. + /// + /// On error the encoder is left as it was. + pub fn set_color_description(&mut self, desc: ColorDescription) -> Result<()> { + let mut config = self.config.clone(); + config.color_description = Some(desc); + match self.conversion { + Conversion::None => self.inner.set_color_description(desc)?, + Conversion::Shader if self.shader_still_converts(&config) => { + let target = target_space(&desc).expect("the plan checked the target"); + let range = if desc.full_range { + ColorRange::Full + } else { + ColorRange::Limited + }; + let converter = self + .converter + .as_mut() + .expect("a shader conversion has a converter"); + let (old_target, old_range) = (converter.config().target, converter.config().range); + converter.set_target(target)?; + converter.set_range(range); + if let Err(e) = self.inner.set_color_description(desc) { + let _ = converter.set_target(old_target); + converter.set_range(old_range); + return Err(e); + } + } + Conversion::Shader | Conversion::Encoder => self.rebuild(config.clone())?, + } + self.config = config; + Ok(()) + } + + /// Whether the colour converter this encoder has can produce what + /// `config` describes, so the encoder can keep it. + fn shader_still_converts(&self, config: &EncodeConfig) -> bool { + let mut probe = config.clone(); + probe.conversion = ConversionPreference::Shader; + plan(&self.context, &probe).is_ok() + } + + /// Make the codec encoder again for `config`, keeping the timestamps + /// going. + fn rebuild(&mut self, config: EncodeConfig) -> Result<()> { + self.inner.flush()?; + let encoded = self.inner.common().input_frame_num; + let mut next = Self::new(self.context.clone(), config)?; + next.pts_offset = self.pts_offset + encoded; + next.id = self.id; + next.generation = if next.conversion == self.conversion { + self.generation + } else { + self.generation + 1 + }; + #[cfg(feature = "dmabuf")] + { + next.sync_files = self.sync_files.take(); + } + *self = next; + Ok(()) + } +} + +fn create_codec_encoder( + context: &VideoContext, + config: &EncodeConfig, + rgb_input: Option, +) -> Result> { + let context = context.clone(); + let config = config.clone(); + Ok(match config.codec { + Codec::H264 => Box::new(super::h264::H264::create(context, config, rgb_input)?), + Codec::H265 => Box::new(super::h265::H265::create(context, config, rgb_input)?), + Codec::AV1 => Box::new(super::av1::Av1::create(context, config, rgb_input)?), + }) +} diff --git a/src/encoder/av1/init.rs b/src/encoder/av1/init.rs index adc00971..7366db52 100644 --- a/src/encoder/av1/init.rs +++ b/src/encoder/av1/init.rs @@ -1,10 +1,11 @@ use super::{Av1, MIN_BITSTREAM_BUFFER_SIZE, SUPERBLOCK_SIZE}; +use crate::converter::InputFormat; use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, }; -use crate::encoder::profile::av1_profile; -use crate::encoder::rgb::{RgbConversionCaps, rgb_conversion_profile}; +use crate::encoder::profile::EncodeProfile; +use crate::encoder::rgb::RgbConversionCaps; use crate::encoder::{ColorDescription, EncodeConfig}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; @@ -14,7 +15,11 @@ use tracing::info; impl Av1 { /// Create a new AV1 encoder. - pub fn create(context: VideoContext, config: EncodeConfig) -> Result> { + pub(crate) fn create( + context: VideoContext, + config: EncodeConfig, + rgb_input: Option, + ) -> Result> { // Reachable from safe caller code via `with_b_frames`, so report it // rather than panicking inside the library. if config.b_frame_count != 0 { @@ -29,33 +34,8 @@ impl Av1 { config.dimensions.width, config.dimensions.height, config.pixel_format ); - let profile = av1_profile(config.pixel_format); - - let chroma_subsampling: vk::VideoChromaSubsamplingFlagsKHR = config.pixel_format.into(); - let bit_depth: vk::VideoComponentBitDepthFlagsKHR = config.bit_depth.into(); - - // Get encoder tuning mode, usage and content hints. - let encode_usage_hint: vk::VideoEncodeUsageFlagsKHR = config.encode_usage_hint.into(); - let encode_content_hint: vk::VideoEncodeContentFlagsKHR = config.encode_content_hint.into(); - let encoder_tuning_mode: vk::VideoEncodeTuningModeKHR = config.encoder_tuning_mode.into(); - - let mut video_encode_usage_info = vk::VideoEncodeUsageInfoKHR::default() - .video_usage_hints(encode_usage_hint) - .video_content_hints(encode_content_hint) - .tuning_mode(encoder_tuning_mode); - - let mut av1_profile_info = vk::VideoEncodeAV1ProfileInfoKHR::default().std_profile(profile); - let mut rgb_profile = rgb_conversion_profile(); - let mut profile_info = vk::VideoProfileInfoKHR::default() - .video_codec_operation(vk::VideoCodecOperationFlagsKHR::ENCODE_AV1) - .chroma_subsampling(chroma_subsampling) - .luma_bit_depth(bit_depth) - .chroma_bit_depth(bit_depth) - .push(&mut av1_profile_info) - .push(&mut video_encode_usage_info); - if config.rgb_input.is_some() { - profile_info = profile_info.push(&mut rgb_profile); - } + let profile = EncodeProfile::new(&config, rgb_input.is_some())?; + let profile_info = profile.info(); let bitstream_buffer_size = MIN_BITSTREAM_BUFFER_SIZE .max(config.dimensions.width as usize * config.dimensions.height as usize); @@ -68,16 +48,17 @@ impl Av1 { .push(&mut encode_caps) .push(&mut av1_caps); let mut rgb_caps = vk::VideoEncodeRgbConversionCapabilitiesVALVE::default(); - if config.rgb_input.is_some() { + if rgb_input.is_some() { capabilities = capabilities.push(&mut rgb_caps); } - let caps = query_video_caps(&context, &profile_info, &mut capabilities)?; - let rgb_caps = config.rgb_input.map(|_| RgbConversionCaps::from(&rgb_caps)); + let caps = query_video_caps(&context, profile_info, &mut capabilities)?; + let rgb_caps = rgb_input.map(|_| RgbConversionCaps::from(&rgb_caps)); - let init = build_encoder_common(&CommonInitRequest { + let init = build_encoder_common(CommonInitRequest { context: &context, config: &config, - profile_info: &profile_info, + profile, + rgb_input, caps: &caps, align_unit: SUPERBLOCK_SIZE, // AV1 has NUM_REF_FRAMES = 8 DPB slots, and `refresh_frame_flags` diff --git a/src/encoder/av1/record.rs b/src/encoder/av1/record.rs index 62611ec1..93fe1e7b 100644 --- a/src/encoder/av1/record.rs +++ b/src/encoder/av1/record.rs @@ -32,7 +32,7 @@ impl Av1 { let timestamp_query_pool = slot.timestamp_query_pool; let bitstream_buffer = slot.bitstream_buffer; let bitstream_buffer_size = slot.bitstream_buffer_size; - let input_image_view = slot.input_image_view; + let input_image_view = common.source_view(); debug!( "av1 record: key={}, refs_len={}, dpb_slot={}", @@ -58,6 +58,7 @@ impl Av1 { &ref_dpb_slots, common.dpb_slot_active[current_dpb_slot as usize], ); + common.record_source_acquire(command_buffer); } let frame_type = if is_key_frame { @@ -428,6 +429,8 @@ impl Av1 { .video_queue_fn .cmd_end_video_coding(command_buffer, &end_coding_info); + common.record_source_release(command_buffer); + device .end_command_buffer(command_buffer) .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index be9750f3..adafd2c5 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -15,9 +15,14 @@ use ash::vk; use ash::vk::TaggedStructure; +use crate::converter::InputFormat; use crate::encoder::dpb::MAX_DPB_SLOTS; use crate::encoder::gop::{GopFrameType, GopPosition, GopStructure}; -use crate::encoder::pipeline::{EncodeFuture, EncodePipeline, PipelineConfig, SlotPacketMetadata}; +use crate::encoder::input::{FrameInput, FrameSync, SourceOwnership}; +use crate::encoder::pipeline::{ + ENCODE_PIPELINE_DEPTH, EncodeFuture, EncodePipeline, PipelineConfig, SlotPacketMetadata, +}; +use crate::encoder::profile::EncodeProfile; use crate::encoder::resources::{ EncoderTeardown, UploadParams, align_up, allocate_session_memory, create_command_resources, create_dpb_images, destroy_encoder_resources, get_video_format, lcm, @@ -27,6 +32,7 @@ use crate::encoder::rgb::RgbConversionCaps; use crate::encoder::{ColorDescription, EncodeConfig, FrameType, RateControlMode}; use crate::error::{PixelForgeError, Result}; use crate::sync::TimelinePoint; +use crate::video::{TimelineChain, allocate_command_buffers}; use crate::vulkan::VideoContext; /// Per-encoder state shared by every codec. @@ -73,11 +79,29 @@ pub(crate) struct EncoderCommon { pub command_pool: vk::CommandPool, pub upload_command_pool: vk::CommandPool, - pub upload_command_buffer: vk::CommandBuffer, pub upload_fence: vk::Fence, - /// Caller work this frame's first submission must wait for. The upload - /// takes them when it copies, otherwise the encode submission does. - pub pending_waits: Vec, + /// One upload command buffer per pipeline slot. A slot is handed out again + /// only once its encode is complete, and that encode waited for the + /// slot's upload, so the buffer is free by then. + pub upload_command_buffers: Vec, + /// Signalled by each upload; the encode of the same frame waits on it. + pub upload_timeline: TimelineChain, + /// What the current frame's encode submission waits for. + pub pending_waits: Vec>, + /// What the current frame's encode submission signals besides its + /// timeline: a release the caller asked for, when the encode is what + /// reads the source last. + pub pending_signals: Vec>, + /// What the current frame's encode reads. + pub source: SourceBinding, + /// When the current frame's source is free again, if something before + /// the encode was the last to read it. + pub source_released: Option, + /// The session's video profile, for images made after creation. + #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] + pub profile: EncodeProfile, + /// The RGB format the encoder converts from, when it takes RGB input. + pub rgb_input: Option, } impl EncoderCommon { @@ -96,36 +120,174 @@ impl EncoderCommon { } } - /// Copy a source image, in `src_layout`, into the current slot's input - /// image. No-op when the source already *is* the slot image (e.g. - /// converted in place). - pub fn upload(&mut self, src_image: vk::Image, src_layout: vk::ImageLayout) -> Result<()> { - let (dst_image, input_image_layout) = { + /// Set up what the current frame's encode reads, copying the source into + /// the slot's input image first when it has to. + /// + /// `sync.waits` go to the frame's first submission and + /// `sync.release_signals` to the one that is last to read the source: + /// the copy when there is one, the encode otherwise. + pub fn prepare_source(&mut self, input: FrameInput, sync: FrameSync) -> Result<()> { + let (slot_image, slot_view, slot_layout) = { let slot = self.pipeline.current(); - (slot.input_image, slot.input_image_layout) + ( + slot.input_image, + slot.input_image_view, + slot.input_image_layout, + ) }; - if src_image == dst_image { - return Ok(()); + let slot_binding = SourceBinding { + image: slot_image, + view: slot_view, + extent: None, + ownership: None, + }; + self.source_released = None; + self.pending_signals.clear(); + match input { + FrameInput::Copy { image, .. } if image == slot_image => { + self.source = slot_binding; + self.pending_waits = sync.waits; + self.pending_signals = sync.release_signals; + } + FrameInput::InPlace => { + self.source = slot_binding; + self.pending_waits = sync.waits; + self.pending_signals = sync.release_signals; + } + FrameInput::Copy { image, ownership } => { + let (semaphore, value) = self.upload_timeline.pending_signal(); + let done = vk::SemaphoreSubmitInfo::default() + .semaphore(semaphore) + .value(value) + .stage_mask(vk::PipelineStageFlags2::ALL_COMMANDS); + let mut signals = vec![done]; + signals.extend(sync.release_signals); + let params = UploadParams { + upload_command_buffer: self.upload_command_buffers + [self.pipeline.current_index()], + src_image: image, + src_ownership: ownership, + dst_image: slot_image, + width: self.config.dimensions.width, + height: self.config.dimensions.height, + pixel_format: self.config.pixel_format, + input_image_layout: slot_layout, + rgb: self.rgb_input.is_some(), + upload_queue: self.context.transfer_queue(), + waits: &sync.waits, + signals: &signals, + }; + upload_image_to_input(&self.context, ¶ms)?; + self.upload_timeline.commit(); + self.pipeline.current_mut().input_image_layout = + vk::ImageLayout::VIDEO_ENCODE_SRC_KHR; + let point = TimelinePoint::new(semaphore, value); + self.pending_waits = vec![point.wait_info()]; + self.source_released = Some(point); + self.source = slot_binding; + } + FrameInput::Direct { + image, + view, + extent, + ownership, + } => { + self.source = SourceBinding { + image, + view, + extent: Some(extent), + ownership: Some(ownership), + }; + self.pending_waits = sync.waits; + self.pending_signals = sync.release_signals; + } } - let waits = std::mem::take(&mut self.pending_waits); + Ok(()) + } + + /// The view the current frame's encode reads. + pub fn source_view(&self) -> vk::ImageView { + self.source.view + } - let params = UploadParams { - upload_command_buffer: self.upload_command_buffer, - upload_fence: self.upload_fence, - src_image, - src_layout, - dst_image, - width: self.config.dimensions.width, - height: self.config.dimensions.height, - pixel_format: self.config.pixel_format, - input_image_layout, - rgb: self.config.rgb_input.is_some(), - upload_queue: self.context.transfer_queue(), - waits: &waits, + /// The coded extent of the current frame's source picture. + /// + /// The aligned extent for the encoder's own input images, which are made + /// that large and have their padding zeroed. A source read directly may + /// be only the display size, as a DMA-BUF from a capture protocol has to + /// be. Its coded extent is then the display size, which the spec allows: + /// the stream headers already crop to it, and what the encoder reads + /// beyond the image is the implementation's padding. + pub fn source_coded_extent(&self) -> vk::Extent2D { + let aligned = vk::Extent2D { + width: self.aligned_width, + height: self.aligned_height, }; - upload_image_to_input(&self.context, ¶ms)?; - self.pipeline.current_mut().input_image_layout = vk::ImageLayout::VIDEO_ENCODE_SRC_KHR; - Ok(()) + match self.source.extent { + Some(extent) if extent.width < aligned.width || extent.height < aligned.height => { + vk::Extent2D { + width: self.config.dimensions.width, + height: self.config.dimensions.height, + } + } + _ => aligned, + } + } + + /// Record the barrier that brings a directly read source into + /// `VIDEO_ENCODE_SRC_KHR` on the encode queue. Nothing to do for the + /// encoder's own input images, which are already there. + /// + /// # Safety + /// + /// `command_buffer` must be recording, before the encode. + pub unsafe fn record_source_acquire(&self, command_buffer: vk::CommandBuffer) { + let (Some(ownership), Some(family)) = ( + self.source.ownership, + self.context.video_encode_queue_family(), + ) else { + return; + }; + let barrier = [ownership.acquire_barrier( + self.source.image, + family, + vk::ImageLayout::VIDEO_ENCODE_SRC_KHR, + vk::PipelineStageFlags2::VIDEO_ENCODE_KHR, + vk::AccessFlags2::VIDEO_ENCODE_READ_KHR, + )]; + unsafe { + self.context.sync2().cmd_pipeline_barrier2( + command_buffer, + &vk::DependencyInfo::default().image_memory_barriers(&barrier), + ); + } + } + + /// Record the barrier that hands a directly read source back after the + /// encode, in `GENERAL`. + /// + /// # Safety + /// + /// `command_buffer` must be recording, after the encode. + pub unsafe fn record_source_release(&self, command_buffer: vk::CommandBuffer) { + let (Some(ownership), Some(family)) = ( + self.source.ownership, + self.context.video_encode_queue_family(), + ) else { + return; + }; + let barrier = [ownership.release_barrier( + self.source.image, + family, + vk::ImageLayout::VIDEO_ENCODE_SRC_KHR, + vk::PipelineStageFlags2::VIDEO_ENCODE_KHR, + )]; + unsafe { + self.context.sync2().cmd_pipeline_barrier2( + command_buffer, + &vk::DependencyInfo::default().image_memory_barriers(&barrier), + ); + } } /// Record the metadata for the packet the current slot will produce. @@ -140,12 +302,17 @@ impl EncoderCommon { PixelForgeError::NoSuitableDevice("No video encode queue available".to_string()) })?; let waits = std::mem::take(&mut self.pending_waits); - let future = self.pipeline.submit_current( + let signals = std::mem::take(&mut self.pending_signals); + let mut future = self.pipeline.submit_current( self.context.device(), self.context.sync2(), encode_queue, &waits, + &signals, )?; + if let Some(point) = self.source_released.take() { + future.set_released(point); + } self.dpb_slot_active[self.current_dpb_slot as usize] = true; Ok(future) } @@ -161,6 +328,19 @@ impl EncoderCommon { } } +/// What one frame's encode reads: the slot's own input image, or a source +/// read directly. +#[derive(Default)] +pub(crate) struct SourceBinding { + pub image: vk::Image, + pub view: vk::ImageView, + /// The image's size, for a source read directly; `None` for the slot's + /// input image, which has the aligned extent. + pub extent: Option, + /// The ownership transfers the encode does, for a source read directly. + pub ownership: Option, +} + /// Everything a single frame needs to know about its schedule, snapshotted by /// [`EncoderCommon::begin_frame`] so the codec hooks all see a consistent view. pub(crate) struct FramePlan { @@ -327,18 +507,11 @@ impl CodecEncoder { self.common.pipeline.input_image() } - /// Encode one frame once `wait` is reached, returning a future for its - /// packet. See [`crate::Encoder::encode_after`]. - pub fn encode_after( - &mut self, - src_image: vk::Image, - src_layout: vk::ImageLayout, - wait: &[TimelinePoint], - ) -> Result { + /// Encode one frame from `input`, returning a future for its packet. + /// See [`crate::Encoder::submit`]. + pub fn encode_frame(&mut self, input: FrameInput, sync: FrameSync) -> Result { let plan = self.common.begin_frame(); - self.common.pending_waits.clear(); - self.common.pending_waits.extend_from_slice(wait); - self.common.upload(src_image, src_layout)?; + self.common.prepare_source(input, sync)?; let setup = self.codec.begin_picture(&mut self.common, &plan)?; self.common.set_pending_metadata(SlotPacketMetadata { @@ -415,6 +588,13 @@ impl Drop for CodecEncoder { } common.pipeline.destroy(device); + // A caller may still wait on points these handed out. + common + .context + .retire_semaphore(common.pipeline.timeline_semaphore()); + common + .context + .retire_semaphore(common.upload_timeline.semaphore()); destroy_encoder_resources( device, @@ -441,8 +621,11 @@ impl Drop for CodecEncoder { pub(crate) struct CommonInitRequest<'a> { pub context: &'a VideoContext, pub config: &'a EncodeConfig, - /// The codec profile, with its `VideoEncode*ProfileInfoKHR` already chained. - pub profile_info: &'a vk::VideoProfileInfoKHR<'a>, + /// The codec profile. Moved into [`EncoderCommon`], since images made + /// later have to name it too. + pub profile: EncodeProfile, + /// The RGB format the encoder converts from, when it takes RGB input. + pub rgb_input: Option, /// Device capabilities for this profile, resolved by [`query_video_caps`]. pub caps: &'a DeviceVideoCaps, /// Codec block size for coded-extent alignment (macroblock / CTB / superblock). @@ -454,8 +637,7 @@ pub(crate) struct CommonInitRequest<'a> { /// `SEPARATE_REFERENCE_IMAGES` (H.264/H.265 yes; AV1 no). pub allow_layered_dpb: bool, /// What the hardware RGB conversion supports for this profile, queried - /// alongside the codec capabilities when [`EncodeConfig::rgb_input`] is - /// set. + /// alongside the codec capabilities when `rgb_input` is set. pub rgb_caps: Option, } @@ -529,9 +711,10 @@ pub(crate) fn query_video_caps( /// Query capabilities, create the video session, DPB images, command resources, /// the encode pipeline and the GOP structure — the ~85% of encoder /// initialization that is identical across codecs. -pub(crate) fn build_encoder_common(req: &CommonInitRequest) -> Result { +pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result { let context = req.context; let config = req.config; + let profile_info = req.profile.info(); let width = config.dimensions.width; let height = config.dimensions.height; @@ -587,7 +770,7 @@ pub(crate) fn build_encoder_common(req: &CommonInitRequest) -> Result { let caps = req.rgb_caps.ok_or_else(|| { PixelForgeError::NoSuitableDevice( @@ -609,12 +792,12 @@ pub(crate) fn build_encoder_common(req: &CommonInitRequest) -> Result Result Result Result Result Result u32 { + (code[0] as u32) | (code[1] as u32) << 8 | (code[2] as u32) << 16 | (code[3] as u32) << 24 +} + +/// DRM fourcc codes for the formats an encoder reads. +pub mod drm_format { + use super::fourcc; + + /// 8-bit 4:2:0, Y plane then interleaved CbCr. + pub const NV12: u32 = fourcc(b"NV12"); + /// 10-bit 4:2:0 in 16-bit words, Y plane then interleaved CbCr. + pub const P010: u32 = fourcc(b"P010"); + /// 8-bit 4:4:4, Y plane then interleaved CbCr. + pub const NV24: u32 = fourcc(b"NV24"); + /// 32-bit, B G R X in memory. + pub const XRGB8888: u32 = fourcc(b"XR24"); + /// 32-bit, B G R A in memory. + pub const ARGB8888: u32 = fourcc(b"AR24"); + /// 32-bit, R G B X in memory. + pub const XBGR8888: u32 = fourcc(b"XB24"); + /// 32-bit, R G B A in memory. + pub const ABGR8888: u32 = fourcc(b"AB24"); + /// 10 bits per colour, 2 of alpha, red in the low bits. + pub const ABGR2101010: u32 = fourcc(b"AB30"); + /// 10 bits per colour, red in the low bits. + pub const XBGR2101010: u32 = fourcc(b"XB30"); + /// 16-bit float per channel, red first. + pub const ABGR16161616F: u32 = fourcc(b"AB4H"); +} + +/// The Vulkan format a DRM fourcc holds, for the formats an encoder reads. +pub fn vk_format_of(drm: u32) -> Option { + use drm_format::*; + Some(match drm { + NV12 => vk::Format::G8_B8R8_2PLANE_420_UNORM, + P010 => vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, + NV24 => vk::Format::G8_B8R8_2PLANE_444_UNORM, + XRGB8888 | ARGB8888 => vk::Format::B8G8R8A8_UNORM, + XBGR8888 | ABGR8888 => vk::Format::R8G8B8A8_UNORM, + XBGR2101010 | ABGR2101010 => vk::Format::A2B10G10R10_UNORM_PACK32, + ABGR16161616F => vk::Format::R16G16B16A16_SFLOAT, + _ => return None, + }) +} + +/// The DRM fourcc for a Vulkan format an encoder reads. For the RGB formats, +/// the one with alpha: a buffer without it is taken too. +fn drm_format_of(format: vk::Format) -> Option { + use drm_format::*; + Some(match format { + vk::Format::G8_B8R8_2PLANE_420_UNORM => NV12, + vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 => P010, + vk::Format::G8_B8R8_2PLANE_444_UNORM => NV24, + vk::Format::B8G8R8A8_UNORM => ARGB8888, + vk::Format::R8G8B8A8_UNORM => ABGR8888, + vk::Format::A2B10G10R10_UNORM_PACK32 => ABGR2101010, + vk::Format::R16G16B16A16_SFLOAT => ABGR16161616F, + _ => return None, + }) +} + +/// One plane of a [`DmaBuf`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DmaBufPlane { + /// Where the plane starts in the buffer, in bytes. + pub offset: u64, + /// Bytes from one row to the next. + pub stride: u64, +} + +/// A DMA-BUF holding one picture: one buffer, every plane in it at its own +/// offset. +#[derive(Debug)] +pub struct DmaBuf { + /// The buffer. + pub fd: OwnedFd, + /// Width in pixels. + pub width: u32, + /// Height in pixels. + pub height: u32, + /// DRM fourcc of the pixel format, see [`drm_format`]. + pub format: u32, + /// DRM format modifier. + pub modifier: u64, + /// The memory planes, as many as the modifier has. + pub planes: Vec, +} + +/// What an encoder can do with buffers of one DRM format modifier. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InputModifier { + /// The DRM format modifier. + pub modifier: u64, + /// How many memory planes a buffer with it has. + pub plane_count: u32, + /// The encoder reads such a buffer with no copy in pixelforge: the + /// encode itself, or the colour converter's shader where it converts. + pub zero_copy: bool, + /// [`Encoder::import`] takes such a buffer, reading it directly where + /// `zero_copy` says so and copying it otherwise. + pub importable: bool, + /// [`Encoder::allocate_input`] can make such a buffer: read directly + /// where `zero_copy` says so, copied otherwise. + pub exportable: bool, +} + +/// The buffers an encoder takes, from [`Encoder::input_formats`]. +#[derive(Debug, Clone)] +pub struct InputFormats { + /// The DRM fourcc of the pixel format, see [`drm_format`]. + pub format: u32, + /// The same format, as Vulkan names it. + pub vk_format: vk::Format, + /// Every modifier the device knows for the format and the encoder can do + /// something with. Offer the `zero_copy` ones to a producer first. + pub modifiers: Vec, +} + +/// How an [`ExternalImage`] is read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Access { + /// By the encode, made with its video profile. + Encode, + /// By a copy into the encoder's input image. + Copy, + /// By the colour converter's shader. + Sample, +} + +/// Which modifier a new image gets. +enum ModifierChoice<'a> { + /// The driver picks one of these, for an image to export. + OneOf(&'a [u64]), + /// This one, with these plane layouts, for an imported buffer. + Exactly(u64, &'a [vk::SubresourceLayout]), +} + +/// What the device says about one modifier for one use. +struct ModifierSupport { + importable: bool, + exportable: bool, +} + +/// A DMA-BUF an [`Encoder`] can read, made by [`Encoder::import`] or +/// [`Encoder::allocate_input`]. Hand it to +/// [`Frame::external`](crate::Frame::external). +/// +/// Only the encoder that made it can read it. Dropping it waits until its last +/// frame is done with it. +pub struct ExternalImage { + context: VideoContext, + image: vk::Image, + memory: vk::DeviceMemory, + view: Option, + access: Access, + extent: vk::Extent2D, + format: u32, + modifier: u64, + plane_count: u32, + /// The encoder it belongs to, and its conversion's generation. + owner: (u64, u64), + /// Whether no frame has read it yet, so its layout is still undefined. + fresh: AtomicBool, + /// When the last frame that read it is done with it. + last_use: Mutex>, +} + +impl std::fmt::Debug for ExternalImage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExternalImage") + .field("extent", &self.extent) + .field("format", &self.format) + .field("modifier", &self.modifier) + .field("access", &self.access) + .finish() + } +} + +impl ExternalImage { + /// Whether the encoder reads this image directly, with no copy. + pub fn is_zero_copy(&self) -> bool { + self.access != Access::Copy + } + + /// The DRM format modifier. + pub fn modifier(&self) -> u64 { + self.modifier + } + + /// The image, for a caller that also uses it with Vulkan itself. + pub fn image(&self) -> vk::Image { + self.image + } + + /// Export the image as a DMA-BUF, for a producer to write into. Each call + /// gives a new fd for the same memory. + pub fn export(&self) -> Result { + let device = self.context.device(); + let memory_fd = ash::khr::external_memory_fd::Device::load(self.context.instance(), device); + let info = vk::MemoryGetFdInfoKHR::default() + .memory(self.memory) + .handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let fd = unsafe { memory_fd.get_memory_fd(&info) } + .map_err(|e| PixelForgeError::ResourceCreation(format!("DMA-BUF export: {e}")))?; + // SAFETY: vkGetMemoryFdKHR hands over a new fd the caller owns. + let fd = unsafe { OwnedFd::from_raw_fd(fd) }; + let planes = (0..self.plane_count) + .map(|plane| { + let layout = unsafe { + device.get_image_subresource_layout( + self.image, + vk::ImageSubresource { + aspect_mask: memory_plane(plane), + mip_level: 0, + array_layer: 0, + }, + ) + }; + DmaBufPlane { + offset: layout.offset, + stride: layout.row_pitch, + } + }) + .collect(); + Ok(DmaBuf { + fd, + width: self.extent.width, + height: self.extent.height, + format: self.format, + modifier: self.modifier, + planes, + }) + } + + /// The ownership transfers of the next frame that reads this image. + fn ownership(&self) -> SourceOwnership { + let foreign = self.context.external().foreign_family(); + SourceOwnership { + old_layout: if self.fresh.load(Ordering::Acquire) { + vk::ImageLayout::UNDEFINED + } else { + vk::ImageLayout::GENERAL + }, + acquire_from: Some(foreign), + release_to: Some(foreign), + } + } + + fn last_use(&self) -> Option { + *self.last_use.lock().unwrap_or_else(|p| p.into_inner()) + } + + fn mark_used(&self, released: TimelinePoint) { + self.fresh.store(false, Ordering::Release); + *self.last_use.lock().unwrap_or_else(|p| p.into_inner()) = Some(released); + } +} + +impl Drop for ExternalImage { + fn drop(&mut self) { + if let Some(point) = self.last_use() { + let _ = self.context.wait_point(point); + } + let device = self.context.device(); + unsafe { + if let Some(view) = self.view { + device.destroy_image_view(view, None); + } + device.destroy_image(self.image, None); + device.free_memory(self.memory, None); + } + } +} + +fn memory_plane(plane: u32) -> vk::ImageAspectFlags { + match plane { + 0 => vk::ImageAspectFlags::MEMORY_PLANE_0_EXT, + 1 => vk::ImageAspectFlags::MEMORY_PLANE_1_EXT, + 2 => vk::ImageAspectFlags::MEMORY_PLANE_2_EXT, + _ => vk::ImageAspectFlags::MEMORY_PLANE_3_EXT, + } +} + +/// Whether a DMA-BUF with `modifier` is safe to use for `access` on this +/// driver, past what the driver itself reports. +/// +/// NVIDIA's driver (seen on 615.78.08) reports more than works, in two ways, +/// both of which give a wrong picture rather than an error: +/// +/// - An image made with a video profile reads a block-linear buffer as all +/// black, even one the encoder allocated and exported itself. Imported +/// without the profile, the same buffer reads correctly. So the encode never +/// reads a DMA-BUF directly there; it is copied. +/// - A two-plane YUV buffer with a block-linear modifier loses its chroma plane +/// between the image that wrote it and one that imports it, even with no +/// pixelforge code involved. LINEAR survives, so it is the only modifier +/// offered for YUV there. Single-plane RGB is not affected. +/// +/// `PIXELFORGE_NO_DMABUF_QUIRKS=1` trusts the driver anyway, to try a newer +/// one. +fn quirks_allow(context: &VideoContext, format: vk::Format, modifier: u64, access: Access) -> bool { + const NVIDIA: u32 = 0x10de; + const LINEAR: u64 = 0; + if context.device_properties().vendor_id != NVIDIA + || std::env::var_os("PIXELFORGE_NO_DMABUF_QUIRKS").is_some_and(|v| v == "1") + { + return true; + } + let multi_planar = matches!( + format, + vk::Format::G8_B8R8_2PLANE_420_UNORM + | vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16 + | vk::Format::G8_B8R8_2PLANE_444_UNORM + ); + access != Access::Encode && (!multi_planar || modifier == LINEAR) +} + +/// What an encoder reads and how, worked out from its conversion. +struct InputPlan { + vk_format: vk::Format, + zero_copy: Access, + /// A way to read buffers the zero-copy path does not take, if any. + fallback: Option, +} + +impl Encoder { + fn input_plan(&self) -> Result { + let config = self.config(); + let rgb_format = match config.source { + Source::Rgb(rgb) => Some(rgb.format.vk_format()), + Source::Yuv => None, + }; + Ok(match self.conversion() { + Conversion::None => { + if config.pixel_format == PixelFormat::Yuv422 { + return Err(PixelForgeError::InvalidInput( + "DMA-BUF input: no 4:2:2 encode".to_string(), + )); + } + InputPlan { + vk_format: get_video_format(config.pixel_format, config.bit_depth), + zero_copy: Access::Encode, + fallback: Some(Access::Copy), + } + } + Conversion::Encoder => InputPlan { + vk_format: rgb_format.expect("an encoder conversion has an RGB source"), + zero_copy: Access::Encode, + fallback: Some(Access::Copy), + }, + Conversion::Shader => InputPlan { + vk_format: rgb_format.expect("a shader conversion has an RGB source"), + zero_copy: Access::Sample, + fallback: None, + }, + }) + } + + fn require_dmabuf(&self) -> Result<()> { + if self.context().external().dmabuf { + Ok(()) + } else { + Err(PixelForgeError::NoSuitableDevice( + "DMA-BUF input needs VK_EXT_image_drm_format_modifier, \ + VK_EXT_external_memory_dma_buf and VK_KHR_external_memory_fd" + .to_string(), + )) + } + } + + /// The image usage `access` needs, on an image to `export` or not. + /// + /// An image made to be exported also gets `TRANSFER_DST`, the usage a + /// producer writing into it needs. A driver may lay a buffer out + /// differently by usage, and then follow its own layout over the one an + /// importer gives it: NVIDIA's (615) pads LINEAR rows for `TRANSFER_DST` + /// and not for `TRANSFER_SRC` alone, and a producer importing the + /// unpadded export then writes every row in the wrong place. + fn usage(access: Access, export: bool) -> vk::ImageUsageFlags { + let usage = match access { + Access::Encode => vk::ImageUsageFlags::VIDEO_ENCODE_SRC_KHR, + Access::Copy => vk::ImageUsageFlags::TRANSFER_SRC, + Access::Sample => vk::ImageUsageFlags::SAMPLED, + }; + if export { + usage | vk::ImageUsageFlags::TRANSFER_DST + } else { + usage + } + } + + /// What the device says about `modifier` for `access`, on an image to + /// `export` or one imported, or `None` if it cannot make such an image at + /// all. + fn modifier_support( + &self, + format: vk::Format, + modifier: u64, + access: Access, + export: bool, + ) -> Option { + let context = self.context(); + if !quirks_allow(context, format, modifier, access) { + return None; + } + let profile = self.inner().common().profile.info(); + let profiles = std::slice::from_ref(profile); + let mut profile_list = vk::VideoProfileListInfoKHR::default().profiles(profiles); + let mut external = vk::PhysicalDeviceExternalImageFormatInfo::default() + .handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let mut drm = vk::PhysicalDeviceImageDrmFormatModifierInfoEXT::default() + .drm_format_modifier(modifier) + .sharing_mode(vk::SharingMode::EXCLUSIVE); + let mut info = vk::PhysicalDeviceImageFormatInfo2::default() + .format(format) + .ty(vk::ImageType::TYPE_2D) + .tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT) + .usage(Self::usage(access, export)) + .push(&mut external) + .push(&mut drm); + if access == Access::Encode { + info = info.push(&mut profile_list); + } + let mut external_props = vk::ExternalImageFormatProperties::default(); + let mut props = vk::ImageFormatProperties2::default().push(&mut external_props); + unsafe { + context + .instance() + .get_physical_device_image_format_properties2( + context.physical_device(), + &info, + &mut props, + ) + } + .ok()?; + let max = props.image_format_properties.max_extent; + let size = self.config().dimensions; + if max.width < size.width || max.height < size.height { + return None; + } + let features = external_props + .external_memory_properties + .external_memory_features; + Some(ModifierSupport { + importable: features.contains(vk::ExternalMemoryFeatureFlags::IMPORTABLE), + exportable: features.contains(vk::ExternalMemoryFeatureFlags::EXPORTABLE), + }) + } + + /// The modifiers the device knows for `format`, with their plane counts. + fn known_modifiers(&self, format: vk::Format) -> Vec<(u64, u32)> { + let context = self.context(); + let query = |list: &mut vk::DrmFormatModifierPropertiesListEXT| { + let mut props = vk::FormatProperties2::default().push(list); + unsafe { + context.instance().get_physical_device_format_properties2( + context.physical_device(), + format, + &mut props, + ) + }; + }; + let mut list = vk::DrmFormatModifierPropertiesListEXT::default(); + query(&mut list); + let mut entries = vec![ + vk::DrmFormatModifierPropertiesEXT::default(); + list.drm_format_modifier_count as usize + ]; + let mut list = vk::DrmFormatModifierPropertiesListEXT::default() + .drm_format_modifier_properties(&mut entries); + query(&mut list); + let count = list.drm_format_modifier_count as usize; + entries + .iter() + .take(count) + .map(|e| (e.drm_format_modifier, e.drm_format_modifier_plane_count)) + .collect() + } + + /// The DRM format this encoder takes as a DMA-BUF, and what it can do + /// with each modifier the device knows for it. + /// + /// The format follows the [`Source`](crate::encoder::Source): the + /// encoder's YUV for a YUV source, the RGB format for an RGB one. A + /// producer that can choose should be offered the `zero_copy` modifiers + /// first. + pub fn input_formats(&self) -> Result { + self.require_dmabuf()?; + let plan = self.input_plan()?; + let format = drm_format_of(plan.vk_format).ok_or_else(|| { + PixelForgeError::InvalidInput(format!( + "DMA-BUF input: no DRM format for {:?}", + plan.vk_format + )) + })?; + let modifiers = self + .known_modifiers(plan.vk_format) + .into_iter() + .filter_map(|(modifier, plane_count)| { + let support = |access: Access, export: bool| { + self.modifier_support(plan.vk_format, modifier, access, export) + }; + let imports = |access| support(access, false).is_some_and(|s| s.importable); + let exports = |access| support(access, true).is_some_and(|s| s.exportable); + let direct_import = imports(plan.zero_copy); + let direct_export = exports(plan.zero_copy); + let zero_copy = direct_import || direct_export; + let importable = direct_import || plan.fallback.is_some_and(imports); + let exportable = direct_export || plan.fallback.is_some_and(exports); + (importable || exportable).then_some(InputModifier { + modifier, + plane_count, + zero_copy, + importable, + exportable, + }) + }) + .collect(); + Ok(InputFormats { + format, + vk_format: plan.vk_format, + modifiers, + }) + } + + /// Make an image with one of `modifiers`, which a producer can write into + /// once [exported](ExternalImage::export). + /// + /// For producers that write into buffers the consumer supplies, such as + /// `ext-image-copy-capture`. The image has the encoder's display size, + /// which is what such a protocol asks for. `modifiers` are the ones the + /// producer takes. The driver picks among those the encoder reads directly + /// (`zero_copy` in [`Self::input_formats`]); only if there are none, among + /// those it can copy from, and then each frame is copied + /// ([`ExternalImage::is_zero_copy`] says which). It fails if none of + /// `modifiers` can be allocated either way. + pub fn allocate_input(&self, modifiers: &[u64]) -> Result { + self.require_dmabuf()?; + let plan = self.input_plan()?; + let format = drm_format_of(plan.vk_format).ok_or_else(|| { + PixelForgeError::InvalidInput(format!( + "DMA-BUF input: no DRM format for {:?}", + plan.vk_format + )) + })?; + let exportable_for = |access: Access| -> Vec { + modifiers + .iter() + .copied() + .filter(|&m| { + self.modifier_support(plan.vk_format, m, access, true) + .is_some_and(|s| s.exportable) + }) + .collect() + }; + let direct = exportable_for(plan.zero_copy); + let (access, usable) = if !direct.is_empty() { + (plan.zero_copy, direct) + } else if let Some(fallback) = plan.fallback { + (fallback, exportable_for(fallback)) + } else { + (plan.zero_copy, Vec::new()) + }; + if usable.is_empty() { + return Err(PixelForgeError::NoSuitableDevice(format!( + "DMA-BUF input: none of the modifiers {modifiers:x?} can be allocated for this \ + encoder" + ))); + } + self.make_external_image( + plan.vk_format, + format, + access, + ModifierChoice::OneOf(&usable), + None, + ) + } + + /// Import a producer's DMA-BUF for this encoder to read. + /// + /// The buffer is checked here, once: a modifier the encoder reads directly + /// gives an image it does ([`ExternalImage::is_zero_copy`]); one it can only + /// copy from gives an image each frame copies; any other fails. The format + /// must be the one [`Self::input_formats`] names, or an RGB format that + /// differs from it only in whether the fourth channel is alpha. The size + /// must be at least the encoder's. + pub fn import(&self, buffer: &DmaBuf) -> Result { + self.require_dmabuf()?; + let plan = self.input_plan()?; + if vk_format_of(buffer.format) != Some(plan.vk_format) { + return Err(PixelForgeError::InvalidInput(format!( + "DMA-BUF import: format {:#x} is not {:?}, which this encoder reads", + buffer.format, plan.vk_format + ))); + } + let size = self.config().dimensions; + if buffer.width < size.width || buffer.height < size.height { + return Err(PixelForgeError::InvalidInput(format!( + "DMA-BUF import: {}x{} is smaller than the encoder's {}x{}", + buffer.width, buffer.height, size.width, size.height + ))); + } + let can_import = |access: Access| { + self.modifier_support(plan.vk_format, buffer.modifier, access, false) + .is_some_and(|s| s.importable) + }; + let access = if can_import(plan.zero_copy) { + plan.zero_copy + } else if let Some(fallback) = plan.fallback.filter(|&a| can_import(a)) { + fallback + } else { + return Err(PixelForgeError::NoSuitableDevice(format!( + "DMA-BUF import: the encoder can neither read nor copy modifier {:#x}", + buffer.modifier + ))); + }; + let layouts: Vec = buffer + .planes + .iter() + .map(|p| vk::SubresourceLayout { + offset: p.offset, + size: 0, + row_pitch: p.stride, + array_pitch: 0, + depth_pitch: 0, + }) + .collect(); + self.make_external_image( + plan.vk_format, + buffer.format, + access, + ModifierChoice::Exactly(buffer.modifier, &layouts), + Some(buffer), + ) + } + + /// Create the image and its memory: imported from `import`, or allocated + /// for export without it. + fn make_external_image( + &self, + vk_format: vk::Format, + drm: u32, + access: Access, + modifier: ModifierChoice<'_>, + import: Option<&DmaBuf>, + ) -> Result { + let context = self.context(); + let device = context.device(); + let extent = match import { + Some(buffer) => vk::Extent2D { + width: buffer.width, + height: buffer.height, + }, + None => vk::Extent2D { + width: self.config().dimensions.width, + height: self.config().dimensions.height, + }, + }; + let usage = Self::usage(access, import.is_none()); + let profile = self.inner().common().profile.info(); + let profiles = std::slice::from_ref(profile); + let mut profile_list = vk::VideoProfileListInfoKHR::default().profiles(profiles); + let mut external = vk::ExternalMemoryImageCreateInfo::default() + .handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let (mut list, mut explicit) = match modifier { + ModifierChoice::OneOf(modifiers) => ( + Some( + vk::ImageDrmFormatModifierListCreateInfoEXT::default() + .drm_format_modifiers(modifiers), + ), + None, + ), + ModifierChoice::Exactly(modifier, layouts) => ( + None, + Some( + vk::ImageDrmFormatModifierExplicitCreateInfoEXT::default() + .drm_format_modifier(modifier) + .plane_layouts(layouts), + ), + ), + }; + let mut info = vk::ImageCreateInfo::default() + .image_type(vk::ImageType::TYPE_2D) + .format(vk_format) + .extent(vk::Extent3D { + width: extent.width, + height: extent.height, + depth: 1, + }) + .mip_levels(1) + .array_layers(1) + .samples(vk::SampleCountFlags::TYPE_1) + .tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT) + .usage(usage) + .sharing_mode(vk::SharingMode::EXCLUSIVE) + .initial_layout(vk::ImageLayout::UNDEFINED) + .push(&mut external); + if let Some(list) = list.as_mut() { + info = info.push(list); + } + if let Some(explicit) = explicit.as_mut() { + info = info.push(explicit); + } + if access == Access::Encode { + info = info.push(&mut profile_list); + } + let image = unsafe { device.create_image(&info, None) } + .map_err(|e| PixelForgeError::ResourceCreation(format!("DMA-BUF image: {e}")))?; + + let memory = match self.bind_external_memory(image, import) { + Ok(memory) => memory, + Err(e) => { + unsafe { device.destroy_image(image, None) }; + return Err(e); + } + }; + + let modifier_fn = + ash::ext::image_drm_format_modifier::Device::load(context.instance(), device); + let mut modifier_props = vk::ImageDrmFormatModifierPropertiesEXT::default(); + let modifier = unsafe { + modifier_fn.get_image_drm_format_modifier_properties(image, &mut modifier_props) + } + .map(|()| modifier_props.drm_format_modifier) + .unwrap_or_else(|_| import.map_or(0, |b| b.modifier)); + let plane_count = self + .known_modifiers(vk_format) + .into_iter() + .find(|&(m, _)| m == modifier) + .map_or(1, |(_, planes)| planes); + + let view = if access == Access::Encode { + match create_video_image_view(context, image, vk_format, 0, usage) { + Ok(view) => Some(view), + Err(e) => { + unsafe { + device.destroy_image(image, None); + device.free_memory(memory, None); + } + return Err(e); + } + } + } else { + None + }; + + tracing::debug!( + "DMA-BUF {}: {}x{} {vk_format:?}, modifier {modifier:#x}, {plane_count} planes, \ + read by {access:?}", + if import.is_some() { + "imported" + } else { + "allocated" + }, + extent.width, + extent.height, + ); + Ok(ExternalImage { + context: context.clone(), + image, + memory, + view, + access, + extent, + format: drm, + modifier, + plane_count, + owner: (self.id(), self.generation()), + fresh: AtomicBool::new(true), + last_use: Mutex::new(None), + }) + } + + /// Give `image` dedicated memory, imported from `import`'s fd or + /// allocated to be exported. + fn bind_external_memory( + &self, + image: vk::Image, + import: Option<&DmaBuf>, + ) -> Result { + let context = self.context(); + let device = context.device(); + let requirements = unsafe { device.get_image_memory_requirements(image) }; + let mut dedicated = vk::MemoryDedicatedAllocateInfo::default().image(image); + let handle = vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT; + + let memory = match import { + Some(buffer) => { + let memory_fd = + ash::khr::external_memory_fd::Device::load(context.instance(), device); + let mut fd_props = vk::MemoryFdPropertiesKHR::default(); + unsafe { + memory_fd.get_memory_fd_properties(handle, buffer.fd.as_raw_fd(), &mut fd_props) + } + .map_err(|e| PixelForgeError::ResourceCreation(format!("DMA-BUF fd: {e}")))?; + let type_index = context + .find_memory_type( + requirements.memory_type_bits & fd_props.memory_type_bits, + vk::MemoryPropertyFlags::empty(), + ) + .ok_or_else(|| { + PixelForgeError::ResourceCreation( + "DMA-BUF import: no memory type fits".to_string(), + ) + })?; + // Vulkan owns the fd once the import is attempted. The spec + // gives it back on failure, but at least one driver (NVIDIA + // 615) closes it anyway, so it is never closed here: a leaked + // fd on a failed import is better than closing whatever file + // took its number. + let fd = buffer + .fd + .try_clone() + .map_err(|e| PixelForgeError::ResourceCreation(format!("DMA-BUF fd: {e}")))?; + let raw = fd.into_raw_fd(); + let mut import_info = vk::ImportMemoryFdInfoKHR::default() + .handle_type(handle) + .fd(raw); + let info = vk::MemoryAllocateInfo::default() + .allocation_size(requirements.size) + .memory_type_index(type_index) + .push(&mut import_info) + .push(&mut dedicated); + unsafe { device.allocate_memory(&info, None) }.map_err(|e| { + PixelForgeError::ResourceCreation(format!("DMA-BUF import: {e}")) + })? + } + None => { + let type_index = context + .find_memory_type( + requirements.memory_type_bits, + vk::MemoryPropertyFlags::DEVICE_LOCAL, + ) + .or_else(|| { + context.find_memory_type( + requirements.memory_type_bits, + vk::MemoryPropertyFlags::empty(), + ) + }) + .ok_or_else(|| { + PixelForgeError::ResourceCreation( + "DMA-BUF allocation: no memory type fits".to_string(), + ) + })?; + let mut export = vk::ExportMemoryAllocateInfo::default().handle_types(handle); + let info = vk::MemoryAllocateInfo::default() + .allocation_size(requirements.size) + .memory_type_index(type_index) + .push(&mut export) + .push(&mut dedicated); + unsafe { device.allocate_memory(&info, None) }.map_err(|e| { + PixelForgeError::ResourceCreation(format!("DMA-BUF allocation: {e}")) + })? + } + }; + if let Err(e) = unsafe { device.bind_image_memory(image, memory, 0) } { + unsafe { device.free_memory(memory, None) }; + return Err(PixelForgeError::ResourceCreation(format!( + "DMA-BUF bind: {e}" + ))); + } + Ok(memory) + } + + /// Encode from an [`ExternalImage`] this encoder made. + pub(crate) fn submit_external( + &mut self, + external: &ExternalImage, + mut sync: FrameSync, + ) -> Result { + if external.owner != (self.id(), self.generation()) { + return Err(PixelForgeError::InvalidInput( + "this ExternalImage was made by another encoder, or before a colour change \ + changed the conversion; import or allocate it again" + .to_string(), + )); + } + // The previous frame's release has to come before this frame's + // acquire, even when both run on the same queue. + if let Some(point) = external.last_use() { + sync.waits.push(point.wait_info()); + } + let ownership = external.ownership(); + let future = match external.access { + Access::Encode => self.encode_input( + FrameInput::Direct { + image: external.image, + view: external.view.expect("an encode-read image has a view"), + extent: external.extent, + ownership, + }, + sync, + )?, + Access::Copy => self.encode_input( + FrameInput::Copy { + image: external.image, + ownership, + }, + sync, + )?, + Access::Sample => self.convert_and_encode(external.image, ownership, sync)?, + }; + external.mark_used(future.source_released()); + Ok(future) + } +} + +/// Per-slot binary semaphores for sync_file fences. +/// +/// A slot's semaphores are used again only once the slot is free, which is +/// after every submission of its last frame has completed. +pub(crate) struct SyncFiles { + context: VideoContext, + fd_fn: ash::khr::external_semaphore_fd::Device, + acquire: Vec, + release: Vec, +} + +impl SyncFiles { + pub(crate) fn new(context: &VideoContext) -> Result { + if !context.external().sync_file { + return Err(PixelForgeError::NoSuitableDevice( + "sync_file fences need VK_KHR_external_semaphore_fd".to_string(), + )); + } + let device = context.device(); + let slots = crate::encoder::pipeline::ENCODE_PIPELINE_DEPTH; + let mut files = Self { + context: context.clone(), + fd_fn: ash::khr::external_semaphore_fd::Device::load(context.instance(), device), + acquire: Vec::with_capacity(slots), + release: Vec::with_capacity(slots), + }; + for _ in 0..slots { + let semaphore = + unsafe { device.create_semaphore(&vk::SemaphoreCreateInfo::default(), None) } + .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; + files.acquire.push(semaphore); + let mut export = vk::ExportSemaphoreCreateInfo::default() + .handle_types(vk::ExternalSemaphoreHandleTypeFlags::SYNC_FD); + let semaphore = unsafe { + device.create_semaphore(&vk::SemaphoreCreateInfo::default().push(&mut export), None) + } + .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; + files.release.push(semaphore); + } + Ok(files) + } + + /// Put `fence` into the slot's acquire semaphore, and return the wait on + /// it. + pub(crate) fn import_wait( + &mut self, + slot: usize, + fence: OwnedFd, + ) -> Result> { + let semaphore = self.acquire[slot]; + let raw = fence.into_raw_fd(); + let info = vk::ImportSemaphoreFdInfoKHR::default() + .semaphore(semaphore) + .flags(vk::SemaphoreImportFlags::TEMPORARY) + .handle_type(vk::ExternalSemaphoreHandleTypeFlags::SYNC_FD) + .fd(raw); + // Never closed here on failure, as for memory imports. + if let Err(e) = unsafe { self.fd_fn.import_semaphore_fd(&info) } { + return Err(PixelForgeError::Synchronization(format!( + "sync_file import: {e}" + ))); + } + Ok(vk::SemaphoreSubmitInfo::default() + .semaphore(semaphore) + .stage_mask(vk::PipelineStageFlags2::ALL_COMMANDS)) + } + + /// The signal of the slot's release semaphore. + pub(crate) fn release_signal(&self, slot: usize) -> vk::SemaphoreSubmitInfo<'static> { + vk::SemaphoreSubmitInfo::default() + .semaphore(self.release[slot]) + .stage_mask(vk::PipelineStageFlags2::ALL_COMMANDS) + } + + /// The slot's release semaphore as a sync_file, once a submission that + /// signals it was made. `None` when it has already signalled, which the + /// driver may report as fd -1. + pub(crate) fn export_release(&self, slot: usize) -> Result> { + let info = vk::SemaphoreGetFdInfoKHR::default() + .semaphore(self.release[slot]) + .handle_type(vk::ExternalSemaphoreHandleTypeFlags::SYNC_FD); + let fd = unsafe { self.fd_fn.get_semaphore_fd(&info) } + .map_err(|e| PixelForgeError::Synchronization(format!("sync_file export: {e}")))?; + if fd < 0 { + return Ok(None); + } + // SAFETY: vkGetSemaphoreFdKHR hands over a new fd the caller owns. + Ok(Some(unsafe { OwnedFd::from_raw_fd(fd) })) + } +} + +impl Drop for SyncFiles { + fn drop(&mut self) { + // A submission may still wait on or signal these. + for &semaphore in self.acquire.iter().chain(&self.release) { + self.context.retire_semaphore(semaphore); + } + } +} diff --git a/src/encoder/frame.rs b/src/encoder/frame.rs new file mode 100644 index 00000000..1f1e9d2f --- /dev/null +++ b/src/encoder/frame.rs @@ -0,0 +1,120 @@ +//! One frame handed to [`Encoder::submit`](crate::Encoder::submit). + +use ash::vk; + +use crate::sync::TimelinePoint; + +/// Where a frame's pixels are. +pub(crate) enum FrameSource<'a> { + /// A `vk::Image` the caller owns, in the given layout. + Image { + image: vk::Image, + layout: vk::ImageLayout, + }, + /// A DMA-BUF pixelforge imported or allocated. + #[cfg(feature = "dmabuf")] + External(&'a crate::encoder::external::ExternalImage), + #[cfg(not(feature = "dmabuf"))] + #[allow(dead_code)] + Never(std::marker::PhantomData<&'a ()>), +} + +/// One frame to encode: its source, and what has to happen on the GPU before +/// it can be read. +/// +/// ```no_run +/// # use pixelforge::{Encoder, Frame, TimelinePoint}; +/// # use ash::vk; +/// # fn f(encoder: &mut Encoder, image: vk::Image, rendered: TimelinePoint) -> Result<(), pixelforge::PixelForgeError> { +/// let packet = encoder.submit(Frame::image(image, vk::ImageLayout::GENERAL).wait(&[rendered]))?; +/// // Write `image` again only after this point. +/// let reusable = packet.source_released(); +/// # Ok(()) +/// # } +/// ``` +#[must_use] +pub struct Frame<'a> { + pub(crate) source: FrameSource<'a>, + pub(crate) waits: &'a [TimelinePoint], + #[cfg(feature = "dmabuf")] + pub(crate) wait_sync_file: Option, + #[cfg(feature = "dmabuf")] + pub(crate) signal_sync_file: bool, +} + +impl<'a> Frame<'a> { + /// A frame in `image`, a `vk::Image` the caller owns, currently in + /// `layout`. + /// + /// The image must be in the format the encoder's + /// [`Source`](crate::encoder::Source) says and have the encoder's + /// dimensions. Pixelforge reads it with a transfer, or with the colour + /// converter's shader, so it needs `TRANSFER_SRC` or `SAMPLED` usage to + /// match. + /// + /// `UNDEFINED` means a first-time external-memory import, which is + /// acquired from `VK_QUEUE_FAMILY_EXTERNAL`. The image is left in + /// `GENERAL`, owned by the queue family that read it. + /// + /// [`Encoder::input_image`](crate::Encoder::input_image) is read in place, + /// with no copy, for a YUV source. + /// + /// The image is read after [`Encoder::submit`](crate::Encoder::submit) + /// returns: keep it alive, and do not write it, until + /// [`EncodeFuture::source_released`](crate::EncodeFuture::source_released). + pub fn image(image: vk::Image, layout: vk::ImageLayout) -> Self { + Self::new(FrameSource::Image { image, layout }) + } + + /// A frame in a DMA-BUF that the encoder imported or allocated, see + /// [`Encoder::import`](crate::Encoder::import) and + /// [`Encoder::allocate_input`](crate::Encoder::allocate_input). + /// + /// It is read directly when the device can, and copied otherwise. + /// Pixelforge acquires it from the foreign queue family before it reads, + /// and releases it back after. + #[cfg(feature = "dmabuf")] + pub fn external(image: &'a crate::encoder::external::ExternalImage) -> Self { + Self::new(FrameSource::External(image)) + } + + fn new(source: FrameSource<'a>) -> Self { + Self { + source, + waits: &[], + #[cfg(feature = "dmabuf")] + wait_sync_file: None, + #[cfg(feature = "dmabuf")] + signal_sync_file: false, + } + } + + /// Read the frame only once every point in `points` is reached, on the + /// GPU. + pub fn wait(mut self, points: &'a [TimelinePoint]) -> Self { + self.waits = points; + self + } + + /// Read the frame only once `fence`, a sync_file, signals, on the GPU. + /// + /// For a DMA-BUF whose producer gives its write fence as a sync_file, + /// such as one taken from the buffer with `DMA_BUF_IOCTL_EXPORT_SYNC_FILE`. + /// Pixelforge takes ownership of the fd. + #[cfg(feature = "dmabuf")] + pub fn wait_sync_file(mut self, fence: std::os::fd::OwnedFd) -> Self { + self.wait_sync_file = Some(fence); + self + } + + /// Also give the point after which the source is free as a sync_file, + /// from [`EncodeFuture::take_release_sync_file`](crate::EncodeFuture::take_release_sync_file). + /// + /// For a producer that waits on sync_files, or to attach to the DMA-BUF + /// with `DMA_BUF_IOCTL_IMPORT_SYNC_FILE` before handing it back. + #[cfg(feature = "dmabuf")] + pub fn signal_sync_file(mut self) -> Self { + self.signal_sync_file = true; + self + } +} diff --git a/src/encoder/h264/init.rs b/src/encoder/h264/init.rs index 904a870c..29a66966 100644 --- a/src/encoder/h264/init.rs +++ b/src/encoder/h264/init.rs @@ -1,12 +1,14 @@ use super::{H264, MB_SIZE}; +use crate::converter::InputFormat; use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, }; use crate::encoder::dpb::{DecodedPictureBuffer, DecodedPictureBufferTrait, DpbConfig}; +use crate::encoder::profile::EncodeProfile; use crate::encoder::profile::h264_profile_idc; use crate::encoder::resources::MIN_BITSTREAM_BUFFER_SIZE; -use crate::encoder::rgb::{RgbConversionCaps, rgb_conversion_profile}; +use crate::encoder::rgb::RgbConversionCaps; use crate::encoder::{ColorDescription, EncodeConfig}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; @@ -16,7 +18,11 @@ use tracing::{debug, info}; impl H264 { /// Create a new H.264 encoder. - pub fn create(context: VideoContext, config: EncodeConfig) -> Result> { + pub(crate) fn create( + context: VideoContext, + config: EncodeConfig, + rgb_input: Option, + ) -> Result> { // B-frames are not yet supported. `with_b_frames` is public and accepts // any count, so this is reachable from safe caller code: report it // rather than panicking inside the library. @@ -35,36 +41,12 @@ impl H264 { // Select the profile based on chroma subsampling. let profile_idc = h264_profile_idc(config.pixel_format); - let chroma_subsampling: vk::VideoChromaSubsamplingFlagsKHR = config.pixel_format.into(); - let bit_depth: vk::VideoComponentBitDepthFlagsKHR = config.bit_depth.into(); - - // Get encoder tuning mode, usage and content hints. - let encode_usage_hint: vk::VideoEncodeUsageFlagsKHR = config.encode_usage_hint.into(); - let encode_content_hint: vk::VideoEncodeContentFlagsKHR = config.encode_content_hint.into(); - let encoder_tuning_mode: vk::VideoEncodeTuningModeKHR = config.encoder_tuning_mode.into(); - - let mut video_encode_usage_info = vk::VideoEncodeUsageInfoKHR::default() - .video_usage_hints(encode_usage_hint) - .video_content_hints(encode_content_hint) - .tuning_mode(encoder_tuning_mode); - - let mut h264_profile_info = - vk::VideoEncodeH264ProfileInfoKHR::default().std_profile_idc(profile_idc); - let mut rgb_profile = rgb_conversion_profile(); - let mut profile_info = vk::VideoProfileInfoKHR::default() - .video_codec_operation(vk::VideoCodecOperationFlagsKHR::ENCODE_H264) - .chroma_subsampling(chroma_subsampling) - .luma_bit_depth(bit_depth) - .chroma_bit_depth(bit_depth) - .push(&mut h264_profile_info) - .push(&mut video_encode_usage_info); - if config.rgb_input.is_some() { - profile_info = profile_info.push(&mut rgb_profile); - } + let profile = EncodeProfile::new(&config, rgb_input.is_some())?; + let profile_info = profile.info(); // The driver's preferred entropy mode is H.264-specific (some require // CAVLC for High 4:4:4 Predictive). - let preferred_entropy_cabac = query_preferred_entropy_cabac(&context, &profile_info); + let preferred_entropy_cabac = query_preferred_entropy_cabac(&context, profile_info); // Query device capabilities with the H.264-specific capability struct // chained in (required by the driver). @@ -74,20 +56,21 @@ impl H264 { .push(&mut encode_caps) .push(&mut h264_caps); let mut rgb_caps = vk::VideoEncodeRgbConversionCapabilitiesVALVE::default(); - if config.rgb_input.is_some() { + if rgb_input.is_some() { capabilities = capabilities.push(&mut rgb_caps); } - let caps = query_video_caps(&context, &profile_info, &mut capabilities)?; - let rgb_caps = config.rgb_input.map(|_| RgbConversionCaps::from(&rgb_caps)); + let caps = query_video_caps(&context, profile_info, &mut capabilities)?; + let rgb_caps = rgb_input.map(|_| RgbConversionCaps::from(&rgb_caps)); // Scale the bitstream buffer by resolution so a single frame can't overflow (matches AV1). let bitstream_buffer_size = MIN_BITSTREAM_BUFFER_SIZE .max(config.dimensions.width as usize * config.dimensions.height as usize); - let init = build_encoder_common(&CommonInitRequest { + let init = build_encoder_common(CommonInitRequest { context: &context, config: &config, - profile_info: &profile_info, + profile, + rgb_input, caps: &caps, align_unit: MB_SIZE, max_active_refs_cap: 32, diff --git a/src/encoder/h264/record.rs b/src/encoder/h264/record.rs index 5f100401..7412715d 100644 --- a/src/encoder/h264/record.rs +++ b/src/encoder/h264/record.rs @@ -31,7 +31,7 @@ impl H264 { let timestamp_query_pool = slot.timestamp_query_pool; let bitstream_buffer = slot.bitstream_buffer; let bitstream_buffer_size = slot.bitstream_buffer_size; - let input_image_view = slot.input_image_view; + let input_image_view = common.source_view(); debug!( "h264 record: frame_num={}, poc={}, is_idr={}, refs_len={}, cur_slot={}", @@ -95,6 +95,7 @@ impl H264 { &ref_dpb_slots, common.dpb_slot_active[common.current_dpb_slot as usize], ); + common.record_source_acquire(command_buffer); } let slice_type = if is_idr { @@ -259,7 +260,7 @@ impl H264 { let src_picture_resource = vk::VideoPictureResourceInfoKHR::default() .coded_offset(vk::Offset2D { x: 0, y: 0 }) - .coded_extent(coded_extent) + .coded_extent(common.source_coded_extent()) .base_array_layer(0) .image_view_binding(input_image_view); @@ -532,6 +533,8 @@ impl H264 { let end_info = vk::VideoEndCodingInfoKHR::default(); (common.video_queue_fn.fp().cmd_end_video_coding_khr)(command_buffer, &end_info); + common.record_source_release(command_buffer); + device .end_command_buffer(command_buffer) .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; diff --git a/src/encoder/h265/init.rs b/src/encoder/h265/init.rs index fa0167a1..73a15d94 100644 --- a/src/encoder/h265/init.rs +++ b/src/encoder/h265/init.rs @@ -1,12 +1,14 @@ use super::{CTB_SIZE, H265}; +use crate::converter::InputFormat; use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, }; use crate::encoder::dpb::{DecodedPictureBuffer, DecodedPictureBufferTrait, DpbConfig}; +use crate::encoder::profile::EncodeProfile; use crate::encoder::profile::h265_profile_idc; use crate::encoder::resources::MIN_BITSTREAM_BUFFER_SIZE; -use crate::encoder::rgb::{RgbConversionCaps, rgb_conversion_profile}; +use crate::encoder::rgb::RgbConversionCaps; use crate::encoder::{ColorDescription, EncodeConfig}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; @@ -16,7 +18,11 @@ use tracing::info; impl H265 { /// Create a new H.265/HEVC encoder. - pub fn create(context: VideoContext, config: EncodeConfig) -> Result> { + pub(crate) fn create( + context: VideoContext, + config: EncodeConfig, + rgb_input: Option, + ) -> Result> { // Reachable from safe caller code via `with_b_frames`, so report it // rather than panicking inside the library. if config.b_frame_count != 0 { @@ -39,32 +45,8 @@ impl H265 { )) })?; - let chroma_subsampling: vk::VideoChromaSubsamplingFlagsKHR = config.pixel_format.into(); - let bit_depth: vk::VideoComponentBitDepthFlagsKHR = config.bit_depth.into(); - - // Get encoder tuning mode, usage and content hints. - let encode_usage_hint: vk::VideoEncodeUsageFlagsKHR = config.encode_usage_hint.into(); - let encode_content_hint: vk::VideoEncodeContentFlagsKHR = config.encode_content_hint.into(); - let encoder_tuning_mode: vk::VideoEncodeTuningModeKHR = config.encoder_tuning_mode.into(); - - let mut video_encode_usage_info = vk::VideoEncodeUsageInfoKHR::default() - .video_usage_hints(encode_usage_hint) - .video_content_hints(encode_content_hint) - .tuning_mode(encoder_tuning_mode); - - let mut h265_profile_info = - vk::VideoEncodeH265ProfileInfoKHR::default().std_profile_idc(profile_idc); - let mut rgb_profile = rgb_conversion_profile(); - let mut profile_info = vk::VideoProfileInfoKHR::default() - .video_codec_operation(vk::VideoCodecOperationFlagsKHR::ENCODE_H265) - .chroma_subsampling(chroma_subsampling) - .luma_bit_depth(bit_depth) - .chroma_bit_depth(bit_depth) - .push(&mut h265_profile_info) - .push(&mut video_encode_usage_info); - if config.rgb_input.is_some() { - profile_info = profile_info.push(&mut rgb_profile); - } + let profile = EncodeProfile::new(&config, rgb_input.is_some())?; + let profile_info = profile.info(); // Query device capabilities with the H.265-specific capability struct // chained in (required by the driver). @@ -74,20 +56,21 @@ impl H265 { .push(&mut encode_caps) .push(&mut h265_caps); let mut rgb_caps = vk::VideoEncodeRgbConversionCapabilitiesVALVE::default(); - if config.rgb_input.is_some() { + if rgb_input.is_some() { capabilities = capabilities.push(&mut rgb_caps); } - let caps = query_video_caps(&context, &profile_info, &mut capabilities)?; - let rgb_caps = config.rgb_input.map(|_| RgbConversionCaps::from(&rgb_caps)); + let caps = query_video_caps(&context, profile_info, &mut capabilities)?; + let rgb_caps = rgb_input.map(|_| RgbConversionCaps::from(&rgb_caps)); // Scale the bitstream buffer by resolution so a single frame can't overflow (matches AV1). let bitstream_buffer_size = MIN_BITSTREAM_BUFFER_SIZE .max(config.dimensions.width as usize * config.dimensions.height as usize); - let init = build_encoder_common(&CommonInitRequest { + let init = build_encoder_common(CommonInitRequest { context: &context, config: &config, - profile_info: &profile_info, + profile, + rgb_input, caps: &caps, align_unit: CTB_SIZE, max_active_refs_cap: 15, diff --git a/src/encoder/h265/record.rs b/src/encoder/h265/record.rs index 10ce65b6..2a7472ed 100644 --- a/src/encoder/h265/record.rs +++ b/src/encoder/h265/record.rs @@ -31,7 +31,7 @@ impl H265 { let timestamp_query_pool = slot.timestamp_query_pool; let bitstream_buffer = slot.bitstream_buffer; let bitstream_buffer_size = slot.bitstream_buffer_size; - let input_image_view = slot.input_image_view; + let input_image_view = common.source_view(); unsafe { prepare_encode_command_buffer(common.device(), command_buffer, query_pool)?; @@ -47,6 +47,7 @@ impl H265 { &ref_dpb_slots, common.dpb_slot_active[common.current_dpb_slot as usize], ); + common.record_source_acquire(command_buffer); } let slice_type = if is_idr { @@ -293,7 +294,7 @@ impl H265 { let src_picture_resource = vk::VideoPictureResourceInfoKHR::default() .coded_offset(vk::Offset2D { x: 0, y: 0 }) - .coded_extent(coded_extent) + .coded_extent(common.source_coded_extent()) .base_array_layer(0) .image_view_binding(input_image_view); @@ -528,6 +529,8 @@ impl H265 { let end_coding_info = vk::VideoEndCodingInfoKHR::default(); (common.video_queue_fn.fp().cmd_end_video_coding_khr)(command_buffer, &end_coding_info); + common.record_source_release(command_buffer); + device .end_command_buffer(command_buffer) .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; diff --git a/src/encoder/input.rs b/src/encoder/input.rs new file mode 100644 index 00000000..5f1c3b5f --- /dev/null +++ b/src/encoder/input.rs @@ -0,0 +1,152 @@ +//! How one frame's source reaches the encoder, inside the crate. +//! +//! [`Encoder::submit`](crate::Encoder::submit) turns a public +//! [`Frame`](crate::Frame) into a [`FrameInput`] for the codec encoder: the +//! slot's own input image, an image to copy into it, or an image the encode +//! reads directly. The submission that reads a source also does its queue +//! family ownership transfers, which [`SourceOwnership`] describes. + +use ash::vk; + +/// The layout a source is in, and the queue family transfers around the one +/// submission that reads it. +#[derive(Debug, Clone, Copy)] +pub(crate) struct SourceOwnership { + /// Layout the source is in when the reading submission starts. + pub old_layout: vk::ImageLayout, + /// Queue family to acquire the source from, if another one owns it + /// (`EXTERNAL` or `FOREIGN_EXT` for memory shared with another API or + /// process). + pub acquire_from: Option, + /// Queue family to release the source to once it has been read. + pub release_to: Option, +} + +impl SourceOwnership { + /// A caller's image in `layout`. `UNDEFINED` means a first-time + /// external-memory import, which is acquired from `EXTERNAL` and then + /// stays with the queue that read it. + pub(crate) fn caller_image(layout: vk::ImageLayout) -> Self { + Self { + old_layout: layout, + acquire_from: (layout == vk::ImageLayout::UNDEFINED) + .then_some(vk::QUEUE_FAMILY_EXTERNAL), + release_to: None, + } + } + + /// The barrier that brings the source into `new_layout` for `family`, + /// acquiring it first when another family owns it. + pub(crate) fn acquire_barrier( + &self, + image: vk::Image, + family: u32, + new_layout: vk::ImageLayout, + dst_stage: vk::PipelineStageFlags2, + dst_access: vk::AccessFlags2, + ) -> vk::ImageMemoryBarrier2<'static> { + let (src_family, dst_family) = match self.acquire_from { + Some(from) => (from, family), + None => (vk::QUEUE_FAMILY_IGNORED, vk::QUEUE_FAMILY_IGNORED), + }; + // An acquire has no access of this queue's to make available, and + // anything before it on this queue is ordered by the semaphore waits. + let (src_stage, src_access) = if self.acquire_from.is_some() { + (vk::PipelineStageFlags2::NONE, vk::AccessFlags2::NONE) + } else { + ( + vk::PipelineStageFlags2::ALL_COMMANDS, + vk::AccessFlags2::MEMORY_READ | vk::AccessFlags2::MEMORY_WRITE, + ) + }; + vk::ImageMemoryBarrier2::default() + .src_stage_mask(src_stage) + .src_access_mask(src_access) + .dst_stage_mask(dst_stage) + .dst_access_mask(dst_access) + .old_layout(self.old_layout) + .new_layout(new_layout) + .src_queue_family_index(src_family) + .dst_queue_family_index(dst_family) + .image(image) + .subresource_range(color_range()) + } + + /// The barrier that leaves the source in `GENERAL` after it was read in + /// `layout`, releasing it when another family takes it back. + pub(crate) fn release_barrier( + &self, + image: vk::Image, + family: u32, + layout: vk::ImageLayout, + src_stage: vk::PipelineStageFlags2, + ) -> vk::ImageMemoryBarrier2<'static> { + let (src_family, dst_family, dst_stage, dst_access) = match self.release_to { + // The acquiring side's barrier does the rest. + Some(to) => ( + family, + to, + vk::PipelineStageFlags2::NONE, + vk::AccessFlags2::NONE, + ), + None => ( + vk::QUEUE_FAMILY_IGNORED, + vk::QUEUE_FAMILY_IGNORED, + vk::PipelineStageFlags2::ALL_COMMANDS, + vk::AccessFlags2::MEMORY_READ | vk::AccessFlags2::MEMORY_WRITE, + ), + }; + vk::ImageMemoryBarrier2::default() + .src_stage_mask(src_stage) + .src_access_mask(vk::AccessFlags2::NONE) + .dst_stage_mask(dst_stage) + .dst_access_mask(dst_access) + .old_layout(layout) + .new_layout(vk::ImageLayout::GENERAL) + .src_queue_family_index(src_family) + .dst_queue_family_index(dst_family) + .image(image) + .subresource_range(color_range()) + } +} + +/// The source of one frame, as the codec encoder sees it. +#[derive(Debug, Clone, Copy)] +pub(crate) enum FrameInput { + /// The current slot's input image, already written. + InPlace, + /// Copy `image` into the slot's input image first. + Copy { + image: vk::Image, + ownership: SourceOwnership, + }, + /// Encode straight from `view`, an image made with the session's profile. + #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] + Direct { + image: vk::Image, + view: vk::ImageView, + /// The image's size, which may be smaller than the aligned coded + /// extent. + extent: vk::Extent2D, + ownership: SourceOwnership, + }, +} + +/// Semaphore operations one frame adds to the submissions pixelforge makes. +#[derive(Default)] +pub(crate) struct FrameSync { + /// What the frame's first submission waits for. + pub waits: Vec>, + /// What the submission that is last to read the source signals. + pub release_signals: Vec>, +} + +fn color_range() -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_mip_level: 0, + level_count: 1, + base_array_layer: 0, + layer_count: 1, + } +} diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index 9f99082d..ef323e7d 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -5,18 +5,24 @@ //! - GOP structure management (`gop` module) - reusable for H.264/H.265. //! - Frame reordering for B-frame support (`reorder` module) - reusable for H.264/H.265. +mod api; pub mod av1; pub mod bitwriter; pub(crate) mod codec; pub mod dpb; +#[cfg(feature = "dmabuf")] +pub mod external; +mod frame; pub mod gop; pub mod h264; pub mod h265; +pub(crate) mod input; pub(crate) mod pipeline; pub(crate) mod profile; pub mod reorder; pub mod resources; pub(crate) mod rgb; +mod source; pub(crate) mod support; use ash::vk; @@ -44,10 +50,6 @@ pub const DEFAULT_H265_QP: u32 = 28; /// Default maximum number of reference frames. pub const DEFAULT_MAX_REFERENCE_FRAMES: u32 = 4; -use crate::error::Result; -use crate::sync::TimelinePoint; -use crate::vulkan::VideoContext; - /// Video codec types. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Codec { @@ -345,9 +347,12 @@ pub struct EncodeConfig { /// Color description for VUI signaling. /// Defaults to BT.709, limited range, when `None`. pub color_description: Option, - /// Take RGB input and let the encoder convert it to YUV itself, through - /// `VK_VALVE_video_encode_rgb_conversion`. See [`Self::with_rgb_input`]. - pub rgb_input: Option, + /// What the frames handed to the encoder are. YUV by default; see + /// [`Self::with_source`]. + pub source: Source, + /// Which conversion to use for an RGB source; see + /// [`Self::with_conversion`]. + pub conversion: ConversionPreference, /// Usage hint for encoding. pub encode_usage_hint: EncodeUsageHint, /// Content hint for encoding. @@ -379,7 +384,8 @@ impl EncodeConfig { virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, color_description: None, - rgb_input: None, + source: Source::Yuv, + conversion: ConversionPreference::Auto, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, encoder_tuning_mode: EncoderTuningMode::Default, @@ -408,7 +414,8 @@ impl EncodeConfig { virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, color_description: None, - rgb_input: None, + source: Source::Yuv, + conversion: ConversionPreference::Auto, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, encoder_tuning_mode: EncoderTuningMode::Default, @@ -437,7 +444,8 @@ impl EncodeConfig { virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, color_description: None, - rgb_input: None, + source: Source::Yuv, + conversion: ConversionPreference::Auto, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, encoder_tuning_mode: EncoderTuningMode::Default, @@ -526,24 +534,21 @@ impl EncodeConfig { self } - /// Take `format` RGB images as input and convert them to YUV in the - /// encoder itself, with no colour conversion shader. - /// - /// Needs `VK_VALVE_video_encode_rgb_conversion` (see - /// [`VideoContext::has_video_encode_rgb_conversion`]), and a driver that - /// accepts `format` as encode input for this codec, profile and bit depth; - /// [`Encoder::new`] fails otherwise. The YUV matrix and range come from - /// [`Self::color_description`], so set that to what the stream should - /// carry. The hardware applies only the matrix: a source that needs a - /// transfer function or gamut change still has to go through - /// [`ColorConverter`](crate::ColorConverter), which - /// [`ColorConverterConfig::rgb_encode_input`](crate::ColorConverterConfig::rgb_encode_input) - /// decides for you. + /// Say what the frames handed to the encoder are. /// - /// With this set, [`Encoder::input_image`] is an RGB image, and - /// [`Encoder::encode`] copies an RGB source into it. - pub fn with_rgb_input(mut self, format: crate::converter::InputFormat) -> Self { - self.rgb_input = Some(format); + /// For [`Source::Rgb`], the encoder converts them to the YUV that + /// [`Self::color_description`] describes, which then also picks the + /// conversion's target: BT.709 for SDR, BT.2020 with PQ for HDR10. + /// [`Encoder::new`] picks how, see [`Conversion`]. + pub fn with_source(mut self, source: Source) -> Self { + self.source = source; + self + } + + /// Choose the conversion for an RGB source, rather than let + /// [`Encoder::new`] pick. For tests, and for comparing the two. + pub fn with_conversion(mut self, preference: ConversionPreference) -> Self { + self.conversion = preference; self } @@ -566,7 +571,10 @@ impl EncodeConfig { } } +pub use api::Encoder; +pub use frame::Frame; pub use pipeline::EncodeFuture; +pub use source::{Conversion, ConversionPreference, RgbSource, Source}; /// Statistic about the encoded video packet. #[derive(Debug, Clone)] @@ -596,198 +604,6 @@ pub struct EncodedPacket { pub stats: Option, } -/// The codec-erased operations every [`codec::CodecEncoder`] exposes. -/// -/// One blanket impl covers all codecs, so [`Encoder`] can hold any of them -/// behind a single boxed pointer instead of an enum that re-dispatches by hand. -trait EncoderApi: Send { - fn input_image(&self) -> vk::Image; - fn encode_after( - &mut self, - src_image: vk::Image, - src_layout: vk::ImageLayout, - wait: &[TimelinePoint], - ) -> Result; - fn flush(&mut self) -> Result<()>; - fn request_idr(&mut self); - fn invalidate_reference_frames(&mut self, first_lost_display_order: u64); - fn set_color_description(&mut self, desc: ColorDescription) -> Result<()>; -} - -impl EncoderApi for codec::CodecEncoder { - fn input_image(&self) -> vk::Image { - codec::CodecEncoder::input_image(self) - } - fn encode_after( - &mut self, - src_image: vk::Image, - src_layout: vk::ImageLayout, - wait: &[TimelinePoint], - ) -> Result { - codec::CodecEncoder::encode_after(self, src_image, src_layout, wait) - } - fn flush(&mut self) -> Result<()> { - codec::CodecEncoder::flush(self) - } - fn request_idr(&mut self) { - codec::CodecEncoder::request_idr(self) - } - fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) { - codec::CodecEncoder::invalidate_reference_frames(self, first_lost_display_order) - } - fn set_color_description(&mut self, desc: ColorDescription) -> Result<()> { - codec::CodecEncoder::set_color_description(self, desc) - } -} - -/// Video encoder supporting multiple codecs. -/// -/// Constructed via [`Encoder::new`], which selects the codec from the config and -/// boxes the corresponding `codec::CodecEncoder`. All codecs share one generic -/// implementation; this type just erases which one is in use. -pub struct Encoder(Box); - -impl Encoder { - /// Create a new encoder for the codec named in `config`. - pub fn new(context: VideoContext, config: EncodeConfig) -> Result { - if let Some(format) = config.rgb_input { - rgb::check_rgb_input(&context, &config, format)?; - } - let inner: Box = match config.codec { - Codec::H264 => Box::new(self::h264::H264::create(context, config)?), - Codec::H265 => Box::new(self::h265::H265::create(context, config)?), - Codec::AV1 => Box::new(self::av1::Av1::create(context, config)?), - }; - Ok(Encoder(inner)) - } - - /// Get the internal input image. - /// - /// This image can be used as a target for `ColorConverter::convert` to avoid - /// an intermediate copy. - pub fn input_image(&self) -> vk::Image { - self.0.input_image() - } - - /// Encode a frame from a GPU image. - /// - /// This accepts a source NV12 (YUV420) or planar YUV444 image on the GPU and encodes it directly. - /// The source image must match the format and dimensions in the encoder configuration. - /// - /// Encoding is asynchronous: this submits the frame without blocking and - /// returns an [`EncodeFuture`] that resolves with the encoded packet once the - /// GPU finishes and a background readback thread reads it back. The call only - /// blocks if every pipeline slot is still in flight (backpressure). - /// - /// Use `InputImage` to create an image from YUV data: - /// ```no_run - /// use pixelforge::{InputImage, Encoder, EncodeConfig, EncodeBitDepth, PixelFormat, VideoContext, Codec}; - /// - /// # fn example(context: VideoContext) -> Result<(), Box> { - /// let config = EncodeConfig::h264(1920, 1080); - /// let mut encoder = Encoder::new(context.clone(), config)?; - /// let mut input = InputImage::new( - /// context, - /// Codec::H264, - /// 1920, - /// 1080, - /// EncodeBitDepth::Eight, - /// PixelFormat::Yuv420, - /// )?; - /// - /// // Upload YUV420 data to the input image - /// # let yuv_data = vec![0u8; 1920 * 1080 * 3 / 2]; - /// input.upload_yuv420(&yuv_data)?; - /// - /// // Submit the frame and await its packet. - /// let future = encoder.encode(input.image())?; - /// let packet = pollster::block_on(future)?; - /// // ... use packet.data ... - /// # Ok(()) - /// # } - /// ``` - pub fn encode(&mut self, src_image: vk::Image) -> Result { - self.encode_after(src_image, vk::ImageLayout::GENERAL, &[]) - } - - /// Encode a frame once GPU work the caller submitted has finished. - /// - /// Like [`Self::encode`], but the frame's first GPU submission waits for - /// every point in `wait` before touching `src_image`. Pass the point - /// returned by [`ColorConverter::convert_async`](crate::ColorConverter::convert_async) - /// to encode what it converted without waiting for the conversion on the - /// CPU, or a point the caller's own rendering signals. - /// - /// Only a `src_image` that is [`Self::input_image`] avoids the CPU - /// entirely. Any other image is first copied into the input image, and - /// that copy is waited for on the CPU before the encode is recorded, so - /// this call then blocks until the points in `wait` are reached and the - /// copy is done. Render or convert into [`Self::input_image`] to keep the - /// whole handover on the GPU. - /// - /// `src_layout` is the layout `src_image` is in, as for - /// [`ColorConverter::convert_async`](crate::ColorConverter::convert_async): - /// `UNDEFINED` is a first-time external-memory import, which the copy - /// acquires from `VK_QUEUE_FAMILY_EXTERNAL`. The copy leaves the image in - /// `GENERAL`, owned by - /// [`VideoContext::transfer_queue_family`](crate::VideoContext::transfer_queue_family). - /// It is ignored for [`Self::input_image`]. - pub fn encode_after( - &mut self, - src_image: vk::Image, - src_layout: vk::ImageLayout, - wait: &[TimelinePoint], - ) -> Result { - self.0.encode_after(src_image, src_layout, wait) - } - - /// Wait for all in-flight frames to finish encoding (end-of-stream barrier). - /// - /// Packets are delivered through the [`EncodeFuture`]s returned by - /// [`Encoder::encode`]; await those to obtain them. Once this returns, every - /// outstanding future has been resolved. - pub fn flush(&mut self) -> Result<()> { - self.0.flush() - } - - /// Request that the next frame be an IDR frame. - pub fn request_idr(&mut self) { - self.0.request_idr() - } - - /// Recover from packet loss without a full IDR (reference frame invalidation). - /// - /// When a client reports that it could not decode a range of frames, every - /// reference picture the encoder still holds from the earliest lost frame - /// onward is transitively undecodable on the client. This drops those - /// references so the next P-frame is predicted from the most recent - /// *surviving* reference instead — a much cheaper recovery than re-sending a - /// full keyframe. If no reference survives (the loss covers the encoder's - /// whole reference window), it transparently falls back to forcing an IDR. - /// - /// `first_lost_display_order` is the display order — the `pts` reported on - /// [`EncodedPacket`]s — of the earliest frame the client lost. - /// - /// Effective recovery requires the encoder to keep more than one reference - /// (see [`EncodeConfig::with_max_reference_frames`]); with a single - /// reference this necessarily falls back to an IDR. This applies uniformly - /// across H.264, H.265, and AV1 — each keeps a multi-reference window and - /// re-anchors prediction to the most recent survivor, falling back to an - /// IDR (AV1: key frame) only when the loss covers the entire window. - pub fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) { - self.0.invalidate_reference_frames(first_lost_display_order) - } - - /// Update the color description (VUI parameters) for the encoder. - /// - /// This recreates the video session parameters with an updated SPS/VPS/sequence - /// header containing the new color description. The next frame will be encoded as - /// an IDR/key frame with the new parameters. - pub fn set_color_description(&mut self, desc: ColorDescription) -> Result<()> { - self.0.set_color_description(desc) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/encoder/pipeline.rs b/src/encoder/pipeline.rs index 4d6ea167..4551a6f0 100644 --- a/src/encoder/pipeline.rs +++ b/src/encoder/pipeline.rs @@ -64,14 +64,56 @@ use crate::vulkan::VideoContext; /// still reads the slot back (its send simply finds no receiver) and frees it. pub struct EncodeFuture { rx: oneshot::Receiver>, + released: TimelinePoint, + /// Added to the packet's timestamps, so they carry on across an encoder + /// that was made again for a colour change. + pub(crate) pts_offset: u64, + #[cfg(feature = "dmabuf")] + pub(crate) release_sync_file: Option, +} + +impl EncodeFuture { + /// The point after which this frame's source may be written again, or + /// handed back to whatever produced it. + /// + /// Which submission reaches it depends on how the source was read: the + /// copy into the encoder's input image, the colour conversion, or the + /// encode itself for a source read directly. It is usually long before the + /// packet is ready. Waiting on it on the GPU, before the next write to the + /// source, is enough. + /// + /// The semaphore stays valid for as long as the encoder's + /// [`VideoContext`] does. + pub fn source_released(&self) -> TimelinePoint { + self.released + } + + /// The sync_file asked for with + /// [`Frame::signal_sync_file`](crate::Frame::signal_sync_file): it + /// signals at [`Self::source_released`]. `None` if it was not asked for, + /// was already taken, or had already signalled when it was made, which + /// the driver may report that way. + #[cfg(feature = "dmabuf")] + pub fn take_release_sync_file(&mut self) -> Option { + self.release_sync_file.take() + } + + pub(crate) fn set_released(&mut self, point: TimelinePoint) { + self.released = point; + } } impl Future for EncodeFuture { type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let offset = self.pts_offset; match Pin::new(&mut self.rx).poll(cx) { - Poll::Ready(Ok(result)) => Poll::Ready(result), + Poll::Ready(Ok(result)) => Poll::Ready(result.map(|mut packet| { + packet.pts += offset; + packet.dts += offset; + packet + })), // The sender was dropped without sending. The completion thread only // drops a sender after sending, so this indicates encoder teardown. Poll::Ready(Err(oneshot::Canceled)) => { @@ -363,7 +405,8 @@ impl EncodePipeline { device: &ash::Device, sync2: &ash::khr::synchronization2::Device, encode_queue: vk::Queue, - external_waits: &[TimelinePoint], + external_waits: &[vk::SemaphoreSubmitInfo<'static>], + extra_signals: &[vk::SemaphoreSubmitInfo<'static>], ) -> Result { let wait = self.timeline.wait(); let signal = self.timeline.pending_signal(); @@ -409,8 +452,19 @@ impl EncodePipeline { .value(value) .stage_mask(vk::PipelineStageFlags2::VIDEO_ENCODE_KHR) }) - .chain(external_waits.iter().map(TimelinePoint::wait_info)) + .chain(external_waits.iter().copied()) .collect(); + // ALL_COMMANDS rather than the encode stage: the timeline value is + // also handed out as the point after which a directly read source is + // free, and its release barrier comes after the encode. + let signals: Vec = std::iter::once( + vk::SemaphoreSubmitInfo::default() + .semaphore(signal.0) + .value(signal.1) + .stage_mask(vk::PipelineStageFlags2::ALL_COMMANDS), + ) + .chain(extra_signals.iter().copied()) + .collect(); unsafe { submit_encode_only( device, @@ -419,7 +473,7 @@ impl EncodePipeline { fence, encode_queue, &waits, - Some(signal), + &signals, )?; } let submit_time = std::time::Instant::now(); @@ -449,7 +503,24 @@ impl EncodePipeline { let _ = tx.send(work); } - Ok(EncodeFuture { rx: result_rx }) + Ok(EncodeFuture { + rx: result_rx, + released: TimelinePoint::new(signal.0, signal.1), + pts_offset: 0, + #[cfg(feature = "dmabuf")] + release_sync_file: None, + }) + } + + /// Index of the slot the next frame will be encoded into. + pub(crate) fn current_index(&self) -> usize { + self.current_slot + } + + /// The timeline the encodes signal. Retired to the context rather than + /// destroyed, since a caller may still wait on a point it handed out. + pub(crate) fn timeline_semaphore(&self) -> vk::Semaphore { + self.timeline.semaphore() } /// Advance to the next slot after a frame has been submitted. @@ -478,7 +549,8 @@ impl EncodePipeline { } } - /// Destroy all slot resources and the timeline semaphore. + /// Destroy all slot resources. The timeline semaphore is left for the + /// caller to retire, see [`Self::timeline_semaphore`]. /// /// # Safety /// @@ -509,9 +581,6 @@ impl EncodePipeline { device.free_memory(slot.input_image_memory, None); } } - unsafe { - self.timeline.destroy(device); - } } } diff --git a/src/encoder/profile.rs b/src/encoder/profile.rs index 6d6ba9a7..9fe321a7 100644 --- a/src/encoder/profile.rs +++ b/src/encoder/profile.rs @@ -4,7 +4,9 @@ //! both choose their profile here, so the query answers for exactly the profile an //! encoder would request. -use super::{BitDepth, PixelFormat}; +use super::{BitDepth, Codec, EncodeConfig, PixelFormat}; +use crate::error::{PixelForgeError, Result}; +use ash::vk; /// H.264 profile for a pixel format: High 4:4:4 Predictive for 4:4:4, High otherwise. pub(crate) fn h264_profile_idc(pixel_format: PixelFormat) -> u32 { @@ -40,3 +42,90 @@ pub(crate) fn av1_profile(pixel_format: PixelFormat) -> u32 { _ => ash::vk::native::StdVideoAV1Profile_STD_VIDEO_AV1_PROFILE_HIGH, } } + +/// The video profile an encoder uses, owned so it can be named again after +/// creation. +/// +/// Everything that touches the encoder's pictures has to name the session's +/// exact profile: the session itself, its input and DPB images, its query +/// pools, and any image a caller hands over to be read directly. The chain +/// lives in one heap allocation, so the pointers between its structs stay +/// valid for as long as this does. +pub(crate) struct EncodeProfile(Box); + +struct ProfileChain { + info: vk::VideoProfileInfoKHR<'static>, + h264: vk::VideoEncodeH264ProfileInfoKHR<'static>, + h265: vk::VideoEncodeH265ProfileInfoKHR<'static>, + av1: vk::VideoEncodeAV1ProfileInfoKHR<'static>, + usage: vk::VideoEncodeUsageInfoKHR<'static>, + rgb: vk::VideoEncodeProfileRgbConversionInfoVALVE<'static>, +} + +// SAFETY: the raw pointers in the chain only point into the same boxed +// allocation, which moves with this value and is never shared. +unsafe impl Send for EncodeProfile {} + +impl EncodeProfile { + /// The profile `config` asks for, with the RGB conversion struct when the + /// encoder takes RGB input. + pub(crate) fn new(config: &EncodeConfig, rgb_input: bool) -> Result { + let operation = match config.codec { + Codec::H264 => vk::VideoCodecOperationFlagsKHR::ENCODE_H264, + Codec::H265 => vk::VideoCodecOperationFlagsKHR::ENCODE_H265, + Codec::AV1 => vk::VideoCodecOperationFlagsKHR::ENCODE_AV1, + }; + let h265_idc = match config.codec { + Codec::H265 => { + h265_profile_idc(config.pixel_format, config.bit_depth).ok_or_else(|| { + PixelForgeError::InvalidInput(format!( + "Unsupported pixel format / bit depth for H.265: {:?} / {:?}", + config.pixel_format, config.bit_depth + )) + })? + } + _ => 0, + }; + let bit_depth: vk::VideoComponentBitDepthFlagsKHR = config.bit_depth.into(); + let mut chain = Box::new(ProfileChain { + info: vk::VideoProfileInfoKHR::default() + .video_codec_operation(operation) + .chroma_subsampling(config.pixel_format.into()) + .luma_bit_depth(bit_depth) + .chroma_bit_depth(bit_depth), + h264: vk::VideoEncodeH264ProfileInfoKHR::default() + .std_profile_idc(h264_profile_idc(config.pixel_format)), + h265: vk::VideoEncodeH265ProfileInfoKHR::default().std_profile_idc(h265_idc), + av1: vk::VideoEncodeAV1ProfileInfoKHR::default() + .std_profile(av1_profile(config.pixel_format)), + usage: vk::VideoEncodeUsageInfoKHR::default() + .video_usage_hints(config.encode_usage_hint.into()) + .video_content_hints(config.encode_content_hint.into()) + .tuning_mode(config.encoder_tuning_mode.into()), + rgb: vk::VideoEncodeProfileRgbConversionInfoVALVE::default() + .perform_encode_rgb_conversion(true), + }); + + // info -> codec profile -> usage [-> rgb] + let chain_ref = &mut *chain; + let codec_profile: *mut vk::BaseOutStructure = match config.codec { + Codec::H264 => (&mut chain_ref.h264 as *mut vk::VideoEncodeH264ProfileInfoKHR).cast(), + Codec::H265 => (&mut chain_ref.h265 as *mut vk::VideoEncodeH265ProfileInfoKHR).cast(), + Codec::AV1 => (&mut chain_ref.av1 as *mut vk::VideoEncodeAV1ProfileInfoKHR).cast(), + }; + let usage: *mut vk::BaseOutStructure = + (&mut chain_ref.usage as *mut vk::VideoEncodeUsageInfoKHR).cast(); + chain_ref.info.p_next = codec_profile.cast_const().cast(); + unsafe { (*codec_profile).p_next = usage }; + if rgb_input { + chain_ref.usage.p_next = + (&chain_ref.rgb as *const vk::VideoEncodeProfileRgbConversionInfoVALVE).cast(); + } + Ok(Self(chain)) + } + + /// The profile, with its chain. + pub(crate) fn info(&self) -> &vk::VideoProfileInfoKHR<'_> { + &self.0.info + } +} diff --git a/src/encoder/resources.rs b/src/encoder/resources.rs index 805a5c9c..a3555bfc 100644 --- a/src/encoder/resources.rs +++ b/src/encoder/resources.rs @@ -1,6 +1,6 @@ +use crate::encoder::input::SourceOwnership; use crate::encoder::{BitDepth, PixelFormat}; use crate::error::{PixelForgeError, Result}; -use crate::sync::TimelinePoint; use crate::vulkan::VideoContext; use ash::vk::TaggedStructure; use ash::vk::{self, Handle}; @@ -412,16 +412,12 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam /// Parameters for uploading an image to the encoder's input image. pub(crate) struct UploadParams<'a> { - /// The command buffer to use for the upload. + /// The command buffer to record into, the current slot's own. pub upload_command_buffer: vk::CommandBuffer, - /// The fence to use for synchronization. - pub upload_fence: vk::Fence, /// The source image to copy from. pub src_image: vk::Image, - /// The layout the source image is in. `UNDEFINED` means a first-time - /// external-memory import, which the copy acquires from - /// `VK_QUEUE_FAMILY_EXTERNAL` for the transfer queue family. - pub src_layout: vk::ImageLayout, + /// The source's layout and the ownership transfers around this copy. + pub src_ownership: SourceOwnership, /// The destination image to copy to. pub dst_image: vk::Image, /// The width of the image. @@ -436,100 +432,67 @@ pub(crate) struct UploadParams<'a> { pub rgb: bool, /// The queue to submit transfer operations to. pub upload_queue: vk::Queue, - /// Caller work the copy must wait for. - pub waits: &'a [TimelinePoint], + /// What the copy waits for. + pub waits: &'a [vk::SemaphoreSubmitInfo<'static>], + /// What the copy signals once done: the point the encode waits on, and + /// any release the caller asked for. + pub signals: &'a [vk::SemaphoreSubmitInfo<'static>], } -/// Upload an image to the encoder's input image via GPU-to-GPU copy. -/// -/// This function handles: -/// - Resetting and beginning the command buffer -/// - Transitioning source image from `src_layout` to TRANSFER_SRC, acquiring -/// a first-time external-memory import -/// - Transitioning destination image from `input_image_layout` to TRANSFER_DST -/// - Copying Y and UV planes (NV12 format) -/// - Transitioning destination image to VIDEO_ENCODE_SRC -/// - Transitioning source image back to GENERAL -/// - Submitting the command buffer and waiting for completion +/// Copy a source image into the encoder's input image on the transfer queue. /// -/// Returns Ok(()) on success, or an error if any Vulkan operation fails. +/// Records the copy into the slot's own command buffer and submits it without +/// waiting: the encode waits for one of `params.signals` on the GPU instead. +/// The source is acquired and released as `params.src_ownership` says and is +/// left in `GENERAL`; the input image is left in `VIDEO_ENCODE_SRC_KHR`. pub(crate) fn upload_image_to_input( context: &crate::vulkan::VideoContext, params: &UploadParams<'_>, ) -> Result<()> { let device = context.device(); + let family = context.transfer_queue_family(); + let command_buffer = params.upload_command_buffer; - unsafe { - device.reset_command_buffer( - params.upload_command_buffer, - vk::CommandBufferResetFlags::empty(), - ) - } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + unsafe { device.reset_command_buffer(command_buffer, vk::CommandBufferResetFlags::empty()) } + .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; let begin_info = vk::CommandBufferBeginInfo::default().flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT); - unsafe { device.begin_command_buffer(params.upload_command_buffer, &begin_info) } + unsafe { device.begin_command_buffer(command_buffer, &begin_info) } .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; - // Transition source image to TRANSFER_SRC. A first-time external-memory - // import (UNDEFINED) is still owned by VK_QUEUE_FAMILY_EXTERNAL: acquire it - // for the transfer queue family. - let needs_acquire = params.src_layout == vk::ImageLayout::UNDEFINED; - let src_barrier = vk::ImageMemoryBarrier::default() - .old_layout(params.src_layout) - .new_layout(vk::ImageLayout::TRANSFER_SRC_OPTIMAL) - .src_queue_family_index(if needs_acquire { - vk::QUEUE_FAMILY_EXTERNAL - } else { - vk::QUEUE_FAMILY_IGNORED - }) - .dst_queue_family_index(if needs_acquire { - context.transfer_queue_family() - } else { - vk::QUEUE_FAMILY_IGNORED - }) - .image(params.src_image) - .subresource_range(vk::ImageSubresourceRange { - aspect_mask: vk::ImageAspectFlags::COLOR, - base_mip_level: 0, - level_count: 1, - base_array_layer: 0, - layer_count: 1, - }) - .src_access_mask(if needs_acquire { - vk::AccessFlags::empty() - } else { - vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE - }) - .dst_access_mask(vk::AccessFlags::TRANSFER_READ); - - // Transition destination image to TRANSFER_DST. - let dst_barrier = vk::ImageMemoryBarrier::default() + let color = vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_mip_level: 0, + level_count: 1, + base_array_layer: 0, + layer_count: 1, + }; + let src_barrier = params.src_ownership.acquire_barrier( + params.src_image, + family, + vk::ImageLayout::TRANSFER_SRC_OPTIMAL, + vk::PipelineStageFlags2::COPY, + vk::AccessFlags2::TRANSFER_READ, + ); + // The input image was last read by this slot's previous encode, which + // was complete before the slot was handed out again. + let dst_barrier = vk::ImageMemoryBarrier2::default() + .src_stage_mask(vk::PipelineStageFlags2::NONE) + .src_access_mask(vk::AccessFlags2::NONE) + .dst_stage_mask(vk::PipelineStageFlags2::COPY) + .dst_access_mask(vk::AccessFlags2::TRANSFER_WRITE) .old_layout(params.input_image_layout) .new_layout(vk::ImageLayout::TRANSFER_DST_OPTIMAL) .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) .image(params.dst_image) - .subresource_range(vk::ImageSubresourceRange { - aspect_mask: vk::ImageAspectFlags::COLOR, - base_mip_level: 0, - level_count: 1, - base_array_layer: 0, - layer_count: 1, - }) - .src_access_mask(vk::AccessFlags::empty()) - .dst_access_mask(vk::AccessFlags::TRANSFER_WRITE); - + .subresource_range(color); + let barriers = [src_barrier, dst_barrier]; unsafe { - device.cmd_pipeline_barrier( - params.upload_command_buffer, - vk::PipelineStageFlags::TOP_OF_PIPE, - vk::PipelineStageFlags::TRANSFER, - vk::DependencyFlags::empty(), - &[], - &[], - &[src_barrier, dst_barrier], + context.sync2().cmd_pipeline_barrier2( + command_buffer, + &vk::DependencyInfo::default().image_memory_barriers(&barriers), ); } @@ -610,7 +573,7 @@ pub(crate) fn upload_image_to_input( unsafe { device.cmd_copy_image( - params.upload_command_buffer, + command_buffer, params.src_image, vk::ImageLayout::TRANSFER_SRC_OPTIMAL, params.dst_image, @@ -619,76 +582,49 @@ pub(crate) fn upload_image_to_input( ); } - // Transition destination image to VIDEO_ENCODE_SRC. - let barrier = vk::ImageMemoryBarrier::default() + // The encode that reads the input image waits for this submission's + // signal, which makes the copy's writes visible to it. + let dst_to_encode = vk::ImageMemoryBarrier2::default() + .src_stage_mask(vk::PipelineStageFlags2::COPY) + .src_access_mask(vk::AccessFlags2::TRANSFER_WRITE) + .dst_stage_mask(vk::PipelineStageFlags2::NONE) + .dst_access_mask(vk::AccessFlags2::NONE) .old_layout(vk::ImageLayout::TRANSFER_DST_OPTIMAL) .new_layout(vk::ImageLayout::VIDEO_ENCODE_SRC_KHR) .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) .image(params.dst_image) - .subresource_range(vk::ImageSubresourceRange { - aspect_mask: vk::ImageAspectFlags::COLOR, - base_mip_level: 0, - level_count: 1, - base_array_layer: 0, - layer_count: 1, - }) - .src_access_mask(vk::AccessFlags::TRANSFER_WRITE) - .dst_access_mask(vk::AccessFlags::empty()); - - // Also transition source image back to GENERAL for reuse. - let src_barrier_back = vk::ImageMemoryBarrier::default() - .old_layout(vk::ImageLayout::TRANSFER_SRC_OPTIMAL) - .new_layout(vk::ImageLayout::GENERAL) - .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) - .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) - .image(params.src_image) - .subresource_range(vk::ImageSubresourceRange { - aspect_mask: vk::ImageAspectFlags::COLOR, - base_mip_level: 0, - level_count: 1, - base_array_layer: 0, - layer_count: 1, - }) - .src_access_mask(vk::AccessFlags::TRANSFER_READ) - .dst_access_mask(vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE); - + .subresource_range(color); + let src_back = params.src_ownership.release_barrier( + params.src_image, + family, + vk::ImageLayout::TRANSFER_SRC_OPTIMAL, + vk::PipelineStageFlags2::COPY, + ); + let barriers = [dst_to_encode, src_back]; unsafe { - device.cmd_pipeline_barrier( - params.upload_command_buffer, - vk::PipelineStageFlags::TRANSFER, - vk::PipelineStageFlags::BOTTOM_OF_PIPE, - vk::DependencyFlags::empty(), - &[], - &[], - &[barrier, src_barrier_back], + context.sync2().cmd_pipeline_barrier2( + command_buffer, + &vk::DependencyInfo::default().image_memory_barriers(&barriers), ); } - unsafe { device.end_command_buffer(params.upload_command_buffer) } + unsafe { device.end_command_buffer(command_buffer) } .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; - let command_buffers = - [vk::CommandBufferSubmitInfo::default().command_buffer(params.upload_command_buffer)]; - let waits: Vec = - params.waits.iter().map(TimelinePoint::wait_info).collect(); + let command_buffers = [vk::CommandBufferSubmitInfo::default().command_buffer(command_buffer)]; let submit_info = vk::SubmitInfo2::default() - .wait_semaphore_infos(&waits) - .command_buffer_infos(&command_buffers); + .wait_semaphore_infos(params.waits) + .command_buffer_infos(&command_buffers) + .signal_semaphore_infos(params.signals); unsafe { context .sync2() - .queue_submit2(params.upload_queue, &[submit_info], params.upload_fence) + .queue_submit2(params.upload_queue, &[submit_info], vk::Fence::null()) } .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; - unsafe { device.wait_for_fences(&[params.upload_fence], true, u64::MAX) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; - - unsafe { device.reset_fences(&[params.upload_fence]) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; - Ok(()) } @@ -891,25 +827,15 @@ pub(crate) unsafe fn submit_encode_only( fence: vk::Fence, encode_queue: vk::Queue, wait_infos: &[vk::SemaphoreSubmitInfo<'_>], - signal_timeline: Option<(vk::Semaphore, u64)>, + signal_infos: &[vk::SemaphoreSubmitInfo<'_>], ) -> Result<()> { let command_buffer_info = vk::CommandBufferSubmitInfo::default().command_buffer(command_buffer); let command_buffer_infos = [command_buffer_info]; - let signal_infos: Vec = signal_timeline - .into_iter() - .map(|(semaphore, value)| { - vk::SemaphoreSubmitInfo::default() - .semaphore(semaphore) - .value(value) - .stage_mask(vk::PipelineStageFlags2::VIDEO_ENCODE_KHR) - }) - .collect(); - let submit_info = vk::SubmitInfo2::default() .wait_semaphore_infos(wait_infos) .command_buffer_infos(&command_buffer_infos) - .signal_semaphore_infos(&signal_infos); + .signal_semaphore_infos(signal_infos); unsafe { device diff --git a/src/encoder/rgb.rs b/src/encoder/rgb.rs index a29aee40..3b922b97 100644 --- a/src/encoder/rgb.rs +++ b/src/encoder/rgb.rs @@ -162,11 +162,9 @@ impl VideoContext { /// [`InputFormat::ABGR2101010`]. Resolution and rate control are not /// considered. /// - /// When this is true, build the encoder with - /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input) - /// and [`EncodeConfig::with_color_description`](crate::EncodeConfig::with_color_description) - /// set to `desc`. [`ColorConverterConfig::rgb_encode_input`](crate::ColorConverterConfig::rgb_encode_input) - /// asks this for a conversion it would otherwise run. + /// When this is true, an encoder for an RGB [`Source`](crate::Source) in + /// `format`, with `desc` as its colour description, picks + /// [`Conversion::Encoder`](crate::Conversion::Encoder) by itself. pub fn supports_rgb_encode_input( &self, codec: Codec, diff --git a/src/encoder/source.rs b/src/encoder/source.rs new file mode 100644 index 00000000..41c6ec06 --- /dev/null +++ b/src/encoder/source.rs @@ -0,0 +1,296 @@ +//! What the frames an encoder is given are, and how they become YUV. +//! +//! The caller describes the source once, in [`EncodeConfig::source`]. The +//! encoder then picks the [`Conversion`] that gets it to the stream's YUV, so +//! a caller never chooses between the colour converter and the encoder's own +//! RGB conversion, and always submits frames the same way. + +use super::rgb::rgb_input_bit_depth; +use super::{BitDepth, ColorDescription, EncodeConfig, PixelFormat}; +use crate::converter::{ColorConverterConfig, ColorRange, ColorSpec, InputFormat, OutputFormat}; +use crate::error::{PixelForgeError, Result}; +use crate::vulkan::VideoContext; + +/// What the frames handed to an [`Encoder`](crate::Encoder) are. +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub enum Source { + /// YUV frames, already in the encoder's pixel format and bit depth. + #[default] + Yuv, + /// RGB frames, which pixelforge converts to YUV. + Rgb(RgbSource), +} + +/// RGB frames, and what their pixels mean. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct RgbSource { + /// How the pixels are laid out. + pub format: InputFormat, + /// What the pixel values mean. + pub color: ColorSpec, + /// How bright the source's white is, in nits, overriding + /// [`ColorSpec::reference_white_nits`]. Only used when encoding SDR + /// content to HDR. + pub reference_white_nits: Option, +} + +impl RgbSource { + /// RGB frames in `format`, holding colours in `color`. + pub fn new(format: InputFormat, color: ColorSpec) -> Self { + Self { + format, + color, + reference_white_nits: None, + } + } + + /// Override how bright the source's white is, in nits. + pub fn with_reference_white_nits(mut self, nits: f32) -> Self { + self.reference_white_nits = Some(nits); + self + } +} + +/// Which conversion an encoder should use for an RGB [`Source`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ConversionPreference { + /// The encoder's own conversion where the device can do it, the colour + /// converter otherwise. + #[default] + Auto, + /// The encoder's own conversion, or fail. + Encoder, + /// The colour converter's compute shader, even where the encoder could + /// convert. + Shader, +} + +/// The conversion an [`Encoder`](crate::Encoder) uses, see +/// [`Encoder::conversion`](crate::Encoder::conversion). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Conversion { + /// The source is already YUV. + None, + /// The encoder converts RGB itself, through + /// `VK_VALVE_video_encode_rgb_conversion`. Its input image is RGB. + Encoder, + /// The colour converter converts RGB into the encoder's YUV input image. + Shader, +} + +/// The conversion an encoder will use, and what it needs for it. +pub(crate) struct Plan { + pub conversion: Conversion, + /// The RGB format the encoder converts from, for [`Conversion::Encoder`]. + pub rgb_input: Option, + /// The colour converter's configuration, for [`Conversion::Shader`]. + pub converter: Option, +} + +/// The colour space a stream described by `desc` is in, if a conversion +/// can produce it. +pub(crate) fn target_space(desc: &ColorDescription) -> Option { + [ColorSpec::Srgb, ColorSpec::Bt2020Pq] + .into_iter() + .find(|space| { + space.color_description().is_some_and(|d| { + ( + d.color_primaries, + d.transfer_characteristics, + d.matrix_coefficients, + ) == ( + desc.color_primaries, + desc.transfer_characteristics, + desc.matrix_coefficients, + ) + }) + }) +} + +/// The YUV the colour converter has to write for this encoder. +fn shader_output(pixel_format: PixelFormat, bit_depth: BitDepth) -> Option { + match (pixel_format, bit_depth) { + (PixelFormat::Yuv420, BitDepth::Eight) => Some(OutputFormat::NV12), + (PixelFormat::Yuv420, BitDepth::Ten) => Some(OutputFormat::P010), + (PixelFormat::Yuv444, BitDepth::Eight) => Some(OutputFormat::YUV444), + (PixelFormat::Yuv444, BitDepth::Ten) => Some(OutputFormat::YUV444P10), + _ => None, + } +} + +/// Pick the conversion for `config` on this device. +pub(crate) fn plan(context: &VideoContext, config: &EncodeConfig) -> Result { + let rgb = match config.source { + Source::Yuv => { + if config.conversion != ConversionPreference::Auto { + return Err(PixelForgeError::InvalidInput(format!( + "conversion preference {:?} needs an RGB source", + config.conversion + ))); + } + return Ok(Plan { + conversion: Conversion::None, + rgb_input: None, + converter: None, + }); + } + Source::Rgb(rgb) => rgb, + }; + + let desc = config + .color_description + .unwrap_or_else(ColorDescription::bt709); + let target = target_space(&desc).ok_or_else(|| { + PixelForgeError::InvalidInput(format!( + "no conversion from RGB produces the colour description {desc:?}; \ + use BT.709 or BT.2020 with PQ" + )) + })?; + + let encoder = encoder_conversion(context, config, &rgb, target, &desc); + let shader = shader_conversion(context, config, &rgb, target, &desc); + + let use_encoder = match config.conversion { + ConversionPreference::Encoder => { + encoder?; + true + } + ConversionPreference::Shader => false, + ConversionPreference::Auto => match &encoder { + Ok(()) => true, + Err(e) => { + tracing::debug!("RGB input: the encoder cannot convert ({e}); using the shader"); + false + } + }, + }; + if use_encoder { + return Ok(Plan { + conversion: Conversion::Encoder, + rgb_input: Some(rgb.format), + converter: None, + }); + } + Ok(Plan { + conversion: Conversion::Shader, + rgb_input: None, + converter: Some(shader?), + }) +} + +/// Why the encoder cannot do this conversion itself, if it cannot. +fn encoder_conversion( + context: &VideoContext, + config: &EncodeConfig, + rgb: &RgbSource, + target: ColorSpec, + desc: &ColorDescription, +) -> Result<()> { + let no = |reason: String| { + Err(PixelForgeError::InvalidInput(format!( + "RGB input: {reason}" + ))) + }; + if !context.has_video_encode_rgb_conversion() { + return no("the device has no VK_VALVE_video_encode_rgb_conversion".to_string()); + } + if rgb.color != target { + return no(format!( + "{:?} to {target:?} needs more than a YUV matrix", + rgb.color + )); + } + match rgb_input_bit_depth(rgb.format) { + Some(depth) if config.pixel_format == PixelFormat::Yuv420 && config.bit_depth == depth => {} + Some(depth) => { + return no(format!( + "{:?} converts to {depth:?} 4:2:0, the encoder is set up for {:?} {:?}", + rgb.format, config.bit_depth, config.pixel_format + )); + } + None => return no(format!("{:?} needs tone mapping first", rgb.format)), + } + if !context.supports_rgb_encode_input(config.codec, rgb.format, desc) { + return no(format!( + "the driver does not convert {:?} to {desc:?} for {:?}", + rgb.format, config.codec + )); + } + Ok(()) +} + +/// The colour converter's configuration for this conversion, or why the +/// shader cannot do it. +fn shader_conversion( + context: &VideoContext, + config: &EncodeConfig, + rgb: &RgbSource, + target: ColorSpec, + desc: &ColorDescription, +) -> Result { + if !context.has_push_descriptor() { + return Err(PixelForgeError::NoSuitableDevice( + "RGB input: the colour converter needs VK_KHR_push_descriptor".to_string(), + )); + } + let output = shader_output(config.pixel_format, config.bit_depth).ok_or_else(|| { + PixelForgeError::InvalidInput(format!( + "RGB input: the colour converter does not write {:?} {:?}", + config.pixel_format, config.bit_depth + )) + })?; + let range = if desc.full_range { + ColorRange::Full + } else { + ColorRange::Limited + }; + let mut converter = ColorConverterConfig::new( + config.dimensions.width, + config.dimensions.height, + rgb.format, + output, + rgb.color, + target, + range, + ); + converter.reference_white_nits = rgb.reference_white_nits; + if !converter.conversion_supported() { + return Err(converter.unsupported_conversion()); + } + Ok(converter) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn target_space_follows_the_description() { + assert_eq!( + target_space(&ColorDescription::bt709()), + Some(ColorSpec::Srgb) + ); + assert_eq!( + target_space(&ColorDescription::bt709().with_full_range(true)), + Some(ColorSpec::Srgb) + ); + assert_eq!( + target_space(&ColorDescription::bt2020_pq()), + Some(ColorSpec::Bt2020Pq) + ); + let bt601 = ColorDescription { + matrix_coefficients: 6, + ..ColorDescription::bt709() + }; + assert_eq!(target_space(&bt601), None); + } + + #[test] + fn shader_output_matches_the_encoder() { + assert_eq!( + shader_output(PixelFormat::Yuv420, BitDepth::Ten), + Some(OutputFormat::P010) + ); + assert_eq!(shader_output(PixelFormat::Yuv422, BitDepth::Eight), None); + } +} diff --git a/src/encoder/support.rs b/src/encoder/support.rs index cd5de019..f4600a8c 100644 --- a/src/encoder/support.rs +++ b/src/encoder/support.rs @@ -48,7 +48,7 @@ impl VideoContext { /// hand the profile to `then` for any further query about it. /// /// With `rgb_caps`, the profile is the RGB input one an encoder built with - /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input) + /// [`Conversion::Encoder`](crate::Conversion::Encoder) /// requests, and the driver fills in what its conversion supports. /// /// The encoders build their profiles themselves, since theirs also carry diff --git a/src/lib.rs b/src/lib.rs index 0a978891..e4d57f8f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,8 +8,10 @@ //! - **Asynchronous pipelines**: both directions submit without waiting. //! Encoding hands back an [`EncodeFuture`]; decoding delivers frames through a //! [`DecodeSource`] as the GPU finishes with them. -//! - **GPU color conversion**: RGB/BGR → YUV via Vulkan compute shaders (BT.709, BT.2020, sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ), -//! or in the encoder itself where `VK_VALVE_video_encode_rgb_conversion` is available. +//! - **RGB input**: describe the source once and hand the encoder RGB frames. +//! It converts them in the encoder itself where `VK_VALVE_video_encode_rgb_conversion` +//! allows, and with a compute shader otherwise (BT.709, BT.2020, +//! sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ). //! - **Shared devices**: encode and decode on a Vulkan device your application //! created, from Vulkan 1.1 up, on queues you choose, ordered against your own //! work with timeline semaphores. @@ -18,7 +20,9 @@ //! - **Flexible configuration**: Rate control (CBR, VBR, CQP), quality levels, GOP settings. //! - **Multiple input formats**: BGRx, RGBx, BGRA, RGBA, ABGR2101010 (10-bit packed), RGBA16F (FP16). //! - **Utility helpers**: [`InputImage`] for easy YUV data upload to GPU. -//! - **Optional DMA-BUF support**: Zero-copy image import from external processes (Linux only). +//! - **Optional DMA-BUF input**: import a producer's buffers, or allocate buffers +//! for it, read with no copy where the driver allows, with sync_file fences +//! (Linux only). //! //! > **Note**: B-frame support is not yet implemented. Setting `b_frame_count > 0` will panic. //! @@ -52,7 +56,7 @@ //! //! | Feature | Description | //! |---------|-------------| -//! | `dmabuf` | Enable DMA-BUF support for zero-copy image import from external processes (Linux only). Adds Vulkan extensions: `VK_KHR_external_memory`, `VK_KHR_external_memory_fd`, `VK_EXT_external_memory_dma_buf`, `VK_EXT_image_drm_format_modifier`. | +//! | `dmabuf` | DMA-BUF input and sync_file fences (Linux only). Enables, where the device has them, `VK_KHR_external_memory_fd`, `VK_EXT_external_memory_dma_buf`, `VK_EXT_image_drm_format_modifier`, `VK_KHR_external_semaphore_fd` and `VK_EXT_queue_family_foreign`. | //! //! To enable DMA-BUF support: //! @@ -181,11 +185,51 @@ //! pictures out instead of handing over its own. Reading a frame back to the //! CPU is the consumer's job; `examples/common` shows one way. //! -//! ## Color Conversion (RGB → YUV) +//! ## RGB input //! -//! PixelForge includes a GPU compute shader for converting RGB input to YUV -//! output. The source color describes the input. The target color describes the stream. -//! Both are a [`ColorSpec`]. +//! An encoder takes RGB frames once it is told what they are, with a +//! [`Source`]. It then converts them to the YUV that the stream's +//! [`ColorDescription`] describes, and that description also picks the target: +//! BT.709 for SDR, BT.2020 with PQ for HDR10. The source and the stream are +//! described once each, so the conversion and the stream's label cannot +//! disagree, and a colour change through [`Encoder::set_color_description`] +//! changes both. +//! +//! ```rust,no_run +//! use pixelforge::{ +//! Codec, ColorDescription, ColorSpec, EncodeConfig, Encoder, InputFormat, RgbSource, +//! Source, VideoContextBuilder, +//! }; +//! +//! # fn main() -> Result<(), Box> { +//! let context = VideoContextBuilder::new() +//! .require_encode(Codec::H265) +//! .build()?; +//! +//! // SDR desktop content, encoded as HDR10 for an HDR streaming session. +//! let config = EncodeConfig::h265(1920, 1080) +//! .with_bit_depth(pixelforge::EncodeBitDepth::Ten) +//! .with_source(Source::Rgb(RgbSource::new(InputFormat::BGRx, ColorSpec::Srgb))) +//! .with_color_description(ColorDescription::bt2020_pq().with_full_range(true)); +//! let mut encoder = Encoder::new(context, config)?; +//! println!("converting with {:?}", encoder.conversion()); +//! // let packet = encoder.encode(rgb_image)?; +//! # Ok(()) +//! # } +//! ``` +//! +//! [`Encoder::new`] picks the [`Conversion`]: +//! +//! - [`Conversion::Encoder`] where the conversion is nothing but the YUV +//! matrix, as for ordinary SDR content, and the device has +//! `VK_VALVE_video_encode_rgb_conversion` with that matrix and range. No +//! shader runs at all. +//! - [`Conversion::Shader`] otherwise: the colour converter's compute shader +//! writes the encoder's YUV input. +//! +//! [`EncodeConfig::with_conversion`] can force one, for tests and comparisons. +//! +//! The shader's colour spaces are [`ColorSpec`]s: //! //! | `ColorSpec` | | Can be a target | //! |-------------|-|-----------------| @@ -198,81 +242,27 @@ //! record that it holds linear light. Any source can be converted to //! `Bt2020Pq`. Only `Srgb` can be converted to `Srgb`; the others would need //! tone mapping or a gamma curve applied, which the shader does not do. -//! [`ColorConverter::new`] rejects the combinations it cannot do. +//! [`Encoder::new`] rejects the combinations nothing can do. //! //! Encoding to HDR needs to know how bright the source's white is, since HDR //! carries real brightness values and SDR does not. Each space has a sensible //! default, see [`ColorSpec::reference_white_nits`], overridable with -//! [`ColorConverterConfig::with_reference_white_nits`]. +//! [`RgbSource::with_reference_white_nits`]. //! //! Supported input formats: BGRx, RGBx, BGRA, RGBA, ABGR2101010 (10-bit packed), RGBA16F (FP16). -//! Supported output formats: NV12 (8-bit), I420 (8-bit), YUV444 (8-bit), P010 (10-bit), YUV444P10 (10-bit). -//! -//! Pass [`ColorConverter::color_description`] to the encoder rather than -//! describing the colours a second time by hand. The two have to agree: if the -//! converter writes full-range pixels and the stream says limited, players -//! stretch the range again and the picture comes out wrong. -//! -//! ```rust,no_run -//! use pixelforge::{ -//! Codec, ColorConverter, ColorConverterConfig, ColorRange, ColorSpec, EncodeConfig, -//! Encoder, InputFormat, OutputFormat, VideoContextBuilder, -//! }; -//! -//! # fn main() -> Result<(), Box> { -//! let context = VideoContextBuilder::new() -//! .app_name("Color Converter") -//! .require_encode(Codec::H265) -//! .build()?; //! -//! // SDR desktop content, encoded as HDR10 for an HDR streaming session. -//! let config = ColorConverterConfig::new( -//! 1920, -//! 1080, -//! InputFormat::BGRx, -//! OutputFormat::P010, -//! ColorSpec::Srgb, -//! ColorSpec::Bt2020Pq, -//! ColorRange::Full, -//! ); -//! -//! let mut converter = ColorConverter::new(context.clone(), config)?; -//! -//! // The encoder declares exactly what the shader wrote. -//! let encode_config = EncodeConfig::h265(1920, 1080) -//! .with_color_description(converter.color_description()); -//! -//! let mut encoder = Encoder::new(context, encode_config)?; -//! // converter.convert(input_image, layout, encoder.input_image())?; -//! # Ok(()) -//! # } -//! ``` +//! [`ColorConverter`] is the same shader on its own, for converting without +//! encoding. //! -//! Where the conversion is nothing but the YUV matrix, as for ordinary SDR -//! content, some encoders can do it themselves and the shader need not run at -//! all. [`ColorConverterConfig::rgb_encode_input`] says whether that works for -//! a given conversion and codec on this device. When it does, build the -//! encoder with [`EncodeConfig::with_rgb_input`] and hand it the RGB frames, -//! passing `UNDEFINED` to [`Encoder::encode_after`] for first-time DMA-BUF -//! imports. +//! ## Submitting frames //! -//! ```rust,no_run -//! # use pixelforge::{Codec, ColorConverterConfig, EncodeConfig, Encoder, VideoContext}; -//! # fn pick( -//! # context: VideoContext, -//! # config: ColorConverterConfig, -//! # ) -> Result<(), pixelforge::PixelForgeError> { -//! let encode_config = EncodeConfig::h264(1920, 1080) -//! .with_color_description(config.color_description().expect("SDR is encodable")); -//! let encoder = match config.rgb_encode_input(&context, Codec::H264) { -//! // The encoder converts: give it the RGB images as they are. -//! Some(format) => Encoder::new(context, encode_config.with_rgb_input(format))?, -//! // The shader converts into the encoder's input image, as above. -//! None => Encoder::new(context, encode_config)?, -//! }; -//! # Ok(()) -//! # } -//! ``` +//! [`Encoder::submit`] takes a [`Frame`]: where the pixels are, and what has +//! to happen on the GPU before they can be read. It does not wait on the CPU, +//! except when every pipeline slot is in flight. The source is read by a copy, +//! the shader, or the encode itself, after the call returns, so write it again +//! only after [`EncodeFuture::source_released`]. [`Encoder::encode`] is the +//! short form that also waits for that point, which keeps the simple loop of +//! writing one image and encoding it correct. //! //! ## Encoding on your own device //! @@ -288,36 +278,45 @@ //! the application does not use, with //! [`VideoContextBuilder::with_encode_queue`] and friends. //! -//! [`ColorConverter::convert_async`] and [`Encoder::encode_after`] then order -//! each frame on the GPU: the application's submission signals a -//! [`TimelinePoint`], the conversion waits for it and signals its own, and the -//! encode waits for that. This stays off the CPU as long as the encoder's own -//! [`Encoder::input_image`] is what gets encoded, as below: any other image is -//! copied into it first, and that copy is waited for on the CPU. +//! Each frame is then ordered on the GPU: the application's submission signals +//! a [`TimelinePoint`], the encoder's first submission for the frame waits for +//! it, and the application waits for [`EncodeFuture::source_released`] before +//! it draws into the image again. //! //! ```rust,no_run //! # use ash::vk; -//! # use pixelforge::{ColorConverter, Encoder, TimelinePoint}; +//! # use pixelforge::{Encoder, Frame, TimelinePoint}; //! # fn frame( -//! # converter: &mut ColorConverter, //! # encoder: &mut Encoder, //! # rendered: vk::Semaphore, //! # frame_number: u64, //! # image: vk::Image, //! # ) -> Result<(), Box> { //! // `rendered` reaches `frame_number` once the application has drawn `image`. -//! let converted = converter.convert_async( -//! image, -//! vk::ImageLayout::GENERAL, -//! encoder.input_image(), -//! &[TimelinePoint::new(rendered, frame_number)], -//! )?; -//! let packet = encoder.encode_after(encoder.input_image(), vk::ImageLayout::GENERAL, &[converted])?; +//! let ready = [TimelinePoint::new(rendered, frame_number)]; +//! let packet = encoder.submit(Frame::image(image, vk::ImageLayout::GENERAL).wait(&ready))?; +//! // Wait on this before drawing into `image` again. +//! let reusable = packet.source_released(); //! # Ok(()) //! # } //! ``` //! -//! # Benchmarking +//! ## DMA-BUF input +//! +//! With the `dmabuf` feature, an encoder reads DMA-BUFs from another process. +//! `Encoder::input_formats` gives the DRM format it wants and, per DRM +//! modifier, whether it reads such a buffer with no copy. Offer those +//! modifiers to the producer first. `Encoder::import` takes a producer's +//! buffer; `Encoder::allocate_input` makes one for a producer that writes +//! into buffers the consumer supplies, such as `ext-image-copy-capture`. +//! Either is checked once and gives an `ExternalImage`, which +//! `Frame::external` submits. A buffer the encoder cannot read directly is +//! copied on the GPU instead. +//! +//! The producer's write fence and the encoder's read fence can travel as +//! sync_files: `Frame::wait_sync_file` and `Frame::signal_sync_file`. +//! +//! //! # Benchmarking //! //! Run the encode latency benchmark with: //! @@ -395,12 +394,15 @@ pub use converter::{ pub use decoder::{ DecodeConfig, DecodeSink, DecodeSource, DecodeStatus, DecodedFrame, Decoder, FramePoll, Framing, }; +#[cfg(feature = "dmabuf")] +pub use encoder::external::{DmaBuf, DmaBufPlane, ExternalImage, InputFormats, InputModifier}; pub use encoder::{ BitDepth as EncodeBitDepth, Codec, ColorDescription, DEFAULT_FRAME_RATE, DEFAULT_GOP_SIZE, DEFAULT_H264_QP, DEFAULT_H265_QP, DEFAULT_MAX_BITRATE, DEFAULT_MAX_REFERENCE_FRAMES, DEFAULT_TARGET_BITRATE, EncodeConfig, EncodeContentHint, EncodeFuture, EncodeUsageHint, - EncodedPacket, Encoder, EncoderTuningMode, FrameType, PixelFormat, RateControlMode, + EncodedPacket, Encoder, EncoderTuningMode, Frame, FrameType, PixelFormat, RateControlMode, }; +pub use encoder::{Conversion, ConversionPreference, RgbSource, Source}; pub use error::PixelForgeError; pub use image::InputImage; pub use sync::TimelinePoint; diff --git a/src/video.rs b/src/video.rs index 449de659..5cc6c52e 100644 --- a/src/video.rs +++ b/src/video.rs @@ -162,7 +162,7 @@ fn create_video_image_raw( /// must carry a `VkSamplerYcbcrConversion`, and a video picture resource must /// not have one. Declaring the narrower usage says what this view is actually /// for and keeps both requirements satisfiable at once. -fn create_video_image_view( +pub(crate) fn create_video_image_view( context: &VideoContext, image: vk::Image, format: vk::Format, diff --git a/src/vulkan.rs b/src/vulkan.rs index 5a77765e..da5661a2 100644 --- a/src/vulkan.rs +++ b/src/vulkan.rs @@ -524,6 +524,66 @@ impl DeviceExtensions { } } +/// Which of the extensions behind DMA-BUF and sync_file input a context has +/// enabled. All off without the `dmabuf` feature. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct ExternalSupport { + /// Import and export of DMA-BUF images with DRM format modifiers. + pub dmabuf: bool, + /// Import and export of sync_file fences. + pub sync_file: bool, + /// `VK_QUEUE_FAMILY_FOREIGN_EXT`, the right owner for memory shared with + /// another process or API. + pub foreign_queue: bool, +} + +impl ExternalSupport { + const DMABUF: [&'static CStr; 4] = [ + ash::khr::external_memory_fd::NAME, + ash::ext::external_memory_dma_buf::NAME, + ash::ext::image_drm_format_modifier::NAME, + // Required by the modifier extension; core since 1.2. + ash::khr::image_format_list::NAME, + ]; + + /// What a device offering the extensions `has` says it has can enable. + pub(crate) fn detect(has: impl Fn(&CStr) -> bool) -> Self { + if !cfg!(feature = "dmabuf") { + return Self::default(); + } + Self { + dmabuf: Self::DMABUF.iter().all(|name| has(name)), + sync_file: has(ash::khr::external_semaphore_fd::NAME), + foreign_queue: has(ash::ext::queue_family_foreign::NAME), + } + } + + /// The extensions to enable for this. + pub(crate) fn extension_names(&self) -> Vec<&'static CStr> { + let mut names = Vec::new(); + if self.dmabuf { + names.extend(Self::DMABUF); + } + if self.sync_file { + names.push(ash::khr::external_semaphore_fd::NAME); + } + if self.foreign_queue { + names.push(ash::ext::queue_family_foreign::NAME); + } + names + } + + /// The queue family to acquire shared memory from and release it to. + #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] + pub(crate) fn foreign_family(&self) -> u32 { + if self.foreign_queue { + vk::QUEUE_FAMILY_FOREIGN_EXT + } else { + vk::QUEUE_FAMILY_EXTERNAL + } + } +} + /// Whether `physical_device` can support unified image layouts *for video*. /// /// Both the extension and both of its feature bits are needed: @@ -612,6 +672,9 @@ struct VideoContextInner { /// calling them crashes. The KHR names are valid on any device with /// `VK_KHR_synchronization2` enabled, which pixelforge requires either way. sync2: ash::khr::synchronization2::Device, + /// `VK_KHR_timeline_semaphore` entry points, for the same reason as + /// `sync2`. + timeline: ash::khr::timeline_semaphore::Device, video_encode_queue_family: Option, video_encode_timestamp_valid_bits: u32, video_encode_queue: Option, @@ -628,6 +691,8 @@ struct VideoContextInner { has_push_descriptor: bool, has_video_encode_rgb_conversion: bool, has_unified_image_layouts: bool, + #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] + external: ExternalSupport, /// Whether this context created (and therefore must destroy) the device and /// instance. A context adopted from a caller's device borrows them and /// destroys neither. @@ -690,6 +755,23 @@ impl VideoContext { &self.inner.sync2 } + /// The DMA-BUF and sync_file extensions this context enabled. + #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] + pub(crate) fn external(&self) -> ExternalSupport { + self.inner.external + } + + /// Wait on the CPU until `point` is reached. + pub(crate) fn wait_point(&self, point: crate::sync::TimelinePoint) -> crate::error::Result<()> { + let semaphores = [point.semaphore]; + let values = [point.value]; + let info = vk::SemaphoreWaitInfo::default() + .semaphores(&semaphores) + .values(&values); + unsafe { self.inner.timeline.wait_semaphores(&info, u64::MAX) } + .map_err(|e| PixelForgeError::Synchronization(e.to_string())) + } + /// Hand over a semaphore to be destroyed with the context rather than now. /// /// For a semaphore whose owner is being dropped while a submission from @@ -777,7 +859,7 @@ impl VideoContext { /// Whether `VK_VALVE_video_encode_rgb_conversion` is enabled, extension /// and feature both: encoders can then take RGB input and convert it /// themselves, see - /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input). + /// [`Conversion::Encoder`](crate::Conversion::Encoder). pub fn has_video_encode_rgb_conversion(&self) -> bool { self.inner.has_video_encode_rgb_conversion } @@ -1161,16 +1243,17 @@ impl VideoContext { let mut extension_names = vec![ ash::khr::video_queue::NAME.as_ptr(), ash::khr::synchronization2::NAME.as_ptr(), + ash::khr::timeline_semaphore::NAME.as_ptr(), ]; - // External memory extensions for DMA-BUF support (optional, enabled with "dmabuf" feature). - #[cfg(feature = "dmabuf")] - { - extension_names.push(ash::khr::external_memory::NAME.as_ptr()); - extension_names.push(ash::khr::external_memory_fd::NAME.as_ptr()); - extension_names.push(ash::ext::external_memory_dma_buf::NAME.as_ptr()); - extension_names.push(ash::ext::image_drm_format_modifier::NAME.as_ptr()); - } + // DMA-BUF and sync_file input, with the `dmabuf` feature, wherever the + // device offers them. + let external = ExternalSupport::detect(|name| { + selected_device_exts + .as_ref() + .is_some_and(|exts| has_extension(exts, name)) + }); + extension_names.extend(external.extension_names().iter().map(|name| name.as_ptr())); let mut push_ext = |name: *const std::ffi::c_char| { if !extension_names.contains(&name) { @@ -1377,6 +1460,7 @@ impl VideoContext { inner: std::sync::Arc::new(VideoContextInner { entry, sync2: ash::khr::synchronization2::Device::load(&instance, &device), + timeline: ash::khr::timeline_semaphore::Device::load(&instance, &device), instance, physical_device, device, @@ -1396,6 +1480,7 @@ impl VideoContext { has_push_descriptor, has_video_encode_rgb_conversion: has_rgb_conversion, has_unified_image_layouts: has_unified_layouts, + external, owns_device: true, retired_semaphores: Default::default(), debug_messenger, @@ -1482,6 +1567,7 @@ impl VideoContext { inner: std::sync::Arc::new(VideoContextInner { entry, sync2: ash::khr::synchronization2::Device::load(&instance, &device), + timeline: ash::khr::timeline_semaphore::Device::load(&instance, &device), instance, physical_device, device, @@ -1501,6 +1587,7 @@ impl VideoContext { has_push_descriptor: false, has_video_encode_rgb_conversion: false, has_unified_image_layouts: declared_unified_image_layouts, + external: ExternalSupport::default(), owns_device: false, retired_semaphores: Default::default(), // The caller owns the instance; reporting is theirs to set up. diff --git a/src/vulkan/encode_device.rs b/src/vulkan/encode_device.rs index c6b462b8..ccd21e00 100644 --- a/src/vulkan/encode_device.rs +++ b/src/vulkan/encode_device.rs @@ -7,6 +7,7 @@ //! external memory, no export and import, and a semaphore instead of a CPU //! wait where one side hands a frame to the other. +use super::ExternalSupport; use super::queues::{self, QueueOverrides}; use super::{ DeviceExtensions, DeviceFeatures, DeviceQueue, DeviceRequirements, QueueRoles, VideoContext, @@ -42,6 +43,7 @@ struct EncodePlan { sampler_ycbcr_conversion: bool, internally_synchronized_queues: bool, rgb_conversion: bool, + external: ExternalSupport, } impl EncodePlan { @@ -158,6 +160,7 @@ impl EncodePlan { ), rgb_conversion: has(ash::valve::video_encode_rgb_conversion::NAME) && rgb.video_encode_rgb_conversion != 0, + external: ExternalSupport::detect(has), }) } @@ -187,6 +190,7 @@ impl EncodePlan { if self.rgb_conversion { names.push(ash::valve::video_encode_rgb_conversion::NAME); } + names.extend(self.external.extension_names()); names } @@ -225,7 +229,7 @@ impl VideoContextBuilder { /// whenever the device has it. On a device without it, encoding from YUV /// input still works and only the converter is unavailable. /// `VK_VALVE_video_encode_rgb_conversion` is included the same way, for - /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input). + /// [`Conversion::Encoder`](crate::Conversion::Encoder). pub fn encode_device_requirements( &self, entry: &ash::Entry, @@ -335,6 +339,7 @@ impl VideoContext { Ok(VideoContext { inner: std::sync::Arc::new(VideoContextInner { sync2: ash::khr::synchronization2::Device::load(&instance, &device), + timeline: ash::khr::timeline_semaphore::Device::load(&instance, &device), entry, instance, physical_device, @@ -355,6 +360,7 @@ impl VideoContext { has_push_descriptor: plan.push_descriptor, has_video_encode_rgb_conversion: plan.rgb_conversion, has_unified_image_layouts: false, + external: plan.external, owns_device: false, retired_semaphores: Default::default(), // The caller owns the instance; reporting is theirs to set up. diff --git a/tests/dmabuf.rs b/tests/dmabuf.rs new file mode 100644 index 00000000..7418af21 --- /dev/null +++ b/tests/dmabuf.rs @@ -0,0 +1,663 @@ +//! DMA-BUF input: the encoder reads a buffer straight into the encode, copies +//! it, or converts it with the shader, and orders all of it with sync_files. +//! +//! The tests play the producer themselves: they import the same DMA-BUF into +//! an image of their own and write frames into it, the way a compositor would +//! from another process. Each path is checked against encoding the same +//! frames from an ordinary image, which has to give the same stream. +#![cfg(feature = "dmabuf")] + +#[allow(dead_code)] +mod common; +use common::source::{host_buffer, one_shot}; +use common::{Readback, decode_stream}; + +use ash::vk; +use ash::vk::TaggedStructure; +use pixelforge::decoder::{DecodeConfig, Decoder}; +use pixelforge::encoder::external::drm_format; +use pixelforge::{ + Codec, ColorSpec, Conversion, ConversionPreference, DmaBuf, DmaBufPlane, EncodeBitDepth, + EncodeConfig, Encoder, ExternalImage, Frame, InputFormat, InputImage, PixelFormat, + RateControlMode, RgbSource, Source, VideoContext, VideoContextBuilder, +}; +use std::os::fd::{FromRawFd, OwnedFd}; + +const FRAMES: u32 = 6; + +fn context() -> Result> { + common::init_logging(); + Ok(VideoContextBuilder::new() + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .build()?) +} + +fn config(codec: Codec, width: u32, height: u32) -> EncodeConfig { + match codec { + Codec::H264 => EncodeConfig::h264(width, height), + Codec::H265 => EncodeConfig::h265(width, height), + Codec::AV1 => EncodeConfig::av1(width, height), + } + .with_rate_control(RateControlMode::Cqp) + .with_quality_level(22) + .with_b_frames(0) +} + +/// An NV12 frame that moves with `n`. +fn nv12_frame(width: u32, height: u32, n: u32) -> Vec { + let (w, h) = (width as usize, height as usize); + let mut data = vec![0u8; w * h * 3 / 2]; + for y in 0..h { + for x in 0..w { + data[y * w + x] = ((x + y * 2 + n as usize * 5) ^ (x / 8)) as u8; + } + } + for i in 0..w * h / 2 { + data[w * h + i] = (i / 3 + n as usize * 7) as u8; + } + data +} + +/// A BGRA frame that moves with `n`. +fn bgra_frame(width: u32, height: u32, n: u32) -> Vec { + let mut data = Vec::with_capacity((width * height * 4) as usize); + for y in 0..height { + for x in 0..width { + let (sx, sy) = (x + n * 3, y + n); + data.extend_from_slice(&[(sy * 3) as u8, (sx * 2 + sy) as u8, (sx ^ sy) as u8, 255]); + } + } + data +} + +/// Which modifier a producer image gets. +enum Modifier<'a> { + OneOf(&'a [u64]), + Exactly(u64, &'a [vk::SubresourceLayout]), +} + +/// The producer's side of a DMA-BUF: its own image over the buffer's memory, +/// which it writes frames into. +struct Producer { + context: VideoContext, + image: vk::Image, + memory: vk::DeviceMemory, + format: vk::Format, + width: u32, + height: u32, + /// Whether it was written before, and so was released to the foreign + /// family. + written: bool, +} + +impl Producer { + /// Make an image over `buffer`'s memory. + fn import(context: &VideoContext, buffer: &DmaBuf) -> Result> { + let format = pixelforge::encoder::external::vk_format_of(buffer.format).unwrap(); + let layouts: Vec = buffer + .planes + .iter() + .map(|p| vk::SubresourceLayout { + offset: p.offset, + size: 0, + row_pitch: p.stride, + array_pitch: 0, + depth_pitch: 0, + }) + .collect(); + let image = Self::create_image( + context, + format, + buffer.width, + buffer.height, + Modifier::Exactly(buffer.modifier, &layouts), + )?; + let device = context.device(); + let requirements = unsafe { device.get_image_memory_requirements(image) }; + let fd_fn = ash::khr::external_memory_fd::Device::load(context.instance(), device); + let mut fd_props = vk::MemoryFdPropertiesKHR::default(); + unsafe { + fd_fn.get_memory_fd_properties( + vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT, + std::os::fd::AsRawFd::as_raw_fd(&buffer.fd), + &mut fd_props, + ) + }?; + let type_index = context + .find_memory_type( + requirements.memory_type_bits & fd_props.memory_type_bits, + vk::MemoryPropertyFlags::empty(), + ) + .unwrap(); + let fd = std::os::fd::IntoRawFd::into_raw_fd(buffer.fd.try_clone()?); + // Owned by Vulkan from here on, whatever the import does. + let mut import = vk::ImportMemoryFdInfoKHR::default() + .handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT) + .fd(fd); + let mut dedicated = vk::MemoryDedicatedAllocateInfo::default().image(image); + let info = vk::MemoryAllocateInfo::default() + .allocation_size(requirements.size) + .memory_type_index(type_index) + .push(&mut import) + .push(&mut dedicated); + let memory = unsafe { device.allocate_memory(&info, None) }?; + unsafe { device.bind_image_memory(image, memory, 0) }?; + Ok(Self { + context: context.clone(), + image, + memory, + format, + width: buffer.width, + height: buffer.height, + written: false, + }) + } + + /// Make a new buffer with `modifier`, the producer allocating it. + fn allocate( + context: &VideoContext, + drm: u32, + width: u32, + height: u32, + modifier: u64, + ) -> Result<(Self, DmaBuf), Box> { + let format = pixelforge::encoder::external::vk_format_of(drm).unwrap(); + let image = + Self::create_image(context, format, width, height, Modifier::OneOf(&[modifier]))?; + let device = context.device(); + let requirements = unsafe { device.get_image_memory_requirements(image) }; + let type_index = context + .find_memory_type( + requirements.memory_type_bits, + vk::MemoryPropertyFlags::empty(), + ) + .unwrap(); + let mut export = vk::ExportMemoryAllocateInfo::default() + .handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let mut dedicated = vk::MemoryDedicatedAllocateInfo::default().image(image); + let info = vk::MemoryAllocateInfo::default() + .allocation_size(requirements.size) + .memory_type_index(type_index) + .push(&mut export) + .push(&mut dedicated); + let memory = unsafe { device.allocate_memory(&info, None) }?; + unsafe { device.bind_image_memory(image, memory, 0) }?; + let fd_fn = ash::khr::external_memory_fd::Device::load(context.instance(), device); + let fd = unsafe { + fd_fn.get_memory_fd( + &vk::MemoryGetFdInfoKHR::default() + .memory(memory) + .handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT), + ) + }?; + let planes = (0..if drm == drm_format::NV12 { 2 } else { 1 }) + .map(|plane| { + let aspect = [ + vk::ImageAspectFlags::MEMORY_PLANE_0_EXT, + vk::ImageAspectFlags::MEMORY_PLANE_1_EXT, + ][plane]; + let layout = unsafe { + device.get_image_subresource_layout( + image, + vk::ImageSubresource { + aspect_mask: aspect, + mip_level: 0, + array_layer: 0, + }, + ) + }; + DmaBufPlane { + offset: layout.offset, + stride: layout.row_pitch, + } + }) + .collect(); + let buffer = DmaBuf { + fd: unsafe { OwnedFd::from_raw_fd(fd) }, + width, + height, + format: drm, + modifier, + planes, + }; + let producer = Self { + context: context.clone(), + image, + memory, + format, + width, + height, + written: false, + }; + Ok((producer, buffer)) + } + + fn create_image( + context: &VideoContext, + format: vk::Format, + width: u32, + height: u32, + modifier: Modifier<'_>, + ) -> Result> { + let mut external = vk::ExternalMemoryImageCreateInfo::default() + .handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + let (mut list, mut explicit) = match modifier { + Modifier::OneOf(modifiers) => ( + Some( + vk::ImageDrmFormatModifierListCreateInfoEXT::default() + .drm_format_modifiers(modifiers), + ), + None, + ), + Modifier::Exactly(modifier, layouts) => ( + None, + Some( + vk::ImageDrmFormatModifierExplicitCreateInfoEXT::default() + .drm_format_modifier(modifier) + .plane_layouts(layouts), + ), + ), + }; + let mut info = vk::ImageCreateInfo::default() + .image_type(vk::ImageType::TYPE_2D) + .format(format) + .extent(vk::Extent3D { + width, + height, + depth: 1, + }) + .mip_levels(1) + .array_layers(1) + .samples(vk::SampleCountFlags::TYPE_1) + .tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT) + .usage(vk::ImageUsageFlags::TRANSFER_DST) + .sharing_mode(vk::SharingMode::EXCLUSIVE) + .initial_layout(vk::ImageLayout::UNDEFINED) + .push(&mut external); + if let Some(list) = list.as_mut() { + info = info.push(list); + } + if let Some(explicit) = explicit.as_mut() { + info = info.push(explicit); + } + Ok(unsafe { context.device().create_image(&info, None) }?) + } + + /// Write `data` into the buffer and hand it to the foreign family, as a + /// producer in another process would leave it. + fn write(&mut self, data: &[u8]) -> Result<(), Box> { + let context = self.context.clone(); + let device = context.device(); + let (staging, staging_memory) = unsafe { + host_buffer( + &context, + data.len() as u64, + vk::BufferUsageFlags::TRANSFER_SRC, + )? + }; + unsafe { + let ptr = device.map_memory( + staging_memory, + 0, + vk::WHOLE_SIZE, + vk::MemoryMapFlags::empty(), + )?; + std::ptr::copy_nonoverlapping(data.as_ptr(), ptr.cast::(), data.len()); + device.unmap_memory(staging_memory); + } + let family = context.compute_queue_family(); + let color = vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_mip_level: 0, + level_count: 1, + base_array_layer: 0, + layer_count: 1, + }; + let (w, h) = (self.width, self.height); + let regions: Vec = + if self.format == vk::Format::G8_B8R8_2PLANE_420_UNORM { + vec![ + plane_copy(0, vk::ImageAspectFlags::PLANE_0, w, h), + plane_copy((w * h) as u64, vk::ImageAspectFlags::PLANE_1, w / 2, h / 2), + ] + } else { + vec![plane_copy(0, vk::ImageAspectFlags::COLOR, w, h)] + }; + let written = self.written; + unsafe { + one_shot(&context, |cb| { + let to_dst = vk::ImageMemoryBarrier::default() + .old_layout(if written { + vk::ImageLayout::GENERAL + } else { + vk::ImageLayout::UNDEFINED + }) + .new_layout(vk::ImageLayout::TRANSFER_DST_OPTIMAL) + .src_queue_family_index(if written { + vk::QUEUE_FAMILY_FOREIGN_EXT + } else { + vk::QUEUE_FAMILY_IGNORED + }) + .dst_queue_family_index(if written { + family + } else { + vk::QUEUE_FAMILY_IGNORED + }) + .image(self.image) + .subresource_range(color) + .dst_access_mask(vk::AccessFlags::TRANSFER_WRITE); + device.cmd_pipeline_barrier( + cb, + vk::PipelineStageFlags::TOP_OF_PIPE, + vk::PipelineStageFlags::TRANSFER, + vk::DependencyFlags::empty(), + &[], + &[], + &[to_dst], + ); + device.cmd_copy_buffer_to_image( + cb, + staging, + self.image, + vk::ImageLayout::TRANSFER_DST_OPTIMAL, + ®ions, + ); + let release = vk::ImageMemoryBarrier::default() + .old_layout(vk::ImageLayout::TRANSFER_DST_OPTIMAL) + .new_layout(vk::ImageLayout::GENERAL) + .src_queue_family_index(family) + .dst_queue_family_index(vk::QUEUE_FAMILY_FOREIGN_EXT) + .image(self.image) + .subresource_range(color) + .src_access_mask(vk::AccessFlags::TRANSFER_WRITE); + device.cmd_pipeline_barrier( + cb, + vk::PipelineStageFlags::TRANSFER, + vk::PipelineStageFlags::BOTTOM_OF_PIPE, + vk::DependencyFlags::empty(), + &[], + &[], + &[release], + ); + })?; + device.destroy_buffer(staging, None); + device.free_memory(staging_memory, None); + } + self.written = true; + Ok(()) + } +} + +impl Drop for Producer { + fn drop(&mut self) { + let device = self.context.device(); + unsafe { + device.destroy_image(self.image, None); + device.free_memory(self.memory, None); + } + } +} + +fn plane_copy( + offset: u64, + aspect: vk::ImageAspectFlags, + width: u32, + height: u32, +) -> vk::BufferImageCopy { + vk::BufferImageCopy { + buffer_offset: offset, + buffer_row_length: 0, + buffer_image_height: 0, + image_subresource: vk::ImageSubresourceLayers { + aspect_mask: aspect, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + }, + image_offset: vk::Offset3D::default(), + image_extent: vk::Extent3D { + width, + height, + depth: 1, + }, + } +} + +/// The stream of encoding `frames` from an ordinary YUV image. +fn encode_reference( + context: &VideoContext, + config: EncodeConfig, + frames: &[Vec], +) -> Result, Box> { + let (w, h) = (config.dimensions.width, config.dimensions.height); + let mut encoder = Encoder::new(context.clone(), config.clone())?; + let mut input = InputImage::new( + context.clone(), + config.codec, + w, + h, + EncodeBitDepth::Eight, + PixelFormat::Yuv420, + )?; + let mut stream = Vec::new(); + for frame in frames { + input.upload_nv12(frame)?; + stream.extend(pollster::block_on(encoder.encode(input.image())?)?.data); + } + Ok(stream) +} + +/// The stream of encoding `frames` written by `producer` into `external`. +fn encode_external( + encoder: &mut Encoder, + producer: &mut Producer, + external: &ExternalImage, + frames: &[Vec], +) -> Result, Box> { + let mut stream = Vec::new(); + for frame in frames { + producer.write(frame)?; + let packet = pollster::block_on(encoder.submit(Frame::external(external))?)?; + stream.extend(packet.data); + } + Ok(stream) +} + +/// Every modifier the encoder can allocate. +fn exportable_modifiers(encoder: &Encoder) -> Vec { + encoder.input_formats().map_or(Vec::new(), |formats| { + formats + .modifiers + .iter() + .filter(|m| m.exportable) + .map(|m| m.modifier) + .collect() + }) +} + +/// A buffer the encoder allocated encodes exactly what an ordinary image with +/// the same pixels does, whether the encode reads it directly or copies it. +/// Then the same buffer, imported back, does too. +#[test] +#[ignore = "requires a Vulkan Video device with DMA-BUF support"] +fn allocated_and_imported_inputs_encode_the_same() -> Result<(), Box> { + let context = context()?; + // A size every codec's block size divides, so the reference's coded + // extent is the display size too and the streams can match exactly. + let (w, h) = (320, 256); + let frames: Vec> = (0..FRAMES).map(|n| nv12_frame(w, h, n)).collect(); + for codec in [Codec::H264, Codec::H265, Codec::AV1] { + if !context.supports_encode(codec) { + continue; + } + let reference = encode_reference(&context, config(codec, w, h), &frames)?; + + let mut encoder = Encoder::new(context.clone(), config(codec, w, h))?; + let modifiers = exportable_modifiers(&encoder); + if modifiers.is_empty() { + eprintln!("skipping {codec:?}: no modifier the encoder can allocate"); + continue; + } + let allocated = encoder.allocate_input(&modifiers)?; + println!( + "{codec:?}: allocated modifier {:#x}, zero copy {}", + allocated.modifier(), + allocated.is_zero_copy() + ); + let exported = allocated.export()?; + let mut producer = Producer::import(&context, &exported)?; + let stream = encode_external(&mut encoder, &mut producer, &allocated, &frames)?; + assert!(stream == reference, "{codec:?}: allocated input differs"); + + let mut encoder = Encoder::new(context.clone(), config(codec, w, h))?; + let imported = encoder.import(&exported)?; + assert_eq!(imported.is_zero_copy(), allocated.is_zero_copy()); + let stream = encode_external(&mut encoder, &mut producer, &imported, &frames)?; + assert!(stream == reference, "{codec:?}: imported input differs"); + } + Ok(()) +} + +/// A buffer with a modifier the encode cannot read is copied instead, and +/// encodes the same. +#[test] +#[ignore = "requires a Vulkan Video device with DMA-BUF support"] +fn copy_only_modifiers_are_copied() -> Result<(), Box> { + let context = context()?; + let (w, h) = (320, 256); + let frames: Vec> = (0..FRAMES).map(|n| nv12_frame(w, h, n)).collect(); + let reference = encode_reference(&context, config(Codec::H264, w, h), &frames)?; + let mut encoder = Encoder::new(context.clone(), config(Codec::H264, w, h))?; + let formats = encoder.input_formats()?; + let Some(copy_only) = formats + .modifiers + .iter() + .find(|m| m.importable && !m.zero_copy) + else { + eprintln!("skipping: every importable modifier is read directly here"); + return Ok(()); + }; + let (mut producer, buffer) = + Producer::allocate(&context, drm_format::NV12, w, h, copy_only.modifier)?; + let imported = encoder.import(&buffer)?; + assert!(!imported.is_zero_copy()); + let stream = encode_external(&mut encoder, &mut producer, &imported, &frames)?; + assert!(stream == reference, "copied input differs"); + Ok(()) +} + +/// An RGB buffer goes to the colour converter, which reads it directly, and +/// encodes what converting an ordinary RGB image does. +#[test] +#[ignore = "requires a Vulkan Video device with DMA-BUF support"] +fn rgb_buffers_are_converted_directly() -> Result<(), Box> { + let context = context()?; + let (w, h) = (320, 256); + let rgb = Source::Rgb(RgbSource::new(InputFormat::BGRA, ColorSpec::Srgb)); + let shader = |c: EncodeConfig| { + c.with_source(rgb) + .with_conversion(ConversionPreference::Shader) + }; + + let mut reference_encoder = Encoder::new(context.clone(), shader(config(Codec::H264, w, h)))?; + let mut reference = Vec::new(); + let mut encoder = Encoder::new(context.clone(), shader(config(Codec::H264, w, h)))?; + assert_eq!(encoder.conversion(), Conversion::Shader); + let modifiers = exportable_modifiers(&encoder); + if modifiers.is_empty() { + eprintln!("skipping: no RGB modifier the shader reads"); + return Ok(()); + } + let allocated = encoder.allocate_input(&modifiers)?; + assert!( + allocated.is_zero_copy(), + "the shader reads what it can import" + ); + let mut producer = Producer::import(&context, &allocated.export()?)?; + let mut stream = Vec::new(); + for n in 0..FRAMES { + let data = bgra_frame(w, h, n); + let src = unsafe { common::source::create_src_image(&context, w, h, &data)? }; + reference.extend(pollster::block_on(reference_encoder.encode(src.image)?)?.data); + producer.write(&data)?; + stream.extend(pollster::block_on(encoder.submit(Frame::external(&allocated))?)?.data); + } + assert!(stream == reference, "converted DMA-BUF differs"); + Ok(()) +} + +/// A source smaller than the aligned coded extent is read directly at the +/// display size, and decodes to the frames it was given. +#[test] +#[ignore = "requires a Vulkan Video device with DMA-BUF support"] +fn unaligned_sizes_are_read_at_the_display_size() -> Result<(), Box> { + let context = VideoContextBuilder::new() + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .require_encode(Codec::H264) + .require_decode(Codec::H264) + .build()?; + // 232 rows is not a whole number of macroblocks. + let (w, h) = (320, 232); + let frames: Vec> = (0..FRAMES).map(|n| nv12_frame(w, h, n)).collect(); + let mut encoder = Encoder::new(context.clone(), config(Codec::H264, w, h))?; + let allocated = encoder.allocate_input(&exportable_modifiers(&encoder))?; + if !allocated.is_zero_copy() { + eprintln!("skipping: the encoder copies DMA-BUFs on this device"); + return Ok(()); + } + let mut producer = Producer::import(&context, &allocated.export()?)?; + let stream = encode_external(&mut encoder, &mut producer, &allocated, &frames)?; + + let mut readback = Readback::new(&context)?; + let mut decoder = Decoder::new(context.clone(), DecodeConfig::h264().with_byte_stream())?; + let mut decoded = Vec::new(); + decode_stream(&mut decoder, &stream, |frame| { + decoded.push(readback.read(&frame)?.y); + Ok(()) + })?; + assert_eq!(decoded.len(), FRAMES as usize); + for (n, (got, want)) in decoded.iter().zip(&frames).enumerate() { + let want = &want[..(w * h) as usize]; + assert_eq!(got.len(), want.len(), "frame {n}: decoded size"); + let mse = got + .iter() + .zip(want) + .map(|(&a, &b)| (a as f64 - b as f64).powi(2)) + .sum::() + / want.len() as f64; + let psnr = 10.0 * (255.0f64 * 255.0 / mse.max(1e-9)).log10(); + assert!(psnr > 30.0, "frame {n}: {psnr:.1} dB"); + } + Ok(()) +} + +/// The release of one frame, as a sync_file, gates the next one's read. +#[test] +#[ignore = "requires a Vulkan Video device with sync_file support"] +fn sync_files_chain_frames() -> Result<(), Box> { + let context = context()?; + let (w, h) = (320, 256); + let mut encoder = Encoder::new(context.clone(), config(Codec::H264, w, h))?; + let modifiers = exportable_modifiers(&encoder); + if modifiers.is_empty() { + eprintln!("skipping: no modifier the encoder can allocate"); + return Ok(()); + } + let allocated = encoder.allocate_input(&modifiers)?; + let mut producer = Producer::import(&context, &allocated.export()?)?; + let mut fence = None; + let mut released = 0; + for n in 0..FRAMES { + producer.write(&nv12_frame(w, h, n))?; + let mut frame = Frame::external(&allocated).signal_sync_file(); + if let Some(fence) = fence.take() { + frame = frame.wait_sync_file(fence); + } + let mut future = encoder.submit(frame)?; + fence = future.take_release_sync_file(); + released += usize::from(fence.is_some()); + assert!(!pollster::block_on(future)?.data.is_empty()); + } + println!("{released} of {FRAMES} releases came as a pending sync_file"); + Ok(()) +} diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs index e2d25681..d77b5943 100644 --- a/tests/rgb_encode.rs +++ b/tests/rgb_encode.rs @@ -1,13 +1,12 @@ -//! Encoding RGB input with the encoder's own colour conversion, through -//! `VK_VALVE_video_encode_rgb_conversion`, against the colour converter. +//! Encoding an RGB [`Source`], through each [`Conversion`] the encoder can +//! pick. //! -//! When the conversion is nothing but the YUV matrix, the encoder can take the -//! RGB frame as it is and the shader does not run at all. These tests encode -//! the same frames both ways and check the hardware path lands where the -//! shader does. -//! -//! Skipped on devices without the extension, which today is everything but -//! AMD under RADV. +//! With [`Conversion::Shader`] the encoder runs the colour converter itself, +//! so its stream must be byte-identical to converting by hand and encoding +//! the result. With [`Conversion::Encoder`] the encoder converts through +//! `VK_VALVE_video_encode_rgb_conversion`; those tests check that it lands +//! where the shader does, and skip on devices without the extension, which +//! today is everything but AMD under RADV. #[allow(dead_code)] mod common; @@ -18,8 +17,8 @@ use ash::vk; use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::{ Codec, ColorConverter, ColorConverterConfig, ColorDescription, ColorRange, ColorSpec, - EncodeBitDepth, EncodeConfig, Encoder, InputFormat, OutputFormat, RateControlMode, - VideoContext, VideoContextBuilder, + Conversion, ConversionPreference, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputFormat, + OutputFormat, RateControlMode, RgbSource, Source, VideoContext, VideoContextBuilder, }; const WIDTH: u32 = 320; @@ -61,8 +60,8 @@ fn moving_frame(n: u32) -> Vec { data } -/// Upload frame `n`. It is left in `GENERAL`, which is where [`Encoder::encode`] -/// expects a source it copies from. +/// Upload frame `n`. It is left in `GENERAL`, the layout [`Encoder::encode`] +/// expects. fn source(context: &VideoContext, n: u32) -> Result> { Ok(unsafe { create_src_image(context, WIDTH, HEIGHT, &moving_frame(n))? }) } @@ -81,8 +80,13 @@ fn encode_config(codec: Codec) -> EncodeConfig { .with_color_description(converter_config().color_description().unwrap()) } -/// Encode the frames with the colour converter doing the conversion. -fn encode_with_shader( +fn rgb(format: InputFormat) -> Source { + Source::Rgb(RgbSource::new(format, ColorSpec::Srgb)) +} + +/// Encode the frames with a YUV encoder and a colour converter the test runs +/// itself. +fn encode_by_hand( context: &VideoContext, codec: Codec, ) -> Result, Box> { @@ -98,13 +102,19 @@ fn encode_with_shader( Ok(stream) } -/// Encode the frames with the encoder doing the conversion. +/// Encode the frames with an RGB source and the given conversion. fn encode_rgb( context: &VideoContext, codec: Codec, format: InputFormat, + preference: ConversionPreference, ) -> Result, Box> { - let mut encoder = Encoder::new(context.clone(), encode_config(codec).with_rgb_input(format))?; + let mut encoder = Encoder::new( + context.clone(), + encode_config(codec) + .with_source(rgb(format)) + .with_conversion(preference), + )?; let mut stream = Vec::new(); for n in 0..FRAMES { let src = source(context, n)?; @@ -142,24 +152,76 @@ fn psnr(a: &[u8], b: &[u8]) -> f64 { } } -/// The format to hand a `codec` encoder, or `None` to skip it on this device. -fn rgb_format(context: &VideoContext, codec: Codec, test: &str) -> Option { - let format = converter_config().rgb_encode_input(context, codec); - if format.is_none() { - eprintln!("skipping {test} for {codec:?}: this device has no RGB input for it"); +/// Whether the encoder can convert `format` for `codec` here; prints why a +/// test is skipped when not. +fn encoder_converts(context: &VideoContext, codec: Codec, test: &str) -> bool { + let desc = converter_config().color_description().unwrap(); + let ok = context.supports_rgb_encode_input(codec, InputFormat::BGRA, &desc); + if !ok { + eprintln!("skipping {test} for {codec:?}: the encoder cannot convert RGB here"); + } + ok +} + +/// The shader path inside the encoder does exactly what converting by hand +/// does, so the streams match byte for byte. +#[test] +#[ignore = "requires a Vulkan Video device"] +fn shader_source_matches_converting_by_hand() -> Result<(), Box> { + for codec in [Codec::H264, Codec::H265, Codec::AV1] { + let Ok(context) = context(codec) else { + eprintln!("{codec:?}: no encoder on this device"); + continue; + }; + let by_hand = encode_by_hand(&context, codec)?; + let shader = encode_rgb( + &context, + codec, + InputFormat::BGRA, + ConversionPreference::Shader, + )?; + assert!(!by_hand.is_empty(), "{codec:?}: empty stream"); + assert!(by_hand == shader, "{codec:?}: the streams differ"); } - format + Ok(()) +} + +/// `Auto` picks the encoder's own conversion exactly where it is available. +#[test] +#[ignore = "requires a Vulkan Video device"] +fn auto_picks_the_encoder_where_it_can() -> Result<(), Box> { + let context = context(Codec::H264)?; + let encoder = Encoder::new( + context.clone(), + encode_config(Codec::H264).with_source(rgb(InputFormat::BGRA)), + )?; + let desc = converter_config().color_description().unwrap(); + let expected = if context.supports_rgb_encode_input(Codec::H264, InputFormat::BGRA, &desc) { + Conversion::Encoder + } else { + Conversion::Shader + }; + assert_eq!(encoder.conversion(), expected); + Ok(()) } #[test] #[ignore = "requires a Vulkan Video device with VK_VALVE_video_encode_rgb_conversion"] fn rgb_input_matches_the_converter() -> Result<(), Box> { let context = context(Codec::H264)?; - let Some(format) = rgb_format(&context, Codec::H264, "rgb_input_matches_the_converter") else { + if !encoder_converts(&context, Codec::H264, "rgb_input_matches_the_converter") { return Ok(()); - }; - let shader = decode_all(&context, &encode_with_shader(&context, Codec::H264)?)?; - let hardware = decode_all(&context, &encode_rgb(&context, Codec::H264, format)?)?; + } + let shader = decode_all(&context, &encode_by_hand(&context, Codec::H264)?)?; + let hardware = decode_all( + &context, + &encode_rgb( + &context, + Codec::H264, + InputFormat::BGRA, + ConversionPreference::Encoder, + )?, + )?; assert_eq!(hardware.len(), FRAMES as usize, "frame count"); let psnrs: Vec = hardware .iter() @@ -187,10 +249,15 @@ fn rgb_input_encodes_every_codec() -> Result<(), Box> { continue; } }; - let Some(format) = rgb_format(&context, codec, "rgb_input_encodes_every_codec") else { + if !encoder_converts(&context, codec, "rgb_input_encodes_every_codec") { continue; - }; - let stream = encode_rgb(&context, codec, format)?; + } + let stream = encode_rgb( + &context, + codec, + InputFormat::BGRA, + ConversionPreference::Encoder, + )?; assert!(!stream.is_empty(), "{codec:?}: empty stream"); if let Ok(dir) = std::env::var("PIXELFORGE_RGB_STREAM_DIR") { std::fs::write(format!("{dir}/rgb.{codec:?}").to_lowercase(), &stream)?; @@ -199,10 +266,11 @@ fn rgb_input_encodes_every_codec() -> Result<(), Box> { Ok(()) } -/// `supports_rgb_encode_input` must agree with what an encoder can actually -/// be created for, on every device: one that answers yes for something the -/// encoder then refuses sends a caller down a path that fails, and one that -/// answers no for something that works leaves the shader running for nothing. +/// `supports_rgb_encode_input` must agree with what an encoder forced to +/// convert can actually be created for, on every device: one that answers +/// yes for something the encoder then refuses would make `Auto` fail, and one +/// that answers no for something that works leaves the shader running for +/// nothing. #[test] #[ignore = "requires a Vulkan Video device"] fn rgb_query_matches_encoder_creation() -> Result<(), Box> { @@ -234,13 +302,19 @@ fn rgb_query_matches_encoder_creation() -> Result<(), Box EncodeBitDepth::Eight }; for (name, desc) in &descriptions { + let color = if desc.is_hdr() { + ColorSpec::Bt2020Pq + } else { + ColorSpec::Srgb + }; let supported = context.supports_rgb_encode_input(codec, format, desc); let created = Encoder::new( context.clone(), encode_config(codec) .with_bit_depth(bit_depth) .with_color_description(*desc) - .with_rgb_input(format), + .with_source(Source::Rgb(RgbSource::new(format, color))) + .with_conversion(ConversionPreference::Encoder), ) .is_ok(); eprintln!("{codec:?} {format:?} {name}: query={supported} encoder={created}"); @@ -253,3 +327,38 @@ fn rgb_query_matches_encoder_creation() -> Result<(), Box } Ok(()) } + +/// A colour change keeps the conversion in step with the label, the next +/// packet is a key frame, and timestamps carry on. +#[test] +#[ignore = "requires a Vulkan Video device"] +fn color_change_follows_into_the_conversion() -> Result<(), Box> { + let context = context(Codec::H264)?; + let mut encoder = Encoder::new( + context.clone(), + encode_config(Codec::H264).with_source(rgb(InputFormat::BGRA)), + )?; + let conversion = encoder.conversion(); + let mut pts = Vec::new(); + let mut keys = Vec::new(); + for n in 0..FRAMES { + if n == FRAMES / 2 { + encoder.set_color_description(ColorDescription::bt709().with_full_range(true))?; + } + let src = source(&context, n)?; + let packet = + pollster::block_on(encoder.submit(Frame::image(src.image, vk::ImageLayout::GENERAL))?)?; + pts.push(packet.pts); + keys.push(packet.is_key_frame); + } + assert_eq!(pts, (0..u64::from(FRAMES)).collect::>()); + assert!(keys[0] && keys[FRAMES as usize / 2], "key frames: {keys:?}"); + assert!( + encoder.config().color_description.unwrap().full_range, + "the description was not kept" + ); + // Full range is not something every encoder converts to; where it does + // not, the shader takes over. + println!("conversion {conversion:?} -> {:?}", encoder.conversion()); + Ok(()) +} From 92f5b8a66b57288c64ccef6367040b94a42b94f9 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 08:40:17 +0100 Subject: [PATCH 04/19] feat(encoder)!: submit frames only through source images and input slots --- CHANGELOG.md | 29 ++++--- README.md | 35 +++++---- benches/encode.rs | 6 +- docs/encode-input.md | 57 ++++++++++---- examples/encode.rs | 4 +- src/converter/mod.rs | 10 ++- src/decoder/mod.rs | 2 +- src/encoder/api.rs | 136 ++++++++++++++++---------------- src/encoder/codec.rs | 5 -- src/encoder/external.rs | 48 +++++------- src/encoder/frame.rs | 83 ++++++++++++-------- src/encoder/input.rs | 84 ++++++++++++++++---- src/encoder/mod.rs | 4 +- src/encoder/pipeline.rs | 18 ++--- src/encoder/source_image.rs | 151 ++++++++++++++++++++++++++++++++++++ src/image.rs | 75 +++++++++++++----- src/lib.rs | 43 ++++++---- src/sync.rs | 13 ++-- src/vulkan.rs | 2 +- tests/adopted.rs | 9 ++- tests/color_conversion.rs | 12 ++- tests/dmabuf.rs | 15 +++- tests/rfi.rs | 8 +- tests/rgb_encode.rs | 31 ++++++-- tests/roundtrip.rs | 4 +- tests/submit.rs | 94 ++++++++++++++++++++++ tests/verify_all.rs | 10 +-- 27 files changed, 703 insertions(+), 285 deletions(-) create mode 100644 src/encoder/source_image.rs create mode 100644 tests/submit.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ad7701d..62668a94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `VK_VALVE_video_encode_rgb_conversion` where the device can and the colour converter's shader otherwise (`Encoder::conversion`, `EncodeConfig::with_conversion`). -- Add `Encoder::submit` and `Frame`: one way to hand over a frame, whatever - the conversion, with its GPU waits. -- Add `EncodeFuture::source_released`: the point after which a frame's source - may be written again. +- Add `Encoder::submit` and `Frame`: the one way to hand over a frame, + whatever the conversion, with its GPU waits. +- Add `SourceImage` and `Handoff`: a caller's image to encode from. It keeps + track of the frames that read it (`SourceImage::released`), and its drop + waits for them. +- Add `Encoder::input_slot` and `InputSlot`: the encoder's own input image, + to write a frame in place and submit with `Frame::input`. +- Add `InputImage::source`. - Add DMA-BUF input (feature `dmabuf`): `Encoder::input_formats`, `Encoder::import`, `Encoder::allocate_input`, `ExternalImage` and `Frame::external`. The encoder reads a buffer directly where the device @@ -31,13 +35,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - The copy of a frame into the encoder's input image no longer waits on the - CPU: the encode waits for it on the GPU. `Encoder::encode_after` and - `Encoder::submit` return before the source is read; `Encoder::encode` still - returns once the source can be written again. -- `Encoder::encode_after` takes the source image's layout, as - `ColorConverter::convert_async` does: for `UNDEFINED`, a first-time - external-memory import, the copy into the input image acquires it from - `VK_QUEUE_FAMILY_EXTERNAL`. + CPU: the encode waits for it on the GPU. `Encoder::submit` returns before + the source is read. +- **Breaking:** `Encoder::encode`, `Encoder::encode_after` and + `Encoder::input_image` are removed. Submit a `Frame` with `Encoder::submit`: + `Frame::image(&source_image)` for an image of the caller's, + `Frame::input(encoder.input_slot()?)` for the input image written in place. + An imported image that `UNDEFINED` used to mark is `Handoff::External`. +- **Breaking:** `InputImage::upload_yuv420_to` and `upload_yuv444_to` take an + `InputSlot` instead of a `vk::Image`. `InputImage` uploads wait until the + frames that read the image are done with it. - `Encoder::set_color_description` also updates the conversion of an RGB source, and makes the encoder again where the encoder converts. Packet timestamps carry on. diff --git a/README.md b/README.md index 67151764..22f363cf 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ fn main() -> Result<(), Box> { // For each frame: upload YUV data and encode. // let yuv_data: &[u8] = ...; // YUV420 frame data // input_image.upload_yuv420(yuv_data)?; - // let packets = encoder.encode(input_image.image())?; + // let packet = encoder.submit(Frame::image(input_image.source()))?; Ok(()) } @@ -259,13 +259,19 @@ encoding. ### Submitting frames -[`Encoder::submit`] takes a [`Frame`]: where the pixels are, and what has -to happen on the GPU before they can be read. It does not wait on the CPU, -except when every pipeline slot is in flight. The source is read by a copy, -the shader, or the encode itself, after the call returns, so write it again -only after [`EncodeFuture::source_released`]. [`Encoder::encode`] is the -short form that also waits for that point, which keeps the simple loop of -writing one image and encoding it correct. +[`Encoder::submit`] is the one way to encode a frame. It takes a [`Frame`]: +where the pixels are, and what has to happen on the GPU before they can be +read. It does not wait on the CPU, except when every pipeline slot is in +flight. The pixels are in one of: + +- an [`InputSlot`], the encoder's own input image, from + [`Encoder::input_slot`]. The encode reads it in place, with no copy. +- a [`SourceImage`], an image of the caller's. Pixelforge copies or + converts it into the input image after the call returns, and the + `SourceImage` keeps track of that: write the image again only after + [`SourceImage::released`]. [`InputImage`], for frames uploaded from the + CPU, waits for that itself. +- an `ExternalImage`, a DMA-BUF (see below). ### Encoding on your own device @@ -283,15 +289,16 @@ the application does not use, with Each frame is then ordered on the GPU: the application's submission signals a [`TimelinePoint`], the encoder's first submission for the frame waits for -it, and the application waits for [`EncodeFuture::source_released`] before -it draws into the image again. +it, and the application waits for [`SourceImage::released`] before it draws +into the image again. ```rust -// `rendered` reaches `frame_number` once the application has drawn `image`. +// `rendered` reaches `frame_number` once the application has drawn +// `source.image()`. let ready = [TimelinePoint::new(rendered, frame_number)]; -let packet = encoder.submit(Frame::image(image, vk::ImageLayout::GENERAL).wait(&ready))?; -// Wait on this before drawing into `image` again. -let reusable = packet.source_released(); +let packet = encoder.submit(Frame::image(source).wait(&ready))?; +// Wait on these before drawing into the image again. +let reusable = source.released(); ``` ### DMA-BUF input diff --git a/benches/encode.rs b/benches/encode.rs index fc597f98..77b4770a 100644 --- a/benches/encode.rs +++ b/benches/encode.rs @@ -9,8 +9,8 @@ //! cargo bench --bench encode use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, EncoderTuningMode, InputImage, PixelFormat, - RateControlMode, VideoContextBuilder, + Codec, EncodeBitDepth, EncodeConfig, Encoder, EncoderTuningMode, Frame, InputImage, + PixelFormat, RateControlMode, VideoContextBuilder, }; use std::fs::File; use std::io::Read; @@ -176,7 +176,7 @@ fn run_encode_test( // Upload YUV420 data to the input image input_image.upload_yuv420(frame)?; - pending.push_back(encoder.encode(input_image.image())?); + pending.push_back(encoder.submit(Frame::image(input_image.source()))?); // drain before submitting when at capacity while pending.len() >= 2 { let packet = pollster::block_on(pending.pop_front().unwrap())?; diff --git a/docs/encode-input.md b/docs/encode-input.md index 4f17bf8f..a6f7c04e 100644 --- a/docs/encode-input.md +++ b/docs/encode-input.md @@ -8,7 +8,7 @@ method for each. ## Goals 1. The caller describes the source one time. Pixelforge selects the path. -2. One encode entry point for all paths. +2. One encode entry point for all paths, and no second, older way. 3. No CPU wait in the hot path. The GPU orders all work. 4. The caller knows when it can reuse a source image. 5. A DMA-BUF source goes to the encoder with no pixelforge copy, when the @@ -61,23 +61,47 @@ not. ## Submitting a frame +`Encoder::submit(Frame)` is the one way to encode a frame: + ```rust -let future = encoder.submit(Frame::image(image, layout).wait(&points))?; +let slot = encoder.input_slot()?; +// ... write the frame into slot.image() ... +let future = encoder.submit(Frame::input(slot).wait(&points))?; + +let future = encoder.submit(Frame::image(&source_image).wait(&points))?; let future = encoder.submit(Frame::external(&imported).wait_sync_file(fence))?; ``` -`Encoder::encode(image)` and `Encoder::encode_after(image, layout, waits)` -stay, as short forms of `submit`. - A `Frame` names its source: -- `Frame::image(image, layout)`: a `vk::Image` the caller owns. If it is - `Encoder::input_image()`, the encoder reads it in place. Otherwise - pixelforge copies it (`None`, `Encoder`) or converts it (`Shader`) into the - input image. +- `Frame::input(InputSlot)`: the encoder's own input image for the current + slot, written in place. `Encoder::input_slot()` waits for the slot and + returns the slot. A slot is valid only until the encoder's next submission: + a later submit of it is an error. With `Shader` there is no slot, because + the input image belongs to the converter. +- `Frame::image(&SourceImage)`: a `vk::Image` the caller owns. Pixelforge + copies it (`None`, `Encoder`) or converts it (`Shader`) into the input + image. - `Frame::external(&ExternalImage)`: a DMA-BUF that pixelforge imported or allocated. See below. +### Source images + +A `SourceImage` wraps a caller's `vk::Image` with a `Handoff`, which says how +each frame gets the image and gives it back: + +- `Handoff::Device { layout }`: the caller writes it on this device and hands + it over in `layout`. Pixelforge leaves it in `layout`. `UNDEFINED` and + `PREINITIALIZED` are refused, because they keep no pixels. +- `Handoff::External`: imported memory, written outside this device. Each + frame acquires it from the foreign queue family, and releases it back in + `GENERAL`. The first acquire uses `UNDEFINED`. + +The handle keeps the release point of each frame that read the image. +`SourceImage::released()` gives those points. `wait_released()` waits for +them on the CPU, and the drop waits for them too. `InputImage` holds one, so +its uploads wait for the previous frames by themselves. + ### No CPU waits The copy from a caller image into the input image signals a timeline @@ -91,20 +115,19 @@ flight, `submit` waits for the oldest one. ### Release point -`EncodeFuture::source_released()` returns the `TimelinePoint` after which the -source can be written again, or given back to its producer: +The image, not the future, holds the point after which the source can be +written again, or given back to its producer: | Source | Released by | |---|---| -| `Encoder::input_image()` | the encode | -| caller image, `None` / `Encoder` | the copy | -| caller image, `Shader` | the conversion | +| `InputSlot` | the encode | +| `SourceImage`, `None` / `Encoder` | the copy | +| `SourceImage`, `Shader` | the conversion | | `ExternalImage`, read directly | the encode | | `ExternalImage`, copied | the copy | -Before this change, `encode()` returned only after the copy was done. Now it -returns at once, so a caller that writes the source again must wait on this -point first. +`SourceImage::released()` and `ExternalImage::released()` give it. A slot +needs none: the next `input_slot()` waits for the slot. The semaphores behind these points are retired to the context when the encoder is dropped, as the converter already does. A wait on a point that the diff --git a/examples/encode.rs b/examples/encode.rs index 20a526b0..c184620e 100644 --- a/examples/encode.rs +++ b/examples/encode.rs @@ -10,7 +10,7 @@ //! The codec defaults to `h264`. use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, InputImage, PixelFormat, RateControlMode, + Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControlMode, VideoContextBuilder, }; use std::fs::File; @@ -144,7 +144,7 @@ fn main() -> Result<(), Box> { input_image.upload_yuv420(frame)?; // Submit the frame (async) and keep the pipeline at most ~2 deep. - pending.push_back(encoder.encode(input_image.image())?); + pending.push_back(encoder.submit(Frame::image(input_image.source()))?); while pending.len() > 2 { let packet = pollster::block_on(pending.pop_front().unwrap())?; write_packet(packet, &mut total_bytes)?; diff --git a/src/converter/mod.rs b/src/converter/mod.rs index 91bd88c1..af38dbb0 100644 --- a/src/converter/mod.rs +++ b/src/converter/mod.rs @@ -779,7 +779,7 @@ impl ColorConverter { /// /// The conversion waits on the GPU for every point in `wait` before /// reading `src_image`, and returns the point it signals when done. Pass - /// that to [`Encoder::encode_after`](crate::Encoder::encode_after) to + /// that to [`Frame::wait`](crate::Frame::wait) to /// encode the result, and wait on it before writing `src_image` again. /// /// On a device shared with the caller, where the caller renders or copies @@ -806,7 +806,13 @@ impl ColorConverter { wait.iter().map(TimelinePoint::wait_info).collect(); self.convert_with( src_image, - SourceOwnership::caller_image(src_layout), + SourceOwnership { + old_layout: src_layout, + acquire_from: (src_layout == vk::ImageLayout::UNDEFINED) + .then_some(vk::QUEUE_FAMILY_EXTERNAL), + release_to: None, + final_layout: vk::ImageLayout::GENERAL, + }, target_image, &waits, &[], diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 1f92b6f9..342728f7 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -14,7 +14,7 @@ //! while the current one decodes. Frames arrive on the //! [`DecodeSource`] as they complete, and the two halves can be driven from //! separate threads. Mirrors -//! [`Encoder::encode`](crate::encoder::Encoder::encode). +//! [`Encoder::submit`](crate::encoder::Encoder::submit). //! - **Stream-driven**: the Vulkan video session is created lazily from the //! stream's own parameter sets, so no dimensions or profile need to be //! configured up front. Mid-stream resolution changes recreate the session diff --git a/src/encoder/api.rs b/src/encoder/api.rs index f4c61a02..0866cf9d 100644 --- a/src/encoder/api.rs +++ b/src/encoder/api.rs @@ -4,9 +4,10 @@ use ash::vk; use super::codec::{CodecEncoder, EncoderCommon, VideoCodec}; -use super::frame::{Frame, FrameSource}; +use super::frame::{Frame, FrameSource, InputSlot}; use super::input::{FrameInput, FrameSync, SourceOwnership}; use super::source::{Conversion, ConversionPreference, plan, target_space}; +use super::source_image::SourceImage; use super::{Codec, ColorDescription, EncodeConfig, EncodeFuture}; use crate::converter::{ColorConverter, ColorRange}; use crate::error::{PixelForgeError, Result}; @@ -70,6 +71,9 @@ pub struct Encoder { pts_offset: u64, /// Tells encoders apart, so an image made for one is not read by another. id: u64, + /// Counts submissions, so an [`InputSlot`] is submitted only before the + /// next one. + ticket: u64, /// Counts the times the codec encoder was made again with a different /// conversion. An [`ExternalImage`](super::external::ExternalImage) made /// for one cannot be read by another. @@ -109,6 +113,7 @@ impl Encoder { converter, pts_offset: 0, id: NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed), + ticket: 0, generation: 0, #[cfg(feature = "dmabuf")] sync_files: None, @@ -125,36 +130,23 @@ impl Encoder { &self.config } - /// Get the current slot's input image. + /// The encoder's input image for the next frame, to write it in place. /// - /// For a YUV source, rendering or uploading into this image and passing it - /// to [`Frame::image`] encodes it in place, with no copy. It is in - /// `VIDEO_ENCODE_SRC_KHR` layout, and must be left there. This call waits - /// until the slot's previous encode has finished with it. + /// Write the frame into [`InputSlot::image`] and submit it with + /// [`Frame::input`]: the encode reads it with no copy. This waits until + /// the slot's previous encode is done with the image. /// - /// For an RGB source the input image belongs to the conversion: hand the - /// RGB frames to [`Self::submit`] instead. - pub fn input_image(&self) -> vk::Image { - self.inner.input_image() - } - - /// Encode a frame from a GPU image in `GENERAL` layout, and return once - /// the image can be written again. - /// - /// Like `submit(Frame::image(src_image, vk::ImageLayout::GENERAL))`, but - /// for an image other than [`Self::input_image`] it also waits on the CPU - /// for [`EncodeFuture::source_released`]: the copy or conversion out of - /// it. That keeps the simple loop of writing one image and encoding it - /// correct. [`Self::submit`] does not wait, for callers that order their - /// writes on the GPU instead. + /// The input image holds what the encode reads: YUV, or RGB where the + /// encoder converts ([`Conversion::Encoder`]). With + /// [`Conversion::Shader`] it belongs to the colour converter, and this is + /// an error: submit the RGB frames from a [`SourceImage`] instead. /// /// ```no_run - /// use pixelforge::{InputImage, Encoder, EncodeConfig, EncodeBitDepth, PixelFormat, VideoContext, Codec}; + /// use pixelforge::{Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, VideoContext}; /// /// # fn example(context: VideoContext) -> Result<(), Box> { - /// let config = EncodeConfig::h264(1920, 1080); - /// let mut encoder = Encoder::new(context.clone(), config)?; - /// let mut input = InputImage::new( + /// let mut encoder = Encoder::new(context.clone(), EncodeConfig::h264(1920, 1080))?; + /// let mut upload = InputImage::new( /// context, /// Codec::H264, /// 1920, @@ -162,41 +154,26 @@ impl Encoder { /// EncodeBitDepth::Eight, /// PixelFormat::Yuv420, /// )?; - /// - /// // Upload YUV420 data to the input image /// # let yuv_data = vec![0u8; 1920 * 1080 * 3 / 2]; - /// input.upload_yuv420(&yuv_data)?; - /// - /// // Submit the frame and await its packet. - /// let future = encoder.encode(input.image())?; - /// let packet = pollster::block_on(future)?; - /// // ... use packet.data ... + /// let slot = encoder.input_slot()?; + /// upload.upload_yuv420_to(&slot, &yuv_data)?; + /// let packet = pollster::block_on(encoder.submit(Frame::input(slot))?)?; /// # Ok(()) /// # } /// ``` - pub fn encode(&mut self, src_image: vk::Image) -> Result { - // Read before submitting, since submitting moves to the next slot. - let in_place = - self.conversion != Conversion::Shader && src_image == self.inner.input_image(); - let future = self.submit(Frame::image(src_image, vk::ImageLayout::GENERAL))?; - if !in_place { - self.context.wait_point(future.source_released())?; + pub fn input_slot(&self) -> Result { + if self.conversion == Conversion::Shader { + return Err(PixelForgeError::InvalidInput( + "the input image holds the colour converter's YUV; submit the RGB frame \ + from a SourceImage instead" + .to_string(), + )); } - Ok(future) - } - - /// Encode a frame once GPU work the caller submitted has finished. - /// - /// Short for `submit(Frame::image(src_image, src_layout).wait(wait))`. It - /// does not wait for the source to be read: write it again only after - /// [`EncodeFuture::source_released`]. - pub fn encode_after( - &mut self, - src_image: vk::Image, - src_layout: vk::ImageLayout, - wait: &[TimelinePoint], - ) -> Result { - self.submit(Frame::image(src_image, src_layout).wait(wait)) + Ok(InputSlot { + image: self.inner.input_image(), + encoder: self.id, + ticket: self.ticket, + }) } /// Encode one frame. @@ -208,9 +185,21 @@ impl Encoder { /// the encoder's input image, by the colour converter, or by the encode /// itself, depending on the [`Conversion`] and the source. /// - /// Because the call returns before the source is read, write the source - /// again only after [`EncodeFuture::source_released`]. + /// The call returns before the source is read. A [`SourceImage`] or an + /// `ExternalImage` keeps track of that itself, see + /// [`SourceImage::released`]. pub fn submit(&mut self, frame: Frame<'_>) -> Result { + if let FrameSource::Input(slot) = &frame.source + && (slot.encoder, slot.ticket) != (self.id, self.ticket) + { + return Err(PixelForgeError::InvalidInput( + "this input slot is from before the encoder's last submission; \ + take a new one with input_slot()" + .to_string(), + )); + } + self.ticket += 1; + #[cfg_attr(not(feature = "dmabuf"), allow(unused_mut))] let mut sync = FrameSync { waits: frame.waits.iter().map(TimelinePoint::wait_info).collect(), @@ -238,13 +227,10 @@ impl Encoder { }; let mut future = match frame.source { - FrameSource::Image { image, layout } => { - self.submit_image(image, SourceOwnership::caller_image(layout), sync)? - } + FrameSource::Input(_) => self.inner.encode_frame(FrameInput::InPlace, sync)?, + FrameSource::Image(source) => self.submit_source(source, sync)?, #[cfg(feature = "dmabuf")] FrameSource::External(external) => self.submit_external(external, sync)?, - #[cfg(not(feature = "dmabuf"))] - FrameSource::Never(_) => unreachable!(), }; future.pts_offset = self.pts_offset; @@ -257,6 +243,24 @@ impl Encoder { Ok(future) } + /// Encode from a caller's [`SourceImage`]. + fn submit_source(&mut self, source: &SourceImage, mut sync: FrameSync) -> Result { + if source.image() == self.inner.input_image() { + return Err(PixelForgeError::InvalidInput( + "the encoder's input image is submitted with Frame::input".to_string(), + )); + } + sync.waits.extend( + source + .pending_release() + .iter() + .map(TimelinePoint::wait_info), + ); + let future = self.submit_image(source.image(), source.ownership(), sync)?; + source.mark_used(future.source_released()); + Ok(future) + } + /// Encode from an image the caller owns. pub(crate) fn submit_image( &mut self, @@ -280,12 +284,6 @@ impl Encoder { sync: FrameSync, ) -> Result { let target = self.inner.input_image(); - if image == target { - return Err(PixelForgeError::InvalidInput( - "the encoder's input image holds converted YUV; hand it the RGB frame instead" - .to_string(), - )); - } let converter = self .converter .as_mut() @@ -431,6 +429,8 @@ impl Encoder { let mut next = Self::new(self.context.clone(), config)?; next.pts_offset = self.pts_offset + encoded; next.id = self.id; + // A slot taken before is not a slot of the new encoder. + next.ticket = self.ticket + 1; next.generation = if next.conversion == self.conversion { self.generation } else { diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index adafd2c5..60da4a54 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -144,11 +144,6 @@ impl EncoderCommon { self.source_released = None; self.pending_signals.clear(); match input { - FrameInput::Copy { image, .. } if image == slot_image => { - self.source = slot_binding; - self.pending_waits = sync.waits; - self.pending_signals = sync.release_signals; - } FrameInput::InPlace => { self.source = slot_binding; self.pending_waits = sync.waits; diff --git a/src/encoder/external.rs b/src/encoder/external.rs index c5bbc395..04f9193b 100644 --- a/src/encoder/external.rs +++ b/src/encoder/external.rs @@ -14,14 +14,12 @@ //! sync_files, see [`Frame::wait_sync_file`](crate::Frame::wait_sync_file). use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd}; -use std::sync::Mutex; -use std::sync::atomic::{AtomicBool, Ordering}; use ash::vk; use ash::vk::TaggedStructure; use super::api::Encoder; -use super::input::{FrameInput, FrameSync, SourceOwnership}; +use super::input::{FrameInput, FrameSync, SourceOwnership, Uses}; use super::source::{Conversion, Source}; use super::{EncodeFuture, PixelFormat}; use crate::error::{PixelForgeError, Result}; @@ -191,10 +189,8 @@ pub struct ExternalImage { plane_count: u32, /// The encoder it belongs to, and its conversion's generation. owner: (u64, u64), - /// Whether no frame has read it yet, so its layout is still undefined. - fresh: AtomicBool, - /// When the last frame that read it is done with it. - last_use: Mutex>, + /// The frames that read it. + uses: Uses, } impl std::fmt::Debug for ExternalImage { @@ -214,6 +210,14 @@ impl ExternalImage { self.access != Access::Copy } + /// The points after which every frame submitted so far is done with the + /// buffer, so its producer can write it again. Empty before the first + /// frame. For a producer that takes a sync_file instead, see + /// [`Frame::signal_sync_file`](crate::Frame::signal_sync_file). + pub fn released(&self) -> Vec { + self.uses.points() + } + /// The DRM format modifier. pub fn modifier(&self) -> u64 { self.modifier @@ -268,31 +272,21 @@ impl ExternalImage { fn ownership(&self) -> SourceOwnership { let foreign = self.context.external().foreign_family(); SourceOwnership { - old_layout: if self.fresh.load(Ordering::Acquire) { - vk::ImageLayout::UNDEFINED - } else { + old_layout: if self.uses.was_read() { vk::ImageLayout::GENERAL + } else { + vk::ImageLayout::UNDEFINED }, acquire_from: Some(foreign), release_to: Some(foreign), + final_layout: vk::ImageLayout::GENERAL, } } - - fn last_use(&self) -> Option { - *self.last_use.lock().unwrap_or_else(|p| p.into_inner()) - } - - fn mark_used(&self, released: TimelinePoint) { - self.fresh.store(false, Ordering::Release); - *self.last_use.lock().unwrap_or_else(|p| p.into_inner()) = Some(released); - } } impl Drop for ExternalImage { fn drop(&mut self) { - if let Some(point) = self.last_use() { - let _ = self.context.wait_point(point); - } + let _ = self.uses.wait(&self.context); let device = self.context.device(); unsafe { if let Some(view) = self.view { @@ -798,8 +792,7 @@ impl Encoder { modifier, plane_count, owner: (self.id(), self.generation()), - fresh: AtomicBool::new(true), - last_use: Mutex::new(None), + uses: Uses::default(), }) } @@ -909,9 +902,8 @@ impl Encoder { } // The previous frame's release has to come before this frame's // acquire, even when both run on the same queue. - if let Some(point) = external.last_use() { - sync.waits.push(point.wait_info()); - } + sync.waits + .extend(external.uses.points().iter().map(TimelinePoint::wait_info)); let ownership = external.ownership(); let future = match external.access { Access::Encode => self.encode_input( @@ -932,7 +924,7 @@ impl Encoder { )?, Access::Sample => self.convert_and_encode(external.image, ownership, sync)?, }; - external.mark_used(future.source_released()); + external.uses.record(future.source_released()); Ok(future) } } diff --git a/src/encoder/frame.rs b/src/encoder/frame.rs index 1f1e9d2f..8e79da7a 100644 --- a/src/encoder/frame.rs +++ b/src/encoder/frame.rs @@ -2,33 +2,58 @@ use ash::vk; +use super::source_image::SourceImage; use crate::sync::TimelinePoint; /// Where a frame's pixels are. pub(crate) enum FrameSource<'a> { - /// A `vk::Image` the caller owns, in the given layout. - Image { - image: vk::Image, - layout: vk::ImageLayout, - }, + /// The encoder's own input image for the current slot. + Input(InputSlot), + /// A `vk::Image` of the caller's. + Image(&'a SourceImage), /// A DMA-BUF pixelforge imported or allocated. #[cfg(feature = "dmabuf")] External(&'a crate::encoder::external::ExternalImage), - #[cfg(not(feature = "dmabuf"))] - #[allow(dead_code)] - Never(std::marker::PhantomData<&'a ()>), +} + +/// The encoder's input image for the next frame, from +/// [`Encoder::input_slot`](crate::Encoder::input_slot). +/// +/// Write the frame into [`Self::image`], then submit the slot with +/// [`Frame::input`]: the encoder reads it in place, with no copy. The slot is +/// valid until the encoder's next submission; submitting it later is an +/// error. +#[derive(Debug)] +#[must_use] +pub struct InputSlot { + pub(crate) image: vk::Image, + pub(crate) encoder: u64, + pub(crate) ticket: u64, +} + +impl InputSlot { + /// The image to write the frame into. It is in `VIDEO_ENCODE_SRC_KHR` + /// layout, and must be in that layout when the slot is submitted. + pub fn image(&self) -> vk::Image { + self.image + } } /// One frame to encode: its source, and what has to happen on the GPU before /// it can be read. /// /// ```no_run -/// # use pixelforge::{Encoder, Frame, TimelinePoint}; +/// # use pixelforge::{Encoder, Frame, Handoff, SourceImage, TimelinePoint, VideoContext}; /// # use ash::vk; -/// # fn f(encoder: &mut Encoder, image: vk::Image, rendered: TimelinePoint) -> Result<(), pixelforge::PixelForgeError> { -/// let packet = encoder.submit(Frame::image(image, vk::ImageLayout::GENERAL).wait(&[rendered]))?; -/// // Write `image` again only after this point. -/// let reusable = packet.source_released(); +/// # fn f(context: &VideoContext, encoder: &mut Encoder, image: vk::Image, rendered: TimelinePoint) -> Result<(), pixelforge::PixelForgeError> { +/// let source = SourceImage::new( +/// context, +/// image, +/// Handoff::Device { layout: vk::ImageLayout::GENERAL }, +/// )?; +/// let packet = encoder.submit(Frame::image(&source).wait(&[rendered]))?; +/// // Render the next frame into `image` only after these points. +/// let reusable = source.released(); /// # Ok(()) /// # } /// ``` @@ -43,27 +68,19 @@ pub struct Frame<'a> { } impl<'a> Frame<'a> { - /// A frame in `image`, a `vk::Image` the caller owns, currently in - /// `layout`. - /// - /// The image must be in the format the encoder's - /// [`Source`](crate::encoder::Source) says and have the encoder's - /// dimensions. Pixelforge reads it with a transfer, or with the colour - /// converter's shader, so it needs `TRANSFER_SRC` or `SAMPLED` usage to - /// match. - /// - /// `UNDEFINED` means a first-time external-memory import, which is - /// acquired from `VK_QUEUE_FAMILY_EXTERNAL`. The image is left in - /// `GENERAL`, owned by the queue family that read it. - /// - /// [`Encoder::input_image`](crate::Encoder::input_image) is read in place, - /// with no copy, for a YUV source. + /// A frame in the encoder's own input image, written in place. + pub fn input(slot: InputSlot) -> Self { + Self::new(FrameSource::Input(slot)) + } + + /// A frame in `source`, an image of the caller's. /// - /// The image is read after [`Encoder::submit`](crate::Encoder::submit) - /// returns: keep it alive, and do not write it, until - /// [`EncodeFuture::source_released`](crate::EncodeFuture::source_released). - pub fn image(image: vk::Image, layout: vk::ImageLayout) -> Self { - Self::new(FrameSource::Image { image, layout }) + /// Pixelforge copies it into the encoder's input image, or converts it + /// there for an RGB source. That happens after + /// [`Encoder::submit`](crate::Encoder::submit) returns, so write the image + /// again only after [`SourceImage::released`]. + pub fn image(source: &'a SourceImage) -> Self { + Self::new(FrameSource::Image(source)) } /// A frame in a DMA-BUF that the encoder imported or allocated, see diff --git a/src/encoder/input.rs b/src/encoder/input.rs index 5f1c3b5f..4cee07e4 100644 --- a/src/encoder/input.rs +++ b/src/encoder/input.rs @@ -6,8 +6,15 @@ //! reads directly. The submission that reads a source also does its queue //! family ownership transfers, which [`SourceOwnership`] describes. +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; + use ash::vk; +use crate::error::Result; +use crate::sync::TimelinePoint; +use crate::vulkan::VideoContext; + /// The layout a source is in, and the queue family transfers around the one /// submission that reads it. #[derive(Debug, Clone, Copy)] @@ -20,21 +27,11 @@ pub(crate) struct SourceOwnership { pub acquire_from: Option, /// Queue family to release the source to once it has been read. pub release_to: Option, + /// Layout the source is left in once it has been read. + pub final_layout: vk::ImageLayout, } impl SourceOwnership { - /// A caller's image in `layout`. `UNDEFINED` means a first-time - /// external-memory import, which is acquired from `EXTERNAL` and then - /// stays with the queue that read it. - pub(crate) fn caller_image(layout: vk::ImageLayout) -> Self { - Self { - old_layout: layout, - acquire_from: (layout == vk::ImageLayout::UNDEFINED) - .then_some(vk::QUEUE_FAMILY_EXTERNAL), - release_to: None, - } - } - /// The barrier that brings the source into `new_layout` for `family`, /// acquiring it first when another family owns it. pub(crate) fn acquire_barrier( @@ -72,8 +69,8 @@ impl SourceOwnership { .subresource_range(color_range()) } - /// The barrier that leaves the source in `GENERAL` after it was read in - /// `layout`, releasing it when another family takes it back. + /// The barrier that leaves the source in its final layout after it was + /// read in `layout`, releasing it when another family takes it back. pub(crate) fn release_barrier( &self, image: vk::Image, @@ -102,7 +99,7 @@ impl SourceOwnership { .dst_stage_mask(dst_stage) .dst_access_mask(dst_access) .old_layout(layout) - .new_layout(vk::ImageLayout::GENERAL) + .new_layout(self.final_layout) .src_queue_family_index(src_family) .dst_queue_family_index(dst_family) .image(image) @@ -115,7 +112,7 @@ impl SourceOwnership { pub(crate) enum FrameInput { /// The current slot's input image, already written. InPlace, - /// Copy `image` into the slot's input image first. + /// Copy `image`, which is not the slot's input image, into it first. Copy { image: vk::Image, ownership: SourceOwnership, @@ -150,3 +147,58 @@ fn color_range() -> vk::ImageSubresourceRange { layer_count: 1, } } + +/// When the frames that read a source are done with it. +/// +/// One point per timeline semaphore, so a source read by two encoders keeps +/// the last read of each. +#[derive(Debug)] +pub(crate) struct Uses { + read: AtomicBool, + points: Mutex>, +} + +impl Default for Uses { + fn default() -> Self { + Self { + read: AtomicBool::new(false), + points: Mutex::new(Vec::new()), + } + } +} + +impl Uses { + /// Whether a frame has read the source yet. + pub(crate) fn was_read(&self) -> bool { + self.read.load(Ordering::Acquire) + } + + /// Note a frame that is done with the source at `released`. + pub(crate) fn record(&self, released: TimelinePoint) { + self.read.store(true, Ordering::Release); + let mut points = self.points.lock().unwrap_or_else(|p| p.into_inner()); + match points + .iter_mut() + .find(|p| p.semaphore == released.semaphore) + { + Some(point) => point.value = point.value.max(released.value), + None => points.push(released), + } + } + + /// The points after which every frame so far is done with the source. + pub(crate) fn points(&self) -> Vec { + self.points + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone() + } + + /// Wait on the CPU until every frame so far is done with the source. + pub(crate) fn wait(&self, context: &VideoContext) -> Result<()> { + for point in self.points() { + context.wait_point(point)?; + } + Ok(()) + } +} diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index ef323e7d..001b5f32 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -23,6 +23,7 @@ pub mod reorder; pub mod resources; pub(crate) mod rgb; mod source; +mod source_image; pub(crate) mod support; use ash::vk; @@ -572,9 +573,10 @@ impl EncodeConfig { } pub use api::Encoder; -pub use frame::Frame; +pub use frame::{Frame, InputSlot}; pub use pipeline::EncodeFuture; pub use source::{Conversion, ConversionPreference, RgbSource, Source}; +pub use source_image::{Handoff, SourceImage}; /// Statistic about the encoded video packet. #[derive(Debug, Clone)] diff --git a/src/encoder/pipeline.rs b/src/encoder/pipeline.rs index 4551a6f0..f375e1f9 100644 --- a/src/encoder/pipeline.rs +++ b/src/encoder/pipeline.rs @@ -54,7 +54,7 @@ use crate::vulkan::VideoContext; /// A handle to the packet a single `EncodePipeline::submit_current` will /// eventually produce. /// -/// Returned by `Encoder::encode`, one per submitted frame. Awaiting it yields +/// Returned by `Encoder::submit`, one per submitted frame. Awaiting it yields /// that frame's [`EncodedPacket`] once the GPU finishes encoding and the /// completion thread reads the bitstream back. Futures resolve in submission /// order (one submission → one readback → one packet) as long as you await them @@ -73,24 +73,16 @@ pub struct EncodeFuture { } impl EncodeFuture { - /// The point after which this frame's source may be written again, or - /// handed back to whatever produced it. - /// - /// Which submission reaches it depends on how the source was read: the + /// The point after which this frame's source may be written again: the /// copy into the encoder's input image, the colour conversion, or the - /// encode itself for a source read directly. It is usually long before the - /// packet is ready. Waiting on it on the GPU, before the next write to the - /// source, is enough. - /// - /// The semaphore stays valid for as long as the encoder's - /// [`VideoContext`] does. - pub fn source_released(&self) -> TimelinePoint { + /// encode itself for a source read directly. + pub(crate) fn source_released(&self) -> TimelinePoint { self.released } /// The sync_file asked for with /// [`Frame::signal_sync_file`](crate::Frame::signal_sync_file): it - /// signals at [`Self::source_released`]. `None` if it was not asked for, + /// signals once the frame is done with its source. `None` if it was not asked for, /// was already taken, or had already signalled when it was made, which /// the driver may report that way. #[cfg(feature = "dmabuf")] diff --git a/src/encoder/source_image.rs b/src/encoder/source_image.rs new file mode 100644 index 00000000..cb204f09 --- /dev/null +++ b/src/encoder/source_image.rs @@ -0,0 +1,151 @@ +//! A `vk::Image` of the caller's that frames are encoded from. + +use ash::vk; + +use super::input::{SourceOwnership, Uses}; +use crate::error::{PixelForgeError, Result}; +use crate::sync::TimelinePoint; +use crate::vulkan::VideoContext; + +/// How the caller hands a [`SourceImage`] to pixelforge for each frame, and +/// how pixelforge gives it back. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Handoff { + /// The caller writes the image on this device, and hands it over in + /// `layout`. Pixelforge gives it back in `layout`. + /// + /// Pixelforge reads it on its own queues with no queue family transfer, + /// so make the image with `CONCURRENT` sharing across the caller's queue + /// families and pixelforge's, or write it on the queue family pixelforge + /// reads it on. + Device { + /// The layout the image is in when a frame is submitted, and is left + /// in once the frame is done with it. Not `UNDEFINED` or + /// `PREINITIALIZED`, which do not keep the pixels. + layout: vk::ImageLayout, + }, + /// The image's memory is imported, and written outside this device: by + /// another process, another API or another Vulkan instance. + /// + /// Each frame acquires the image from `VK_QUEUE_FAMILY_FOREIGN_EXT` (or + /// `VK_QUEUE_FAMILY_EXTERNAL` without `VK_EXT_queue_family_foreign`), and + /// releases it back in `GENERAL`. + External, +} + +/// A `vk::Image` of the caller's that frames are encoded from, see +/// [`Frame::image`](crate::Frame::image). +/// +/// It keeps track of the frames that read the image. Write the image again +/// only after [`Self::released`]: wait for those points on the GPU, or call +/// [`Self::wait_released`]. Dropping it waits for them too, so drop it before +/// the image is destroyed. +/// +/// The image must be in the format and the size the encoder's +/// [`Source`](crate::Source) says. Pixelforge reads it with a transfer, or +/// with the colour converter's shader for an RGB source, so it needs +/// `TRANSFER_SRC` or `SAMPLED` usage to match. +pub struct SourceImage { + context: VideoContext, + image: vk::Image, + handoff: Handoff, + uses: Uses, +} + +impl std::fmt::Debug for SourceImage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SourceImage") + .field("image", &self.image) + .field("handoff", &self.handoff) + .finish() + } +} + +impl SourceImage { + /// Encode frames from `image`, handed over as `handoff` says. + pub fn new(context: &VideoContext, image: vk::Image, handoff: Handoff) -> Result { + if let Handoff::Device { layout } = handoff + && matches!( + layout, + vk::ImageLayout::UNDEFINED | vk::ImageLayout::PREINITIALIZED + ) + { + return Err(PixelForgeError::InvalidInput(format!( + "a source image handed over in {layout:?} has no pixels to read" + ))); + } + Ok(Self { + context: context.clone(), + image, + handoff, + uses: Uses::default(), + }) + } + + /// The image. + pub fn image(&self) -> vk::Image { + self.image + } + + /// How the image is handed over. + pub fn handoff(&self) -> Handoff { + self.handoff + } + + /// The points after which every frame submitted so far is done with the + /// image, so it can be written again. Empty before the first frame. + pub fn released(&self) -> Vec { + self.uses.points() + } + + /// Wait on the CPU until every frame submitted so far is done with the + /// image. + pub fn wait_released(&self) -> Result<()> { + self.uses.wait(&self.context) + } + + /// The layout and queue family transfers for the next frame that reads + /// the image. + pub(crate) fn ownership(&self) -> SourceOwnership { + match self.handoff { + Handoff::Device { layout } => SourceOwnership { + old_layout: layout, + acquire_from: None, + release_to: None, + final_layout: layout, + }, + Handoff::External => { + let foreign = self.context.external().foreign_family(); + SourceOwnership { + old_layout: if self.uses.was_read() { + vk::ImageLayout::GENERAL + } else { + vk::ImageLayout::UNDEFINED + }, + acquire_from: Some(foreign), + release_to: Some(foreign), + final_layout: vk::ImageLayout::GENERAL, + } + } + } + } + + /// What the next frame has to wait for before it reads the image: the + /// previous release, for an image that goes back to another owner. + pub(crate) fn pending_release(&self) -> Vec { + match self.handoff { + Handoff::Device { .. } => Vec::new(), + Handoff::External => self.uses.points(), + } + } + + pub(crate) fn mark_used(&self, released: TimelinePoint) { + self.uses.record(released); + } +} + +impl Drop for SourceImage { + fn drop(&mut self) { + let _ = self.uses.wait(&self.context); + } +} diff --git a/src/image.rs b/src/image.rs index 23be5084..769ad192 100644 --- a/src/image.rs +++ b/src/image.rs @@ -3,13 +3,13 @@ //! This module provides [`InputImage`], a helper for uploading YUV data from the CPU //! to the GPU. The image it creates is a transfer-only staging image (not a Vulkan Video //! image) and must be copied into an encoder's input image before encoding. -//! Use [`InputImage::upload_yuv420`] to upload to the internal image, then pass -//! `input_image.image()` to [`Encoder::encode`](crate::encoder::Encoder::encode). +//! Use [`InputImage::upload_yuv420`] to upload to the internal image, then submit +//! [`InputImage::source`] with [`Frame::image`](crate::Frame::image). //! Alternatively, use [`InputImage::upload_yuv420_to`] / [`InputImage::upload_yuv444_to`] -//! to upload directly into the encoder's input image (obtained via -//! [`Encoder::input_image`](crate::encoder::Encoder::input_image)). +//! to upload directly into an encoder's [`InputSlot`] (from +//! [`Encoder::input_slot`](crate::encoder::Encoder::input_slot)). -use crate::encoder::{BitDepth, Codec, PixelFormat}; +use crate::encoder::{BitDepth, Codec, Handoff, InputSlot, PixelFormat, SourceImage}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; use ash::vk; @@ -23,12 +23,15 @@ use ash::vk; /// /// The image is **not** directly usable as a Vulkan Video encode source. To /// encode, either: -/// - Upload to this image and pass `self.image()` to [`Encoder::encode`](crate::encoder::Encoder::encode), which -/// will copy it into the encoder's internal input image, or +/// - Upload to this image and submit [`Self::source`] with +/// [`Frame::image`](crate::Frame::image), which copies it into the encoder's +/// input image, or /// - Use [`upload_yuv420_to`](Self::upload_yuv420_to) / /// [`upload_yuv444_to`](Self::upload_yuv444_to) to upload directly into the -/// encoder's input image. +/// encoder's input image, through an [`InputSlot`]. pub struct InputImage { + /// Keeps track of the frames that read `image`. + source: SourceImage, context: VideoContext, image: vk::Image, /// Current layout of `self.image`. @@ -204,7 +207,15 @@ impl InputImage { let fence = unsafe { device.create_fence(&fence_info, None) } .map_err(|e| PixelForgeError::ResourceCreation(format!("fence creation: {}", e)))?; - Ok(Self { + let source = SourceImage::new( + &context, + image, + Handoff::Device { + layout: vk::ImageLayout::GENERAL, + }, + )?; + let mut input = Self { + source, context, image, image_layout: vk::ImageLayout::UNDEFINED, @@ -219,7 +230,24 @@ impl InputImage { height, bit_depth, pixel_format, - }) + }; + // Start from zeroed planes in GENERAL, so the image is a valid source + // before the first upload. + unsafe { + let device = input.context.device(); + let ptr = device + .map_memory( + staging_memory, + 0, + staging_size as vk::DeviceSize, + vk::MemoryMapFlags::empty(), + ) + .map_err(|e| PixelForgeError::MemoryAllocation(e.to_string()))?; + std::ptr::write_bytes(ptr as *mut u8, 0, staging_size); + device.unmap_memory(staging_memory); + } + input.copy_staging_to_image()?; + Ok(input) } /// Upload YUV420 (I420) data to this image. @@ -435,10 +463,11 @@ impl InputImage { /// Upload YUV420 (I420) data to an external image. /// - /// Same as `upload_yuv420()` but copies the data to the specified target image - /// instead of this InputImage's own image. Useful for uploading directly to an - /// encoder's input image to avoid cross-queue copy issues. - pub fn upload_yuv420_to(&mut self, target_image: vk::Image, yuv_data: &[u8]) -> Result<()> { + /// Same as `upload_yuv420()` but copies the data into an encoder's input + /// image instead of this InputImage's own image, so the encode reads it + /// with no further copy. Submit the slot with + /// [`Frame::input`](crate::Frame::input) afterwards. + pub fn upload_yuv420_to(&mut self, slot: &InputSlot, yuv_data: &[u8]) -> Result<()> { if self.pixel_format != PixelFormat::Yuv420 { return Err(PixelForgeError::InvalidInput( "upload_yuv420_to can only be used with YUV420 InputImage".to_string(), @@ -506,7 +535,7 @@ impl InputImage { // Record and submit copy commands to the target image. self.copy_staging_to_image_internal( - target_image, + slot.image(), vk::ImageLayout::UNDEFINED, vk::ImageLayout::VIDEO_ENCODE_SRC_KHR, )?; @@ -514,8 +543,9 @@ impl InputImage { Ok(()) } - /// Upload YUV444 (planar) data to an external image. - pub fn upload_yuv444_to(&mut self, target_image: vk::Image, yuv_data: &[u8]) -> Result<()> { + /// Upload YUV444 (planar) data into an encoder's input image, like + /// [`Self::upload_yuv420_to`]. + pub fn upload_yuv444_to(&mut self, slot: &InputSlot, yuv_data: &[u8]) -> Result<()> { if self.pixel_format != PixelFormat::Yuv444 { return Err(PixelForgeError::InvalidInput( "upload_yuv444_to can only be used with YUV444 InputImage".to_string(), @@ -594,7 +624,7 @@ impl InputImage { // Record and submit copy commands. self.copy_staging_to_image_internal( - target_image, + slot.image(), vk::ImageLayout::UNDEFINED, vk::ImageLayout::VIDEO_ENCODE_SRC_KHR, )?; @@ -603,6 +633,8 @@ impl InputImage { } fn copy_staging_to_image(&mut self) -> Result<()> { + // A frame submitted before may still be reading the image. + self.source.wait_released()?; let old_layout = self.image_layout; self.copy_staging_to_image_internal(self.image, old_layout, vk::ImageLayout::GENERAL)?; self.image_layout = vk::ImageLayout::GENERAL; @@ -826,6 +858,13 @@ impl InputImage { self.image } + /// The image as a source to encode from, see + /// [`Frame::image`](crate::Frame::image). The uploads wait until the + /// frames that read it are done with it. + pub fn source(&self) -> &SourceImage { + &self.source + } + /// Get the image dimensions. pub fn dimensions(&self) -> (u32, u32) { (self.width, self.height) diff --git a/src/lib.rs b/src/lib.rs index e4d57f8f..0cf1f59d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -121,7 +121,7 @@ //! // For each frame: upload YUV data and encode. //! // let yuv_data: &[u8] = ...; // YUV420 frame data //! // input_image.upload_yuv420(yuv_data)?; -//! // let packets = encoder.encode(input_image.image())?; +//! // let packet = encoder.submit(Frame::image(input_image.source()))?; //! //! Ok(()) //! } @@ -256,13 +256,19 @@ //! //! ## Submitting frames //! -//! [`Encoder::submit`] takes a [`Frame`]: where the pixels are, and what has -//! to happen on the GPU before they can be read. It does not wait on the CPU, -//! except when every pipeline slot is in flight. The source is read by a copy, -//! the shader, or the encode itself, after the call returns, so write it again -//! only after [`EncodeFuture::source_released`]. [`Encoder::encode`] is the -//! short form that also waits for that point, which keeps the simple loop of -//! writing one image and encoding it correct. +//! [`Encoder::submit`] is the one way to encode a frame. It takes a [`Frame`]: +//! where the pixels are, and what has to happen on the GPU before they can be +//! read. It does not wait on the CPU, except when every pipeline slot is in +//! flight. The pixels are in one of: +//! +//! - an [`InputSlot`], the encoder's own input image, from +//! [`Encoder::input_slot`]. The encode reads it in place, with no copy. +//! - a [`SourceImage`], an image of the caller's. Pixelforge copies or +//! converts it into the input image after the call returns, and the +//! `SourceImage` keeps track of that: write the image again only after +//! [`SourceImage::released`]. [`InputImage`], for frames uploaded from the +//! CPU, waits for that itself. +//! - an `ExternalImage`, a DMA-BUF (see below). //! //! ## Encoding on your own device //! @@ -280,23 +286,24 @@ //! //! Each frame is then ordered on the GPU: the application's submission signals //! a [`TimelinePoint`], the encoder's first submission for the frame waits for -//! it, and the application waits for [`EncodeFuture::source_released`] before -//! it draws into the image again. +//! it, and the application waits for [`SourceImage::released`] before it draws +//! into the image again. //! //! ```rust,no_run //! # use ash::vk; -//! # use pixelforge::{Encoder, Frame, TimelinePoint}; +//! # use pixelforge::{Encoder, Frame, SourceImage, TimelinePoint}; //! # fn frame( //! # encoder: &mut Encoder, //! # rendered: vk::Semaphore, //! # frame_number: u64, -//! # image: vk::Image, +//! # source: &SourceImage, //! # ) -> Result<(), Box> { -//! // `rendered` reaches `frame_number` once the application has drawn `image`. +//! // `rendered` reaches `frame_number` once the application has drawn +//! // `source.image()`. //! let ready = [TimelinePoint::new(rendered, frame_number)]; -//! let packet = encoder.submit(Frame::image(image, vk::ImageLayout::GENERAL).wait(&ready))?; -//! // Wait on this before drawing into `image` again. -//! let reusable = packet.source_released(); +//! let packet = encoder.submit(Frame::image(source).wait(&ready))?; +//! // Wait on these before drawing into the image again. +//! let reusable = source.released(); //! # Ok(()) //! # } //! ``` @@ -402,7 +409,9 @@ pub use encoder::{ DEFAULT_TARGET_BITRATE, EncodeConfig, EncodeContentHint, EncodeFuture, EncodeUsageHint, EncodedPacket, Encoder, EncoderTuningMode, Frame, FrameType, PixelFormat, RateControlMode, }; -pub use encoder::{Conversion, ConversionPreference, RgbSource, Source}; +pub use encoder::{ + Conversion, ConversionPreference, Handoff, InputSlot, RgbSource, Source, SourceImage, +}; pub use error::PixelForgeError; pub use image::InputImage; pub use sync::TimelinePoint; diff --git a/src/sync.rs b/src/sync.rs index 66ca372f..709f5e27 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -5,13 +5,12 @@ use ash::vk; /// A point on a timeline semaphore: reached once the semaphore's value is at /// least `value`. /// -/// Returned by [`ColorConverter::convert_async`](crate::ColorConverter::convert_async) -/// for the work it submitted, and accepted by it and by -/// [`Encoder::encode_after`](crate::Encoder::encode_after) as work to wait -/// for. On a device shared with the caller this is how a frame moves from the -/// caller's rendering to the converter to the encoder with every step ordered -/// on the GPU: each submission waits on the point the previous one signals, -/// and nothing waits on the CPU. +/// Given by [`SourceImage::released`](crate::SourceImage::released) for the +/// frames that read an image, and accepted by +/// [`Frame::wait`](crate::Frame::wait) as work to wait for. On a device shared +/// with the caller this is how a frame moves from the caller's rendering to +/// the encoder and back with every step ordered on the GPU: each submission +/// waits on the point the previous one signals, and nothing waits on the CPU. /// /// The semaphore must be a timeline semaphore on the same device as the /// context. A point pixelforge hands out stays valid for as long as the diff --git a/src/vulkan.rs b/src/vulkan.rs index da5661a2..bb8ff5bb 100644 --- a/src/vulkan.rs +++ b/src/vulkan.rs @@ -776,7 +776,7 @@ impl VideoContext { /// /// For a semaphore whose owner is being dropped while a submission from /// elsewhere may still wait on it: the colour converter's timeline, which - /// an encode submitted with [`crate::Encoder::encode_after`] can still be + /// an encode that waits on it with [`crate::Frame::wait`] can still be /// waiting on when the converter goes. Every encoder holds the context and /// waits for its queues when dropped, so by the time the context goes, /// nothing of pixelforge's waits on these any more. diff --git a/tests/adopted.rs b/tests/adopted.rs index 0bb547ca..2e515e56 100644 --- a/tests/adopted.rs +++ b/tests/adopted.rs @@ -19,7 +19,7 @@ use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::encoder::Codec; use pixelforge::vulkan::{DeviceQueue, DeviceRequirements, VideoContext, VideoContextBuilder}; use pixelforge::{ - ColorConverter, ColorConverterConfig, ColorRange, ColorSpec, EncodeConfig, Encoder, + ColorConverter, ColorConverterConfig, ColorRange, ColorSpec, EncodeConfig, Encoder, Frame, InputFormat, OutputFormat, RateControlMode, TimelinePoint, }; @@ -292,11 +292,12 @@ fn encode_all( for n in 0..ENCODE_FRAMES { let src = unsafe { create_src_image(context, ENCODE_WIDTH, ENCODE_HEIGHT, &moving_frame(n))? }; - let target = encoder.input_image(); + let slot = encoder.input_slot()?; + let target = slot.image(); let future = match handoff { Handoff::Cpu => { converter.convert(src.image, vk::ImageLayout::GENERAL, target)?; - encoder.encode(target)? + encoder.submit(Frame::input(slot))? } Handoff::Gpu => { let ready = TimelinePoint::new(rendered, u64::from(n) + 1); @@ -322,7 +323,7 @@ fn encode_all( if n + 1 == ENCODE_FRAMES { waits.push(TimelinePoint::new(rendered, after_drop)); } - encoder.encode_after(target, vk::ImageLayout::GENERAL, &waits)? + encoder.submit(Frame::input(slot).wait(&waits))? } }; sources.push(src); diff --git a/tests/color_conversion.rs b/tests/color_conversion.rs index 8069164f..d823bc63 100644 --- a/tests/color_conversion.rs +++ b/tests/color_conversion.rs @@ -229,7 +229,11 @@ fn convert( ); let mut converter = ColorConverter::new(context.clone(), config)?; let description = converter.color_description(); - converter.convert(src.image, vk::ImageLayout::GENERAL, encoder.input_image())?; + converter.convert( + src.image, + vk::ImageLayout::GENERAL, + encoder.input_slot()?.image(), + )?; Ok((read_luma(context, &converter, output_format)?, description)) } @@ -443,7 +447,11 @@ fn a_replaced_source_image_is_read_afresh() -> Result<(), Box>(luma) diff --git a/tests/dmabuf.rs b/tests/dmabuf.rs index 7418af21..d5c323af 100644 --- a/tests/dmabuf.rs +++ b/tests/dmabuf.rs @@ -18,8 +18,8 @@ use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::encoder::external::drm_format; use pixelforge::{ Codec, ColorSpec, Conversion, ConversionPreference, DmaBuf, DmaBufPlane, EncodeBitDepth, - EncodeConfig, Encoder, ExternalImage, Frame, InputFormat, InputImage, PixelFormat, - RateControlMode, RgbSource, Source, VideoContext, VideoContextBuilder, + EncodeConfig, Encoder, ExternalImage, Frame, Handoff, InputFormat, InputImage, PixelFormat, + RateControlMode, RgbSource, Source, SourceImage, VideoContext, VideoContextBuilder, }; use std::os::fd::{FromRawFd, OwnedFd}; @@ -442,7 +442,7 @@ fn encode_reference( let mut stream = Vec::new(); for frame in frames { input.upload_nv12(frame)?; - stream.extend(pollster::block_on(encoder.encode(input.image())?)?.data); + stream.extend(pollster::block_on(encoder.submit(Frame::image(input.source()))?)?.data); } Ok(stream) } @@ -578,7 +578,14 @@ fn rgb_buffers_are_converted_directly() -> Result<(), Box for n in 0..FRAMES { let data = bgra_frame(w, h, n); let src = unsafe { common::source::create_src_image(&context, w, h, &data)? }; - reference.extend(pollster::block_on(reference_encoder.encode(src.image)?)?.data); + let image = SourceImage::new( + &context, + src.image, + Handoff::Device { + layout: vk::ImageLayout::GENERAL, + }, + )?; + reference.extend(pollster::block_on(reference_encoder.submit(Frame::image(&image))?)?.data); producer.write(&data)?; stream.extend(pollster::block_on(encoder.submit(Frame::external(&allocated))?)?.data); } diff --git a/tests/rfi.rs b/tests/rfi.rs index b873f750..887cf0d4 100644 --- a/tests/rfi.rs +++ b/tests/rfi.rs @@ -20,7 +20,7 @@ mod common; use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, InputImage, PixelFormat, RateControlMode, + Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControlMode, VideoContextBuilder, }; use std::collections::VecDeque; @@ -155,9 +155,9 @@ fn run_codec( encoder.invalidate_reference_frames(first_lost); } - let encoder_image = encoder.input_image(); - input_image.upload_yuv420_to(encoder_image, &yuv_data[start..end])?; - pending.push_back(encoder.encode(encoder_image)?); + let slot = encoder.input_slot()?; + input_image.upload_yuv420_to(&slot, &yuv_data[start..end])?; + pending.push_back(encoder.submit(Frame::input(slot))?); while pending.len() > 2 { drain_one(&mut pending, &mut output_file, &mut recovery_is_key)?; } diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs index d77b5943..4660e93d 100644 --- a/tests/rgb_encode.rs +++ b/tests/rgb_encode.rs @@ -17,8 +17,9 @@ use ash::vk; use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::{ Codec, ColorConverter, ColorConverterConfig, ColorDescription, ColorRange, ColorSpec, - Conversion, ConversionPreference, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputFormat, - OutputFormat, RateControlMode, RgbSource, Source, VideoContext, VideoContextBuilder, + Conversion, ConversionPreference, EncodeBitDepth, EncodeConfig, Encoder, Frame, Handoff, + InputFormat, OutputFormat, RateControlMode, RgbSource, Source, SourceImage, VideoContext, + VideoContextBuilder, }; const WIDTH: u32 = 320; @@ -84,6 +85,20 @@ fn rgb(format: InputFormat) -> Source { Source::Rgb(RgbSource::new(format, ColorSpec::Srgb)) } +/// `src` as a source to encode from, handed over in `GENERAL`. +fn source_image( + context: &VideoContext, + src: &SrcImage, +) -> Result { + SourceImage::new( + context, + src.image, + Handoff::Device { + layout: vk::ImageLayout::GENERAL, + }, + ) +} + /// Encode the frames with a YUV encoder and a colour converter the test runs /// itself. fn encode_by_hand( @@ -95,8 +110,9 @@ fn encode_by_hand( let mut stream = Vec::new(); for n in 0..FRAMES { let src = source(context, n)?; - converter.convert(src.image, vk::ImageLayout::GENERAL, encoder.input_image())?; - let packet = pollster::block_on(encoder.encode(encoder.input_image())?)?; + let slot = encoder.input_slot()?; + converter.convert(src.image, vk::ImageLayout::GENERAL, slot.image())?; + let packet = pollster::block_on(encoder.submit(Frame::input(slot))?)?; stream.extend_from_slice(&packet.data); } Ok(stream) @@ -118,7 +134,8 @@ fn encode_rgb( let mut stream = Vec::new(); for n in 0..FRAMES { let src = source(context, n)?; - let packet = pollster::block_on(encoder.encode(src.image)?)?; + let image = source_image(context, &src)?; + let packet = pollster::block_on(encoder.submit(Frame::image(&image))?)?; stream.extend_from_slice(&packet.data); } Ok(stream) @@ -346,8 +363,8 @@ fn color_change_follows_into_the_conversion() -> Result<(), Box Result<(), Box> { let mut pending: VecDeque = VecDeque::new(); for i in 0..frame_count { input_image.upload_yuv420(&yuv[i * frame_size..(i + 1) * frame_size])?; - pending.push_back(encoder.encode(input_image.image())?); + pending.push_back(encoder.submit(Frame::image(input_image.source()))?); while pending.len() > 2 { let packet = pollster::block_on(pending.pop_front().expect("non-empty"))?; bitstream.extend_from_slice(&packet.data); diff --git a/tests/submit.rs b/tests/submit.rs new file mode 100644 index 00000000..2c6f6795 --- /dev/null +++ b/tests/submit.rs @@ -0,0 +1,94 @@ +//! The rules `Encoder::submit` holds a caller to. +//! +//! Ignored by default: requires a Vulkan Video device. Run with +//! `cargo test -- --ignored`. + +#[allow(dead_code)] +mod common; + +use ash::vk; +use pixelforge::{ + Codec, ColorSpec, ConversionPreference, EncodeBitDepth, EncodeConfig, Encoder, Frame, Handoff, + InputFormat, InputImage, PixelFormat, RgbSource, Source, SourceImage, VideoContext, + VideoContextBuilder, +}; + +const WIDTH: u32 = 320; +const HEIGHT: u32 = 240; + +fn context() -> Result> { + common::init_logging(); + Ok(VideoContextBuilder::new() + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .require_encode(Codec::H264) + .build()?) +} + +#[test] +#[ignore = "requires a Vulkan Video device"] +fn an_input_slot_is_valid_until_the_next_submission() -> Result<(), Box> { + let context = context()?; + let mut encoder = Encoder::new(context.clone(), EncodeConfig::h264(WIDTH, HEIGHT))?; + let input = InputImage::new( + context.clone(), + Codec::H264, + WIDTH, + HEIGHT, + EncodeBitDepth::Eight, + PixelFormat::Yuv420, + )?; + + let stale = encoder.input_slot()?; + let packet = encoder.submit(Frame::image(input.source()))?; + assert!( + encoder.submit(Frame::input(stale)).is_err(), + "a slot from before a submission was accepted" + ); + pollster::block_on(packet)?; + + let slot = encoder.input_slot()?; + assert!( + !pollster::block_on(encoder.submit(Frame::input(slot))?)? + .data + .is_empty() + ); + Ok(()) +} + +#[test] +#[ignore = "requires a Vulkan Video device"] +fn a_source_image_needs_pixels() -> Result<(), Box> { + let context = context()?; + let input = InputImage::new( + context.clone(), + Codec::H264, + WIDTH, + HEIGHT, + EncodeBitDepth::Eight, + PixelFormat::Yuv420, + )?; + for layout in [vk::ImageLayout::UNDEFINED, vk::ImageLayout::PREINITIALIZED] { + assert!( + SourceImage::new(&context, input.image(), Handoff::Device { layout }).is_err(), + "{layout:?} was accepted" + ); + } + Ok(()) +} + +#[test] +#[ignore = "requires a Vulkan Video device"] +fn a_shader_conversion_has_no_input_slot() -> Result<(), Box> { + let context = context()?; + let encoder = Encoder::new( + context, + EncodeConfig::h264(WIDTH, HEIGHT) + .with_source(Source::Rgb(RgbSource::new( + InputFormat::BGRA, + ColorSpec::Srgb, + ))) + .with_conversion(ConversionPreference::Shader), + )?; + assert!(encoder.input_slot().is_err()); + Ok(()) +} diff --git a/tests/verify_all.rs b/tests/verify_all.rs index 4ccad3a2..3d1f78a7 100644 --- a/tests/verify_all.rs +++ b/tests/verify_all.rs @@ -11,7 +11,7 @@ mod common; use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, InputImage, PixelFormat, RateControlMode, + Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControlMode, VideoContextBuilder, }; use std::fs::File; @@ -174,14 +174,14 @@ fn run_test( // Upload directly to encoder's input image to avoid cross-queue // copy issues (InputImage uses the transfer queue, encoder uses the // video encode queue which doesn't support transfer ops). - let encoder_image = encoder.input_image(); + let slot = encoder.input_slot()?; match format { - PixelFormat::Yuv420 => input_image.upload_yuv420_to(encoder_image, frame)?, - PixelFormat::Yuv444 => input_image.upload_yuv444_to(encoder_image, frame)?, + PixelFormat::Yuv420 => input_image.upload_yuv420_to(&slot, frame)?, + PixelFormat::Yuv444 => input_image.upload_yuv444_to(&slot, frame)?, _ => return Err("Unsupported format".into()), } - pending.push_back(encoder.encode(encoder_image)?); + pending.push_back(encoder.submit(Frame::input(slot))?); while pending.len() > 2 { let packet = pollster::block_on(pending.pop_front().unwrap())?; output_file.write_all(&packet.data)?; From 1cd4c89588cf781b06aaa68020d44882b4479c8f Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 08:45:17 +0100 Subject: [PATCH 05/19] refactor(converter)!: make the colour converter crate-private --- AGENTS.md | 9 +- CHANGELOG.md | 3 + README.md | 3 - docs/encode-input.md | 3 + .../converter/device_tests.rs | 170 +++++++++++-- src/converter/mod.rs | 224 +++--------------- src/lib.rs | 11 +- tests/adopted.rs | 126 ++++------ tests/rgb_encode.rs | 86 ++----- 9 files changed, 269 insertions(+), 366 deletions(-) rename tests/color_conversion.rs => src/converter/device_tests.rs (74%) diff --git a/AGENTS.md b/AGENTS.md index 441b85ca..9d909790 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,9 +119,12 @@ run it after touching submission or barrier code. An RGB source (`Source::Rgb`) is converted by the encoder itself (`Conversion::Encoder`) only with `VK_VALVE_video_encode_rgb_conversion`, -which today only RADV has. `tests/rgb_encode.rs` checks the shader path -everywhere (byte-identical to converting by hand), and on RADV compares the -encoder's conversion against it and checks that `supports_rgb_encode_input` +which today only RADV has. The colour converter is crate-private, so its +device tests are in the crate (`src/converter/device_tests.rs`, run with +`cargo test --lib -- --ignored`): they check the shader against the same maths +on the CPU, and the shader path inside the encoder against converting by hand +(byte-identical). `tests/rgb_encode.rs` on RADV compares the encoder's +conversion against the shader's and checks that `supports_rgb_encode_input` agrees with encoder creation, so a green run elsewhere says little about the encoder path. Under validation that path draws two messages that are the layer's, not pixelforge's: the layer does not model that extension, so it diff --git a/CHANGELOG.md b/CHANGELOG.md index 62668a94..b723d944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `Frame::image(&source_image)` for an image of the caller's, `Frame::input(encoder.input_slot()?)` for the input image written in place. An imported image that `UNDEFINED` used to mark is `Handoff::External`. +- **Breaking:** `ColorConverter`, `ColorConverterConfig` and `OutputFormat` + are no longer public. Describe an RGB source with `Source::Rgb` instead, and + the encoder runs the converter itself, with the right target and range. - **Breaking:** `InputImage::upload_yuv420_to` and `upload_yuv444_to` take an `InputSlot` instead of a `vk::Image`. `InputImage` uploads wait until the frames that read the image are done with it. diff --git a/README.md b/README.md index 22f363cf..45e80187 100644 --- a/README.md +++ b/README.md @@ -254,9 +254,6 @@ default, see [`ColorSpec::reference_white_nits`], overridable with Supported input formats: BGRx, RGBx, BGRA, RGBA, ABGR2101010 (10-bit packed), RGBA16F (FP16). -[`ColorConverter`] is the same shader on its own, for converting without -encoding. - ### Submitting frames [`Encoder::submit`] is the one way to encode a frame. It takes a [`Frame`]: diff --git a/docs/encode-input.md b/docs/encode-input.md index a6f7c04e..be005c75 100644 --- a/docs/encode-input.md +++ b/docs/encode-input.md @@ -46,6 +46,9 @@ because no conversion produces it. work is an error from `Encoder::new`. `Encoder::conversion()` tells which path was selected. +The colour converter is not public. An RGB source is the only way to use it, +so its target and range always follow the stream's colour description. + ### Colour changes `Encoder::set_color_description` is the one way to change colour: diff --git a/tests/color_conversion.rs b/src/converter/device_tests.rs similarity index 74% rename from tests/color_conversion.rs rename to src/converter/device_tests.rs index d823bc63..b795fabf 100644 --- a/tests/color_conversion.rs +++ b/src/converter/device_tests.rs @@ -18,19 +18,28 @@ //! muddy the comparison. The test frame is deliberately not grey: grey converts //! to exact values and would hide the very errors being looked for. //! -//! Ignored by default: requires a Vulkan Video device. Run with -//! `cargo test -- --ignored`. +//! The last test checks the converter inside the encoder: an RGB source +//! converted by the shader must encode to the same stream as converting by +//! hand and encoding the result. +//! +//! These live in the crate, not in `tests/`, because the converter is not +//! public. Ignored by default: requires a Vulkan Video device. Run with +//! `cargo test --lib -- --ignored`. #[allow(dead_code)] -mod common; -use common::source::{SrcImage, create_src_image, host_buffer, one_shot}; +#[path = "../../tests/common/source.rs"] +mod source; +use source::{SrcImage, create_src_image, host_buffer, one_shot}; -use ash::vk; -use pixelforge::{ - Codec, ColorConverter, ColorConverterConfig, ColorDescription, ColorRange, ColorSpec, - EncodeBitDepth, EncodeConfig, Encoder, InputFormat, OutputFormat, RateControlMode, - VideoContext, VideoContextBuilder, +use super::{ + ColorConverter, ColorConverterConfig, ColorRange, ColorSpec, InputFormat, OutputFormat, +}; +use crate::encoder::input::SourceOwnership; +use crate::{ + Codec, ColorDescription, ConversionPreference, EncodeBitDepth, EncodeConfig, Encoder, Frame, + Handoff, RateControlMode, RgbSource, Source, SourceImage, VideoContext, VideoContextBuilder, }; +use ash::vk; const WIDTH: u32 = 256; const HEIGHT: u32 = 64; @@ -200,7 +209,7 @@ fn encoders( } fn context() -> Result> { - common::init_logging(); + init_logging(); Ok(VideoContextBuilder::new() .app_name("pixelforge-color-conversion") .require_encode(Codec::H264) @@ -208,6 +217,36 @@ fn context() -> Result> { .build()?) } +/// Route the validation layer's messages to the test output, as +/// `tests/common` does. +fn init_logging() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| { + let filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn")); + let _ = tracing_subscriber::fmt() + .with_env_filter(filter) + .with_test_writer() + .try_init(); + }); +} + +/// Convert `src`, in `GENERAL`, into `target`, and wait for it. +fn convert_into( + converter: &mut ColorConverter, + src: vk::Image, + target: vk::Image, +) -> crate::error::Result<()> { + let general = SourceOwnership { + old_layout: vk::ImageLayout::GENERAL, + acquire_from: None, + release_to: None, + final_layout: vk::ImageLayout::GENERAL, + }; + converter.convert_with(src, general, target, &[], &[])?; + converter.wait_idle() +} + /// Convert one frame and read its luma plane back as code values. fn convert( context: &VideoContext, @@ -229,11 +268,7 @@ fn convert( ); let mut converter = ColorConverter::new(context.clone(), config)?; let description = converter.color_description(); - converter.convert( - src.image, - vk::ImageLayout::GENERAL, - encoder.input_slot()?.image(), - )?; + convert_into(&mut converter, src.image, encoder.input_slot()?.image())?; Ok((read_luma(context, &converter, output_format)?, description)) } @@ -447,11 +482,7 @@ fn a_replaced_source_image_is_read_afresh() -> Result<(), Box>(luma) @@ -470,3 +501,102 @@ fn a_replaced_source_image_is_read_afresh() -> Result<(), Box Vec { + let mut data = Vec::with_capacity((ENCODE_WIDTH * ENCODE_HEIGHT * 4) as usize); + for y in 0..ENCODE_HEIGHT { + for x in 0..ENCODE_WIDTH { + let (sx, sy) = (x + n * 3, y + n); + data.extend_from_slice(&[(sy * 3) as u8, (sx * 2 + sy) as u8, (sx ^ sy) as u8, 255]); + } + } + data +} + +fn encode_config(codec: Codec) -> EncodeConfig { + match codec { + Codec::H264 => EncodeConfig::h264(ENCODE_WIDTH, ENCODE_HEIGHT), + Codec::H265 => EncodeConfig::h265(ENCODE_WIDTH, ENCODE_HEIGHT), + Codec::AV1 => EncodeConfig::av1(ENCODE_WIDTH, ENCODE_HEIGHT), + } + .with_rate_control(RateControlMode::Cqp) + .with_quality_level(22) + .with_frame_rate(30, 1) + .with_b_frames(0) + .with_color_description(ColorDescription::bt709()) +} + +/// The streams of 8 moving frames, converted by hand and by an encoder with +/// an RGB source and the shader conversion. +fn by_hand_and_by_encoder( + context: &VideoContext, + codec: Codec, +) -> Result<(Vec, Vec), Box> { + let mut by_hand = Encoder::new(context.clone(), encode_config(codec))?; + let mut converter = ColorConverter::new( + context.clone(), + ColorConverterConfig::new( + ENCODE_WIDTH, + ENCODE_HEIGHT, + InputFormat::BGRA, + OutputFormat::NV12, + ColorSpec::Srgb, + ColorSpec::Srgb, + ColorRange::Limited, + ), + )?; + let mut by_encoder = Encoder::new( + context.clone(), + encode_config(codec) + .with_source(Source::Rgb(RgbSource::new( + InputFormat::BGRA, + ColorSpec::Srgb, + ))) + .with_conversion(ConversionPreference::Shader), + )?; + let (mut a, mut b) = (Vec::new(), Vec::new()); + for n in 0..8 { + let src = + unsafe { create_src_image(context, ENCODE_WIDTH, ENCODE_HEIGHT, &moving_frame(n))? }; + let slot = by_hand.input_slot()?; + convert_into(&mut converter, src.image, slot.image())?; + a.extend(pollster::block_on(by_hand.submit(Frame::input(slot))?)?.data); + + let image = SourceImage::new( + context, + src.image, + Handoff::Device { + layout: vk::ImageLayout::GENERAL, + }, + )?; + b.extend(pollster::block_on(by_encoder.submit(Frame::image(&image))?)?.data); + } + Ok((a, b)) +} + +/// The shader path inside the encoder does exactly what converting by hand +/// does, so the streams match byte for byte. +#[test] +#[ignore = "requires a Vulkan Video device"] +fn shader_source_matches_converting_by_hand() -> Result<(), Box> { + init_logging(); + for codec in [Codec::H264, Codec::H265, Codec::AV1] { + let Ok(context) = VideoContextBuilder::new() + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .require_encode(codec) + .build() + else { + eprintln!("{codec:?}: no encoder on this device"); + continue; + }; + let (by_hand, by_encoder) = by_hand_and_by_encoder(&context, codec)?; + assert!(!by_hand.is_empty(), "{codec:?}: empty stream"); + assert!(by_hand == by_encoder, "{codec:?}: the streams differ"); + } + Ok(()) +} diff --git a/src/converter/mod.rs b/src/converter/mod.rs index af38dbb0..1c7ff887 100644 --- a/src/converter/mod.rs +++ b/src/converter/mod.rs @@ -15,11 +15,10 @@ use crate::sync::TimelinePoint; use crate::video::TimelineChain; use crate::vulkan::VideoContext; use ash::vk; -use tracing::debug; /// A colour space, used for both ends of a conversion: -/// [`ColorConverterConfig::source`] for the input and -/// [`ColorConverterConfig::target`] for the encoded output. +/// [`RgbSource::color`](crate::RgbSource::color) for the input, and the +/// stream's colour description for the encoded output. /// /// The two linear spaces can only be an input. Video files have no way to say /// "linear", so nothing could be decoded correctly; see @@ -56,7 +55,7 @@ impl ColorSpec { /// /// `None` for `Bt2020Pq`, which already carries real brightness values. /// Override it with - /// [`ColorConverterConfig::with_reference_white_nits`]. + /// [`RgbSource::with_reference_white_nits`](crate::RgbSource::with_reference_white_nits). pub fn reference_white_nits(&self) -> Option { match self { Self::Srgb | Self::Bt2020Linear => Some(203.0), @@ -76,7 +75,7 @@ impl ColorSpec { /// What to tell a player about video in this space. /// /// `None` when [`Self::is_encodable`] is false. The range is set - /// separately, by [`ColorConverterConfig::color_description`]. + /// separately. pub fn color_description(&self) -> Option { match self { // sRGB and BT.709 differ slightly in their curves, but video @@ -157,10 +156,12 @@ impl InputFormat { /// Supported output YUV formats. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum OutputFormat { +pub(crate) enum OutputFormat { /// NV12: Y plane followed by interleaved UV (4:2:0), 8-bit. NV12, - /// I420: Y plane, U plane, V plane (4:2:0), 8-bit. + /// I420: Y plane, U plane, V plane (4:2:0), 8-bit. The shader has the + /// mode, and no encoder input takes it. + #[allow(dead_code)] I420, /// YUV444 8-bit: 2-plane semi-planar (Y plane + interleaved UV) at full resolution. YUV444, @@ -193,23 +194,14 @@ impl OutputFormat { crate::align4(raw) } - /// Get the Vulkan format for this output format. - pub fn vulkan_format(&self) -> vk::Format { - match self { - OutputFormat::NV12 => vk::Format::G8_B8R8_2PLANE_420_UNORM, - OutputFormat::I420 => vk::Format::G8_B8_R8_3PLANE_420_UNORM, - OutputFormat::YUV444 => vk::Format::G8_B8R8_2PLANE_444_UNORM, - OutputFormat::P010 => vk::Format::G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, - OutputFormat::YUV444P10 => vk::Format::G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16, - } - } - /// Returns true if this is a 10-bit format. + #[cfg(test)] pub fn is_10bit(&self) -> bool { matches!(self, OutputFormat::P010 | OutputFormat::YUV444P10) } /// Bytes per sample for this format. + #[cfg(test)] pub fn bytes_per_sample(&self) -> usize { if self.is_10bit() { 2 } else { 1 } } @@ -221,7 +213,7 @@ impl OutputFormat { /// are, [`target`](Self::target) for what the encoded stream should be. #[derive(Clone, Debug)] #[non_exhaustive] -pub struct ColorConverterConfig { +pub(crate) struct ColorConverterConfig { /// Input frame width. pub width: u32, /// Input frame height. @@ -270,54 +262,14 @@ impl ColorConverterConfig { } } - /// Override how bright the source's white is, in nits. - /// - /// Rarely needed; each space already has a sensible default, see - /// [`ColorSpec::reference_white_nits`]. - pub fn with_reference_white_nits(mut self, nits: f32) -> Self { - self.reference_white_nits = Some(nits); - self - } - /// The value the shader gets: the override, or the source's default. fn effective_reference_white_nits(&self) -> f32 { self.reference_white_nits .or_else(|| self.source.reference_white_nits()) .unwrap_or(0.0) } - - /// What to tell the encoder about the colours this conversion produces, - /// for - /// [`EncodeConfig::with_color_description`](crate::EncodeConfig::with_color_description). - /// - /// Taken from [`target`](Self::target) and [`range`](Self::range), so the - /// stream cannot end up describing something the shader did not write. - /// - /// `None` when the target is one [`ColorConverter::new`] would reject - /// anyway. [`ColorConverter::color_description`] gives the same answer - /// without the `Option`. - /// - /// ```no_run - /// use pixelforge::{ - /// ColorConverterConfig, ColorRange, ColorSpec, EncodeConfig, InputFormat, OutputFormat, - /// }; - /// - /// # fn main() -> Result<(), Box> { - /// let conv = ColorConverterConfig::new( - /// 1920, - /// 1080, - /// InputFormat::BGRA, - /// OutputFormat::P010, - /// ColorSpec::Bt709Linear, - /// ColorSpec::Bt2020Pq, - /// ColorRange::Full, - /// ); - /// - /// let enc = EncodeConfig::h265(1920, 1080) - /// .with_color_description(conv.color_description().expect("PQ is encodable")); - /// # Ok(()) - /// # } - /// ``` + /// The stream colour this configuration converts to. + #[cfg(test)] pub fn color_description(&self) -> Option { Some( self.target @@ -360,7 +312,7 @@ impl ColorConverterConfig { /// /// Uses Vulkan compute shaders to convert RGB/BGR formats to YUV. /// with minimal latency and high throughput. -pub struct ColorConverter { +pub(crate) struct ColorConverter { context: VideoContext, config: ColorConverterConfig, @@ -394,7 +346,7 @@ pub struct ColorConverter { /// next conversion reuses the command buffer and output buffer, and /// destroys this one's source view, so it waits for this one first. in_flight: bool, - /// Signalled by each conversion, for [`Self::convert_async`] to hand out. + /// Signalled by each conversion, for [`Self::convert_with`] to hand out. timeline: TimelineChain, } @@ -411,18 +363,8 @@ impl ColorConverter { pipeline::create_converter(context, config) } - /// Get the video context. - /// - /// Returns a reference to the VideoContext used by this converter. - pub fn context(&self) -> &VideoContext { - &self.context - } - - /// What to tell the encoder about the colours this converter produces, for - /// [`EncodeConfig::with_color_description`](crate::EncodeConfig::with_color_description). - /// - /// [`ColorConverterConfig::color_description`] without the `Option`: this - /// converter exists, so its target was already checked. + /// The stream colour this converter produces. + #[cfg(test)] pub fn color_description(&self) -> ColorDescription { self.config .color_description() @@ -434,47 +376,19 @@ impl ColorConverter { &self.config } - /// Get the output buffer. - /// - /// Returns the Vulkan buffer containing raw YUV data. - /// This is useful for direct GPU-to-GPU transfers. + /// The buffer the shader writes the YUV planes to. + #[cfg(test)] pub fn output_buffer(&self) -> vk::Buffer { self.output_buffer } - /// Change what the input pixels are, for subsequent conversions. - /// - /// Takes effect on the next `convert()` without recreating the pipeline, - /// since the source is passed via push constants. - /// - /// Returns an error, and leaves the converter untouched, if the new source - /// cannot reach the configured target. - pub fn set_source(&mut self, source: ColorSpec) -> Result<()> { - let candidate = ColorConverterConfig { - source, - ..self.config.clone() - }; - if !candidate.conversion_supported() { - return Err(candidate.unsupported_conversion()); - } - self.config.source = source; - Ok(()) - } - /// Change what the encoded stream should be, for subsequent conversions. /// - /// Takes effect on the next `convert()` without recreating the pipeline, + /// Takes effect on the next conversion without recreating the pipeline, /// since the target is passed via push constants. /// /// Returns an error, and leaves the converter untouched, if the configured /// source cannot reach the new target. - /// - /// The target is half of what [`ColorConverterConfig::color_description`] - /// derives from, so an encoder told about the old one is now describing the - /// wrong signal. Pass the new description to - /// [`Encoder::set_color_description`](crate::Encoder::set_color_description), - /// which rebuilds the session parameters and makes the next frame an IDR - /// carrying the updated header. pub fn set_target(&mut self, target: ColorSpec) -> Result<()> { let candidate = ColorConverterConfig { target, @@ -489,10 +403,8 @@ impl ColorConverter { /// Change the quantization range for subsequent conversions. /// - /// Takes effect on the next `convert()` without recreating the pipeline, - /// since the range is passed via push constants. The range is the other - /// half of [`ColorConverterConfig::color_description`], so the same - /// re-declaration as [`Self::set_target`] applies. + /// Takes effect on the next conversion without recreating the pipeline, + /// since the range is passed via push constants. pub fn set_range(&mut self, range: ColorRange) { self.config.range = range; } @@ -741,86 +653,14 @@ impl ColorConverter { } } - /// Convert an input image directly to a target image (zero-copy encoder path). - /// - /// This is the most efficient path for encoding: it converts the source image. - /// directly into the encoder's input image, eliminating an intermediate copy. - /// - /// The target image must: - /// - Have the same dimensions as the converter's configuration - /// - Be in a format compatible with NV12/YUV (G8_B8R8_2PLANE_420_UNORM) - /// - Have TRANSFER_DST usage flag - /// - /// After this call, the target image will be in VIDEO_ENCODE_SRC_KHR layout, - /// ready for encoding. + /// Convert `src_image` into `target_image`, the encoder's input image, + /// which is left in `VIDEO_ENCODE_SRC_KHR` layout. /// - /// # Arguments - /// * `src_image` - Source RGB/BGR image (e.g., from DMA-BUF import) - /// * `src_layout` - Current layout of the source image (e.g., `GENERAL` for cached - /// imports, `UNDEFINED` for first-time imports that haven't been transitioned yet) - /// * `target_image` - Target image to write YUV data to (e.g., encoder's input_image) - /// - /// # Returns - /// Returns `Ok(())` on success. The target_image is transitioned to VIDEO_ENCODE_SRC_KHR. - pub fn convert( - &mut self, - src_image: vk::Image, - src_layout: vk::ImageLayout, - target_image: vk::Image, - ) -> Result<()> { - let start = std::time::Instant::now(); - self.convert_async(src_image, src_layout, target_image, &[])?; - self.wait_idle()?; - debug!("ColorConverter::convert() took {:?}", start.elapsed()); - Ok(()) - } - - /// Convert like [`Self::convert`], but without waiting on the CPU. - /// - /// The conversion waits on the GPU for every point in `wait` before - /// reading `src_image`, and returns the point it signals when done. Pass - /// that to [`Frame::wait`](crate::Frame::wait) to - /// encode the result, and wait on it before writing `src_image` again. - /// - /// On a device shared with the caller, where the caller renders or copies - /// into `src_image` itself, this is the whole handover: the caller's - /// submission signals a [`TimelinePoint`], the conversion waits on it, the - /// encode waits on the conversion. If the caller's queue is in another - /// family than [`VideoContext::compute_queue_family`], create `src_image` - /// with `VK_SHARING_MODE_CONCURRENT` across both, and pass the layout it - /// is actually in as `src_layout`: `UNDEFINED` is taken to mean a - /// first-time external-memory import and acquires ownership from - /// `VK_QUEUE_FAMILY_EXTERNAL`, which an image that never left the device - /// does not have to give. - /// - /// The next call to either method waits for this conversion to finish - /// before recording, since both reuse the same command buffer. - pub fn convert_async( - &mut self, - src_image: vk::Image, - src_layout: vk::ImageLayout, - target_image: vk::Image, - wait: &[TimelinePoint], - ) -> Result { - let waits: Vec> = - wait.iter().map(TimelinePoint::wait_info).collect(); - self.convert_with( - src_image, - SourceOwnership { - old_layout: src_layout, - acquire_from: (src_layout == vk::ImageLayout::UNDEFINED) - .then_some(vk::QUEUE_FAMILY_EXTERNAL), - release_to: None, - final_layout: vk::ImageLayout::GENERAL, - }, - target_image, - &waits, - &[], - ) - } - - /// [`Self::convert_async`] with the source's ownership transfers spelled - /// out, and extra semaphores to wait on and signal. + /// The conversion waits for `waits` on the GPU, acquires and releases the + /// source as `ownership` says, signals `extra_signals` with its own + /// timeline, and returns the point it signals when done. The next call + /// waits for this conversion to finish before recording, since every + /// call reuses the same command buffer. pub(crate) fn convert_with( &mut self, src_image: vk::Image, @@ -1165,6 +1005,9 @@ impl Drop for ColorConverter { } } +#[cfg(test)] +mod device_tests; + #[cfg(test)] mod tests { use super::*; @@ -1281,7 +1124,8 @@ mod tests { assert_eq!(base.reference_white_nits, None); assert_eq!(base.effective_reference_white_nits(), 80.0); - let overridden = base.clone().with_reference_white_nits(203.0); + let mut overridden = base.clone(); + overridden.reference_white_nits = Some(203.0); assert_eq!(overridden.effective_reference_white_nits(), 203.0); // The space itself is untouched; the override lives on the conversion. assert_eq!(overridden.source, ColorSpec::Bt709Linear); diff --git a/src/lib.rs b/src/lib.rs index 0cf1f59d..884536e7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -251,9 +251,6 @@ //! //! Supported input formats: BGRx, RGBx, BGRA, RGBA, ABGR2101010 (10-bit packed), RGBA16F (FP16). //! -//! [`ColorConverter`] is the same shader on its own, for converting without -//! encoding. -//! //! ## Submitting frames //! //! [`Encoder::submit`] is the one way to encode a frame. It takes a [`Frame`]: @@ -378,6 +375,10 @@ //! This project was heavily inspired by the [vk_video_samples](https://github.com/nvpro-samples/vk_video_samples) //! repository by NVIDIA, which provided invaluable reference for Vulkan Video encoding. +// The device tests share `tests/common/source.rs`, which names the crate. +#[cfg(test)] +extern crate self as pixelforge; + pub mod converter; pub mod decoder; pub mod encoder; @@ -395,9 +396,7 @@ pub(crate) const fn align4(size: usize) -> usize { (size + 3) & !3 } -pub use converter::{ - ColorConverter, ColorConverterConfig, ColorRange, ColorSpec, InputFormat, OutputFormat, -}; +pub use converter::{ColorRange, ColorSpec, InputFormat}; pub use decoder::{ DecodeConfig, DecodeSink, DecodeSource, DecodeStatus, DecodedFrame, Decoder, FramePoll, Framing, }; diff --git a/tests/adopted.rs b/tests/adopted.rs index 2e515e56..3c6fb810 100644 --- a/tests/adopted.rs +++ b/tests/adopted.rs @@ -19,8 +19,8 @@ use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::encoder::Codec; use pixelforge::vulkan::{DeviceQueue, DeviceRequirements, VideoContext, VideoContextBuilder}; use pixelforge::{ - ColorConverter, ColorConverterConfig, ColorRange, ColorSpec, EncodeConfig, Encoder, Frame, - InputFormat, OutputFormat, RateControlMode, TimelinePoint, + ColorSpec, ConversionPreference, EncodeConfig, Encoder, Frame, Handoff, InputFormat, + RateControlMode, RgbSource, Source, SourceImage, TimelinePoint, }; /// An instance and device the test owns, destroyed on drop after everything @@ -228,24 +228,22 @@ fn moving_frame(n: u32) -> Vec { data } -/// How [`encode_all`] hands each frame from the converter to the encoder. +/// How [`encode_all`] orders each frame against the caller's rendering. #[derive(Clone, Copy, PartialEq)] -enum Handoff { - /// `convert`, which waits on the CPU, then `encode`. +enum Wait { + /// Each frame is submitted once rendered, and its source is waited for + /// on the CPU before the next one. Cpu, - /// `convert_async` waiting on a point the caller signals, then - /// `encode_after` waiting on the conversion: ordered on the GPU, with - /// nothing waited for on the CPU until the packets are collected. + /// Each frame waits on the GPU for a point the caller signals only after + /// the submission: the conversion has to wait for it, and nothing waits + /// on the CPU until the packets are collected. Gpu, } -/// Convert and encode `ENCODE_FRAMES` frames as H.264 on `context`, the way a -/// capture pipeline would: an RGB image goes through the colour converter -/// straight into the encoder's input. -fn encode_all( - context: &VideoContext, - handoff: Handoff, -) -> Result, Box> { +/// Encode `ENCODE_FRAMES` RGB frames as H.264 on `context`, the way a capture +/// pipeline would: the encoder converts each one with the colour converter's +/// shader, straight into its input image. +fn encode_all(context: &VideoContext, wait: Wait) -> Result, Box> { let device = context.device(); let mut encoder = Encoder::new( context.clone(), @@ -253,24 +251,17 @@ fn encode_all( .with_rate_control(RateControlMode::Cqp) .with_quality_level(22) .with_frame_rate(30, 1) - .with_b_frames(0), - )?; - let mut converter = ColorConverter::new( - context.clone(), - ColorConverterConfig::new( - ENCODE_WIDTH, - ENCODE_HEIGHT, - InputFormat::BGRA, - OutputFormat::NV12, - ColorSpec::Srgb, - ColorSpec::Srgb, - ColorRange::Limited, - ), + .with_b_frames(0) + .with_source(Source::Rgb(RgbSource::new( + InputFormat::BGRA, + ColorSpec::Srgb, + ))) + .with_conversion(ConversionPreference::Shader), )?; - // Stands in for the caller's own rendering: each frame's conversion waits - // for a value the "renderer" signals once the frame is ready. The KHR - // entry points, since the adopted device is Vulkan 1.1. + // Stands in for the caller's own rendering: each frame waits for a value + // the "renderer" signals once the frame is ready. The KHR entry points, + // since the adopted device is Vulkan 1.1. let timeline_fns = ash::khr::timeline_semaphore::Device::load(context.instance(), device); let mut type_info = vk::SemaphoreTypeCreateInfo::default() .semaphore_type(vk::SemaphoreType::TIMELINE) @@ -282,33 +273,32 @@ fn encode_all( )? }; - // Signalled after the converter is dropped, see the last encode below. - let after_drop = u64::from(ENCODE_FRAMES) + 1; - - // Every source stays alive until the end: with a GPU handoff nothing says - // a conversion has finished reading one until its packet is back. + // Every source stays alive until the end. The images are dropped after + // their `SourceImage`s, which wait for the frames that read them. + let mut images = Vec::new(); let mut sources = Vec::new(); let mut pending = Vec::new(); for n in 0..ENCODE_FRAMES { - let src = + let image = unsafe { create_src_image(context, ENCODE_WIDTH, ENCODE_HEIGHT, &moving_frame(n))? }; - let slot = encoder.input_slot()?; - let target = slot.image(); - let future = match handoff { - Handoff::Cpu => { - converter.convert(src.image, vk::ImageLayout::GENERAL, target)?; - encoder.submit(Frame::input(slot))? + let source = SourceImage::new( + context, + image.image, + Handoff::Device { + layout: vk::ImageLayout::GENERAL, + }, + )?; + let future = match wait { + Wait::Cpu => { + let future = encoder.submit(Frame::image(&source))?; + source.wait_released()?; + future } - Handoff::Gpu => { + Wait::Gpu => { let ready = TimelinePoint::new(rendered, u64::from(n) + 1); - let converted = converter.convert_async( - src.image, - vk::ImageLayout::GENERAL, - target, - &[ready], - )?; - // Signalled only after the conversion was submitted, so it - // really has to wait. + let future = encoder.submit(Frame::image(&source).wait(&[ready]))?; + // Signalled only after the frame was submitted, so it really + // has to wait. unsafe { timeline_fns.signal_semaphore( &vk::SemaphoreSignalInfo::default() @@ -316,38 +306,20 @@ fn encode_all( .value(ready.value), )? }; - // The last encode also waits for a point signalled only once - // the converter is gone, so it is certainly still queued, - // waiting on the converter's semaphore, when that happens. - let mut waits = vec![converted]; - if n + 1 == ENCODE_FRAMES { - waits.push(TimelinePoint::new(rendered, after_drop)); - } - encoder.submit(Frame::input(slot).wait(&waits))? + future } }; - sources.push(src); + images.push(image); + sources.push(source); pending.push(future); } - // Dropped while an encode that waits on its points is still queued, as a - // caller rebuilding its converter might: its semaphore has to outlive - // that wait. - drop(converter); - if matches!(handoff, Handoff::Gpu) { - unsafe { - timeline_fns.signal_semaphore( - &vk::SemaphoreSignalInfo::default() - .semaphore(rendered) - .value(after_drop), - )? - }; - } let mut stream = Vec::new(); for future in pending { stream.extend_from_slice(&pollster::block_on(future)?.data); } - drop(encoder); drop(sources); + drop(encoder); + drop(images); unsafe { device.destroy_semaphore(rendered, None) }; Ok(stream) } @@ -377,7 +349,7 @@ fn encode_on_a_vulkan_1_1_device() -> Result<(), Box> { .require_encode(Codec::H264) .require_decode(Codec::H264) .build()?; - let expected = encode_all(&own, Handoff::Cpu)?; + let expected = encode_all(&own, Wait::Cpu)?; let (entry, instance) = instance_1_1()?; let builder = VideoContextBuilder::new().require_encode(Codec::H264); @@ -398,7 +370,7 @@ fn encode_on_a_vulkan_1_1_device() -> Result<(), Box> { app.physical_device, app.device.clone(), )?; - let actual = encode_all(&context, Handoff::Gpu)?; + let actual = encode_all(&context, Wait::Gpu)?; drop(context); // Both streams decoded by the same decoder, so any difference is the diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs index 4660e93d..689f5182 100644 --- a/tests/rgb_encode.rs +++ b/tests/rgb_encode.rs @@ -1,9 +1,9 @@ //! Encoding an RGB [`Source`], through each [`Conversion`] the encoder can //! pick. //! -//! With [`Conversion::Shader`] the encoder runs the colour converter itself, -//! so its stream must be byte-identical to converting by hand and encoding -//! the result. With [`Conversion::Encoder`] the encoder converts through +//! With [`Conversion::Shader`] the encoder runs the colour converter itself; +//! the crate's own tests check that it is byte-identical to converting by +//! hand. With [`Conversion::Encoder`] the encoder converts through //! `VK_VALVE_video_encode_rgb_conversion`; those tests check that it lands //! where the shader does, and skip on devices without the extension, which //! today is everything but AMD under RADV. @@ -16,10 +16,9 @@ use common::{Readback, decode_stream}; use ash::vk; use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::{ - Codec, ColorConverter, ColorConverterConfig, ColorDescription, ColorRange, ColorSpec, - Conversion, ConversionPreference, EncodeBitDepth, EncodeConfig, Encoder, Frame, Handoff, - InputFormat, OutputFormat, RateControlMode, RgbSource, Source, SourceImage, VideoContext, - VideoContextBuilder, + Codec, ColorDescription, ColorSpec, Conversion, ConversionPreference, EncodeBitDepth, + EncodeConfig, Encoder, Frame, Handoff, InputFormat, RateControlMode, RgbSource, Source, + SourceImage, VideoContext, VideoContextBuilder, }; const WIDTH: u32 = 320; @@ -37,18 +36,6 @@ fn context(codec: Codec) -> Result> { Ok(builder.build()?) } -fn converter_config() -> ColorConverterConfig { - ColorConverterConfig::new( - WIDTH, - HEIGHT, - InputFormat::BGRA, - OutputFormat::NV12, - ColorSpec::Srgb, - ColorSpec::Srgb, - ColorRange::Limited, - ) -} - /// A BGRA frame that moves with `n`. fn moving_frame(n: u32) -> Vec { let mut data = Vec::with_capacity((WIDTH * HEIGHT * 4) as usize); @@ -61,8 +48,7 @@ fn moving_frame(n: u32) -> Vec { data } -/// Upload frame `n`. It is left in `GENERAL`, the layout [`Encoder::encode`] -/// expects. +/// Upload frame `n`. It is left in `GENERAL`. fn source(context: &VideoContext, n: u32) -> Result> { Ok(unsafe { create_src_image(context, WIDTH, HEIGHT, &moving_frame(n))? }) } @@ -78,7 +64,7 @@ fn encode_config(codec: Codec) -> EncodeConfig { .with_quality_level(22) .with_frame_rate(30, 1) .with_b_frames(0) - .with_color_description(converter_config().color_description().unwrap()) + .with_color_description(ColorDescription::bt709()) } fn rgb(format: InputFormat) -> Source { @@ -99,25 +85,6 @@ fn source_image( ) } -/// Encode the frames with a YUV encoder and a colour converter the test runs -/// itself. -fn encode_by_hand( - context: &VideoContext, - codec: Codec, -) -> Result, Box> { - let mut encoder = Encoder::new(context.clone(), encode_config(codec))?; - let mut converter = ColorConverter::new(context.clone(), converter_config())?; - let mut stream = Vec::new(); - for n in 0..FRAMES { - let src = source(context, n)?; - let slot = encoder.input_slot()?; - converter.convert(src.image, vk::ImageLayout::GENERAL, slot.image())?; - let packet = pollster::block_on(encoder.submit(Frame::input(slot))?)?; - stream.extend_from_slice(&packet.data); - } - Ok(stream) -} - /// Encode the frames with an RGB source and the given conversion. fn encode_rgb( context: &VideoContext, @@ -172,7 +139,7 @@ fn psnr(a: &[u8], b: &[u8]) -> f64 { /// Whether the encoder can convert `format` for `codec` here; prints why a /// test is skipped when not. fn encoder_converts(context: &VideoContext, codec: Codec, test: &str) -> bool { - let desc = converter_config().color_description().unwrap(); + let desc = ColorDescription::bt709(); let ok = context.supports_rgb_encode_input(codec, InputFormat::BGRA, &desc); if !ok { eprintln!("skipping {test} for {codec:?}: the encoder cannot convert RGB here"); @@ -180,29 +147,6 @@ fn encoder_converts(context: &VideoContext, codec: Codec, test: &str) -> bool { ok } -/// The shader path inside the encoder does exactly what converting by hand -/// does, so the streams match byte for byte. -#[test] -#[ignore = "requires a Vulkan Video device"] -fn shader_source_matches_converting_by_hand() -> Result<(), Box> { - for codec in [Codec::H264, Codec::H265, Codec::AV1] { - let Ok(context) = context(codec) else { - eprintln!("{codec:?}: no encoder on this device"); - continue; - }; - let by_hand = encode_by_hand(&context, codec)?; - let shader = encode_rgb( - &context, - codec, - InputFormat::BGRA, - ConversionPreference::Shader, - )?; - assert!(!by_hand.is_empty(), "{codec:?}: empty stream"); - assert!(by_hand == shader, "{codec:?}: the streams differ"); - } - Ok(()) -} - /// `Auto` picks the encoder's own conversion exactly where it is available. #[test] #[ignore = "requires a Vulkan Video device"] @@ -212,7 +156,7 @@ fn auto_picks_the_encoder_where_it_can() -> Result<(), Box Result<(), Box> { if !encoder_converts(&context, Codec::H264, "rgb_input_matches_the_converter") { return Ok(()); } - let shader = decode_all(&context, &encode_by_hand(&context, Codec::H264)?)?; + let shader = decode_all( + &context, + &encode_rgb( + &context, + Codec::H264, + InputFormat::BGRA, + ConversionPreference::Shader, + )?, + )?; let hardware = decode_all( &context, &encode_rgb( From 425db7f00320ca051ef7702d1a12b20aecb1ebd5 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 08:48:06 +0100 Subject: [PATCH 06/19] feat(encoder)!: carry each rate control mode's values in a RateControl enum --- CHANGELOG.md | 7 + README.md | 8 +- benches/encode.rs | 12 +- examples/encode.rs | 9 +- src/converter/device_tests.rs | 7 +- src/encoder/api.rs | 1 + src/encoder/codec.rs | 22 ++-- src/encoder/h264/record.rs | 6 +- src/encoder/h265/record.rs | 6 +- src/encoder/mod.rs | 235 +++++++++++++++++++--------------- src/lib.rs | 16 ++- tests/adopted.rs | 5 +- tests/dmabuf.rs | 5 +- tests/rfi.rs | 5 +- tests/rgb_encode.rs | 5 +- tests/roundtrip.rs | 5 +- tests/verify_all.rs | 5 +- 17 files changed, 188 insertions(+), 171 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b723d944..226c871c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** `ColorConverter`, `ColorConverterConfig` and `OutputFormat` are no longer public. Describe an RGB source with `Source::Rgb` instead, and the encoder runs the converter itself, with the right target and range. +- **Breaking:** `RateControl` replaces `RateControlMode`, `quality_level`, + `target_bitrate` and `max_bitrate`: `Cqp { qp }`, `Cbr { bitrate }` or + `Vbr { target, max }`, so each mode carries exactly the values it uses. + `Encoder::new` refuses a quantizer out of the codec's range, a bitrate of 0, + and a peak below the target. `RateControlMode::Disabled` was the same as + `Cqp` and is gone. `DEFAULT_TARGET_BITRATE` and `DEFAULT_MAX_BITRATE` are + removed; `DEFAULT_AV1_Q_INDEX` is added. - **Breaking:** `InputImage::upload_yuv420_to` and `upload_yuv444_to` take an `InputSlot` instead of a `vk::Image`. `InputImage` uploads wait until the frames that read the image are done with it. diff --git a/README.md b/README.md index 45e80187..0b926aa2 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ fn main() -> Result<(), Box> { ```rust use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, InputImage, PixelFormat, RateControlMode, + Codec, EncodeBitDepth, EncodeConfig, Encoder, InputImage, PixelFormat, RateControl, VideoContextBuilder, }; @@ -108,8 +108,10 @@ fn main() -> Result<(), Box> { .build()?; let config = EncodeConfig::h264(1920, 1080) - .with_rate_control(RateControlMode::Vbr) - .with_target_bitrate(5_000_000) + .with_rate_control(RateControl::Vbr { + target: 5_000_000, + max: 7_500_000, + }) .with_frame_rate(30, 1) .with_gop_size(60); diff --git a/benches/encode.rs b/benches/encode.rs index 77b4770a..5de3ef2c 100644 --- a/benches/encode.rs +++ b/benches/encode.rs @@ -10,7 +10,7 @@ use pixelforge::{ Codec, EncodeBitDepth, EncodeConfig, Encoder, EncoderTuningMode, Frame, InputImage, - PixelFormat, RateControlMode, VideoContextBuilder, + PixelFormat, RateControl, VideoContextBuilder, }; use std::fs::File; use std::io::Read; @@ -122,12 +122,10 @@ fn run_encode_test( Codec::AV1 => EncodeConfig::av1(WIDTH, HEIGHT), } .with_rate_control(if matches!(tuning_mode, EncoderTuningMode::Lossless) { - RateControlMode::Cqp + RateControl::default_for(codec) } else { - RateControlMode::Cbr + RateControl::Cbr { bitrate: 8_000_000 } }) - .with_target_bitrate(8_000_000) - .with_max_bitrate(12_000_000) .with_frame_rate(60, 1) .with_gop_size(120) .with_b_frames(0) @@ -136,8 +134,8 @@ fn run_encode_test( .with_encoder_tuning_mode(tuning_mode); println!( - "Config: {:?}, bitrate={}, GOP={}, B-frames={}", - config.rate_control_mode, config.target_bitrate, config.gop_size, config.b_frame_count + "Config: {:?}, GOP={}, B-frames={}", + config.rate_control, config.gop_size, config.b_frame_count ); // Create input image for uploading frames. diff --git a/examples/encode.rs b/examples/encode.rs index c184620e..0ec2a5ad 100644 --- a/examples/encode.rs +++ b/examples/encode.rs @@ -10,7 +10,7 @@ //! The codec defaults to `h264`. use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControlMode, + Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, VideoContextBuilder, }; use std::fs::File; @@ -92,15 +92,14 @@ fn main() -> Result<(), Box> { // Configure encoder. let config = config - .with_rate_control(RateControlMode::Cqp) - .with_quality_level(26) + .with_rate_control(RateControl::Cqp { qp: 26 }) .with_frame_rate(30, 1) .with_gop_size(30) .with_b_frames(0); println!( - "Config: {:?}, QP={}, GOP={}, B-frames={}\n", - config.rate_control_mode, config.quality_level, config.gop_size, config.b_frame_count + "Config: {:?}, GOP={}, B-frames={}\n", + config.rate_control, config.gop_size, config.b_frame_count ); // Create input image for uploading frames. diff --git a/src/converter/device_tests.rs b/src/converter/device_tests.rs index b795fabf..6fb153b5 100644 --- a/src/converter/device_tests.rs +++ b/src/converter/device_tests.rs @@ -37,7 +37,7 @@ use super::{ use crate::encoder::input::SourceOwnership; use crate::{ Codec, ColorDescription, ConversionPreference, EncodeBitDepth, EncodeConfig, Encoder, Frame, - Handoff, RateControlMode, RgbSource, Source, SourceImage, VideoContext, VideoContextBuilder, + Handoff, RateControl, RgbSource, Source, SourceImage, VideoContext, VideoContextBuilder, }; use ash::vk; @@ -189,7 +189,6 @@ fn encoders( let eight = Encoder::new( context.clone(), EncodeConfig::h264(WIDTH, HEIGHT) - .with_rate_control(RateControlMode::Cqp) .with_frame_rate(30, 1) .with_b_frames(0), )?; @@ -197,7 +196,6 @@ fn encoders( context.clone(), EncodeConfig::h265(WIDTH, HEIGHT) .with_bit_depth(EncodeBitDepth::Ten) - .with_rate_control(RateControlMode::Cqp) .with_frame_rate(30, 1) .with_b_frames(0), ) @@ -524,8 +522,7 @@ fn encode_config(codec: Codec) -> EncodeConfig { Codec::H265 => EncodeConfig::h265(ENCODE_WIDTH, ENCODE_HEIGHT), Codec::AV1 => EncodeConfig::av1(ENCODE_WIDTH, ENCODE_HEIGHT), } - .with_rate_control(RateControlMode::Cqp) - .with_quality_level(22) + .with_rate_control(RateControl::Cqp { qp: 22 }) .with_frame_rate(30, 1) .with_b_frames(0) .with_color_description(ColorDescription::bt709()) diff --git a/src/encoder/api.rs b/src/encoder/api.rs index 0866cf9d..685eefb8 100644 --- a/src/encoder/api.rs +++ b/src/encoder/api.rs @@ -90,6 +90,7 @@ impl Encoder { /// gets from the source to [`EncodeConfig::color_description`], or if /// [`EncodeConfig::conversion`] asks for one that cannot. pub fn new(context: VideoContext, config: EncodeConfig) -> Result { + config.rate_control.check(config.codec)?; let plan = plan(&context, &config)?; if let Some(format) = plan.rgb_input { super::rgb::check_rgb_input(&context, &config, format)?; diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 60da4a54..478dc59a 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -29,7 +29,7 @@ use crate::encoder::resources::{ query_supported_video_formats, upload_image_to_input, }; use crate::encoder::rgb::RgbConversionCaps; -use crate::encoder::{ColorDescription, EncodeConfig, FrameType, RateControlMode}; +use crate::encoder::{ColorDescription, EncodeConfig, FrameType, RateControl}; use crate::error::{PixelForgeError, Result}; use crate::sync::TimelinePoint; use crate::video::{TimelineChain, allocate_command_buffers}; @@ -391,30 +391,30 @@ pub(crate) struct RateControlPlan { pub mode: vk::VideoEncodeRateControlModeFlagsKHR, pub average_bitrate: u32, pub max_bitrate: u32, - /// QP/q-index: the configured quality level for CQP/Disabled, otherwise the + /// QP/q-index: the configured quantizer for CQP, otherwise the /// codec's default starting point for the bitrate controller. pub qp: u32, } impl RateControlPlan { pub fn new(config: &EncodeConfig, controller_default_qp: u32) -> Self { - match config.rate_control_mode { - RateControlMode::Cqp | RateControlMode::Disabled => Self { + match config.rate_control { + RateControl::Cqp { qp } => Self { mode: vk::VideoEncodeRateControlModeFlagsKHR::DISABLED, average_bitrate: 0, max_bitrate: 0, - qp: config.quality_level, + qp, }, - RateControlMode::Cbr => Self { + RateControl::Cbr { bitrate } => Self { mode: vk::VideoEncodeRateControlModeFlagsKHR::CBR, - average_bitrate: config.target_bitrate, - max_bitrate: config.target_bitrate, + average_bitrate: bitrate, + max_bitrate: bitrate, qp: controller_default_qp, }, - RateControlMode::Vbr => Self { + RateControl::Vbr { target, max } => Self { mode: vk::VideoEncodeRateControlModeFlagsKHR::VBR, - average_bitrate: config.target_bitrate, - max_bitrate: config.max_bitrate, + average_bitrate: target, + max_bitrate: max, qp: controller_default_qp, }, } diff --git a/src/encoder/h264/record.rs b/src/encoder/h264/record.rs index 7412715d..e79da965 100644 --- a/src/encoder/h264/record.rs +++ b/src/encoder/h264/record.rs @@ -129,10 +129,8 @@ impl H264 { ), }; - let slice_qp_delta = match common.config.rate_control_mode { - crate::encoder::RateControlMode::Cqp | crate::encoder::RateControlMode::Disabled => { - (common.config.quality_level as i32 - 26) as i8 - } + let slice_qp_delta = match common.config.rate_control { + crate::encoder::RateControl::Cqp { qp } => (qp as i32 - 26) as i8, _ => 0, }; diff --git a/src/encoder/h265/record.rs b/src/encoder/h265/record.rs index 2a7472ed..0e6ec693 100644 --- a/src/encoder/h265/record.rs +++ b/src/encoder/h265/record.rs @@ -274,11 +274,7 @@ impl H265 { }; let rc = RateControlPlan::new(&common.config, 26); - let constant_qp = if rc.is_disabled() { - common.config.quality_level as i32 - } else { - 0 - }; + let constant_qp = if rc.is_disabled() { rc.qp as i32 } else { 0 }; let nalu_slice_entries = [vk::VideoEncodeH265NaluSliceSegmentInfoKHR::default() .constant_qp(constant_qp) .std_slice_segment_header(&slice_header)]; diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index 001b5f32..312496f2 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -30,12 +30,6 @@ use ash::vk; // Default encoder configuration constants. -/// Default target bitrate in bits per second (4 Mbps). -pub const DEFAULT_TARGET_BITRATE: u32 = 4_000_000; - -/// Default maximum bitrate in bits per second (6 Mbps). -pub const DEFAULT_MAX_BITRATE: u32 = 6_000_000; - /// Default frame rate (frames per second). pub const DEFAULT_FRAME_RATE: u32 = 30; @@ -48,6 +42,9 @@ pub const DEFAULT_H264_QP: u32 = 26; /// Default QP (quantization parameter) for H.265. pub const DEFAULT_H265_QP: u32 = 28; +/// Default quantizer index for AV1. +pub const DEFAULT_AV1_Q_INDEX: u32 = 128; + /// Default maximum number of reference frames. pub const DEFAULT_MAX_REFERENCE_FRAMES: u32 = 4; @@ -124,18 +121,65 @@ impl From for vk::VideoComponentBitDepthFlagsKHR { } } -/// Rate control modes. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum RateControlMode { - /// Disabled rate control - constant QP. - #[default] - Disabled, - /// Constant QP mode. - Cqp, - /// Constant bitrate mode. - Cbr, - /// Variable bitrate mode. - Vbr, +/// How the encoder spends bits, with the values that mode needs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RateControl { + /// Every frame at one quantizer, and no bitrate target. + Cqp { + /// The QP, 0 to 51, for H.264 and H.265; the quantizer index, 0 to + /// 255, for AV1. Lower is better quality and more bits. + qp: u32, + }, + /// A constant bitrate. + Cbr { + /// Bits per second. + bitrate: u32, + }, + /// A variable bitrate around a target, up to a peak. + Vbr { + /// Average bits per second. + target: u32, + /// Peak bits per second, at least `target`. + max: u32, + }, +} + +impl RateControl { + /// The default for `codec`: a constant quantizer of middling quality. + pub fn default_for(codec: Codec) -> Self { + Self::Cqp { + qp: match codec { + Codec::H264 => DEFAULT_H264_QP, + Codec::H265 => DEFAULT_H265_QP, + Codec::AV1 => DEFAULT_AV1_Q_INDEX, + }, + } + } + + /// Why this cannot be used for `codec`, if it cannot. + pub(crate) fn check(&self, codec: Codec) -> crate::error::Result<()> { + let invalid = |reason: String| Err(crate::error::PixelForgeError::InvalidInput(reason)); + match *self { + Self::Cqp { qp } => { + let max = if codec == Codec::AV1 { 255 } else { 51 }; + if qp > max { + return invalid(format!( + "{codec:?} takes a quantizer of 0 to {max}, not {qp}" + )); + } + } + Self::Cbr { bitrate: 0 } | Self::Vbr { target: 0, .. } => { + return invalid("a bitrate of 0".to_string()); + } + Self::Vbr { target, max } if max < target => { + return invalid(format!( + "a peak bitrate of {max} is below the target of {target}" + )); + } + Self::Cbr { .. } | Self::Vbr { .. } => {} + } + Ok(()) + } } /// Encode usage hints. @@ -316,14 +360,8 @@ pub struct EncodeConfig { pub pixel_format: PixelFormat, /// Bit depth per component. pub bit_depth: BitDepth, - /// Rate control mode. - pub rate_control_mode: RateControlMode, - /// Target bitrate in bits per second. - pub target_bitrate: u32, - /// Maximum bitrate in bits per second. - pub max_bitrate: u32, - /// Quality level for CQP mode (QP value). - pub quality_level: u32, + /// How the encoder spends bits. Checked by [`Encoder::new`]. + pub rate_control: RateControl, /// Frame rate numerator. pub frame_rate_numerator: u32, /// Frame rate denominator. @@ -334,12 +372,14 @@ pub struct EncodeConfig { pub b_frame_count: u32, /// Maximum number of reference frames. pub max_reference_frames: u32, - /// VBV/HRD virtual buffer size in milliseconds. + /// VBV/HRD virtual buffer size in milliseconds, for + /// [`RateControl::Cbr`] and [`RateControl::Vbr`]. /// Controls how much the encoder can deviate from the target bitrate /// on a per-frame basis. Smaller values produce more uniform frame /// sizes. pub virtual_buffer_size_ms: u32, - /// Initial VBV buffer fullness in milliseconds. + /// Initial VBV buffer fullness in milliseconds, for + /// [`RateControl::Cbr`] and [`RateControl::Vbr`]. /// Controls how much budget the encoder has for IDR/I-frames. /// Setting this to 0 constrains IDR frames to the same budget as /// P-frames. Setting it equal to `virtual_buffer_size_ms` gives @@ -373,10 +413,7 @@ impl EncodeConfig { dimensions: Dimensions { width, height }, pixel_format: PixelFormat::Yuv420, bit_depth: BitDepth::Eight, - rate_control_mode: RateControlMode::Disabled, - target_bitrate: DEFAULT_TARGET_BITRATE, - max_bitrate: DEFAULT_MAX_BITRATE, - quality_level: DEFAULT_H264_QP, + rate_control: RateControl::default_for(Codec::H264), frame_rate_numerator: DEFAULT_FRAME_RATE, frame_rate_denominator: 1, gop_size: DEFAULT_GOP_SIZE, @@ -403,10 +440,7 @@ impl EncodeConfig { dimensions: Dimensions { width, height }, pixel_format: PixelFormat::Yuv420, bit_depth: BitDepth::Eight, - rate_control_mode: RateControlMode::Disabled, - target_bitrate: DEFAULT_TARGET_BITRATE, - max_bitrate: DEFAULT_MAX_BITRATE, - quality_level: DEFAULT_H265_QP, + rate_control: RateControl::default_for(Codec::H265), frame_rate_numerator: DEFAULT_FRAME_RATE, frame_rate_denominator: 1, gop_size: DEFAULT_GOP_SIZE, @@ -433,10 +467,7 @@ impl EncodeConfig { dimensions: Dimensions { width, height }, pixel_format: PixelFormat::Yuv420, bit_depth: BitDepth::Eight, - rate_control_mode: RateControlMode::Disabled, - target_bitrate: DEFAULT_TARGET_BITRATE, - max_bitrate: DEFAULT_MAX_BITRATE, - quality_level: 128, // AV1 uses 0-255 QP range + rate_control: RateControl::default_for(Codec::AV1), frame_rate_numerator: DEFAULT_FRAME_RATE, frame_rate_denominator: 1, gop_size: DEFAULT_GOP_SIZE, @@ -453,9 +484,9 @@ impl EncodeConfig { } } - /// Set the rate control mode. - pub fn with_rate_control(mut self, mode: RateControlMode) -> Self { - self.rate_control_mode = mode; + /// Set how the encoder spends bits. + pub fn with_rate_control(mut self, rate_control: RateControl) -> Self { + self.rate_control = rate_control; self } @@ -471,12 +502,6 @@ impl EncodeConfig { self } - /// Set the quality level (QP for CQP mode). - pub fn with_quality_level(mut self, level: u32) -> Self { - self.quality_level = level; - self - } - /// Set the frame rate. pub fn with_frame_rate(mut self, numerator: u32, denominator: u32) -> Self { self.frame_rate_numerator = numerator; @@ -502,18 +527,6 @@ impl EncodeConfig { self } - /// Set the target bitrate. - pub fn with_target_bitrate(mut self, bitrate: u32) -> Self { - self.target_bitrate = bitrate; - self - } - - /// Set the maximum bitrate. - pub fn with_max_bitrate(mut self, bitrate: u32) -> Self { - self.max_bitrate = bitrate; - self - } - /// Set the VBV/HRD virtual buffer size in milliseconds. /// Smaller values produce more uniform frame sizes at the cost of /// quality variation during scene changes. @@ -691,13 +704,44 @@ mod tests { } } - // RateControlMode tests. + // RateControl tests. mod rate_control_tests { use super::*; #[test] - fn test_default() { - assert_eq!(RateControlMode::default(), RateControlMode::Disabled); + fn defaults_are_a_constant_quantizer() { + assert_eq!( + RateControl::default_for(Codec::H264), + RateControl::Cqp { qp: 26 } + ); + assert_eq!( + RateControl::default_for(Codec::AV1), + RateControl::Cqp { qp: 128 } + ); + } + + #[test] + fn check_refuses_what_cannot_work() { + assert!(RateControl::Cqp { qp: 51 }.check(Codec::H264).is_ok()); + assert!(RateControl::Cqp { qp: 52 }.check(Codec::H265).is_err()); + assert!(RateControl::Cqp { qp: 255 }.check(Codec::AV1).is_ok()); + assert!(RateControl::Cbr { bitrate: 0 }.check(Codec::H264).is_err()); + assert!( + RateControl::Vbr { + target: 8_000_000, + max: 6_000_000 + } + .check(Codec::H264) + .is_err() + ); + assert!( + RateControl::Vbr { + target: 6_000_000, + max: 8_000_000 + } + .check(Codec::H264) + .is_ok() + ); } } @@ -714,8 +758,7 @@ mod tests { assert_eq!(config.dimensions.height, 1080); assert_eq!(config.pixel_format, PixelFormat::Yuv420); assert_eq!(config.bit_depth, BitDepth::Eight); - assert_eq!(config.rate_control_mode, RateControlMode::Disabled); - assert_eq!(config.quality_level, 26); + assert_eq!(config.rate_control, RateControl::Cqp { qp: 26 }); assert_eq!(config.gop_size, 30); assert_eq!(config.b_frame_count, 0); assert_eq!(config.frame_rate_numerator, 30); @@ -729,14 +772,16 @@ mod tests { assert_eq!(config.codec, Codec::H265); assert_eq!(config.dimensions.width, 3840); assert_eq!(config.dimensions.height, 2160); - assert_eq!(config.quality_level, 28); // H.265 uses slightly higher QP + // H.265 uses slightly higher QP. + assert_eq!(config.rate_control, RateControl::Cqp { qp: 28 }); } #[test] fn test_with_rate_control() { - let config = EncodeConfig::h264(1920, 1080).with_rate_control(RateControlMode::Cbr); + let rate_control = RateControl::Cbr { bitrate: 8_000_000 }; + let config = EncodeConfig::h264(1920, 1080).with_rate_control(rate_control); - assert_eq!(config.rate_control_mode, RateControlMode::Cbr); + assert_eq!(config.rate_control, rate_control); } #[test] @@ -753,13 +798,6 @@ mod tests { assert_eq!(config.bit_depth, BitDepth::Ten); } - #[test] - fn test_with_quality_level() { - let config = EncodeConfig::h264(1920, 1080).with_quality_level(20); - - assert_eq!(config.quality_level, 20); - } - #[test] fn test_with_frame_rate() { let config = EncodeConfig::h264(1920, 1080).with_frame_rate(60, 1); @@ -789,20 +827,6 @@ mod tests { assert_eq!(config.max_reference_frames, 8); } - #[test] - fn test_with_target_bitrate() { - let config = EncodeConfig::h264(1920, 1080).with_target_bitrate(8_000_000); - - assert_eq!(config.target_bitrate, 8_000_000); - } - - #[test] - fn test_with_max_bitrate() { - let config = EncodeConfig::h264(1920, 1080).with_max_bitrate(12_000_000); - - assert_eq!(config.max_bitrate, 12_000_000); - } - #[test] fn test_av1_defaults() { let config = EncodeConfig::av1(2560, 1440); @@ -812,8 +836,8 @@ mod tests { assert_eq!(config.dimensions.height, 1440); assert_eq!(config.pixel_format, PixelFormat::Yuv420); assert_eq!(config.bit_depth, BitDepth::Eight); - assert_eq!(config.rate_control_mode, RateControlMode::Disabled); - assert_eq!(config.quality_level, 128); // AV1 uses 0-255 QP range + // AV1 uses a 0-255 quantizer index. + assert_eq!(config.rate_control, RateControl::Cqp { qp: 128 }); assert_eq!(config.gop_size, 30); assert_eq!(config.b_frame_count, 0); assert_eq!(config.frame_rate_numerator, 30); @@ -823,39 +847,38 @@ mod tests { #[test] fn test_av1_builder_chaining() { let config = EncodeConfig::av1(1920, 1080) - .with_rate_control(RateControlMode::Vbr) - .with_target_bitrate(8_000_000) - .with_max_bitrate(12_000_000) + .with_rate_control(RateControl::Cqp { qp: 100 }) .with_gop_size(60) .with_frame_rate(60, 1) - .with_quality_level(100) .with_max_reference_frames(2); assert_eq!(config.codec, Codec::AV1); - assert_eq!(config.rate_control_mode, RateControlMode::Vbr); - assert_eq!(config.target_bitrate, 8_000_000); - assert_eq!(config.max_bitrate, 12_000_000); + assert_eq!(config.rate_control, RateControl::Cqp { qp: 100 }); assert_eq!(config.gop_size, 60); assert_eq!(config.frame_rate_numerator, 60); - assert_eq!(config.quality_level, 100); assert_eq!(config.max_reference_frames, 2); } #[test] fn test_builder_chaining() { let config = EncodeConfig::h264(1920, 1080) - .with_rate_control(RateControlMode::Vbr) - .with_target_bitrate(6_000_000) - .with_max_bitrate(10_000_000) + .with_rate_control(RateControl::Vbr { + target: 6_000_000, + max: 10_000_000, + }) .with_gop_size(120) .with_b_frames(2) .with_frame_rate(60, 1) .with_pixel_format(PixelFormat::Yuv420) .with_bit_depth(BitDepth::Eight); - assert_eq!(config.rate_control_mode, RateControlMode::Vbr); - assert_eq!(config.target_bitrate, 6_000_000); - assert_eq!(config.max_bitrate, 10_000_000); + assert_eq!( + config.rate_control, + RateControl::Vbr { + target: 6_000_000, + max: 10_000_000 + } + ); assert_eq!(config.gop_size, 120); assert_eq!(config.b_frame_count, 2); assert_eq!(config.frame_rate_numerator, 60); diff --git a/src/lib.rs b/src/lib.rs index 884536e7..1e23f0bc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,7 +91,7 @@ //! //! ```rust,no_run //! use pixelforge::{ -//! Codec, EncodeBitDepth, EncodeConfig, Encoder, InputImage, PixelFormat, RateControlMode, +//! Codec, EncodeBitDepth, EncodeConfig, Encoder, InputImage, PixelFormat, RateControl, //! VideoContextBuilder, //! }; //! @@ -102,8 +102,10 @@ //! .build()?; //! //! let config = EncodeConfig::h264(1920, 1080) -//! .with_rate_control(RateControlMode::Vbr) -//! .with_target_bitrate(5_000_000) +//! .with_rate_control(RateControl::Vbr { +//! target: 5_000_000, +//! max: 7_500_000, +//! }) //! .with_frame_rate(30, 1) //! .with_gop_size(60); //! @@ -403,10 +405,10 @@ pub use decoder::{ #[cfg(feature = "dmabuf")] pub use encoder::external::{DmaBuf, DmaBufPlane, ExternalImage, InputFormats, InputModifier}; pub use encoder::{ - BitDepth as EncodeBitDepth, Codec, ColorDescription, DEFAULT_FRAME_RATE, DEFAULT_GOP_SIZE, - DEFAULT_H264_QP, DEFAULT_H265_QP, DEFAULT_MAX_BITRATE, DEFAULT_MAX_REFERENCE_FRAMES, - DEFAULT_TARGET_BITRATE, EncodeConfig, EncodeContentHint, EncodeFuture, EncodeUsageHint, - EncodedPacket, Encoder, EncoderTuningMode, Frame, FrameType, PixelFormat, RateControlMode, + BitDepth as EncodeBitDepth, Codec, ColorDescription, DEFAULT_AV1_Q_INDEX, DEFAULT_FRAME_RATE, + DEFAULT_GOP_SIZE, DEFAULT_H264_QP, DEFAULT_H265_QP, DEFAULT_MAX_REFERENCE_FRAMES, EncodeConfig, + EncodeContentHint, EncodeFuture, EncodeUsageHint, EncodedPacket, Encoder, EncoderTuningMode, + Frame, FrameType, PixelFormat, RateControl, }; pub use encoder::{ Conversion, ConversionPreference, Handoff, InputSlot, RgbSource, Source, SourceImage, diff --git a/tests/adopted.rs b/tests/adopted.rs index 3c6fb810..9661027c 100644 --- a/tests/adopted.rs +++ b/tests/adopted.rs @@ -20,7 +20,7 @@ use pixelforge::encoder::Codec; use pixelforge::vulkan::{DeviceQueue, DeviceRequirements, VideoContext, VideoContextBuilder}; use pixelforge::{ ColorSpec, ConversionPreference, EncodeConfig, Encoder, Frame, Handoff, InputFormat, - RateControlMode, RgbSource, Source, SourceImage, TimelinePoint, + RateControl, RgbSource, Source, SourceImage, TimelinePoint, }; /// An instance and device the test owns, destroyed on drop after everything @@ -248,8 +248,7 @@ fn encode_all(context: &VideoContext, wait: Wait) -> Result, Box EncodeConfig { Codec::H265 => EncodeConfig::h265(width, height), Codec::AV1 => EncodeConfig::av1(width, height), } - .with_rate_control(RateControlMode::Cqp) - .with_quality_level(22) + .with_rate_control(RateControl::Cqp { qp: 22 }) .with_b_frames(0) } diff --git a/tests/rfi.rs b/tests/rfi.rs index 887cf0d4..1be993e1 100644 --- a/tests/rfi.rs +++ b/tests/rfi.rs @@ -20,7 +20,7 @@ mod common; use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControlMode, + Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, VideoContextBuilder, }; use std::collections::VecDeque; @@ -103,8 +103,7 @@ fn run_codec( Codec::H265 => EncodeConfig::h265(WIDTH, HEIGHT), Codec::AV1 => EncodeConfig::av1(WIDTH, HEIGHT), } - .with_rate_control(RateControlMode::Cqp) - .with_quality_level(10) + .with_rate_control(RateControl::Cqp { qp: 10 }) .with_pixel_format(PixelFormat::Yuv420) .with_bit_depth(EncodeBitDepth::Eight) // Infinite GOP so the recovery can only come from RFI, never a periodic IDR. diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs index 689f5182..62f222d4 100644 --- a/tests/rgb_encode.rs +++ b/tests/rgb_encode.rs @@ -17,7 +17,7 @@ use ash::vk; use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::{ Codec, ColorDescription, ColorSpec, Conversion, ConversionPreference, EncodeBitDepth, - EncodeConfig, Encoder, Frame, Handoff, InputFormat, RateControlMode, RgbSource, Source, + EncodeConfig, Encoder, Frame, Handoff, InputFormat, RateControl, RgbSource, Source, SourceImage, VideoContext, VideoContextBuilder, }; @@ -60,8 +60,7 @@ fn encode_config(codec: Codec) -> EncodeConfig { Codec::AV1 => EncodeConfig::av1(WIDTH, HEIGHT), }; config - .with_rate_control(RateControlMode::Cqp) - .with_quality_level(22) + .with_rate_control(RateControl::Cqp { qp: 22 }) .with_frame_rate(30, 1) .with_b_frames(0) .with_color_description(ColorDescription::bt709()) diff --git a/tests/roundtrip.rs b/tests/roundtrip.rs index 929042d4..2a34d17d 100644 --- a/tests/roundtrip.rs +++ b/tests/roundtrip.rs @@ -22,7 +22,7 @@ use common::{Readback, decode_stream, write_nv12}; use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControlMode, + Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, VideoContextBuilder, }; @@ -57,8 +57,7 @@ fn roundtrip_h264() -> Result<(), Box> { let frame_count = yuv.len() / frame_size; let config = EncodeConfig::h264(WIDTH, HEIGHT) - .with_rate_control(RateControlMode::Cqp) - .with_quality_level(26) + .with_rate_control(RateControl::Cqp { qp: 26 }) .with_frame_rate(30, 1) .with_gop_size(GOP) .with_b_frames(B_FRAMES); diff --git a/tests/verify_all.rs b/tests/verify_all.rs index 3d1f78a7..5ed73e96 100644 --- a/tests/verify_all.rs +++ b/tests/verify_all.rs @@ -11,7 +11,7 @@ mod common; use pixelforge::{ - Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControlMode, + Codec, EncodeBitDepth, EncodeConfig, Encoder, Frame, InputImage, PixelFormat, RateControl, VideoContextBuilder, }; use std::fs::File; @@ -129,8 +129,7 @@ fn run_test( Codec::H265 => EncodeConfig::h265(WIDTH, HEIGHT), Codec::AV1 => EncodeConfig::av1(WIDTH, HEIGHT), } - .with_rate_control(RateControlMode::Cqp) - .with_quality_level(10) + .with_rate_control(RateControl::Cqp { qp: 10 }) .with_pixel_format(format) .with_bit_depth(depth); From 7f74ff3da58bd839b1ad6a3c223520a1483487c1 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 08:51:35 +0100 Subject: [PATCH 07/19] feat(encoder)!: describe the stream colour with a StreamColor enum --- CHANGELOG.md | 12 +++-- README.md | 20 ++++---- docs/encode-input.md | 17 ++++--- src/converter/device_tests.rs | 5 +- src/encoder/api.rs | 29 +++++------ src/encoder/av1/init.rs | 6 +-- src/encoder/codec.rs | 21 +++----- src/encoder/h264/init.rs | 6 +-- src/encoder/h264/session_params.rs | 2 +- src/encoder/h265/init.rs | 6 +-- src/encoder/mod.rs | 68 ++++++++++++++++++++----- src/encoder/rgb.rs | 15 +++--- src/encoder/source.rs | 80 ++++++++++++------------------ src/lib.rs | 21 ++++---- tests/rgb_encode.rs | 56 +++++++++++++-------- 15 files changed, 204 insertions(+), 160 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 226c871c..13535ced 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,9 +55,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** `InputImage::upload_yuv420_to` and `upload_yuv444_to` take an `InputSlot` instead of a `vk::Image`. `InputImage` uploads wait until the frames that read the image are done with it. -- `Encoder::set_color_description` also updates the conversion of an RGB - source, and makes the encoder again where the encoder converts. Packet - timestamps carry on. +- **Breaking:** `StreamColor` replaces the raw `ColorDescription` in the + encode API: `EncodeConfig::with_color` replaces `with_color_description`, + `Encoder::set_color` replaces `set_color_description`, and + `VideoContext::supports_rgb_encode_input` takes a `StreamColor`. `Sdr` and + `Hdr10` name the two colours an RGB source converts to, with their range; + `Other` carries any H.273 description, for a YUV source only. +- `Encoder::set_color` also updates the conversion of an RGB source, and + makes the encoder again where the encoder converts. Packet timestamps carry + on. - The `dmabuf` feature enables its extensions only where the device has them, rather than failing device creation where it does not. diff --git a/README.md b/README.md index 0b926aa2..117996b4 100644 --- a/README.md +++ b/README.md @@ -197,16 +197,16 @@ CPU is the consumer's job; `examples/common` shows one way. An encoder takes RGB frames once it is told what they are, with a [`Source`]. It then converts them to the YUV that the stream's -[`ColorDescription`] describes, and that description also picks the target: -BT.709 for SDR, BT.2020 with PQ for HDR10. The source and the stream are -described once each, so the conversion and the stream's label cannot -disagree, and a colour change through [`Encoder::set_color_description`] -changes both. +[`StreamColor`] says: [`StreamColor::Sdr`] (BT.709) or +[`StreamColor::Hdr10`] (BT.2020 with PQ), each with its range. The source +and the stream are described once each, so the conversion and the +stream's label cannot disagree, and a colour change through +[`Encoder::set_color`] changes both. ```rust use pixelforge::{ - Codec, ColorDescription, ColorSpec, EncodeConfig, Encoder, InputFormat, RgbSource, - Source, VideoContextBuilder, + Codec, ColorRange, ColorSpec, EncodeConfig, Encoder, InputFormat, RgbSource, Source, + StreamColor, VideoContextBuilder, }; let context = VideoContextBuilder::new() @@ -217,10 +217,12 @@ let context = VideoContextBuilder::new() let config = EncodeConfig::h265(1920, 1080) .with_bit_depth(pixelforge::EncodeBitDepth::Ten) .with_source(Source::Rgb(RgbSource::new(InputFormat::BGRx, ColorSpec::Srgb))) - .with_color_description(ColorDescription::bt2020_pq().with_full_range(true)); + .with_color(StreamColor::Hdr10 { + range: ColorRange::Full, + }); let mut encoder = Encoder::new(context, config)?; println!("converting with {:?}", encoder.conversion()); -// let packet = encoder.encode(rgb_image)?; +// let packet = encoder.submit(Frame::image(&rgb_image))?; ``` [`Encoder::new`] picks the [`Conversion`]: diff --git a/docs/encode-input.md b/docs/encode-input.md index be005c75..4251a495 100644 --- a/docs/encode-input.md +++ b/docs/encode-input.md @@ -25,11 +25,14 @@ method for each. RGB. `format` is the pixel layout (`InputFormat`) and `color` is what the pixels mean (`ColorSpec`). -The stream's colour stays `EncodeConfig::color_description`. For an RGB -source, pixelforge derives the conversion target from it: BT.709 primaries, -transfer and matrix are `ColorSpec::Srgb`; BT.2020 and PQ are -`ColorSpec::Bt2020Pq`. Any other description is an error for an RGB source, -because no conversion produces it. +The stream's colour is `EncodeConfig::color`, a `StreamColor`: + +- `Sdr { range }`: BT.709. An RGB source converts to `ColorSpec::Srgb`. +- `Hdr10 { range }`: BT.2020 with PQ. An RGB source converts to + `ColorSpec::Bt2020Pq`. +- `Other(ColorDescription)`: any other H.273 description, for a YUV source + whose pixels already are in it. It is an error for an RGB source, because + no conversion produces it, even when its codes say BT.709. ### Path selection @@ -47,11 +50,11 @@ work is an error from `Encoder::new`. `Encoder::conversion()` tells which path was selected. The colour converter is not public. An RGB source is the only way to use it, -so its target and range always follow the stream's colour description. +so its target and range always follow the stream's colour. ### Colour changes -`Encoder::set_color_description` is the one way to change colour: +`Encoder::set_color(StreamColor)` is the one way to change colour: - `None` and `Shader`: the session parameters are made again (as before), and the shader gets the new target and range. diff --git a/src/converter/device_tests.rs b/src/converter/device_tests.rs index 6fb153b5..5fcbad5e 100644 --- a/src/converter/device_tests.rs +++ b/src/converter/device_tests.rs @@ -37,7 +37,8 @@ use super::{ use crate::encoder::input::SourceOwnership; use crate::{ Codec, ColorDescription, ConversionPreference, EncodeBitDepth, EncodeConfig, Encoder, Frame, - Handoff, RateControl, RgbSource, Source, SourceImage, VideoContext, VideoContextBuilder, + Handoff, RateControl, RgbSource, Source, SourceImage, StreamColor, VideoContext, + VideoContextBuilder, }; use ash::vk; @@ -525,7 +526,7 @@ fn encode_config(codec: Codec) -> EncodeConfig { .with_rate_control(RateControl::Cqp { qp: 22 }) .with_frame_rate(30, 1) .with_b_frames(0) - .with_color_description(ColorDescription::bt709()) + .with_color(StreamColor::default()) } /// The streams of 8 moving frames, converted by hand and by an encoder with diff --git a/src/encoder/api.rs b/src/encoder/api.rs index 685eefb8..1707a2dd 100644 --- a/src/encoder/api.rs +++ b/src/encoder/api.rs @@ -8,8 +8,8 @@ use super::frame::{Frame, FrameSource, InputSlot}; use super::input::{FrameInput, FrameSync, SourceOwnership}; use super::source::{Conversion, ConversionPreference, plan, target_space}; use super::source_image::SourceImage; -use super::{Codec, ColorDescription, EncodeConfig, EncodeFuture}; -use crate::converter::{ColorConverter, ColorRange}; +use super::{Codec, EncodeConfig, EncodeFuture, StreamColor}; +use crate::converter::ColorConverter; use crate::error::{PixelForgeError, Result}; use crate::sync::TimelinePoint; use crate::vulkan::VideoContext; @@ -27,7 +27,7 @@ pub(crate) trait EncoderApi: Send { fn flush(&mut self) -> Result<()>; fn request_idr(&mut self); fn invalidate_reference_frames(&mut self, first_lost_display_order: u64); - fn set_color_description(&mut self, desc: ColorDescription) -> Result<()>; + fn set_color(&mut self, color: StreamColor) -> Result<()>; } impl EncoderApi for CodecEncoder { @@ -49,8 +49,8 @@ impl EncoderApi for CodecEncoder { fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) { CodecEncoder::invalidate_reference_frames(self, first_lost_display_order) } - fn set_color_description(&mut self, desc: ColorDescription) -> Result<()> { - CodecEncoder::set_color_description(self, desc) + fn set_color(&mut self, color: StreamColor) -> Result<()> { + CodecEncoder::set_color(self, color) } } @@ -87,7 +87,7 @@ impl Encoder { /// /// For an RGB [`Source`](super::Source), this also picks the /// [`Conversion`], see [`Self::conversion`]. It fails if no conversion - /// gets from the source to [`EncodeConfig::color_description`], or if + /// gets from the source to [`EncodeConfig::color`], or if /// [`EncodeConfig::conversion`] asks for one that cannot. pub fn new(context: VideoContext, config: EncodeConfig) -> Result { config.rate_control.check(config.codec)?; @@ -378,23 +378,18 @@ impl Encoder { /// source the conversion follows: the colour converter gets the new target /// and range, and where the encoder converts, it is made again with the /// new matrix and range, since those are fixed in its video session. - /// Packet timestamps carry on across that. If the new description needs a + /// Packet timestamps carry on across that. If the new colour needs a /// different [`Conversion`], images imported or allocated for the old one /// have to be made again. /// /// On error the encoder is left as it was. - pub fn set_color_description(&mut self, desc: ColorDescription) -> Result<()> { + pub fn set_color(&mut self, color: StreamColor) -> Result<()> { let mut config = self.config.clone(); - config.color_description = Some(desc); + config.color = color; match self.conversion { - Conversion::None => self.inner.set_color_description(desc)?, + Conversion::None => self.inner.set_color(color)?, Conversion::Shader if self.shader_still_converts(&config) => { - let target = target_space(&desc).expect("the plan checked the target"); - let range = if desc.full_range { - ColorRange::Full - } else { - ColorRange::Limited - }; + let (target, range) = target_space(&color).expect("the plan checked the target"); let converter = self .converter .as_mut() @@ -402,7 +397,7 @@ impl Encoder { let (old_target, old_range) = (converter.config().target, converter.config().range); converter.set_target(target)?; converter.set_range(range); - if let Err(e) = self.inner.set_color_description(desc) { + if let Err(e) = self.inner.set_color(color) { let _ = converter.set_target(old_target); converter.set_range(old_range); return Err(e); diff --git a/src/encoder/av1/init.rs b/src/encoder/av1/init.rs index 7366db52..b5e1a454 100644 --- a/src/encoder/av1/init.rs +++ b/src/encoder/av1/init.rs @@ -1,12 +1,12 @@ use super::{Av1, MIN_BITSTREAM_BUFFER_SIZE, SUPERBLOCK_SIZE}; use crate::converter::InputFormat; +use crate::encoder::EncodeConfig; use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, }; use crate::encoder::profile::EncodeProfile; use crate::encoder::rgb::RgbConversionCaps; -use crate::encoder::{ColorDescription, EncodeConfig}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; use ash::vk; @@ -86,9 +86,7 @@ impl Av1 { }; let mut encoder = CodecEncoder { common, codec }; - let color_desc = config - .color_description - .unwrap_or(ColorDescription::bt709()); + let color_desc = config.color.description(); encoder.common.session_params = encoder .codec .build_session_params(&encoder.common, &color_desc)?; diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 478dc59a..144ebcc7 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -2,7 +2,7 @@ //! //! Everything that is identical across H.264, H.265 and AV1 lives here: //! the shared per-encoder state ([`EncoderCommon`]), the generic driver -//! ([`CodecEncoder`]) that owns the public `encode`/`flush`/`set_color_description` +//! ([`CodecEncoder`]) that owns the public `submit`/`flush`/`set_color` //! flow, the shared initialization scaffolding ([`build_encoder_common`]) and the //! shared rate-control decision ([`RateControlPlan`]). //! @@ -29,7 +29,7 @@ use crate::encoder::resources::{ query_supported_video_formats, upload_image_to_input, }; use crate::encoder::rgb::RgbConversionCaps; -use crate::encoder::{ColorDescription, EncodeConfig, FrameType, RateControl}; +use crate::encoder::{ColorDescription, EncodeConfig, FrameType, RateControl, StreamColor}; use crate::error::{PixelForgeError, Result}; use crate::sync::TimelinePoint; use crate::video::{TimelineChain, allocate_command_buffers}; @@ -469,7 +469,7 @@ pub(crate) trait VideoCodec: Sized + Send { ) -> bool; /// Build the codec parameter sets and create Vulkan session parameters. - /// Used at init and by `set_color_description`. + /// Used at init and by `set_color`. fn create_session_params( &self, common: &EncoderCommon, @@ -549,11 +549,12 @@ impl CodecEncoder { } } - /// Rebuild session parameters with a new color description; the next frame is + /// Rebuild session parameters with a new stream colour; the next frame is /// an IDR/key frame carrying the updated header. - pub fn set_color_description(&mut self, desc: ColorDescription) -> Result<()> { + pub fn set_color(&mut self, color: StreamColor) -> Result<()> { // Drain in-flight encodes before mutating shared session parameters. self.common.pipeline.wait_all_free(); + let desc = color.description(); let old_session_params = self.common.session_params; let new_session_params = self.codec.create_session_params(&self.common, &desc)?; @@ -564,7 +565,7 @@ impl CodecEncoder { } self.common.session_params = new_session_params; - self.common.config.color_description = Some(desc); + self.common.config.color = color; self.codec.invalidate_header_cache(); self.common.gop.request_idr(); Ok(()) @@ -772,13 +773,7 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result "RGB input: the driver reported no conversion capabilities".to_string(), ) })?; - Some( - caps.session_info( - &config - .color_description - .unwrap_or_else(ColorDescription::bt709), - )?, - ) + Some(caps.session_info(&config.color.description())?) } None => None, }; diff --git a/src/encoder/h264/init.rs b/src/encoder/h264/init.rs index 29a66966..e22bbb1e 100644 --- a/src/encoder/h264/init.rs +++ b/src/encoder/h264/init.rs @@ -1,6 +1,7 @@ use super::{H264, MB_SIZE}; use crate::converter::InputFormat; +use crate::encoder::EncodeConfig; use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, }; @@ -9,7 +10,6 @@ use crate::encoder::profile::EncodeProfile; use crate::encoder::profile::h264_profile_idc; use crate::encoder::resources::MIN_BITSTREAM_BUFFER_SIZE; use crate::encoder::rgb::RgbConversionCaps; -use crate::encoder::{ColorDescription, EncodeConfig}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; use ash::vk; @@ -109,9 +109,7 @@ impl H264 { }; let mut encoder = CodecEncoder { common, codec }; - let color_desc = config - .color_description - .unwrap_or(ColorDescription::bt709()); + let color_desc = config.color.description(); encoder.common.session_params = encoder .codec .build_session_params(&encoder.common, &color_desc)?; diff --git a/src/encoder/h264/session_params.rs b/src/encoder/h264/session_params.rs index 01aae437..0578c796 100644 --- a/src/encoder/h264/session_params.rs +++ b/src/encoder/h264/session_params.rs @@ -13,7 +13,7 @@ use std::ptr; impl H264 { /// Build SPS/PPS and create Vulkan video session parameters. /// - /// Used both at init and by `set_color_description`; a single implementation + /// Used both at init and by `set_color`; a single implementation /// keeps the parameter sets bit-for-bit consistent. pub(super) fn build_session_params( &self, diff --git a/src/encoder/h265/init.rs b/src/encoder/h265/init.rs index 73a15d94..ed8784f9 100644 --- a/src/encoder/h265/init.rs +++ b/src/encoder/h265/init.rs @@ -1,6 +1,7 @@ use super::{CTB_SIZE, H265}; use crate::converter::InputFormat; +use crate::encoder::EncodeConfig; use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, }; @@ -9,7 +10,6 @@ use crate::encoder::profile::EncodeProfile; use crate::encoder::profile::h265_profile_idc; use crate::encoder::resources::MIN_BITSTREAM_BUFFER_SIZE; use crate::encoder::rgb::RgbConversionCaps; -use crate::encoder::{ColorDescription, EncodeConfig}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; use ash::vk; @@ -104,9 +104,7 @@ impl H265 { }; let mut encoder = CodecEncoder { common, codec }; - let color_desc = config - .color_description - .unwrap_or(ColorDescription::bt709()); + let color_desc = config.color.description(); encoder.common.session_params = encoder .codec .build_session_params(&encoder.common, &color_desc)?; diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index 312496f2..7d8032d6 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -28,6 +28,8 @@ pub(crate) mod support; use ash::vk; +use crate::converter::ColorRange; + // Default encoder configuration constants. /// Default frame rate (frames per second). @@ -287,6 +289,46 @@ pub struct Dimensions { pub height: u32, } +/// What the stream says about its colours: for an RGB source, also what the +/// encoder converts to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamColor { + /// SDR: BT.709 primaries, transfer and matrix. + Sdr { + /// The luma and chroma range. + range: ColorRange, + }, + /// HDR10: BT.2020 primaries and matrix, with the PQ transfer. + Hdr10 { + /// The luma and chroma range. + range: ColorRange, + }, + /// Any other H.273 description, for a YUV source whose pixels already + /// are in it. No conversion from RGB produces one, so [`Encoder::new`] + /// refuses it for [`Source::Rgb`]. + Other(ColorDescription), +} + +impl Default for StreamColor { + /// SDR, limited range: what players assume when a stream says nothing. + fn default() -> Self { + Self::Sdr { + range: ColorRange::Limited, + } + } +} + +impl StreamColor { + /// The H.273 description written into the stream. + pub fn description(&self) -> ColorDescription { + match *self { + Self::Sdr { range } => ColorDescription::bt709().with_full_range(range.is_full()), + Self::Hdr10 { range } => ColorDescription::bt2020_pq().with_full_range(range.is_full()), + Self::Other(desc) => desc, + } + } +} + /// Video signal color description for VUI parameters. /// /// Describes how color is encoded in the video stream, allowing decoders @@ -385,9 +427,9 @@ pub struct EncodeConfig { /// P-frames. Setting it equal to `virtual_buffer_size_ms` gives /// IDR frames maximum headroom. pub initial_virtual_buffer_size_ms: u32, - /// Color description for VUI signaling. - /// Defaults to BT.709, limited range, when `None`. - pub color_description: Option, + /// What the stream says about its colours. SDR, limited range, by + /// default; see [`Self::with_color`]. + pub color: StreamColor, /// What the frames handed to the encoder are. YUV by default; see /// [`Self::with_source`]. pub source: Source, @@ -421,7 +463,7 @@ impl EncodeConfig { max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, - color_description: None, + color: StreamColor::default(), source: Source::Yuv, conversion: ConversionPreference::Auto, encode_usage_hint: EncodeUsageHint::Default, @@ -448,7 +490,7 @@ impl EncodeConfig { max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, - color_description: None, + color: StreamColor::default(), source: Source::Yuv, conversion: ConversionPreference::Auto, encode_usage_hint: EncodeUsageHint::Default, @@ -475,7 +517,7 @@ impl EncodeConfig { max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, - color_description: None, + color: StreamColor::default(), source: Source::Yuv, conversion: ConversionPreference::Auto, encode_usage_hint: EncodeUsageHint::Default, @@ -542,18 +584,19 @@ impl EncodeConfig { self } - /// Set the color description for VUI signaling. - pub fn with_color_description(mut self, desc: ColorDescription) -> Self { - self.color_description = Some(desc); + /// Set what the stream says about its colours. + /// + /// For [`Source::Rgb`] it is also what the encoder converts to: + /// [`StreamColor::Sdr`] or [`StreamColor::Hdr10`]. + pub fn with_color(mut self, color: StreamColor) -> Self { + self.color = color; self } /// Say what the frames handed to the encoder are. /// /// For [`Source::Rgb`], the encoder converts them to the YUV that - /// [`Self::color_description`] describes, which then also picks the - /// conversion's target: BT.709 for SDR, BT.2020 with PQ for HDR10. - /// [`Encoder::new`] picks how, see [`Conversion`]. + /// [`Self::color`] says. [`Encoder::new`] picks how, see [`Conversion`]. pub fn with_source(mut self, source: Source) -> Self { self.source = source; self @@ -589,6 +632,7 @@ pub use api::Encoder; pub use frame::{Frame, InputSlot}; pub use pipeline::EncodeFuture; pub use source::{Conversion, ConversionPreference, RgbSource, Source}; + pub use source_image::{Handoff, SourceImage}; /// Statistic about the encoded video packet. diff --git a/src/encoder/rgb.rs b/src/encoder/rgb.rs index 3b922b97..9992d4a9 100644 --- a/src/encoder/rgb.rs +++ b/src/encoder/rgb.rs @@ -8,7 +8,7 @@ //! matrix, range and chroma siting to convert with. use super::support::report; -use super::{BitDepth, Codec, ColorDescription, EncodeConfig, PixelFormat}; +use super::{BitDepth, Codec, ColorDescription, EncodeConfig, PixelFormat, StreamColor}; use crate::converter::InputFormat; use crate::error::{PixelForgeError, Result}; use crate::video::query_supported_video_formats; @@ -151,26 +151,29 @@ impl RgbConversionCaps { impl VideoContext { /// Whether an [`Encoder`](crate::Encoder) for `codec` can take `format` - /// RGB input and convert it to the YUV that `desc` describes itself, + /// RGB input and convert it to the YUV that `color` says itself, /// through `VK_VALVE_video_encode_rgb_conversion`. /// /// Asks the driver everything [`Encoder::new`](crate::Encoder::new) will /// need: that the device has the extension, that the codec's RGB input - /// profile exists, that its conversion does `desc`'s matrix and range, + /// profile exists, that its conversion does `color`'s matrix and range, /// and that it accepts `format` as an input image. The output is always /// 4:2:0, at 8 bits for 8-bit RGB and 10 bits for /// [`InputFormat::ABGR2101010`]. Resolution and rate control are not /// considered. /// /// When this is true, an encoder for an RGB [`Source`](crate::Source) in - /// `format`, with `desc` as its colour description, picks + /// `format`, with `color` as its stream colour, picks /// [`Conversion::Encoder`](crate::Conversion::Encoder) by itself. pub fn supports_rgb_encode_input( &self, codec: Codec, format: InputFormat, - desc: &ColorDescription, + color: StreamColor, ) -> bool { + if matches!(color, StreamColor::Other(_)) { + return false; + } if !self.has_video_encode_rgb_conversion() || !self.supports_encode(codec) { return false; } @@ -211,7 +214,7 @@ impl VideoContext { ); return false; } - if let Err(e) = RgbConversionCaps::from(&caps).session_info(desc) { + if let Err(e) = RgbConversionCaps::from(&caps).session_info(&color.description()) { tracing::debug!("{what}: {e}"); return false; } diff --git a/src/encoder/source.rs b/src/encoder/source.rs index 41c6ec06..d5f4a042 100644 --- a/src/encoder/source.rs +++ b/src/encoder/source.rs @@ -6,7 +6,7 @@ //! RGB conversion, and always submits frames the same way. use super::rgb::rgb_input_bit_depth; -use super::{BitDepth, ColorDescription, EncodeConfig, PixelFormat}; +use super::{BitDepth, EncodeConfig, PixelFormat, StreamColor}; use crate::converter::{ColorConverterConfig, ColorRange, ColorSpec, InputFormat, OutputFormat}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; @@ -87,24 +87,14 @@ pub(crate) struct Plan { pub converter: Option, } -/// The colour space a stream described by `desc` is in, if a conversion -/// can produce it. -pub(crate) fn target_space(desc: &ColorDescription) -> Option { - [ColorSpec::Srgb, ColorSpec::Bt2020Pq] - .into_iter() - .find(|space| { - space.color_description().is_some_and(|d| { - ( - d.color_primaries, - d.transfer_characteristics, - d.matrix_coefficients, - ) == ( - desc.color_primaries, - desc.transfer_characteristics, - desc.matrix_coefficients, - ) - }) - }) +/// The colour space and range a conversion from RGB writes for `color`, if +/// one can. +pub(crate) fn target_space(color: &StreamColor) -> Option<(ColorSpec, ColorRange)> { + match *color { + StreamColor::Sdr { range } => Some((ColorSpec::Srgb, range)), + StreamColor::Hdr10 { range } => Some((ColorSpec::Bt2020Pq, range)), + StreamColor::Other(_) => None, + } } /// The YUV the colour converter has to write for this encoder. @@ -137,18 +127,14 @@ pub(crate) fn plan(context: &VideoContext, config: &EncodeConfig) -> Result rgb, }; - let desc = config - .color_description - .unwrap_or_else(ColorDescription::bt709); - let target = target_space(&desc).ok_or_else(|| { + let (target, range) = target_space(&config.color).ok_or_else(|| { PixelForgeError::InvalidInput(format!( - "no conversion from RGB produces the colour description {desc:?}; \ - use BT.709 or BT.2020 with PQ" + "no conversion from RGB produces {:?}; use StreamColor::Sdr or StreamColor::Hdr10", + config.color )) })?; - - let encoder = encoder_conversion(context, config, &rgb, target, &desc); - let shader = shader_conversion(context, config, &rgb, target, &desc); + let encoder = encoder_conversion(context, config, &rgb, target); + let shader = shader_conversion(context, config, &rgb, target, range); let use_encoder = match config.conversion { ConversionPreference::Encoder => { @@ -184,7 +170,6 @@ fn encoder_conversion( config: &EncodeConfig, rgb: &RgbSource, target: ColorSpec, - desc: &ColorDescription, ) -> Result<()> { let no = |reason: String| { Err(PixelForgeError::InvalidInput(format!( @@ -210,10 +195,10 @@ fn encoder_conversion( } None => return no(format!("{:?} needs tone mapping first", rgb.format)), } - if !context.supports_rgb_encode_input(config.codec, rgb.format, desc) { + if !context.supports_rgb_encode_input(config.codec, rgb.format, config.color) { return no(format!( - "the driver does not convert {:?} to {desc:?} for {:?}", - rgb.format, config.codec + "the driver does not convert {:?} to {:?} for {:?}", + rgb.format, config.color, config.codec )); } Ok(()) @@ -226,7 +211,7 @@ fn shader_conversion( config: &EncodeConfig, rgb: &RgbSource, target: ColorSpec, - desc: &ColorDescription, + range: ColorRange, ) -> Result { if !context.has_push_descriptor() { return Err(PixelForgeError::NoSuitableDevice( @@ -239,11 +224,6 @@ fn shader_conversion( config.pixel_format, config.bit_depth )) })?; - let range = if desc.full_range { - ColorRange::Full - } else { - ColorRange::Limited - }; let mut converter = ColorConverterConfig::new( config.dimensions.width, config.dimensions.height, @@ -263,26 +243,30 @@ fn shader_conversion( #[cfg(test)] mod tests { use super::*; + use crate::encoder::ColorDescription; #[test] - fn target_space_follows_the_description() { + fn target_space_follows_the_stream_colour() { assert_eq!( - target_space(&ColorDescription::bt709()), - Some(ColorSpec::Srgb) + target_space(&StreamColor::default()), + Some((ColorSpec::Srgb, ColorRange::Limited)) ); assert_eq!( - target_space(&ColorDescription::bt709().with_full_range(true)), - Some(ColorSpec::Srgb) - ); - assert_eq!( - target_space(&ColorDescription::bt2020_pq()), - Some(ColorSpec::Bt2020Pq) + target_space(&StreamColor::Hdr10 { + range: ColorRange::Full + }), + Some((ColorSpec::Bt2020Pq, ColorRange::Full)) ); let bt601 = ColorDescription { matrix_coefficients: 6, ..ColorDescription::bt709() }; - assert_eq!(target_space(&bt601), None); + assert_eq!(target_space(&StreamColor::Other(bt601)), None); + // Even when the codes say BT.709: an RGB source names its target. + assert_eq!( + target_space(&StreamColor::Other(ColorDescription::bt709())), + None + ); } #[test] diff --git a/src/lib.rs b/src/lib.rs index 1e23f0bc..8b43308d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -191,16 +191,16 @@ //! //! An encoder takes RGB frames once it is told what they are, with a //! [`Source`]. It then converts them to the YUV that the stream's -//! [`ColorDescription`] describes, and that description also picks the target: -//! BT.709 for SDR, BT.2020 with PQ for HDR10. The source and the stream are -//! described once each, so the conversion and the stream's label cannot -//! disagree, and a colour change through [`Encoder::set_color_description`] -//! changes both. +//! [`StreamColor`] says: [`StreamColor::Sdr`] (BT.709) or +//! [`StreamColor::Hdr10`] (BT.2020 with PQ), each with its range. The source +//! and the stream are described once each, so the conversion and the +//! stream's label cannot disagree, and a colour change through +//! [`Encoder::set_color`] changes both. //! //! ```rust,no_run //! use pixelforge::{ -//! Codec, ColorDescription, ColorSpec, EncodeConfig, Encoder, InputFormat, RgbSource, -//! Source, VideoContextBuilder, +//! Codec, ColorRange, ColorSpec, EncodeConfig, Encoder, InputFormat, RgbSource, Source, +//! StreamColor, VideoContextBuilder, //! }; //! //! # fn main() -> Result<(), Box> { @@ -212,10 +212,12 @@ //! let config = EncodeConfig::h265(1920, 1080) //! .with_bit_depth(pixelforge::EncodeBitDepth::Ten) //! .with_source(Source::Rgb(RgbSource::new(InputFormat::BGRx, ColorSpec::Srgb))) -//! .with_color_description(ColorDescription::bt2020_pq().with_full_range(true)); +//! .with_color(StreamColor::Hdr10 { +//! range: ColorRange::Full, +//! }); //! let mut encoder = Encoder::new(context, config)?; //! println!("converting with {:?}", encoder.conversion()); -//! // let packet = encoder.encode(rgb_image)?; +//! // let packet = encoder.submit(Frame::image(&rgb_image))?; //! # Ok(()) //! # } //! ``` @@ -412,6 +414,7 @@ pub use encoder::{ }; pub use encoder::{ Conversion, ConversionPreference, Handoff, InputSlot, RgbSource, Source, SourceImage, + StreamColor, }; pub use error::PixelForgeError; pub use image::InputImage; diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs index 62f222d4..897f763c 100644 --- a/tests/rgb_encode.rs +++ b/tests/rgb_encode.rs @@ -16,9 +16,9 @@ use common::{Readback, decode_stream}; use ash::vk; use pixelforge::decoder::{DecodeConfig, Decoder}; use pixelforge::{ - Codec, ColorDescription, ColorSpec, Conversion, ConversionPreference, EncodeBitDepth, - EncodeConfig, Encoder, Frame, Handoff, InputFormat, RateControl, RgbSource, Source, - SourceImage, VideoContext, VideoContextBuilder, + Codec, ColorRange, ColorSpec, Conversion, ConversionPreference, EncodeBitDepth, EncodeConfig, + Encoder, Frame, Handoff, InputFormat, RateControl, RgbSource, Source, SourceImage, StreamColor, + VideoContext, VideoContextBuilder, }; const WIDTH: u32 = 320; @@ -63,7 +63,7 @@ fn encode_config(codec: Codec) -> EncodeConfig { .with_rate_control(RateControl::Cqp { qp: 22 }) .with_frame_rate(30, 1) .with_b_frames(0) - .with_color_description(ColorDescription::bt709()) + .with_color(StreamColor::default()) } fn rgb(format: InputFormat) -> Source { @@ -138,8 +138,7 @@ fn psnr(a: &[u8], b: &[u8]) -> f64 { /// Whether the encoder can convert `format` for `codec` here; prints why a /// test is skipped when not. fn encoder_converts(context: &VideoContext, codec: Codec, test: &str) -> bool { - let desc = ColorDescription::bt709(); - let ok = context.supports_rgb_encode_input(codec, InputFormat::BGRA, &desc); + let ok = context.supports_rgb_encode_input(codec, InputFormat::BGRA, StreamColor::default()); if !ok { eprintln!("skipping {test} for {codec:?}: the encoder cannot convert RGB here"); } @@ -155,8 +154,11 @@ fn auto_picks_the_encoder_where_it_can() -> Result<(), Box Result<(), Box let context = VideoContextBuilder::new() .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) .build()?; - let descriptions = [ - ("BT.709 limited", ColorDescription::bt709()), + let colors = [ + ("SDR limited", StreamColor::default()), ( - "BT.709 full", - ColorDescription::bt709().with_full_range(true), + "SDR full", + StreamColor::Sdr { + range: ColorRange::Full, + }, + ), + ( + "HDR10 limited", + StreamColor::Hdr10 { + range: ColorRange::Limited, + }, ), - ("BT.2020", ColorDescription::bt2020_pq()), ]; for codec in [Codec::H264, Codec::H265, Codec::AV1] { if !context.supports_encode(codec) { @@ -269,18 +278,18 @@ fn rgb_query_matches_encoder_creation() -> Result<(), Box } else { EncodeBitDepth::Eight }; - for (name, desc) in &descriptions { - let color = if desc.is_hdr() { + for &(name, stream) in &colors { + let color = if matches!(stream, StreamColor::Hdr10 { .. }) { ColorSpec::Bt2020Pq } else { ColorSpec::Srgb }; - let supported = context.supports_rgb_encode_input(codec, format, desc); + let supported = context.supports_rgb_encode_input(codec, format, stream); let created = Encoder::new( context.clone(), encode_config(codec) .with_bit_depth(bit_depth) - .with_color_description(*desc) + .with_color(stream) .with_source(Source::Rgb(RgbSource::new(format, color))) .with_conversion(ConversionPreference::Encoder), ) @@ -311,7 +320,9 @@ fn color_change_follows_into_the_conversion() -> Result<(), Box Result<(), Box>()); assert!(keys[0] && keys[FRAMES as usize / 2], "key frames: {keys:?}"); - assert!( - encoder.config().color_description.unwrap().full_range, - "the description was not kept" + assert_eq!( + encoder.config().color, + StreamColor::Sdr { + range: ColorRange::Full + }, + "the colour was not kept" ); // Full range is not something every encoder converts to; where it does // not, the shader takes over. From ebea57d1c259cb942b81be056a3458adc22ac3d7 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 08:52:58 +0100 Subject: [PATCH 08/19] refactor(encoder)!: remove the B-frame option until B-frames are encoded --- CHANGELOG.md | 3 +++ README.md | 2 +- benches/encode.rs | 6 +----- examples/encode.rs | 7 +++---- src/converter/device_tests.rs | 8 ++------ src/encoder/av1/init.rs | 11 +---------- src/encoder/h264/init.rs | 12 +----------- src/encoder/h265/init.rs | 9 --------- src/encoder/mod.rs | 20 +++----------------- src/lib.rs | 2 +- tests/adopted.rs | 1 - tests/dmabuf.rs | 1 - tests/rfi.rs | 3 +-- tests/rgb_encode.rs | 1 - tests/roundtrip.rs | 7 ++----- 15 files changed, 19 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13535ced..a86800fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and a peak below the target. `RateControlMode::Disabled` was the same as `Cqp` and is gone. `DEFAULT_TARGET_BITRATE` and `DEFAULT_MAX_BITRATE` are removed; `DEFAULT_AV1_Q_INDEX` is added. +- **Breaking:** `EncodeConfig::with_b_frames` and the public `b_frame_count` + field are removed. No codec encodes B-frames yet, and every encoder refused + a count above 0 at runtime. - **Breaking:** `InputImage::upload_yuv420_to` and `upload_yuv444_to` take an `InputSlot` instead of a `vk::Image`. `InputImage` uploads wait until the frames that read the image are done with it. diff --git a/README.md b/README.md index 117996b4..e422a2ed 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ H.265 and AV1 encode, and H.264 decode. for it, read with no copy where the driver allows, with sync_file fences (Linux only). -> **Note**: B-frame support is not yet implemented. Setting `b_frame_count > 0` will panic. +> **Note**: B-frame encoding is not yet implemented, so there is no option for it. ## Supported Codecs diff --git a/benches/encode.rs b/benches/encode.rs index 5de3ef2c..c4fc445e 100644 --- a/benches/encode.rs +++ b/benches/encode.rs @@ -128,15 +128,11 @@ fn run_encode_test( }) .with_frame_rate(60, 1) .with_gop_size(120) - .with_b_frames(0) .with_encode_content_hint(pixelforge::EncodeContentHint::Rendered) .with_encode_usage_hint(pixelforge::EncodeUsageHint::Streaming) .with_encoder_tuning_mode(tuning_mode); - println!( - "Config: {:?}, GOP={}, B-frames={}", - config.rate_control, config.gop_size, config.b_frame_count - ); + println!("Config: {:?}, GOP={}", config.rate_control, config.gop_size); // Create input image for uploading frames. let mut input_image = InputImage::new( diff --git a/examples/encode.rs b/examples/encode.rs index 0ec2a5ad..b61231f8 100644 --- a/examples/encode.rs +++ b/examples/encode.rs @@ -94,12 +94,11 @@ fn main() -> Result<(), Box> { let config = config .with_rate_control(RateControl::Cqp { qp: 26 }) .with_frame_rate(30, 1) - .with_gop_size(30) - .with_b_frames(0); + .with_gop_size(30); println!( - "Config: {:?}, GOP={}, B-frames={}\n", - config.rate_control, config.gop_size, config.b_frame_count + "Config: {:?}, GOP={}\n", + config.rate_control, config.gop_size ); // Create input image for uploading frames. diff --git a/src/converter/device_tests.rs b/src/converter/device_tests.rs index 5fcbad5e..cd3a595c 100644 --- a/src/converter/device_tests.rs +++ b/src/converter/device_tests.rs @@ -189,16 +189,13 @@ fn encoders( ) -> Result<(Encoder, Option), Box> { let eight = Encoder::new( context.clone(), - EncodeConfig::h264(WIDTH, HEIGHT) - .with_frame_rate(30, 1) - .with_b_frames(0), + EncodeConfig::h264(WIDTH, HEIGHT).with_frame_rate(30, 1), )?; let ten = Encoder::new( context.clone(), EncodeConfig::h265(WIDTH, HEIGHT) .with_bit_depth(EncodeBitDepth::Ten) - .with_frame_rate(30, 1) - .with_b_frames(0), + .with_frame_rate(30, 1), ) .ok(); if ten.is_none() { @@ -525,7 +522,6 @@ fn encode_config(codec: Codec) -> EncodeConfig { } .with_rate_control(RateControl::Cqp { qp: 22 }) .with_frame_rate(30, 1) - .with_b_frames(0) .with_color(StreamColor::default()) } diff --git a/src/encoder/av1/init.rs b/src/encoder/av1/init.rs index b5e1a454..2aa24493 100644 --- a/src/encoder/av1/init.rs +++ b/src/encoder/av1/init.rs @@ -7,7 +7,7 @@ use crate::encoder::codec::{ }; use crate::encoder::profile::EncodeProfile; use crate::encoder::rgb::RgbConversionCaps; -use crate::error::{PixelForgeError, Result}; +use crate::error::Result; use crate::vulkan::VideoContext; use ash::vk; use ash::vk::TaggedStructure; @@ -20,15 +20,6 @@ impl Av1 { config: EncodeConfig, rgb_input: Option, ) -> Result> { - // Reachable from safe caller code via `with_b_frames`, so report it - // rather than panicking inside the library. - if config.b_frame_count != 0 { - return Err(PixelForgeError::InvalidInput(format!( - "AV1 encode: B-frames are not yet supported; set b_frame_count = 0 (got {})", - config.b_frame_count - ))); - } - info!( "Creating AV1 encoder: {}x{}, pixel_format={:?}", config.dimensions.width, config.dimensions.height, config.pixel_format diff --git a/src/encoder/h264/init.rs b/src/encoder/h264/init.rs index e22bbb1e..13b33291 100644 --- a/src/encoder/h264/init.rs +++ b/src/encoder/h264/init.rs @@ -10,7 +10,7 @@ use crate::encoder::profile::EncodeProfile; use crate::encoder::profile::h264_profile_idc; use crate::encoder::resources::MIN_BITSTREAM_BUFFER_SIZE; use crate::encoder::rgb::RgbConversionCaps; -use crate::error::{PixelForgeError, Result}; +use crate::error::Result; use crate::vulkan::VideoContext; use ash::vk; use ash::vk::TaggedStructure; @@ -23,16 +23,6 @@ impl H264 { config: EncodeConfig, rgb_input: Option, ) -> Result> { - // B-frames are not yet supported. `with_b_frames` is public and accepts - // any count, so this is reachable from safe caller code: report it - // rather than panicking inside the library. - if config.b_frame_count != 0 { - return Err(PixelForgeError::InvalidInput(format!( - "H.264 encode: B-frames are not yet supported; set b_frame_count = 0 (got {})", - config.b_frame_count - ))); - } - info!( "Creating H.264 encoder: {}x{}, pixel_format={:?}", config.dimensions.width, config.dimensions.height, config.pixel_format diff --git a/src/encoder/h265/init.rs b/src/encoder/h265/init.rs index ed8784f9..ec582a74 100644 --- a/src/encoder/h265/init.rs +++ b/src/encoder/h265/init.rs @@ -23,15 +23,6 @@ impl H265 { config: EncodeConfig, rgb_input: Option, ) -> Result> { - // Reachable from safe caller code via `with_b_frames`, so report it - // rather than panicking inside the library. - if config.b_frame_count != 0 { - return Err(PixelForgeError::InvalidInput(format!( - "H.265 encode: B-frames are not yet supported; set b_frame_count = 0 (got {})", - config.b_frame_count - ))); - } - info!( "Creating H.265 encoder: {}x{}, pixel_format={:?}", config.dimensions.width, config.dimensions.height, config.pixel_format diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index 7d8032d6..a1060c22 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -410,8 +410,9 @@ pub struct EncodeConfig { pub frame_rate_denominator: u32, /// GOP size (distance between IDR frames). pub gop_size: u32, - /// Number of consecutive B-frames. - pub b_frame_count: u32, + /// Number of consecutive B-frames. Always 0: no codec encodes B-frames + /// yet, so there is no setter. + pub(crate) b_frame_count: u32, /// Maximum number of reference frames. pub max_reference_frames: u32, /// VBV/HRD virtual buffer size in milliseconds, for @@ -557,12 +558,6 @@ impl EncodeConfig { self } - /// Set the number of B-frames. - pub fn with_b_frames(mut self, count: u32) -> Self { - self.b_frame_count = count; - self - } - /// Set the maximum reference frames. pub fn with_max_reference_frames(mut self, count: u32) -> Self { self.max_reference_frames = count; @@ -857,13 +852,6 @@ mod tests { assert_eq!(config.gop_size, 60); } - #[test] - fn test_with_b_frames() { - let config = EncodeConfig::h264(1920, 1080).with_b_frames(2); - - assert_eq!(config.b_frame_count, 2); - } - #[test] fn test_with_max_reference_frames() { let config = EncodeConfig::h264(1920, 1080).with_max_reference_frames(8); @@ -911,7 +899,6 @@ mod tests { max: 10_000_000, }) .with_gop_size(120) - .with_b_frames(2) .with_frame_rate(60, 1) .with_pixel_format(PixelFormat::Yuv420) .with_bit_depth(BitDepth::Eight); @@ -924,7 +911,6 @@ mod tests { } ); assert_eq!(config.gop_size, 120); - assert_eq!(config.b_frame_count, 2); assert_eq!(config.frame_rate_numerator, 60); } } diff --git a/src/lib.rs b/src/lib.rs index 8b43308d..d3642927 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,7 +24,7 @@ //! for it, read with no copy where the driver allows, with sync_file fences //! (Linux only). //! -//! > **Note**: B-frame support is not yet implemented. Setting `b_frame_count > 0` will panic. +//! > **Note**: B-frame encoding is not yet implemented, so there is no option for it. //! //! # Supported Codecs //! diff --git a/tests/adopted.rs b/tests/adopted.rs index 9661027c..f79cf274 100644 --- a/tests/adopted.rs +++ b/tests/adopted.rs @@ -250,7 +250,6 @@ fn encode_all(context: &VideoContext, wait: Wait) -> Result, Box EncodeConfig { Codec::AV1 => EncodeConfig::av1(width, height), } .with_rate_control(RateControl::Cqp { qp: 22 }) - .with_b_frames(0) } /// An NV12 frame that moves with `n`. diff --git a/tests/rfi.rs b/tests/rfi.rs index 1be993e1..14d5f7fc 100644 --- a/tests/rfi.rs +++ b/tests/rfi.rs @@ -107,8 +107,7 @@ fn run_codec( .with_pixel_format(PixelFormat::Yuv420) .with_bit_depth(EncodeBitDepth::Eight) // Infinite GOP so the recovery can only come from RFI, never a periodic IDR. - .with_gop_size(0) - .with_b_frames(0); + .with_gop_size(0); let mut encoder = Encoder::new(context.clone(), config)?; let mut input_image = InputImage::new( diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs index 897f763c..c79ff32f 100644 --- a/tests/rgb_encode.rs +++ b/tests/rgb_encode.rs @@ -62,7 +62,6 @@ fn encode_config(codec: Codec) -> EncodeConfig { config .with_rate_control(RateControl::Cqp { qp: 22 }) .with_frame_rate(30, 1) - .with_b_frames(0) .with_color(StreamColor::default()) } diff --git a/tests/roundtrip.rs b/tests/roundtrip.rs index 2a34d17d..b26622be 100644 --- a/tests/roundtrip.rs +++ b/tests/roundtrip.rs @@ -28,8 +28,6 @@ use pixelforge::{ const WIDTH: u32 = 320; const HEIGHT: u32 = 240; -/// The encoder does not support B-frames yet, so this is an I-P GOP. -const B_FRAMES: u32 = 0; const GOP: u32 = 30; #[test] @@ -59,8 +57,7 @@ fn roundtrip_h264() -> Result<(), Box> { let config = EncodeConfig::h264(WIDTH, HEIGHT) .with_rate_control(RateControl::Cqp { qp: 26 }) .with_frame_rate(30, 1) - .with_gop_size(GOP) - .with_b_frames(B_FRAMES); + .with_gop_size(GOP); let mut input_image = InputImage::new( context.clone(), @@ -89,7 +86,7 @@ fn roundtrip_h264() -> Result<(), Box> { } File::create(&bitstream_path)?.write_all(&bitstream)?; println!( - "Encoded {} frames -> {} bytes (b_frames={B_FRAMES}, gop={GOP})", + "Encoded {} frames -> {} bytes (gop={GOP})", frame_count, bitstream.len() ); From dc547c9a656216046a747826b5742e7097eff1e3 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 08:55:27 +0100 Subject: [PATCH 09/19] feat(error): report a lost device as PixelForgeError::DeviceLost --- CHANGELOG.md | 4 +++ src/converter/mod.rs | 4 +-- src/decoder/pipeline.rs | 9 ++++--- src/encoder/resources.rs | 22 +++++++++++------ src/encoder/source.rs | 11 +++++++-- src/error.rs | 53 ++++++++++++++++++++++++++++++++++++++-- src/image.rs | 8 ++++-- src/vulkan.rs | 2 +- 8 files changed, 93 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a86800fa..f5605b6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 can, and copies it otherwise. - Add sync_file fences (feature `dmabuf`): `Frame::wait_sync_file`, `Frame::signal_sync_file` and `EncodeFuture::take_release_sync_file`. +- Add `PixelForgeError::DeviceLost`: a submit or wait that finds the device + lost reports it as such, so a caller knows to make a new context. - Add `VideoContext::supports_rgb_encode_input`, `VideoContext::has_video_encode_rgb_conversion` and `DeviceFeatures::video_encode_rgb_conversion`. @@ -52,6 +54,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and a peak below the target. `RateControlMode::Disabled` was the same as `Cqp` and is gone. `DEFAULT_TARGET_BITRATE` and `DEFAULT_MAX_BITRATE` are removed; `DEFAULT_AV1_Q_INDEX` is added. +- An RGB conversion the device cannot do is `PixelForgeError::NoSuitableDevice`, + as other missing device support is, rather than `InvalidInput`. - **Breaking:** `EncodeConfig::with_b_frames` and the public `b_frame_count` field are removed. No codec encodes B-frames yet, and every encoder refused a count above 0 at runtime. diff --git a/src/converter/mod.rs b/src/converter/mod.rs index 1c7ff887..dcad159d 100644 --- a/src/converter/mod.rs +++ b/src/converter/mod.rs @@ -920,7 +920,7 @@ impl ColorConverter { self.context .sync2() .queue_submit2(self.context.compute_queue(), &[submit_info], self.fence) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(|e| PixelForgeError::vk(e, PixelForgeError::CommandBuffer))?; } self.timeline.commit(); self.in_flight = true; @@ -938,7 +938,7 @@ impl ColorConverter { .device() .wait_for_fences(&[self.fence], true, u64::MAX) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(|e| PixelForgeError::vk(e, PixelForgeError::CommandBuffer))?; self.in_flight = false; } self.destroy_src_view(); diff --git a/src/decoder/pipeline.rs b/src/decoder/pipeline.rs index ef22383e..93c5a24e 100644 --- a/src/decoder/pipeline.rs +++ b/src/decoder/pipeline.rs @@ -288,7 +288,7 @@ impl DecodePipeline { unsafe { sync2 .queue_submit2(decode_queue, &[submit], slot.decode_fence) - .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; + .map_err(|e| PixelForgeError::vk(e, PixelForgeError::Synchronization))?; } self.decodes.commit(); @@ -344,7 +344,7 @@ impl DecodePipeline { unsafe { sync2 .queue_submit2(transfer_queue, &[submit], slot.transfer_fence) - .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; + .map_err(|e| PixelForgeError::vk(e, PixelForgeError::Synchronization))?; } self.copies.commit(); @@ -451,7 +451,10 @@ fn run_completion_thread( if let Err(e) = wait && let Some(tx) = &frames_tx { - let _ = tx.unbounded_send(Err(PixelForgeError::Synchronization(e.to_string()))); + let _ = tx.unbounded_send(Err(PixelForgeError::vk( + e, + PixelForgeError::Synchronization, + ))); } } diff --git a/src/encoder/resources.rs b/src/encoder/resources.rs index a3555bfc..f0434a5a 100644 --- a/src/encoder/resources.rs +++ b/src/encoder/resources.rs @@ -394,10 +394,16 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam vk::SubmitInfo::default().command_buffers(std::slice::from_ref(¶ms.command_buffer)); unsafe { device.reset_fences(&[params.fence]) } .map_err(|e| PixelForgeError::CommandBuffer(format!("reset fence: {}", e)))?; - unsafe { device.queue_submit(params.queue, &[submit_info], params.fence) } - .map_err(|e| PixelForgeError::CommandBuffer(format!("submit clear: {}", e)))?; - unsafe { device.wait_for_fences(&[params.fence], true, u64::MAX) } - .map_err(|e| PixelForgeError::CommandBuffer(format!("wait clear: {}", e)))?; + unsafe { device.queue_submit(params.queue, &[submit_info], params.fence) }.map_err(|e| { + PixelForgeError::vk(e, |e| { + PixelForgeError::CommandBuffer(format!("submit clear: {e}")) + }) + })?; + unsafe { device.wait_for_fences(&[params.fence], true, u64::MAX) }.map_err(|e| { + PixelForgeError::vk(e, |e| { + PixelForgeError::CommandBuffer(format!("wait clear: {e}")) + }) + })?; unsafe { device.reset_fences(&[params.fence]) } .map_err(|e| PixelForgeError::CommandBuffer(format!("reset fence after clear: {}", e)))?; @@ -623,7 +629,7 @@ pub(crate) fn upload_image_to_input( .sync2() .queue_submit2(params.upload_queue, &[submit_info], vk::Fence::null()) } - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(|e| PixelForgeError::vk(e, PixelForgeError::CommandBuffer))?; Ok(()) } @@ -846,7 +852,7 @@ pub(crate) unsafe fn submit_encode_only( unsafe { sync2 .queue_submit2(encode_queue, &[submit_info], fence) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(|e| PixelForgeError::vk(e, PixelForgeError::CommandBuffer))?; } Ok(()) @@ -876,7 +882,7 @@ pub(crate) unsafe fn wait_and_read_bitstream( unsafe { device .wait_for_fences(&[fence], true, u64::MAX) - .map_err(|e| PixelForgeError::CommandBuffer(e.to_string()))?; + .map_err(|e| PixelForgeError::vk(e, PixelForgeError::CommandBuffer))?; } // Read query results (offset + bytes_written). @@ -899,7 +905,7 @@ pub(crate) unsafe fn wait_and_read_bitstream( &mut query_results, vk::QueryResultFlags::WAIT, ) - .map_err(|e| PixelForgeError::QueryPool(e.to_string()))?; + .map_err(|e| PixelForgeError::vk(e, PixelForgeError::QueryPool))?; } let offset = query_results[0].offset as usize; diff --git a/src/encoder/source.rs b/src/encoder/source.rs index d5f4a042..a5019700 100644 --- a/src/encoder/source.rs +++ b/src/encoder/source.rs @@ -171,13 +171,20 @@ fn encoder_conversion( rgb: &RgbSource, target: ColorSpec, ) -> Result<()> { + // What no device can do is the configuration's fault; what this one + // cannot is the device's. let no = |reason: String| { Err(PixelForgeError::InvalidInput(format!( "RGB input: {reason}" ))) }; + let not_here = |reason: String| { + Err(PixelForgeError::NoSuitableDevice(format!( + "RGB input: {reason}" + ))) + }; if !context.has_video_encode_rgb_conversion() { - return no("the device has no VK_VALVE_video_encode_rgb_conversion".to_string()); + return not_here("the device has no VK_VALVE_video_encode_rgb_conversion".to_string()); } if rgb.color != target { return no(format!( @@ -196,7 +203,7 @@ fn encoder_conversion( None => return no(format!("{:?} needs tone mapping first", rgb.format)), } if !context.supports_rgb_encode_input(config.codec, rgb.format, config.color) { - return no(format!( + return not_here(format!( "the driver does not convert {:?} to {:?} for {:?}", rgb.format, config.color, config.codec )); diff --git a/src/error.rs b/src/error.rs index b5cc6738..b928f169 100644 --- a/src/error.rs +++ b/src/error.rs @@ -9,10 +9,19 @@ pub enum PixelForgeError { #[error("Failed to create Vulkan instance: {0}")] InstanceCreation(String), - /// Vulkan physical device selection failed. + /// The device cannot do what was asked: it lacks a queue, an extension, + /// a feature, a format or a profile. The same request can work on another + /// device, or with another configuration on this one; nothing is wrong + /// with the device itself. #[error("No suitable Vulkan physical device found: {0}")] NoSuitableDevice(String), + /// The device was lost: a driver reset, a GPU hang, or the GPU going + /// away. Nothing made from this context works any more. Drop the + /// encoders, decoders and the context, and make a new context. + #[error("The Vulkan device was lost")] + DeviceLost, + /// Vulkan logical device creation failed. #[error("Failed to create Vulkan device: {0}")] DeviceCreation(String), @@ -69,9 +78,49 @@ pub enum PixelForgeError { impl From for PixelForgeError { fn from(result: ash::vk::Result) -> Self { - PixelForgeError::Vulkan(result) + Self::vk(result, |_| PixelForgeError::Vulkan(result)) + } +} + +impl PixelForgeError { + /// `wrap` the description of a failed Vulkan call, except a lost device, + /// which is always [`Self::DeviceLost`]. + pub(crate) fn vk(result: ash::vk::Result, wrap: impl FnOnce(String) -> Self) -> Self { + if result == ash::vk::Result::ERROR_DEVICE_LOST { + Self::DeviceLost + } else { + wrap(result.to_string()) + } } } /// Result type for PixelForge operations. pub type Result = std::result::Result; + +#[cfg(test)] +mod tests { + use super::*; + use ash::vk; + + #[test] + fn a_lost_device_is_always_device_lost() { + assert!(matches!( + PixelForgeError::vk( + vk::Result::ERROR_DEVICE_LOST, + PixelForgeError::CommandBuffer + ), + PixelForgeError::DeviceLost + )); + assert!(matches!( + PixelForgeError::from(vk::Result::ERROR_DEVICE_LOST), + PixelForgeError::DeviceLost + )); + assert!(matches!( + PixelForgeError::vk( + vk::Result::ERROR_OUT_OF_DEVICE_MEMORY, + PixelForgeError::CommandBuffer + ), + PixelForgeError::CommandBuffer(_) + )); + } +} diff --git a/src/image.rs b/src/image.rs index 769ad192..3f7fabd9 100644 --- a/src/image.rs +++ b/src/image.rs @@ -842,10 +842,14 @@ impl InputImage { vk::SubmitInfo::default().command_buffers(std::slice::from_ref(&self.command_buffer)); unsafe { device.queue_submit(self.context.transfer_queue(), &[submit_info], self.fence) } - .map_err(|e| PixelForgeError::CommandBuffer(format!("Queue submit failed: {}", e)))?; + .map_err(|e| { + PixelForgeError::vk(e, |e| { + PixelForgeError::CommandBuffer(format!("Queue submit failed: {e}")) + }) + })?; unsafe { device.wait_for_fences(&[self.fence], true, u64::MAX) } - .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; + .map_err(|e| PixelForgeError::vk(e, PixelForgeError::Synchronization))?; unsafe { device.reset_fences(&[self.fence]) } .map_err(|e| PixelForgeError::Synchronization(e.to_string()))?; diff --git a/src/vulkan.rs b/src/vulkan.rs index bb8ff5bb..97056ee9 100644 --- a/src/vulkan.rs +++ b/src/vulkan.rs @@ -769,7 +769,7 @@ impl VideoContext { .semaphores(&semaphores) .values(&values); unsafe { self.inner.timeline.wait_semaphores(&info, u64::MAX) } - .map_err(|e| PixelForgeError::Synchronization(e.to_string())) + .map_err(|e| PixelForgeError::vk(e, PixelForgeError::Synchronization)) } /// Hand over a semaphore to be destroyed with the context rather than now. From 9ec0718c54c8f3841306ae6bc0c619e65073ae5e Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 09:16:30 +0100 Subject: [PATCH 10/19] feat(encoder)!: change the source and stream colour together with Encoder::reconfigure --- AGENTS.md | 7 ++++ CHANGELOG.md | 9 +++-- README.md | 2 +- docs/encode-input.md | 27 ++++++++----- src/converter/mod.rs | 48 ++++++++++++----------- src/encoder/api.rs | 84 +++++++++++++++++++++++------------------ src/encoder/codec.rs | 2 +- src/encoder/external.rs | 4 +- src/lib.rs | 2 +- tests/common/source.rs | 14 ++++++- tests/rgb_encode.rs | 66 ++++++++++++++++++++++++++++++-- 11 files changed, 184 insertions(+), 81 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9d909790..1c30eef8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -158,6 +158,13 @@ see a passing test's. Without `VK_LAYER_KHRONOS_validation` installed, pixelforge logs a warning and carries on with validation disabled, so absence of errors means nothing if the layer is missing. +On NVIDIA (615.78.08) a decode submit under validation crashes inside the +driver (`vkQueueSubmit2` from `DecodePipeline::submit_decode`), so every test +that decodes (`roundtrip`, `adopted`, and the `rgb_encode` tests that compare +decoded frames) segfaults there with `PIXELFORGE_VALIDATION=1`. Run those +without validation on that driver, and check their encode half under +validation separately. + Run the ignored tests once without validation as well. The layer gives every object a unique handle, while drivers reuse the handles of destroyed objects freely, so anything that remembers state by handle can pass with validation diff --git a/CHANGELOG.md b/CHANGELOG.md index f5605b6a..478335a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,13 +64,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 frames that read the image are done with it. - **Breaking:** `StreamColor` replaces the raw `ColorDescription` in the encode API: `EncodeConfig::with_color` replaces `with_color_description`, - `Encoder::set_color` replaces `set_color_description`, and + `Encoder::reconfigure` replaces `set_color_description`, and `VideoContext::supports_rgb_encode_input` takes a `StreamColor`. `Sdr` and `Hdr10` name the two colours an RGB source converts to, with their range; `Other` carries any H.273 description, for a YUV source only. -- `Encoder::set_color` also updates the conversion of an RGB source, and - makes the encoder again where the encoder converts. Packet timestamps carry - on. +- `Encoder::reconfigure(source, color)` changes the source and the stream + colour together, so the conversion follows both: the colour converter + switches in place, and where the encoder converts it is made again. Packet + timestamps carry on. - The `dmabuf` feature enables its extensions only where the device has them, rather than failing device creation where it does not. diff --git a/README.md b/README.md index e422a2ed..26284fdf 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ An encoder takes RGB frames once it is told what they are, with a [`StreamColor::Hdr10`] (BT.2020 with PQ), each with its range. The source and the stream are described once each, so the conversion and the stream's label cannot disagree, and a colour change through -[`Encoder::set_color`] changes both. +[`Encoder::reconfigure`] changes both. ```rust use pixelforge::{ diff --git a/docs/encode-input.md b/docs/encode-input.md index 4251a495..84581ea4 100644 --- a/docs/encode-input.md +++ b/docs/encode-input.md @@ -52,15 +52,24 @@ was selected. The colour converter is not public. An RGB source is the only way to use it, so its target and range always follow the stream's colour. -### Colour changes - -`Encoder::set_color(StreamColor)` is the one way to change colour: - -- `None` and `Shader`: the session parameters are made again (as before), and - the shader gets the new target and range. -- `Encoder`: the matrix and range are fixed in the video session, so the codec - encoder is made again, and the path is selected again. Packet timestamps - continue from where they were. The next frame is an IDR in all cases. +### Source and colour changes + +`Encoder::reconfigure(Source, StreamColor)` is the one way to change the +source or the colour. Both decide the conversion, so they change together: +there is no moment where a new source meets the old colour, a pair the +conversion may not do. A call that changes nothing does nothing, so a caller +whose frames change format or colour space (a compositor) calls it for every +frame. + +- `None` kept: the session parameters are made again for a new colour. +- `Shader` kept: the converter switches in place. Its input format, source, + target and range are all push constants, and its source view is made per + conversion. The session parameters are made again for a new colour. +- `Encoder`, or a change of conversion: the input format, matrix and range + are fixed in the video session, so the codec encoder is made again, and the + path is selected again. Packet timestamps continue from where they were. + +A new colour makes the next frame an IDR in all cases. This removes the bug where the label changed and the hardware conversion did not. diff --git a/src/converter/mod.rs b/src/converter/mod.rs index dcad159d..417a56ac 100644 --- a/src/converter/mod.rs +++ b/src/converter/mod.rs @@ -382,31 +382,35 @@ impl ColorConverter { self.output_buffer } - /// Change what the encoded stream should be, for subsequent conversions. - /// - /// Takes effect on the next conversion without recreating the pipeline, - /// since the target is passed via push constants. - /// - /// Returns an error, and leaves the converter untouched, if the configured - /// source cannot reach the new target. - pub fn set_target(&mut self, target: ColorSpec) -> Result<()> { - let candidate = ColorConverterConfig { - target, - ..self.config.clone() - }; - if !candidate.conversion_supported() { - return Err(candidate.unsupported_conversion()); - } - self.config.target = target; - Ok(()) + /// Whether [`Self::reconfigure`] can switch this converter to `config`: + /// the same size and output, which its buffers are made for. + pub fn can_become(&self, config: &ColorConverterConfig) -> bool { + (config.width, config.height, config.output_format) + == ( + self.config.width, + self.config.height, + self.config.output_format, + ) } - /// Change the quantization range for subsequent conversions. + /// Switch to `config` for subsequent conversions: its input format, + /// source, target, range and reference white. They all reach the shader + /// as push constants, and the source view is made per conversion, so + /// nothing is made again. /// - /// Takes effect on the next conversion without recreating the pipeline, - /// since the range is passed via push constants. - pub fn set_range(&mut self, range: ColorRange) { - self.config.range = range; + /// Returns an error, and leaves the converter untouched, if the shader + /// cannot do that conversion or [`Self::can_become`] is false. + pub fn reconfigure(&mut self, config: ColorConverterConfig) -> Result<()> { + if !self.can_become(&config) { + return Err(PixelForgeError::InvalidInput( + "a converter cannot change its size or output format".to_string(), + )); + } + if !config.conversion_supported() { + return Err(config.unsupported_conversion()); + } + self.config = config; + Ok(()) } /// Build buffer-to-image copy regions for multi-planar YUV formats. diff --git a/src/encoder/api.rs b/src/encoder/api.rs index 1707a2dd..3b8d56a2 100644 --- a/src/encoder/api.rs +++ b/src/encoder/api.rs @@ -6,7 +6,7 @@ use ash::vk; use super::codec::{CodecEncoder, EncoderCommon, VideoCodec}; use super::frame::{Frame, FrameSource, InputSlot}; use super::input::{FrameInput, FrameSync, SourceOwnership}; -use super::source::{Conversion, ConversionPreference, plan, target_space}; +use super::source::{Conversion, Source, plan}; use super::source_image::SourceImage; use super::{Codec, EncodeConfig, EncodeFuture, StreamColor}; use crate::converter::ColorConverter; @@ -371,52 +371,63 @@ impl Encoder { self.inner.invalidate_reference_frames(local) } - /// Change what the stream says about its colours, and what the encoder - /// makes its YUV to match. + /// Change what the frames are and what the stream says about its + /// colours, together. /// - /// The next frame is an IDR/key frame carrying the new header. For an RGB - /// source the conversion follows: the colour converter gets the new target - /// and range, and where the encoder converts, it is made again with the - /// new matrix and range, since those are fixed in its video session. - /// Packet timestamps carry on across that. If the new colour needs a - /// different [`Conversion`], images imported or allocated for the old one - /// have to be made again. + /// Both decide the conversion, so they change in one call: there is no + /// moment where the new source meets the old colour, a pair the + /// conversion may not be able to do. A call that changes nothing does + /// nothing, so it is cheap to make for every frame. /// - /// On error the encoder is left as it was. - pub fn set_color(&mut self, color: StreamColor) -> Result<()> { + /// A new colour makes the next frame an IDR/key frame carrying the new + /// header. The conversion follows: + /// + /// - with [`Conversion::Shader`] kept, the colour converter switches in + /// place; + /// - where the encoder converts, or the conversion changes, the codec + /// encoder is made again, because its video session fixes the input + /// format, matrix and range. Packet timestamps carry on across that. + /// + /// An `ExternalImage` made before a change of source or conversion has to + /// be made again. On error the encoder is left as it was. + pub fn reconfigure(&mut self, source: Source, color: StreamColor) -> Result<()> { + if (source, color) == (self.config.source, self.config.color) { + return Ok(()); + } let mut config = self.config.clone(); + config.source = source; config.color = color; - match self.conversion { - Conversion::None => self.inner.set_color(color)?, - Conversion::Shader if self.shader_still_converts(&config) => { - let (target, range) = target_space(&color).expect("the plan checked the target"); + let next = plan(&self.context, &config)?; + match (self.conversion, next.conversion, next.converter) { + (Conversion::None, Conversion::None, _) => self.inner.set_color(color)?, + (Conversion::Shader, Conversion::Shader, Some(next_converter)) + if self + .converter + .as_ref() + .is_some_and(|c| c.can_become(&next_converter)) => + { let converter = self .converter .as_mut() .expect("a shader conversion has a converter"); - let (old_target, old_range) = (converter.config().target, converter.config().range); - converter.set_target(target)?; - converter.set_range(range); - if let Err(e) = self.inner.set_color(color) { - let _ = converter.set_target(old_target); - converter.set_range(old_range); + let old = converter.config().clone(); + converter.reconfigure(next_converter)?; + if color != self.config.color + && let Err(e) = self.inner.set_color(color) + { + let _ = converter.reconfigure(old); return Err(e); } + if source != self.config.source { + self.generation += 1; + } } - Conversion::Shader | Conversion::Encoder => self.rebuild(config.clone())?, + _ => self.rebuild(config.clone())?, } self.config = config; Ok(()) } - /// Whether the colour converter this encoder has can produce what - /// `config` describes, so the encoder can keep it. - fn shader_still_converts(&self, config: &EncodeConfig) -> bool { - let mut probe = config.clone(); - probe.conversion = ConversionPreference::Shader; - plan(&self.context, &probe).is_ok() - } - /// Make the codec encoder again for `config`, keeping the timestamps /// going. fn rebuild(&mut self, config: EncodeConfig) -> Result<()> { @@ -427,11 +438,12 @@ impl Encoder { next.id = self.id; // A slot taken before is not a slot of the new encoder. next.ticket = self.ticket + 1; - next.generation = if next.conversion == self.conversion { - self.generation - } else { - self.generation + 1 - }; + next.generation = + if (next.conversion, next.config.source) == (self.conversion, self.config.source) { + self.generation + } else { + self.generation + 1 + }; #[cfg(feature = "dmabuf")] { next.sync_files = self.sync_files.take(); diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 144ebcc7..15f75cf6 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -2,7 +2,7 @@ //! //! Everything that is identical across H.264, H.265 and AV1 lives here: //! the shared per-encoder state ([`EncoderCommon`]), the generic driver -//! ([`CodecEncoder`]) that owns the public `submit`/`flush`/`set_color` +//! ([`CodecEncoder`]) that owns the public `submit`/`flush`/`reconfigure` //! flow, the shared initialization scaffolding ([`build_encoder_common`]) and the //! shared rate-control decision ([`RateControlPlan`]). //! diff --git a/src/encoder/external.rs b/src/encoder/external.rs index 04f9193b..7cbbbe81 100644 --- a/src/encoder/external.rs +++ b/src/encoder/external.rs @@ -895,8 +895,8 @@ impl Encoder { ) -> Result { if external.owner != (self.id(), self.generation()) { return Err(PixelForgeError::InvalidInput( - "this ExternalImage was made by another encoder, or before a colour change \ - changed the conversion; import or allocate it again" + "this ExternalImage was made by another encoder, or before a reconfigure \ + changed the source or the conversion; import or allocate it again" .to_string(), )); } diff --git a/src/lib.rs b/src/lib.rs index d3642927..b3c62f3a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -195,7 +195,7 @@ //! [`StreamColor::Hdr10`] (BT.2020 with PQ), each with its range. The source //! and the stream are described once each, so the conversion and the //! stream's label cannot disagree, and a colour change through -//! [`Encoder::set_color`] changes both. +//! [`Encoder::reconfigure`] changes both. //! //! ```rust,no_run //! use pixelforge::{ diff --git a/tests/common/source.rs b/tests/common/source.rs index ecfa0c28..e140843e 100644 --- a/tests/common/source.rs +++ b/tests/common/source.rs @@ -100,11 +100,23 @@ pub unsafe fn create_src_image( width: u32, height: u32, pixels: &[u8], +) -> Result> { + unsafe { create_src_image_as(context, width, height, vk::Format::B8G8R8A8_UNORM, pixels) } +} + +/// [`create_src_image`] for `pixels` in `format`, any format of 4 bytes per +/// pixel. +pub unsafe fn create_src_image_as( + context: &VideoContext, + width: u32, + height: u32, + format: vk::Format, + pixels: &[u8], ) -> Result> { let device = context.device(); let info = vk::ImageCreateInfo::default() .image_type(vk::ImageType::TYPE_2D) - .format(vk::Format::B8G8R8A8_UNORM) + .format(format) .extent(vk::Extent3D { width, height, diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs index c79ff32f..c4b41db0 100644 --- a/tests/rgb_encode.rs +++ b/tests/rgb_encode.rs @@ -10,7 +10,7 @@ #[allow(dead_code)] mod common; -use common::source::{SrcImage, create_src_image}; +use common::source::{SrcImage, create_src_image, create_src_image_as}; use common::{Readback, decode_stream}; use ash::vk; @@ -319,9 +319,12 @@ fn color_change_follows_into_the_conversion() -> Result<(), Box Result<(), Box {:?}", encoder.conversion()); Ok(()) } + +/// A source that changes its pixel layout mid-stream is read in its new +/// layout from the next frame on: the shader switches in place, with no new +/// key frame, and the picture stays the same. +#[test] +#[ignore = "requires a Vulkan Video device"] +fn a_format_change_switches_the_shader_in_place() -> Result<(), Box> { + let context = context(Codec::H264)?; + let reference = encode_rgb( + &context, + Codec::H264, + InputFormat::BGRA, + ConversionPreference::Shader, + )?; + let mut encoder = Encoder::new( + context.clone(), + encode_config(Codec::H264) + .with_source(rgb(InputFormat::BGRA)) + .with_conversion(ConversionPreference::Shader), + )?; + let mut stream = Vec::new(); + let mut keys = Vec::new(); + for n in 0..FRAMES { + let mut pixels = moving_frame(n); + let mut format = vk::Format::B8G8R8A8_UNORM; + if n >= FRAMES / 2 { + encoder.reconfigure(rgb(InputFormat::RGBA), StreamColor::default())?; + for pixel in pixels.as_chunks_mut::<4>().0 { + pixel.swap(0, 2); + } + format = vk::Format::R8G8B8A8_UNORM; + } + let src = unsafe { create_src_image_as(&context, WIDTH, HEIGHT, format, &pixels)? }; + let image = source_image(&context, &src)?; + let packet = pollster::block_on(encoder.submit(Frame::image(&image))?)?; + keys.push(packet.is_key_frame); + stream.extend_from_slice(&packet.data); + } + assert_eq!(encoder.conversion(), Conversion::Shader); + assert!( + !keys[FRAMES as usize / 2], + "the format change made a key frame: {keys:?}" + ); + let expected = decode_all(&context, &reference)?; + let actual = decode_all(&context, &stream)?; + assert_eq!(actual.len(), FRAMES as usize, "frame count"); + for (i, (a, e)) in actual.iter().zip(&expected).enumerate() { + let p = psnr(a, e); + assert!( + p >= 40.0, + "frame {i}: {p:.2} dB against the unchanged source" + ); + } + Ok(()) +} From 3dfc229d9604973b89680b2ec0eefa6cb6c39473 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 09:31:43 +0100 Subject: [PATCH 11/19] feat(encoder): add Encoder::repeat to encode the last frame again --- AGENTS.md | 4 +-- CHANGELOG.md | 3 ++ docs/encode-input.md | 11 +++++++ src/encoder/api.rs | 62 ++++++++++++++++++++++++++++++++++ src/encoder/external.rs | 6 ++++ src/encoder/input.rs | 1 - src/encoder/pipeline.rs | 5 +++ tests/submit.rs | 73 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 162 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1c30eef8..b17c4068 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -160,8 +160,8 @@ of errors means nothing if the layer is missing. On NVIDIA (615.78.08) a decode submit under validation crashes inside the driver (`vkQueueSubmit2` from `DecodePipeline::submit_decode`), so every test -that decodes (`roundtrip`, `adopted`, and the `rgb_encode` tests that compare -decoded frames) segfaults there with `PIXELFORGE_VALIDATION=1`. Run those +that decodes (`roundtrip`, `adopted`, `submit`'s repeat test, and the +`rgb_encode` tests that compare decoded frames) segfaults there with `PIXELFORGE_VALIDATION=1`. Run those without validation on that driver, and check their encode half under validation separately. diff --git a/CHANGELOG.md b/CHANGELOG.md index 478335a9..9664768b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `Encoder::input_slot` and `InputSlot`: the encoder's own input image, to write a frame in place and submit with `Frame::input`. - Add `InputImage::source`. +- Add `Encoder::repeat`: encode the last frame again from the encoder's own + input image, for a key frame while no new frame arrives. The caller's + source does not have to stay alive for it. - Add DMA-BUF input (feature `dmabuf`): `Encoder::input_formats`, `Encoder::import`, `Encoder::allocate_input`, `ExternalImage` and `Frame::external`. The encoder reads a buffer directly where the device diff --git a/docs/encode-input.md b/docs/encode-input.md index 84581ea4..542fb88b 100644 --- a/docs/encode-input.md +++ b/docs/encode-input.md @@ -117,6 +117,17 @@ The handle keeps the release point of each frame that read the image. them on the CPU, and the drop waits for them too. `InputImage` holds one, so its uploads wait for the previous frames by themselves. +### Repeating the last frame + +`Encoder::repeat()` encodes the last frame again, for a key frame while no new +frame arrives. The frame is still in an input image (copied or converted +there), so the caller's source does not have to stay alive. The encoder keeps +which slot holds it. When that is the current slot, the encode reads it in +place. Otherwise it reads the other slot's image directly, and waits on the +CPU until it is done, because the next frame can be written into that image. +A frame that the encode read straight from a DMA-BUF is in no input image, so +it cannot be repeated. + ### No CPU waits The copy from a caller image into the input image signals a timeline diff --git a/src/encoder/api.rs b/src/encoder/api.rs index 3b8d56a2..87e072ff 100644 --- a/src/encoder/api.rs +++ b/src/encoder/api.rs @@ -74,6 +74,8 @@ pub struct Encoder { /// Counts submissions, so an [`InputSlot`] is submitted only before the /// next one. ticket: u64, + /// The slot whose input image holds the last frame, for [`Self::repeat`]. + last_frame_slot: Option, /// Counts the times the codec encoder was made again with a different /// conversion. An [`ExternalImage`](super::external::ExternalImage) made /// for one cannot be read by another. @@ -115,6 +117,7 @@ impl Encoder { pts_offset: 0, id: NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed), ticket: 0, + last_frame_slot: None, generation: 0, #[cfg(feature = "dmabuf")] sync_files: None, @@ -200,6 +203,12 @@ impl Encoder { )); } self.ticket += 1; + let fills_input_image = match &frame.source { + #[cfg(feature = "dmabuf")] + FrameSource::External(external) => external.fills_input_image(), + _ => true, + }; + let frame_slot = self.inner.common().pipeline.current_index(); #[cfg_attr(not(feature = "dmabuf"), allow(unused_mut))] let mut sync = FrameSync { @@ -234,6 +243,7 @@ impl Encoder { FrameSource::External(external) => self.submit_external(external, sync)?, }; future.pts_offset = self.pts_offset; + self.last_frame_slot = fills_input_image.then_some(frame_slot); #[cfg(feature = "dmabuf")] if frame.signal_sync_file @@ -302,6 +312,58 @@ impl Encoder { Ok(future) } + /// Encode the last frame again: an IDR if one was requested, for a client + /// that needs a key frame while no new frame arrives. + /// + /// The encoder still holds the last frame in an input image, so the + /// caller's source does not have to stay alive or unchanged. When that + /// image is another slot's, this waits on the CPU until the encode is + /// done, because the next frame can be written into it. + /// + /// An error if there is no last frame to repeat: nothing was submitted + /// since the encoder was made or reconfigured into a new encoder, or the + /// last frame was read straight from a DMA-BUF. + pub fn repeat(&mut self) -> Result { + let Some(last) = self.last_frame_slot else { + return Err(PixelForgeError::InvalidInput( + "no frame to repeat: none was submitted since the encoder was made, \ + or the last one was read straight from a DMA-BUF" + .to_string(), + )); + }; + self.ticket += 1; + let common = self.inner.common(); + // The encode writes nothing into its own slot's image, so the frame + // stays where it is for the next repeat. + let other_slot = last != common.pipeline.current_index(); + let input = if other_slot { + let slot = common.pipeline.slot(last); + FrameInput::Direct { + image: slot.input_image, + view: slot.input_image_view, + extent: vk::Extent2D { + width: common.aligned_width, + height: common.aligned_height, + }, + ownership: SourceOwnership { + old_layout: vk::ImageLayout::VIDEO_ENCODE_SRC_KHR, + acquire_from: None, + release_to: None, + final_layout: vk::ImageLayout::VIDEO_ENCODE_SRC_KHR, + }, + } + } else { + FrameInput::InPlace + }; + let mut future = self.inner.encode_frame(input, FrameSync::default())?; + future.pts_offset = self.pts_offset; + if other_slot { + // The next frame may be written into that slot's image. + self.context.wait_point(future.source_released())?; + } + Ok(future) + } + #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] pub(crate) fn inner(&self) -> &dyn EncoderApi { &*self.inner diff --git a/src/encoder/external.rs b/src/encoder/external.rs index 7cbbbe81..3f93469e 100644 --- a/src/encoder/external.rs +++ b/src/encoder/external.rs @@ -210,6 +210,12 @@ impl ExternalImage { self.access != Access::Copy } + /// Whether a frame from this image ends up in the encoder's input image, + /// copied or converted, rather than read where it is. + pub(crate) fn fills_input_image(&self) -> bool { + self.access != Access::Encode + } + /// The points after which every frame submitted so far is done with the /// buffer, so its producer can write it again. Empty before the first /// frame. For a producer that takes a sync_file instead, see diff --git a/src/encoder/input.rs b/src/encoder/input.rs index 4cee07e4..5613fab7 100644 --- a/src/encoder/input.rs +++ b/src/encoder/input.rs @@ -118,7 +118,6 @@ pub(crate) enum FrameInput { ownership: SourceOwnership, }, /// Encode straight from `view`, an image made with the session's profile. - #[cfg_attr(not(feature = "dmabuf"), allow(dead_code))] Direct { image: vk::Image, view: vk::ImageView, diff --git a/src/encoder/pipeline.rs b/src/encoder/pipeline.rs index f375e1f9..061038b5 100644 --- a/src/encoder/pipeline.rs +++ b/src/encoder/pipeline.rs @@ -504,6 +504,11 @@ impl EncodePipeline { }) } + /// The slot at `index`. + pub(crate) fn slot(&self, index: usize) -> &EncodeSlot { + &self.slots[index] + } + /// Index of the slot the next frame will be encoded into. pub(crate) fn current_index(&self) -> usize { self.current_slot diff --git a/tests/submit.rs b/tests/submit.rs index 2c6f6795..7375145c 100644 --- a/tests/submit.rs +++ b/tests/submit.rs @@ -92,3 +92,76 @@ fn a_shader_conversion_has_no_input_slot() -> Result<(), Box Result<(), Box> { + common::init_logging(); + let context = VideoContextBuilder::new() + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .require_encode(Codec::H264) + .require_decode(Codec::H264) + .build()?; + let mut encoder = Encoder::new(context.clone(), EncodeConfig::h264(WIDTH, HEIGHT))?; + assert!( + encoder.repeat().is_err(), + "repeated a frame that was never there" + ); + + let mut input = InputImage::new( + context.clone(), + Codec::H264, + WIDTH, + HEIGHT, + EncodeBitDepth::Eight, + PixelFormat::Yuv420, + )?; + let luma_levels = [40u8, 120, 200]; + let plane = (WIDTH * HEIGHT) as usize; + let mut stream = Vec::new(); + let mut keys = Vec::new(); + for level in luma_levels { + let mut yuv = vec![128u8; plane * 3 / 2]; + yuv[..plane].fill(level); + input.upload_yuv420(&yuv)?; + let packet = pollster::block_on(encoder.submit(Frame::image(input.source()))?)?; + keys.push(packet.is_key_frame); + stream.extend(packet.data); + } + // The source changes, and the repeats must not see it. + let mut yuv = vec![128u8; plane * 3 / 2]; + yuv[..plane].fill(10); + input.upload_yuv420(&yuv)?; + for _ in 0..2 { + encoder.request_idr(); + let packet = pollster::block_on(encoder.repeat()?)?; + keys.push(packet.is_key_frame); + stream.extend(packet.data); + } + assert_eq!(keys, [true, false, false, true, true]); + + let mut readback = common::Readback::new(&context)?; + let mut decoder = pixelforge::Decoder::new( + context.clone(), + pixelforge::DecodeConfig::h264().with_byte_stream(), + )?; + let mut means = Vec::new(); + common::decode_stream(&mut decoder, &stream, |frame| { + let y = readback.read(&frame)?.y; + means.push(y.iter().map(|&v| f64::from(v)).sum::() / y.len() as f64); + Ok(()) + })?; + println!("decoded luma means: {means:?}"); + assert_eq!(means.len(), 5, "frame count"); + for (i, mean) in means.iter().enumerate() { + let want = f64::from(luma_levels[i.min(2)]); + assert!( + (mean - want).abs() < 2.0, + "frame {i}: luma {mean:.1}, want {want}" + ); + } + Ok(()) +} From 8d4f85e20432e905475da3a79542836feae5d5c5 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 14:56:20 +0100 Subject: [PATCH 12/19] feat(encoder)!: always report packet stats, with the GPU time optional --- CHANGELOG.md | 5 +++++ benches/encode.rs | 9 +++++---- src/encoder/mod.rs | 13 +++++++------ src/encoder/pipeline.rs | 39 ++++++++++++++++++++++----------------- src/encoder/resources.rs | 33 ++++++++++++++++++++++++++++++++- src/vulkan.rs | 5 +++++ 6 files changed, 76 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9664768b..aa1e1687 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,11 +77,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 timestamps carry on. - The `dmabuf` feature enables its extensions only where the device has them, rather than failing device creation where it does not. +- `EncodedPacket::stats` is always there, and only its `gpu_time_ns` is + optional. Without encode queue timestamps, a packet still has its wall + times. ### Fixed - The source image barrier of the input copy now waits for earlier work on the image, rather than starting at `TOP_OF_PIPE`. +- The GPU encode time is correct when the timestamp counter wraps between + the two timestamps. ## [0.10.0] - 19-10-2026 diff --git a/benches/encode.rs b/benches/encode.rs index c4fc445e..30441748 100644 --- a/benches/encode.rs +++ b/benches/encode.rs @@ -155,11 +155,12 @@ fn run_encode_test( let mut write_packet = |packet: pixelforge::EncodedPacket| { total_bytes += packet.data.len(); - if let Some(stats) = packet.stats { - gpu_durations.push(Duration::from_nanos(stats.gpu_time_ns)); - frame_latency_durations.push(Duration::from_nanos(stats.frame_latency_ns)); - wall_latency_durations.push(Duration::from_nanos(stats.wall_latency_ns)); + let stats = packet.stats; + if let Some(gpu_time_ns) = stats.gpu_time_ns { + gpu_durations.push(Duration::from_nanos(gpu_time_ns)); } + frame_latency_durations.push(Duration::from_nanos(stats.frame_latency_ns)); + wall_latency_durations.push(Duration::from_nanos(stats.wall_latency_ns)); Ok::<(), Box>(()) }; diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index a1060c22..b3befc9c 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -631,10 +631,11 @@ pub use source::{Conversion, ConversionPreference, RgbSource, Source}; pub use source_image::{Handoff, SourceImage}; /// Statistic about the encoded video packet. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct EncodedPacketStats { - /// GPU encode time in nanoseconds - pub gpu_time_ns: u64, + /// GPU encode time in nanoseconds, or `None` when the video encode queue + /// has no timestamps. + pub gpu_time_ns: Option, /// CPU wall time in nanoseconds (submission + fence wait + readback) pub frame_latency_ns: u64, /// Wall latency in nanoseconds (time between submit and bitstream ready) @@ -654,8 +655,8 @@ pub struct EncodedPacket { pub pts: u64, /// Decode timestamp. pub dts: u64, - /// Optional stats about the packet - pub stats: Option, + /// Stats about the packet. + pub stats: EncodedPacketStats, } #[cfg(test)] @@ -941,7 +942,7 @@ mod tests { is_key_frame: true, pts: 0, dts: 0, - stats: None, + stats: EncodedPacketStats::default(), }; assert!(packet.is_key_frame); diff --git a/src/encoder/pipeline.rs b/src/encoder/pipeline.rs index 061038b5..6ac1d71d 100644 --- a/src/encoder/pipeline.rs +++ b/src/encoder/pipeline.rs @@ -161,6 +161,7 @@ struct WorkItem { metadata: SlotPacketMetadata, /// Timestamping resources timestamp_period: f32, + timestamp_valid_bits: u32, timestamp_query_pool: vk::QueryPool, submit_time: std::time::Instant, /// Resolves this frame's [`EncodeFuture`] once the bitstream is read back. @@ -186,6 +187,7 @@ pub(crate) struct EncodeSlot { pub query_pool: vk::QueryPool, pub timestamp_period: f32, + pub timestamp_valid_bits: u32, pub timestamp_query_pool: vk::QueryPool, /// Packet metadata recorded before submit, moved to the completion thread @@ -248,6 +250,7 @@ impl EncodePipeline { // a null pool as "timestamps disabled". let timestamps_supported = context.encode_timestamps_supported(); let timestamp_period = context.device_properties().limits.timestamp_period; + let timestamp_valid_bits = context.encode_timestamp_valid_bits(); if !timestamps_supported { tracing::info!( "Video encode queue family {:?} reports timestampValidBits=0; \ @@ -322,6 +325,7 @@ impl EncodePipeline { encode_fence, query_pool, timestamp_period, + timestamp_valid_bits, timestamp_query_pool, pending_metadata: None, }); @@ -413,6 +417,7 @@ impl EncodePipeline { bitstream_ptr, bitstream_buffer_size, timestamp_period, + timestamp_valid_bits, timestamp_query_pool, metadata, ) = { @@ -429,6 +434,7 @@ impl EncodePipeline { slot.bitstream_buffer_ptr as *const u8, slot.bitstream_buffer_size, slot.timestamp_period, + slot.timestamp_valid_bits, slot.timestamp_query_pool, metadata, ) @@ -484,6 +490,7 @@ impl EncodePipeline { bitstream_ptr: SendPtr(bitstream_ptr), bitstream_buffer_size, timestamp_period, + timestamp_valid_bits, timestamp_query_pool, submit_time, metadata, @@ -604,23 +611,21 @@ fn run_completion_thread( }; let bitstream_ready_time = std::time::Instant::now(); - let mut stats: Option = None; - if let Some(gpu_encode_ns) = unsafe { - query_timestamp_diff( - &device, - work.timestamp_query_pool, - work.metadata.timestamps, - work.timestamp_period, - ) - } { - stats = Some(super::EncodedPacketStats { - gpu_time_ns: gpu_encode_ns, - frame_latency_ns: work.metadata.now.elapsed().as_nanos() as u64, - wall_latency_ns: bitstream_ready_time - .duration_since(work.submit_time) - .as_nanos() as u64, - }); - } + let stats = super::EncodedPacketStats { + gpu_time_ns: unsafe { + query_timestamp_diff( + &device, + work.timestamp_query_pool, + work.metadata.timestamps, + work.timestamp_period, + work.timestamp_valid_bits, + ) + }, + frame_latency_ns: work.metadata.now.elapsed().as_nanos() as u64, + wall_latency_ns: bitstream_ready_time + .duration_since(work.submit_time) + .as_nanos() as u64, + }; let packet = result.map(|()| EncodedPacket { data, diff --git a/src/encoder/resources.rs b/src/encoder/resources.rs index f0434a5a..e1a41beb 100644 --- a/src/encoder/resources.rs +++ b/src/encoder/resources.rs @@ -1109,6 +1109,7 @@ pub(crate) unsafe fn query_timestamp_diff( query_pool: vk::QueryPool, mut timestamps: [u64; 2], timestamp_period: f32, + timestamp_valid_bits: u32, ) -> Option { if query_pool.is_null() { return None; @@ -1123,11 +1124,27 @@ pub(crate) unsafe fn query_timestamp_diff( ) }; if result.is_ok() { - encode_time_ns = Some(((timestamps[1] - timestamps[0]) as f32 * timestamp_period) as u64); + encode_time_ns = Some(timestamp_delta_ns( + timestamps, + timestamp_period, + timestamp_valid_bits, + )); } encode_time_ns } +/// The time between two timestamps of a counter with `valid_bits` bits, which +/// wraps around. +fn timestamp_delta_ns([start, end]: [u64; 2], period: f32, valid_bits: u32) -> u64 { + let mask = if valid_bits >= 64 { + u64::MAX + } else { + (1u64 << valid_bits) - 1 + }; + let ticks = end.wrapping_sub(start) & mask; + (ticks as f64 * f64::from(period)) as u64 +} + pub(crate) struct ClearImageParams { pub command_buffer: vk::CommandBuffer, pub fence: vk::Fence, @@ -1146,6 +1163,20 @@ pub(crate) struct ClearImageParams { mod tests { use super::*; + #[test] + fn timestamp_delta_scales_by_the_period() { + assert_eq!(timestamp_delta_ns([100, 1100], 1.0, 64), 1000); + assert_eq!(timestamp_delta_ns([100, 1100], 40.0, 64), 40_000); + } + + #[test] + fn timestamp_delta_survives_a_wrap() { + // A 36-bit counter that wrapped between the two timestamps. + let max = (1u64 << 36) - 1; + assert_eq!(timestamp_delta_ns([max - 9, 10], 1.0, 36), 20); + assert_eq!(timestamp_delta_ns([u64::MAX - 9, 10], 1.0, 64), 20); + } + #[test] fn test_gcd() { assert_eq!(gcd(12, 8), 4); diff --git a/src/vulkan.rs b/src/vulkan.rs index 97056ee9..72d07cb3 100644 --- a/src/vulkan.rs +++ b/src/vulkan.rs @@ -804,6 +804,11 @@ impl VideoContext { self.inner.video_encode_timestamp_valid_bits > 0 } + /// The selected video encode queue family's `timestampValidBits`. + pub(crate) fn encode_timestamp_valid_bits(&self) -> u32 { + self.inner.video_encode_timestamp_valid_bits + } + pub(crate) fn video_decode_queue_family(&self) -> Option { self.inner.video_decode_queue_family } From 18560fc4762717c7f5cb428bfdb09d304a205f87 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 14:58:32 +0100 Subject: [PATCH 13/19] fix(encoder): start the packet wall latency before the encode submit --- CHANGELOG.md | 2 ++ src/encoder/mod.rs | 3 ++- src/encoder/pipeline.rs | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa1e1687..2e662a8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 the image, rather than starting at `TOP_OF_PIPE`. - The GPU encode time is correct when the timestamp counter wraps between the two timestamps. +- `wall_latency_ns` starts before the encode submit. It started after it, so + a submit that blocked in the driver made it shorter than the GPU time. ## [0.10.0] - 19-10-2026 diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index b3befc9c..81fee163 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -638,7 +638,8 @@ pub struct EncodedPacketStats { pub gpu_time_ns: Option, /// CPU wall time in nanoseconds (submission + fence wait + readback) pub frame_latency_ns: u64, - /// Wall latency in nanoseconds (time between submit and bitstream ready) + /// Wall latency in nanoseconds, from the start of the encode submit until + /// the bitstream is read back. pub wall_latency_ns: u64, } diff --git a/src/encoder/pipeline.rs b/src/encoder/pipeline.rs index 6ac1d71d..05be333b 100644 --- a/src/encoder/pipeline.rs +++ b/src/encoder/pipeline.rs @@ -463,6 +463,9 @@ impl EncodePipeline { ) .chain(extra_signals.iter().copied()) .collect(); + // Before the submit, not after: the driver can block in the submit + // while the GPU already runs the work. + let submit_time = std::time::Instant::now(); unsafe { submit_encode_only( device, @@ -474,7 +477,6 @@ impl EncodePipeline { &signals, )?; } - let submit_time = std::time::Instant::now(); self.timeline.commit(); From 0395d0d4eabd63a02477f871f523e08da1d6cdd0 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 17:05:59 +0100 Subject: [PATCH 14/19] fix(shader): compile the shaders to SPIR-V 1.3 for Vulkan 1.1 devices --- CHANGELOG.md | 2 ++ examples/shader/sample_frame.spv | Bin 1536 -> 1580 bytes examples/shader/sample_planes.spv | Bin 1728 -> 1764 bytes shader/README.md | 7 ++++--- shader/color_convert.spv | Bin 33156 -> 33184 bytes shader/compile.sh | 4 ++-- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e662a8c..60a84add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 the image, rather than starting at `TOP_OF_PIPE`. - The GPU encode time is correct when the timestamp counter wraps between the two timestamps. +- The shaders are SPIR-V 1.3, which a Vulkan 1.1 device must accept. They + were SPIR-V 1.6, which only a Vulkan 1.3 device must accept. - `wall_latency_ns` starts before the encode submit. It started after it, so a submit that blocked in the driver made it shorter than the GPU time. diff --git a/examples/shader/sample_frame.spv b/examples/shader/sample_frame.spv index 78132082742220baf1a5ed007bb4f7715fbac8f3..fd1f82758761b90a0b00e283070c3162db1fc2a8 100644 GIT binary patch delta 102 zcmZqRS;ND|%%sfDz&w%7MwpF(m4ShQ1&DJKGxLBH1JA_7)x7RNp%5U48;B)0b29QX jX=nq5*nr$%Am#vK9UzYjNQ0Dt#6Va9h&M|xzhD9YYq$#} delta 59 zcmZ3()4;>V%%sfDz&4T1MxK*_m4ShQ1&DJKGxLBH0}l|30kJ(0drS;ny;*>fhiS75 H^9v>b=O_so diff --git a/examples/shader/sample_planes.spv b/examples/shader/sample_planes.spv index 06b9907d165552617269901a2ffe3879a28fa28b..009f19e7fbec32d4bebdbd347af861e382b85526 100644 GIT binary patch delta 99 zcmX@W`-GQ`nMs+Qfq5dEi!d7lD+2=q3lQffX66AY2A+w9k9pmJLg7FTHxNs14rKhy hsHzPVVgqu+fS3b_b$~oBAPrIm5(8m{&5q3V%mB!z4CVj; delta 67 zcmaFDdw`dXnMs+Qfo&q2i#j(0D+2=q3lQffX66AY1|A?517ckuHUVOPAP$-s`*?Ez M;|Ipg8O$}z077O9JOBUy diff --git a/shader/README.md b/shader/README.md index cb13c7a6..57e7b68f 100644 --- a/shader/README.md +++ b/shader/README.md @@ -15,9 +15,10 @@ Requires `glslc` from the Vulkan SDK. ./compile.sh ``` -This compiles both shaders to SPIR-V 1.6 (Vulkan 1.3, optimized). The `.spv` -files are included in source via `include_bytes!`, so rerun this after editing a -`.comp` and commit the result. +This compiles the shaders to SPIR-V 1.3 (Vulkan 1.1, optimized), the newest +version a Vulkan 1.1 device must accept: pixelforge also runs on a device the +caller created, which can be 1.1. The `.spv` files are included in source via +`include_bytes!`, so rerun this after editing a `.comp` and commit the result. The examples carry one more, outside the library: `examples/shader/sample_frame.comp` reads a decoded frame through a sampler-YCbCr conversion and writes RGBA, which diff --git a/shader/color_convert.spv b/shader/color_convert.spv index ef2895e7c92851511fef6966d45cda245e44777c..65bbda2f0f8c795279ea1379c50585ef7217720a 100644 GIT binary patch delta 83 zcmZo^W?In9#Kz2|%+A0(k-DB3}gTR diff --git a/shader/compile.sh b/shader/compile.sh index 9bd29a46..2f599267 100755 --- a/shader/compile.sh +++ b/shader/compile.sh @@ -11,7 +11,7 @@ if ! command -v glslc &>/dev/null; then fi echo "Compiling color_convert.comp → color_convert.spv" -glslc -x glsl --target-env=vulkan1.3 -O \ +glslc -x glsl --target-env=vulkan1.1 -O \ "$SCRIPT_DIR/color_convert.comp" \ -o "$SPV_PATH" @@ -23,7 +23,7 @@ for name in sample_frame sample_planes; do EXAMPLE_SRC="$PROJECT_ROOT/examples/shader/$name.comp" EXAMPLE_SPV="$PROJECT_ROOT/examples/shader/$name.spv" echo "Compiling $name.comp → $name.spv" - glslc -x glsl --target-env=vulkan1.3 -O "$EXAMPLE_SRC" -o "$EXAMPLE_SPV" + glslc -x glsl --target-env=vulkan1.1 -O "$EXAMPLE_SRC" -o "$EXAMPLE_SPV" FILE_SIZE=$(stat -c%s "$EXAMPLE_SPV" 2>/dev/null || stat -f%z "$EXAMPLE_SPV" 2>/dev/null) echo "Written $FILE_SIZE bytes to $EXAMPLE_SPV" done From 08b90ac7b28af7caaa081cc9c9ddbed514ebbb47 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 17:05:59 +0100 Subject: [PATCH 15/19] fix(h264): query the quality level only for a supported profile --- CHANGELOG.md | 2 ++ src/encoder/h264/init.rs | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60a84add..7e5e1f8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 the two timestamps. - The shaders are SPIR-V 1.3, which a Vulkan 1.1 device must accept. They were SPIR-V 1.6, which only a Vulkan 1.3 device must accept. +- An H.264 encoder for a profile the device does not support no longer asks + for its quality level properties before it fails. - `wall_latency_ns` starts before the encode submit. It started after it, so a submit that blocked in the driver made it shorter than the GPU time. diff --git a/src/encoder/h264/init.rs b/src/encoder/h264/init.rs index 13b33291..4a9bb3c0 100644 --- a/src/encoder/h264/init.rs +++ b/src/encoder/h264/init.rs @@ -34,10 +34,6 @@ impl H264 { let profile = EncodeProfile::new(&config, rgb_input.is_some())?; let profile_info = profile.info(); - // The driver's preferred entropy mode is H.264-specific (some require - // CAVLC for High 4:4:4 Predictive). - let preferred_entropy_cabac = query_preferred_entropy_cabac(&context, profile_info); - // Query device capabilities with the H.264-specific capability struct // chained in (required by the driver). let mut h264_caps = vk::VideoEncodeH264CapabilitiesKHR::default(); @@ -52,6 +48,12 @@ impl H264 { let caps = query_video_caps(&context, profile_info, &mut capabilities)?; let rgb_caps = rgb_input.map(|_| RgbConversionCaps::from(&rgb_caps)); + // The driver's preferred entropy mode is H.264-specific (some require + // CAVLC for High 4:4:4 Predictive). Only after the capability query, + // which fails for an unsupported profile: a quality level query for one + // is invalid (VUID-VkPhysicalDeviceVideoEncodeQualityLevelInfoKHR-pVideoProfile-08259). + let preferred_entropy_cabac = query_preferred_entropy_cabac(&context, profile_info); + // Scale the bitstream buffer by resolution so a single frame can't overflow (matches AV1). let bitstream_buffer_size = MIN_BITSTREAM_BUFFER_SIZE .max(config.dimensions.width as usize * config.dimensions.height as usize); From 0d2ca39679130f31c88226d7804b4d5f97a5c62f Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Mon, 2 Nov 2026 17:21:46 +0100 Subject: [PATCH 16/19] test(converter): check shader frames submitted without waiting --- src/converter/device_tests.rs | 64 ++++++++++++++++++++++++++++++----- src/encoder/api.rs | 2 ++ 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/src/converter/device_tests.rs b/src/converter/device_tests.rs index cd3a595c..9d6c4f5d 100644 --- a/src/converter/device_tests.rs +++ b/src/converter/device_tests.rs @@ -526,10 +526,12 @@ fn encode_config(codec: Codec) -> EncodeConfig { } /// The streams of 8 moving frames, converted by hand and by an encoder with -/// an RGB source and the shader conversion. +/// an RGB source and the shader conversion. `pipelined` submits the encoder's +/// frames without waiting for each packet, so up to every slot is in flight. fn by_hand_and_by_encoder( context: &VideoContext, codec: Codec, + pipelined: bool, ) -> Result<(Vec, Vec), Box> { let mut by_hand = Encoder::new(context.clone(), encode_config(codec))?; let mut converter = ColorConverter::new( @@ -553,14 +555,12 @@ fn by_hand_and_by_encoder( ))) .with_conversion(ConversionPreference::Shader), )?; - let (mut a, mut b) = (Vec::new(), Vec::new()); + // All sources first, so that nothing else runs between the encoder's + // submits. Each `SourceImage` drops (and waits) before the image it reads. + let mut sources: Vec<(SourceImage, SrcImage)> = Vec::new(); for n in 0..8 { let src = unsafe { create_src_image(context, ENCODE_WIDTH, ENCODE_HEIGHT, &moving_frame(n))? }; - let slot = by_hand.input_slot()?; - convert_into(&mut converter, src.image, slot.image())?; - a.extend(pollster::block_on(by_hand.submit(Frame::input(slot))?)?.data); - let image = SourceImage::new( context, src.image, @@ -568,8 +568,30 @@ fn by_hand_and_by_encoder( layout: vk::ImageLayout::GENERAL, }, )?; - b.extend(pollster::block_on(by_encoder.submit(Frame::image(&image))?)?.data); + sources.push((image, src)); + } + + let mut a = Vec::new(); + for (_, src) in &sources { + let slot = by_hand.input_slot()?; + convert_into(&mut converter, src.image, slot.image())?; + a.extend(pollster::block_on(by_hand.submit(Frame::input(slot))?)?.data); + } + + let mut b = Vec::new(); + let mut pending = Vec::new(); + for (image, _) in &sources { + let packet = by_encoder.submit(Frame::image(image))?; + if pipelined { + pending.push(packet); + } else { + b.extend(pollster::block_on(packet)?.data); + } } + for packet in pending { + b.extend(pollster::block_on(packet)?.data); + } + drop(sources); Ok((a, b)) } @@ -588,9 +610,35 @@ fn shader_source_matches_converting_by_hand() -> Result<(), Box Result<(), Box> { + init_logging(); + for codec in [Codec::H264, Codec::H265, Codec::AV1] { + let Ok(context) = VideoContextBuilder::new() + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .require_encode(codec) + .build() + else { + eprintln!("{codec:?}: no encoder on this device"); + continue; + }; + let (by_hand, by_encoder) = by_hand_and_by_encoder(&context, codec, true)?; + assert!(!by_hand.is_empty(), "{codec:?}: empty stream"); + assert!( + by_hand == by_encoder, + "{codec:?}: a pipelined frame was overwritten" + ); + } + Ok(()) +} diff --git a/src/encoder/api.rs b/src/encoder/api.rs index 87e072ff..9ca2db0b 100644 --- a/src/encoder/api.rs +++ b/src/encoder/api.rs @@ -294,6 +294,8 @@ impl Encoder { ownership: SourceOwnership, sync: FrameSync, ) -> Result { + // This waits until the slot is free, so the conversion never writes + // into an image that an earlier encode still reads. let target = self.inner.input_image(); let converter = self .converter From f5a9f23a8a7d23fd07afeb24a23ab67986e0b812 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Wed, 4 Nov 2026 18:59:44 +0100 Subject: [PATCH 17/19] test(converter): check that encoding is deterministic --- src/converter/device_tests.rs | 45 +++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/converter/device_tests.rs b/src/converter/device_tests.rs index 9d6c4f5d..e98907ac 100644 --- a/src/converter/device_tests.rs +++ b/src/converter/device_tests.rs @@ -642,3 +642,48 @@ fn pipelined_shader_frames_do_not_overwrite_each_other() -> Result<(), Box Result<(), Box> { + init_logging(); + for codec in [Codec::H264, Codec::H265, Codec::AV1] { + let Ok(context) = VideoContextBuilder::new().require_encode(codec).build() else { + eprintln!("{codec:?}: no encoder on this device"); + continue; + }; + let encode = || -> Result>, Box> { + let mut encoder = Encoder::new(context.clone(), encode_config(codec))?; + let mut converter = ColorConverter::new( + context.clone(), + ColorConverterConfig::new( + ENCODE_WIDTH, + ENCODE_HEIGHT, + InputFormat::BGRA, + OutputFormat::NV12, + ColorSpec::Srgb, + ColorSpec::Srgb, + ColorRange::Limited, + ), + )?; + let mut packets = Vec::new(); + for n in 0..8 { + let src = unsafe { + create_src_image(&context, ENCODE_WIDTH, ENCODE_HEIGHT, &moving_frame(n))? + }; + let slot = encoder.input_slot()?; + convert_into(&mut converter, src.image, slot.image())?; + packets.push(pollster::block_on(encoder.submit(Frame::input(slot))?)?.data); + } + Ok(packets) + }; + let first = encode()?; + for run in 1..5 { + let other = encode()?; + let differ: Vec = (0..first.len()).filter(|&i| first[i] != other[i]).collect(); + println!("{codec:?} run {run}: frames that differ from run 0: {differ:?}"); + } + } + Ok(()) +} From d3f7988a6f908b6cb6bb1870fcd03d1fc4c18914 Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Wed, 4 Nov 2026 20:05:06 +0100 Subject: [PATCH 18/19] feat(examples): print per-frame reference limits --- examples/query_capabilities.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/query_capabilities.rs b/examples/query_capabilities.rs index 37e48b26..aaff6a24 100644 --- a/examples/query_capabilities.rs +++ b/examples/query_capabilities.rs @@ -195,6 +195,26 @@ fn query_detailed_capabilities( caps.max_active_reference_pictures ); println!(" Max DPB Slots: {}", caps.max_dpb_slots); + // The references one frame may predict from, which can be fewer than + // the active reference pictures. + match codec { + Codec::H264 => println!( + " Max L0 References (P / B), L1: {} / {}, {}", + h264_caps.max_p_picture_l0_reference_count, + h264_caps.max_b_picture_l0_reference_count, + h264_caps.max_l1_reference_count + ), + Codec::H265 => println!( + " Max L0 References (P / B), L1: {} / {}, {}", + h265_caps.max_p_picture_l0_reference_count, + h265_caps.max_b_picture_l0_reference_count, + h265_caps.max_l1_reference_count + ), + Codec::AV1 => println!( + " Max Single References: {} (names {:#09b})", + av1_caps.max_single_reference_count, av1_caps.single_reference_name_mask + ), + } // 2. Query Supported Formats let mut format_props_count = 0; From f046afeb171203483aa3f5e9f06d6a50d7abde5d Mon Sep 17 00:00:00 2001 From: Hans Gaiser Date: Wed, 4 Nov 2026 20:05:06 +0100 Subject: [PATCH 19/19] fix(encoder): keep a reference window larger than the references per frame --- AGENTS.md | 13 +++++++ src/encoder/api.rs | 4 +++ src/encoder/av1/init.rs | 15 +++++--- src/encoder/av1/mod.rs | 42 +++++++++++++++++++--- src/encoder/av1/record.rs | 58 +++++++++++++++++------------- src/encoder/codec.rs | 57 ++++++++++++++++++++++------- src/encoder/h264/init.rs | 23 ++++++++++-- src/encoder/h264/mod.rs | 12 ++++--- src/encoder/h264/record.rs | 13 ++++--- src/encoder/h264/session_params.rs | 7 ++-- src/encoder/h265/init.rs | 5 ++- src/encoder/h265/mod.rs | 22 ++++++++++-- src/encoder/h265/record.rs | 19 ++++++---- src/encoder/mod.rs | 9 +++-- 14 files changed, 228 insertions(+), 71 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b17c4068..eefd553a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -143,6 +143,19 @@ direct read and the unaligned-size test are skipped, and only RADV can check them. `PIXELFORGE_NO_DMABUF_QUIRKS=1` turns the quirks off, which makes two of those tests fail on that driver. +Reference frame invalidation (`tests/rfi.rs`) recovers from the frames the +encoder keeps in the DPB (the window, `max_reference_frames`), which can be +more than each frame predicts from. A device that allows many references per +frame never exercises that. `PIXELFORGE_MAX_ACTIVE_REFERENCES=1` limits the +references per frame without changing the window, to test it: + +```bash +PIXELFORGE_MAX_ACTIVE_REFERENCES=1 cargo test --test rfi -- --ignored --nocapture +``` + +`RUST_LOG=pixelforge::encoder::codec=debug` reports the negotiated window and +per-frame count, on the `Reference window` line. + Tests run one at a time (`RUST_TEST_THREADS=1` in `.cargo/config.toml`): the NVIDIA driver can deadlock when one thread destroys a device while another creates objects on a different one. diff --git a/src/encoder/api.rs b/src/encoder/api.rs index 9ca2db0b..9f6cccc6 100644 --- a/src/encoder/api.rs +++ b/src/encoder/api.rs @@ -430,6 +430,10 @@ impl Encoder { /// across H.264, H.265, and AV1 — each keeps a multi-reference window and /// re-anchors prediction to the most recent survivor, falling back to an /// IDR (AV1: key frame) only when the loss covers the entire window. + /// + /// The window does not depend on how many references the device can + /// predict from per frame. A device that predicts from one or two still + /// keeps the whole window, and recovers from the older frames in it. pub fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) { let local = first_lost_display_order.saturating_sub(self.pts_offset); self.inner.invalidate_reference_frames(local) diff --git a/src/encoder/av1/init.rs b/src/encoder/av1/init.rs index 2aa24493..d8b60aa6 100644 --- a/src/encoder/av1/init.rs +++ b/src/encoder/av1/init.rs @@ -54,10 +54,14 @@ impl Av1 { align_unit: SUPERBLOCK_SIZE, // AV1 has NUM_REF_FRAMES = 8 DPB slots, and `refresh_frame_flags` // is an 8-bit syntax element. `build_encoder_common` adds one setup - // slot on top of the active count, so cap active refs at 7 to keep - // every DPB slot index in 0..=7 (the range the [u8; 8] order-hint - // array and `1 << slot` refresh mask can represent). - max_active_refs_cap: 7, + // slot on top of the window, so cap the window at 7 to keep every + // DPB slot index in 0..=7 (the range the [u8; 8] order-hint array + // and `1 << slot` refresh mask can represent). + max_reference_window: 7, + // Inter frames use single-reference prediction, so each may name + // at most this many references, and only names in the mask. + max_frame_references: (av1_caps.max_single_reference_count as usize) + .min(av1_caps.single_reference_name_mask.count_ones() as usize), bitstream_buffer_size, // Allow the shared infrastructure to fall back to a layered DPB // when the driver does not support separate reference images @@ -65,6 +69,7 @@ impl Av1 { allow_layered_dpb: true, rgb_caps, })?; + let reference_window = init.reference_window; let active_reference_count = init.active_reference_count; let common = init.common; @@ -73,7 +78,9 @@ impl Av1 { order_hint: 0, header_data: None, references: Vec::new(), + reference_window, active_reference_count, + single_reference_name_mask: av1_caps.single_reference_name_mask, }; let mut encoder = CodecEncoder { common, codec }; diff --git a/src/encoder/av1/mod.rs b/src/encoder/av1/mod.rs index 2dbdc085..bb9843ac 100644 --- a/src/encoder/av1/mod.rs +++ b/src/encoder/av1/mod.rs @@ -5,8 +5,9 @@ //! and its sequence-header generation (`session_params`). //! //! Inter frames predict from up to `Av1::active_reference_count` recent -//! references, mapped onto the forward AV1 reference names. Keeping a window of -//! references (rather than just the previous frame) is what lets reference frame +//! references, mapped onto the forward AV1 reference names the device allows. +//! A window of `Av1::reference_window` references stays in the DPB (rather than +//! just the ones predicted from), which is what lets reference frame //! invalidation recover from loss by re-anchoring to an older surviving frame //! instead of forcing a key frame. @@ -41,10 +42,41 @@ pub(crate) struct Av1 { order_hint: u32, /// Cached sequence-header OBU (invalidated when session params change). header_data: Option>, - /// Active references, most-recent first. + /// References held in the DPB, most-recent first. Each frame predicts + /// from the first `active_reference_count`; the rest are kept for + /// reference frame invalidation to fall back on. references: Vec, - /// Negotiated number of active references kept for prediction. + /// How many references stay in the DPB. + reference_window: u32, + /// How many of the newest references one frame predicts from. active_reference_count: u32, + /// The reference names single-reference prediction may use (bit 0 is + /// LAST), from `VkVideoEncodeAV1CapabilitiesKHR`. + single_reference_name_mask: u32, +} + +impl Av1 { + /// The references this frame predicts from: the newest held ones. + fn used_references(&self) -> &[ReferenceInfo] { + let n = self + .references + .len() + .min(self.active_reference_count as usize); + &self.references[..n] + } + + /// The DPB slot each of the 7 AV1 reference names predicts from, or -1. + /// + /// The used references take the names the device allows, most recent + /// first. A name left at -1 is not predicted from. + fn reference_names(&self) -> [i32; 7] { + let mut names = [-1i32; 7]; + let allowed = (0..7).filter(|name| self.single_reference_name_mask & (1 << name) != 0); + for (name, r) in allowed.zip(self.used_references()) { + names[name] = r.dpb_slot as i32; + } + names + } } impl VideoCodec for Av1 { @@ -110,7 +142,7 @@ impl VideoCodec for Av1 { self.references.insert(0, ref_info); // Keep a sliding window of the most recent references for prediction and // loss recovery. - while self.references.len() > self.active_reference_count.max(1) as usize { + while self.references.len() > self.reference_window.max(1) as usize { self.references.pop(); } diff --git a/src/encoder/av1/record.rs b/src/encoder/av1/record.rs index 93fe1e7b..4b99b4ca 100644 --- a/src/encoder/av1/record.rs +++ b/src/encoder/av1/record.rs @@ -47,7 +47,9 @@ impl Av1 { unsafe { prepare_encode_command_buffer(common.device(), command_buffer, query_pool)?; } - let ref_dpb_slots: Vec = self.references.iter().map(|r| r.dpb_slot).collect(); + // Only the references this frame predicts from go to the encode. The + // rest of the window stays in its DPB slots, untouched. + let ref_dpb_slots: Vec = self.used_references().iter().map(|r| r.dpb_slot).collect(); unsafe { record_dpb_barriers( common.context.sync2(), @@ -115,15 +117,15 @@ impl Av1 { .picture_resource(&setup_picture_resource) .push(&mut setup_av1_dpb_info_ref0); - // Reference frames for inter frames: declare each distinct DPB slot we - // still hold, most-recent first. The vectors hold the data so the + // Reference frames for inter frames: declare each distinct DPB slot + // this frame predicts from, most-recent first. The vectors hold the data so the // pointers the slot infos take into them stay stable. // // `distinct_refs` dedups by DPB slot (a reference can be mapped to more // than one AV1 reference name below, but each slot is declared once). let mut distinct_refs: Vec<&super::ReferenceInfo> = Vec::new(); if !is_key_frame { - for r in &self.references { + for r in self.used_references() { if !distinct_refs.iter().any(|d| d.dpb_slot == r.dpb_slot) { distinct_refs.push(r); } @@ -245,18 +247,14 @@ impl Av1 { pBufferRemovalTimes: std::ptr::null(), }; - // Map the 7 AV1 reference names to DPB slots. The N most-recent - // references take the first N forward names (LAST, LAST2, LAST3, - // GOLDEN, ...); any remaining names reuse the oldest surviving - // reference so every name resolves to a valid, decodable DPB slot. This + // Map the AV1 reference names this frame predicts from to DPB slots + // (see `reference_names`); the others stay -1. The names that are set // must mirror `ref_frame_idx` from `calculate_reference_frame_mapping`. - let mut reference_name_slot_indices = [-1i32; 7]; - if !is_key_frame && !self.references.is_empty() { - let n = self.references.len(); - for (name, idx) in reference_name_slot_indices.iter_mut().enumerate() { - *idx = self.references[name.min(n - 1)].dpb_slot as i32; - } - } + let reference_name_slot_indices = if !is_key_frame && !self.references.is_empty() { + self.reference_names() + } else { + [-1i32; 7] + }; let (prediction_mode, rate_control_group) = if is_key_frame { ( @@ -464,19 +462,31 @@ impl Av1 { let mut ref_frame_idx = [0i8; 7]; let mut ref_order_hint = [0u8; 8]; - let n = self.references.len(); - // Most-recent references take the first forward names; surplus names - // reuse the oldest survivor. Matches `reference_name_slot_indices`. - for (name, idx) in ref_frame_idx.iter_mut().enumerate() { - *idx = self.references[name.min(n - 1)].dpb_slot as i8; + // The names this frame predicts from match `reference_name_slot_indices`. + // The bitstream needs a slot for every name, so the others point at + // the most recent reference, which no block then uses. + let names = self.reference_names(); + let newest = self.references[0].dpb_slot as i8; + for (idx, &slot) in ref_frame_idx.iter_mut().zip(&names) { + *idx = if slot >= 0 { slot as i8 } else { newest }; } - // Order hints keyed by DPB slot. + // Order hints keyed by DPB slot, for every reference still held. for r in &self.references { ref_order_hint[r.dpb_slot as usize] = r.order_hint as u8; } - // Load the frame context from the most recent (LAST) reference, and - // refresh only the current slot so this frame becomes the new LAST. + // Load the frame context from the most recent reference, under the + // first name it took, and refresh only the current slot so this frame + // becomes the newest reference. + let primary_ref_frame = names + .iter() + .position(|&slot| slot >= 0) + .map_or(PRIMARY_REF_NONE, |name| name as u8); let refresh_flags = 1u8 << current_dpb_slot; - (ref_frame_idx, ref_order_hint, 0u8, refresh_flags) + ( + ref_frame_idx, + ref_order_hint, + primary_ref_frame, + refresh_flags, + ) } } diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 15f75cf6..8712123a 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -626,8 +626,11 @@ pub(crate) struct CommonInitRequest<'a> { pub caps: &'a DeviceVideoCaps, /// Codec block size for coded-extent alignment (macroblock / CTB / superblock). pub align_unit: u32, - /// Upper bound on active reference pictures the codec's syntax allows. - pub max_active_refs_cap: usize, + /// Upper bound on the reference window the codec's syntax allows. + pub max_reference_window: usize, + /// How many references one frame may predict from, from the codec's own + /// capabilities (`maxPPictureL0ReferenceCount`, `maxSingleReferenceCount`). + pub max_frame_references: usize, pub bitstream_buffer_size: usize, /// Whether the codec can use a layered DPB image when the driver lacks /// `SEPARATE_REFERENCE_IMAGES` (H.264/H.265 yes; AV1 no). @@ -638,9 +641,12 @@ pub(crate) struct CommonInitRequest<'a> { } /// Result of [`build_encoder_common`]: the assembled common state plus the -/// negotiated active-reference count the codec needs for its parameter sets. +/// negotiated reference counts the codec needs for its parameter sets. pub(crate) struct CommonInit { pub common: EncoderCommon, + /// How many reconstructed frames stay in the DPB as references. + pub reference_window: u32, + /// How many of the newest of those one frame predicts from. pub active_reference_count: u32, } @@ -818,7 +824,13 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result .find(|f| *f == preferred_src_format) .unwrap_or(supported_dpb_formats[0].format); - // Negotiate DPB slots and active references. + // Negotiate the reference window and the references each frame uses. + // + // The window is how many reconstructed frames stay in the DPB, which is + // what reference frame invalidation can recover from. Each frame predicts + // from only the newest `active` of them, so a device that predicts from + // one or two references still keeps older frames to fall back on. This is + // the model NVENC documents for `NvEncInvalidateRefFrames`. let max_dpb_slots_supported = capabilities.max_dpb_slots as usize; let max_active_supported = capabilities.max_active_reference_pictures as usize; if max_dpb_slots_supported < 2 { @@ -827,18 +839,36 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result max_dpb_slots_supported ))); } - let mut target_active_refs = (config.max_reference_frames as usize) + // At least 1: `max_dpb_slots_supported` is at least 2, and every codec's + // window cap and `MAX_DPB_SLOTS` are larger. + let window_limit = req + .max_reference_window + .min(max_dpb_slots_supported - 1) + .min(MAX_DPB_SLOTS - 1); + let reference_window = (config.max_reference_frames as usize).clamp(1, window_limit); + let mut max_active_reference_pictures = reference_window .min(max_active_supported) - .min(req.max_active_refs_cap); - if target_active_refs < 1 && max_active_supported >= 1 { - target_active_refs = 1; + .min(req.max_frame_references); + // `PIXELFORGE_MAX_ACTIVE_REFERENCES=1` predicts from fewer references than + // the device allows, to reproduce a device with tighter limits. + if let Some(cap) = std::env::var("PIXELFORGE_MAX_ACTIVE_REFERENCES") + .ok() + .and_then(|v| v.parse::().ok()) + { + max_active_reference_pictures = max_active_reference_pictures.min(cap); + } + if max_active_reference_pictures < 1 && max_active_supported >= 1 { + max_active_reference_pictures = 1; } // B-frames are not yet supported, so a reconstructed-frame slot plus the - // active references is enough. - let dpb_slot_count = (target_active_refs + 1) - .min(max_dpb_slots_supported) - .min(MAX_DPB_SLOTS); - let max_active_reference_pictures = target_active_refs.min(dpb_slot_count.saturating_sub(1)); + // window is enough. + let dpb_slot_count = reference_window + 1; + tracing::debug!( + "Reference window {reference_window}, {max_active_reference_pictures} used per frame \ + (device: {max_dpb_slots_supported} DPB slots, {max_active_supported} active, \ + {} per frame)", + req.max_frame_references + ); let encode_queue_family = context.video_encode_queue_family().ok_or_else(|| { PixelForgeError::NoSuitableDevice("No video encode queue family available".to_string()) @@ -965,6 +995,7 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result Ok(CommonInit { common, + reference_window: reference_window as u32, active_reference_count: max_active_reference_pictures as u32, }) } diff --git a/src/encoder/h264/init.rs b/src/encoder/h264/init.rs index 4a9bb3c0..fce70b00 100644 --- a/src/encoder/h264/init.rs +++ b/src/encoder/h264/init.rs @@ -1,4 +1,4 @@ -use super::{H264, MB_SIZE}; +use super::{H264, MB_SIZE, ReferenceInfo}; use crate::converter::InputFormat; use crate::encoder::EncodeConfig; @@ -65,11 +65,13 @@ impl H264 { rgb_input, caps: &caps, align_unit: MB_SIZE, - max_active_refs_cap: 32, + max_reference_window: 32, + max_frame_references: h264_caps.max_p_picture_l0_reference_count as usize, bitstream_buffer_size, allow_layered_dpb: true, rgb_caps, })?; + let reference_window = init.reference_window; let active_reference_count = init.active_reference_count; let common = init.common; @@ -94,6 +96,7 @@ impl H264 { backward_reference_poc: 0, backward_reference_dpb_slot: 2, l0_references: Vec::new(), + reference_window, active_reference_count, pending_unmark_frame_nums: Vec::new(), profile_idc, @@ -120,10 +123,24 @@ impl H264 { self.preferred_entropy_cabac } - /// Negotiated active reference count. + /// How many of the newest references one frame predicts from. pub(super) fn active_reference_count(&self) -> u32 { self.active_reference_count } + + /// How many references stay in the DPB. + pub(super) fn reference_window(&self) -> u32 { + self.reference_window + } + + /// The references this frame predicts from: the newest held ones. + pub(super) fn used_references(&self) -> &[ReferenceInfo] { + let n = self + .l0_references + .len() + .min(self.active_reference_count as usize); + &self.l0_references[..n] + } } /// Query the driver's preferred H.264 entropy coding mode for quality level 0. diff --git a/src/encoder/h264/mod.rs b/src/encoder/h264/mod.rs index f94a53a6..f1f0732e 100644 --- a/src/encoder/h264/mod.rs +++ b/src/encoder/h264/mod.rs @@ -43,9 +43,13 @@ pub(crate) struct H264 { backward_reference_frame_num: u32, backward_reference_poc: i32, backward_reference_dpb_slot: u8, - /// Active L0 references, most-recent first. + /// References held in the DPB, most-recent first. Each frame predicts + /// from the first `active_reference_count`; the rest are kept for + /// reference frame invalidation to fall back on. l0_references: Vec, - /// Negotiated number of active references. + /// How many references stay in the DPB. + reference_window: u32, + /// How many of the newest references one frame predicts from. active_reference_count: u32, /// `frame_num`s of references dropped by reference frame invalidation that /// still need to be marked unused-for-reference (via MMCO) in the next @@ -69,7 +73,7 @@ impl VideoCodec for H264 { // Reset the DPB for the new coded video sequence. let dpb_config = DpbConfig { dpb_size: common.dpb_slot_count as u32, - max_num_ref_frames: common.config.max_reference_frames, + max_num_ref_frames: self.reference_window, use_multiple_references: common.config.b_frame_count > 0, log2_max_frame_num_minus4: 4, log2_max_pic_order_cnt_lsb_minus4: 4, @@ -142,7 +146,7 @@ impl VideoCodec for H264 { display_order: plan.display_order, }, ); - while self.l0_references.len() > self.active_reference_count as usize { + while self.l0_references.len() > self.reference_window as usize { self.l0_references.pop(); } diff --git a/src/encoder/h264/record.rs b/src/encoder/h264/record.rs index e79da965..eb8bf1ca 100644 --- a/src/encoder/h264/record.rs +++ b/src/encoder/h264/record.rs @@ -84,7 +84,10 @@ impl H264 { unsafe { prepare_encode_command_buffer(common.device(), command_buffer, query_pool)?; } - let ref_dpb_slots: Vec = self.l0_references.iter().map(|r| r.dpb_slot).collect(); + // Only the references this frame predicts from go to the encode. The + // rest of the window stays in its DPB slots, untouched. + let used_references = self.used_references(); + let ref_dpb_slots: Vec = used_references.iter().map(|r| r.dpb_slot).collect(); unsafe { record_dpb_barriers( common.context.sync2(), @@ -277,10 +280,10 @@ impl H264 { }; // Vectors hold the data so pointers into them stay stable. - let mut l0_resources = Vec::with_capacity(self.l0_references.len()); - let mut l0_std_infos = Vec::with_capacity(self.l0_references.len()); + let mut l0_resources = Vec::with_capacity(used_references.len()); + let mut l0_std_infos = Vec::with_capacity(used_references.len()); - for ref_info in &self.l0_references { + for ref_info in used_references { l0_resources.push( vk::VideoPictureResourceInfoKHR::default() .coded_offset(vk::Offset2D { x: 0, y: 0 }) @@ -312,7 +315,7 @@ impl H264 { .zip(l0_dpb_slot_infos.iter_mut()) .enumerate() { - let ref_info = &self.l0_references[i]; + let ref_info = &used_references[i]; l0_slots.push( vk::VideoReferenceSlotInfoKHR::default() .slot_index(ref_info.dpb_slot as i32) diff --git a/src/encoder/h264/session_params.rs b/src/encoder/h264/session_params.rs index 0578c796..6320341d 100644 --- a/src/encoder/h264/session_params.rs +++ b/src/encoder/h264/session_params.rs @@ -69,6 +69,9 @@ impl H264 { }; let max_active = self.active_reference_count() as u8; + // The decoder's sliding window holds as many frames as the encoder + // keeps, so older ones survive for reference frame invalidation. + let window = self.reference_window() as u8; let mut vui_flags: ash::vk::native::StdVideoH264SpsVuiFlags = unsafe { std::mem::zeroed() }; vui_flags.set_aspect_ratio_info_present_flag(1); @@ -98,7 +101,7 @@ impl H264 { num_units_in_tick: 0, time_scale: 0, max_num_reorder_frames: if config.b_frame_count > 0 { 1 } else { 0 }, - max_dec_frame_buffering: max_active + 1, + max_dec_frame_buffering: window + 1, chroma_sample_loc_type_top_field: 0, chroma_sample_loc_type_bottom_field: 0, reserved1: 0, @@ -123,7 +126,7 @@ impl H264 { offset_for_top_to_bottom_field: 0, log2_max_pic_order_cnt_lsb_minus4: 4, num_ref_frames_in_pic_order_cnt_cycle: 0, - max_num_ref_frames: max_active, + max_num_ref_frames: window, reserved1: 0, pic_width_in_mbs_minus1: pic_width_in_mbs - 1, pic_height_in_map_units_minus1: pic_height_in_map_units - 1, diff --git a/src/encoder/h265/init.rs b/src/encoder/h265/init.rs index ec582a74..88bfcac0 100644 --- a/src/encoder/h265/init.rs +++ b/src/encoder/h265/init.rs @@ -64,11 +64,13 @@ impl H265 { rgb_input, caps: &caps, align_unit: CTB_SIZE, - max_active_refs_cap: 15, + max_reference_window: 15, + max_frame_references: h265_caps.max_p_picture_l0_reference_count as usize, bitstream_buffer_size, allow_layered_dpb: true, rgb_caps, })?; + let reference_window = init.reference_window; let active_reference_count = init.active_reference_count; let common = init.common; @@ -90,6 +92,7 @@ impl H265 { backward_reference_poc: 0, backward_reference_dpb_slot: 2, l0_references: Vec::new(), + reference_window, active_reference_count, profile_idc, }; diff --git a/src/encoder/h265/mod.rs b/src/encoder/h265/mod.rs index 533e5b9f..8eff3e17 100644 --- a/src/encoder/h265/mod.rs +++ b/src/encoder/h265/mod.rs @@ -36,11 +36,29 @@ pub(crate) struct H265 { has_backward_reference: bool, backward_reference_poc: i32, backward_reference_dpb_slot: u8, + /// References held in the DPB, most-recent first. Each frame predicts + /// from the first `active_reference_count`; the rest stay in its + /// reference picture set, unused, for reference frame invalidation to fall + /// back on. l0_references: Vec, + /// How many references stay in the DPB. + reference_window: u32, + /// How many of the newest references one frame predicts from. active_reference_count: u32, profile_idc: u32, } +impl H265 { + /// The references this frame predicts from: the newest held ones. + fn used_references(&self) -> &[ReferenceInfo] { + let n = self + .l0_references + .len() + .min(self.active_reference_count as usize); + &self.l0_references[..n] + } +} + impl VideoCodec for H265 { fn begin_picture( &mut self, @@ -50,7 +68,7 @@ impl VideoCodec for H265 { if plan.is_idr() { let dpb_config = DpbConfig { dpb_size: common.dpb_slot_count as u32, - max_num_ref_frames: common.config.max_reference_frames, + max_num_ref_frames: self.reference_window, use_multiple_references: common.config.b_frame_count > 0, log2_max_frame_num_minus4: 0, log2_max_pic_order_cnt_lsb_minus4: 4, @@ -117,7 +135,7 @@ impl VideoCodec for H265 { display_order: plan.display_order, }, ); - while self.l0_references.len() > self.active_reference_count as usize { + while self.l0_references.len() > self.reference_window as usize { self.l0_references.pop(); } diff --git a/src/encoder/h265/record.rs b/src/encoder/h265/record.rs index 0e6ec693..a8172bb9 100644 --- a/src/encoder/h265/record.rs +++ b/src/encoder/h265/record.rs @@ -36,7 +36,10 @@ impl H265 { unsafe { prepare_encode_command_buffer(common.device(), command_buffer, query_pool)?; } - let ref_dpb_slots: Vec = self.l0_references.iter().map(|r| r.dpb_slot).collect(); + // Only the references this frame predicts from go to the encode. The + // rest of the window stays in its DPB slots, untouched. + let used_references = self.used_references(); + let ref_dpb_slots: Vec = used_references.iter().map(|r| r.dpb_slot).collect(); unsafe { record_dpb_barriers( common.context.sync2(), @@ -127,7 +130,11 @@ impl H265 { let diff = prev_delta_poc - delta_poc; delta_poc_s0_minus1[num_negative_pics as usize] = (diff - 1).max(0) as u16; prev_delta_poc = delta_poc; - used_by_curr_pic_s0_flag |= 1 << num_negative_pics; + // Every held reference stays in the set, so the decoder keeps + // it too, but only the newest are used by this picture. + if i < used_references.len() { + used_by_curr_pic_s0_flag |= 1 << num_negative_pics; + } num_negative_pics += 1; } if is_b_frame && self.has_backward_reference { @@ -213,8 +220,8 @@ impl H265 { (0, 0) } } else if !is_idr && !self.l0_references.is_empty() { - let count = self.l0_references.len(); - for (i, ref_info) in self.l0_references.iter().enumerate() { + let count = used_references.len(); + for (i, ref_info) in used_references.iter().enumerate() { if i < 15 { ref_list0[i] = ref_info.dpb_slot; } @@ -339,7 +346,7 @@ impl H265 { let has_l1_ref = is_b_frame && self.has_backward_reference; if has_l0_ref { - for ref_info in &self.l0_references { + for ref_info in used_references { ref_resources.push( vk::VideoPictureResourceInfoKHR::default() .coded_offset(vk::Offset2D { x: 0, y: 0 }) @@ -379,7 +386,7 @@ impl H265 { let mut stored_indices_count = 0; let mut slot_infos_iter = h265_slot_infos.iter_mut(); if has_l0_ref { - for ref_info in &self.l0_references { + for ref_info in used_references { if let Some(h265_slot_info) = slot_infos_iter.next() { let ref_slot = vk::VideoReferenceSlotInfoKHR::default() .slot_index(ref_info.dpb_slot as i32) diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index 81fee163..b732aa14 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -413,7 +413,11 @@ pub struct EncodeConfig { /// Number of consecutive B-frames. Always 0: no codec encodes B-frames /// yet, so there is no setter. pub(crate) b_frame_count: u32, - /// Maximum number of reference frames. + /// Maximum number of reference frames kept in the DPB: the window that + /// [`Encoder::invalidate_reference_frames`](crate::Encoder::invalidate_reference_frames) + /// can recover from. Each frame predicts from only as many of the newest + /// as the device allows, which can be fewer. The device's DPB size can + /// also lower it. pub max_reference_frames: u32, /// VBV/HRD virtual buffer size in milliseconds, for /// [`RateControl::Cbr`] and [`RateControl::Vbr`]. @@ -558,7 +562,8 @@ impl EncodeConfig { self } - /// Set the maximum reference frames. + /// Set the maximum reference frames kept in the DPB (see + /// [`max_reference_frames`](Self::max_reference_frames)). pub fn with_max_reference_frames(mut self, count: u32) -> Self { self.max_reference_frames = count; self