diff --git a/crates/recording/src/instant_recording.rs b/crates/recording/src/instant_recording.rs index 759025f1f01..2a2ac9f303e 100644 --- a/crates/recording/src/instant_recording.rs +++ b/crates/recording/src/instant_recording.rs @@ -1332,6 +1332,8 @@ async fn build_instant_recording_actor( max_capture_size, timestamps.system_time(), inputs.capture_system_audio, + #[cfg(windows)] + inputs.capture_target.window(), #[cfg(target_os = "linux")] crate::sources::screen_capture::LinuxCaptureSource::from_target( &inputs.capture_target, diff --git a/crates/recording/src/sources/screen_capture/mod.rs b/crates/recording/src/sources/screen_capture/mod.rs index 6a61479f0fd..e5818a720fe 100644 --- a/crates/recording/src/sources/screen_capture/mod.rs +++ b/crates/recording/src/sources/screen_capture/mod.rs @@ -376,6 +376,8 @@ pub struct Config { crop_bounds: Option, fps: u32, show_cursor: bool, + #[cfg(windows)] + window: Option, #[cfg(target_os = "linux")] linux_source: LinuxCaptureSource, } @@ -469,6 +471,7 @@ impl ScreenCaptureConfig { max_capture_size: Option<(u32, u32)>, start_time: SystemTime, system_audio: bool, + #[cfg(windows)] window: Option, #[cfg(target_os = "linux")] linux_source: LinuxCaptureSource, #[cfg(windows)] d3d_device: ::windows::Win32::Graphics::Direct3D11::ID3D11Device, #[cfg(target_os = "macos")] shareable_content: SendableShareableContent, @@ -529,6 +532,8 @@ impl ScreenCaptureConfig { crop_bounds, fps, show_cursor, + #[cfg(windows)] + window, #[cfg(target_os = "linux")] linux_source, }, diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index 36d5d448386..d554675f0c9 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -1,6 +1,8 @@ use crate::{ AudioFrame, SetupCtx, output_pipeline, - screen_capture::{ScreenCaptureConfig, ScreenCaptureFormat, cadence::FrameCadenceGate}, + screen_capture::{ + CropBounds, ScreenCaptureConfig, ScreenCaptureFormat, cadence::FrameCadenceGate, + }, }; use ::windows::Win32::Graphics::Direct3D11::{ D3D11_BIND_SHADER_RESOURCE, D3D11_BOX, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, @@ -16,7 +18,7 @@ use futures::{ channel::{mpsc, oneshot}, }; use scap_ffmpeg::*; -use scap_targets::{Display, DisplayId}; +use scap_targets::{Display, DisplayId, Window, WindowId}; use std::{ sync::{ Arc, Mutex, @@ -351,13 +353,29 @@ impl output_pipeline::VideoFrame for VideoFrame { } } +/// A window target captures the window's own GraphicsCaptureItem, so the crop +/// that carves the window out of the display is neither needed nor correct: +/// WGC already hands back just that window, and the crop would additionally +/// pin the recording to wherever the window started out. +fn crop_for_target( + window: Option<&WindowId>, + crop_bounds: Option, +) -> Option { + match window { + Some(_) => None, + None => crop_bounds, + } +} + impl ScreenCaptureConfig { pub async fn to_sources( &self, ) -> anyhow::Result<(VideoSourceConfig, Option)> { + let crop_bounds = crop_for_target(self.config.window.as_ref(), self.config.crop_bounds); + let mut settings = scap_direct3d::Settings { pixel_format: Direct3DCapture::PIXEL_FORMAT, - crop: self.config.crop_bounds.map(|b| { + crop: crop_bounds.map(|b| { let position = b.position(); let size = b.size().map(|v| (v / 2.0).floor() * 2.0); @@ -393,12 +411,13 @@ impl ScreenCaptureConfig { settings.fps = Some(self.config.fps); - // Store the display ID instead of GraphicsCaptureItem to avoid COM threading issues + // Store the target's ID instead of GraphicsCaptureItem to avoid COM threading issues // The GraphicsCaptureItem will be created on the capture thread Ok(( VideoSourceConfig { video_info: self.video_info, display_id: self.config.display.clone(), + window_id: self.config.window.clone(), settings, d3d_device: self.d3d_device.clone(), }, @@ -416,6 +435,7 @@ pub enum VideoSourceError { pub struct VideoSourceConfig { video_info: VideoInfo, display_id: DisplayId, + window_id: Option, settings: scap_direct3d::Settings, pub d3d_device: ID3D11Device, } @@ -487,6 +507,7 @@ struct CaptureClosureEvent { struct CreateCapturerParams<'a> { display_id: &'a DisplayId, + window_id: Option<&'a WindowId>, settings: &'a scap_direct3d::Settings, d3d_device: &'a ID3D11Device, video_tx: &'a mpsc::Sender, @@ -511,11 +532,18 @@ fn create_d3d_capturer( params: &CreateCapturerParams, error_tx: &mpsc::Sender, ) -> anyhow::Result { - let capture_item = Display::from_id(params.display_id) - .ok_or_else(|| anyhow!("Display not found for ID: {:?}", params.display_id))? - .raw_handle() - .try_as_capture_item() - .map_err(|e| anyhow!("Failed to create GraphicsCaptureItem: {}", e))?; + let capture_item = match params.window_id { + Some(window_id) => Window::from_id(window_id) + .ok_or_else(|| anyhow!("Window not found for ID: {:?}", window_id))? + .raw_handle() + .try_as_capture_item() + .map_err(|e| anyhow!("Failed to create window GraphicsCaptureItem: {}", e))?, + None => Display::from_id(params.display_id) + .ok_or_else(|| anyhow!("Display not found for ID: {:?}", params.display_id))? + .raw_handle() + .try_as_capture_item() + .map_err(|e| anyhow!("Failed to create GraphicsCaptureItem: {}", e))?, + }; scap_direct3d::Capturer::new( capture_item, @@ -661,6 +689,7 @@ impl output_pipeline::VideoSource for VideoSource { VideoSourceConfig { video_info, display_id, + window_id, settings, d3d_device, }: Self::Config, @@ -723,6 +752,7 @@ impl output_pipeline::VideoSource for VideoSource { ($device:expr) => { CreateCapturerParams { display_id: &display_id, + window_id: window_id.as_ref(), settings: &settings, d3d_device: $device, video_tx: &video_tx, @@ -1598,3 +1628,30 @@ mod first_screen_frame_tests { assert!(error.to_string().contains("closed before its first frame")); } } + +#[cfg(test)] +mod capture_target_tests { + use super::*; + use scap_targets::bounds::{PhysicalPosition, PhysicalSize}; + + fn bounds() -> CropBounds { + CropBounds::new( + PhysicalPosition::new(120.0, 80.0), + PhysicalSize::new(640.0, 480.0), + ) + } + + #[test] + fn a_display_target_keeps_its_crop() { + let kept = crop_for_target(None, Some(bounds())).expect("crop kept"); + assert_eq!(kept.position().x(), 120.0); + assert_eq!(kept.size().width(), 640.0); + assert!(crop_for_target(None, None).is_none()); + } + + #[test] + fn a_window_target_captures_the_window_rather_than_a_crop_of_the_display() { + let window: WindowId = "1234".parse().unwrap(); + assert!(crop_for_target(Some(&window), Some(bounds())).is_none()); + } +} diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 1ff44d06f1b..4ab4e1787ec 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -3270,6 +3270,8 @@ async fn create_segment_pipeline( max_capture_size, start_time.system_time(), base_inputs.capture_system_audio, + #[cfg(windows)] + capture_target.window(), #[cfg(target_os = "linux")] sources::screen_capture::LinuxCaptureSource::from_target(&capture_target), #[cfg(windows)] diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index dbe42566731..9c81a109372 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -506,14 +506,18 @@ impl Capturer { .map(|fps| ((fps as f32 / 30.0 * 2.0).ceil() as i32).clamp(2, 4)) .unwrap_or(2); + let item_size = item.Size().map_err(NewCapturerError::ItemSize)?; + let frame_pool = Direct3D11CaptureFramePool::CreateFreeThreaded( &direct3d_device, settings.pixel_format.as_directx(), frame_pool_size, - item.Size().map_err(NewCapturerError::ItemSize)?, + item_size, ) .map_err(NewCapturerError::FramePool)?; + let pool_size = Arc::new(Mutex::new((item_size.Width, item_size.Height))); + let session = frame_pool .CreateCaptureSession(&item) .map_err(NewCapturerError::CaptureSession)?; @@ -575,6 +579,8 @@ impl Capturer { let stop_flag = stop_flag.clone(); let staging_pool = staging_pool.clone(); let callback_activity = callback_activity.clone(); + let pixel_format = settings.pixel_format; + let pool_size = pool_size.clone(); move |frame_pool, _| { let _activity_guard = callback_activity.enter(); @@ -591,11 +597,51 @@ impl Capturer { let size = frame.ContentSize()?; + // The item can change size mid-capture (a recorded + // window is resized, or the display's resolution + // changes). The pool keeps handing out surfaces at its + // creation size, so the frame's own ContentSize stops + // describing its texture and every later frame reads + // back as black. Recreate the pool for the new size and + // skip this frame; the next one arrives correctly sized. + { + let mut pool_size = pool_size + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if (size.Width, size.Height) != *pool_size { + drop(frame); + // IDirect3DDevice is not Send, so it is rebuilt + // here from the capture device rather than held + // across the event handler. + let dxgi_device = d3d_device.cast::()?; + let direct3d_device = + unsafe { CreateDirect3D11DeviceFromDXGIDevice(&dxgi_device) }? + .cast::()?; + frame_pool.Recreate( + &direct3d_device, + pixel_format.as_directx(), + frame_pool_size, + size, + )?; + *pool_size = (size.Width, size.Height); + return Ok(()); + } + } + let surface = frame.Surface()?; let dxgi_interface = surface.cast::()?; let texture = unsafe { dxgi_interface.GetInterface::() }?; let frame = if let Some((cropped_texture, crop)) = crop_data.clone() { + // A shrunk item leaves the crop box hanging off the + // edge of the source texture, which copies nothing + // and would vend a stale surface as if it were live. + if crop.right > size.Width.max(0) as u32 + || crop.bottom > size.Height.max(0) as u32 + { + return Ok(()); + } + unsafe { d3d_context.CopySubresourceRegion( &cropped_texture, diff --git a/crates/scap-direct3d/tests/window_capture.rs b/crates/scap-direct3d/tests/window_capture.rs new file mode 100644 index 00000000000..bc0769c6944 --- /dev/null +++ b/crates/scap-direct3d/tests/window_capture.rs @@ -0,0 +1,414 @@ +#![cfg(target_os = "windows")] + +//! Window capture regressions for CapSoftware/Cap#2151. +//! +//! The fixture process owns the windows because `Window::list()` skips the +//! calling process, the same arrangement `scap-targets`' window discovery test +//! uses. + +use scap_direct3d::{Capturer, PixelFormat, Settings}; +use scap_targets::{Window, WindowId}; +use std::{ + io::{BufRead, BufReader, Write}, + process::{Child, Command, Stdio}, + sync::{Arc, Mutex, Once, mpsc}, + time::{Duration, Instant}, +}; +use windows::{ + Win32::{ + Foundation::{COLORREF, HWND, LPARAM, LRESULT, WPARAM}, + Graphics::{ + Direct3D11::D3D11_BOX, + Gdi::{CreateSolidBrush, InvalidateRect}, + }, + UI::{ + HiDpi::{PROCESS_PER_MONITOR_DPI_AWARE, SetProcessDpiAwareness}, + WindowsAndMessaging::{ + CS_HREDRAW, CS_VREDRAW, CreateWindowExW, DefWindowProcW, DispatchMessageW, + HWND_TOPMOST, MSG, PM_REMOVE, PeekMessageW, RegisterClassW, SWP_NOACTIVATE, + SWP_SHOWWINDOW, SetWindowPos, TranslateMessage, WINDOW_EX_STYLE, WNDCLASSW, + WS_EX_TOOLWINDOW, WS_EX_TOPMOST, WS_OVERLAPPEDWINDOW, WS_POPUP, WS_VISIBLE, + }, + }, + }, + core::{PCWSTR, w}, +}; + +const TARGET_X: i32 = 200; +const TARGET_Y: i32 = 200; +const TARGET_W: i32 = 800; +const TARGET_H: i32 = 600; +const OVERLAY_W: i32 = 300; +const OVERLAY_H: i32 = 200; +const RESIZED_W: i32 = 1100; +const RESIZED_H: i32 = 700; + +struct FixtureProcess(Child); + +impl Drop for FixtureProcess { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +/// Process-wide and only settable once, so both tests here share one call. +fn match_cli_dpi_awareness() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + unsafe { SetProcessDpiAwareness(PROCESS_PER_MONITOR_DPI_AWARE) } + .expect("match CLI per-monitor DPI awareness"); + }); +} + +unsafe extern "system" fn window_proc( + window: HWND, + message: u32, + wparam: WPARAM, + lparam: LPARAM, +) -> LRESULT { + unsafe { DefWindowProcW(window, message, wparam, lparam) } +} + +fn register_class(class: PCWSTR, colour: u32) { + let class = WNDCLASSW { + style: CS_HREDRAW | CS_VREDRAW, + lpfnWndProc: Some(window_proc), + hbrBackground: unsafe { CreateSolidBrush(COLORREF(colour)) }, + lpszClassName: class, + ..Default::default() + }; + assert!(unsafe { RegisterClassW(&class) } != 0, "register class"); +} + +/// Runs in the fixture process: a blue window with a red always-on-top window +/// covering part of it, resized once `resize` is read on stdin. +#[test] +fn window_capture_fixture_process() { + if std::env::var_os("CAP_WINDOW_CAPTURE_FIXTURE").is_none() { + return; + } + + match_cli_dpi_awareness(); + + // COLORREF is 0x00BBGGRR. + register_class(w!("Cap2151Target"), 0x00FF_0000); + register_class(w!("Cap2151Overlay"), 0x0000_00FF); + + let target = unsafe { + CreateWindowExW( + WINDOW_EX_STYLE(0), + w!("Cap2151Target"), + w!("Cap2151 target"), + WS_OVERLAPPEDWINDOW | WS_VISIBLE, + TARGET_X, + TARGET_Y, + TARGET_W, + TARGET_H, + None, + None, + None, + None, + ) + } + .expect("create target window"); + let overlay = unsafe { + CreateWindowExW( + WS_EX_TOPMOST | WS_EX_TOOLWINDOW, + w!("Cap2151Overlay"), + w!("Cap2151 overlay"), + WS_POPUP | WS_VISIBLE, + TARGET_X + 200, + TARGET_Y + 150, + OVERLAY_W, + OVERLAY_H, + None, + None, + None, + None, + ) + } + .expect("create overlay window"); + unsafe { + SetWindowPos( + overlay, + Some(HWND_TOPMOST), + TARGET_X + 200, + TARGET_Y + 150, + OVERLAY_W, + OVERLAY_H, + SWP_SHOWWINDOW | SWP_NOACTIVATE, + ) + } + .expect("raise overlay"); + + println!("WINDOW_CAPTURE_FIXTURE {}", target.0 as u64); + std::io::stdout().flush().unwrap(); + + let (command_tx, command_rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(std::io::stdin()).lines() { + let Ok(line) = line else { break }; + if command_tx.send(line).is_err() { + break; + } + } + }); + + let started = Instant::now(); + while started.elapsed() < Duration::from_secs(60) { + match command_rx.try_recv() { + Ok(line) if line.trim() == "resize" => unsafe { + SetWindowPos( + target, + None, + TARGET_X, + TARGET_Y, + RESIZED_W, + RESIZED_H, + SWP_SHOWWINDOW | SWP_NOACTIVATE, + ) + .expect("resize target"); + }, + Ok(line) if line.trim() == "stop" => return, + Ok(_) | Err(mpsc::TryRecvError::Empty) => {} + Err(mpsc::TryRecvError::Disconnected) => return, + } + + // WGC only delivers a frame when the content changes, so keep the + // window repainting for the duration of the capture. + let _ = unsafe { InvalidateRect(Some(target), None, true) }; + let mut message = MSG::default(); + while unsafe { PeekMessageW(&mut message, None, 0, 0, PM_REMOVE) }.as_bool() { + let _ = unsafe { TranslateMessage(&message) }; + unsafe { DispatchMessageW(&message) }; + } + std::thread::sleep(Duration::from_millis(16)); + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Sample { + width: u32, + height: u32, + red: usize, + blue: usize, +} + +fn capture_samples( + item: windows::Graphics::Capture::GraphicsCaptureItem, + crop: Option, + duration: Duration, + during: impl FnOnce(), +) -> Vec { + let samples: Arc>> = Arc::new(Mutex::new(Vec::new())); + let sink = samples.clone(); + let mut capturer = Capturer::new( + item, + Settings { + is_border_required: Some(false), + is_cursor_capture_enabled: Some(false), + pixel_format: PixelFormat::R8G8B8A8Unorm, + crop, + fps: Some(30), + ..Default::default() + }, + move |frame| { + let buffer = frame.as_buffer()?; + let (width, height, stride) = ( + buffer.width() as usize, + buffer.height() as usize, + buffer.stride() as usize, + ); + let data = buffer.data(); + let (mut red, mut blue) = (0usize, 0usize); + for y in 0..height { + for x in 0..width { + let pixel = &data[y * stride + x * 4..y * stride + x * 4 + 4]; + let (r, g, b) = (pixel[0], pixel[1], pixel[2]); + if r > 200 && g < 60 && b < 60 { + red += 1; + } else if b > 200 && r < 60 && g < 60 { + blue += 1; + } + } + } + sink.lock().unwrap().push(Sample { + width: width as u32, + height: height as u32, + red, + blue, + }); + Ok(()) + }, + || Ok(()), + None, + ) + .expect("create capturer"); + + capturer.start().expect("start capture"); + during(); + std::thread::sleep(duration); + capturer.stop().expect("stop capture"); + let samples = samples.lock().unwrap(); + samples.clone() +} + +struct Fixture { + process: FixtureProcess, + target: WindowId, +} + +impl Fixture { + fn start() -> Self { + let mut process = FixtureProcess( + Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "window_capture_fixture_process", "--nocapture"]) + .env("CAP_WINDOW_CAPTURE_FIXTURE", "1") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("start window capture fixture"), + ); + let stdout = process.0.stdout.take().unwrap(); + let (ready_tx, ready_rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + let Ok(line) = line else { break }; + if let Some((_, id)) = line.split_once("WINDOW_CAPTURE_FIXTURE ") { + let _ = ready_tx.send(id.to_string()); + } + } + }); + let target: WindowId = ready_rx + .recv_timeout(Duration::from_secs(15)) + .expect("fixture must become ready") + .trim() + .parse() + .expect("parse fixture window id"); + // Give the windows a moment to paint before capturing them. + std::thread::sleep(Duration::from_millis(800)); + Self { process, target } + } + + fn window(&self) -> Window { + Window::from_id(&self.target).expect("fixture window must be discoverable") + } + + fn send(&mut self, command: &str) { + let stdin = self.process.0.stdin.as_mut().expect("fixture stdin"); + stdin.write_all(command.as_bytes()).unwrap(); + stdin.write_all(b"\n").unwrap(); + stdin.flush().unwrap(); + } +} + +/// Capturing a window through the display's capture item sees whatever is +/// stacked on top of it; capturing the window's own item does not. +#[test] +fn window_item_excludes_windows_stacked_on_top() { + match_cli_dpi_awareness(); + + let fixture = Fixture::start(); + let window = fixture.window(); + let display = window.display().expect("window display"); + let window_bounds = window + .raw_handle() + .physical_bounds() + .expect("window bounds"); + let display_position = display + .raw_handle() + .physical_position() + .expect("display position"); + + let crop = D3D11_BOX { + left: (window_bounds.position().x() - display_position.x()).max(0.0) as u32, + top: (window_bounds.position().y() - display_position.y()).max(0.0) as u32, + right: (window_bounds.position().x() - display_position.x() + window_bounds.size().width()) + .max(0.0) as u32, + bottom: (window_bounds.position().y() - display_position.y() + + window_bounds.size().height()) + .max(0.0) as u32, + front: 0, + back: 1, + }; + + let cropped = capture_samples( + display + .raw_handle() + .try_as_capture_item() + .expect("display capture item"), + Some(crop), + Duration::from_secs(2), + || {}, + ); + let cropped = cropped.last().copied().expect("cropped display frames"); + assert!( + cropped.red > 0, + "a display capture cropped to the window must show the window stacked on top of it, \ + which is the bug being fixed: {cropped:?}" + ); + + let window_item = capture_samples( + window + .raw_handle() + .try_as_capture_item() + .expect("window capture item"), + None, + Duration::from_secs(2), + || {}, + ); + let window_item = window_item.last().copied().expect("window frames"); + assert_eq!( + window_item.red, 0, + "the window's own capture item must not contain the window stacked on top of it: \ + {window_item:?}" + ); + assert!( + window_item.blue > 0, + "the window's own capture item must contain the window: {window_item:?}" + ); +} + +/// The frame pool is created at the item's size; when the item resizes, frames +/// keep arriving with the new ContentSize but the pool's surfaces are still the +/// old size, so every frame reads back empty until the pool is recreated. +#[test] +fn frames_survive_the_capture_item_resizing() { + match_cli_dpi_awareness(); + + let mut fixture = Fixture::start(); + let window = fixture.window(); + let before = window.physical_size().expect("window size"); + + let samples = capture_samples( + window + .raw_handle() + .try_as_capture_item() + .expect("window capture item"), + None, + Duration::from_secs(4), + || { + std::thread::sleep(Duration::from_millis(500)); + fixture.send("resize"); + }, + ); + + let grown: Vec<_> = samples + .iter() + .filter(|sample| sample.width > before.width() as u32) + .collect(); + assert!( + !grown.is_empty(), + "the capture must follow the window's new size, saw {:?}", + samples + .iter() + .map(|s| (s.width, s.height)) + .collect::>() + ); + let last = grown.last().expect("a frame after the resize"); + assert!( + last.blue > 0, + "frames after a resize must still carry the window's contents: {last:?}" + ); +}