diff --git a/.github/workflows/build-native.yml b/.github/workflows/build-native.yml index 4c137c691..28d4ae9f7 100644 --- a/.github/workflows/build-native.yml +++ b/.github/workflows/build-native.yml @@ -14,6 +14,8 @@ # # audio-context-windows- audiocontext.node packages/audio-context/platforms/windows// # +# canvas-media-windows- canvasmedia.node packages/canvas-media/platforms/windows// +# # # xcframeworks are tarred before upload because upload-artifact does not # preserve file permissions. @@ -23,15 +25,17 @@ # (what `npm run build.canvas.ios.framework.release` runs, minus nx) # Android: make android, then gradle :canvas:assembleRelease and copy the aar # -# Windows: tools/scripts/build-napi.sh release-napi [canvas-svg-napi|audio-context-napi] -# (`make windows`, `make windows-svg`, `make windows-audio`); Skia builds from source +# Windows: tools/scripts/build-napi.sh release-napi [] (`make windows`, +# `make windows-svg`, `make windows-audio`, `make windows-media`); the modules other +# than canvasnative.node share one matrix job (napi-windows). Skia builds from source # with the runner's LLVM and ninja, as locally with .tools/env.sh # (.github/actions/setup-windows-native). arm64 is cross-compiled; the x64 jobs also # run the Node-API test suites on the shipped modules (on WARP: runners have no GPU; # audio renders offline and on the 'none' sink: they have no audio device either). # -# canvas-svg and audio-context change rarely, so their jobs cache the final -# artifact keyed on a hash of their sources and skip the whole build on a hit. +# The napi-windows modules and the canvas-svg / audio-context mobile builds change rarely, so +# their jobs cache the final artifact keyed on a hash of their sources and skip the whole build +# on a hit. name: Build Native permissions: @@ -54,7 +58,7 @@ on: default: '' env: - # bump to invalidate the canvas-svg / audio-context artifact caches + # bump to invalidate the canvas-svg / audio-context / canvas-media artifact caches NATIVE_CACHE_VERSION: v1 jobs: @@ -284,12 +288,32 @@ jobs: path: packages/canvas/platforms/windows/${{ matrix.arch }}/ if-no-files-found: error - canvas-svg-windows: - name: Canvas SVG Windows (${{ matrix.arch }}) + # The Node-API modules other than canvasnative.node (no Skia, no ANGLE): one entry per module. + napi-windows: + name: ${{ matrix.module.name }} Windows (${{ matrix.arch }}) runs-on: windows-2025 strategy: fail-fast: false matrix: + module: + # `sources`: every tracked directory the module is built from (its cache key). + - name: canvas-svg + crate: canvas-svg-napi + file: canvassvg.node + addon-env: CANVAS_SVG_NAPI_ADDON + sources: crates/canvas-svg crates/canvas-svg-c crates/canvas-svg-napi crates/canvas-2d crates/canvas-core + # Runners have no audio device: its suite renders offline and on the 'none' sink. + - name: audio-context + crate: audio-context-napi + file: audiocontext.node + addon-env: AUDIO_CONTEXT_NAPI_ADDON + sources: crates/audio-context-napi + - name: canvas-media + crate: canvas-media-napi + file: canvasmedia.node + addon-env: CANVAS_MEDIA_NAPI_ADDON + sources: crates/canvas-media-napi + arch: [x64, arm64] include: - arch: x64 target: x86_64-pc-windows-msvc @@ -298,83 +322,48 @@ jobs: defaults: run: shell: bash + env: + MODULE_DIR: packages/${{ matrix.module.name }}/platforms/windows/${{ matrix.arch }} steps: - uses: actions/checkout@v4 with: ref: ${{ github.event_name == 'workflow_dispatch' && github.event.repository.default_branch || inputs.ref }} + # The tree hashes of what the module is built from: equal hashes, equal module. + - name: Source key + id: sources + run: | + key=$(for path in ${{ matrix.module.sources }} crates/vendor/napi-sys Cargo.toml tools/scripts/build-napi.sh; do git rev-parse "HEAD:$path" || exit 1; done | sha256sum | cut -c1-32) + echo "key=$key" >> "$GITHUB_OUTPUT" + # Not on manual runs (CodeQL: cache poisoning via untrusted files); those always build. - name: Restore cached module if: github.event_name != 'workflow_dispatch' - id: svg-node + id: cache uses: actions/cache@v4 with: - path: packages/canvas-svg/platforms/windows/${{ matrix.arch }}/canvassvg.node - key: ${{ env.NATIVE_CACHE_VERSION }}-canvas-svg-windows-${{ matrix.arch }}-${{ hashFiles('crates/canvas-svg/**', 'crates/canvas-svg-c/**', 'crates/canvas-svg-napi/**', 'crates/canvas-2d/**', 'crates/canvas-core/**', 'crates/vendor/napi-sys/**', 'Cargo.toml', 'Cargo.lock', 'tools/scripts/build-napi.sh') }} + path: ${{ env.MODULE_DIR }}/${{ matrix.module.file }} + key: ${{ env.NATIVE_CACHE_VERSION }}-${{ matrix.module.name }}-windows-${{ matrix.arch }}-${{ steps.sources.outputs.key }} + # No Skia here; the action's Rust toolchain and cargo cache are what these need. - uses: ./.github/actions/setup-windows-native - if: steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch' + if: steps.cache.outputs.cache-hit != 'true' with: target: ${{ matrix.target }} - uses: actions/setup-node@v4 - if: (steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch') && matrix.arch == 'x64' + if: steps.cache.outputs.cache-hit != 'true' && matrix.arch == 'x64' with: node-version: '22' - - name: Build canvassvg.node - if: steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch' - run: ./tools/scripts/build-napi.sh ${{ matrix.target }} release-napi canvas-svg-napi + - name: Build ${{ matrix.module.file }} + if: steps.cache.outputs.cache-hit != 'true' + run: ./tools/scripts/build-napi.sh ${{ matrix.target }} release-napi ${{ matrix.module.crate }} + # arm64 is cross-compiled, so only x64 runs them. The timeout bounds a test process that + # hangs on exit (an addon releasing Media Foundation / D3D objects during process shutdown). - name: Node-API tests - if: (steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch') && matrix.arch == 'x64' - env: - CANVAS_SVG_NAPI_ADDON: ${{ github.workspace }}\packages\canvas-svg\platforms\windows\x64\canvassvg.node - run: node --expose-gc --test "crates/canvas-svg-napi/__test__/*.test.mjs" - - uses: actions/upload-artifact@v4 - with: - name: canvas-svg-windows-${{ matrix.arch }} - path: packages/canvas-svg/platforms/windows/${{ matrix.arch }}/ - if-no-files-found: error - - audio-context-windows: - name: Audio Context Windows (${{ matrix.arch }}) - runs-on: windows-2025 - strategy: - fail-fast: false - matrix: - include: - - arch: x64 - target: x86_64-pc-windows-msvc - - arch: arm64 - target: aarch64-pc-windows-msvc - defaults: - run: - shell: bash - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref }} - - name: Restore cached module - id: audio-node - uses: actions/cache@v4 - with: - path: packages/audio-context/platforms/windows/${{ matrix.arch }}/audiocontext.node - key: ${{ env.NATIVE_CACHE_VERSION }}-audio-context-windows-${{ matrix.arch }}-${{ hashFiles('crates/audio-context-napi/**', 'crates/vendor/napi-sys/**', 'Cargo.toml', 'Cargo.lock', 'tools/scripts/build-napi.sh') }} - # No Skia here; the action's Rust toolchain and cargo cache are what this job needs. - - uses: ./.github/actions/setup-windows-native - if: steps.audio-node.outputs.cache-hit != 'true' - with: - target: ${{ matrix.target }} - - uses: actions/setup-node@v4 - if: steps.audio-node.outputs.cache-hit != 'true' && matrix.arch == 'x64' - with: - node-version: '22' - - name: Build audiocontext.node - if: steps.audio-node.outputs.cache-hit != 'true' - run: ./tools/scripts/build-napi.sh ${{ matrix.target }} release-napi audio-context-napi - # Runners have no audio device: the suite renders offline and on the 'none' sink. - - name: Node-API tests - if: steps.audio-node.outputs.cache-hit != 'true' && matrix.arch == 'x64' - env: - AUDIO_CONTEXT_NAPI_ADDON: ${{ github.workspace }}\packages\audio-context\platforms\windows\x64\audiocontext.node - run: node --expose-gc --test "crates/audio-context-napi/__test__/*.test.mjs" + if: steps.cache.outputs.cache-hit != 'true' && matrix.arch == 'x64' + timeout-minutes: 5 + run: | + export ${{ matrix.module.addon-env }}="$(cygpath -w "$PWD/$MODULE_DIR/${{ matrix.module.file }}")" + node --expose-gc --test "crates/${{ matrix.module.crate }}/__test__/*.test.mjs" - uses: actions/upload-artifact@v4 with: - name: audio-context-windows-${{ matrix.arch }} - path: packages/audio-context/platforms/windows/${{ matrix.arch }}/ + name: ${{ matrix.module.name }}-windows-${{ matrix.arch }} + path: ${{ env.MODULE_DIR }}/ if-no-files-found: error diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 15c7506e2..e9bf092c1 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -96,7 +96,7 @@ jobs: tar -xzf native-artifacts/audio-context-ios/audio-context-ios.tar.gz -C packages/audio-context/platforms/ios # Node-API modules (+ ANGLE DLLs) per Windows arch for arch in x64 arm64; do - for pkg in canvas canvas-svg audio-context; do + for pkg in canvas canvas-svg audio-context canvas-media; do rm -rf "packages/$pkg/platforms/windows/$arch" mkdir -p "packages/$pkg/platforms/windows/$arch" cp native-artifacts/$pkg-windows-$arch/* "packages/$pkg/platforms/windows/$arch/" diff --git a/Cargo.toml b/Cargo.toml index c81120cc4..2675f76fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "crates/canvas-napi", "crates/canvas-svg-napi", "crates/audio-context-napi", + "crates/canvas-media-napi", "crates/canvas-svg-ios", "crates/canvas-svg-android", ] diff --git a/Makefile b/Makefile index f5d0cade1..8517b8711 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,8 @@ windows-svg: $(addsuffix _svg_napi,$(ARCHS_WINDOWS)) windows-audio: $(addsuffix _audio_napi,$(ARCHS_WINDOWS)) +windows-media: $(addsuffix _media_napi,$(ARCHS_WINDOWS)) + ios-svg: GENERATE_IOS_SVG visionos-svg: GENERATE_VISIONOS_SVG @@ -98,6 +100,10 @@ $(addsuffix _svg_napi,$(ARCHS_WINDOWS)): %_svg_napi: $(addsuffix _audio_napi,$(ARCHS_WINDOWS)): %_audio_napi: ./tools/scripts/build-napi.sh $* release-napi audio-context-napi +.PHONY: windows-media $(addsuffix _media_napi,$(ARCHS_WINDOWS)) +$(addsuffix _media_napi,$(ARCHS_WINDOWS)): %_media_napi: + ./tools/scripts/build-napi.sh $* release-napi canvas-media-napi + .PHONY: GENERATE_ANDROID GENERATE_ANDROID: $(ARCHS_ANDROID) diff --git a/crates/audio-context-napi/src/context.rs b/crates/audio-context-napi/src/context.rs index 014e941b3..8a7bbc51f 100644 --- a/crates/audio-context-napi/src/context.rs +++ b/crates/audio-context-napi/src/context.rs @@ -232,6 +232,15 @@ impl AudioContext { Some(AudioParam::wrap(param)) } + #[napi] + pub fn create_media_element_source_from_tap(&self, address: f64) -> Result { + let context = self.online()?; + let stream = unsafe { crate::tap::TapStream::new(address as usize) } + .ok_or_else(|| error("InvalidStateError: the media element has no audio tap"))?; + let track = web_audio_api::media_streams::MediaStreamTrack::from_iter(stream); + guard(|| AudioNode::new(Kind::MediaElementSource(context.create_media_stream_track_source(&track)))) + } + #[napi] pub fn create_gain(&self) -> Result { self.node(|base| Kind::Gain(base.create_gain())) diff --git a/crates/audio-context-napi/src/lib.rs b/crates/audio-context-napi/src/lib.rs index 1af192897..48107b73d 100644 --- a/crates/audio-context-napi/src/lib.rs +++ b/crates/audio-context-napi/src/lib.rs @@ -10,6 +10,7 @@ mod buffer; mod context; mod node; mod param; +mod tap; pub use buffer::{AudioBuffer, PeriodicWave}; pub use context::AudioContext; diff --git a/crates/audio-context-napi/src/node.rs b/crates/audio-context-napi/src/node.rs index 71eba487f..755ba589d 100644 --- a/crates/audio-context-napi/src/node.rs +++ b/crates/audio-context-napi/src/node.rs @@ -28,6 +28,7 @@ pub(crate) enum Kind { DynamicsCompressor(node::DynamicsCompressorNode), ChannelSplitter(node::ChannelSplitterNode), ChannelMerger(node::ChannelMergerNode), + MediaElementSource(node::MediaStreamTrackAudioSourceNode), } /// Evaluates `$body` with `$node` bound to whichever node `$kind` holds. @@ -50,6 +51,7 @@ macro_rules! any_node { Kind::DynamicsCompressor($node) => $body, Kind::ChannelSplitter($node) => $body, Kind::ChannelMerger($node) => $body, + Kind::MediaElementSource($node) => $body, } }; } @@ -102,6 +104,7 @@ impl AudioNode { Kind::DynamicsCompressor(_) => "DynamicsCompressorNode", Kind::ChannelSplitter(_) => "ChannelSplitterNode", Kind::ChannelMerger(_) => "ChannelMergerNode", + Kind::MediaElementSource(_) => "MediaElementAudioSourceNode", } } diff --git a/crates/audio-context-napi/src/tap.rs b/crates/audio-context-napi/src/tap.rs new file mode 100644 index 000000000..07432cb17 --- /dev/null +++ b/crates/audio-context-napi/src/tap.rs @@ -0,0 +1,79 @@ +use std::ffi::c_void; + +use web_audio_api::AudioBuffer; + +/// Same layout as `tap::AudioTapSource` in canvas-media's module. +#[repr(C)] +#[derive(Clone, Copy)] +struct AudioTapSource { + size: u32, + reserved: u32, + tap: *const c_void, + read: unsafe extern "C" fn(tap: *const c_void, out: *mut f32, capacity: usize, channels: *mut u32, sample_rate: *mut u32) -> usize, + retain: unsafe extern "C" fn(tap: *const c_void), + release: unsafe extern "C" fn(tap: *const c_void), +} + +const FRAMES: usize = 256; +const MAX_CHANNELS: usize = 8; + +/// Pulled on the render thread: a read never waits, and an empty tap is silence. +pub struct TapStream { + source: AudioTapSource, + scratch: Vec, + channels: usize, + sample_rate: f32, +} + +// The tap is Send + Sync on the producer's side and only reached through `read`. +unsafe impl Send for TapStream {} +unsafe impl Sync for TapStream {} + +impl TapStream { + /// # Safety + /// `address` must be an `NSCAudioTap.address` whose tap is alive for the duration of the call; + /// the stream then holds its own reference. + pub unsafe fn new(address: usize) -> Option { + let source = (address as *const AudioTapSource).as_ref()?; + if source.size as usize != std::mem::size_of::() || source.tap.is_null() { + return None; + } + (source.retain)(source.tap); + Some(TapStream { + source: *source, + scratch: vec![0.; FRAMES * MAX_CHANNELS], + channels: 2, + sample_rate: 48000., + }) + } +} + +impl Drop for TapStream { + fn drop(&mut self) { + unsafe { (self.source.release)(self.source.tap) }; + } +} + +impl Iterator for TapStream { + type Item = Result>; + + fn next(&mut self) -> Option { + let (mut channels, mut sample_rate) = (0u32, 0u32); + let frames = unsafe { + (self.source.read)(self.source.tap, self.scratch.as_mut_ptr(), self.scratch.len(), &mut channels, &mut sample_rate) + }; + if channels > 0 && (channels as usize) <= MAX_CHANNELS && sample_rate > 0 { + self.channels = channels as usize; + self.sample_rate = sample_rate as f32; + } + let channel_count = self.channels; + let data: Vec> = if frames == 0 { + vec![vec![0.; FRAMES]; channel_count] + } else { + (0..channel_count) + .map(|channel| (0..frames).map(|frame| self.scratch[frame * channel_count + channel]).collect()) + .collect() + }; + Some(Ok(AudioBuffer::from(data, self.sample_rate))) + } +} diff --git a/crates/canvas-c/Cargo.toml b/crates/canvas-c/Cargo.toml index ebd10bb0d..629fed236 100644 --- a/crates/canvas-c/Cargo.toml +++ b/crates/canvas-c/Cargo.toml @@ -61,6 +61,8 @@ objc2-core-foundation = { workspace = true } [target.'cfg(target_os="windows")'.dependencies] wgpu-core = { workspace = true, features = ["wgsl", "dx12"] } wgpu-hal = { workspace = true, features = ["dx12"] } +# gpu_shared_frame: D3D12 types for wgpu-hal's texture_from_raw (it does not re-export them). +windows = { workspace = true, features = ["Win32_Foundation", "Win32_Graphics_Direct3D12"] } [target.'cfg(target_os="android")'.dependencies] ndk = { version = "0.7.0", features = ["bitmap"] } diff --git a/crates/canvas-c/src/webgpu/gpu_adapter.rs b/crates/canvas-c/src/webgpu/gpu_adapter.rs index 254325884..2ef1fabde 100644 --- a/crates/canvas-c/src/webgpu/gpu_adapter.rs +++ b/crates/canvas-c/src/webgpu/gpu_adapter.rs @@ -168,6 +168,8 @@ pub extern "C" fn canvas_native_webgpu_adapter_request_device( }), error_sink: error_sink.clone(), blit: Arc::new(parking_lot::Mutex::new(None)), + #[cfg(target_os = "windows")] + shared_frames: Default::default(), }); let ret = Arc::into_raw(Arc::new(CanvasGPUDevice { diff --git a/crates/canvas-c/src/webgpu/gpu_external_texture.rs b/crates/canvas-c/src/webgpu/gpu_external_texture.rs index c3c55e6ed..961686182 100644 --- a/crates/canvas-c/src/webgpu/gpu_external_texture.rs +++ b/crates/canvas-c/src/webgpu/gpu_external_texture.rs @@ -56,12 +56,11 @@ pub unsafe extern "C" fn canvas_native_webgpu_device_import_external_texture( return std::ptr::null(); } - let Some(plane) = super::gpu_native_texture::import_platform_texture( - &device.device, - native_texture, - width, - height, - ) else { + #[cfg(target_os = "windows")] + let plane = super::gpu_shared_frame::import_external_plane(device, native_texture, width, height); + #[cfg(not(target_os = "windows"))] + let plane = super::gpu_native_texture::import_platform_texture(&device.device, native_texture, width, height); + let Some(plane) = plane else { return std::ptr::null(); }; diff --git a/crates/canvas-c/src/webgpu/gpu_native_texture.rs b/crates/canvas-c/src/webgpu/gpu_native_texture.rs index 001a17a01..af06d9c9c 100644 --- a/crates/canvas-c/src/webgpu/gpu_native_texture.rs +++ b/crates/canvas-c/src/webgpu/gpu_native_texture.rs @@ -340,6 +340,7 @@ pub(crate) unsafe fn import_platform_texture( target_os = "tvos", target_os = "android" )))] +#[allow(dead_code)] // Windows imports through gpu_shared_frame. pub(crate) unsafe fn import_platform_texture( _device: &Arc, _handle: *mut c_void, @@ -366,13 +367,49 @@ unsafe fn blit_frame_into_texture( ) -> bool { let device = &queue.device_id; - let Some(source_texture) = import_platform_texture(device, handle, frame_width, frame_height) - else { + #[cfg(target_os = "windows")] + let source_texture = super::gpu_shared_frame::import_and_stage(queue, handle, frame_width, frame_height); + #[cfg(not(target_os = "windows"))] + let source_texture = import_platform_texture(device, handle, frame_width, frame_height); + let Some(source_texture) = source_texture else { return false; }; + let _ = device; - let destination_texture = &*destination.texture; - let destination_descriptor = destination_texture.texture.descriptor(); + blit_texture( + queue, + &source_texture, + frame_width, + frame_height, + source_origin_x, + source_origin_y, + flip_y, + &(*destination.texture).texture, + destination.mip_level, + destination.origin.z, + (destination.origin.x, destination.origin.y), + size, + ) +} + +/// Only samples `source_texture`, and submits. `false` when the destination is not renderable. +#[allow(clippy::too_many_arguments)] +pub(crate) unsafe fn blit_texture( + queue: &CanvasGPUQueue, + source_texture: &Arc, + frame_width: u32, + frame_height: u32, + source_origin_x: u32, + source_origin_y: u32, + flip_y: bool, + destination_texture: &Arc, + destination_mip_level: u32, + destination_layer: u32, + destination_origin: (u32, u32), + size: wgt::Extent3d, +) -> bool { + let device = &queue.device_id; + let destination_descriptor = destination_texture.descriptor(); let destination_format = destination_descriptor.format; // The blit renders into the destination, so it needs RENDER_ATTACHMENT. The WebGPU @@ -392,21 +429,18 @@ unsafe fn blit_frame_into_texture( }); // Render into the destination's requested mip level and array layer only. - let destination_view = - destination_texture - .texture - .create_view(&wgpu_core::resource::TextureViewDescriptor { - label: Some(Cow::Borrowed("videoBlit:DestinationView")), - dimension: Some(wgt::TextureViewDimension::D2), - range: wgt::ImageSubresourceRange { - aspect: wgt::TextureAspect::All, - base_mip_level: destination.mip_level, - mip_level_count: Some(1), - base_array_layer: destination.origin.z, - array_layer_count: Some(1), - }, - ..Default::default() - }); + let destination_view = destination_texture.create_view(&wgpu_core::resource::TextureViewDescriptor { + label: Some(Cow::Borrowed("videoBlit:DestinationView")), + dimension: Some(wgt::TextureViewDimension::D2), + range: wgt::ImageSubresourceRange { + aspect: wgt::TextureAspect::All, + base_mip_level: destination_mip_level, + mip_level_count: Some(1), + base_array_layer: destination_layer, + array_layer_count: Some(1), + }, + ..Default::default() + }); // UV transform selecting the source sub-rect, with flipY folded into the y scale. let frame_width = frame_width as f32; @@ -487,16 +521,16 @@ unsafe fn blit_frame_into_texture( // The triangle deliberately overhangs the viewport, so the scissor — not the viewport // — is what keeps the write inside the destination rect. pass.set_viewport( - destination.origin.x as f32, - destination.origin.y as f32, + destination_origin.0 as f32, + destination_origin.1 as f32, size.width as f32, size.height as f32, 0.0, 1.0, ); pass.set_scissor_rect( - destination.origin.x, - destination.origin.y, + destination_origin.0, + destination_origin.1, size.width, size.height, ); diff --git a/crates/canvas-c/src/webgpu/gpu_queue.rs b/crates/canvas-c/src/webgpu/gpu_queue.rs index 06d48f1e7..d97ca94f4 100644 --- a/crates/canvas-c/src/webgpu/gpu_queue.rs +++ b/crates/canvas-c/src/webgpu/gpu_queue.rs @@ -31,6 +31,9 @@ pub struct CanvasGPUQueue { /// Lazily built pipelines for the video blit, shared across clones of this handle. pub(crate) blit: Arc>>, + #[cfg(target_os = "windows")] + pub(crate) shared_frames: + Arc>, } unsafe impl Send for CanvasGPUQueue {} diff --git a/crates/canvas-c/src/webgpu/gpu_shared_frame.rs b/crates/canvas-c/src/webgpu/gpu_shared_frame.rs new file mode 100644 index 000000000..1ae3d0654 --- /dev/null +++ b/crates/canvas-c/src/webgpu/gpu_shared_frame.rs @@ -0,0 +1,180 @@ +use std::borrow::Cow; +use std::collections::HashMap; +use std::os::raw::c_void; +use std::sync::Arc; + +use windows::Win32::Foundation::HANDLE; +use windows::Win32::Graphics::Direct3D12::{ID3D12Device, ID3D12Fence, ID3D12Resource}; + +use super::gpu_device::CanvasGPUDevice; +use super::gpu_queue::CanvasGPUQueue; + +/// Same layout as `SharedFrameDesc` in canvas-media's module, which shares it as `nativeTexture`. +/// Reads wait for `ready_value`; signalling `release_value` after them lets the producer reuse the +/// texture. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct CanvasD3DSharedFrame { + pub size: u32, + /// Set once the release signal is staged; the producer frees an unconsumed frame at once. + pub consumed: u32, + pub adapter_luid: u64, + /// The cache keys: handle values can be reused. + pub texture_id: u64, + pub texture: *mut c_void, + pub ready_fence_id: u64, + pub ready_fence: *mut c_void, + pub ready_value: u64, + pub release_fence_id: u64, + pub release_fence: *mut c_void, + pub release_value: u64, +} + +#[derive(Default)] +pub struct SharedFrameCache { + textures: HashMap>, + fences: HashMap, +} + +// The fences are only used from the queue's thread; wgpu-core's objects are Send + Sync. +unsafe impl Send for SharedFrameCache {} + +// Entries keep the producer's objects alive (it recreates them when the video size changes). +const MAX_CACHED: usize = 16; + +/// By value: `consumed` is written through `handle` afterwards. +unsafe fn frame(handle: *mut c_void) -> Option { + let frame = (handle as *const CanvasD3DSharedFrame).as_ref()?; + (frame.size as usize == std::mem::size_of::()).then_some(*frame) +} + +unsafe fn raw_device(device: &Arc) -> Option { + let hal = Arc::clone(device).as_hal::()?; + Some(hal.raw_device().clone()) +} + +unsafe fn open_fence(raw: &ID3D12Device, cache: &mut SharedFrameCache, id: u64, handle: *mut c_void) -> Option { + if let Some(fence) = cache.fences.get(&id) { + return Some(fence.clone()); + } + let mut fence: Option = None; + raw.OpenSharedHandle(HANDLE(handle), &mut fence).ok()?; + let fence = fence?; + if cache.fences.len() >= MAX_CACHED { + cache.fences.clear(); + } + cache.fences.insert(id, fence.clone()); + Some(fence) +} + +/// The fences are staged for the queue's next submit, which must be the one reading the texture. +/// It may only be sampled: it stays in COMMON between submits, which promotes implicitly to shader +/// reads (a copy would need a barrier from a state wgpu does not know it is in). +pub(crate) unsafe fn import_and_stage( + queue: &CanvasGPUQueue, + handle: *mut c_void, + width: u32, + height: u32, +) -> Option> { + let frame = frame(handle)?; + if frame.texture.is_null() || frame.ready_fence.is_null() || frame.release_fence.is_null() { + return None; + } + let device = &queue.device_id; + let raw = raw_device(device)?; + let luid = raw.GetAdapterLuid(); + if (luid.LowPart as u64 | ((luid.HighPart as u32 as u64) << 32)) != frame.adapter_luid { + return None; + } + + let mut cache = queue.shared_frames.lock(); + let ready = open_fence(&raw, &mut cache, frame.ready_fence_id, frame.ready_fence)?; + let release = open_fence(&raw, &mut cache, frame.release_fence_id, frame.release_fence)?; + + let texture = match cache.textures.get(&frame.texture_id) { + Some(texture) => Arc::clone(texture), + None => { + let mut resource: Option = None; + raw.OpenSharedHandle(HANDLE(frame.texture), &mut resource).ok()?; + let size = wgt::Extent3d { + width, + height, + depth_or_array_layers: 1, + }; + let hal_texture = wgpu_hal::dx12::Device::texture_from_raw( + resource?, + wgt::TextureFormat::Bgra8Unorm, + wgt::TextureDimension::D2, + size, + 1, + 1, + ); + let descriptor = wgpu_core::resource::TextureDescriptor { + label: Some(Cow::Borrowed("videoBlit:SharedFrame")), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgt::TextureDimension::D2, + format: wgt::TextureFormat::Bgra8Unorm, + usage: wgt::TextureUsages::TEXTURE_BINDING, + view_formats: vec![], + }; + let (texture, error) = + device.create_texture_from_hal(Box::new(hal_texture), &descriptor, wgt::TextureUses::RESOURCE, true); + if let Some(error) = error { + log::error!("importing a shared video frame failed: {error:?}"); + return None; + } + if cache.textures.len() >= MAX_CACHED { + cache.textures.clear(); + } + cache.textures.insert(frame.texture_id, Arc::clone(&texture)); + texture + } + }; + drop(cache); + + let hal_queue = Arc::clone(&queue.queue.id).as_hal::()?; + hal_queue.add_wait_fence(ready, frame.ready_value); + hal_queue.add_signal_fence(release, frame.release_value); + (*(handle as *mut CanvasD3DSharedFrame)).consumed = 1; + Some(texture) +} + +/// Drawn into a plane of our own at once: the external texture is sampled after this call, when the +/// producer may be writing the frame again. +pub(crate) unsafe fn import_external_plane( + device: &CanvasGPUDevice, + handle: *mut c_void, + width: u32, + height: u32, +) -> Option> { + let queue = &*device.queue; + let size = wgt::Extent3d { + width, + height, + depth_or_array_layers: 1, + }; + let plane = device.device.create_texture(&wgpu_core::resource::TextureDescriptor { + label: Some(Cow::Borrowed("externalTexture:Plane")), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgt::TextureDimension::D2, + format: wgt::TextureFormat::Bgra8Unorm, + usage: wgt::TextureUsages::TEXTURE_BINDING | wgt::TextureUsages::RENDER_ATTACHMENT, + view_formats: vec![], + }); + let source = import_and_stage(queue, handle, width, height)?; + super::gpu_native_texture::blit_texture(queue, &source, width, height, 0, 0, false, &plane, 0, 0, (0, 0), size) + .then_some(plane) +} + +/// Videos share frames only with devices on their adapter. 0 if unknown. +#[no_mangle] +pub unsafe extern "C" fn canvas_native_webgpu_device_get_adapter_luid(device: *const CanvasGPUDevice) -> u64 { + let Some(device) = device.as_ref() else { return 0 }; + let Some(raw) = raw_device(&device.device) else { return 0 }; + let luid = raw.GetAdapterLuid(); + luid.LowPart as u64 | ((luid.HighPart as u32 as u64) << 32) +} diff --git a/crates/canvas-c/src/webgpu/mod.rs b/crates/canvas-c/src/webgpu/mod.rs index 9f57e3f9c..4214f7713 100644 --- a/crates/canvas-c/src/webgpu/mod.rs +++ b/crates/canvas-c/src/webgpu/mod.rs @@ -65,6 +65,8 @@ pub mod gpu_render_pass_encoder; pub mod gpu_render_pipeline; pub mod gpu_sampler; pub mod gpu_shader_module; +#[cfg(target_os = "windows")] +pub mod gpu_shared_frame; pub mod gpu_supported_limits; pub mod gpu_texture; pub mod gpu_texture_view; diff --git a/crates/canvas-media-napi/Cargo.toml b/crates/canvas-media-napi/Cargo.toml new file mode 100644 index 000000000..c00fc0f0a --- /dev/null +++ b/crates/canvas-media-napi/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "canvas-media-napi" +version = "0.1.0" +edition = "2021" +publish = false +description = "Node-API binding for @nativescript/canvas-media on Windows: MediaPlayer events and frame-server frames for the Video and Audio views." + +[lib] +crate-type = ["cdylib"] + +[features] +default = ["napi8"] +napi8 = ["napi/napi8"] +# Hosts whose Node-API lacks type tags (NativeScript Windows on JavaScriptCore). +napi7 = ["napi/napi7"] + +[dependencies] +napi = { version = "3.13.0", default-features = false, features = ["dyn-symbols"] } +napi-derive = "3.6.9" + +[target.'cfg(target_os = "windows")'.dependencies] +# `#[windows::core::interface]` (ISurfaceImageSourceNative) expands to `windows_core` paths. +windows-core = "0.62" +# IVectorView for the audio tap's supported formats. +windows-collections = "0.3" +windows = { workspace = true, features = [ + "Foundation", + "Foundation_Collections", + "Graphics_DirectX_Direct3D11", + "Media_Core", + "Media_Effects", + "Media_MediaProperties", + "Media_Playback", + "Win32_Foundation", + "Win32_Graphics_Direct3D", + "Win32_Graphics_Direct3D11", + "Win32_Graphics_Dxgi", + "Win32_Graphics_Dxgi_Common", + "Win32_System_WinRT", + "Win32_Security", + "Win32_System_WinRT_Direct3D11", +] } + +[build-dependencies] +napi-build = "2.5.0" diff --git a/crates/canvas-media-napi/__test__/media.test.mjs b/crates/canvas-media-napi/__test__/media.test.mjs new file mode 100644 index 000000000..4f6d808eb --- /dev/null +++ b/crates/canvas-media-napi/__test__/media.test.mjs @@ -0,0 +1,83 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import url from 'node:url'; + +const here = path.dirname(url.fileURLToPath(import.meta.url)); +const root = path.resolve(here, '../../..'); +const file = 'canvas_media_napi.dll'; +const addon = [process.env.CANVAS_MEDIA_NAPI_ADDON, path.join(root, 'target', 'debug', file), path.join(root, 'target', 'release-napi', file)].filter(Boolean).find((f) => fs.existsSync(f)); +const module = { exports: {} }; +process.dlopen(module, addon); +const Media = module.exports; +const clip = url.pathToFileURL(path.join(root, 'tools', 'demo', 'canvas', 'assets', 'webgpu', 'pano.mp4')).href; + +function watch(frames, done, timeoutMs = 20000) { + const events = []; + let bridge; + const promise = new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`timed out; saw ${events.map((e) => e[0]).join(', ')}`)), timeoutMs); + bridge = new Media.NSCMediaPlayerBridge( + Media.__createTestPlayer(clip), + (type, detail) => { + events.push([type, detail]); + if (type === 'error') { + clearTimeout(timer); + reject(new Error(detail)); + } else if (done(events, bridge)) { + clearTimeout(timer); + resolve(events); + } + }, + frames, + ); + }); + return { promise, bridge: () => bridge }; +} + +test('exports the bridge index.windows.ts builds on', () => { + assert.equal(typeof Media.NSCMediaPlayerBridge, 'function'); + assert.throws(() => new Media.NSCMediaPlayerBridge('0x0', () => {}), /Invalid MediaPlayer pointer/); +}); + +test('delivers the player events on the JS thread', async () => { + const { promise, bridge } = watch(false, (events) => events.some(([type, state]) => type === 'state' && state === '3')); + const events = await promise; + assert.ok(events.some(([type]) => type === 'opened'), 'opened'); + assert.equal(bridge().frameId, 0); + assert.equal(bridge().readPixels(), null); + bridge().close(); +}); + +test('copies frames and reads them back as RGBA', async () => { + const { promise, bridge } = watch(true, (events) => events.filter(([type]) => type === 'frame').length >= 3); + await promise; + const b = bridge(); + const [width, height] = [b.videoWidth, b.videoHeight]; + assert.ok(width > 0 && height > 0, `${width}x${height}`); + assert.ok(b.frameId >= 3); + const pixels = b.readPixels(); + assert.equal(pixels.length, width * height * 4); + let lit = 0; + for (let i = 0; i < pixels.length; i += 4) { + assert.equal(pixels[i + 3], 255); + if (pixels[i] + pixels[i + 1] + pixels[i + 2] > 0) lit++; + } + assert.ok(lit > (width * height) / 10, 'the frame has content'); + assert.equal(b.sharesFrames, true); + assert.ok(b.adapterLuid > 0, `adapter ${b.adapterLuid}`); + const shared = b.gpuFrame(); + assert.ok(shared.address > 0); + assert.equal(shared.width, width); + assert.equal(shared.height, height); + shared.close(); + shared.close(); + // No SurfaceImageSource attached. + assert.equal(b.present(), false); + b.close(); + const id = b.frameId; + await new Promise((resolve) => setTimeout(resolve, 300)); + assert.equal(b.frameId, id, 'no frames after close'); + assert.equal(b.readPixels().length, width * height * 4, 'the last frame stays readable'); +}); diff --git a/crates/canvas-media-napi/build.rs b/crates/canvas-media-napi/build.rs new file mode 100644 index 000000000..0f1b01002 --- /dev/null +++ b/crates/canvas-media-napi/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/crates/canvas-media-napi/package.json b/crates/canvas-media-napi/package.json new file mode 100644 index 000000000..5bc95ffe3 --- /dev/null +++ b/crates/canvas-media-napi/package.json @@ -0,0 +1,19 @@ +{ + "name": "@nativescript/canvas-media-napi", + "private": true, + "version": "0.1.0", + "description": "Node-API build of the canvas-media native module (MediaPlayer bridge for Windows).", + "napi": { + "binaryName": "canvasmedia", + "targets": [ + "x86_64-pc-windows-msvc", + "aarch64-pc-windows-msvc" + ] + }, + "scripts": { + "test": "node --expose-gc --test \"__test__/*.test.mjs\"" + }, + "engines": { + "node": ">=20" + } +} diff --git a/crates/canvas-media-napi/src/lib.rs b/crates/canvas-media-napi/src/lib.rs new file mode 100644 index 000000000..ec0ed78d5 --- /dev/null +++ b/crates/canvas-media-napi/src/lib.rs @@ -0,0 +1,795 @@ +#![cfg(target_os = "windows")] +#![deny(clippy::all)] + +mod tap; + +use std::cell::{Cell, RefCell}; +use std::mem::ManuallyDrop; +use std::rc::{Rc, Weak}; +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + +use napi::bindgen_prelude::{FnArgs, Function, ObjectFinalize, Uint8Array}; +use napi::threadsafe_function::{ThreadsafeFunctionCallMode, UnknownReturnValue}; +use napi::{Env, Error, Result}; +use napi_derive::napi; +use windows::core::{IInspectable, IUnknown, Interface, RuntimeType, HRESULT}; +use windows::Foundation::TypedEventHandler; +use windows::Graphics::DirectX::Direct3D11::IDirect3DSurface; +use windows::Media::Playback::{MediaPlaybackSession, MediaPlayer, MediaPlayerFailedEventArgs}; +use windows::core::PCWSTR; +use windows::Win32::Foundation::{CloseHandle, GENERIC_ALL, HANDLE, HMODULE, POINT, RECT}; +use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP}; +use windows::Win32::Graphics::Direct3D11::{ + D3D11CreateDevice, ID3D11Device, ID3D11Device5, ID3D11DeviceContext, ID3D11DeviceContext4, ID3D11Fence, + ID3D11Multithread, ID3D11Resource, ID3D11Texture2D, D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, + D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_FENCE_FLAG_SHARED, + D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_READ, D3D11_RESOURCE_MISC_SHARED, D3D11_RESOURCE_MISC_SHARED_NTHANDLE, + D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, +}; +use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_SAMPLE_DESC}; +use windows::Win32::Graphics::Dxgi::{ + IDXGIDevice, IDXGIResource1, IDXGISurface, DXGI_SHARED_RESOURCE_READ, DXGI_SHARED_RESOURCE_WRITE, +}; +use windows::Win32::System::WinRT::Direct3D11::CreateDirect3D11SurfaceFromDXGISurface; + +/// WinUI 3's `ISurfaceImageSourceNative` (microsoft.ui.xaml.media.dxinterop.h; not the UWP IID). +#[windows::core::interface("e4cecd6c-f14b-4f46-83c3-8bbda27c6504")] +unsafe trait ISurfaceImageSourceNative: windows::core::IUnknown { + fn SetDevice(&self, device: *mut c_void) -> HRESULT; + fn BeginDraw(&self, update_rect: RECT, surface: *mut *mut c_void, offset: *mut POINT) -> HRESULT; + fn EndDraw(&self) -> HRESULT; +} + +fn parse_pointer_key(key: &str) -> Option<*mut c_void> { + let key = key.trim(); + let address = match key.strip_prefix("0x").or_else(|| key.strip_prefix("0X")) { + Some(hex) => usize::from_str_radix(hex, 16).ok(), + None => key.parse().ok(), + }?; + (address != 0).then_some(address as *mut c_void) +} + +fn to_napi(error: windows::core::Error) -> Error { + Error::from_reason(error.message()) +} + +struct Device { + device: ID3D11Device, + context: ID3D11DeviceContext, + /// WebGPU shares frames only on the same adapter. + luid: u64, + /// Windows 10 1703+; without fences frames are not shared. + fences: Option<(ID3D11Device5, ID3D11DeviceContext4)>, +} + +// Multithread-protected (below), and D3D11 devices are free-threaded. +unsafe impl Send for Device {} +unsafe impl Sync for Device {} + +/// One device for every player. MediaPlayer copies frames into it on its own threads while the JS +/// thread reads them back or presents them, so its immediate context is multithread-protected. +fn device() -> Option<&'static Device> { + static DEVICE: OnceLock> = OnceLock::new(); + DEVICE + .get_or_init(|| create_device(D3D_DRIVER_TYPE_HARDWARE).or_else(|| create_device(D3D_DRIVER_TYPE_WARP))) + .as_ref() +} + +fn create_device(driver: D3D_DRIVER_TYPE) -> Option { + let (mut device, mut context) = (None, None); + unsafe { + D3D11CreateDevice( + None, + driver, + HMODULE::default(), + // BGRA for XAML SurfaceImageSources; video support for MediaPlayer's video processor. + D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + None, + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + } + .ok()?; + let (device, context) = (device?, context?); + let multithread: ID3D11Multithread = context.cast().ok()?; + let _ = unsafe { multithread.SetMultithreadProtected(true) }; + let luid = (|| -> windows::core::Result { + let adapter = unsafe { device.cast::()?.GetAdapter()? }; + let luid = unsafe { adapter.GetDesc()? }.AdapterLuid; + Ok(luid.LowPart as u64 | ((luid.HighPart as u32 as u64) << 32)) + })() + .unwrap_or(0); + let fences = device.cast::().ok().zip(context.cast::().ok()); + Some(Device { device, context, luid, fences }) +} + +fn texture_desc(width: u32, height: u32, shared: bool) -> D3D11_TEXTURE2D_DESC { + D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_B8G8R8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, + CPUAccessFlags: 0, + // Opened by WebGPU on its D3D12 device (canvas-c's gpu_shared_frame). + MiscFlags: if shared { (D3D11_RESOURCE_MISC_SHARED_NTHANDLE.0 | D3D11_RESOURCE_MISC_SHARED.0) as u32 } else { 0 }, + } +} + +static NEXT_SHARED_ID: AtomicU64 = AtomicU64::new(1); + +struct SharedHandle { + handle: HANDLE, + /// Unique for the process's lifetime, unlike handle values: consumers cache by it. + id: u64, +} + +impl SharedHandle { + fn new(handle: HANDLE) -> Self { + Self { handle, id: NEXT_SHARED_ID.fetch_add(1, Ordering::Relaxed) } + } +} + +impl Drop for SharedHandle { + fn drop(&mut self) { + let _ = unsafe { CloseHandle(self.handle) }; + } +} + +struct Fences { + ready: ID3D11Fence, + ready_handle: SharedHandle, + release: ID3D11Fence, + release_handle: SharedHandle, +} + +impl Fences { + fn new(device: &ID3D11Device5) -> Option { + let create = || -> windows::core::Result<(ID3D11Fence, SharedHandle)> { + let mut fence: Option = None; + unsafe { device.CreateFence(0, D3D11_FENCE_FLAG_SHARED, &mut fence)? }; + let fence = fence.ok_or_else(windows::core::Error::empty)?; + let handle = unsafe { fence.CreateSharedHandle(None, GENERIC_ALL.0, PCWSTR::null())? }; + Ok((fence, SharedHandle::new(handle))) + }; + let (ready, ready_handle) = create().ok()?; + let (release, release_handle) = create().ok()?; + Some(Fences { ready, ready_handle, release, release_handle }) + } +} + +/// `CanvasD3DSharedFrame` in canvas-c (crates/canvas-c/src/webgpu/gpu_shared_frame.rs): same layout. +#[repr(C)] +struct SharedFrameDesc { + size: u32, + consumed: u32, + adapter_luid: u64, + texture_id: u64, + texture: *mut c_void, + ready_fence_id: u64, + ready_fence: *mut c_void, + ready_value: u64, + release_fence_id: u64, + release_fence: *mut c_void, + release_value: u64, +} + +struct Frame { + texture: ID3D11Texture2D, + surface: IDirect3DSurface, + shared: Option, + /// The release value to wait for before writing the texture again. + busy_until: u64, +} + +impl Frame { + fn new(device: &Device, width: u32, height: u32) -> Option { + let shared = device.fences.is_some(); + let mut texture = None; + unsafe { device.device.CreateTexture2D(&texture_desc(width, height, shared), None, Some(&mut texture)) }.ok()?; + let texture = texture?; + let dxgi: IDXGISurface = texture.cast().ok()?; + let surface: IDirect3DSurface = unsafe { CreateDirect3D11SurfaceFromDXGISurface(&dxgi) }.ok()?.cast().ok()?; + let shared = shared + .then(|| { + let resource: IDXGIResource1 = texture.cast().ok()?; + let access = DXGI_SHARED_RESOURCE_READ.0 | DXGI_SHARED_RESOURCE_WRITE.0; + unsafe { resource.CreateSharedHandle(None, access, PCWSTR::null()) }.ok().map(SharedHandle::new) + }) + .flatten(); + Some(Frame { texture, surface, shared, busy_until: 0 }) + } +} + +/// A texture a consumer may still be reading is not written; with every other one busy, a decoded +/// frame is dropped rather than waited for. +const POOL_SIZE: usize = 3; + +#[derive(Default)] +struct Pool { + frames: Vec, + latest: Option, + width: u32, + height: u32, + fences: Option, + latest_ready: u64, + next_ready: u64, + next_release: u64, +} + +unsafe impl Send for Pool {} + +impl Pool { + fn latest(&self) -> Option<&Frame> { + self.frames.get(self.latest?) + } +} + +#[derive(Default)] +struct Frames { + pool: Mutex, + generation: AtomicU64, + /// Set under `pool`'s lock: a `VideoFrameAvailable` still running when its handler is removed + /// copies nothing after the bridge closes. + closed: AtomicBool, +} + +impl Frames { + /// On the player's thread, from `VideoFrameAvailable`. + fn copy_from(&self, player: &MediaPlayer) -> bool { + let Some(device) = device() else { return false }; + let Ok(session) = player.PlaybackSession() else { return false }; + let (width, height) = (session.NaturalVideoWidth().unwrap_or(0), session.NaturalVideoHeight().unwrap_or(0)); + if width == 0 || height == 0 { + return false; + } + let mut pool = self.pool.lock().unwrap(); + if self.closed.load(Ordering::Acquire) { + return false; + } + if pool.width != width || pool.height != height { + pool.frames.clear(); + pool.latest = None; + pool.width = width; + pool.height = height; + } + if pool.fences.is_none() { + pool.fences = device.fences.as_ref().and_then(|(device5, _)| Fences::new(device5)); + } + let released = pool.fences.as_ref().map_or(u64::MAX, |fences| unsafe { fences.release.GetCompletedValue() }); + let latest = pool.latest; + let free = |i: usize| Some(i) != latest && pool.frames.get(i).is_none_or(|frame| frame.busy_until <= released); + let Some(index) = (0..POOL_SIZE).find(|&i| free(i)) else { + return false; + }; + if index == pool.frames.len() { + let Some(frame) = Frame::new(device, width, height) else { return false }; + pool.frames.push(frame); + } + if player.CopyFrameToVideoSurface(&pool.frames[index].surface).is_err() { + return false; + } + if let (Some(fences), Some((_, context4))) = (pool.fences.as_ref(), device.fences.as_ref()) { + let value = pool.next_ready + 1; + if unsafe { context4.Signal(&fences.ready, value) }.is_ok() { + pool.next_ready = value; + } + } + pool.latest = Some(index); + pool.latest_ready = pool.next_ready; + self.generation.fetch_add(1, Ordering::AcqRel); + true + } + + fn close(&self) { + let _pool = self.pool.lock().unwrap(); + self.closed.store(true, Ordering::Release); + } + + fn size(&self) -> (u32, u32) { + let pool = self.pool.lock().unwrap(); + if pool.latest.is_some() { + (pool.width, pool.height) + } else { + (0, 0) + } + } +} + +struct XamlSurface { + native: ISurfaceImageSourceNative, + width: u32, + height: u32, +} + +impl XamlSurface { + /// UI thread. + fn present(&self, device: &Device, texture: &ID3D11Texture2D) -> windows::core::Result<()> { + let rect = RECT { left: 0, top: 0, right: self.width as i32, bottom: self.height as i32 }; + let mut surface = std::ptr::null_mut(); + let mut offset = POINT::default(); + unsafe { self.native.BeginDraw(rect, &mut surface, &mut offset) }.ok()?; + // The update rectangle lives in XAML's atlas at `offset`. + let copied = unsafe { IDXGISurface::from_raw(surface) }.cast::().map(|target| unsafe { + device.context.CopySubresourceRegion(&target, 0, offset.x.max(0) as u32, offset.y.max(0) as u32, 0, texture, 0, None) + }); + let ended = unsafe { self.native.EndDraw() }.ok(); + copied.and(ended) + } +} + +/// `address`: a `tap::AudioTapSource` for audiocontext.node. +#[napi(js_name = "NSCAudioTap")] +pub struct NSCAudioTap { + tap: Arc, + source: Box, +} + +#[napi] +impl NSCAudioTap { + #[napi(getter)] + pub fn address(&self) -> f64 { + &*self.source as *const tap::AudioTapSource as usize as f64 + } + + #[napi] + pub fn set_gain(&self, gain: f64) { + self.tap.set_gain(gain as f32); + } + + /// Routed: the player is silent and its audio goes to the graph. + #[napi] + pub fn set_routed(&self, routed: bool) { + self.tap.set_routed(routed); + } + + #[napi(getter)] + pub fn frames_tapped(&self) -> f64 { + self.tap.frames.load(std::sync::atomic::Ordering::Relaxed) as f64 + } +} + +/// Holds the pool, so the handles it names stay open. +#[napi(js_name = "NSCSharedFrame", custom_finalize)] +pub struct NSCSharedFrame { + desc: Box, + frames: Arc, + width: u32, + height: u32, + released: bool, +} + +impl ObjectFinalize for NSCSharedFrame { + fn finalize(mut self, _: Env) -> Result<()> { + self.close(); + Ok(()) + } +} + +#[napi] +impl NSCSharedFrame { + #[napi(getter)] + pub fn address(&self) -> f64 { + &*self.desc as *const SharedFrameDesc as usize as f64 + } + + #[napi(getter)] + pub fn width(&self) -> u32 { + self.width + } + + #[napi(getter)] + pub fn height(&self) -> u32 { + self.height + } + + /// An unconsumed frame (its import failed) frees its texture at once; a consumed one when the + /// consumer's release signal completes. + #[napi] + pub fn close(&mut self) { + if std::mem::replace(&mut self.released, true) { + return; + } + // Written by the consumer (another module) through the address. + let consumed = unsafe { std::ptr::read_volatile(&self.desc.consumed) }; + if consumed == 0 { + let mut pool = self.frames.pool.lock().unwrap(); + let (texture_id, release_value) = (self.desc.texture_id, self.desc.release_value); + if let Some(frame) = pool.frames.iter_mut().find(|f| f.shared.as_ref().is_some_and(|s| s.id == texture_id)) { + if frame.busy_until == release_value { + frame.busy_until = 0; + } + } + } + } +} + +/// What a bridge holds on the player. A MediaPlayer released while the process exits (after +/// ExitProcess has killed Media Foundation's threads, e.g. from a thread-local destructor) spins +/// forever in the graphics driver, so an env cleanup hook closes every live player while its threads +/// still run, and nothing here is released from process shutdown. +struct Live { + player: MediaPlayer, + unsubscribe: Vec>, + frames: Option>, +} + +impl Live { + fn close(&mut self) { + for unsubscribe in self.unsubscribe.drain(..) { + unsubscribe(); + } + if let Some(frames) = self.frames.as_ref() { + frames.close(); + } + } +} + +type LiveCell = RefCell>; + +thread_local! { + // Weak and ManuallyDrop: dropping these at thread exit releases nothing. + static LIVE: RefCell>> = const { RefCell::new(Vec::new()) }; + static TEST_PLAYERS: RefCell>> = const { RefCell::new(Vec::new()) }; + static CLEANUP_HOOKED: Cell = const { Cell::new(false) }; +} + +fn close_everything() { + for live in LIVE.with(|live| std::mem::take(&mut *live.borrow_mut())) { + if let Some(live) = live.upgrade() { + if let Some(mut live) = live.borrow_mut().take() { + live.close(); + let _ = live.player.Close(); + } + } + } + close_test_players(); +} + +fn ensure_cleanup_hook(env: &Env) -> Result<()> { + if !CLEANUP_HOOKED.get() { + env.add_env_cleanup_hook((), |_| close_everything())?; + CLEANUP_HOOKED.set(true); + } + Ok(()) +} + +type Emit = Arc) + Send + Sync>; +type EventCallback<'a> = Function<'a, FnArgs<(String, Option)>, UnknownReturnValue>; + +fn notify(emit: &Emit, kind: &'static str) -> TypedEventHandler { + let emit = Arc::clone(emit); + TypedEventHandler::new(move |_, _| { + emit(kind, None); + Ok(()) + }) +} + +/// MediaPlayer raises its events on Media Foundation threads, where the runtime cannot run JS +/// delegates: they are subscribed here and delivered on the JS thread. +#[napi(js_name = "NSCMediaPlayerBridge", custom_finalize)] +pub struct NSCMediaPlayerBridge { + live: Rc, + frames: Option>, + staging: RefCell>, + xaml: RefCell>, +} + +impl ObjectFinalize for NSCMediaPlayerBridge { + fn finalize(mut self, _: Env) -> Result<()> { + self.close(); + Ok(()) + } +} + +#[napi] +impl NSCMediaPlayerBridge { + /// `onEvent(type, detail)`: `opened`, `ended`, `error` (detail: the message), `state` (detail: the + /// `MediaPlaybackState`), `seeked`, `durationchange`, `resize`, `waiting`, `buffered` and, with + /// `frames`, `frame` (at most one queued). + #[napi( + constructor, + ts_args_type = "playerKey: string, onEvent: (type: string, detail?: string) => void, frames?: boolean" + )] + pub fn new(env: Env, player_key: String, on_event: EventCallback, frames: Option) -> Result { + ensure_cleanup_hook(&env)?; + let raw = parse_pointer_key(&player_key) + .ok_or_else(|| Error::from_reason(format!("Invalid MediaPlayer pointer: {player_key}")))?; + let player: MediaPlayer = unsafe { IUnknown::from_raw_borrowed(&raw) } + .ok_or_else(|| Error::from_reason("Invalid MediaPlayer pointer"))? + .cast() + .map_err(to_napi)?; + let session = player.PlaybackSession().map_err(to_napi)?; + + let frame_queued = Arc::new(AtomicBool::new(false)); + let queued = Arc::clone(&frame_queued); + let tsfn = on_event + .build_threadsafe_function::<(String, Option)>() + .weak::() + .build_callback(move |ctx| { + if ctx.value.0 == "frame" { + queued.store(false, Ordering::Release); + } + Ok(FnArgs::from(ctx.value)) + })?; + let emit: Emit = Arc::new(move |kind, detail| { + tsfn.call((kind.to_string(), detail), ThreadsafeFunctionCallMode::NonBlocking); + }); + + let mut unsubscribe: Vec> = Vec::new(); + macro_rules! subscribe { + ($target:expr, $add:ident, $remove:ident, $handler:expr) => {{ + let target = $target.clone(); + let token = target.$add(&$handler).map_err(to_napi)?; + unsubscribe.push(Box::new(move || { + let _ = target.$remove(token); + })); + }}; + } + type OnPlayer = IInspectable; + + subscribe!(player, MediaOpened, RemoveMediaOpened, notify::(&emit, "opened")); + subscribe!(player, MediaEnded, RemoveMediaEnded, notify::(&emit, "ended")); + let failed = { + let emit = Arc::clone(&emit); + TypedEventHandler::::new(move |_, args| { + let message = args.ok().ok().map(|args| { + let message = args.ErrorMessage().map(|m| m.to_string()).unwrap_or_default(); + if message.is_empty() { + let code = args.ExtendedErrorCode().map(|code| code.0).unwrap_or(0); + format!("Media playback failed (0x{:08x})", code as u32) + } else { + message + } + }); + emit("error", message); + Ok(()) + }) + }; + subscribe!(player, MediaFailed, RemoveMediaFailed, failed); + let state = { + let emit = Arc::clone(&emit); + TypedEventHandler::::new(move |session, _| { + let state = session.ok().ok().and_then(|s| s.PlaybackState().ok()).map(|s| s.0.to_string()); + emit("state", state); + Ok(()) + }) + }; + subscribe!(session, PlaybackStateChanged, RemovePlaybackStateChanged, state); + subscribe!(session, SeekCompleted, RemoveSeekCompleted, notify::(&emit, "seeked")); + subscribe!( + session, + NaturalDurationChanged, + RemoveNaturalDurationChanged, + notify::(&emit, "durationchange") + ); + subscribe!( + session, + NaturalVideoSizeChanged, + RemoveNaturalVideoSizeChanged, + notify::(&emit, "resize") + ); + subscribe!(session, BufferingStarted, RemoveBufferingStarted, notify::(&emit, "waiting")); + subscribe!(session, BufferingEnded, RemoveBufferingEnded, notify::(&emit, "buffered")); + + let frames = frames.unwrap_or(false).then(|| Arc::new(Frames::default())); + if let Some(frames) = frames.as_ref() { + let frames = Arc::clone(frames); + let emit = Arc::clone(&emit); + let available = TypedEventHandler::::new(move |player, _| { + if let Ok(player) = player.ok() { + if frames.copy_from(player) && !frame_queued.swap(true, Ordering::AcqRel) { + emit("frame", None); + } + } + Ok(()) + }); + subscribe!(player, VideoFrameAvailable, RemoveVideoFrameAvailable, available); + } + + let live = Rc::new(RefCell::new(Some(Live { player, unsubscribe, frames: frames.clone() }))); + LIVE.with(|all| { + let mut all = all.borrow_mut(); + all.retain(|live| live.strong_count() > 0); + all.push(Rc::downgrade(&live)); + }); + Ok(Self { live, frames, staging: RefCell::new(None), xaml: RefCell::new(None) }) + } + + /// The last frame stays readable. + #[napi] + pub fn close(&mut self) { + if let Some(mut live) = self.live.borrow_mut().take() { + live.close(); + } + self.xaml.borrow_mut().take(); + } + + #[napi(getter)] + pub fn video_width(&self) -> u32 { + self.frames.as_ref().map_or(0, |frames| frames.size().0) + } + + #[napi(getter)] + pub fn video_height(&self) -> u32 { + self.frames.as_ref().map_or(0, |frames| frames.size().1) + } + + #[napi(getter)] + pub fn adapter_luid(&self) -> f64 { + device().map_or(0., |device| device.luid as f64) + } + + #[napi(getter)] + pub fn shares_frames(&self) -> bool { + self.frames.is_some() && device().is_some_and(|device| device.fences.is_some()) + } + + /// `address` is a `CanvasD3DSharedFrame` for `nativeTexture`; the texture is not written again + /// until released. + #[napi] + pub fn gpu_frame(&self) -> Option { + let frames = self.frames.as_ref()?; + let device = device()?; + let mut pool = frames.pool.lock().unwrap(); + let index = pool.latest?; + let (width, height, ready_value) = (pool.width, pool.height, pool.latest_ready); + let fences = pool.fences.as_ref()?; + let (ready_fence_id, ready_fence) = (fences.ready_handle.id, fences.ready_handle.handle.0); + let (release_fence_id, release_fence) = (fences.release_handle.id, fences.release_handle.handle.0); + pool.next_release += 1; + let release_value = pool.next_release; + let frame = &mut pool.frames[index]; + let shared = frame.shared.as_ref()?; + let (texture_id, texture) = (shared.id, shared.handle.0); + frame.busy_until = frame.busy_until.max(release_value); + Some(NSCSharedFrame { + desc: Box::new(SharedFrameDesc { + size: std::mem::size_of::() as u32, + consumed: 0, + adapter_luid: device.luid, + texture_id, + texture, + ready_fence_id, + ready_fence, + ready_value, + release_fence_id, + release_fence, + release_value, + }), + frames: Arc::clone(frames), + width, + height, + released: false, + }) + } + + /// Passes the audio through until routed. Effects only apply to the sources set after them, so + /// this runs before any; optional, so a player whose effect cannot be activated just plays. + #[napi] + pub fn create_audio_tap(&self) -> Result { + let live = self.live.borrow(); + let player = &live.as_ref().ok_or_else(|| Error::from_reason("The player is closed"))?.player; + let tap = tap::Tap::new(); + let configuration = windows::Foundation::Collections::PropertySet::new().map_err(to_napi)?; + configuration + .Insert(&windows::core::HSTRING::from(tap::TAP_KEY), &windows::Foundation::PropertyValue::CreateUInt64(tap.id).map_err(to_napi)?) + .map_err(to_napi)?; + player + .AddAudioEffect(&windows::core::HSTRING::from(tap::TAP_CLASS), true, &configuration) + .map_err(to_napi)?; + let source = Box::new(tap::AudioTapSource::new(&tap)); + Ok(NSCAudioTap { tap, source }) + } + + #[napi(getter)] + pub fn frame_id(&self) -> f64 { + self.frames.as_ref().map_or(0., |frames| frames.generation.load(Ordering::Acquire) as f64) + } + + /// UI thread; replaces any previous one. + #[napi] + pub fn attach_surface_image_source(&self, key: String, width: u32, height: u32) -> bool { + let (Some(device), Some(raw)) = (device(), parse_pointer_key(&key)) else { return false }; + let attached = (|| -> windows::core::Result { + let unknown = unsafe { IUnknown::from_raw_borrowed(&raw) }.ok_or_else(windows::core::Error::empty)?; + let native: ISurfaceImageSourceNative = unknown.cast()?; + let dxgi: IDXGIDevice = device.device.cast()?; + unsafe { native.SetDevice(dxgi.as_raw()) }.ok()?; + Ok(XamlSurface { native, width, height }) + })(); + let attached = attached.ok(); + let ok = attached.is_some(); + *self.xaml.borrow_mut() = attached; + ok + } + + #[napi] + pub fn detach_surface_image_source(&self) { + self.xaml.borrow_mut().take(); + } + + /// UI thread. `false` without a surface of the frame's size. + #[napi] + pub fn present(&self) -> bool { + let (Some(frames), Some(device)) = (self.frames.as_ref(), device()) else { return false }; + let xaml = self.xaml.borrow(); + let Some(surface) = xaml.as_ref() else { return false }; + let pool = frames.pool.lock().unwrap(); + if pool.width != surface.width || pool.height != surface.height { + return false; + } + let Some(frame) = pool.latest() else { return false }; + surface.present(device, &frame.texture).is_ok() + } + + /// RGBA, top row first; `null` before the first frame. + #[napi] + pub fn read_pixels(&self) -> Option { + let (frames, device) = (self.frames.as_ref()?, device()?); + let mut staging = self.staging.borrow_mut(); + let pool = frames.pool.lock().unwrap(); + let frame = pool.latest()?; + let (width, height) = (pool.width, pool.height); + if staging.as_ref().is_none_or(|(_, w, h)| *w != width || *h != height) { + let desc = D3D11_TEXTURE2D_DESC { + Usage: D3D11_USAGE_STAGING, + BindFlags: 0, + CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32, + ..texture_desc(width, height, false) + }; + let mut texture = None; + unsafe { device.device.CreateTexture2D(&desc, None, Some(&mut texture)) }.ok()?; + *staging = Some((texture?, width, height)); + } + let (texture, ..) = staging.as_ref()?; + unsafe { device.context.CopyResource(texture, &frame.texture) }; + drop(pool); + + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + unsafe { device.context.Map(texture, 0, D3D11_MAP_READ, 0, Some(&mut mapped)) }.ok()?; + let row = width as usize * 4; + let mut rgba = vec![0u8; row * height as usize]; + for (y, dst) in rgba.chunks_exact_mut(row).enumerate() { + let src = unsafe { std::slice::from_raw_parts((mapped.pData as *const u8).add(y * mapped.RowPitch as usize), row) }; + for (dst, src) in dst.chunks_exact_mut(4).zip(src.chunks_exact(4)) { + dst.copy_from_slice(&[src[2], src[1], src[0], 255]); + } + } + unsafe { device.context.Unmap(texture, 0) }; + Some(Uint8Array::new(rgba)) + } +} + +/// A muted, autoplaying frame-server player for tests: Node has no WinRT projection to make one. +#[napi(js_name = "__createTestPlayer")] +pub fn create_test_player(env: Env, uri: String) -> Result { + ensure_cleanup_hook(&env)?; + let player = MediaPlayer::new().map_err(to_napi)?; + let setup = || -> windows::core::Result<()> { + player.SetIsVideoFrameServerEnabled(true)?; + player.SetIsMuted(true)?; + player.SetAutoPlay(true)?; + let uri = windows::Foundation::Uri::CreateUri(&windows::core::HSTRING::from(uri.as_str()))?; + player.SetSource(&windows::Media::Core::MediaSource::CreateFromUri(&uri)?) + }; + setup().map_err(to_napi)?; + let key = format!("0x{:x}", player.as_raw() as usize); + TEST_PLAYERS.with(|players| players.borrow_mut().push(ManuallyDrop::new(player))); + Ok(key) +} + +#[napi(js_name = "__closeTestPlayers")] +pub fn close_test_players() { + for player in TEST_PLAYERS.with(|players| std::mem::take(&mut *players.borrow_mut())) { + let player = ManuallyDrop::into_inner(player); + let _ = player.Close(); + } +} diff --git a/crates/canvas-media-napi/src/tap.rs b/crates/canvas-media-napi/src/tap.rs new file mode 100644 index 000000000..514756934 --- /dev/null +++ b/crates/canvas-media-napi/src/tap.rs @@ -0,0 +1,264 @@ +use std::collections::VecDeque; +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, Weak}; + +use windows::core::{implement, IInspectable, Interface, Ref, HRESULT, HSTRING}; +use windows::Foundation::Collections::IPropertySet; +use windows::Foundation::{IMemoryBufferReference, IPropertyValue}; +use windows::Media::Effects::{IBasicAudioEffect, IBasicAudioEffect_Impl, MediaEffectClosedReason, ProcessAudioFrameContext}; +use windows::Media::MediaProperties::{AudioEncodingProperties, MediaEncodingSubtypes}; +use windows::Media::{AudioBufferAccessMode, IMediaExtension, IMediaExtension_Impl}; +use windows::Win32::Foundation::{CLASS_E_CLASSNOTAVAILABLE, E_POINTER, S_FALSE, S_OK}; +use windows::Win32::System::WinRT::{IActivationFactory, IActivationFactory_Impl, IMemoryBufferByteAccess}; +use windows_collections::IVectorView; + +/// Registered in the app manifest by platforms/windows/plugin.targets, this module its server. +pub const TAP_CLASS: &str = "NativeScript.CanvasMedia.AudioTap"; +pub const TAP_KEY: &str = "tap"; +/// Beyond this the oldest samples go: the graph has fallen behind. +const MAX_BUFFERED_SECONDS: usize = 1; + +pub struct Tap { + pub id: u64, + ring: Mutex, + routed: AtomicBool, + /// f32 bits: the element's volume, 0 when muted. + gain: AtomicU32, + pub frames: AtomicU64, +} + +#[derive(Default)] +struct Ring { + samples: VecDeque, + channels: u32, + sample_rate: u32, +} + +static TAPS: Mutex>> = Mutex::new(Vec::new()); +static NEXT_TAP: AtomicU64 = AtomicU64::new(1); + +impl Tap { + pub fn new() -> Arc { + let tap = Arc::new(Tap { + id: NEXT_TAP.fetch_add(1, Ordering::Relaxed), + ring: Mutex::new(Ring::default()), + routed: AtomicBool::new(false), + gain: AtomicU32::new(1f32.to_bits()), + frames: AtomicU64::new(0), + }); + let mut taps = TAPS.lock().unwrap(); + taps.retain(|tap| tap.strong_count() > 0); + taps.push(Arc::downgrade(&tap)); + tap + } + + fn find(id: u64) -> Option> { + TAPS.lock().unwrap().iter().filter_map(Weak::upgrade).find(|tap| tap.id == id) + } + + pub fn set_gain(&self, gain: f32) { + self.gain.store(gain.max(0.).to_bits(), Ordering::Relaxed); + } + + pub fn set_routed(&self, routed: bool) { + self.routed.store(routed, Ordering::Release); + if !routed { + self.clear(); + } + } + + fn clear(&self) { + self.ring.lock().unwrap().samples.clear(); + } + + /// Media Foundation's thread. + fn push(&self, samples: &[f32], channels: u32, sample_rate: u32) { + let gain = f32::from_bits(self.gain.load(Ordering::Relaxed)); + let mut ring = self.ring.lock().unwrap(); + if ring.channels != channels || ring.sample_rate != sample_rate { + ring.samples.clear(); + ring.channels = channels; + ring.sample_rate = sample_rate; + } + ring.samples.extend(samples.iter().map(|sample| sample * gain)); + self.frames.fetch_add((samples.len() / channels.max(1) as usize) as u64, Ordering::Relaxed); + let max = sample_rate as usize * channels as usize * MAX_BUFFERED_SECONDS; + let excess = ring.samples.len().saturating_sub(max); + ring.samples.drain(..excess - excess % channels.max(1) as usize); + } + + /// The graph's render thread; whole frames only. + fn read(&self, out: &mut [f32], channels: &mut u32, sample_rate: &mut u32) -> usize { + let mut ring = self.ring.lock().unwrap(); + *channels = ring.channels; + *sample_rate = ring.sample_rate; + if ring.channels == 0 { + return 0; + } + let count = ring.samples.len().min(out.len()); + let count = count - count % ring.channels as usize; + for (out, sample) in out.iter_mut().zip(ring.samples.drain(..count)) { + *out = sample; + } + count / ring.channels as usize + } +} + +/// Read by audiocontext.node through `NSCAudioTap.address`; `tap` lives while it holds a `retain`ed +/// reference. +#[repr(C)] +pub struct AudioTapSource { + pub size: u32, + pub reserved: u32, + pub tap: *const c_void, + /// Reports the format too (0 channels: nothing decoded yet). + pub read: unsafe extern "C" fn(tap: *const c_void, out: *mut f32, capacity: usize, channels: *mut u32, sample_rate: *mut u32) -> usize, + pub retain: unsafe extern "C" fn(tap: *const c_void), + pub release: unsafe extern "C" fn(tap: *const c_void), +} + +unsafe extern "C" fn source_read(tap: *const c_void, out: *mut f32, capacity: usize, channels: *mut u32, sample_rate: *mut u32) -> usize { + let tap = &*(tap as *const Tap); + tap.read(std::slice::from_raw_parts_mut(out, capacity), &mut *channels, &mut *sample_rate) +} + +unsafe extern "C" fn source_retain(tap: *const c_void) { + Arc::increment_strong_count(tap as *const Tap); +} + +unsafe extern "C" fn source_release(tap: *const c_void) { + Arc::decrement_strong_count(tap as *const Tap); +} + +impl AudioTapSource { + pub fn new(tap: &Arc) -> Self { + Self { + size: std::mem::size_of::() as u32, + reserved: 0, + tap: Arc::as_ptr(tap) as *const c_void, + read: source_read, + retain: source_retain, + release: source_release, + } + } +} + +fn bytes(reference: &IMemoryBufferReference) -> windows::core::Result<(*mut u8, usize)> { + let access: IMemoryBufferByteAccess = reference.cast()?; + let (mut data, mut capacity) = (std::ptr::null_mut(), 0u32); + unsafe { access.GetBuffer(&mut data, &mut capacity)? }; + Ok((data, capacity as usize)) +} + +#[implement(IBasicAudioEffect, IMediaExtension)] +struct AudioTapEffect { + tap: Mutex>>, + /// (sample rate, channels) of the float frames it is given. + format: Mutex<(u32, u32)>, +} + +impl IMediaExtension_Impl for AudioTapEffect_Impl { + fn SetProperties(&self, configuration: Ref) -> windows::core::Result<()> { + let id = configuration + .ok() + .ok() + .and_then(|configuration| configuration.Lookup(&HSTRING::from(TAP_KEY)).ok()) + .and_then(|value| value.cast::().ok()) + .and_then(|value| value.GetUInt64().ok()); + *self.tap.lock().unwrap() = id.and_then(Tap::find); + Ok(()) + } +} + +impl IBasicAudioEffect_Impl for AudioTapEffect_Impl { + fn UseInputFrameForOutput(&self) -> windows::core::Result { + Ok(false) + } + + fn SupportedEncodingProperties(&self) -> windows::core::Result> { + let float = MediaEncodingSubtypes::Float()?; + let mut formats = Vec::new(); + for sample_rate in [48000, 44100] { + for channels in [2, 1] { + let format = AudioEncodingProperties::CreatePcm(sample_rate, channels, 32)?; + format.SetSubtype(&float)?; + formats.push(Some(format)); + } + } + Ok(IVectorView::from(formats)) + } + + fn SetEncodingProperties(&self, encoding: Ref) -> windows::core::Result<()> { + let encoding = encoding.ok()?; + *self.format.lock().unwrap() = (encoding.SampleRate()?, encoding.ChannelCount()?); + Ok(()) + } + + fn ProcessFrame(&self, context: Ref) -> windows::core::Result<()> { + let context = context.ok()?; + let (input, output) = (context.InputFrame()?, context.OutputFrame()?); + let input = input.LockBuffer(AudioBufferAccessMode::Read)?; + let output = output.LockBuffer(AudioBufferAccessMode::Write)?; + let (input_ref, output_ref) = (input.CreateReference()?, output.CreateReference()?); + let ((input_data, input_capacity), (output_data, output_capacity)) = (bytes(&input_ref)?, bytes(&output_ref)?); + let length = (input.Length()? as usize).min(input_capacity).min(output_capacity) / 4 * 4; + let samples = unsafe { std::slice::from_raw_parts(input_data as *const f32, length / 4) }; + let out = unsafe { std::slice::from_raw_parts_mut(output_data as *mut f32, length / 4) }; + let tap = self.tap.lock().unwrap().clone(); + match tap.filter(|tap| tap.routed.load(Ordering::Acquire)) { + Some(tap) => { + let (sample_rate, channels) = *self.format.lock().unwrap(); + tap.push(samples, channels, sample_rate); + out.fill(0.); + } + None => out.copy_from_slice(samples), + } + output.SetLength(length as u32)?; + input_ref.Close()?; + output_ref.Close()?; + Ok(()) + } + + fn Close(&self, _reason: MediaEffectClosedReason) -> windows::core::Result<()> { + self.tap.lock().unwrap().take(); + Ok(()) + } + + fn DiscardQueuedFrames(&self) -> windows::core::Result<()> { + if let Some(tap) = self.tap.lock().unwrap().as_ref() { + tap.clear(); + } + Ok(()) + } +} + +#[implement(IActivationFactory)] +struct AudioTapFactory; + +impl IActivationFactory_Impl for AudioTapFactory_Impl { + fn ActivateInstance(&self) -> windows::core::Result { + Ok(AudioTapEffect { tap: Mutex::new(None), format: Mutex::new((48000, 2)) }.into()) + } +} + +#[no_mangle] +pub unsafe extern "system" fn DllGetActivationFactory(class_id: *mut c_void, factory: *mut *mut c_void) -> HRESULT { + if factory.is_null() { + return E_POINTER; + } + // Borrowed: the caller keeps the string. + let class_id = &*(&class_id as *const *mut c_void as *const HSTRING); + if *class_id == TAP_CLASS { + *factory = IActivationFactory::from(AudioTapFactory).into_raw(); + S_OK + } else { + *factory = std::ptr::null_mut(); + CLASS_E_CLASSNOTAVAILABLE + } +} + +#[no_mangle] +pub extern "system" fn DllCanUnloadNow() -> HRESULT { + S_FALSE +} diff --git a/crates/canvas-napi/src/gpu/device.rs b/crates/canvas-napi/src/gpu/device.rs index 9a5ac4cfb..e78a0dd68 100644 --- a/crates/canvas-napi/src/gpu/device.rs +++ b/crates/canvas-napi/src/gpu/device.rs @@ -840,6 +840,12 @@ impl g_p_u_device { Ok(unsafe { g_p_u_texture::from_raw(texture) }) } + #[cfg(target_os = "windows")] + #[napi(js_name = "__getAdapterLuid")] + pub fn get_adapter_luid(&self) -> f64 { + unsafe { canvas_c::webgpu::gpu_shared_frame::canvas_native_webgpu_device_get_adapter_luid(self.ptr()) as f64 } + } + /// `importExternalTexture({ nativeTexture, width, height, label? })`: packages/canvas resolves /// the video to a platform texture; undefined where the backend cannot import it. #[napi( diff --git a/packages/audio-context/index.windows.ts b/packages/audio-context/index.windows.ts index 406787909..4c503af65 100644 --- a/packages/audio-context/index.windows.ts +++ b/packages/audio-context/index.windows.ts @@ -22,15 +22,19 @@ import { PeriodicWaveOptions, StereoPannerOptions, WaveShaperOptions, + assertMediaElementUsable, context_, distanceModelFromNumber, distanceModelToNumber, looksLikePath, + markMediaElementUsed, native_, nativeCtor_, normalizeSourcePath, panningModelFromNumber, panningModelToNumber, + throwInvalidMediaElement, + unmarkMediaElementUsed, } from './common'; declare const __non_webpack_require__: (specifier: string) => any; @@ -659,29 +663,45 @@ export class AudioBufferSourceNode extends AudioScheduledSourceNode { } } -function notSupported(message: string): Error { - if (typeof DOMException !== 'undefined') { - return new DOMException(message, 'NotSupportedError'); - } - return new Error(`NotSupportedError: ${message}`); -} - -/** Media elements have no Windows backend yet (canvas-media), so nothing can be tapped. */ export class MediaElementAudioSourceNode extends AudioNode { + private _mediaElement: MediaElementLike; + constructor(context: AudioContext, mediaElement: MediaElementLike) { - super(context, null as never); - throw notSupported('MediaElementAudioSourceNode is not supported on Windows yet'); + super(context, MediaElementAudioSourceNode._createNative(context, mediaElement)); + this._mediaElement = mediaElement; + markMediaElementUsed(mediaElement); + } + + /** canvas-polyfill's