feat(windows): canvas-media, WebGPU video frames and MediaElementAudioSourceNode

.github/workflows/build-native.yml

@@ -14,6 +14,8 @@
 #     <arch>
 #   audio-context-windows- audiocontext.node                     packages/audio-context/platforms/windows/<arch>/
 #     <arch>
+#   canvas-media-windows-  canvasmedia.node                      packages/canvas-media/platforms/windows/<arch>/
+#     <arch>
 #
 # xcframeworks are tarred before upload because upload-artifact does not
 # preserve file permissions.
@@ -23,15 +25,17 @@
 #            (what `npm run build.canvas.ios.framework.release` runs, minus nx)
 #   Android: make android, then gradle :canvas:assembleRelease and copy the aar
 #
-#   Windows: tools/scripts/build-napi.sh <triple> release-napi [canvas-svg-napi|audio-context-napi]
-#            (`make windows`, `make windows-svg`, `make windows-audio`); Skia builds from source
+#   Windows: tools/scripts/build-napi.sh <triple> release-napi [<crate>] (`make windows`,
+#            `make windows-svg`, `make windows-audio`, `make windows-media`); the modules other
+#            than canvasnative.node share one matrix job (napi-windows). Skia builds from source
 #            with the runner's LLVM and ninja, as locally with .tools/env.sh
 #            (.github/actions/setup-windows-native). arm64 is cross-compiled; the x64 jobs also
 #            run the Node-API test suites on the shipped modules (on WARP: runners have no GPU;
 #            audio renders offline and on the 'none' sink: they have no audio device either).
 #
-# canvas-svg and audio-context change rarely, so their jobs cache the final
-# artifact keyed on a hash of their sources and skip the whole build on a hit.
+# The napi-windows modules and the canvas-svg / audio-context mobile builds change rarely, so
+# their jobs cache the final artifact keyed on a hash of their sources and skip the whole build
+# on a hit.
 name: Build Native
 
 permissions:
@@ -54,7 +58,7 @@ on:
         default: ''
 
 env:
-  # bump to invalidate the canvas-svg / audio-context artifact caches
+  # bump to invalidate the canvas-svg / audio-context / canvas-media artifact caches
   NATIVE_CACHE_VERSION: v1
 
 jobs:
@@ -284,12 +288,32 @@ jobs:
           path: packages/canvas/platforms/windows/${{ matrix.arch }}/
           if-no-files-found: error
 
-  canvas-svg-windows:
-    name: Canvas SVG Windows (${{ matrix.arch }})
+  # The Node-API modules other than canvasnative.node (no Skia, no ANGLE): one entry per module.
+  napi-windows:
+    name: ${{ matrix.module.name }} Windows (${{ matrix.arch }})
     runs-on: windows-2025
     strategy:
       fail-fast: false
       matrix:
+        module:
+          # `sources`: every tracked directory the module is built from (its cache key).
+          - name: canvas-svg
+            crate: canvas-svg-napi
+            file: canvassvg.node
+            addon-env: CANVAS_SVG_NAPI_ADDON
+            sources: crates/canvas-svg crates/canvas-svg-c crates/canvas-svg-napi crates/canvas-2d crates/canvas-core
+          # Runners have no audio device: its suite renders offline and on the 'none' sink.
+          - name: audio-context
+            crate: audio-context-napi
+            file: audiocontext.node
+            addon-env: AUDIO_CONTEXT_NAPI_ADDON
+            sources: crates/audio-context-napi
+          - name: canvas-media
+            crate: canvas-media-napi
+            file: canvasmedia.node
+            addon-env: CANVAS_MEDIA_NAPI_ADDON
+            sources: crates/canvas-media-napi
+        arch: [x64, arm64]
         include:
           - arch: x64
             target: x86_64-pc-windows-msvc
@@ -298,83 +322,48 @@ jobs:
     defaults:
       run:
         shell: bash
+    env:
+      MODULE_DIR: packages/${{ matrix.module.name }}/platforms/windows/${{ matrix.arch }}
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event_name == 'workflow_dispatch' && github.event.repository.default_branch || inputs.ref }}
+      # The tree hashes of what the module is built from: equal hashes, equal module.
+      - name: Source key
+        id: sources
+        run: |
+          key=$(for path in ${{ matrix.module.sources }} crates/vendor/napi-sys Cargo.toml tools/scripts/build-napi.sh; do git rev-parse "HEAD:$path" || exit 1; done | sha256sum | cut -c1-32)
+          echo "key=$key" >> "$GITHUB_OUTPUT"
+      # Not on manual runs (CodeQL: cache poisoning via untrusted files); those always build.
       - name: Restore cached module
         if: github.event_name != 'workflow_dispatch'
-        id: svg-node
+        id: cache
         uses: actions/cache@v4
         with:
-          path: packages/canvas-svg/platforms/windows/${{ matrix.arch }}/canvassvg.node
-          key: ${{ env.NATIVE_CACHE_VERSION }}-canvas-svg-windows-${{ matrix.arch }}-${{ hashFiles('crates/canvas-svg/**', 'crates/canvas-svg-c/**', 'crates/canvas-svg-napi/**', 'crates/canvas-2d/**', 'crates/canvas-core/**', 'crates/vendor/napi-sys/**', 'Cargo.toml', 'Cargo.lock', 'tools/scripts/build-napi.sh') }}
+          path: ${{ env.MODULE_DIR }}/${{ matrix.module.file }}
+          key: ${{ env.NATIVE_CACHE_VERSION }}-${{ matrix.module.name }}-windows-${{ matrix.arch }}-${{ steps.sources.outputs.key }}
+      # No Skia here; the action's Rust toolchain and cargo cache are what these need.
       - uses: ./.github/actions/setup-windows-native
-        if: steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch'
+        if: steps.cache.outputs.cache-hit != 'true'
         with:
           target: ${{ matrix.target }}
       - uses: actions/setup-node@v4
-        if: (steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch') && matrix.arch == 'x64'
+        if: steps.cache.outputs.cache-hit != 'true' && matrix.arch == 'x64'
         with:
           node-version: '22'
-      - name: Build canvassvg.node
-        if: steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch'
-        run: ./tools/scripts/build-napi.sh ${{ matrix.target }} release-napi canvas-svg-napi
+      - name: Build ${{ matrix.module.file }}
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: ./tools/scripts/build-napi.sh ${{ matrix.target }} release-napi ${{ matrix.module.crate }}
+      # arm64 is cross-compiled, so only x64 runs them. The timeout bounds a test process that
+      # hangs on exit (an addon releasing Media Foundation / D3D objects during process shutdown).
       - name: Node-API tests
-        if: (steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch') && matrix.arch == 'x64'
-        env:
-          CANVAS_SVG_NAPI_ADDON: ${{ github.workspace }}\packages\canvas-svg\platforms\windows\x64\canvassvg.node
-        run: node --expose-gc --test "crates/canvas-svg-napi/__test__/*.test.mjs"
-      - uses: actions/upload-artifact@v4
-        with:
-          name: canvas-svg-windows-${{ matrix.arch }}
-          path: packages/canvas-svg/platforms/windows/${{ matrix.arch }}/
-          if-no-files-found: error
-
-  audio-context-windows:
-    name: Audio Context Windows (${{ matrix.arch }})
-    runs-on: windows-2025
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: x64
-            target: x86_64-pc-windows-msvc
-          - arch: arm64
-            target: aarch64-pc-windows-msvc
-    defaults:
-      run:
-        shell: bash
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.ref }}
-      - name: Restore cached module
-        id: audio-node
-        uses: actions/cache@v4
-        with:
-          path: packages/audio-context/platforms/windows/${{ matrix.arch }}/audiocontext.node
-          key: ${{ env.NATIVE_CACHE_VERSION }}-audio-context-windows-${{ matrix.arch }}-${{ hashFiles('crates/audio-context-napi/**', 'crates/vendor/napi-sys/**', 'Cargo.toml', 'Cargo.lock', 'tools/scripts/build-napi.sh') }}
-      # No Skia here; the action's Rust toolchain and cargo cache are what this job needs.
-      - uses: ./.github/actions/setup-windows-native
-        if: steps.audio-node.outputs.cache-hit != 'true'
-        with:
-          target: ${{ matrix.target }}
-      - uses: actions/setup-node@v4
-        if: steps.audio-node.outputs.cache-hit != 'true' && matrix.arch == 'x64'
-        with:
-          node-version: '22'
-      - name: Build audiocontext.node
-        if: steps.audio-node.outputs.cache-hit != 'true'
-        run: ./tools/scripts/build-napi.sh ${{ matrix.target }} release-napi audio-context-napi
-      # Runners have no audio device: the suite renders offline and on the 'none' sink.
-      - name: Node-API tests
-        if: steps.audio-node.outputs.cache-hit != 'true' && matrix.arch == 'x64'
-        env:
-          AUDIO_CONTEXT_NAPI_ADDON: ${{ github.workspace }}\packages\audio-context\platforms\windows\x64\audiocontext.node
-        run: node --expose-gc --test "crates/audio-context-napi/__test__/*.test.mjs"
+        if: steps.cache.outputs.cache-hit != 'true' && matrix.arch == 'x64'
+        timeout-minutes: 5
+        run: |
+          export ${{ matrix.module.addon-env }}="$(cygpath -w "$PWD/$MODULE_DIR/${{ matrix.module.file }}")"
+          node --expose-gc --test "crates/${{ matrix.module.crate }}/__test__/*.test.mjs"
       - uses: actions/upload-artifact@v4
         with:
-          name: audio-context-windows-${{ matrix.arch }}
-          path: packages/audio-context/platforms/windows/${{ matrix.arch }}/
+          name: ${{ matrix.module.name }}-windows-${{ matrix.arch }}
+          path: ${{ env.MODULE_DIR }}/
           if-no-files-found: error

.github/workflows/pull-request.yml

@@ -96,7 +96,7 @@ jobs:
           tar -xzf native-artifacts/audio-context-ios/audio-context-ios.tar.gz -C packages/audio-context/platforms/ios
           # Node-API modules (+ ANGLE DLLs) per Windows arch
           for arch in x64 arm64; do
-            for pkg in canvas canvas-svg audio-context; do
+            for pkg in canvas canvas-svg audio-context canvas-media; do
               rm -rf "packages/$pkg/platforms/windows/$arch"
               mkdir -p "packages/$pkg/platforms/windows/$arch"
               cp native-artifacts/$pkg-windows-$arch/* "packages/$pkg/platforms/windows/$arch/"

Cargo.toml

@@ -15,6 +15,7 @@ members = [
     "crates/canvas-napi",
     "crates/canvas-svg-napi",
     "crates/audio-context-napi",
+    "crates/canvas-media-napi",
     "crates/canvas-svg-ios",
     "crates/canvas-svg-android",
 ]

Makefile

@@ -24,6 +24,8 @@ windows-svg: $(addsuffix _svg_napi,$(ARCHS_WINDOWS))
 
 windows-audio: $(addsuffix _audio_napi,$(ARCHS_WINDOWS))
 
+windows-media: $(addsuffix _media_napi,$(ARCHS_WINDOWS))
+
 ios-svg: GENERATE_IOS_SVG
 
 visionos-svg: GENERATE_VISIONOS_SVG
@@ -98,6 +100,10 @@ $(addsuffix _svg_napi,$(ARCHS_WINDOWS)): %_svg_napi:
 $(addsuffix _audio_napi,$(ARCHS_WINDOWS)): %_audio_napi:
 	./tools/scripts/build-napi.sh $* release-napi audio-context-napi
 
+.PHONY: windows-media $(addsuffix _media_napi,$(ARCHS_WINDOWS))
+$(addsuffix _media_napi,$(ARCHS_WINDOWS)): %_media_napi:
+	./tools/scripts/build-napi.sh $* release-napi canvas-media-napi
+
 .PHONY: GENERATE_ANDROID
 GENERATE_ANDROID: $(ARCHS_ANDROID)
 

crates/audio-context-napi/src/context.rs

@@ -232,6 +232,15 @@ impl AudioContext {
     Some(AudioParam::wrap(param))
   }
 
+  #[napi]
+  pub fn create_media_element_source_from_tap(&self, address: f64) -> Result<AudioNode> {
+    let context = self.online()?;
+    let stream = unsafe { crate::tap::TapStream::new(address as usize) }
+      .ok_or_else(|| error("InvalidStateError: the media element has no audio tap"))?;
+    let track = web_audio_api::media_streams::MediaStreamTrack::from_iter(stream);
+    guard(|| AudioNode::new(Kind::MediaElementSource(context.create_media_stream_track_source(&track))))
+  }
+
   #[napi]
   pub fn create_gain(&self) -> Result<AudioNode> {
     self.node(|base| Kind::Gain(base.create_gain()))

crates/audio-context-napi/src/lib.rs

@@ -10,6 +10,7 @@ mod buffer;
 mod context;
 mod node;
 mod param;
+mod tap;
 
 pub use buffer::{AudioBuffer, PeriodicWave};
 pub use context::AudioContext;

crates/audio-context-napi/src/node.rs

@@ -28,6 +28,7 @@ pub(crate) enum Kind {
   DynamicsCompressor(node::DynamicsCompressorNode),
   ChannelSplitter(node::ChannelSplitterNode),
   ChannelMerger(node::ChannelMergerNode),
+  MediaElementSource(node::MediaStreamTrackAudioSourceNode),
 }
 
 /// Evaluates `$body` with `$node` bound to whichever node `$kind` holds.
@@ -50,6 +51,7 @@ macro_rules! any_node {
       Kind::DynamicsCompressor($node) => $body,
       Kind::ChannelSplitter($node) => $body,
       Kind::ChannelMerger($node) => $body,
+      Kind::MediaElementSource($node) => $body,
     }
   };
 }
@@ -102,6 +104,7 @@ impl AudioNode {
       Kind::DynamicsCompressor(_) => "DynamicsCompressorNode",
       Kind::ChannelSplitter(_) => "ChannelSplitterNode",
       Kind::ChannelMerger(_) => "ChannelMergerNode",
+      Kind::MediaElementSource(_) => "MediaElementAudioSourceNode",
     }
   }
 

crates/audio-context-napi/src/tap.rs

@@ -0,0 +1,79 @@
+use std::ffi::c_void;
+
+use web_audio_api::AudioBuffer;
+
+/// Same layout as `tap::AudioTapSource` in canvas-media's module.
+#[repr(C)]
+#[derive(Clone, Copy)]
+struct AudioTapSource {
+  size: u32,
+  reserved: u32,
+  tap: *const c_void,
+  read: unsafe extern "C" fn(tap: *const c_void, out: *mut f32, capacity: usize, channels: *mut u32, sample_rate: *mut u32) -> usize,
+  retain: unsafe extern "C" fn(tap: *const c_void),
+  release: unsafe extern "C" fn(tap: *const c_void),
+}
+
+const FRAMES: usize = 256;
+const MAX_CHANNELS: usize = 8;
+
+/// Pulled on the render thread: a read never waits, and an empty tap is silence.
+pub struct TapStream {
+  source: AudioTapSource,
+  scratch: Vec<f32>,
+  channels: usize,
+  sample_rate: f32,
+}
+
+// The tap is Send + Sync on the producer's side and only reached through `read`.
+unsafe impl Send for TapStream {}
+unsafe impl Sync for TapStream {}
+
+impl TapStream {
+  /// # Safety
+  /// `address` must be an `NSCAudioTap.address` whose tap is alive for the duration of the call;
+  /// the stream then holds its own reference.
+  pub unsafe fn new(address: usize) -> Option<TapStream> {
+    let source = (address as *const AudioTapSource).as_ref()?;
+    if source.size as usize != std::mem::size_of::<AudioTapSource>() || source.tap.is_null() {
+      return None;
+    }
+    (source.retain)(source.tap);
+    Some(TapStream {
+      source: *source,
+      scratch: vec![0.; FRAMES * MAX_CHANNELS],
+      channels: 2,
+      sample_rate: 48000.,
+    })
+  }
+}
+
+impl Drop for TapStream {
+  fn drop(&mut self) {
+    unsafe { (self.source.release)(self.source.tap) };
+  }
+}
+
+impl Iterator for TapStream {
+  type Item = Result<AudioBuffer, Box<dyn std::error::Error + Send + Sync>>;
+
+  fn next(&mut self) -> Option<Self::Item> {
+    let (mut channels, mut sample_rate) = (0u32, 0u32);
+    let frames = unsafe {
+      (self.source.read)(self.source.tap, self.scratch.as_mut_ptr(), self.scratch.len(), &mut channels, &mut sample_rate)
+    };
+    if channels > 0 && (channels as usize) <= MAX_CHANNELS && sample_rate > 0 {
+      self.channels = channels as usize;
+      self.sample_rate = sample_rate as f32;
+    }
+    let channel_count = self.channels;
+    let data: Vec<Vec<f32>> = if frames == 0 {
+      vec![vec![0.; FRAMES]; channel_count]
+    } else {
+      (0..channel_count)
+        .map(|channel| (0..frames).map(|frame| self.scratch[frame * channel_count + channel]).collect())
+        .collect()
+    };
+    Some(Ok(AudioBuffer::from(data, self.sample_rate)))
+  }
+}

crates/canvas-c/Cargo.toml

@@ -61,6 +61,8 @@ objc2-core-foundation = { workspace = true }
 [target.'cfg(target_os="windows")'.dependencies]
 wgpu-core = { workspace = true, features = ["wgsl", "dx12"] }
 wgpu-hal = { workspace = true, features = ["dx12"] }
+# gpu_shared_frame: D3D12 types for wgpu-hal's texture_from_raw (it does not re-export them).
+windows = { workspace = true, features = ["Win32_Foundation", "Win32_Graphics_Direct3D12"] }
 
 [target.'cfg(target_os="android")'.dependencies]
 ndk = { version = "0.7.0", features = ["bitmap"] }

crates/canvas-c/src/webgpu/gpu_adapter.rs

@@ -168,6 +168,8 @@ pub extern "C" fn canvas_native_webgpu_adapter_request_device(
                     }),
                     error_sink: error_sink.clone(),
                     blit: Arc::new(parking_lot::Mutex::new(None)),
+                    #[cfg(target_os = "windows")]
+                    shared_frames: Default::default(),
                 });
 
                 let ret = Arc::into_raw(Arc::new(CanvasGPUDevice {

crates/canvas-c/src/webgpu/gpu_external_texture.rs

@@ -56,12 +56,11 @@ pub unsafe extern "C" fn canvas_native_webgpu_device_import_external_texture(
         return std::ptr::null();
     }
 
-    let Some(plane) = super::gpu_native_texture::import_platform_texture(
-        &device.device,
-        native_texture,
-        width,
-        height,
-    ) else {
+    #[cfg(target_os = "windows")]
+    let plane = super::gpu_shared_frame::import_external_plane(device, native_texture, width, height);
+    #[cfg(not(target_os = "windows"))]
+    let plane = super::gpu_native_texture::import_platform_texture(&device.device, native_texture, width, height);
+    let Some(plane) = plane else {
         return std::ptr::null();
     };