From 97c181f1256b7278d08d7c88ae076446b94716ad Mon Sep 17 00:00:00 2001 From: Osei Fortune Date: Sun, 27 Sep 2026 17:51:38 -0400 Subject: [PATCH 1/7] fix(canvas): fillText's maxWidth is optional; deleteShader takes a WebGLShader - CanvasRenderingContext2D.fillText(text, x, y, maxWidth?): maxWidth is optional, as on the web (strokeText already was). - WebGLRenderingContext.deleteShader(shader: WebGLShader | null) was declared as taking a WebGLRenderbuffer; null is a no-op, as in WebGL (it threw on `shader.native`). --- packages/canvas/Canvas2D/CanvasRenderingContext2D/index.ts | 2 +- packages/canvas/WebGL/WebGLRenderingContext/common.ts | 2 +- packages/canvas/WebGL/WebGLRenderingContext/index.ts | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/canvas/Canvas2D/CanvasRenderingContext2D/index.ts b/packages/canvas/Canvas2D/CanvasRenderingContext2D/index.ts index 1217274c0..3a24a52ff 100644 --- a/packages/canvas/Canvas2D/CanvasRenderingContext2D/index.ts +++ b/packages/canvas/Canvas2D/CanvasRenderingContext2D/index.ts @@ -1176,7 +1176,7 @@ export class CanvasRenderingContext2D implements CanvasRenderingContext { this.context.fillRect(x, y, width, height); } - fillText(text: string, x: number, y: number, maxWidth: number): void { + fillText(text: string, x: number, y: number, maxWidth?: number): void { if (typeof maxWidth === 'number') { this.context.fillText(text + '', x, y, maxWidth); } else { diff --git a/packages/canvas/WebGL/WebGLRenderingContext/common.ts b/packages/canvas/WebGL/WebGLRenderingContext/common.ts index 4e3f52a6f..0949502ce 100644 --- a/packages/canvas/WebGL/WebGLRenderingContext/common.ts +++ b/packages/canvas/WebGL/WebGLRenderingContext/common.ts @@ -1380,7 +1380,7 @@ export abstract class WebGLRenderingCommon implements CanvasRenderingContext { abstract deleteRenderbuffer(renderBuffer: WebGLRenderbuffer): void; - abstract deleteShader(shader: WebGLRenderbuffer): void; + abstract deleteShader(shader: WebGLShader | null): void; abstract deleteTexture(texture: WebGLTexture): void; diff --git a/packages/canvas/WebGL/WebGLRenderingContext/index.ts b/packages/canvas/WebGL/WebGLRenderingContext/index.ts index dab748978..bd064fbe2 100644 --- a/packages/canvas/WebGL/WebGLRenderingContext/index.ts +++ b/packages/canvas/WebGL/WebGLRenderingContext/index.ts @@ -320,7 +320,10 @@ export class WebGLRenderingContextBase extends WebGLRenderingCommon { } @profile - deleteShader(shader: WebGLRenderbuffer): void { + deleteShader(shader: WebGLShader | null): void { + if (!shader) { + return; + } this.native.deleteShader(shader.native); } From 332009e7e75dd9ead85d420bf5a93b683a967267 Mon Sep 17 00:00:00 2001 From: Osei Fortune Date: Sun, 27 Sep 2026 17:56:36 -0400 Subject: [PATCH 2/7] feat(windows): canvas-media on Media Foundation canvas-media's Video, Audio and VideoFrame on NativeScript Windows, over a WinRT MediaPlayer. - crates/canvas-media-napi (canvasmedia.node): NSCMediaPlayerBridge subscribes the player's events natively (Media Foundation raises them on its own threads, where the runtime cannot run JS delegates) and delivers them through threadsafe functions. Video players run in frame-server mode: each frame is copied (CopyFrameToVideoSurface) into a ring of BGRA textures on the module's D3D11 device, presented into the view's XAML SurfaceImageSource and read back (RGBA) on demand. Frames are also shareable with WebGPU (gpuFrame(): NT handles plus ready / release fences) and the audio can be tapped (createAudioTap(): an IBasicAudioEffect the module serves as an in-process WinRT class, registered in the app manifest by plugin.targets). - Exit safety: a MediaPlayer released during process exit (after ExitProcess has killed Media Foundation's threads, e.g. from a thread-local destructor) spins forever in the graphics driver and leaves an unkillable process. An env cleanup hook closes every live player while its threads run, nothing releasable lives in TLS, and no frame copy starts after close(). - player-windows.ts: HTMLMediaElement state and events over the player (readyState, play() promises with AbortError / NotSupportedError, timeupdate while playing, seeking / seeked, volumechange, fallback in order, ms-appx: / file / http sources, canPlayType). - Video: frames shown by an Image; 2D drawImage, WebGL texImage2D / texImage3D / texSubImage3D, WebGPU copyExternalImageToTexture (ImageAsset) and VideoFrame read the current frame once per decoded frame. `controls`: WinUI's transport controls over the frames (the stock MediaPlayerElement template with its presenter kept for layout but invisible: in frame-server mode it only paints black), so canvases keep getting frames. - Audio: WinUI's compact transport controls. - Packaging: platforms/windows plugin.props / plugin.targets, build-napi.sh canvas-media-napi, `make windows-media`, the PR workflow's artifact placement. - CI: canvas-svg, audio-context and canvas-media share one napi-windows matrix job (one entry per module), with the earlier canvas-svg fixes applied to all: no cache on manual runs, the default branch checked out on them, a source key from git tree hashes (Cargo.lock is untracked), and a timeout on the Node-API tests. - demo: a `media` spec suite on Windows (events, drawImage, texImage2D, VideoFrame, copyExternalImageToTexture, importExternalTexture, seeking, errors, audio, createMediaElementSource). --- .github/workflows/build-native.yml | 128 ++- .github/workflows/pull-request.yml | 2 +- Cargo.toml | 1 + Makefile | 6 + crates/canvas-media-napi/Cargo.toml | 45 + .../canvas-media-napi/__test__/media.test.mjs | 87 ++ crates/canvas-media-napi/build.rs | 3 + crates/canvas-media-napi/package.json | 19 + crates/canvas-media-napi/src/lib.rs | 829 ++++++++++++++++++ crates/canvas-media-napi/src/tap.rs | 273 ++++++ .../canvas-media/VideoFrame/index.windows.ts | 116 ++- packages/canvas-media/audio/index.windows.ts | 197 ++++- packages/canvas-media/package.json | 6 +- .../platforms/windows/plugin.props | 5 + .../platforms/windows/plugin.targets | 37 + packages/canvas-media/player-windows.ts | 391 +++++++++ packages/canvas-media/video/index.windows.ts | 409 ++++++++- tools/demo/canvas/spec/index.ts | 4 +- tools/demo/canvas/spec/media.ts | 279 ++++++ tools/scripts/build-napi.sh | 6 +- 20 files changed, 2759 insertions(+), 84 deletions(-) create mode 100644 crates/canvas-media-napi/Cargo.toml create mode 100644 crates/canvas-media-napi/__test__/media.test.mjs create mode 100644 crates/canvas-media-napi/build.rs create mode 100644 crates/canvas-media-napi/package.json create mode 100644 crates/canvas-media-napi/src/lib.rs create mode 100644 crates/canvas-media-napi/src/tap.rs create mode 100644 packages/canvas-media/platforms/windows/plugin.props create mode 100644 packages/canvas-media/platforms/windows/plugin.targets create mode 100644 packages/canvas-media/player-windows.ts create mode 100644 tools/demo/canvas/spec/media.ts diff --git a/.github/workflows/build-native.yml b/.github/workflows/build-native.yml index 4c137c691..7527b3a5a 100644 --- a/.github/workflows/build-native.yml +++ b/.github/workflows/build-native.yml @@ -14,6 +14,8 @@ # # audio-context-windows- audiocontext.node packages/audio-context/platforms/windows// # +# canvas-media-windows- canvasmedia.node packages/canvas-media/platforms/windows// +# # # xcframeworks are tarred before upload because upload-artifact does not # preserve file permissions. @@ -23,15 +25,17 @@ # (what `npm run build.canvas.ios.framework.release` runs, minus nx) # Android: make android, then gradle :canvas:assembleRelease and copy the aar # -# Windows: tools/scripts/build-napi.sh release-napi [canvas-svg-napi|audio-context-napi] -# (`make windows`, `make windows-svg`, `make windows-audio`); Skia builds from source +# Windows: tools/scripts/build-napi.sh release-napi [] (`make windows`, +# `make windows-svg`, `make windows-audio`, `make windows-media`); the modules other +# than canvasnative.node share one matrix job (napi-windows). Skia builds from source # with the runner's LLVM and ninja, as locally with .tools/env.sh # (.github/actions/setup-windows-native). arm64 is cross-compiled; the x64 jobs also # run the Node-API test suites on the shipped modules (on WARP: runners have no GPU; # audio renders offline and on the 'none' sink: they have no audio device either). # -# canvas-svg and audio-context change rarely, so their jobs cache the final -# artifact keyed on a hash of their sources and skip the whole build on a hit. +# The napi-windows modules and the canvas-svg / audio-context mobile builds change rarely, so +# their jobs cache the final artifact keyed on a hash of their sources and skip the whole build +# on a hit. name: Build Native permissions: @@ -54,7 +58,7 @@ on: default: '' env: - # bump to invalidate the canvas-svg / audio-context artifact caches + # bump to invalidate the canvas-svg / audio-context / canvas-media artifact caches NATIVE_CACHE_VERSION: v1 jobs: @@ -284,12 +288,33 @@ jobs: path: packages/canvas/platforms/windows/${{ matrix.arch }}/ if-no-files-found: error - canvas-svg-windows: - name: Canvas SVG Windows (${{ matrix.arch }}) + # The Node-API modules other than canvasnative.node (no Skia, no ANGLE): one entry per module. + # Each is cached on its sources, and the whole build is skipped on a hit. + napi-windows: + name: ${{ matrix.module.name }} Windows (${{ matrix.arch }}) runs-on: windows-2025 strategy: fail-fast: false matrix: + module: + # `sources`: every tracked directory the module is built from (its cache key). + - name: canvas-svg + crate: canvas-svg-napi + file: canvassvg.node + addon-env: CANVAS_SVG_NAPI_ADDON + sources: crates/canvas-svg crates/canvas-svg-c crates/canvas-svg-napi crates/canvas-2d crates/canvas-core + # Runners have no audio device: its suite renders offline and on the 'none' sink. + - name: audio-context + crate: audio-context-napi + file: audiocontext.node + addon-env: AUDIO_CONTEXT_NAPI_ADDON + sources: crates/audio-context-napi + - name: canvas-media + crate: canvas-media-napi + file: canvasmedia.node + addon-env: CANVAS_MEDIA_NAPI_ADDON + sources: crates/canvas-media-napi + arch: [x64, arm64] include: - arch: x64 target: x86_64-pc-windows-msvc @@ -298,83 +323,48 @@ jobs: defaults: run: shell: bash + env: + MODULE_DIR: packages/${{ matrix.module.name }}/platforms/windows/${{ matrix.arch }} steps: - uses: actions/checkout@v4 with: ref: ${{ github.event_name == 'workflow_dispatch' && github.event.repository.default_branch || inputs.ref }} + # The tree hashes of what the module is built from: equal hashes, equal module. + - name: Source key + id: sources + run: | + key=$(for path in ${{ matrix.module.sources }} crates/vendor/napi-sys Cargo.toml tools/scripts/build-napi.sh; do git rev-parse "HEAD:$path" || exit 1; done | sha256sum | cut -c1-32) + echo "key=$key" >> "$GITHUB_OUTPUT" + # Not on manual runs (CodeQL: cache poisoning via untrusted files); those always build. - name: Restore cached module if: github.event_name != 'workflow_dispatch' - id: svg-node + id: cache uses: actions/cache@v4 with: - path: packages/canvas-svg/platforms/windows/${{ matrix.arch }}/canvassvg.node - key: ${{ env.NATIVE_CACHE_VERSION }}-canvas-svg-windows-${{ matrix.arch }}-${{ hashFiles('crates/canvas-svg/**', 'crates/canvas-svg-c/**', 'crates/canvas-svg-napi/**', 'crates/canvas-2d/**', 'crates/canvas-core/**', 'crates/vendor/napi-sys/**', 'Cargo.toml', 'Cargo.lock', 'tools/scripts/build-napi.sh') }} + path: ${{ env.MODULE_DIR }}/${{ matrix.module.file }} + key: ${{ env.NATIVE_CACHE_VERSION }}-${{ matrix.module.name }}-windows-${{ matrix.arch }}-${{ steps.sources.outputs.key }} + # No Skia here; the action's Rust toolchain and cargo cache are what these need. - uses: ./.github/actions/setup-windows-native - if: steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch' + if: steps.cache.outputs.cache-hit != 'true' with: target: ${{ matrix.target }} - uses: actions/setup-node@v4 - if: (steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch') && matrix.arch == 'x64' + if: steps.cache.outputs.cache-hit != 'true' && matrix.arch == 'x64' with: node-version: '22' - - name: Build canvassvg.node - if: steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch' - run: ./tools/scripts/build-napi.sh ${{ matrix.target }} release-napi canvas-svg-napi + - name: Build ${{ matrix.module.file }} + if: steps.cache.outputs.cache-hit != 'true' + run: ./tools/scripts/build-napi.sh ${{ matrix.target }} release-napi ${{ matrix.module.crate }} + # arm64 is cross-compiled, so only x64 runs them. The timeout bounds a test process that + # hangs on exit (an addon releasing Media Foundation / D3D objects during process shutdown). - name: Node-API tests - if: (steps.svg-node.outputs.cache-hit != 'true' || github.event_name == 'workflow_dispatch') && matrix.arch == 'x64' - env: - CANVAS_SVG_NAPI_ADDON: ${{ github.workspace }}\packages\canvas-svg\platforms\windows\x64\canvassvg.node - run: node --expose-gc --test "crates/canvas-svg-napi/__test__/*.test.mjs" - - uses: actions/upload-artifact@v4 - with: - name: canvas-svg-windows-${{ matrix.arch }} - path: packages/canvas-svg/platforms/windows/${{ matrix.arch }}/ - if-no-files-found: error - - audio-context-windows: - name: Audio Context Windows (${{ matrix.arch }}) - runs-on: windows-2025 - strategy: - fail-fast: false - matrix: - include: - - arch: x64 - target: x86_64-pc-windows-msvc - - arch: arm64 - target: aarch64-pc-windows-msvc - defaults: - run: - shell: bash - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref }} - - name: Restore cached module - id: audio-node - uses: actions/cache@v4 - with: - path: packages/audio-context/platforms/windows/${{ matrix.arch }}/audiocontext.node - key: ${{ env.NATIVE_CACHE_VERSION }}-audio-context-windows-${{ matrix.arch }}-${{ hashFiles('crates/audio-context-napi/**', 'crates/vendor/napi-sys/**', 'Cargo.toml', 'Cargo.lock', 'tools/scripts/build-napi.sh') }} - # No Skia here; the action's Rust toolchain and cargo cache are what this job needs. - - uses: ./.github/actions/setup-windows-native - if: steps.audio-node.outputs.cache-hit != 'true' - with: - target: ${{ matrix.target }} - - uses: actions/setup-node@v4 - if: steps.audio-node.outputs.cache-hit != 'true' && matrix.arch == 'x64' - with: - node-version: '22' - - name: Build audiocontext.node - if: steps.audio-node.outputs.cache-hit != 'true' - run: ./tools/scripts/build-napi.sh ${{ matrix.target }} release-napi audio-context-napi - # Runners have no audio device: the suite renders offline and on the 'none' sink. - - name: Node-API tests - if: steps.audio-node.outputs.cache-hit != 'true' && matrix.arch == 'x64' - env: - AUDIO_CONTEXT_NAPI_ADDON: ${{ github.workspace }}\packages\audio-context\platforms\windows\x64\audiocontext.node - run: node --expose-gc --test "crates/audio-context-napi/__test__/*.test.mjs" + if: steps.cache.outputs.cache-hit != 'true' && matrix.arch == 'x64' + timeout-minutes: 5 + run: | + export ${{ matrix.module.addon-env }}="$(cygpath -w "$PWD/$MODULE_DIR/${{ matrix.module.file }}")" + node --expose-gc --test "crates/${{ matrix.module.crate }}/__test__/*.test.mjs" - uses: actions/upload-artifact@v4 with: - name: audio-context-windows-${{ matrix.arch }} - path: packages/audio-context/platforms/windows/${{ matrix.arch }}/ + name: ${{ matrix.module.name }}-windows-${{ matrix.arch }} + path: ${{ env.MODULE_DIR }}/ if-no-files-found: error diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 15c7506e2..e9bf092c1 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -96,7 +96,7 @@ jobs: tar -xzf native-artifacts/audio-context-ios/audio-context-ios.tar.gz -C packages/audio-context/platforms/ios # Node-API modules (+ ANGLE DLLs) per Windows arch for arch in x64 arm64; do - for pkg in canvas canvas-svg audio-context; do + for pkg in canvas canvas-svg audio-context canvas-media; do rm -rf "packages/$pkg/platforms/windows/$arch" mkdir -p "packages/$pkg/platforms/windows/$arch" cp native-artifacts/$pkg-windows-$arch/* "packages/$pkg/platforms/windows/$arch/" diff --git a/Cargo.toml b/Cargo.toml index c81120cc4..2675f76fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "crates/canvas-napi", "crates/canvas-svg-napi", "crates/audio-context-napi", + "crates/canvas-media-napi", "crates/canvas-svg-ios", "crates/canvas-svg-android", ] diff --git a/Makefile b/Makefile index f5d0cade1..8517b8711 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,8 @@ windows-svg: $(addsuffix _svg_napi,$(ARCHS_WINDOWS)) windows-audio: $(addsuffix _audio_napi,$(ARCHS_WINDOWS)) +windows-media: $(addsuffix _media_napi,$(ARCHS_WINDOWS)) + ios-svg: GENERATE_IOS_SVG visionos-svg: GENERATE_VISIONOS_SVG @@ -98,6 +100,10 @@ $(addsuffix _svg_napi,$(ARCHS_WINDOWS)): %_svg_napi: $(addsuffix _audio_napi,$(ARCHS_WINDOWS)): %_audio_napi: ./tools/scripts/build-napi.sh $* release-napi audio-context-napi +.PHONY: windows-media $(addsuffix _media_napi,$(ARCHS_WINDOWS)) +$(addsuffix _media_napi,$(ARCHS_WINDOWS)): %_media_napi: + ./tools/scripts/build-napi.sh $* release-napi canvas-media-napi + .PHONY: GENERATE_ANDROID GENERATE_ANDROID: $(ARCHS_ANDROID) diff --git a/crates/canvas-media-napi/Cargo.toml b/crates/canvas-media-napi/Cargo.toml new file mode 100644 index 000000000..c00fc0f0a --- /dev/null +++ b/crates/canvas-media-napi/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "canvas-media-napi" +version = "0.1.0" +edition = "2021" +publish = false +description = "Node-API binding for @nativescript/canvas-media on Windows: MediaPlayer events and frame-server frames for the Video and Audio views." + +[lib] +crate-type = ["cdylib"] + +[features] +default = ["napi8"] +napi8 = ["napi/napi8"] +# Hosts whose Node-API lacks type tags (NativeScript Windows on JavaScriptCore). +napi7 = ["napi/napi7"] + +[dependencies] +napi = { version = "3.13.0", default-features = false, features = ["dyn-symbols"] } +napi-derive = "3.6.9" + +[target.'cfg(target_os = "windows")'.dependencies] +# `#[windows::core::interface]` (ISurfaceImageSourceNative) expands to `windows_core` paths. +windows-core = "0.62" +# IVectorView for the audio tap's supported formats. +windows-collections = "0.3" +windows = { workspace = true, features = [ + "Foundation", + "Foundation_Collections", + "Graphics_DirectX_Direct3D11", + "Media_Core", + "Media_Effects", + "Media_MediaProperties", + "Media_Playback", + "Win32_Foundation", + "Win32_Graphics_Direct3D", + "Win32_Graphics_Direct3D11", + "Win32_Graphics_Dxgi", + "Win32_Graphics_Dxgi_Common", + "Win32_System_WinRT", + "Win32_Security", + "Win32_System_WinRT_Direct3D11", +] } + +[build-dependencies] +napi-build = "2.5.0" diff --git a/crates/canvas-media-napi/__test__/media.test.mjs b/crates/canvas-media-napi/__test__/media.test.mjs new file mode 100644 index 000000000..d42276622 --- /dev/null +++ b/crates/canvas-media-napi/__test__/media.test.mjs @@ -0,0 +1,87 @@ +// A muted MediaPlayer on the WebGPU demo's clip; frames are copied on WARP where there is no GPU. +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import url from 'node:url'; + +const here = path.dirname(url.fileURLToPath(import.meta.url)); +const root = path.resolve(here, '../../..'); +const file = 'canvas_media_napi.dll'; +const addon = [process.env.CANVAS_MEDIA_NAPI_ADDON, path.join(root, 'target', 'debug', file), path.join(root, 'target', 'release-napi', file)].filter(Boolean).find((f) => fs.existsSync(f)); +const module = { exports: {} }; +process.dlopen(module, addon); +const Media = module.exports; +const clip = url.pathToFileURL(path.join(root, 'tools', 'demo', 'canvas', 'assets', 'webgpu', 'pano.mp4')).href; + +/** Resolves with the events seen once `done(events)` is true; rejects on `error` or timeout. */ +function watch(frames, done, timeoutMs = 20000) { + const events = []; + let bridge; + const promise = new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`timed out; saw ${events.map((e) => e[0]).join(', ')}`)), timeoutMs); + bridge = new Media.NSCMediaPlayerBridge( + Media.__createTestPlayer(clip), + (type, detail) => { + events.push([type, detail]); + if (type === 'error') { + clearTimeout(timer); + reject(new Error(detail)); + } else if (done(events, bridge)) { + clearTimeout(timer); + resolve(events); + } + }, + frames, + ); + }); + return { promise, bridge: () => bridge }; +} + +test('exports the bridge index.windows.ts builds on', () => { + assert.equal(typeof Media.NSCMediaPlayerBridge, 'function'); + assert.throws(() => new Media.NSCMediaPlayerBridge('0x0', () => {}), /Invalid MediaPlayer pointer/); +}); + +test('delivers the player events on the JS thread', async () => { + const { promise, bridge } = watch(false, (events) => events.some(([type, state]) => type === 'state' && state === '3')); + const events = await promise; + assert.ok(events.some(([type]) => type === 'opened'), 'opened'); + // Without frames nothing is copied. + assert.equal(bridge().frameId, 0); + assert.equal(bridge().readPixels(), null); + bridge().close(); +}); + +test('copies frames and reads them back as RGBA', async () => { + const { promise, bridge } = watch(true, (events) => events.filter(([type]) => type === 'frame').length >= 3); + await promise; + const b = bridge(); + const [width, height] = [b.videoWidth, b.videoHeight]; + assert.ok(width > 0 && height > 0, `${width}x${height}`); + assert.ok(b.frameId >= 3); + const pixels = b.readPixels(); + assert.equal(pixels.length, width * height * 4); + let lit = 0; + for (let i = 0; i < pixels.length; i += 4) { + assert.equal(pixels[i + 3], 255); + if (pixels[i] + pixels[i + 1] + pixels[i + 2] > 0) lit++; + } + assert.ok(lit > (width * height) / 10, 'the frame has content'); + // Shared for WebGPU: a descriptor naming the latest frame, on this device's adapter. + assert.equal(b.sharesFrames, true); + assert.ok(b.adapterLuid > 0, `adapter ${b.adapterLuid}`); + const shared = b.gpuFrame(); + assert.ok(shared.address > 0); + assert.equal(shared.width, width); + assert.equal(shared.height, height); + shared.close(); + shared.close(); + // No SurfaceImageSource attached. + assert.equal(b.present(), false); + b.close(); + const id = b.frameId; + await new Promise((resolve) => setTimeout(resolve, 300)); + assert.equal(b.frameId, id, 'no frames after close'); + assert.equal(b.readPixels().length, width * height * 4, 'the last frame stays readable'); +}); diff --git a/crates/canvas-media-napi/build.rs b/crates/canvas-media-napi/build.rs new file mode 100644 index 000000000..0f1b01002 --- /dev/null +++ b/crates/canvas-media-napi/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/crates/canvas-media-napi/package.json b/crates/canvas-media-napi/package.json new file mode 100644 index 000000000..5bc95ffe3 --- /dev/null +++ b/crates/canvas-media-napi/package.json @@ -0,0 +1,19 @@ +{ + "name": "@nativescript/canvas-media-napi", + "private": true, + "version": "0.1.0", + "description": "Node-API build of the canvas-media native module (MediaPlayer bridge for Windows).", + "napi": { + "binaryName": "canvasmedia", + "targets": [ + "x86_64-pc-windows-msvc", + "aarch64-pc-windows-msvc" + ] + }, + "scripts": { + "test": "node --expose-gc --test \"__test__/*.test.mjs\"" + }, + "engines": { + "node": ">=20" + } +} diff --git a/crates/canvas-media-napi/src/lib.rs b/crates/canvas-media-napi/src/lib.rs new file mode 100644 index 000000000..c7e9212e3 --- /dev/null +++ b/crates/canvas-media-napi/src/lib.rs @@ -0,0 +1,829 @@ +#![cfg(target_os = "windows")] +#![deny(clippy::all)] + +mod tap; + +use std::cell::{Cell, RefCell}; +use std::mem::ManuallyDrop; +use std::rc::{Rc, Weak}; +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + +use napi::bindgen_prelude::{FnArgs, Function, ObjectFinalize, Uint8Array}; +use napi::threadsafe_function::{ThreadsafeFunctionCallMode, UnknownReturnValue}; +use napi::{Env, Error, Result}; +use napi_derive::napi; +use windows::core::{IInspectable, IUnknown, Interface, RuntimeType, HRESULT}; +use windows::Foundation::TypedEventHandler; +use windows::Graphics::DirectX::Direct3D11::IDirect3DSurface; +use windows::Media::Playback::{MediaPlaybackSession, MediaPlayer, MediaPlayerFailedEventArgs}; +use windows::core::PCWSTR; +use windows::Win32::Foundation::{CloseHandle, GENERIC_ALL, HANDLE, HMODULE, POINT, RECT}; +use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP}; +use windows::Win32::Graphics::Direct3D11::{ + D3D11CreateDevice, ID3D11Device, ID3D11Device5, ID3D11DeviceContext, ID3D11DeviceContext4, ID3D11Fence, + ID3D11Multithread, ID3D11Resource, ID3D11Texture2D, D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, + D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_FENCE_FLAG_SHARED, + D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_READ, D3D11_RESOURCE_MISC_SHARED, D3D11_RESOURCE_MISC_SHARED_NTHANDLE, + D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, +}; +use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_SAMPLE_DESC}; +use windows::Win32::Graphics::Dxgi::{ + IDXGIDevice, IDXGIResource1, IDXGISurface, DXGI_SHARED_RESOURCE_READ, DXGI_SHARED_RESOURCE_WRITE, +}; +use windows::Win32::System::WinRT::Direct3D11::CreateDirect3D11SurfaceFromDXGISurface; + +/// WinUI 3's `ISurfaceImageSourceNative` (microsoft.ui.xaml.media.dxinterop.h; not the UWP IID). +#[windows::core::interface("e4cecd6c-f14b-4f46-83c3-8bbda27c6504")] +unsafe trait ISurfaceImageSourceNative: windows::core::IUnknown { + fn SetDevice(&self, device: *mut c_void) -> HRESULT; + fn BeginDraw(&self, update_rect: RECT, surface: *mut *mut c_void, offset: *mut POINT) -> HRESULT; + fn EndDraw(&self) -> HRESULT; +} + +/// Parses `NSWinRT.interop.pointerKey(...)` output (`"0x…"`) or a decimal address. +fn parse_pointer_key(key: &str) -> Option<*mut c_void> { + let key = key.trim(); + let address = match key.strip_prefix("0x").or_else(|| key.strip_prefix("0X")) { + Some(hex) => usize::from_str_radix(hex, 16).ok(), + None => key.parse().ok(), + }?; + (address != 0).then_some(address as *mut c_void) +} + +fn to_napi(error: windows::core::Error) -> Error { + Error::from_reason(error.message()) +} + +struct Device { + device: ID3D11Device, + context: ID3D11DeviceContext, + /// `LowPart | HighPart << 32`: WebGPU opens shared frames only on the same adapter. + luid: u64, + /// Fences (Windows 10 1703+): without them frames are not shared with other devices. + fences: Option<(ID3D11Device5, ID3D11DeviceContext4)>, +} + +// Multithread-protected (below), and D3D11 devices are free-threaded. +unsafe impl Send for Device {} +unsafe impl Sync for Device {} + +/// One device for every player. MediaPlayer copies frames into it on its own threads while the JS +/// thread reads them back or presents them, so its immediate context is multithread-protected. +fn device() -> Option<&'static Device> { + static DEVICE: OnceLock> = OnceLock::new(); + DEVICE + .get_or_init(|| create_device(D3D_DRIVER_TYPE_HARDWARE).or_else(|| create_device(D3D_DRIVER_TYPE_WARP))) + .as_ref() +} + +fn create_device(driver: D3D_DRIVER_TYPE) -> Option { + let (mut device, mut context) = (None, None); + unsafe { + D3D11CreateDevice( + None, + driver, + HMODULE::default(), + // BGRA for XAML SurfaceImageSources; video support for MediaPlayer's video processor. + D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + None, + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + } + .ok()?; + let (device, context) = (device?, context?); + let multithread: ID3D11Multithread = context.cast().ok()?; + let _ = unsafe { multithread.SetMultithreadProtected(true) }; + let luid = (|| -> windows::core::Result { + let adapter = unsafe { device.cast::()?.GetAdapter()? }; + let luid = unsafe { adapter.GetDesc()? }.AdapterLuid; + Ok(luid.LowPart as u64 | ((luid.HighPart as u32 as u64) << 32)) + })() + .unwrap_or(0); + let fences = device.cast::().ok().zip(context.cast::().ok()); + Some(Device { device, context, luid, fences }) +} + +fn texture_desc(width: u32, height: u32, shared: bool) -> D3D11_TEXTURE2D_DESC { + D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_B8G8R8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, + CPUAccessFlags: 0, + // Opened by WebGPU on its D3D12 device (canvas-c's gpu_shared_frame). + MiscFlags: if shared { (D3D11_RESOURCE_MISC_SHARED_NTHANDLE.0 | D3D11_RESOURCE_MISC_SHARED.0) as u32 } else { 0 }, + } +} + +static NEXT_SHARED_ID: AtomicU64 = AtomicU64::new(1); + +/// An NT handle to share a texture or fence, closed with it. +struct SharedHandle { + handle: HANDLE, + /// Unique for the process's lifetime, unlike handle values: consumers cache by it. + id: u64, +} + +impl SharedHandle { + fn new(handle: HANDLE) -> Self { + Self { handle, id: NEXT_SHARED_ID.fetch_add(1, Ordering::Relaxed) } + } +} + +impl Drop for SharedHandle { + fn drop(&mut self) { + let _ = unsafe { CloseHandle(self.handle) }; + } +} + +/// The fences frames are shared with: `ready` is signalled here after each copy, `release` by the +/// consumer once it has read a frame. +struct Fences { + ready: ID3D11Fence, + ready_handle: SharedHandle, + release: ID3D11Fence, + release_handle: SharedHandle, +} + +impl Fences { + fn new(device: &ID3D11Device5) -> Option { + let create = || -> windows::core::Result<(ID3D11Fence, SharedHandle)> { + let mut fence: Option = None; + unsafe { device.CreateFence(0, D3D11_FENCE_FLAG_SHARED, &mut fence)? }; + let fence = fence.ok_or_else(windows::core::Error::empty)?; + let handle = unsafe { fence.CreateSharedHandle(None, GENERIC_ALL.0, PCWSTR::null())? }; + Ok((fence, SharedHandle::new(handle))) + }; + let (ready, ready_handle) = create().ok()?; + let (release, release_handle) = create().ok()?; + Some(Fences { ready, ready_handle, release, release_handle }) + } +} + +/// `CanvasD3DSharedFrame` in canvas-c (crates/canvas-c/src/webgpu/gpu_shared_frame.rs): same layout. +#[repr(C)] +struct SharedFrameDesc { + size: u32, + consumed: u32, + adapter_luid: u64, + texture_id: u64, + texture: *mut c_void, + ready_fence_id: u64, + ready_fence: *mut c_void, + ready_value: u64, + release_fence_id: u64, + release_fence: *mut c_void, + release_value: u64, +} + +/// A decoded frame, BGRA at the video's natural size. +struct Frame { + texture: ID3D11Texture2D, + /// `texture` as the WinRT surface `CopyFrameToVideoSurface` takes. + surface: IDirect3DSurface, + /// Set when the device has fences: the texture is shareable. + shared: Option, + /// The release-fence value consumers must reach before the texture is written again. + busy_until: u64, +} + +impl Frame { + fn new(device: &Device, width: u32, height: u32) -> Option { + let shared = device.fences.is_some(); + let mut texture = None; + unsafe { device.device.CreateTexture2D(&texture_desc(width, height, shared), None, Some(&mut texture)) }.ok()?; + let texture = texture?; + let dxgi: IDXGISurface = texture.cast().ok()?; + let surface: IDirect3DSurface = unsafe { CreateDirect3D11SurfaceFromDXGISurface(&dxgi) }.ok()?.cast().ok()?; + let shared = shared + .then(|| { + let resource: IDXGIResource1 = texture.cast().ok()?; + let access = DXGI_SHARED_RESOURCE_READ.0 | DXGI_SHARED_RESOURCE_WRITE.0; + unsafe { resource.CreateSharedHandle(None, access, PCWSTR::null()) }.ok().map(SharedHandle::new) + }) + .flatten(); + Some(Frame { texture, surface, shared, busy_until: 0 }) + } +} + +/// A texture a consumer may still be reading is not written; with every other one busy, a decoded +/// frame is dropped rather than waited for. +const POOL_SIZE: usize = 3; + +#[derive(Default)] +struct Pool { + frames: Vec, + latest: Option, + width: u32, + height: u32, + fences: Option, + /// The ready-fence value signalled after the latest copy. + latest_ready: u64, + next_ready: u64, + next_release: u64, +} + +unsafe impl Send for Pool {} + +impl Pool { + fn latest(&self) -> Option<&Frame> { + self.frames.get(self.latest?) + } +} + +#[derive(Default)] +struct Frames { + pool: Mutex, + /// Bumped per copied frame. + generation: AtomicU64, + /// Set under `pool`'s lock: a `VideoFrameAvailable` still running when its handler is removed + /// copies nothing after the bridge closes. + closed: AtomicBool, +} + +impl Frames { + /// On the player's thread, from `VideoFrameAvailable`. + fn copy_from(&self, player: &MediaPlayer) -> bool { + let Some(device) = device() else { return false }; + let Ok(session) = player.PlaybackSession() else { return false }; + let (width, height) = (session.NaturalVideoWidth().unwrap_or(0), session.NaturalVideoHeight().unwrap_or(0)); + if width == 0 || height == 0 { + return false; + } + let mut pool = self.pool.lock().unwrap(); + if self.closed.load(Ordering::Acquire) { + return false; + } + if pool.width != width || pool.height != height { + pool.frames.clear(); + pool.latest = None; + pool.width = width; + pool.height = height; + } + if pool.fences.is_none() { + pool.fences = device.fences.as_ref().and_then(|(device5, _)| Fences::new(device5)); + } + let released = pool.fences.as_ref().map_or(u64::MAX, |fences| unsafe { fences.release.GetCompletedValue() }); + let latest = pool.latest; + let free = |i: usize| Some(i) != latest && pool.frames.get(i).is_none_or(|frame| frame.busy_until <= released); + let Some(index) = (0..POOL_SIZE).find(|&i| free(i)) else { + return false; + }; + if index == pool.frames.len() { + let Some(frame) = Frame::new(device, width, height) else { return false }; + pool.frames.push(frame); + } + if player.CopyFrameToVideoSurface(&pool.frames[index].surface).is_err() { + return false; + } + if let (Some(fences), Some((_, context4))) = (pool.fences.as_ref(), device.fences.as_ref()) { + let value = pool.next_ready + 1; + if unsafe { context4.Signal(&fences.ready, value) }.is_ok() { + pool.next_ready = value; + } + } + pool.latest = Some(index); + pool.latest_ready = pool.next_ready; + self.generation.fetch_add(1, Ordering::AcqRel); + true + } + + fn close(&self) { + let _pool = self.pool.lock().unwrap(); + self.closed.store(true, Ordering::Release); + } + + fn size(&self) -> (u32, u32) { + let pool = self.pool.lock().unwrap(); + if pool.latest.is_some() { + (pool.width, pool.height) + } else { + (0, 0) + } + } +} + +/// A XAML `SurfaceImageSource` frames are copied into, made by the view at the video's size. +struct XamlSurface { + native: ISurfaceImageSourceNative, + width: u32, + height: u32, +} + +impl XamlSurface { + /// UI thread. + fn present(&self, device: &Device, texture: &ID3D11Texture2D) -> windows::core::Result<()> { + let rect = RECT { left: 0, top: 0, right: self.width as i32, bottom: self.height as i32 }; + let mut surface = std::ptr::null_mut(); + let mut offset = POINT::default(); + unsafe { self.native.BeginDraw(rect, &mut surface, &mut offset) }.ok()?; + // The update rectangle lives in XAML's atlas at `offset`. + let copied = unsafe { IDXGISurface::from_raw(surface) }.cast::().map(|target| unsafe { + device.context.CopySubresourceRegion(&target, 0, offset.x.max(0) as u32, offset.y.max(0) as u32, 0, texture, 0, None) + }); + let ended = unsafe { self.native.EndDraw() }.ok(); + copied.and(ended) + } +} + +/// The player's audio routed to a Web Audio graph (`NSCMediaPlayerBridge.createAudioTap()`). +/// `address` is a `tap::AudioTapSource` audiocontext.node reads through. +#[napi(js_name = "NSCAudioTap")] +pub struct NSCAudioTap { + tap: Arc, + source: Box, +} + +#[napi] +impl NSCAudioTap { + #[napi(getter)] + pub fn address(&self) -> f64 { + &*self.source as *const tap::AudioTapSource as usize as f64 + } + + /// The element's volume (0 when muted): the graph gets what the element would have played. + #[napi] + pub fn set_gain(&self, gain: f64) { + self.tap.set_gain(gain as f32); + } + + /// Routed: the player is silent and its audio goes to the graph. Not routed (initially): it plays + /// as before. + #[napi] + pub fn set_routed(&self, routed: bool) { + self.tap.set_routed(routed); + } + + /// Frames handed to the graph so far. + #[napi(getter)] + pub fn frames_tapped(&self) -> f64 { + self.tap.frames.load(std::sync::atomic::Ordering::Relaxed) as f64 + } +} + +/// A frame handed to WebGPU (`NSCMediaPlayerBridge.gpuFrame()`); holds the handles it names. +#[napi(js_name = "NSCSharedFrame", custom_finalize)] +pub struct NSCSharedFrame { + desc: Box, + frames: Arc, + width: u32, + height: u32, + released: bool, +} + +impl ObjectFinalize for NSCSharedFrame { + fn finalize(mut self, _: Env) -> Result<()> { + self.close(); + Ok(()) + } +} + +#[napi] +impl NSCSharedFrame { + /// The descriptor's address, for `nativeTexture`. + #[napi(getter)] + pub fn address(&self) -> f64 { + &*self.desc as *const SharedFrameDesc as usize as f64 + } + + #[napi(getter)] + pub fn width(&self) -> u32 { + self.width + } + + #[napi(getter)] + pub fn height(&self) -> u32 { + self.height + } + + /// Done with the frame. One never consumed (its import failed) frees its texture at once; a + /// consumed one is freed when the consumer's release signal completes. + #[napi] + pub fn close(&mut self) { + if std::mem::replace(&mut self.released, true) { + return; + } + // Written by the consumer (another module) through the address. + let consumed = unsafe { std::ptr::read_volatile(&self.desc.consumed) }; + if consumed == 0 { + let mut pool = self.frames.pool.lock().unwrap(); + let (texture_id, release_value) = (self.desc.texture_id, self.desc.release_value); + if let Some(frame) = pool.frames.iter_mut().find(|f| f.shared.as_ref().is_some_and(|s| s.id == texture_id)) { + if frame.busy_until == release_value { + frame.busy_until = 0; + } + } + } + } +} + +/// What a bridge holds on the player. A MediaPlayer released while the process exits (after +/// ExitProcess has killed Media Foundation's threads, e.g. from a thread-local destructor) spins +/// forever in the graphics driver, so an env cleanup hook closes every live player while its threads +/// still run, and nothing here is released from process shutdown. +struct Live { + player: MediaPlayer, + unsubscribe: Vec>, + frames: Option>, +} + +impl Live { + fn close(&mut self) { + for unsubscribe in self.unsubscribe.drain(..) { + unsubscribe(); + } + if let Some(frames) = self.frames.as_ref() { + frames.close(); + } + } +} + +type LiveCell = RefCell>; + +thread_local! { + // Weak and ManuallyDrop: dropping these at thread exit releases nothing. + static LIVE: RefCell>> = const { RefCell::new(Vec::new()) }; + static TEST_PLAYERS: RefCell>> = const { RefCell::new(Vec::new()) }; + static CLEANUP_HOOKED: Cell = const { Cell::new(false) }; +} + +/// Closes every live player (bridges and test players) before the env goes away. +fn close_everything() { + for live in LIVE.with(|live| std::mem::take(&mut *live.borrow_mut())) { + if let Some(live) = live.upgrade() { + if let Some(mut live) = live.borrow_mut().take() { + live.close(); + let _ = live.player.Close(); + } + } + } + close_test_players(); +} + +fn ensure_cleanup_hook(env: &Env) -> Result<()> { + if !CLEANUP_HOOKED.get() { + env.add_env_cleanup_hook((), |_| close_everything())?; + CLEANUP_HOOKED.set(true); + } + Ok(()) +} + +type Emit = Arc) + Send + Sync>; +type EventCallback<'a> = Function<'a, FnArgs<(String, Option)>, UnknownReturnValue>; + +fn notify(emit: &Emit, kind: &'static str) -> TypedEventHandler { + let emit = Arc::clone(emit); + TypedEventHandler::new(move |_, _| { + emit(kind, None); + Ok(()) + }) +} + +/// `NSCMediaPlayerBridge`: the native side of canvas-media's Windows `Video` and `Audio`, over a +/// `Windows.Media.Playback.MediaPlayer` the TS creates and drives. +/// +/// MediaPlayer raises its events on Media Foundation threads, where the runtime cannot run JS +/// delegates, so they are subscribed here and delivered on the JS thread (`onEvent`). With +/// `frames`, the player is expected in frame-server mode: each decoded frame is copied into a +/// texture as it arrives, for the view's `SurfaceImageSource` and for canvases (`readPixels`). +#[napi(js_name = "NSCMediaPlayerBridge", custom_finalize)] +pub struct NSCMediaPlayerBridge { + live: Rc, + frames: Option>, + staging: RefCell>, + xaml: RefCell>, +} + +impl ObjectFinalize for NSCMediaPlayerBridge { + fn finalize(mut self, _: Env) -> Result<()> { + self.close(); + Ok(()) + } +} + +#[napi] +impl NSCMediaPlayerBridge { + /// `playerKey`: the MediaPlayer's pointer key. `onEvent(type, detail)`: `opened`, `ended`, + /// `error` (detail: the message), `state` (detail: the `MediaPlaybackState`), `seeked`, + /// `durationchange`, `resize`, `waiting`, `buffered` and, with `frames`, `frame` (at most one + /// queued at a time). + #[napi( + constructor, + ts_args_type = "playerKey: string, onEvent: (type: string, detail?: string) => void, frames?: boolean" + )] + pub fn new(env: Env, player_key: String, on_event: EventCallback, frames: Option) -> Result { + ensure_cleanup_hook(&env)?; + let raw = parse_pointer_key(&player_key) + .ok_or_else(|| Error::from_reason(format!("Invalid MediaPlayer pointer: {player_key}")))?; + let player: MediaPlayer = unsafe { IUnknown::from_raw_borrowed(&raw) } + .ok_or_else(|| Error::from_reason("Invalid MediaPlayer pointer"))? + .cast() + .map_err(to_napi)?; + let session = player.PlaybackSession().map_err(to_napi)?; + + let frame_queued = Arc::new(AtomicBool::new(false)); + let queued = Arc::clone(&frame_queued); + let tsfn = on_event + .build_threadsafe_function::<(String, Option)>() + .weak::() + .build_callback(move |ctx| { + if ctx.value.0 == "frame" { + queued.store(false, Ordering::Release); + } + Ok(FnArgs::from(ctx.value)) + })?; + let emit: Emit = Arc::new(move |kind, detail| { + tsfn.call((kind.to_string(), detail), ThreadsafeFunctionCallMode::NonBlocking); + }); + + let mut unsubscribe: Vec> = Vec::new(); + macro_rules! subscribe { + ($target:expr, $add:ident, $remove:ident, $handler:expr) => {{ + let target = $target.clone(); + let token = target.$add(&$handler).map_err(to_napi)?; + unsubscribe.push(Box::new(move || { + let _ = target.$remove(token); + })); + }}; + } + type OnPlayer = IInspectable; + + subscribe!(player, MediaOpened, RemoveMediaOpened, notify::(&emit, "opened")); + subscribe!(player, MediaEnded, RemoveMediaEnded, notify::(&emit, "ended")); + let failed = { + let emit = Arc::clone(&emit); + TypedEventHandler::::new(move |_, args| { + let message = args.ok().ok().map(|args| { + let message = args.ErrorMessage().map(|m| m.to_string()).unwrap_or_default(); + if message.is_empty() { + let code = args.ExtendedErrorCode().map(|code| code.0).unwrap_or(0); + format!("Media playback failed (0x{:08x})", code as u32) + } else { + message + } + }); + emit("error", message); + Ok(()) + }) + }; + subscribe!(player, MediaFailed, RemoveMediaFailed, failed); + let state = { + let emit = Arc::clone(&emit); + TypedEventHandler::::new(move |session, _| { + let state = session.ok().ok().and_then(|s| s.PlaybackState().ok()).map(|s| s.0.to_string()); + emit("state", state); + Ok(()) + }) + }; + subscribe!(session, PlaybackStateChanged, RemovePlaybackStateChanged, state); + subscribe!(session, SeekCompleted, RemoveSeekCompleted, notify::(&emit, "seeked")); + subscribe!( + session, + NaturalDurationChanged, + RemoveNaturalDurationChanged, + notify::(&emit, "durationchange") + ); + subscribe!( + session, + NaturalVideoSizeChanged, + RemoveNaturalVideoSizeChanged, + notify::(&emit, "resize") + ); + subscribe!(session, BufferingStarted, RemoveBufferingStarted, notify::(&emit, "waiting")); + subscribe!(session, BufferingEnded, RemoveBufferingEnded, notify::(&emit, "buffered")); + + let frames = frames.unwrap_or(false).then(|| Arc::new(Frames::default())); + if let Some(frames) = frames.as_ref() { + let frames = Arc::clone(frames); + let emit = Arc::clone(&emit); + let available = TypedEventHandler::::new(move |player, _| { + if let Ok(player) = player.ok() { + if frames.copy_from(player) && !frame_queued.swap(true, Ordering::AcqRel) { + emit("frame", None); + } + } + Ok(()) + }); + subscribe!(player, VideoFrameAvailable, RemoveVideoFrameAvailable, available); + } + + let live = Rc::new(RefCell::new(Some(Live { player, unsubscribe, frames: frames.clone() }))); + LIVE.with(|all| { + let mut all = all.borrow_mut(); + all.retain(|live| live.strong_count() > 0); + all.push(Rc::downgrade(&live)); + }); + Ok(Self { live, frames, staging: RefCell::new(None), xaml: RefCell::new(None) }) + } + + /// Unsubscribes from the player; the last frame stays readable. + #[napi] + pub fn close(&mut self) { + if let Some(mut live) = self.live.borrow_mut().take() { + live.close(); + } + self.xaml.borrow_mut().take(); + } + + /// The size of the frames copied so far, 0 before the first. + #[napi(getter)] + pub fn video_width(&self) -> u32 { + self.frames.as_ref().map_or(0, |frames| frames.size().0) + } + + #[napi(getter)] + pub fn video_height(&self) -> u32 { + self.frames.as_ref().map_or(0, |frames| frames.size().1) + } + + /// The adapter frames are decoded on (`LowPart | HighPart << 32`, 0 if unknown). + #[napi(getter)] + pub fn adapter_luid(&self) -> f64 { + device().map_or(0., |device| device.luid as f64) + } + + /// Whether frames can be shared with a WebGPU device (on the same adapter). + #[napi(getter)] + pub fn shares_frames(&self) -> bool { + self.frames.is_some() && device().is_some_and(|device| device.fences.is_some()) + } + + /// The latest frame, shared for WebGPU: `address` is a `CanvasD3DSharedFrame` for + /// `nativeTexture`. The texture is not written again until the consumer has signalled its + /// release (or `close()` finds it never staged one). `null` before the first frame. + #[napi] + pub fn gpu_frame(&self) -> Option { + let frames = self.frames.as_ref()?; + let device = device()?; + let mut pool = frames.pool.lock().unwrap(); + let index = pool.latest?; + let (width, height, ready_value) = (pool.width, pool.height, pool.latest_ready); + let fences = pool.fences.as_ref()?; + let (ready_fence_id, ready_fence) = (fences.ready_handle.id, fences.ready_handle.handle.0); + let (release_fence_id, release_fence) = (fences.release_handle.id, fences.release_handle.handle.0); + pool.next_release += 1; + let release_value = pool.next_release; + let frame = &mut pool.frames[index]; + let shared = frame.shared.as_ref()?; + let (texture_id, texture) = (shared.id, shared.handle.0); + frame.busy_until = frame.busy_until.max(release_value); + Some(NSCSharedFrame { + desc: Box::new(SharedFrameDesc { + size: std::mem::size_of::() as u32, + consumed: 0, + adapter_luid: device.luid, + texture_id, + texture, + ready_fence_id, + ready_fence, + ready_value, + release_fence_id, + release_fence, + release_value, + }), + frames: Arc::clone(frames), + width, + height, + released: false, + }) + } + + /// A tap on the player's decoded audio (an audio effect Media Foundation runs on it), passing it + /// through until routed, for a MediaElementAudioSourceNode. Effects apply to the next source + /// set, so this is called before any. Optional for the player: where the effect cannot be + /// activated the element just keeps playing. + #[napi] + pub fn create_audio_tap(&self) -> Result { + let live = self.live.borrow(); + let player = &live.as_ref().ok_or_else(|| Error::from_reason("The player is closed"))?.player; + let tap = tap::Tap::new(); + let configuration = windows::Foundation::Collections::PropertySet::new().map_err(to_napi)?; + configuration + .Insert(&windows::core::HSTRING::from(tap::TAP_KEY), &windows::Foundation::PropertyValue::CreateUInt64(tap.id).map_err(to_napi)?) + .map_err(to_napi)?; + player + .AddAudioEffect(&windows::core::HSTRING::from(tap::TAP_CLASS), true, &configuration) + .map_err(to_napi)?; + let source = Box::new(tap::AudioTapSource::new(&tap)); + Ok(NSCAudioTap { tap, source }) + } + + /// Changes with every frame copied; 0 before the first. + #[napi(getter)] + pub fn frame_id(&self) -> f64 { + self.frames.as_ref().map_or(0., |frames| frames.generation.load(Ordering::Acquire) as f64) + } + + /// Presents frames into a XAML `SurfaceImageSource` of `width` x `height` (the video's size) from + /// now on, replacing any previous one. UI thread. + #[napi] + pub fn attach_surface_image_source(&self, key: String, width: u32, height: u32) -> bool { + let (Some(device), Some(raw)) = (device(), parse_pointer_key(&key)) else { return false }; + let attached = (|| -> windows::core::Result { + let unknown = unsafe { IUnknown::from_raw_borrowed(&raw) }.ok_or_else(windows::core::Error::empty)?; + let native: ISurfaceImageSourceNative = unknown.cast()?; + let dxgi: IDXGIDevice = device.device.cast()?; + unsafe { native.SetDevice(dxgi.as_raw()) }.ok()?; + Ok(XamlSurface { native, width, height }) + })(); + let attached = attached.ok(); + let ok = attached.is_some(); + *self.xaml.borrow_mut() = attached; + ok + } + + #[napi] + pub fn detach_surface_image_source(&self) { + self.xaml.borrow_mut().take(); + } + + /// Copies the current frame into the attached `SurfaceImageSource`; `false` without one of the + /// frame's size. UI thread. + #[napi] + pub fn present(&self) -> bool { + let (Some(frames), Some(device)) = (self.frames.as_ref(), device()) else { return false }; + let xaml = self.xaml.borrow(); + let Some(surface) = xaml.as_ref() else { return false }; + let pool = frames.pool.lock().unwrap(); + if pool.width != surface.width || pool.height != surface.height { + return false; + } + let Some(frame) = pool.latest() else { return false }; + surface.present(device, &frame.texture).is_ok() + } + + /// The current frame's pixels, RGBA and top row first (`videoWidth` x `videoHeight`), read back + /// from the GPU; `null` before the first frame. + #[napi] + pub fn read_pixels(&self) -> Option { + let (frames, device) = (self.frames.as_ref()?, device()?); + let mut staging = self.staging.borrow_mut(); + let pool = frames.pool.lock().unwrap(); + let frame = pool.latest()?; + let (width, height) = (pool.width, pool.height); + if staging.as_ref().is_none_or(|(_, w, h)| *w != width || *h != height) { + let desc = D3D11_TEXTURE2D_DESC { + Usage: D3D11_USAGE_STAGING, + BindFlags: 0, + CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32, + ..texture_desc(width, height, false) + }; + let mut texture = None; + unsafe { device.device.CreateTexture2D(&desc, None, Some(&mut texture)) }.ok()?; + *staging = Some((texture?, width, height)); + } + let (texture, ..) = staging.as_ref()?; + unsafe { device.context.CopyResource(texture, &frame.texture) }; + drop(pool); + + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + unsafe { device.context.Map(texture, 0, D3D11_MAP_READ, 0, Some(&mut mapped)) }.ok()?; + let row = width as usize * 4; + let mut rgba = vec![0u8; row * height as usize]; + for (y, dst) in rgba.chunks_exact_mut(row).enumerate() { + let src = unsafe { std::slice::from_raw_parts((mapped.pData as *const u8).add(y * mapped.RowPitch as usize), row) }; + for (dst, src) in dst.chunks_exact_mut(4).zip(src.chunks_exact(4)) { + dst.copy_from_slice(&[src[2], src[1], src[0], 255]); + } + } + unsafe { device.context.Unmap(texture, 0) }; + Some(Uint8Array::new(rgba)) + } +} + +/// `__createTestPlayer(uri)`: the pointer key of a muted, autoplaying frame-server MediaPlayer on +/// `uri` (tests: Node has no WinRT projection to make one). Kept until `__closeTestPlayers()` or the +/// env's cleanup. +#[napi(js_name = "__createTestPlayer")] +pub fn create_test_player(env: Env, uri: String) -> Result { + ensure_cleanup_hook(&env)?; + let player = MediaPlayer::new().map_err(to_napi)?; + let setup = || -> windows::core::Result<()> { + player.SetIsVideoFrameServerEnabled(true)?; + player.SetIsMuted(true)?; + player.SetAutoPlay(true)?; + let uri = windows::Foundation::Uri::CreateUri(&windows::core::HSTRING::from(uri.as_str()))?; + player.SetSource(&windows::Media::Core::MediaSource::CreateFromUri(&uri)?) + }; + setup().map_err(to_napi)?; + let key = format!("0x{:x}", player.as_raw() as usize); + TEST_PLAYERS.with(|players| players.borrow_mut().push(ManuallyDrop::new(player))); + Ok(key) +} + +#[napi(js_name = "__closeTestPlayers")] +pub fn close_test_players() { + for player in TEST_PLAYERS.with(|players| std::mem::take(&mut *players.borrow_mut())) { + let player = ManuallyDrop::into_inner(player); + let _ = player.Close(); + } +} diff --git a/crates/canvas-media-napi/src/tap.rs b/crates/canvas-media-napi/src/tap.rs new file mode 100644 index 000000000..4791604e8 --- /dev/null +++ b/crates/canvas-media-napi/src/tap.rs @@ -0,0 +1,273 @@ +use std::collections::VecDeque; +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, Weak}; + +use windows::core::{implement, IInspectable, Interface, Ref, HRESULT, HSTRING}; +use windows::Foundation::Collections::IPropertySet; +use windows::Foundation::{IMemoryBufferReference, IPropertyValue}; +use windows::Media::Effects::{IBasicAudioEffect, IBasicAudioEffect_Impl, MediaEffectClosedReason, ProcessAudioFrameContext}; +use windows::Media::MediaProperties::{AudioEncodingProperties, MediaEncodingSubtypes}; +use windows::Media::{AudioBufferAccessMode, IMediaExtension, IMediaExtension_Impl}; +use windows::Win32::Foundation::{CLASS_E_CLASSNOTAVAILABLE, E_POINTER, S_FALSE, S_OK}; +use windows::Win32::System::WinRT::{IActivationFactory, IActivationFactory_Impl, IMemoryBufferByteAccess}; +use windows_collections::IVectorView; + +/// Registered in the app's Package.appxmanifest by platforms/windows/plugin.targets, with this +/// module as its in-process server: Media Foundation activates it by name. +pub const TAP_CLASS: &str = "NativeScript.CanvasMedia.AudioTap"; +/// The `AddAudioEffect` configuration key holding the tap's id. +pub const TAP_KEY: &str = "tap"; +/// At most this much audio waits for the graph; older samples go when it falls behind. +const MAX_BUFFERED_SECONDS: usize = 1; + +/// A MediaPlayer's decoded audio on its way to a Web Audio graph (MediaElementAudioSourceNode). +pub struct Tap { + pub id: u64, + ring: Mutex, + /// While set, the player's own output is silent and its audio goes to the graph instead. + routed: AtomicBool, + /// The element's volume, 0 when muted (f32 bits): the graph gets what the element would play. + gain: AtomicU32, + /// Frames handed over so far (tests: the effect runs). + pub frames: AtomicU64, +} + +#[derive(Default)] +struct Ring { + samples: VecDeque, + channels: u32, + sample_rate: u32, +} + +static TAPS: Mutex>> = Mutex::new(Vec::new()); +static NEXT_TAP: AtomicU64 = AtomicU64::new(1); + +impl Tap { + pub fn new() -> Arc { + let tap = Arc::new(Tap { + id: NEXT_TAP.fetch_add(1, Ordering::Relaxed), + ring: Mutex::new(Ring::default()), + routed: AtomicBool::new(false), + gain: AtomicU32::new(1f32.to_bits()), + frames: AtomicU64::new(0), + }); + let mut taps = TAPS.lock().unwrap(); + taps.retain(|tap| tap.strong_count() > 0); + taps.push(Arc::downgrade(&tap)); + tap + } + + fn find(id: u64) -> Option> { + TAPS.lock().unwrap().iter().filter_map(Weak::upgrade).find(|tap| tap.id == id) + } + + pub fn set_gain(&self, gain: f32) { + self.gain.store(gain.max(0.).to_bits(), Ordering::Relaxed); + } + + pub fn set_routed(&self, routed: bool) { + self.routed.store(routed, Ordering::Release); + if !routed { + self.clear(); + } + } + + fn clear(&self) { + self.ring.lock().unwrap().samples.clear(); + } + + /// Media Foundation's thread: interleaved f32 frames as the player decodes them. + fn push(&self, samples: &[f32], channels: u32, sample_rate: u32) { + let gain = f32::from_bits(self.gain.load(Ordering::Relaxed)); + let mut ring = self.ring.lock().unwrap(); + if ring.channels != channels || ring.sample_rate != sample_rate { + ring.samples.clear(); + ring.channels = channels; + ring.sample_rate = sample_rate; + } + ring.samples.extend(samples.iter().map(|sample| sample * gain)); + self.frames.fetch_add((samples.len() / channels.max(1) as usize) as u64, Ordering::Relaxed); + let max = sample_rate as usize * channels as usize * MAX_BUFFERED_SECONDS; + let excess = ring.samples.len().saturating_sub(max); + ring.samples.drain(..excess - excess % channels.max(1) as usize); + } + + /// The graph's render thread: up to `capacity` interleaved samples of whole frames. + fn read(&self, out: &mut [f32], channels: &mut u32, sample_rate: &mut u32) -> usize { + let mut ring = self.ring.lock().unwrap(); + *channels = ring.channels; + *sample_rate = ring.sample_rate; + if ring.channels == 0 { + return 0; + } + let count = ring.samples.len().min(out.len()); + let count = count - count % ring.channels as usize; + for (out, sample) in out.iter_mut().zip(ring.samples.drain(..count)) { + *out = sample; + } + count / ring.channels as usize + } +} + +/// How another module (audiocontext.node) reads a tap: `NSCAudioTap.address` points at one. +/// `tap` stays valid while the reader holds a reference taken with `retain`. +#[repr(C)] +pub struct AudioTapSource { + pub size: u32, + pub reserved: u32, + pub tap: *const c_void, + /// Copies up to `capacity` interleaved samples (whole frames) into `out`; returns the frame count + /// and reports the format (0 channels: nothing decoded yet). + pub read: unsafe extern "C" fn(tap: *const c_void, out: *mut f32, capacity: usize, channels: *mut u32, sample_rate: *mut u32) -> usize, + pub retain: unsafe extern "C" fn(tap: *const c_void), + pub release: unsafe extern "C" fn(tap: *const c_void), +} + +unsafe extern "C" fn source_read(tap: *const c_void, out: *mut f32, capacity: usize, channels: *mut u32, sample_rate: *mut u32) -> usize { + let tap = &*(tap as *const Tap); + tap.read(std::slice::from_raw_parts_mut(out, capacity), &mut *channels, &mut *sample_rate) +} + +unsafe extern "C" fn source_retain(tap: *const c_void) { + Arc::increment_strong_count(tap as *const Tap); +} + +unsafe extern "C" fn source_release(tap: *const c_void) { + Arc::decrement_strong_count(tap as *const Tap); +} + +impl AudioTapSource { + pub fn new(tap: &Arc) -> Self { + Self { + size: std::mem::size_of::() as u32, + reserved: 0, + tap: Arc::as_ptr(tap) as *const c_void, + read: source_read, + retain: source_retain, + release: source_release, + } + } +} + +fn bytes(reference: &IMemoryBufferReference) -> windows::core::Result<(*mut u8, usize)> { + let access: IMemoryBufferByteAccess = reference.cast()?; + let (mut data, mut capacity) = (std::ptr::null_mut(), 0u32); + unsafe { access.GetBuffer(&mut data, &mut capacity)? }; + Ok((data, capacity as usize)) +} + +/// The effect Media Foundation runs on the player's audio: frames go to the tap (and the player +/// plays silence) while it is routed, and pass through otherwise. +#[implement(IBasicAudioEffect, IMediaExtension)] +struct AudioTapEffect { + tap: Mutex>>, + /// (sample rate, channels) of the float frames it is given. + format: Mutex<(u32, u32)>, +} + +impl IMediaExtension_Impl for AudioTapEffect_Impl { + fn SetProperties(&self, configuration: Ref) -> windows::core::Result<()> { + let id = configuration + .ok() + .ok() + .and_then(|configuration| configuration.Lookup(&HSTRING::from(TAP_KEY)).ok()) + .and_then(|value| value.cast::().ok()) + .and_then(|value| value.GetUInt64().ok()); + *self.tap.lock().unwrap() = id.and_then(Tap::find); + Ok(()) + } +} + +impl IBasicAudioEffect_Impl for AudioTapEffect_Impl { + fn UseInputFrameForOutput(&self) -> windows::core::Result { + Ok(false) + } + + fn SupportedEncodingProperties(&self) -> windows::core::Result> { + let float = MediaEncodingSubtypes::Float()?; + let mut formats = Vec::new(); + for sample_rate in [48000, 44100] { + for channels in [2, 1] { + let format = AudioEncodingProperties::CreatePcm(sample_rate, channels, 32)?; + format.SetSubtype(&float)?; + formats.push(Some(format)); + } + } + Ok(IVectorView::from(formats)) + } + + fn SetEncodingProperties(&self, encoding: Ref) -> windows::core::Result<()> { + let encoding = encoding.ok()?; + *self.format.lock().unwrap() = (encoding.SampleRate()?, encoding.ChannelCount()?); + Ok(()) + } + + fn ProcessFrame(&self, context: Ref) -> windows::core::Result<()> { + let context = context.ok()?; + let (input, output) = (context.InputFrame()?, context.OutputFrame()?); + let input = input.LockBuffer(AudioBufferAccessMode::Read)?; + let output = output.LockBuffer(AudioBufferAccessMode::Write)?; + let (input_ref, output_ref) = (input.CreateReference()?, output.CreateReference()?); + let ((input_data, input_capacity), (output_data, output_capacity)) = (bytes(&input_ref)?, bytes(&output_ref)?); + let length = (input.Length()? as usize).min(input_capacity).min(output_capacity) / 4 * 4; + let samples = unsafe { std::slice::from_raw_parts(input_data as *const f32, length / 4) }; + let out = unsafe { std::slice::from_raw_parts_mut(output_data as *mut f32, length / 4) }; + let tap = self.tap.lock().unwrap().clone(); + match tap.filter(|tap| tap.routed.load(Ordering::Acquire)) { + Some(tap) => { + let (sample_rate, channels) = *self.format.lock().unwrap(); + tap.push(samples, channels, sample_rate); + out.fill(0.); + } + None => out.copy_from_slice(samples), + } + output.SetLength(length as u32)?; + input_ref.Close()?; + output_ref.Close()?; + Ok(()) + } + + fn Close(&self, _reason: MediaEffectClosedReason) -> windows::core::Result<()> { + self.tap.lock().unwrap().take(); + Ok(()) + } + + fn DiscardQueuedFrames(&self) -> windows::core::Result<()> { + if let Some(tap) = self.tap.lock().unwrap().as_ref() { + tap.clear(); + } + Ok(()) + } +} + +#[implement(IActivationFactory)] +struct AudioTapFactory; + +impl IActivationFactory_Impl for AudioTapFactory_Impl { + fn ActivateInstance(&self) -> windows::core::Result { + Ok(AudioTapEffect { tap: Mutex::new(None), format: Mutex::new((48000, 2)) }.into()) + } +} + +/// The in-process server entry point Media Foundation activates `TAP_CLASS` through. +#[no_mangle] +pub unsafe extern "system" fn DllGetActivationFactory(class_id: *mut c_void, factory: *mut *mut c_void) -> HRESULT { + if factory.is_null() { + return E_POINTER; + } + // Borrowed: the caller keeps the string. + let class_id = &*(&class_id as *const *mut c_void as *const HSTRING); + if *class_id == TAP_CLASS { + *factory = IActivationFactory::from(AudioTapFactory).into_raw(); + S_OK + } else { + *factory = std::ptr::null_mut(); + CLASS_E_CLASSNOTAVAILABLE + } +} + +#[no_mangle] +pub extern "system" fn DllCanUnloadNow() -> HRESULT { + S_FALSE +} diff --git a/packages/canvas-media/VideoFrame/index.windows.ts b/packages/canvas-media/VideoFrame/index.windows.ts index 32d55d2f5..958c11db9 100644 --- a/packages/canvas-media/VideoFrame/index.windows.ts +++ b/packages/canvas-media/VideoFrame/index.windows.ts @@ -1,3 +1,113 @@ -// No Windows media backend yet (MediaPlayer/MediaFoundation). Importing fails loudly, so callers -// that probe for it (canvas-polyfill) fall back, and nothing half-works. -throw new Error('@nativescript/canvas-media is not supported on Windows yet'); +import { VideoFrameBase, VideoColorSpace, DOMRectReadOnly, VideoFrameInit, VideoFrameBufferInit, VideoFrameCopyToOptions, PlaneLayout, RectInit, VideoPixelFormat, VideoColorSpaceInit, _VideoFrameData, _INTERNAL, bgraToRgba } from './common'; + +export { VideoColorSpace, DOMRectReadOnly, _INTERNAL }; +export type { VideoPixelFormat, VideoFrameInit, VideoFrameBufferInit, VideoFrameCopyToOptions, PlaneLayout, RectInit, VideoColorSpaceInit }; + +export class VideoFrame extends VideoFrameBase { + constructor(source: ArrayBuffer | ArrayBufferView, init: VideoFrameBufferInit); + constructor(source: object, init?: VideoFrameInit); + constructor(source: any, init?: any) { + if (source === _INTERNAL) { + super(init as _VideoFrameData); + return; + } + + if (source instanceof ArrayBuffer || ArrayBuffer.isView(source)) { + super(VideoFrame._dataFromBuffer(source, init as VideoFrameBufferInit)); + return; + } + super(VideoFrame._dataFromVideoSource(source, init as VideoFrameInit)); + } + + static fromVideo(source: any, init?: VideoFrameInit): VideoFrame | null { + try { + const data = VideoFrame._dataFromVideoSource(source, init); + // Return null when no pixel data is available yet (no frame decoded). + // Callers can treat null as "not ready" and fall back to drawImage. + if (!data.pixelData) return null; + return new VideoFrame(_INTERNAL as any, data); + } catch (err: any) { + console.error('[VideoFrame] fromVideo:', err?.message ?? err); + return null; + } + } + + private static _dataFromBuffer(source: ArrayBuffer | ArrayBufferView, init: VideoFrameBufferInit): _VideoFrameData { + VideoFrame._assertBufferInit(init); + + const buf = ArrayBuffer.isView(source) ? new Uint8Array((source as ArrayBufferView).buffer, (source as ArrayBufferView).byteOffset, (source as ArrayBufferView).byteLength) : new Uint8Array(source as ArrayBuffer); + + const pixelData = init.format === 'BGRA' || init.format === 'BGRX' ? bgraToRgba(buf) : new Uint8Array(buf); + + return { + pixelData, + codedWidth: init.codedWidth, + codedHeight: init.codedHeight, + displayWidth: init.displayWidth ?? init.codedWidth, + displayHeight: init.displayHeight ?? init.codedHeight, + timestamp: init.timestamp, + duration: init.duration ?? null, + format: 'RGBA', + colorSpace: init.colorSpace, + }; + } + + private static _dataFromVideoSource(source: any, init?: VideoFrameInit): _VideoFrameData { + const video = source?._video ?? source; + + if (typeof video?._currentFrame !== 'function') { + throw new TypeError('[VideoFrame] Source must be an HTMLVideoElement or a canvas-media Video instance'); + } + + const timestampUs = typeof init?.timestamp === 'number' ? init.timestamp : Math.round((video.currentTime as number) * 1_000_000); + + // A copy: the video hands the same buffer to every reader until its next frame. + const frame = video._currentFrame(); + const pixelData: Uint8Array | null = frame ? new Uint8Array(frame.pixels) : null; + const codedWidth: number = frame?.width ?? video.videoWidth ?? 0; + const codedHeight: number = frame?.height ?? video.videoHeight ?? 0; + + return { + pixelData, + codedWidth, + codedHeight, + displayWidth: init?.displayWidth ?? codedWidth, + displayHeight: init?.displayHeight ?? codedHeight, + timestamp: timestampUs, + duration: init?.duration ?? null, + format: 'RGBA', + colorSpace: init?.colorSpace, + }; + } + + private static _assertBufferInit(init: VideoFrameBufferInit): void { + if (!init?.format) { + throw new TypeError('VideoFrame: init.format is required for the buffer constructor'); + } + if (!init?.codedWidth || init.codedWidth <= 0) { + throw new TypeError('VideoFrame: init.codedWidth must be a positive integer'); + } + if (!init?.codedHeight || init.codedHeight <= 0) { + throw new TypeError('VideoFrame: init.codedHeight must be a positive integer'); + } + if (typeof init?.timestamp !== 'number' || !isFinite(init.timestamp)) { + throw new TypeError('VideoFrame: init.timestamp must be a finite number (microseconds)'); + } + } + + protected _doClone(): VideoFrame { + const pixelCopy = this._pixelData ? new Uint8Array(this._pixelData) : null; + const data: _VideoFrameData = { + pixelData: pixelCopy, + codedWidth: this.codedWidth, + codedHeight: this.codedHeight, + displayWidth: this.displayWidth, + displayHeight: this.displayHeight, + timestamp: this.timestamp, + duration: this.duration, + format: this.format as VideoPixelFormat, + colorSpace: this.colorSpace.toJSON(), + }; + return new VideoFrame(_INTERNAL as any, data as any); + } +} diff --git a/packages/canvas-media/audio/index.windows.ts b/packages/canvas-media/audio/index.windows.ts index 32d55d2f5..cf83c39b3 100644 --- a/packages/canvas-media/audio/index.windows.ts +++ b/packages/canvas-media/audio/index.windows.ts @@ -1,3 +1,194 @@ -// No Windows media backend yet (MediaPlayer/MediaFoundation). Importing fails loudly, so callers -// that probe for it (canvas-polyfill) fall back, and nothing half-works. -throw new Error('@nativescript/canvas-media is not supported on Windows yet'); +import { booleanConverter } from '@nativescript/core'; +import { AudioBase } from './common'; +import { Source } from '../source'; +import { WindowsMediaPlayer, canPlayType } from '../player-windows'; + +declare const Microsoft: any; + +export class Audio extends AudioBase { + _media: WindowsMediaPlayer; + _sourceView: Source[] = []; + /** ``s not tried yet, in order. */ + private _sources: string[] = []; + _isCustom = false; + private _controls = false; + private _grid: any; + /** The transport controls, while `controls` is set. */ + private _element: any; + + constructor() { + super(); + this._media = new WindowsMediaPlayer(this, false); + } + + static createCustomView() { + const audio = new Audio(); + audio._isCustom = true; + audio.width = 300; + audio.height = 40; + return audio; + } + + get _player() { + return this._media.player; + } + + createNativeView() { + this._grid = new Microsoft.UI.Xaml.Controls.Grid(); + this._syncControls(); + return this._grid; + } + + disposeNativeView() { + try { + this._element?.SetMediaPlayer(null); + } catch (e) {} + this._grid = this._element = null; + super.disposeNativeView(); + } + + private _syncControls() { + const grid = this._grid; + if (!grid) { + return; + } + if (this._controls && !this._element) { + const element = new Microsoft.UI.Xaml.Controls.MediaPlayerElement(); + element.SetMediaPlayer(this._player); + element.AreTransportControlsEnabled = true; + try { + const controls = element.TransportControls; + controls.IsCompact = true; + controls.IsFullWindowButtonVisible = false; + controls.IsZoomButtonVisible = false; + } catch (e) {} + grid.Children.Append(element); + this._element = element; + } else if (!this._controls && this._element) { + const element = this._element; + this._element = null; + try { + element.SetMediaPlayer(null); + grid.Children.Clear(); + } catch (e) {} + } + } + + get readyState() { + return this._media.readyState; + } + + canPlayType(type: string) { + return canPlayType(type); + } + + _addChildFromBuilder(name: string, value: any) { + if (value instanceof Source) { + this._sourceView.push(value); + } + } + + onLoaded() { + super.onLoaded(); + if (!this._media.src) { + this._sources = this._sourceView.filter((item) => item.src && (!item.type || canPlayType(item.type) !== '')).map((item) => item.src); + this._onLoadError(); + } + } + + _onLoadError() { + const next = this._sources.shift(); + if (next === undefined) { + return false; + } + this._media.src = next; + return true; + } + + get duration() { + return this._media.duration; + } + + get currentTime() { + return this._media.currentTime; + } + + set currentTime(value: number) { + this._media.currentTime = value; + } + + get muted() { + return this._media.muted; + } + + set muted(value: boolean) { + this._media.muted = booleanConverter(value as any); + } + + get volume() { + return this._media.volume; + } + + set volume(value: number) { + this._media.volume = value; + } + + get paused() { + return this._media.paused; + } + + get src() { + return this._media.src; + } + + set src(value: string) { + this._sources = []; + this._media.src = value; + } + + load() { + this._media.load(); + } + + play() { + return this._media.play(); + } + + pause() { + this._media.pause(); + } + + get controls() { + return this._controls; + } + + set controls(value: boolean) { + this._controls = booleanConverter(value as any); + this._syncControls(); + } + + get autoplay() { + return this._media.autoplay; + } + + set autoplay(value: boolean) { + this._media.autoplay = booleanConverter(value as any); + } + + get loop() { + return this._media.loop; + } + + set loop(value: boolean) { + this._media.loop = booleanConverter(value as any); + } + + /** For audio-context's MediaElementAudioSourceNode: the element's audio, routed into the graph. */ + attachAudioContextTap(): any { + return this._media.attachAudioTap(); + } + + detachAudioContextTap(): void { + this._media.detachAudioTap(); + } +} diff --git a/packages/canvas-media/package.json b/packages/canvas-media/package.json index dfb4b9df1..cf99bf828 100644 --- a/packages/canvas-media/package.json +++ b/packages/canvas-media/package.json @@ -7,7 +7,8 @@ "nativescript": { "platforms": { "ios": "6.0.0", - "android": "6.0.0" + "android": "6.0.0", + "windows": "9.1.0" } }, "repository": { @@ -19,7 +20,8 @@ "JavaScript", "TypeScript", "iOS", - "Android" + "Android", + "Windows" ], "author": { "name": "NativeScript", diff --git a/packages/canvas-media/platforms/windows/plugin.props b/packages/canvas-media/platforms/windows/plugin.props new file mode 100644 index 000000000..c44da6108 --- /dev/null +++ b/packages/canvas-media/platforms/windows/plugin.props @@ -0,0 +1,5 @@ + + + diff --git a/packages/canvas-media/platforms/windows/plugin.targets b/packages/canvas-media/platforms/windows/plugin.targets new file mode 100644 index 000000000..dd4fdb1e9 --- /dev/null +++ b/packages/canvas-media/platforms/windows/plugin.targets @@ -0,0 +1,37 @@ + + + + <_NSCanvasMediaArch Condition="'$(NSLibPlatform)' != ''">$(NSLibPlatform) + <_NSCanvasMediaArch Condition="'$(_NSCanvasMediaArch)' == '' and ('$(Platform)' == 'ARM64' or '$(Platform)' == 'arm64')">arm64 + + <_NSCanvasMediaArch Condition="'$(_NSCanvasMediaArch)' == '' or '$(_NSCanvasMediaArch)' == 'x86_64'">x64 + <_NSCanvasMediaDir>$(MSBuildThisFileDirectory)$(_NSCanvasMediaArch)\ + + + + + + <_NSCanvasMediaFiles Include="$(_NSCanvasMediaDir)*.node" /> + + + + + + + + + + + + + + + + diff --git a/packages/canvas-media/player-windows.ts b/packages/canvas-media/player-windows.ts new file mode 100644 index 000000000..d8c6901f2 --- /dev/null +++ b/packages/canvas-media/player-windows.ts @@ -0,0 +1,391 @@ +import { knownFolders, path } from '@nativescript/core'; +import { MediaBase, currentTimeProperty, durationProperty } from './common'; + +declare const Windows: any, NSWinRT: any, __non_webpack_require__: (specifier: string) => any; + +export interface SharedFrame { + /** A `CanvasD3DSharedFrame` for WebGPU's `nativeTexture`. */ + readonly address: number; + readonly width: number; + readonly height: number; + close(): void; +} + +/** The player's audio routed into a Web Audio graph (MediaElementAudioSourceNode). */ +export interface AudioTap { + /** An `AudioTapSource` for audiocontext.node's `createMediaElementSourceFromTap`. */ + readonly address: number; + readonly framesTapped: number; + setGain(gain: number): void; + setRouted(routed: boolean): void; +} + +export interface MediaPlayerBridge { + readonly videoWidth: number; + readonly videoHeight: number; + /** Changes with every decoded frame; 0 before the first. */ + readonly frameId: number; + readonly adapterLuid: number; + readonly sharesFrames: boolean; + gpuFrame(): SharedFrame | null; + createAudioTap(): AudioTap; + close(): void; + attachSurfaceImageSource(key: string, width: number, height: number): boolean; + detachSurfaceImageSource(): void; + present(): boolean; + readPixels(): Uint8Array | null; +} + +// Loaded on import, so a missing module makes canvas-polyfill's probes fall back. +const Native: { NSCMediaPlayerBridge: new (playerKey: string, onEvent: (type: string, detail?: string) => void, frames?: boolean) => MediaPlayerBridge } = __non_webpack_require__('system_lib://canvasmedia.node'); + +/** Windows.Media.Playback.MediaPlaybackState */ +const enum PlaybackState { + None = 0, + Opening = 1, + Buffering = 2, + Playing = 3, + Paused = 4, +} + +const HAVE_NOTHING = 0; +const HAVE_METADATA = 1; +const HAVE_ENOUGH_DATA = 4; + +const PLAYABLE = /^(video\/(mp4|x-m4v|quicktime|3gpp|3gpp2|mp2t|webm|x-ms-wmv|x-ms-asf)|audio\/(mpeg|mp3|mp4|x-m4a|aac|aacp|3gpp|3gpp2|wav|wave|x-wav|vnd\.wave|flac|x-flac|ogg|opus|webm|x-ms-wma|amr))$/; + +/** Media Foundation's containers and codecs, plus WebM/Ogg (the Web Media Extensions Windows ships with). */ +export function canPlayType(type: string): '' | 'maybe' { + const mime = String(type ?? '') + .split(';')[0] + .trim() + .toLowerCase(); + return PLAYABLE.test(mime) ? 'maybe' : ''; +} + +/** A WinRT TimeSpan (`{ Duration }` in 100 ns ticks) in seconds. */ +function seconds(timeSpan: any): number { + const value = Number(timeSpan?.Duration ?? timeSpan); + return Number.isFinite(value) ? value / 1e7 : NaN; +} + +export function resolveSrc(src: string): string { + if (typeof src === 'string' && src.startsWith('~/')) { + return path.join(knownFolders.currentApp().path, src.replace('~', '')); + } + return src; +} + +function mediaSource(src: string) { + // A scheme (http:, ms-appx:, file:, ...), not a drive letter. + const uri = /^[a-z][a-z0-9+.-]+:/i.test(src) ? src : 'file:///' + encodeURI(src.replace(/\\/g, '/').replace(/^\/+/, '')).replace(/#/g, '%23').replace(/\?/g, '%3F'); + return Windows.Media.Core.MediaSource.CreateFromUri(new Windows.Foundation.Uri(uri)); +} + +type MediaOwner = MediaBase & { + _notifyListener(type: string): void; + _onFrame?(): void; + /** A load failed: `true` when the owner loads another source instead (its next ``). */ + _onLoadError?(): boolean; +}; + +/** + * A `Windows.Media.Playback.MediaPlayer` with HTMLMediaElement state and events, shared by the + * Windows `Audio` and `Video`. Its events arrive through the native bridge (they are raised off the + * JS thread); with `frames` the player runs in frame-server mode and decoded frames reach the owner's + * `_onFrame`. + */ +export class WindowsMediaPlayer { + readonly player: any; + readonly bridge: MediaPlayerBridge; + readyState = HAVE_NOTHING; + private _src = ''; + private _playing = false; + private _play: { promise: Promise; resolve: () => void; reject: (reason?: any) => void } | null = null; + private _timeupdate: any = null; + private _owner: WeakRef; + private _tap: AudioTap | null = null; + + constructor(owner: MediaOwner, frames: boolean) { + this._owner = new WeakRef(owner); + const player = new Windows.Media.Playback.MediaPlayer(); + player.AutoPlay = false; + if (frames) { + player.IsVideoFrameServerEnabled = true; + } + this.player = player; + const ref = new WeakRef(this); + this.bridge = new Native.NSCMediaPlayerBridge(NSWinRT.interop.pointerKey(player), (type, detail) => ref.deref()?._onEvent(type, detail), frames); + // Before any source: an effect only applies to the sources set after it. Passes the audio + // through until an AudioContext takes it (createMediaElementSource). + try { + this._tap = this.bridge.createAudioTap(); + } catch (e) {} + } + + private get _session() { + return this.player.PlaybackSession; + } + + private _notify(type: string) { + this._owner.deref()?._notifyListener(type); + } + + private _onEvent(type: string, detail?: string) { + const owner = this._owner.deref(); + if (!owner) { + return; + } + switch (type) { + case 'opened': + this.readyState = HAVE_METADATA; + durationProperty.nativeValueChange(owner, this.duration); + this._notify('durationchange'); + this._notify('loadedmetadata'); + this.readyState = HAVE_ENOUGH_DATA; + this._notify('loadeddata'); + this._notify('canplay'); + this._notify('canplaythrough'); + break; + case 'state': + this._onState(Number(detail)); + break; + case 'frame': + owner._onFrame?.(); + break; + case 'seeked': + currentTimeProperty.nativeValueChange(owner, this.currentTime); + this._notify('seeked'); + this._notify('timeupdate'); + break; + case 'durationchange': + if (this.readyState >= HAVE_METADATA) { + durationProperty.nativeValueChange(owner, this.duration); + this._notify('durationchange'); + } + break; + case 'ended': + this._setPlaying(false); + currentTimeProperty.nativeValueChange(owner, this.currentTime); + this._notify('timeupdate'); + this._notify('pause'); + this._notify('ended'); + break; + case 'error': { + if (this.readyState === HAVE_NOTHING && owner._onLoadError?.()) { + break; + } + this._setPlaying(false); + this._rejectPlay(Object.assign(new Error(detail || 'Media playback failed'), { name: 'NotSupportedError' })); + this._notify('error'); + break; + } + case 'resize': + case 'waiting': + this._notify(type); + break; + } + } + + private _onState(state: PlaybackState) { + if (state === PlaybackState.Playing) { + if (this._playing) { + return; + } + this._setPlaying(true); + const play = this._play; + this._play = null; + if (play) { + play.resolve(); + } else { + // Started by autoplay. + this._notify('play'); + } + this._notify('playing'); + } else if (state === PlaybackState.Paused || state === PlaybackState.None) { + this._setPlaying(false); + } + } + + private _setPlaying(playing: boolean) { + this._playing = playing; + if (playing && !this._timeupdate) { + // The web fires timeupdate every 15-250 ms while playing. + this._timeupdate = setInterval(() => { + const owner = this._owner.deref(); + if (owner) { + currentTimeProperty.nativeValueChange(owner, this.currentTime); + this._notify('timeupdate'); + } + }, 250); + } else if (!playing && this._timeupdate) { + clearInterval(this._timeupdate); + this._timeupdate = null; + } + } + + private _rejectPlay(error: Error) { + const play = this._play; + this._play = null; + play?.reject(error); + } + + get src() { + return this._src; + } + + set src(value: string) { + this._src = value ?? ''; + this.load(); + } + + load() { + this._setPlaying(false); + this._rejectPlay(Object.assign(new Error('The play() request was interrupted by a new load request.'), { name: 'AbortError' })); + this.readyState = HAVE_NOTHING; + const src = resolveSrc(this._src); + try { + this.player.Source = src ? mediaSource(src) : null; + } catch (error) { + console.warn(`canvas-media: cannot load ${src}:`, error); + this._notify('error'); + } + } + + play(): Promise { + if (this._playing) { + return Promise.resolve(); + } + if (this._play) { + return this._play.promise; + } + if (!this._src) { + return Promise.reject(Object.assign(new Error('The element has no supported sources.'), { name: 'NotSupportedError' })); + } + let resolve: () => void, reject: (reason?: any) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + this._play = { promise, resolve, reject }; + this._notify('play'); + try { + this.player.Play(); + } catch (error) { + this._rejectPlay(error); + } + return promise; + } + + pause() { + const wasPlaying = this._playing || !!this._play; + try { + this.player.Pause(); + } catch (e) {} + this._setPlaying(false); + this._rejectPlay(Object.assign(new Error('The play() request was interrupted by a call to pause().'), { name: 'AbortError' })); + if (wasPlaying) { + this._notify('pause'); + } + } + + get paused() { + return !this._playing && !this._play; + } + + /** + * Routes the audio into a Web Audio graph instead of the speakers (createMediaElementSource): + * an audio effect in the player's pipeline hands the decoded samples over. null where the + * effect is unavailable. + */ + attachAudioTap(): AudioTap | null { + if (!this._tap) { + return null; + } + this._tap.setRouted(true); + this._syncTapGain(); + return this._tap; + } + + /** The player plays through the speakers again. */ + detachAudioTap() { + this._tap?.setRouted(false); + } + + /** The graph gets what the element would play: its volume, or silence when muted. */ + private _syncTapGain() { + this._tap?.setGain(this.muted ? 0 : this.volume); + } + + get currentTime() { + try { + return seconds(this._session.Position) || 0; + } catch (e) { + return 0; + } + } + + set currentTime(value: number) { + value = Number(value); + if (!Number.isFinite(value) || value < 0) { + return; + } + try { + this._session.Position = { Duration: Math.round(value * 1e7) }; + this._notify('seeking'); + } catch (e) {} + } + + get duration() { + if (this.readyState < HAVE_METADATA) { + return NaN; + } + try { + return seconds(this._session.NaturalDuration) || NaN; + } catch (e) { + return NaN; + } + } + + get muted(): boolean { + return !!this.player.IsMuted; + } + + set muted(value: boolean) { + value = !!value; + if (value !== this.muted) { + this.player.IsMuted = value; + this._syncTapGain(); + this._notify('volumechange'); + } + } + + get volume(): number { + return this.player.Volume; + } + + set volume(value: number) { + value = Math.min(1, Math.max(0, Number(value) || 0)); + if (value !== this.volume) { + this.player.Volume = value; + this._syncTapGain(); + this._notify('volumechange'); + } + } + + get loop(): boolean { + return !!this.player.IsLoopingEnabled; + } + + set loop(value: boolean) { + this.player.IsLoopingEnabled = !!value; + } + + get autoplay(): boolean { + return !!this.player.AutoPlay; + } + + set autoplay(value: boolean) { + this.player.AutoPlay = !!value; + } +} diff --git a/packages/canvas-media/video/index.windows.ts b/packages/canvas-media/video/index.windows.ts index 32d55d2f5..a10efa171 100644 --- a/packages/canvas-media/video/index.windows.ts +++ b/packages/canvas-media/video/index.windows.ts @@ -1,3 +1,406 @@ -// No Windows media backend yet (MediaPlayer/MediaFoundation). Importing fails loudly, so callers -// that probe for it (canvas-polyfill) fall back, and nothing half-works. -throw new Error('@nativescript/canvas-media is not supported on Windows yet'); +import { booleanConverter } from '@nativescript/core'; +import { VideoBase } from './common'; +import { Source } from '../source'; +import { WindowsMediaPlayer, canPlayType } from '../player-windows'; + +declare const Microsoft: any, NSWinRT: any; + +const GL_RGBA = 0x1908; +const GL_UNSIGNED_BYTE = 0x1401; +const GL_RGBA8 = 0x8058; +const GL_SRGB8_ALPHA8 = 0x8c43; + +/** Frames are RGBA8: sized RGBA formats (WebGL 2) are kept, anything else uploads as RGBA. */ +function rgbaInternalFormat(internalformat: number) { + return internalformat === GL_RGBA8 || internalformat === GL_SRGB8_ALPHA8 ? internalformat : GL_RGBA; +} + +/** + * The stock MediaPlayerElement template (Windows App SDK 1.6 generic.xaml) with the presenter kept + * for layout (the element measures itself from it) but invisible: in frame-server mode it renders + * nothing yet still paints black, and the frames show from the view's Image under the element. + */ +const CONTROLS_TEMPLATE = ` + + + + + + + +`; +let controlsTemplate: any; + +interface Frame { + id: number; + pixels: Uint8Array; + width: number; + height: number; +} + +export class Video extends VideoBase { + _media: WindowsMediaPlayer; + _sourceView: Source[] = []; + /** ``s not tried yet, in order. */ + private _sources: string[] = []; + _isCustom = false; + private _controls = false; + private _grid: any; + private _image: any; + /** A MediaPlayerElement over the frames for its transport controls, while `controls` is set. */ + private _element: any; + private _surface: any; + private _surfaceWidth = 0; + private _surfaceHeight = 0; + private _frame: Frame | null = null; + /** The current frame as an ImageAsset, for 2D drawImage and WebGPU. */ + private _asset: any; + private _assetFrameId = 0; + + constructor() { + super(); + this._media = new WindowsMediaPlayer(this, true); + } + + static createCustomView() { + const video = new Video(); + video._isCustom = true; + video.width = 300; + video.height = 150; + return video; + } + + get _player() { + return this._media.player; + } + + get _bridge() { + return this._media.bridge; + } + + createNativeView() { + const { Controls, Media } = Microsoft.UI.Xaml; + const grid = new Controls.Grid(); + const image = new Controls.Image(); + image.Stretch = Media.Stretch.Uniform; + grid.Children.Append(image); + this._grid = grid; + this._image = image; + this._surfaceWidth = this._surfaceHeight = 0; + this._syncControls(); + return grid; + } + + disposeNativeView() { + try { + this._element?.SetMediaPlayer(null); + } catch (e) {} + this._bridge.detachSurfaceImageSource(); + this._grid = this._image = this._element = this._surface = null; + super.disposeNativeView(); + } + + private _syncControls() { + const grid = this._grid; + if (!grid) { + return; + } + if (this._controls && !this._element) { + const element = new Microsoft.UI.Xaml.Controls.MediaPlayerElement(); + controlsTemplate ??= Microsoft.UI.Xaml.Markup.XamlReader.Load(CONTROLS_TEMPLATE); + element.Template = controlsTemplate; + element.SetMediaPlayer(this._player); + element.AreTransportControlsEnabled = true; + try { + // Full window and zoom act on the invisible presenter. + element.TransportControls.IsFullWindowButtonVisible = false; + element.TransportControls.IsZoomButtonVisible = false; + } catch (e) {} + grid.Children.Append(element); + this._element = element; + } else if (!this._controls && this._element) { + const element = this._element; + this._element = null; + try { + element.SetMediaPlayer(null); + // Always the last child, over the image. + grid.Children.RemoveAtEnd(); + } catch (e) {} + } + } + + /** Each decoded frame: shown in the view (when it has one), then requestVideoFrameCallback. */ + _onFrame() { + this._present(); + this._notifyVideoFrameCallbacks(); + } + + private _present() { + const image = this._image; + if (!image) { + return; + } + const bridge = this._bridge; + const width = bridge.videoWidth; + const height = bridge.videoHeight; + if (!width || !height) { + return; + } + if (width !== this._surfaceWidth || height !== this._surfaceHeight) { + const surface = new Microsoft.UI.Xaml.Media.Imaging.SurfaceImageSource(width, height, true); + if (!bridge.attachSurfaceImageSource(NSWinRT.interop.pointerKey(surface), width, height)) { + return; + } + image.Source = surface; + this._surface = surface; + this._surfaceWidth = width; + this._surfaceHeight = height; + } + bridge.present(); + } + + /** The current frame's RGBA pixels, read back once per decoded frame. */ + _currentFrame(): Frame | null { + const bridge = this._bridge; + const id = bridge.frameId; + if (!id) { + return null; + } + if (this._frame?.id !== id) { + const width = bridge.videoWidth; + const height = bridge.videoHeight; + const pixels = bridge.readPixels(); + if (!pixels || pixels.length !== width * height * 4) { + return null; + } + this._frame = { id, pixels, width, height }; + } + return this._frame; + } + + private _currentAsset(): any { + const frame = this._currentFrame(); + if (!frame) { + return null; + } + if (this._assetFrameId !== frame.id) { + this._asset ??= new global.CanvasModule.ImageAsset(); + // Video frames are opaque: premultiplied as they are. + if (!this._asset.fromBytesSync(frame.width, frame.height, frame.pixels, true)) { + return null; + } + this._assetFrameId = frame.id; + } + return this._asset; + } + + /** WebGL `texImage2D` / `texSubImage2D` with this video: `(native, context, target, level, internalformat, ...)`. */ + getCurrentFrame(context?: any) { + const native = arguments[0]; + const frame = this._currentFrame(); + if (!frame || typeof native?.texImage2D !== 'function') { + return; + } + native.texImage2D(arguments[2], arguments[3], rgbaInternalFormat(arguments[4]), frame.width, frame.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, frame.pixels); + } + + getFrameForTexImage3D(nativeCtx: any, ctx: any, target: number, level: number, internalformat: number, width: number, height: number, depth: number, border: number, format: number, type: number) { + const frame = this._currentFrame(); + // One frame is one slice. + if (!frame || depth !== 1) { + return; + } + nativeCtx.texImage3D(target, level, rgbaInternalFormat(internalformat), frame.width, frame.height, 1, border, GL_RGBA, GL_UNSIGNED_BYTE, frame.pixels); + } + + getFrameForTexSubImage3D(nativeCtx: any, ctx: any, target: number, level: number, xoffset: number, yoffset: number, zoffset: number, width: number, height: number, depth: number, format: number, type: number) { + const frame = this._currentFrame(); + if (!frame || depth !== 1) { + return; + } + nativeCtx.texSubImage3D(target, level, xoffset, yoffset, zoffset, frame.width, frame.height, 1, GL_RGBA, GL_UNSIGNED_BYTE, frame.pixels); + } + + /** 2D `drawImage(video, ...)`; `args` are drawImage's arguments. */ + drawImageFrame(context2d: any, args: any[]) { + const asset = this._currentAsset(); + if (!asset || !context2d?.context) { + return; + } + if (args.length === 3) { + context2d.context.drawImage(asset, args[1], args[2]); + } else if (args.length === 5) { + context2d.context.drawImage(asset, args[1], args[2], args[3], args[4]); + } else if (args.length === 9) { + context2d.context.drawImage(asset, args[1], args[2], args[3], args[4], args[5], args[6], args[7], args[8]); + } + } + + /** For audio-context's MediaElementAudioSourceNode: the element's audio, routed into the graph. */ + attachAudioContextTap(): any { + return this._media.attachAudioTap(); + } + + detachAudioContextTap(): void { + this._media.detachAudioTap(); + } + + /** The current frame for WebGPU's `copyExternalImageToTexture` (an ImageAsset), or null. */ + getVideoFrameData(): any { + return this._currentAsset(); + } + + private _gpuFrameId = 0; + + /** Whether frames reach a WebGPU device on the GPU: `device` is its adapter LUID (`__frameDevice`). */ + supportsGPUFrames(device: number): boolean { + const bridge = this._bridge; + return !!device && bridge.sharesFrames && bridge.adapterLuid === device; + } + + /** + * The current frame, shared with WebGPU (`texturePointer` for `nativeTexture`), or null when no + * frame was decoded since the last one handed out. `close()` it once the upload is issued. + */ + getGPUFrameTexture(device: number): any { + const bridge = this._bridge; + const id = bridge.frameId; + if (!id || id === this._gpuFrameId || !this.supportsGPUFrames(device)) { + return null; + } + const frame = bridge.gpuFrame(); + if (!frame) { + return null; + } + this._gpuFrameId = id; + return { + texturePointer: frame.address, + width: frame.width, + height: frame.height, + close: () => frame.close(), + }; + } + + //@ts-ignore + get readyState() { + return this._media.readyState; + } + + get videoWidth() { + return this._bridge.videoWidth; + } + + get videoHeight() { + return this._bridge.videoHeight; + } + + canPlayType(type: string) { + return canPlayType(type); + } + + _addChildFromBuilder(name: string, value: any) { + if (value instanceof Source) { + this._sourceView.push(value); + } + } + + onLoaded() { + super.onLoaded(); + if (!this._media.src) { + this._sources = this._sourceView.filter((item) => item.src && (!item.type || canPlayType(item.type) !== '')).map((item) => item.src); + this._onLoadError(); + } + } + + _onLoadError() { + const next = this._sources.shift(); + if (next === undefined) { + return false; + } + this._media.src = next; + return true; + } + + get duration() { + return this._media.duration; + } + + get currentTime() { + return this._media.currentTime; + } + + set currentTime(value: number) { + this._media.currentTime = value; + } + + get muted() { + return this._media.muted; + } + + set muted(value: boolean) { + this._media.muted = booleanConverter(value as any); + } + + get volume() { + return this._media.volume; + } + + set volume(value: number) { + this._media.volume = value; + } + + get paused() { + return this._media.paused; + } + + get src() { + return this._media.src; + } + + set src(value: string) { + this._sources = []; + this._frame = null; + this._assetFrameId = 0; + this._media.src = value; + } + + load() { + this._media.load(); + } + + play() { + return this._media.play(); + } + + pause() { + this._media.pause(); + } + + get controls() { + return this._controls; + } + + set controls(value: boolean) { + this._controls = booleanConverter(value as any); + this._syncControls(); + } + + //@ts-ignore + get autoplay() { + return this._media.autoplay; + } + + set autoplay(value: boolean) { + this._media.autoplay = booleanConverter(value as any); + } + + // @ts-ignore + get loop() { + return this._media.loop; + } + + // @ts-ignore + set loop(value: boolean | string) { + this._media.loop = booleanConverter(value as any); + } +} diff --git a/tools/demo/canvas/spec/index.ts b/tools/demo/canvas/spec/index.ts index a9fbce149..6309ce41a 100644 --- a/tools/demo/canvas/spec/index.ts +++ b/tools/demo/canvas/spec/index.ts @@ -6,7 +6,7 @@ * adb logcat -d | grep 'SPEC|' * * `--es suite ` narrows the run: a group (`2d`, `webgl`, `webgpu`, - * `imagebitmap`, `bitmaprenderer`, `audio` on Windows) or a single suite (`2d.path2d`). + * `imagebitmap`, `bitmaprenderer`, `audio` and `media` on Windows) or a single suite (`2d.path2d`). * `contextlost` (Windows) runs on its own: it removes the GPU device. */ @@ -19,6 +19,7 @@ import { registerCanvasSourceSpec } from './canvassource'; import { registerScalingSpec } from './scaling'; import { registerContextLossSpec } from './contextloss'; import { registerAudioSpec } from './audio'; +import { registerMediaSpec } from './media'; declare const __WINDOWS__: boolean; @@ -37,6 +38,7 @@ export async function runSpecTests(group: SpecGroup = 'all') { registerScalingSpec(); if (typeof __WINDOWS__ !== 'undefined' && __WINDOWS__) { registerAudioSpec(); + registerMediaSpec(); } // Only when asked for: removing the GPU device takes every other GPU suite in the process with it. if (group === 'contextlost') { diff --git a/tools/demo/canvas/spec/media.ts b/tools/demo/canvas/spec/media.ts new file mode 100644 index 000000000..90fa67177 --- /dev/null +++ b/tools/demo/canvas/spec/media.ts @@ -0,0 +1,279 @@ +import { AudioContext } from '@nativescript/audio-context'; +import { suite, test, ok, equal, closeTo, rejects, make2D, makeCanvas, pixelAt } from './harness'; + +declare const document: any, navigator: any, GPUTextureUsage: any, GPUBufferUsage: any, GPUMapMode: any; + +// ms-appx: the packaged app's own files, whatever `~/` resolves to. +const CLIP = 'ms-appx:///app/assets/file-assets/webgpu/pano.mp4'; +const TONE = 'ms-appx:///app/assets/file-assets/audio/gs-16b-1c-44100hz.wav'; +const MAGENTA = [255, 0, 255, 255]; + +function once(target: any, type: string, timeoutMs = 10000): Promise { + return new Promise((resolve, reject) => { + let done = false; + const timer = setTimeout(() => { + done = true; + reject(new Error(`no ${type} event within ${timeoutMs}ms`)); + }, timeoutMs); + target.addEventListener(type, () => { + if (!done) { + done = true; + clearTimeout(timer); + resolve(); + } + }); + }); +} + +function nextFrame(video: any, timeoutMs = 5000): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`no video frame within ${timeoutMs}ms`)), timeoutMs); + video.requestVideoFrameCallback(() => { + clearTimeout(timer); + resolve(); + }); + }); +} + +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function rms(samples: Float32Array) { + let sum = 0; + for (let i = 0; i < samples.length; i++) sum += samples[i] * samples[i]; + return Math.sqrt(sum / samples.length); +} + +function isMagenta(pixel: ArrayLike) { + return pixel[0] === MAGENTA[0] && pixel[1] === MAGENTA[1] && pixel[2] === MAGENTA[2]; +} + +/** The texture's top-left pixel after `encode` renders or copies into it. */ +async function firstPixel(device: any, texture: any, size: number) { + const bytesPerRow = 256; + const readback = device.createBuffer({ size: bytesPerRow * size, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ }); + const encoder = device.createCommandEncoder(); + encoder.copyTextureToBuffer({ texture }, { buffer: readback, bytesPerRow }, { width: size, height: size, depthOrArrayLayers: 1 }); + device.queue.submit([encoder.finish()]); + await readback.mapAsync(GPUMapMode.READ); + const pixel = new Uint8Array(readback.getMappedRange().slice(0, 4)); + readback.unmap(); + return pixel; +} + +const EXTERNAL_WGSL = ` +@group(0) @binding(0) var frame: texture_external; +@group(0) @binding(1) var frameSampler: sampler; + +@vertex +fn vs(@builtin(vertex_index) index: u32) -> @builtin(position) vec4 { + let corners = array(vec2(-1.0, -1.0), vec2(3.0, -1.0), vec2(-1.0, 3.0)); + return vec4(corners[index], 0.0, 1.0); +} + +@fragment +fn fs(@builtin(position) position: vec4) -> @location(0) vec4 { + return textureSampleBaseClampToEdge(frame, frameSampler, position.xy / 64.0); +} +`; + +let playing: Promise | null = null; + +/** One muted, looping clip for the suite, playing with a frame decoded. */ +function playingVideo(): Promise { + playing ??= (async () => { + const video = document.createElement('video'); + video.muted = true; + video.loop = true; + const loaded = once(video, 'loadeddata'); + video.src = CLIP; + await loaded; + await video.play(); + await nextFrame(video); + return video; + })(); + return playing; +} + +export function registerMediaSpec() { + suite('media.video', () => { + test('canvas-polyfill backs