From f331c97132af59c815f79f6613fa98e9b2552b01 Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Sun, 2 Aug 2026 21:12:13 +0200 Subject: [PATCH 01/10] feat: add reliable macOS audio capture streams --- build/app.zig | 4 + changelog.d/macos-audio-capture.md | 1 + docs/src/app/docs/audio-capture/layout.tsx | 7 + docs/src/app/docs/audio-capture/page.mdx | 174 +++ docs/src/app/docs/capabilities/page.mdx | 7 + docs/src/app/docs/platform-support/page.mdx | 10 + docs/src/app/docs/typescript/page.mdx | 4 + docs/src/lib/docs-navigation.ts | 1 + docs/src/lib/page-titles.ts | 1 + examples/README.md | 2 + examples/audio-capture-ts/README.md | 9 + examples/audio-capture-ts/app.zon | 36 + examples/audio-capture-ts/package.json | 7 + examples/audio-capture-ts/src/app.native | 29 + examples/audio-capture-ts/src/core.ts | 125 +++ examples/audio-capture-ts/tsconfig.json | 10 + examples/audio-capture/README.md | 11 + examples/audio-capture/app.zon | 36 + examples/audio-capture/src/main.zig | 251 +++++ examples/browser/build.zig | 4 + examples/capabilities/build.zig | 4 + examples/capabilities/src/main.zig | 2 +- examples/command-app/build.zig | 4 + examples/command-app/src/main.zig | 2 +- examples/gpu-components/src/app.zig | 2 +- examples/gpu-surface/src/main.zig | 2 +- examples/hello/build.zig | 4 + examples/native-panels/build.zig | 4 + examples/native-panels/src/main.zig | 2 +- examples/native-shell/build.zig | 4 + examples/native-shell/src/main.zig | 2 +- examples/next/build.zig | 4 + examples/react/build.zig | 4 + examples/svelte/build.zig | 4 + examples/vue/build.zig | 4 + examples/webview/build.zig | 4 + packages/core/rt/rt.zig | 87 +- packages/core/sdk/core.ts | 164 +++ packages/core/sdk/events.ts | 53 + packages/core/src/emitter.ts | 187 +++- packages/core/test/effects.test.ts | 140 ++- packages/native-sdk/native-sdk.d.ts | 6 + src/app_runner/root.zig | 1 + src/platform/linux/root.zig | 1 + src/platform/macos/appkit_host.h | 29 + src/platform/macos/appkit_host.m | 73 ++ src/platform/macos/audio_capture.h | 58 + src/platform/macos/audio_capture.m | 575 ++++++++++ src/platform/macos/cef_host.mm | 12 + src/platform/macos/root.zig | 199 ++++ src/platform/null_platform.zig | 170 +++ src/platform/root.zig | 21 + src/platform/types.zig | 182 ++++ src/platform/windows/root.zig | 1 + src/primitives/app_manifest/root.zig | 3 + src/primitives/app_manifest/tests.zig | 25 + src/primitives/app_manifest/types.zig | 14 + src/primitives/app_manifest/validation.zig | 21 + src/root.zig | 13 + src/runtime/api.zig | 8 + src/runtime/bridge_payload.zig | 3 + src/runtime/effects.zig | 1063 +++++++++++++++++++ src/runtime/effects_audio_tests.zig | 201 ++++ src/runtime/effects_video_tests.zig | 21 +- src/runtime/flow.zig | 8 + src/runtime/root.zig | 13 + src/runtime/session_journal.zig | 200 ++++ src/runtime/session_replay.zig | 73 +- src/runtime/session_tests.zig | 237 ++++- src/runtime/ts_core_host.zig | 433 +++++++- src/runtime/ts_core_host_tests.zig | 315 ++++++ src/runtime/ui_app.zig | 40 + src/security/root.zig | 2 + src/tooling/manifest.zig | 42 + src/tooling/package.zig | 45 +- src/tooling/raw_manifest.zig | 6 + src/tooling/templates.zig | 5 + 77 files changed, 5499 insertions(+), 32 deletions(-) create mode 100644 changelog.d/macos-audio-capture.md create mode 100644 docs/src/app/docs/audio-capture/layout.tsx create mode 100644 docs/src/app/docs/audio-capture/page.mdx create mode 100644 examples/audio-capture-ts/README.md create mode 100644 examples/audio-capture-ts/app.zon create mode 100644 examples/audio-capture-ts/package.json create mode 100644 examples/audio-capture-ts/src/app.native create mode 100644 examples/audio-capture-ts/src/core.ts create mode 100644 examples/audio-capture-ts/tsconfig.json create mode 100644 examples/audio-capture/README.md create mode 100644 examples/audio-capture/app.zon create mode 100644 examples/audio-capture/src/main.zig create mode 100644 src/platform/macos/audio_capture.h create mode 100644 src/platform/macos/audio_capture.m diff --git a/build/app.zig b/build/app.zig index 09c7010d1..85f0429cc 100644 --- a/build/app.zig +++ b/build/app.zig @@ -934,6 +934,7 @@ fn linkPlatform(b: *std.Build, dep: *std.Build.Dependency, target: std.Build.Res const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = dep.path("src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = dep.path("src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -963,6 +964,9 @@ fn linkPlatform(b: *std.Build, dep: *std.Build.Dependency, target: std.Build.Res app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the video player's // AVPlayerItemVideoOutput frames). CoreMedia's CMTime use stays // header-only, but the pixel-buffer calls are real symbols. diff --git a/changelog.d/macos-audio-capture.md b/changelog.d/macos-audio-capture.md new file mode 100644 index 000000000..3e5f4049a --- /dev/null +++ b/changelog.d/macos-audio-capture.md @@ -0,0 +1 @@ +feature: **macOS audio capture**: add macOS 15+ reliable paired system-audio and selectable-microphone s16le PCM streams, microphone enumeration, explicit permission flows, and Zig/TypeScript pull APIs. diff --git a/docs/src/app/docs/audio-capture/layout.tsx b/docs/src/app/docs/audio-capture/layout.tsx new file mode 100644 index 000000000..b6e5f5035 --- /dev/null +++ b/docs/src/app/docs/audio-capture/layout.tsx @@ -0,0 +1,7 @@ +import { pageMetadata } from "@/lib/page-metadata"; + +export const metadata = pageMetadata("audio-capture"); + +export default function Layout({ children }: { children: React.ReactNode }) { + return children; +} diff --git a/docs/src/app/docs/audio-capture/page.mdx b/docs/src/app/docs/audio-capture/page.mdx new file mode 100644 index 000000000..b9f3f5a2d --- /dev/null +++ b/docs/src/app/docs/audio-capture/page.mdx @@ -0,0 +1,174 @@ +# Audio Capture + +Native SDK exposes system audio and microphone input as a reliable, pull-based PCM stream on macOS 15 and later. System-only and combined capture use ScreenCaptureKit; microphone-only capture uses AVFoundation and does not require Screen Recording access. + +The SDK does not mix the sources or create a file. Combined capture returns separate, timestamp-aligned system and microphone bytes so the app can meter, transcribe, encode, mix, route, or store them itself. + +Check feature support before presenting capture UI: + +```zig +const can_system = runtime.supports(.system_audio_capture); +const can_microphone = runtime.supports(.microphone_capture); +const can_list_microphones = runtime.supports(.microphone_device_enumeration); +``` + + + + + + + + + + + +
HostSystem audioMicrophoneDevice enumeration
macOS 15+ system engineSupportedSupportedSupported
macOS 11–14 system engineUnsupportedUnsupportedUnsupported
macOS ChromiumUnsupportedUnsupportedUnsupported
Linux, Windows, iOS, AndroidUnsupportedUnsupportedUnsupported
+ +## Manifest and privacy strings + +Declare exactly the source permissions the app uses. Nonempty privacy strings are mandatory when their matching permission is present: + +```zig:app.zon +.{ + .permissions = .{ "microphone", "system_audio" }, + .privacy = .{ + .microphone_usage = "Capture the microphone selected for a meeting.", + .system_audio_usage = "Capture meeting audio played by other apps.", + }, +} +``` + +Packaging maps `microphone_usage` to `NSMicrophoneUsageDescription`. It maps `system_audio_usage` to both `NSScreenCaptureUsageDescription` and `NSAudioCaptureUsageDescription`. + +App Sandbox remains opt-in. A sandboxed app that captures microphone input must add `com.apple.security.device.audio-input` to its own entitlement file. Native SDK does not rewrite custom entitlements. + +## Permission flow + +Starting capture never opens a system prompt. Query or request access explicitly first: + +```ts +Cmd.audioCaptureAccess("mic-access", "microphone", "status", { event: "capture_access" }); +Cmd.audioCaptureAccess("system-access", "system_audio", "request", { event: "capture_access" }); +``` + +System audio reports the coarse ScreenCaptureKit truth: `authorized`, `not_authorized`, or `unavailable`. Microphone access additionally reports `not_determined`, `denied`, and `restricted`. A system-audio request can return `restartRequired: true`; tell the user to restart instead of assuming the running process can already capture. + +The manifest is an independent gate. Missing `system_audio` or `microphone` permission rejects a start even when macOS has granted the corresponding TCC access. + +## Stream contract + +Start options select at least one source and define the normalized output format: + +- `system_audio` / `systemAudio`: include system output. +- Microphone: `none`, `default`, or an enumerated device ID. +- Sample rate: 16, 24, 44.1, or 48 kHz; default 48 kHz. +- Channels: mono or stereo; default stereo. +- `exclude_current_process_audio` / `excludeCurrentProcessAudio`: default `true`. +- Buffer duration: 1–30 seconds; default 5 seconds. + +The runtime owns a bounded ring. A `readable` lifecycle event is a coalesced hint that at least one paired block is ready; it is not one notification per chunk. Read until `remainingFrames` reaches zero. Each read returns whole internal 20 ms blocks and must allow at least one block and at most 100 ms. At 48 kHz, `maxFrames: 4800` requests up to 100 ms. + +Each `chunk` contains: + +- `frameOffset` and `frames`, identifying one shared timeline interval. +- Interleaved signed 16-bit little-endian `systemPcm` and `microphonePcm` in the requested channel count. +- An empty byte slice for a source that was not enabled. +- `systemGapFrames` and `microphoneGapFrames`, counting timestamp frames the SDK had to zero-fill for that source. + +Gap frames are not the same as acoustic silence. Real silence arrives normally as zero-valued samples and leaves the gap counter at zero. A gap means the capture source supplied no sample buffer for part of the timeline, so the SDK inserted zeros to preserve alignment and counted those frame positions separately. The PCM is already padded; the counters are diagnostics, not instructions to insert more samples. + +Bytes are borrowed only for the Msg/update call. Consume them immediately or copy them before retaining them in the model. + +Reliability means accepted frames are never overwritten or silently dropped. The app owns backpressure by reading promptly. If the ring fills, capture seals with `failed` / `consumer_too_slow`; every previously accepted frame remains available to drain. There is no separate `mode` option. + +`stopAudioCapture` / `Cmd.audioCaptureStop` stops producers but retains buffered PCM. Continue reading until `endOfStream` is true; the final delivery releases the stream. `discardAudioCapture` / `Cmd.audioCaptureDiscard` stops and releases immediately. Stopping an idle key and discarding an unknown key are no-ops. + +Consumed chunks and runtime-generated readiness events participate in session journal/replay. Recording PCM-heavy sessions therefore grows the session journal in proportion to the audio the app reads. + +## Zig effects + +```zig +pub const Msg = union(enum) { + start_capture, + stop_capture, + capture: native_sdk.EffectAudioCapture, + capture_read: native_sdk.EffectAudioCaptureRead, +}; + +const App = native_sdk.UiApp(Model, Msg); +const Fx = App.Effects; + +fn read(fx: *Fx) void { + fx.readAudioCapture(.{ + .key = 1, + .max_frames = 4_800, + .on_read = Fx.audioCaptureReadMsg(.capture_read), + }); +} + +pub fn update(model: *Model, msg: Msg, fx: *Fx) void { + switch (msg) { + .start_capture => fx.startAudioCapture(.{ + .key = 1, + .system_audio = true, + .microphone = .default, + .sample_rate_hz = 48_000, + .channel_count = 2, + .buffer_duration_ms = 5_000, + .on_event = Fx.audioCaptureMsg(.capture), + }), + .stop_capture => fx.stopAudioCapture(), + .capture => |event| switch (event.state) { + .readable, .stopped, .failed => read(fx), + .started, .rejected => {}, + }, + .capture_read => |event| { + if (event.state == .chunk) { + consumeAlignedPcm(event.system_pcm, event.microphone_pcm, event.frames); + } + if (!event.end_of_stream and event.remaining_frames > 0) read(fx); + model.buffered_frames = event.remaining_frames; + }, + } +} +``` + +Keep at most one read outstanding. A second returns `rejected` / `read_in_progress`; a wrong or retired key returns `not_recording`. + +## TypeScript commands + +The TypeScript bytecode ABI uses fixed event records and byte slices: + +```ts +const start = Cmd.audioCaptureStart( + "meeting", + { + systemAudio: true, + microphone: "default", + sampleRate: 48000, + channels: 2, + excludeCurrentProcessAudio: true, + bufferDurationMs: 5000, + }, + { event: "capture_event" }, +); + +const read = Cmd.audioCaptureRead("meeting", 4800, { event: "capture_read" }); +const stop = Cmd.audioCaptureStop("meeting"); +const discard = Cmd.audioCaptureDiscard("meeting"); +``` + +The lifecycle arm contains `key`, `state`, `reason`, `sampleRate`, `channels`, `availableFrames`, `capacityFrames`, and `framesProduced`. The read arm contains `key`, `state`, `reason`, `sequence`, `frameOffset`, `frames`, `systemPcm`, `microphonePcm`, `systemGapFrames`, `microphoneGapFrames`, `remainingFrames`, and `endOfStream`. + +## Microphone devices + +```ts +const devices = Cmd.microphoneDevices("microphones", { event: "microphone_device" }); +const changes = Sub.microphoneDevicesChanged("microphones_changed"); +``` + +Listing emits zero or more `device` records with `id`, `name`, `isDefault`, `index`, and `total`, followed by exactly one `completed`, `failed`, or `rejected` record. The change subscription is invalidation only: list again to obtain a fresh snapshot. + +`default` resolves when capture starts and remains pinned for that session. A byte ID selects that exact `AVCaptureDevice.uniqueID`. A missing or disconnected selected device fails instead of silently switching. + +Only one capture can be active per app. A duplicate start is rejected with `already_recording` without disturbing the live stream. File containers, WAV headers, mixing policy, encoders, level smoothing, source gains, video capture, and per-application filtering belong in application code or follow-up extensions. A first-party file-writer extension is tracked separately in [vercel-labs/native#263](https://github.com/vercel-labs/native/issues/263). diff --git a/docs/src/app/docs/capabilities/page.mdx b/docs/src/app/docs/capabilities/page.mdx index f87f28f2c..5c36db6fd 100644 --- a/docs/src/app/docs/capabilities/page.mdx +++ b/docs/src/app/docs/capabilities/page.mdx @@ -73,6 +73,13 @@ Web content itself is declare-to-use: an app ships the embedded web layer only w None. No bridge surface. macOS system-engine host (an MTAudioProcessingTap on the app's single AVPlayer feeding a vDSP FFT — local files, cache entries, and streams alike), Windows system WebView (process-scoped WASAPI loopback capture of THIS app's audio session only — never other apps' audio — with an in-box FFT), and Linux system WebView (GStreamer's spectrum element as the playbin's audio-filter; requires gst-plugins-good, probed live); hosts that cannot analyze — macOS Chromium, pre-2004 Windows, spectrum-less GStreamer setups, iOS, Android — report audio_spectrum unsupported and simply never send the events: honest absence, never fabricated bands + + Audio capture + fx.startAudioCapture(options) / fx.readAudioCapture(options) / fx.stopAudioCapture() / fx.discardAudioCapture() / device and access effects + Cmd.audioCaptureStart / Cmd.audioCaptureRead / Cmd.audioCaptureStop / Cmd.audioCaptureDiscard / device and access commands / Sub.microphoneDevicesChanged + system_audio and/or microphone, matching the requested sources + macOS 15+ system-engine host: reliable paired s16le PCM; microphone-only through AVFoundation, system-audio-only or combined capture through ScreenCaptureKit. Older macOS, macOS Chromium, Linux, Windows, iOS, and Android report the three capture features unsupported. + File drops Event.files_dropped diff --git a/docs/src/app/docs/platform-support/page.mdx b/docs/src/app/docs/platform-support/page.mdx index 8d0b802a5..adb360b12 100644 --- a/docs/src/app/docs/platform-support/page.mdx +++ b/docs/src/app/docs/platform-support/page.mdx @@ -298,6 +298,14 @@ macOS, Linux, and Windows run full desktop apps through their own platform hosts Unsupported Supported on Windows 10 2004+ (process-scoped WASAPI loopback of this app only, probed live) + + System audio / microphone capture + Supported on macOS 15+ (ScreenCaptureKit + AVFoundation) + Unsupported + Unsupported + Unsupported + Unsupported + @@ -305,6 +313,8 @@ macOS, Linux, and Windows run full desktop apps through their own platform hosts Audio spectrum analysis reads the app's OWN playback and nothing else: macOS taps the app's single player pre-effects, Windows captures the app's own audio session through process-scoped loopback (never system-wide — other apps' audio can never appear in the bands), and Linux analyzes inside the app's own playbin. Hosts that cannot analyze report `audio_spectrum` unsupported and simply never deliver `.spectrum` events; see the capabilities page for the band shape and cadence contract. +Audio capture is intentionally narrower than playback: only the macOS 15+ system-engine host exposes `system_audio_capture`, `microphone_capture`, and `microphone_device_enumeration`. The feature flags stay false on older macOS and every Chromium, Linux, Windows, iOS, and Android host. See [Audio Capture](/docs/audio-capture) for permissions, source selection, device identity, and the reliable stream contract. + ## How Support Is Verified macOS is the primary development platform and carries the deepest support. The Linux and Windows columns are not aspirational: the repository carries reproducible live-verification loops that build every showcase app and drive it for real. `tools/linux-truth` runs the apps in real windows against the platform toolkit under Xvfb — clicks, keys, wheel input, multi-window flows, resize clamps, window-close paths, and both engine and X-server screenshots. `tools/windows-truth` drives the same scenarios on a real Windows desktop, plus clipboard round-trips, effect-stream cancellation, record/replay, and launching the packaged artifact. CI additionally runs headless Linux and Windows canvas smokes on every change. iOS is exercised on the simulator through the toolkit host and the embed library — the `native dev --target ios` loop launches real apps, and the input and layout verification scripts inject hardware-true touches and keyboard events; a fresh `native package --target ios` output is archived with `xcodebuild` as part of verification. Android is exercised on the emulator the same way: the `native dev --target android` loop assembles, installs, and launches real apps, input is injected over adb (taps, keys, soft-keyboard text), and a fresh `native package --target android` output assembles the debug APK; the embed ABI additionally cross-compiles for both Android architectures in CI. diff --git a/docs/src/app/docs/typescript/page.mdx b/docs/src/app/docs/typescript/page.mdx index 6c960e2cf..f05307580 100644 --- a/docs/src/app/docs/typescript/page.mdx +++ b/docs/src/app/docs/typescript/page.mdx @@ -294,6 +294,10 @@ The runtime interprets the command after the model commits and dispatches any re Cmd.audioPlay(key, source, { event }) + audioPause/audioResume/audioStop/audioSeek/audioSetVolume The audio player: one event stream (loaded, position, completed, failed, spectrum, ...) until audioStop closes it + + Cmd.audioCaptureStart / audioCaptureRead / audioCaptureStop / audioCaptureDiscard / microphone device and access commands + macOS 15+ reliable paired s16le PCM, microphone enumeration, and explicit permission status/request operations; see Audio Capture + Cmd.showWindow(label) / Cmd.quitApp() The menu-bar lifecycle verbs: un-hide + activate the labeled window (the tray "Open" consequence, the counterpart to close_policy = "hide"), and the real graceful terminate diff --git a/docs/src/lib/docs-navigation.ts b/docs/src/lib/docs-navigation.ts index 134a10d0d..e736b5c8a 100644 --- a/docs/src/lib/docs-navigation.ts +++ b/docs/src/lib/docs-navigation.ts @@ -51,6 +51,7 @@ const unprefixedNavSections: NavSection[] = [ title: "Native Platform", items: [ { name: "Windows", href: "/windows" }, + { name: "Audio Capture", href: "/audio-capture" }, { name: "Native Surfaces", href: "/native-surfaces" }, { name: "Menus", href: "/menus" }, { name: "Dialogs", href: "/dialogs" }, diff --git a/docs/src/lib/page-titles.ts b/docs/src/lib/page-titles.ts index 522b14a02..aff7c3864 100644 --- a/docs/src/lib/page-titles.ts +++ b/docs/src/lib/page-titles.ts @@ -22,6 +22,7 @@ export const PAGE_TITLES: Record = { "native-surfaces": "Native Surfaces", "media-producers": "Media Producers", windows: "Windows", + "audio-capture": "Audio Capture", webviews: "Multiple WebViews", "keyboard-shortcuts": "Keyboard Shortcuts", commands: "Commands", diff --git a/examples/README.md b/examples/README.md index c80969a84..4649ecc19 100644 --- a/examples/README.md +++ b/examples/README.md @@ -29,6 +29,8 @@ native build # produce a ReleaseFast binary in zig-out/bin/ | `gpu-components` | The retained GPU widget controls in one native-first component lab. | | `canvas-preview` | Canvas + WebView in one window, panes snapped to canvas anchors, a status item. | | `effects-probe` | The effect system live: spawn/fetch/file effects, cancellation, worker wakes. | +| `audio-capture` | macOS 15+ reliable paired system/microphone PCM through Zig effects. | +| `audio-capture-ts` | The same pull-stream, permission, and device-enumeration flow from a TypeScript core. | | `menu-bar` | The menu-bar app lifecycle: `close_policy = "hide"`, a status item whose Open/Quit rows drive `fx.showWindow`/`fx.quitApp`, Dock reopen. | ## Examples that own their build diff --git a/examples/audio-capture-ts/README.md b/examples/audio-capture-ts/README.md new file mode 100644 index 000000000..9ed1acd4c --- /dev/null +++ b/examples/audio-capture-ts/README.md @@ -0,0 +1,9 @@ +# Audio Capture (TypeScript) + +The TypeScript counterpart to `examples/audio-capture`. It demonstrates fixed lifecycle/read/device/access Msg records, `Sub.microphoneDevicesChanged`, coalesced readiness, paired PCM reads, stop-and-drain, and discard. + +```sh +native dev +``` + +The SDK does not create a file. `systemPcm` and `microphonePcm` are borrowed signed 16-bit little-endian bytes covering the same frame interval; this example consumes their lengths and gap counters during `update`. diff --git a/examples/audio-capture-ts/app.zon b/examples/audio-capture-ts/app.zon new file mode 100644 index 000000000..a34a5575e --- /dev/null +++ b/examples/audio-capture-ts/app.zon @@ -0,0 +1,36 @@ +.{ + .id = "dev.native_sdk.audio_capture_ts", + .name = "audio-capture-ts", + .display_name = "Audio Capture TS", + .description = "Consume aligned macOS system-audio and microphone PCM from a TypeScript core.", + .version = "0.1.0", + .platforms = .{"macos"}, + .permissions = .{ "view", "microphone", "system_audio" }, + .privacy = .{ + .microphone_usage = "Record the microphone selected in the TypeScript audio capture example.", + .system_audio_usage = "Record system audio in the TypeScript audio capture example.", + }, + .capabilities = .{ "native_views", "gpu_surfaces" }, + .shell = .{ + .windows = .{ + .{ + .label = "main", + .title = "Native SDK Audio Capture TS", + .width = 660, + .height = 390, + .restore_state = false, + .restore_policy = "center_on_primary", + .views = .{ + .{ .label = "capture-canvas", .kind = "gpu_surface", .fill = true, .role = "Audio capture canvas", .accessibility_label = "Audio capture TypeScript example", .gpu_backend = "metal", .gpu_pixel_format = "bgra8_unorm", .gpu_present_mode = "timer", .gpu_alpha_mode = "opaque", .gpu_color_space = "srgb", .gpu_vsync = true }, + }, + }, + }, + }, + .security = .{ + .navigation = .{ + .allowed_origins = .{ "zero://app", "zero://inline" }, + .external_links = .{ .action = "deny" }, + }, + }, + .web_engine = "system", +} diff --git a/examples/audio-capture-ts/package.json b/examples/audio-capture-ts/package.json new file mode 100644 index 000000000..a4b980856 --- /dev/null +++ b/examples/audio-capture-ts/package.json @@ -0,0 +1,7 @@ +{ + "name": "audio-capture-ts", + "private": true, + "dependencies": { + "@native-sdk/core": "0.7.1" + } +} diff --git a/examples/audio-capture-ts/src/app.native b/examples/audio-capture-ts/src/app.native new file mode 100644 index 000000000..7073f065a --- /dev/null +++ b/examples/audio-capture-ts/src/app.native @@ -0,0 +1,29 @@ + + Reliable audio stream + macOS 15+ · aligned s16le system/microphone chunks · 5 second buffer + + + + + + Microphone: {microphoneAccess} + System audio: {systemAccess} + {deviceCount} connected microphone(s) + + + + + + + + + Status: {captureState} · {captureReason} + {availableFrames}/{capacityFrames} buffered frames · {framesConsumed} consumed + Last chunk: {lastSystemBytes} system bytes · {lastMicrophoneBytes} mic bytes + Inserted gaps: {systemGapFrames} system · {microphoneGapFrames} mic + Format: {sampleRate} Hz · {channels} channel(s) + + + + The app consumes borrowed PCM immediately; no file is created by the SDK. + diff --git a/examples/audio-capture-ts/src/core.ts b/examples/audio-capture-ts/src/core.ts new file mode 100644 index 000000000..16dfbc149 --- /dev/null +++ b/examples/audio-capture-ts/src/core.ts @@ -0,0 +1,125 @@ +import { Cmd, Sub, type Cmd as Command, type Sub as Subscription } from "@native-sdk/core"; + +export type CaptureState = "started" | "readable" | "stopped" | "failed" | "rejected"; +export type CaptureReason = + | "none" | "invalid_options" | "permission_missing" | "permission_required" + | "already_recording" | "device_not_found" | "device_disconnected" + | "capture_failed" | "no_audio" | "consumer_too_slow" | "discarded" | "unsupported"; +export type ReadState = "chunk" | "empty" | "ended" | "rejected"; +export type ReadReason = "none" | "invalid_options" | "not_recording" | "read_in_progress"; +export type AccessStatus = "authorized" | "not_authorized" | "not_determined" | "denied" | "restricted" | "unavailable"; +export type AccessSource = "system_audio" | "microphone"; +export type DeviceState = "device" | "completed" | "failed" | "rejected"; + +export interface Model { + readonly captureState: CaptureState; + readonly captureReason: CaptureReason; + readonly sampleRate: number; + readonly channels: number; + readonly availableFrames: number; + readonly capacityFrames: number; + readonly framesConsumed: number; + readonly systemGapFrames: number; + readonly microphoneGapFrames: number; + readonly lastSystemBytes: number; + readonly lastMicrophoneBytes: number; + readonly readPending: boolean; + readonly terminalSeen: boolean; + readonly deviceCount: number; + readonly microphoneAccess: AccessStatus; + readonly systemAccess: AccessStatus; + readonly restartRequired: boolean; +} + +export type Msg = + | { readonly kind: "start_capture" } + | { readonly kind: "stop_capture" } + | { readonly kind: "discard_capture" } + | { readonly kind: "list_microphones" } + | { readonly kind: "request_microphone" } + | { readonly kind: "request_system_audio" } + | { readonly kind: "microphones_changed" } + | { readonly kind: "capture_event"; readonly key: string; readonly state: CaptureState; readonly reason: CaptureReason; readonly sampleRate: number; readonly channels: number; readonly availableFrames: number; readonly capacityFrames: number; readonly framesProduced: number } + | { readonly kind: "capture_read"; readonly key: string; readonly state: ReadState; readonly reason: ReadReason; readonly sequence: number; readonly frameOffset: number; readonly frames: number; readonly systemPcm: Uint8Array; readonly microphonePcm: Uint8Array; readonly systemGapFrames: number; readonly microphoneGapFrames: number; readonly remainingFrames: number; readonly endOfStream: boolean } + | { readonly kind: "microphone_device"; readonly key: string; readonly state: DeviceState; readonly id: Uint8Array; readonly name: Uint8Array; readonly isDefault: boolean; readonly index: number; readonly total: number } + | { readonly kind: "capture_access"; readonly key: string; readonly source: AccessSource; readonly status: AccessStatus; readonly restartRequired: boolean }; + +export const viewUnbound = ["microphones_changed", "capture_event", "capture_read", "microphone_device", "capture_access", "readPending", "terminalSeen", "restartRequired"] as const; + +export function initialModel(): Model { + return { + captureState: "stopped", captureReason: "none", sampleRate: 0, channels: 0, + availableFrames: 0, capacityFrames: 0, framesConsumed: 0, + systemGapFrames: 0, microphoneGapFrames: 0, + lastSystemBytes: 0, lastMicrophoneBytes: 0, + readPending: false, terminalSeen: false, deviceCount: 0, + microphoneAccess: "not_determined", systemAccess: "not_authorized", restartRequired: false, + }; +} + +export function update(model: Model, msg: Msg): [Model, Command] { + switch (msg.kind) { + case "start_capture": + return [{ + ...model, captureState: "started", captureReason: "none", sampleRate: 0, channels: 0, + availableFrames: 0, capacityFrames: 0, framesConsumed: 0, + systemGapFrames: 0, microphoneGapFrames: 0, + lastSystemBytes: 0, lastMicrophoneBytes: 0, readPending: false, terminalSeen: false, + }, Cmd.audioCaptureStart("meeting", { + systemAudio: true, + microphone: "default", + sampleRate: 48000, + channels: 2, + excludeCurrentProcessAudio: true, + bufferDurationMs: 5000, + }, { event: "capture_event" })]; + case "stop_capture": + return [model, Cmd.audioCaptureStop("meeting")]; + case "discard_capture": + return [{ ...model, captureState: "stopped", terminalSeen: true, readPending: false }, Cmd.audioCaptureDiscard("meeting")]; + case "list_microphones": + case "microphones_changed": + return [{ ...model, deviceCount: 0 }, Cmd.microphoneDevices("microphones", { event: "microphone_device" })]; + case "request_microphone": + return [model, Cmd.audioCaptureAccess("mic-access", "microphone", "request", { event: "capture_access" })]; + case "request_system_audio": + return [model, Cmd.audioCaptureAccess("system-access", "system_audio", "request", { event: "capture_access" })]; + case "capture_event": { + const terminal = msg.state === "stopped" || msg.state === "failed"; + const next: Model = { + ...model, captureState: msg.state, captureReason: msg.reason, + sampleRate: msg.sampleRate, channels: msg.channels, + availableFrames: msg.availableFrames, capacityFrames: msg.capacityFrames, + terminalSeen: model.terminalSeen || terminal || msg.state === "rejected", + }; + if ((msg.state === "readable" || terminal) && !next.readPending) { + return [{ ...next, readPending: true }, Cmd.audioCaptureRead("meeting", 4800, { event: "capture_read" })]; + } + return [next, Cmd.none]; + } + case "capture_read": { + const next: Model = { + ...model, readPending: false, availableFrames: msg.remainingFrames, + framesConsumed: model.framesConsumed + msg.frames, + systemGapFrames: model.systemGapFrames + msg.systemGapFrames, + microphoneGapFrames: model.microphoneGapFrames + msg.microphoneGapFrames, + lastSystemBytes: msg.systemPcm.length, + lastMicrophoneBytes: msg.microphonePcm.length, + }; + if (!msg.endOfStream && (msg.remainingFrames > 0 || next.terminalSeen)) { + return [{ ...next, readPending: true }, Cmd.audioCaptureRead("meeting", 4800, { event: "capture_read" })]; + } + return [next, Cmd.none]; + } + case "microphone_device": + if (msg.state === "device") return [{ ...model, deviceCount: model.deviceCount + 1 }, Cmd.none]; + return [model, Cmd.none]; + case "capture_access": + if (msg.source === "microphone") return [{ ...model, microphoneAccess: msg.status, restartRequired: msg.restartRequired }, Cmd.none]; + return [{ ...model, systemAccess: msg.status, restartRequired: msg.restartRequired }, Cmd.none]; + } +} + +export function subscriptions(_model: Model): Subscription { + return Sub.microphoneDevicesChanged("microphones_changed"); +} diff --git a/examples/audio-capture-ts/tsconfig.json b/examples/audio-capture-ts/tsconfig.json new file mode 100644 index 000000000..27b52cd12 --- /dev/null +++ b/examples/audio-capture-ts/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "strict": true, + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "noEmit": true + }, + "include": ["src/**/*.ts"] +} diff --git a/examples/audio-capture/README.md b/examples/audio-capture/README.md new file mode 100644 index 000000000..87088a4f5 --- /dev/null +++ b/examples/audio-capture/README.md @@ -0,0 +1,11 @@ +# Audio Capture (Zig) + +A macOS 15+ app demonstrating the reliable pull stream: explicit permission requests, microphone enumeration, system-only/microphone-only/combined capture, coalesced readiness, paired PCM reads, simple real-time peaks, gap counters, stop-and-drain, and discard. + +```sh +native dev +``` + +Starting capture never prompts. Use the permission buttons first and follow any macOS restart instruction reported by the app. + +The SDK does not create a file. Each read borrows aligned signed 16-bit little-endian system and microphone bytes during `update`; this example consumes them immediately. An application can instead copy them into its own encoder, transcription pipeline, meter, or network transport. diff --git a/examples/audio-capture/app.zon b/examples/audio-capture/app.zon new file mode 100644 index 000000000..826a889a7 --- /dev/null +++ b/examples/audio-capture/app.zon @@ -0,0 +1,36 @@ +.{ + .id = "dev.native_sdk.audio_capture", + .name = "audio-capture", + .display_name = "Audio Capture", + .description = "Consume aligned macOS system-audio and microphone PCM streams.", + .version = "0.1.0", + .platforms = .{"macos"}, + .permissions = .{ "view", "microphone", "system_audio" }, + .privacy = .{ + .microphone_usage = "Record the microphone selected in the audio capture example.", + .system_audio_usage = "Record system audio in the audio capture example.", + }, + .capabilities = .{ "native_views", "gpu_surfaces" }, + .shell = .{ + .windows = .{ + .{ + .label = "main", + .title = "Native SDK Audio Capture", + .width = 680, + .height = 430, + .restore_state = false, + .restore_policy = "center_on_primary", + .views = .{ + .{ .label = "capture-canvas", .kind = "gpu_surface", .fill = true, .role = "Audio capture canvas", .accessibility_label = "Audio capture", .gpu_backend = "metal", .gpu_pixel_format = "bgra8_unorm", .gpu_present_mode = "timer", .gpu_alpha_mode = "opaque", .gpu_color_space = "srgb", .gpu_vsync = true }, + }, + }, + }, + }, + .security = .{ + .navigation = .{ + .allowed_origins = .{ "zero://app", "zero://inline" }, + .external_links = .{ .action = "deny" }, + }, + }, + .web_engine = "system", +} diff --git a/examples/audio-capture/src/main.zig b/examples/audio-capture/src/main.zig new file mode 100644 index 000000000..229a3cec7 --- /dev/null +++ b/examples/audio-capture/src/main.zig @@ -0,0 +1,251 @@ +//! macOS 15+ reliable paired system-audio and microphone capture. + +const std = @import("std"); +const runner = @import("runner"); +const native_sdk = @import("native_sdk"); + +pub const panic = std.debug.FullPanic(native_sdk.debug.capturePanic); + +const canvas = native_sdk.canvas; +const geometry = native_sdk.geometry; +const canvas_label = "capture-canvas"; +const window_width: f32 = 720; +const window_height: f32 = 460; + +const shell_views = [_]native_sdk.ShellView{ + .{ .label = canvas_label, .kind = .gpu_surface, .fill = true, .role = "Audio capture canvas", .accessibility_label = "Audio capture", .gpu_backend = .metal, .gpu_pixel_format = .bgra8_unorm, .gpu_present_mode = .timer, .gpu_alpha_mode = .@"opaque", .gpu_color_space = .srgb, .gpu_vsync = true }, +}; +const shell_windows = [_]native_sdk.ShellWindow{.{ + .label = "main", + .title = "Native SDK Audio Capture", + .width = window_width, + .height = window_height, + .restore_state = false, + .views = &shell_views, +}}; +const shell_scene: native_sdk.ShellConfig = .{ .windows = &shell_windows }; + +const CaptureStatus = enum { idle, starting, recording, draining, stopped, failed, rejected }; + +pub const Model = struct { + status: CaptureStatus = .idle, + reason: native_sdk.EffectAudioCaptureReason = .none, + sample_rate_hz: u32 = 0, + channel_count: u8 = 0, + available_frames: u32 = 0, + capacity_frames: u32 = 0, + frames_consumed: u64 = 0, + system_gap_frames: u64 = 0, + microphone_gap_frames: u64 = 0, + system_peak: u16 = 0, + microphone_peak: u16 = 0, + read_pending: bool = false, + terminal_seen: bool = false, + microphone_count: u32 = 0, + microphone_access: native_sdk.EffectAudioCaptureAccessStatus = .not_determined, + system_access: native_sdk.EffectAudioCaptureAccessStatus = .not_authorized, + restart_required: bool = false, + + pub fn statusText(model: *const Model, arena: std.mem.Allocator) []const u8 { + return std.fmt.allocPrint(arena, "{s} · {s} · {d}/{d} buffered frames · {d} consumed", .{ + @tagName(model.status), @tagName(model.reason), model.available_frames, + model.capacity_frames, model.frames_consumed, + }) catch "audio capture"; + } + + pub fn levelsText(model: *const Model, arena: std.mem.Allocator) []const u8 { + return std.fmt.allocPrint(arena, "system peak={d} · mic peak={d} · inserted gaps {d}/{d}", .{ + model.system_peak, model.microphone_peak, + model.system_gap_frames, model.microphone_gap_frames, + }) catch "levels unavailable"; + } +}; + +pub const Msg = union(enum) { + request_microphone, + request_system_audio, + list_microphones, + start_system, + start_microphone, + start_combined, + stop, + discard, + capture: native_sdk.EffectAudioCapture, + capture_read: native_sdk.EffectAudioCaptureRead, + microphone: native_sdk.EffectMicrophoneDevice, + access: native_sdk.EffectAudioCaptureAccess, + microphones_changed, +}; + +const CaptureApp = native_sdk.UiApp(Model, Msg); +pub const Effects = CaptureApp.Effects; + +pub fn boot(_: *Model, fx: *Effects) void { + fx.observeMicrophoneDevices(Effects.microphoneDevicesChangedMsg(.microphones_changed)); + fx.audioCaptureAccess(.{ .key = 1, .source = .microphone, .on_event = Effects.audioCaptureAccessMsg(.access) }); + fx.audioCaptureAccess(.{ .key = 2, .source = .system_audio, .on_event = Effects.audioCaptureAccessMsg(.access) }); +} + +fn requestRead(model: *Model, fx: *Effects) void { + if (model.read_pending) return; + model.read_pending = true; + fx.readAudioCapture(.{ .key = 10, .max_frames = 4_800, .on_read = Effects.audioCaptureReadMsg(.capture_read) }); +} + +fn peak(pcm: []const u8) u16 { + var result: u16 = 0; + var index: usize = 0; + while (index + 1 < pcm.len) : (index += 2) { + const sample = std.mem.readInt(i16, pcm[index..][0..2], .little); + const magnitude: u16 = if (sample == std.math.minInt(i16)) 32_768 else @intCast(@abs(sample)); + result = @max(result, magnitude); + } + return result; +} + +pub fn update(model: *Model, msg: Msg, fx: *Effects) void { + switch (msg) { + .request_microphone => fx.audioCaptureAccess(.{ .key = 3, .source = .microphone, .action = .request, .on_event = Effects.audioCaptureAccessMsg(.access) }), + .request_system_audio => fx.audioCaptureAccess(.{ .key = 4, .source = .system_audio, .action = .request, .on_event = Effects.audioCaptureAccessMsg(.access) }), + .list_microphones, .microphones_changed => { + model.microphone_count = 0; + fx.listMicrophoneDevices(.{ .key = 5, .on_event = Effects.microphoneDeviceMsg(.microphone) }); + }, + .start_system => start(model, fx, .system), + .start_microphone => start(model, fx, .microphone), + .start_combined => start(model, fx, .combined), + .stop => { + model.status = .draining; + fx.stopAudioCapture(); + }, + .discard => { + model.read_pending = false; + model.terminal_seen = true; + model.status = .stopped; + fx.discardAudioCapture(); + }, + .capture => |event| { + model.reason = event.reason; + model.sample_rate_hz = event.sample_rate_hz; + model.channel_count = event.channel_count; + model.available_frames = event.available_frames; + model.capacity_frames = event.capacity_frames; + switch (event.state) { + .started => model.status = .recording, + .readable => requestRead(model, fx), + .stopped => { + model.status = .draining; + model.terminal_seen = true; + requestRead(model, fx); + }, + .failed => { + model.status = .failed; + model.terminal_seen = true; + requestRead(model, fx); + }, + .rejected => { + model.status = .rejected; + model.terminal_seen = true; + }, + } + }, + .capture_read => |event| { + model.read_pending = false; + model.available_frames = event.remaining_frames; + if (event.state == .chunk) { + model.frames_consumed += event.frames; + model.system_gap_frames += event.system_gap_frames; + model.microphone_gap_frames += event.microphone_gap_frames; + model.system_peak = peak(event.system_pcm); + model.microphone_peak = peak(event.microphone_pcm); + } + if (event.end_of_stream or event.state == .ended) { + if (model.status == .draining) model.status = .stopped; + } else if (event.remaining_frames > 0 or model.terminal_seen) { + requestRead(model, fx); + } + }, + .microphone => |event| if (event.state == .device) { + model.microphone_count += 1; + }, + .access => |event| { + switch (event.source) { + .microphone => model.microphone_access = event.status, + .system_audio => model.system_access = event.status, + } + model.restart_required = event.restart_required; + }, + } +} + +const Source = enum { system, microphone, combined }; + +fn start(model: *Model, fx: *Effects, source: Source) void { + model.* = .{ + .status = .starting, + .microphone_count = model.microphone_count, + .microphone_access = model.microphone_access, + .system_access = model.system_access, + .restart_required = model.restart_required, + }; + fx.startAudioCapture(.{ + .key = 10, + .system_audio = source != .microphone, + .microphone = if (source == .system) .none else .default, + .buffer_duration_ms = 5_000, + .on_event = Effects.audioCaptureMsg(.capture), + }); +} + +pub const CaptureUi = canvas.Ui(Msg); + +pub fn view(ui: *CaptureUi, model: *const Model) CaptureUi.Node { + return ui.column(.{ .padding = 20, .gap = 14, .style_tokens = .{ .background = .background } }, .{ + ui.text(.{ .size = .lg }, "Reliable audio stream"), + ui.text(.{ .style_tokens = .{ .foreground = .text_muted } }, "macOS 15+ · aligned s16le system/microphone chunks · 5 second buffer"), + ui.row(.{ .gap = 8 }, .{ + ui.button(.{ .on_press = .request_microphone }, "Request microphone"), + ui.button(.{ .on_press = .request_system_audio }, "Request system audio"), + ui.button(.{ .on_press = .list_microphones }, "List microphones"), + }), + ui.text(.{}, ui.fmt("microphone={s} · system={s} · restart={any}", .{ @tagName(model.microphone_access), @tagName(model.system_access), model.restart_required })), + ui.text(.{}, ui.fmt("{d} connected microphone(s)", .{model.microphone_count})), + ui.separator(.{}), + ui.row(.{ .gap = 8 }, .{ + ui.button(.{ .variant = .primary, .on_press = .start_combined }, "System + default mic"), + ui.button(.{ .on_press = .start_system }, "System only"), + ui.button(.{ .on_press = .start_microphone }, "Mic only"), + ui.button(.{ .variant = .destructive, .on_press = .stop }, "Stop + drain"), + ui.button(.{ .on_press = .discard }, "Discard"), + }), + ui.panel(.{ .padding = 14, .style_tokens = .{ .background = .surface, .radius = .md } }, .{ + ui.text(.{}, model.statusText(ui.arena)), + ui.text(.{}, model.levelsText(ui.arena)), + }), + ui.spacer(1), + ui.statusBar(.{}, "The app consumes borrowed PCM immediately; no file is created by the SDK."), + }); +} + +pub fn main(init: std.process.Init) !void { + const app_state = try std.heap.page_allocator.create(CaptureApp); + defer std.heap.page_allocator.destroy(app_state); + app_state.* = CaptureApp.init(std.heap.page_allocator, .{}, .{ + .name = "audio-capture", + .scene = shell_scene, + .canvas_label = canvas_label, + .init_fx = boot, + .update_fx = update, + .view = view, + }); + defer app_state.deinit(); + try runner.runWithOptions(app_state.app(), .{ + .app_name = "audio-capture", + .window_title = "Native SDK Audio Capture", + .bundle_id = "dev.native_sdk.audio_capture", + .default_frame = geometry.RectF.init(0, 0, window_width, window_height), + .restore_state = false, + .js_window_api = false, + .security = .{ .navigation = .{ .allowed_origins = &.{ "zero://inline", "zero://app" } } }, + }, init); +} diff --git a/examples/browser/build.zig b/examples/browser/build.zig index fa794db1c..02924f151 100644 --- a/examples/browser/build.zig +++ b/examples/browser/build.zig @@ -180,6 +180,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -207,6 +208,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/capabilities/build.zig b/examples/capabilities/build.zig index 42dd10f6b..f5c4173c0 100644 --- a/examples/capabilities/build.zig +++ b/examples/capabilities/build.zig @@ -198,6 +198,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -227,6 +228,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/capabilities/src/main.zig b/examples/capabilities/src/main.zig index 9cf50d226..410b9ac99 100644 --- a/examples/capabilities/src/main.zig +++ b/examples/capabilities/src/main.zig @@ -110,7 +110,7 @@ const CapabilitiesApp = struct { }, else => {}, }, - .appearance_changed, .command, .shortcut, .timer, .effects_wake, .audio, .video, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance => {}, + .appearance_changed, .command, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance => {}, } } }; diff --git a/examples/command-app/build.zig b/examples/command-app/build.zig index 78d86f5f2..770affef6 100644 --- a/examples/command-app/build.zig +++ b/examples/command-app/build.zig @@ -195,6 +195,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -224,6 +225,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/command-app/src/main.zig b/examples/command-app/src/main.zig index 2b670e674..cfaf41450 100644 --- a/examples/command-app/src/main.zig +++ b/examples/command-app/src/main.zig @@ -179,7 +179,7 @@ const CommandApp = struct { try self.handleCommand(runtime, command); } }, - .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, + .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, } } diff --git a/examples/gpu-components/src/app.zig b/examples/gpu-components/src/app.zig index 72463419b..ce1e958de 100644 --- a/examples/gpu-components/src/app.zig +++ b/examples/gpu-components/src/app.zig @@ -190,7 +190,7 @@ pub const GpuComponentsApp = struct { .canvas_widget_scroll => |scroll_event| try self.handleWidgetScroll(runtime, scroll_event), .canvas_widget_dismiss => |dismiss_event| try self.handleWidgetDismiss(runtime, dismiss_event), .appearance_changed => |appearance| try self.applySystemAppearance(runtime, appearance), - .gpu_surface_resized, .gpu_surface_input, .shortcut, .timer, .effects_wake, .audio, .video, .files_dropped, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, + .gpu_surface_resized, .gpu_surface_input, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .files_dropped, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, } } diff --git a/examples/gpu-surface/src/main.zig b/examples/gpu-surface/src/main.zig index 90d892b88..59d0a8bbe 100644 --- a/examples/gpu-surface/src/main.zig +++ b/examples/gpu-surface/src/main.zig @@ -150,7 +150,7 @@ const GpuSurfaceApp = struct { self.gpu_input_count += 1; } }, - .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .video, .files_dropped, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, + .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .files_dropped, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, } } diff --git a/examples/hello/build.zig b/examples/hello/build.zig index eb616fa01..932352f32 100644 --- a/examples/hello/build.zig +++ b/examples/hello/build.zig @@ -195,6 +195,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -224,6 +225,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/native-panels/build.zig b/examples/native-panels/build.zig index 9ecaff009..356dc1517 100644 --- a/examples/native-panels/build.zig +++ b/examples/native-panels/build.zig @@ -195,6 +195,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -224,6 +225,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/native-panels/src/main.zig b/examples/native-panels/src/main.zig index bd7a5bac6..7ee0c1c8f 100644 --- a/examples/native-panels/src/main.zig +++ b/examples/native-panels/src/main.zig @@ -172,7 +172,7 @@ const NativePanelsApp = struct { try self.apply(runtime, command); } }, - .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, + .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, } } diff --git a/examples/native-shell/build.zig b/examples/native-shell/build.zig index 35d71ae98..3564a9095 100644 --- a/examples/native-shell/build.zig +++ b/examples/native-shell/build.zig @@ -195,6 +195,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -224,6 +225,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/native-shell/src/main.zig b/examples/native-shell/src/main.zig index cd29a16fe..ddc6727aa 100644 --- a/examples/native-shell/src/main.zig +++ b/examples/native-shell/src/main.zig @@ -200,7 +200,7 @@ const NativeShellApp = struct { try self.closePreview(runtime); } }, - .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, + .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, } } diff --git a/examples/next/build.zig b/examples/next/build.zig index e67e91068..83af42b8e 100644 --- a/examples/next/build.zig +++ b/examples/next/build.zig @@ -243,6 +243,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -272,6 +273,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/react/build.zig b/examples/react/build.zig index 089a02d67..87a06a334 100644 --- a/examples/react/build.zig +++ b/examples/react/build.zig @@ -243,6 +243,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -272,6 +273,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/svelte/build.zig b/examples/svelte/build.zig index 5afd1d7fd..cdbbdb879 100644 --- a/examples/svelte/build.zig +++ b/examples/svelte/build.zig @@ -243,6 +243,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -272,6 +273,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/vue/build.zig b/examples/vue/build.zig index e60f08db8..2d63418ff 100644 --- a/examples/vue/build.zig +++ b/examples/vue/build.zig @@ -243,6 +243,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -272,6 +273,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/webview/build.zig b/examples/webview/build.zig index eabef87eb..e1ad4f14a 100644 --- a/examples/webview/build.zig +++ b/examples/webview/build.zig @@ -195,6 +195,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -224,6 +225,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/packages/core/rt/rt.zig b/packages/core/rt/rt.zig index e121fa251..d00d4b16a 100644 --- a/packages/core/rt/rt.zig +++ b/packages/core/rt/rt.zig @@ -1410,7 +1410,7 @@ pub fn Kernel(comptime opts: Options) type { // copies) them after the model commit and BEFORE frameReset — the same // boundary the committed model crosses. - pub const cmd_format_version: u32 = 3; + pub const cmd_format_version: u32 = 5; /// An encoded command value: op records per the layout above. pub const Cmd = []const u8; @@ -1444,6 +1444,12 @@ pub fn Kernel(comptime opts: Options) type { pty_write = 0x1A, pty_resize = 0x1B, pty_kill = 0x1C, + audio_capture_start = 0x1D, + audio_capture_stop = 0x1E, + microphone_devices = 0x1F, + audio_capture_access = 0x20, + audio_capture_read = 0x21, + audio_capture_discard = 0x22, }; /// The spawn record's "no line routing" sentinel: a `line_tag` of @@ -1736,6 +1742,76 @@ pub fn Kernel(comptime opts: Options) type { return out; } + pub fn cmdAudioCaptureStart(key: []const u8, event_tag: u8, system_audio: bool, microphone_kind: u8, microphone_id: []const u8, sample_rate: u32, channels: u8, exclude_current_process_audio: bool, buffer_duration_ms: u32) Cmd { + std.debug.assert(key.len <= 255); + std.debug.assert(microphone_id.len <= std.math.maxInt(u32)); + const out = frameAlloc(u8, 2 + key.len + 1 + 1 + 1 + 4 + 1 + 4 + 4 + microphone_id.len); + out[0] = @intFromEnum(CmdOp.audio_capture_start); + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + var off: usize = 2 + key.len; + out[off] = event_tag; + out[off + 1] = @as(u8, @intFromBool(system_audio)) | (@as(u8, @intFromBool(exclude_current_process_audio)) << 1); + out[off + 2] = microphone_kind; + std.mem.writeInt(u32, out[off + 3 ..][0..4], sample_rate, .little); + out[off + 7] = channels; + std.mem.writeInt(u32, out[off + 8 ..][0..4], buffer_duration_ms, .little); + off += 12; + _ = writeLongBytes(out, off, microphone_id); + return out; + } + + pub fn cmdAudioCaptureStop(key: []const u8) Cmd { + std.debug.assert(key.len <= 255); + const out = frameAlloc(u8, 2 + key.len); + out[0] = @intFromEnum(CmdOp.audio_capture_stop); + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + return out; + } + + pub fn cmdAudioCaptureRead(key: []const u8, event_tag: u8, max_frames: u32) Cmd { + std.debug.assert(key.len <= 255); + const out = frameAlloc(u8, 3 + key.len + 4); + out[0] = @intFromEnum(CmdOp.audio_capture_read); + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + out[2 + key.len] = event_tag; + std.mem.writeInt(u32, out[3 + key.len ..][0..4], max_frames, .little); + return out; + } + + pub fn cmdAudioCaptureDiscard(key: []const u8) Cmd { + std.debug.assert(key.len <= 255); + const out = frameAlloc(u8, 2 + key.len); + out[0] = @intFromEnum(CmdOp.audio_capture_discard); + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + return out; + } + + pub fn cmdMicrophoneDevices(key: []const u8, event_tag: u8) Cmd { + std.debug.assert(key.len <= 255); + const out = frameAlloc(u8, 3 + key.len); + out[0] = @intFromEnum(CmdOp.microphone_devices); + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + out[2 + key.len] = event_tag; + return out; + } + + pub fn cmdAudioCaptureAccess(key: []const u8, event_tag: u8, source: u8, action: u8) Cmd { + std.debug.assert(key.len <= 255); + const out = frameAlloc(u8, 5 + key.len); + out[0] = @intFromEnum(CmdOp.audio_capture_access); + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + out[2 + key.len] = event_tag; + out[3 + key.len] = source; + out[4 + key.len] = action; + return out; + } + pub fn cmdWindowShow(label: []const u8) Cmd { // The emitter's byte gate on the literal label is the // build-time teaching; this is the loud runtime backstop. @@ -1954,7 +2030,7 @@ pub fn Kernel(comptime opts: Options) type { /// An encoded subscription set: records per the layout above. pub const Sub = []const u8; - pub const SubOp = enum(u8) { timer = 0x01 }; + pub const SubOp = enum(u8) { timer = 0x01, microphone_devices_changed = 0x02 }; pub const sub_none: Sub = &.{}; @@ -1969,6 +2045,13 @@ pub fn Kernel(comptime opts: Options) type { return out; } + pub fn subMicrophoneDevicesChanged(msg_tag: u8) Sub { + const out = frameAlloc(u8, 2); + out[0] = @intFromEnum(SubOp.microphone_devices_changed); + out[1] = msg_tag; + return out; + } + pub fn subBatch(subs: []const Sub) Sub { return cmdBatch(subs); } diff --git a/packages/core/sdk/core.ts b/packages/core/sdk/core.ts index 52bbd3342..19423a0a1 100644 --- a/packages/core/sdk/core.ts +++ b/packages/core/sdk/core.ts @@ -387,6 +387,98 @@ export type AudioEventKind = M extends Msgish : never : never; +export type AudioCaptureState = "started" | "readable" | "stopped" | "failed" | "rejected"; +export type AudioCaptureReason = + | "none" | "invalid_options" | "permission_missing" | "permission_required" + | "already_recording" | "device_not_found" | "device_disconnected" + | "capture_failed" | "no_audio" | "consumer_too_slow" | "discarded" | "unsupported"; +export type AudioCaptureEventArm = { + readonly key: string; + readonly state: AudioCaptureState; + readonly reason: AudioCaptureReason; + readonly sampleRate: number; + readonly channels: number; + readonly availableFrames: number; + readonly capacityFrames: number; + readonly framesProduced: number; +}; +export type AudioCaptureEventKind = M extends Msgish + ? [Exclude] extends [keyof AudioCaptureEventArm] + ? [keyof AudioCaptureEventArm] extends [Exclude] + ? M extends Msgish & AudioCaptureEventArm + ? [AudioCaptureState] extends [M["state"]] + ? [AudioCaptureReason] extends [M["reason"]] ? M["kind"] : never + : never + : never + : never + : never + : never; + +export type AudioCaptureReadState = "chunk" | "empty" | "ended" | "rejected"; +export type AudioCaptureReadReason = "none" | "invalid_options" | "not_recording" | "read_in_progress"; +export type AudioCaptureReadEventArm = { + readonly key: string; + readonly state: AudioCaptureReadState; + readonly reason: AudioCaptureReadReason; + readonly sequence: number; + readonly frameOffset: number; + readonly frames: number; + readonly systemPcm: Uint8Array; + readonly microphonePcm: Uint8Array; + readonly systemGapFrames: number; + readonly microphoneGapFrames: number; + readonly remainingFrames: number; + readonly endOfStream: boolean; +}; +export type AudioCaptureReadEventKind = M extends Msgish + ? [Exclude] extends [keyof AudioCaptureReadEventArm] + ? [keyof AudioCaptureReadEventArm] extends [Exclude] + ? M extends Msgish & AudioCaptureReadEventArm ? M["kind"] : never + : never + : never + : never; + +export type MicrophoneDeviceState = "device" | "completed" | "failed" | "rejected"; +export type MicrophoneDeviceEventArm = { + readonly key: string; + readonly state: MicrophoneDeviceState; + readonly id: Uint8Array; + readonly name: Uint8Array; + readonly isDefault: boolean; + readonly index: number; + readonly total: number; +}; +export type MicrophoneDeviceEventKind = M extends Msgish + ? [Exclude] extends [keyof MicrophoneDeviceEventArm] + ? [keyof MicrophoneDeviceEventArm] extends [Exclude] + ? M extends Msgish & MicrophoneDeviceEventArm + ? [MicrophoneDeviceState] extends [M["state"]] ? M["kind"] : never + : never + : never + : never + : never; + +export type AudioCaptureAccessSource = "system_audio" | "microphone"; +export type AudioCaptureAccessAction = "status" | "request"; +export type AudioCaptureAccessStatus = "authorized" | "not_authorized" | "not_determined" | "denied" | "restricted" | "unavailable"; +export type AudioCaptureAccessEventArm = { + readonly key: string; + readonly source: AudioCaptureAccessSource; + readonly status: AudioCaptureAccessStatus; + readonly restartRequired: boolean; +}; +export type AudioCaptureAccessEventKind = M extends Msgish + ? [Exclude] extends [keyof AudioCaptureAccessEventArm] + ? [keyof AudioCaptureAccessEventArm] extends [Exclude] + ? M extends Msgish & AudioCaptureAccessEventArm + ? [AudioCaptureAccessSource] extends [M["source"]] + ? [AudioCaptureAccessStatus] extends [M["status"]] ? M["kind"] : never + : never + : never + : never + : never + : never; + /// The video event states, the audio vocabulary without spectrum: `loaded` /// acknowledges a successful load with the player's duration estimate and /// the stream's decoded pixel dimensions; `position` ticks at the @@ -704,6 +796,22 @@ export interface AudioRoute { readonly event: AudioEventKind; } +export type MicrophoneSelection = "none" | "default" | Uint8Array; +/** Sources and normalized PCM format for one reliable capture stream. + * `bufferDurationMs` defaults to 5000 and accepts 1000..30000. */ +export interface AudioCaptureOptions { + readonly systemAudio?: boolean; + readonly microphone?: MicrophoneSelection; + readonly sampleRate?: 16000 | 24000 | 44100 | 48000; + readonly channels?: 1 | 2; + readonly excludeCurrentProcessAudio?: boolean; + readonly bufferDurationMs?: number; +} +export interface AudioCaptureRoute { readonly event: AudioCaptureEventKind; } +export interface AudioCaptureReadRoute { readonly event: AudioCaptureReadEventKind; } +export interface MicrophoneDevicesRoute { readonly event: MicrophoneDeviceEventKind; } +export interface AudioCaptureAccessRoute { readonly event: AudioCaptureAccessEventKind; } + /// A `Cmd.videoLoad` source. `surface` is the model-owned media-surface id /// the markup binds — the texture channel the decoded frames feed. The /// local `path` is tried first; a missing file falls through to `url` @@ -844,6 +952,23 @@ export type Cmd = /// Seek position (ms) / volume (0..1); 0 for the value-less verbs. readonly value: number; } + | { + readonly op: "audio_capture_start"; + readonly key: string; + readonly eventKind: string; + readonly options: AudioCaptureOptions; + } + | { readonly op: "audio_capture_stop"; readonly key: string } + | { readonly op: "audio_capture_read"; readonly key: string; readonly eventKind: string; readonly maxFrames: number } + | { readonly op: "audio_capture_discard"; readonly key: string } + | { readonly op: "microphone_devices"; readonly key: string; readonly eventKind: string } + | { + readonly op: "audio_capture_access"; + readonly key: string; + readonly eventKind: string; + readonly source: AudioCaptureAccessSource; + readonly action: AudioCaptureAccessAction; + } | { readonly op: "video_load"; readonly key: string; @@ -1119,6 +1244,39 @@ export const Cmd = { return { op: "audio_ctl", key, verb: "volume", value: volume }; }, + /** Start the app's single capture stream. This never prompts; request + * access explicitly first. Lifecycle/readability events use `route`. */ + audioCaptureStart(key: string, options: AudioCaptureOptions, route: AudioCaptureRoute): Cmd { + return { op: "audio_capture_start", key, eventKind: route.event, options }; + }, + + /** Stop producers while retaining every accepted frame for draining. */ + audioCaptureStop(key: string): Cmd { + return { op: "audio_capture_stop", key }; + }, + + /** Pull whole aligned blocks, up to 100 ms. PCM byte slices in the + * resulting message are borrowed until update returns. */ + audioCaptureRead(key: string, maxFrames: number, route: AudioCaptureReadRoute): Cmd { + return { op: "audio_capture_read", key, eventKind: route.event, maxFrames }; + }, + + /** Stop and release a capture immediately, discarding buffered PCM. */ + audioCaptureDiscard(key: string): Cmd { + return { op: "audio_capture_discard", key }; + }, + + /** Stream a microphone snapshot as device records plus one terminal. */ + microphoneDevices(key: string, route: MicrophoneDevicesRoute): Cmd { + return { op: "microphone_devices", key, eventKind: route.event }; + }, + + /** Read or explicitly request one TCC permission. Capture start itself + * never opens a system prompt. */ + audioCaptureAccess(key: string, source: AudioCaptureAccessSource, action: AudioCaptureAccessAction, route: AudioCaptureAccessRoute): Cmd { + return { op: "audio_capture_access", key, eventKind: route.event, source, action }; + }, + /// Open (or replace — one player is the whole surface) the keyed video /// event stream: claim the media-surface the source names, resolve the /// source cascade (local path, then url) and start playback (autoplay, @@ -1375,6 +1533,7 @@ export const Cmd = { export type Sub = | { readonly op: "none" } | { readonly op: "timer"; readonly key: string; readonly everyMs: number; readonly msgKind: string } + | { readonly op: "microphone_devices_changed"; readonly msgKind: string } | { readonly op: "batch"; readonly subs: readonly Sub[] }; export const Sub = { @@ -1388,6 +1547,11 @@ export const Sub = { return { op: "timer", key, everyMs, msgKind }; }, + /** Invalidation only: issue `Cmd.microphoneDevices` for a fresh list. */ + microphoneDevicesChanged(msgKind: EmptyKind): Sub { + return { op: "microphone_devices_changed", msgKind }; + }, + /// Several subscriptions at once. batch(subs: readonly Sub[]): Sub { return { op: "batch", subs }; diff --git a/packages/core/sdk/events.ts b/packages/core/sdk/events.ts index 078240897..c20ae5770 100644 --- a/packages/core/sdk/events.ts +++ b/packages/core/sdk/events.ts @@ -181,3 +181,56 @@ export interface AudioEvent { readonly buffering: boolean; readonly bands: Uint8Array; } + +export type AudioCaptureState = "started" | "readable" | "stopped" | "failed" | "rejected"; +export type AudioCaptureReason = "none" | "invalid_options" | "permission_missing" | "permission_required" | "already_recording" | "device_not_found" | "device_disconnected" | "capture_failed" | "no_audio" | "consumer_too_slow" | "discarded" | "unsupported"; +export interface AudioCaptureEvent { + readonly key: string; + readonly state: AudioCaptureState; + readonly reason: AudioCaptureReason; + readonly sampleRate: number; + readonly channels: number; + readonly availableFrames: number; + readonly capacityFrames: number; + readonly framesProduced: number; +} + +export type AudioCaptureReadState = "chunk" | "empty" | "ended" | "rejected"; +export type AudioCaptureReadReason = "none" | "invalid_options" | "not_recording" | "read_in_progress"; +export interface AudioCaptureReadEvent { + readonly key: string; + readonly state: AudioCaptureReadState; + readonly reason: AudioCaptureReadReason; + readonly sequence: number; + readonly frameOffset: number; + readonly frames: number; + /** Borrowed interleaved signed 16-bit little-endian bytes. Consume or + * copy them before update returns. Empty when the source is disabled. */ + readonly systemPcm: Uint8Array; + /** Same frame interval and lifetime as `systemPcm`. */ + readonly microphonePcm: Uint8Array; + readonly systemGapFrames: number; + readonly microphoneGapFrames: number; + readonly remainingFrames: number; + readonly endOfStream: boolean; +} + +export type MicrophoneDeviceState = "device" | "completed" | "failed" | "rejected"; +export interface MicrophoneDeviceEvent { + readonly key: string; + readonly state: MicrophoneDeviceState; + readonly id: Uint8Array; + readonly name: Uint8Array; + readonly isDefault: boolean; + readonly index: number; + readonly total: number; +} + +export type AudioCaptureAccessSource = "system_audio" | "microphone"; +export type AudioCaptureAccessStatus = "authorized" | "not_authorized" | "not_determined" | "denied" | "restricted" | "unavailable"; +export interface AudioCaptureAccessEvent { + readonly key: string; + readonly source: AudioCaptureAccessSource; + readonly status: AudioCaptureAccessStatus; + readonly restartRequired: boolean; +} diff --git a/packages/core/src/emitter.ts b/packages/core/src/emitter.ts index fce7f3210..9f2ba4e14 100644 --- a/packages/core/src/emitter.ts +++ b/packages/core/src/emitter.ts @@ -14,7 +14,7 @@ import path from "node:path"; import { ts, TypedAst, hasExportModifier, exportListBindings, sdkCoreModulePath } from "./typed_ast.ts"; -import { TypeTable, snakeCase, zigDeclName, zigLocalName, isZigPrimitiveName, isStaticMember, mangleZType, type ZType, type ZField, type UnionInfo, type StructInfo, type ClassInfo } from "./types.ts"; +import { TypeTable, snakeCase, zigDeclName, zigLocalName, isZigPrimitiveName, isStaticMember, mangleZType, type ZType, type ZField, type UnionArm, type UnionInfo, type StructInfo, type ClassInfo } from "./types.ts"; import { IntInference, returnExpressionsOf } from "./infer.ts"; import { thrownShapeOf, thrownArmsOfShape, THROWN_UNION_NAME, type CheckResult } from "./checker.ts"; import type { RuleId } from "./diagnostics.ts"; @@ -232,6 +232,13 @@ const FETCH_METHODS = ["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD"]; /// engine's event vocabulary, matched by member NAME (declaration order is /// the app's own). const AUDIO_STATES = ["loaded", "position", "completed", "failed", "rejected", "spectrum"]; +const AUDIO_CAPTURE_STATES = ["started", "readable", "stopped", "failed", "rejected"]; +const AUDIO_CAPTURE_REASONS = ["none", "invalid_options", "permission_missing", "permission_required", "already_recording", "device_not_found", "device_disconnected", "capture_failed", "no_audio", "consumer_too_slow", "discarded", "unsupported"]; +const AUDIO_CAPTURE_READ_STATES = ["chunk", "empty", "ended", "rejected"]; +const AUDIO_CAPTURE_READ_REASONS = ["none", "invalid_options", "not_recording", "read_in_progress"]; +const MICROPHONE_DEVICE_STATES = ["device", "completed", "failed", "rejected"]; +const AUDIO_CAPTURE_ACCESS_SOURCES = ["system_audio", "microphone"]; +const AUDIO_CAPTURE_ACCESS_STATUSES = ["authorized", "not_authorized", "not_determined", "denied", "restricted", "unavailable"]; /// The video event states an event arm's `state` union must carry — the /// engine's event vocabulary, matched by member NAME (declaration order is @@ -2456,6 +2463,18 @@ export class Emitter { if (method === "audioPlay") { return this.emitAudioPlayCmd(e, ctx); } + if (method === "audioCaptureStart") return this.emitAudioCaptureStartCmd(e, ctx); + if (method === "audioCaptureStop") { + const key = this.literalEffectKey(e, e.arguments[0], "Cmd.audioCaptureStop"); + return `rt.cmdAudioCaptureStop("${escapeZigString(key)}")`; + } + if (method === "audioCaptureRead") return this.emitAudioCaptureReadCmd(e, ctx); + if (method === "audioCaptureDiscard") { + const key = this.literalEffectKey(e, e.arguments[0], "Cmd.audioCaptureDiscard"); + return `rt.cmdAudioCaptureDiscard("${escapeZigString(key)}")`; + } + if (method === "microphoneDevices") return this.emitMicrophoneDevicesCmd(e, ctx); + if (method === "audioCaptureAccess") return this.emitAudioCaptureAccessCmd(e, ctx); if (method === "imageLoad") { return this.emitImageLoadCmd(e, ctx); } @@ -2570,7 +2589,7 @@ export class Emitter { } this.fail( e, - `Cmd.${method} (the v3 command set is none, persist, now, host, request, cancel, readFile, writeFile, fetch, clipboardWrite, clipboardRead, delay, spawn, audioPlay, audioPause, audioResume, audioStop, audioSeek, audioSetVolume, videoLoad, videoPlay, videoPause, videoStop, videoSeek, videoSetVolume, videoSetMuted, videoSetLoop, showWindow, quitApp, imageLoad, imageCancel, imageUnregister, channelOpen, channelClose, ptySpawn, ptyWrite, ptyResize, ptyKill, batch)`, + `Cmd.${method} (the v5 command set is none, persist, now, host, request, cancel, readFile, writeFile, fetch, clipboardWrite, clipboardRead, delay, spawn, audioPlay, audioPause, audioResume, audioStop, audioSeek, audioSetVolume, videoLoad, videoPlay, videoPause, videoStop, videoSeek, videoSetVolume, videoSetMuted, videoSetLoop, showWindow, quitApp, imageLoad, imageCancel, imageUnregister, channelOpen, channelClose, ptySpawn, ptyWrite, ptyResize, ptyKill, audioCaptureStart, audioCaptureRead, audioCaptureStop, audioCaptureDiscard, microphoneDevices, audioCaptureAccess, batch)`, ); } this.fail(expr, "command expression (Cmd values are built inline from the Cmd.* factories)"); @@ -2847,6 +2866,160 @@ export class Emitter { return `rt.cmdAudioPlay("${escapeZigString(keyArg.text)}", ${tag}, ${audio_path}, ${url}, ${cache_path}, ${expected})`; } + private literalEffectKey(call: ts.CallExpression, arg: ts.Expression | undefined, factory: string): string { + if (!arg || !ts.isStringLiteral(arg)) this.fail(call, `\`${factory}\` takes its key as a string literal`, "NS1027"); + if (utf8ByteLength(arg.text) > 255) this.fail(arg, `${factory} key over 255 bytes`); + return arg.text; + } + + private eventRouteArm(call: ts.CallExpression, arg: ts.Expression | undefined, factory: string): ts.StringLiteral { + let route = arg; + while (route && (ts.isParenthesizedExpression(route) || ts.isAsExpression(route) || ts.isSatisfiesExpression(route))) route = route.expression; + if (!route || !ts.isObjectLiteralExpression(route)) this.fail(arg ?? call, `\`${factory}\` routing is an inline \`{ event }\` object`, "NS1027"); + let event: ts.StringLiteral | null = null; + for (const p of route.properties) { + if (!ts.isPropertyAssignment(p) || !ts.isIdentifier(p.name) || p.name.text !== "event") this.fail(p, `\`${factory}\` routing accepts only \`event\``, "NS1027"); + let value: ts.Expression = p.initializer; + while (ts.isParenthesizedExpression(value) || ts.isAsExpression(value) || ts.isSatisfiesExpression(value)) value = value.expression; + if (!ts.isStringLiteral(value)) this.fail(value, `\`${factory}\` event arm is not a string literal`, "NS1027"); + event = value; + } + if (!event) this.fail(route, `\`${factory}\` routing without an \`event\` arm`, "NS1027"); + return event; + } + + private emitAudioCaptureStartCmd(e: ts.CallExpression, ctx: Ctx): string { + const key = this.literalEffectKey(e, e.arguments[0], "Cmd.audioCaptureStart"); + let options = e.arguments[1]; + while (options && (ts.isParenthesizedExpression(options) || ts.isAsExpression(options) || ts.isSatisfiesExpression(options))) options = options.expression; + if (!options || !ts.isObjectLiteralExpression(options)) this.fail(e.arguments[1] ?? e, `\`Cmd.audioCaptureStart\` options must be an inline object`, "NS1029"); + let systemAudio = false; + let microphoneKind = 0; + let microphoneID = '""'; + let sampleRate = 48000; + let channels = 2; + let exclude = true; + let bufferDurationMs = 5000; + for (const p of options.properties) { + if (!ts.isPropertyAssignment(p) || !ts.isIdentifier(p.name)) this.fail(p, `invalid Cmd.audioCaptureStart option`, "NS1029"); + const name = p.name.text; + let value: ts.Expression = p.initializer; + while (ts.isParenthesizedExpression(value) || ts.isAsExpression(value) || ts.isSatisfiesExpression(value)) value = value.expression; + if (name === "systemAudio" || name === "excludeCurrentProcessAudio") { + if (value.kind !== ts.SyntaxKind.TrueKeyword && value.kind !== ts.SyntaxKind.FalseKeyword) this.fail(value, `Cmd.audioCaptureStart ${name} must be a boolean literal`, "NS1030"); + const on = value.kind === ts.SyntaxKind.TrueKeyword; + if (name === "systemAudio") systemAudio = on; else exclude = on; + } else if (name === "microphone") { + if (ts.isStringLiteral(value)) { + if (value.text === "none") microphoneKind = 0; + else if (value.text === "default") microphoneKind = 1; + else this.fail(value, `Cmd.audioCaptureStart microphone string must be "none" or "default"`, "NS1030"); + } else { + microphoneKind = 2; + microphoneID = this.effectBytesArg(e, p.initializer, "Cmd.audioCaptureStart microphone device id", 512, ctx); + } + } else if (name === "sampleRate") { + const literal = this.numberLiteralValue(value); + if (literal === null || ![16000, 24000, 44100, 48000].includes(literal)) this.fail(value, `Cmd.audioCaptureStart sampleRate must be 16000, 24000, 44100, or 48000`, "NS1030"); + sampleRate = literal; + } else if (name === "channels") { + const literal = this.numberLiteralValue(value); + if (literal !== 1 && literal !== 2) this.fail(value, `Cmd.audioCaptureStart channels must be 1 or 2`, "NS1030"); + channels = literal; + } else if (name === "bufferDurationMs") { + const literal = this.numberLiteralValue(value); + if (literal === null || !Number.isInteger(literal) || literal < 1000 || literal > 30000) this.fail(value, `Cmd.audioCaptureStart bufferDurationMs must be an integer from 1000 through 30000`, "NS1030"); + bufferDurationMs = literal; + } else this.fail(p, `unknown Cmd.audioCaptureStart option \`${name}\``, "NS1029"); + } + if (!systemAudio && microphoneKind === 0) this.fail(options, `Cmd.audioCaptureStart must enable systemAudio and/or microphone`, "NS1030"); + const event = this.eventRouteArm(e, e.arguments[2], "Cmd.audioCaptureStart"); + const tag = this.audioCaptureEventArmTag(event, ctx); + return `rt.cmdAudioCaptureStart("${escapeZigString(key)}", ${tag}, ${systemAudio}, ${microphoneKind}, ${microphoneID}, ${sampleRate}, ${channels}, ${exclude}, ${bufferDurationMs})`; + } + + private emitAudioCaptureReadCmd(e: ts.CallExpression, ctx: Ctx): string { + const key = this.literalEffectKey(e, e.arguments[0], "Cmd.audioCaptureRead"); + const maxFramesArg = e.arguments[1]; + if (!maxFramesArg) this.fail(e, `Cmd.audioCaptureRead requires maxFrames`, "NS1030"); + const maxFrames = this.numberLiteralValue(maxFramesArg); + if (maxFrames === null || !Number.isInteger(maxFrames) || maxFrames < 1 || maxFrames > 4800) this.fail(maxFramesArg, `Cmd.audioCaptureRead maxFrames must be an integer from 1 through 4800`, "NS1030"); + const event = this.eventRouteArm(e, e.arguments[2], "Cmd.audioCaptureRead"); + return `rt.cmdAudioCaptureRead("${escapeZigString(key)}", ${this.audioCaptureReadEventArmTag(event, ctx)}, ${maxFrames})`; + } + + private emitMicrophoneDevicesCmd(e: ts.CallExpression, ctx: Ctx): string { + const key = this.literalEffectKey(e, e.arguments[0], "Cmd.microphoneDevices"); + const event = this.eventRouteArm(e, e.arguments[1], "Cmd.microphoneDevices"); + return `rt.cmdMicrophoneDevices("${escapeZigString(key)}", ${this.microphoneDeviceEventArmTag(event, ctx)})`; + } + + private emitAudioCaptureAccessCmd(e: ts.CallExpression, ctx: Ctx): string { + const key = this.literalEffectKey(e, e.arguments[0], "Cmd.audioCaptureAccess"); + const source = e.arguments[1]; + const action = e.arguments[2]; + if (!source || !ts.isStringLiteral(source) || !AUDIO_CAPTURE_ACCESS_SOURCES.includes(source.text)) this.fail(source ?? e, `Cmd.audioCaptureAccess source must be "system_audio" or "microphone"`, "NS1030"); + if (!action || !ts.isStringLiteral(action) || (action.text !== "status" && action.text !== "request")) this.fail(action ?? e, `Cmd.audioCaptureAccess action must be "status" or "request"`, "NS1030"); + const event = this.eventRouteArm(e, e.arguments[3], "Cmd.audioCaptureAccess"); + return `rt.cmdAudioCaptureAccess("${escapeZigString(key)}", ${this.audioCaptureAccessEventArmTag(event, ctx)}, ${source.text === "microphone" ? 1 : 0}, ${action.text === "request" ? 1 : 0})`; + } + + private fixedEventArm(arg: ts.StringLiteral, ctx: Ctx): { arm: UnionArm; unionName: string } { + const unionName = ctx.cmdReturn!.msgUnion; + const info = this.table.unions.get(unionName); + if (!info) this.fail(arg, `unknown union ${unionName}`); + const arm = info.arms.find((candidate) => candidate.tag === arg.text); + if (!arm) this.fail(arg, `routing target \`${arg.text}\` is not an arm of ${unionName}`, "NS1027"); + return { arm, unionName }; + } + + private enumFieldMatches(field: ZField | undefined, members: readonly string[]): boolean { + return field !== undefined && field.type.k === "enum" && field.type.members.length === members.length && members.every((member) => field.type.k === "enum" && field.type.members.includes(member)); + } + + private audioCaptureEventArmTag(arg: ts.StringLiteral, ctx: Ctx): string { + const { arm, unionName } = this.fixedEventArm(arg, ctx); + const fields = new Map(arm.fields.map((f) => [f.tsName, f])); + const number = (name: string) => ["number", "i64", "f64"].includes(fields.get(name)?.type.k ?? ""); + const ok = arm.fields.length === 8 && fields.get("key")?.type.k === "string" && + this.enumFieldMatches(fields.get("state"), AUDIO_CAPTURE_STATES) && this.enumFieldMatches(fields.get("reason"), AUDIO_CAPTURE_REASONS) && + number("sampleRate") && number("channels") && number("availableFrames") && number("capacityFrames") && number("framesProduced"); + if (!ok) this.fail(arg, `audio capture event arm must carry key, state, reason, sampleRate, channels, availableFrames, capacityFrames, and framesProduced`, "NS1027"); + return `@intFromEnum(std.meta.Tag(${unionName}).${zigId(arg.text)})`; + } + + private audioCaptureReadEventArmTag(arg: ts.StringLiteral, ctx: Ctx): string { + const { arm, unionName } = this.fixedEventArm(arg, ctx); + const fields = new Map(arm.fields.map((f) => [f.tsName, f])); + const number = (name: string) => ["number", "i64", "f64"].includes(fields.get(name)?.type.k ?? ""); + const ok = arm.fields.length === 12 && fields.get("key")?.type.k === "string" && + this.enumFieldMatches(fields.get("state"), AUDIO_CAPTURE_READ_STATES) && this.enumFieldMatches(fields.get("reason"), AUDIO_CAPTURE_READ_REASONS) && + number("sequence") && number("frameOffset") && number("frames") && fields.get("systemPcm")?.type.k === "bytes" && + fields.get("microphonePcm")?.type.k === "bytes" && number("systemGapFrames") && number("microphoneGapFrames") && + number("remainingFrames") && fields.get("endOfStream")?.type.k === "bool"; + if (!ok) this.fail(arg, `audio capture read event arm must carry key, state, reason, sequence, frameOffset, frames, systemPcm, microphonePcm, systemGapFrames, microphoneGapFrames, remainingFrames, and endOfStream`, "NS1027"); + return `@intFromEnum(std.meta.Tag(${unionName}).${zigId(arg.text)})`; + } + + private microphoneDeviceEventArmTag(arg: ts.StringLiteral, ctx: Ctx): string { + const { arm, unionName } = this.fixedEventArm(arg, ctx); + const fields = new Map(arm.fields.map((f) => [f.tsName, f])); + const number = (name: string) => ["number", "i64", "f64"].includes(fields.get(name)?.type.k ?? ""); + const ok = arm.fields.length === 7 && fields.get("key")?.type.k === "string" && this.enumFieldMatches(fields.get("state"), MICROPHONE_DEVICE_STATES) && + fields.get("id")?.type.k === "bytes" && fields.get("name")?.type.k === "bytes" && fields.get("isDefault")?.type.k === "bool" && number("index") && number("total"); + if (!ok) this.fail(arg, `microphone device event arm must carry key, state, id, name, isDefault, index, and total`, "NS1027"); + return `@intFromEnum(std.meta.Tag(${unionName}).${zigId(arg.text)})`; + } + + private audioCaptureAccessEventArmTag(arg: ts.StringLiteral, ctx: Ctx): string { + const { arm, unionName } = this.fixedEventArm(arg, ctx); + const fields = new Map(arm.fields.map((f) => [f.tsName, f])); + const ok = arm.fields.length === 4 && fields.get("key")?.type.k === "string" && this.enumFieldMatches(fields.get("source"), AUDIO_CAPTURE_ACCESS_SOURCES) && + this.enumFieldMatches(fields.get("status"), AUDIO_CAPTURE_ACCESS_STATUSES) && fields.get("restartRequired")?.type.k === "bool"; + if (!ok) this.fail(arg, `audio capture access event arm must carry key, source, status, and restartRequired`, "NS1027"); + return `@intFromEnum(std.meta.Tag(${unionName}).${zigId(arg.text)})`; + } + /// `Cmd.imageLoad(id, source, route)`: the app's numeric ImageId (any /// number expression — ids are model data), an inline /// `{ path?, url?, cachePath?, expectedBytes? }` source (at least one of @@ -3864,6 +4037,14 @@ export class Emitter { } return `rt.subTimer("${escapeZigString(keyArg.text)}", ${every}, @intFromEnum(std.meta.Tag(${unionName}).${zigId(kindArg.text)}))`; } + if (method === "microphoneDevicesChanged") { + const kindArg = e.arguments[0]; + if (!kindArg || !ts.isStringLiteral(kindArg)) this.fail(e, `Sub.microphoneDevicesChanged takes its target Msg kind as a string literal`, "NS1027"); + const unionName = ctx.subReturn!.msgUnion; + const arm = this.table.unions.get(unionName)?.arms.find((candidate) => candidate.tag === kindArg.text); + if (!arm || arm.fields.length !== 0) this.fail(kindArg, `microphone device change target must carry no payload fields`, "NS1027"); + return `rt.subMicrophoneDevicesChanged(@intFromEnum(std.meta.Tag(${unionName}).${zigId(kindArg.text)}))`; + } if (method === "batch") { const arr = e.arguments[0]; if (!arr || !ts.isArrayLiteralExpression(arr)) this.fail(e, "Sub.batch argument (an array literal)"); @@ -3871,7 +4052,7 @@ export class Emitter { if (parts.length === 0) return "rt.sub_none"; return `rt.subBatch(&.{ ${parts.join(", ")} })`; } - this.fail(e, `Sub.${method} (the subscription set is none, timer, batch)`); + this.fail(e, `Sub.${method} (the subscription set is none, timer, microphoneDevicesChanged, batch)`); } this.fail(expr, "subscription expression (Sub values are built inline from the Sub.* factories)"); } diff --git a/packages/core/test/effects.test.ts b/packages/core/test/effects.test.ts index aff6b8561..3761ec440 100644 --- a/packages/core/test/effects.test.ts +++ b/packages/core/test/effects.test.ts @@ -72,9 +72,9 @@ fn dispatch(msg: core.Msg) []const u8 { } test "cmd bytes flow through update -> commit -> effect log" { - // v3 is additive: the v1/v2 op records asserted below are + // v4 is additive: the v1/v2/v3 op records asserted below are // byte-identical under the bumped version. - try std.testing.expectEqual(@as(u32, 3), rt.cmd_format_version); + try std.testing.expectEqual(@as(u32, 5), rt.cmd_format_version); rt.resetAll(); g_model = core.commitModelRoot(core.initialModel()); @@ -237,7 +237,7 @@ fn expectRecord(bytes: []const u8, expected: []const u8) !void { } test "v2 wire: init command, payloads, routing, cancel, subscriptions" { - try std.testing.expectEqual(@as(u32, 3), rt.cmd_format_version); + try std.testing.expectEqual(@as(u32, 5), rt.cmd_format_version); var log: [512]u8 = undefined; @@ -409,7 +409,7 @@ fn appendLong(list: *std.ArrayList(u8), a: std.mem.Allocator, bytes: []const u8) } test "named-op wire records match the documented additive layout" { - try std.testing.expectEqual(@as(u32, 3), rt.cmd_format_version); + try std.testing.expectEqual(@as(u32, 5), rt.cmd_format_version); const a = std.testing.allocator; var log: [512]u8 = undefined; @@ -529,6 +529,138 @@ test("v2 effects: wire bytes through the real dispatch cycle", { skip: !hasZig, } }); +// ---------------------------------------------------------------- audio capture v5 + +const coreAudioCapture = ` +import { Cmd, Sub, asciiBytes } from "@native-sdk/core"; + +type CaptureState = "started" | "readable" | "stopped" | "failed" | "rejected"; +type CaptureReason = "none" | "invalid_options" | "permission_missing" | "permission_required" | "already_recording" | "device_not_found" | "device_disconnected" | "capture_failed" | "no_audio" | "consumer_too_slow" | "discarded" | "unsupported"; +type ReadState = "chunk" | "empty" | "ended" | "rejected"; +type ReadReason = "none" | "invalid_options" | "not_recording" | "read_in_progress"; +type DeviceState = "device" | "completed" | "failed" | "rejected"; +type AccessSource = "system_audio" | "microphone"; +type AccessStatus = "authorized" | "not_authorized" | "not_determined" | "denied" | "restricted" | "unavailable"; + +export interface Model { readonly events: number; } +export type Msg = + | { readonly kind: "start" } + | { readonly kind: "read" } + | { readonly kind: "stop" } + | { readonly kind: "discard" } + | { readonly kind: "list" } + | { readonly kind: "access" } + | { readonly kind: "capture_event"; readonly key: string; readonly state: CaptureState; readonly reason: CaptureReason; readonly sampleRate: number; readonly channels: number; readonly availableFrames: number; readonly capacityFrames: number; readonly framesProduced: number } + | { readonly kind: "read_event"; readonly key: string; readonly state: ReadState; readonly reason: ReadReason; readonly sequence: number; readonly frameOffset: number; readonly frames: number; readonly systemPcm: Uint8Array; readonly microphonePcm: Uint8Array; readonly systemGapFrames: number; readonly microphoneGapFrames: number; readonly remainingFrames: number; readonly endOfStream: boolean } + | { readonly kind: "device_event"; readonly key: string; readonly state: DeviceState; readonly id: Uint8Array; readonly name: Uint8Array; readonly isDefault: boolean; readonly index: number; readonly total: number } + | { readonly kind: "access_event"; readonly key: string; readonly source: AccessSource; readonly status: AccessStatus; readonly restartRequired: boolean } + | { readonly kind: "devices_changed" }; + +export function initialModel(): Model { return { events: 0 }; } + +export function update(model: Model, msg: Msg): Model | [Model, Cmd] { + switch (msg.kind) { + case "start": return [model, Cmd.audioCaptureStart("capture", { systemAudio: true, microphone: asciiBytes("usb"), sampleRate: 44100, channels: 1, bufferDurationMs: 3000 }, { event: "capture_event" })]; + case "read": return [model, Cmd.audioCaptureRead("capture", 882, { event: "read_event" })]; + case "stop": return [model, Cmd.audioCaptureStop("capture")]; + case "discard": return [model, Cmd.audioCaptureDiscard("capture")]; + case "list": return [model, Cmd.microphoneDevices("list", { event: "device_event" })]; + case "access": return [model, Cmd.audioCaptureAccess("access", "microphone", "request", { event: "access_event" })]; + case "capture_event": case "read_event": case "device_event": case "access_event": case "devices_changed": return { events: model.events + 1 }; + } +} + +export function subscriptions(model: Model): Sub { + return Sub.microphoneDevicesChanged("devices_changed"); +} +`; + +const harnessAudioCapture = ` +const std = @import("std"); +const core = @import("core.zig"); +const rt = core.rt; + +var g_model: *const core.Model = undefined; + +fn dispatch(msg: core.Msg, log: []u8) []const u8 { + const r = core.update(g_model, msg); + g_model = core.commitModelRoot(r.model); + @memcpy(log[0..r.cmd.len], r.cmd); + const out = log[0..r.cmd.len]; + rt.frameReset(); + return out; +} + +fn expectLong(bytes: []const u8, at: *usize, expected: []const u8) !void { + const len = std.mem.readInt(u32, bytes[at.*..][0..4], .little); + try std.testing.expectEqual(@as(u32, @intCast(expected.len)), len); + try std.testing.expectEqualStrings(expected, bytes[at.* + 4 ..][0..expected.len]); + at.* += 4 + expected.len; +} + +test "audio capture command and subscription wire records" { + try std.testing.expectEqual(@as(u32, 5), rt.cmd_format_version); + var log: [512]u8 = undefined; + rt.resetAll(); + g_model = core.commitModelRoot(core.initialModel()); + rt.frameReset(); + + const start = dispatch(.start, &log); + try std.testing.expectEqual(@as(u8, 0x1D), start[0]); + try std.testing.expectEqual(@as(u8, 7), start[1]); + try std.testing.expectEqualStrings("capture", start[2..9]); + try std.testing.expectEqual(@as(u8, @intFromEnum(std.meta.Tag(core.Msg).capture_event)), start[9]); + try std.testing.expectEqual(@as(u8, 3), start[10]); // system audio + exclude current process + try std.testing.expectEqual(@as(u8, 2), start[11]); // explicit microphone id + try std.testing.expectEqual(@as(u32, 44100), std.mem.readInt(u32, start[12..16], .little)); + try std.testing.expectEqual(@as(u8, 1), start[16]); + try std.testing.expectEqual(@as(u32, 3000), std.mem.readInt(u32, start[17..21], .little)); + var at: usize = 21; + try expectLong(start, &at, "usb"); + try std.testing.expectEqual(start.len, at); + + const read = dispatch(.read, &log); + try std.testing.expectEqual(@as(u8, 0x21), read[0]); + try std.testing.expectEqual(@as(u8, 7), read[1]); + try std.testing.expectEqualStrings("capture", read[2..9]); + try std.testing.expectEqual(@as(u8, @intFromEnum(std.meta.Tag(core.Msg).read_event)), read[9]); + try std.testing.expectEqual(@as(u32, 882), std.mem.readInt(u32, read[10..14], .little)); + + try std.testing.expectEqualSlices(u8, &[_]u8{ 0x1E, 7 } ++ "capture", dispatch(.stop, &log)); + try std.testing.expectEqualSlices(u8, &[_]u8{ 0x22, 7 } ++ "capture", dispatch(.discard, &log)); + + const devices = dispatch(.list, &log); + try std.testing.expectEqualSlices(u8, &[_]u8{ 0x1F, 4 } ++ "list" ++ .{@intFromEnum(std.meta.Tag(core.Msg).device_event)}, devices); + + const access = dispatch(.access, &log); + try std.testing.expectEqualSlices(u8, &[_]u8{ 0x20, 6 } ++ "access" ++ .{ @intFromEnum(std.meta.Tag(core.Msg).access_event), 1, 1 }, access); + + const subs = core.subscriptions(g_model); + try std.testing.expectEqualSlices(u8, &.{ 0x02, @intFromEnum(std.meta.Tag(core.Msg).devices_changed) }, subs); + rt.frameReset(); +} +`; + +test("audio capture effects: wire bytes through the real dispatch cycle", { skip: !hasZig, timeout: 300_000 }, () => { + const result = transpile(coreAudioCapture); + const details = result.diagnostics.map((d) => `${d.id} ${d.message}`).join("\n"); + assert.equal(result.ok, true, `transpile failed\n${result.typeErrors.join("\n")}\n${details}`); + const work = fs.mkdtempSync(path.join(os.tmpdir(), "native-core-effects-audio-capture-")); + try { + fs.copyFileSync(path.join(pkg, "rt", "rt.zig"), path.join(work, "rt.zig")); + fs.writeFileSync(path.join(work, "core.zig"), result.zig!); + fs.writeFileSync(path.join(work, "harness.zig"), harnessAudioCapture); + try { + execFileSync("zig", ["test", "harness.zig"], { cwd: work, encoding: "utf8", stdio: "pipe" }); + } catch (e) { + const err = e as { stderr?: string; stdout?: string }; + assert.fail(`audio capture harness failed:\n${err.stderr ?? ""}${err.stdout ?? ""}`); + } + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } +}); + // ------------------------------------------------------------------ streams // The streaming ops end to end: spawn line/collect records (argv elements, diff --git a/packages/native-sdk/native-sdk.d.ts b/packages/native-sdk/native-sdk.d.ts index 2338a6bb4..94b1a7143 100644 --- a/packages/native-sdk/native-sdk.d.ts +++ b/packages/native-sdk/native-sdk.d.ts @@ -450,6 +450,12 @@ export type NativeSdkPlatformFeature = | "audioStreaming" | "audio_spectrum" | "audioSpectrum" + | "system_audio_capture" + | "systemAudioCapture" + | "microphone_capture" + | "microphoneCapture" + | "microphone_device_enumeration" + | "microphoneDeviceEnumeration" | "window_hide_on_close" | "windowHideOnClose" | "video_playback" diff --git a/src/app_runner/root.zig b/src/app_runner/root.zig index 261461974..79dee3e98 100644 --- a/src/app_runner/root.zig +++ b/src/app_runner/root.zig @@ -65,6 +65,7 @@ pub const RunOptions = struct { .description = manifestStringField("description"), .has_web_content = manifestHasWebContent(), .declares_tray = manifestDeclaresTrayCapability(), + .permissions = self.security.permissions, .window_title = self.window_title, .bundle_id = self.bundle_id, .icon_path = self.icon_path, diff --git a/src/platform/linux/root.zig b/src/platform/linux/root.zig index fd77d3910..a273a7c83 100644 --- a/src/platform/linux/root.zig +++ b/src/platform/linux/root.zig @@ -456,6 +456,7 @@ pub const LinuxPlatform = struct { // a host whose plugin set lacks it answers false and the // deck's glass rests honestly instead of dancing on fakes. .audio_spectrum => self.web_engine == .system and audioSpectrumAvailable(self.host), + .system_audio_capture, .microphone_capture, .microphone_device_enumeration => false, .tray => false, // No tray means no affordance to bring a policy-hidden // window back — reporting support would strand windows, so diff --git a/src/platform/macos/appkit_host.h b/src/platform/macos/appkit_host.h index af9ddfc2b..c95ecc7cc 100644 --- a/src/platform/macos/appkit_host.h +++ b/src/platform/macos/appkit_host.h @@ -34,6 +34,10 @@ typedef enum { NATIVE_SDK_APPKIT_EVENT_AUDIO = 20, NATIVE_SDK_APPKIT_EVENT_VIDEO = 21, NATIVE_SDK_APPKIT_EVENT_VIEW_FOCUSED = 22, + NATIVE_SDK_APPKIT_EVENT_AUDIO_CAPTURE = 23, + NATIVE_SDK_APPKIT_EVENT_MICROPHONE_DEVICE = 24, + NATIVE_SDK_APPKIT_EVENT_MICROPHONE_DEVICES_CHANGED = 25, + NATIVE_SDK_APPKIT_EVENT_AUDIO_CAPTURE_ACCESS = 26, } native_sdk_appkit_event_kind_t; /* Audio player reports (EVENT_AUDIO payloads). LOADED acknowledges a @@ -356,6 +360,21 @@ typedef struct { * event kind. */ uint64_t video_width; uint64_t video_height; + int audio_capture_state; + int audio_capture_reason; + uint32_t audio_capture_sample_rate_hz; + uint8_t audio_capture_channel_count; + int microphone_device_state; + const char *microphone_device_id; + size_t microphone_device_id_len; + const char *microphone_device_name; + size_t microphone_device_name_len; + int microphone_device_is_default; + uint32_t microphone_device_index; + uint32_t microphone_device_total; + int audio_capture_access_source; + int audio_capture_access_status; + int audio_capture_restart_required; } native_sdk_appkit_event_t; typedef void (*native_sdk_appkit_event_callback_t)(void *context, const native_sdk_appkit_event_t *event); @@ -530,6 +549,16 @@ int native_sdk_appkit_audio_pause(native_sdk_appkit_host_t *host); int native_sdk_appkit_audio_stop(native_sdk_appkit_host_t *host); int native_sdk_appkit_audio_seek(native_sdk_appkit_host_t *host, uint64_t position_ms); int native_sdk_appkit_audio_set_volume(native_sdk_appkit_host_t *host, double volume); +typedef int (*native_sdk_appkit_audio_capture_frame_push_t)(void *context, uint64_t token, + uint64_t frame_offset, uint32_t frame_count, + const uint8_t *system_pcm, size_t system_pcm_len, + const uint8_t *microphone_pcm, size_t microphone_pcm_len, + uint32_t system_gap_frames, uint32_t microphone_gap_frames); +int native_sdk_appkit_audio_capture_start(native_sdk_appkit_host_t *host, int system_audio, int microphone_kind, const char *microphone_id, size_t microphone_id_len, uint32_t sample_rate_hz, uint8_t channel_count, int exclude_current_process_audio, native_sdk_appkit_audio_capture_frame_push_t frame_push, void *frame_context, uint64_t frame_token); +void native_sdk_appkit_audio_capture_stop(native_sdk_appkit_host_t *host); +void native_sdk_appkit_microphone_devices(native_sdk_appkit_host_t *host); +void native_sdk_appkit_audio_capture_access(native_sdk_appkit_host_t *host, int source, int action); +void native_sdk_appkit_observe_microphone_devices(native_sdk_appkit_host_t *host, int enabled); /* Where the video player delivers decoded frames: one tightly packed, * row-major, straight-alpha RGBA8 frame per call (len = width * height diff --git a/src/platform/macos/appkit_host.m b/src/platform/macos/appkit_host.m index 2db878e9f..516daa8fe 100644 --- a/src/platform/macos/appkit_host.m +++ b/src/platform/macos/appkit_host.m @@ -1,4 +1,5 @@ #import "appkit_host.h" +#import "audio_capture.h" #import #import @@ -25,6 +26,7 @@ #include @class NativeSdkAppKitHost; +static void NativeSdkAudioCaptureCallback(void *context, const native_sdk_audio_capture_event_t *captureEvent); static const NSUInteger NativeSdkMaxChildWebViews = 16; static const NSUInteger NativeSdkMaxNativeViews = 32; @@ -831,6 +833,7 @@ @interface NativeSdkAppKitHost : NSObject * installed by the load entries. Every timer callback runs on the main * run loop, so no locking guards the pair. */ @property(nonatomic, assign) native_sdk_appkit_video_sink_push_t videoSinkPush; +@property(nonatomic, assign) native_sdk_audio_capture_t *audioCapture; @property(nonatomic, assign) void *videoSinkContext; /* The reusable RGBA conversion target: malloc'd once per load to the * output's max frame size, freed on stop/replace. Frames are converted @@ -1077,6 +1080,44 @@ - (BOOL)handleShortcutEvent:(NSEvent *)event; - (void)emitShortcutWithId:(NSString *)identifier key:(NSString *)key modifiers:(uint32_t)modifiers event:(NSEvent *)event; @end +static void NativeSdkAudioCaptureCallback(void *context, const native_sdk_audio_capture_event_t *captureEvent) { + NativeSdkAppKitHost *host = (__bridge NativeSdkAppKitHost *)context; + if (!host || !captureEvent) return; + native_sdk_appkit_event_t event = { .timestamp_ns = NativeSdkTimestampNanoseconds() }; + switch (captureEvent->kind) { + case NATIVE_SDK_AUDIO_CAPTURE_EVENT_CAPTURE: + event.kind = NATIVE_SDK_APPKIT_EVENT_AUDIO_CAPTURE; + event.audio_capture_state = captureEvent->state; + event.audio_capture_reason = captureEvent->reason; + event.audio_capture_sample_rate_hz = captureEvent->sample_rate_hz; + event.audio_capture_channel_count = captureEvent->channel_count; + break; + case NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICE: + event.kind = NATIVE_SDK_APPKIT_EVENT_MICROPHONE_DEVICE; + event.microphone_device_state = captureEvent->state; + event.microphone_device_id = captureEvent->device_id; + event.microphone_device_id_len = captureEvent->device_id_len; + event.microphone_device_name = captureEvent->device_name; + event.microphone_device_name_len = captureEvent->device_name_len; + event.microphone_device_is_default = captureEvent->device_is_default; + event.microphone_device_index = captureEvent->device_index; + event.microphone_device_total = captureEvent->device_total; + break; + case NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICES_CHANGED: + event.kind = NATIVE_SDK_APPKIT_EVENT_MICROPHONE_DEVICES_CHANGED; + break; + case NATIVE_SDK_AUDIO_CAPTURE_EVENT_ACCESS: + event.kind = NATIVE_SDK_APPKIT_EVENT_AUDIO_CAPTURE_ACCESS; + event.audio_capture_access_source = captureEvent->access_source; + event.audio_capture_access_status = captureEvent->access_status; + event.audio_capture_restart_required = captureEvent->restart_required; + break; + default: + return; + } + [host emitEvent:event]; +} + // Recursively re-emit the gpu-surface resize event for every metal // surface under `view` (the tall-titlebar chrome re-query path). static void NativeSdkEmitGpuSurfaceResizes(NSView *view) { @@ -7254,6 +7295,7 @@ - (instancetype)initWithAppName:(NSString *)appName displayName:(NSString *)disp self.allowedExternalURLs = @[]; self.externalLinkAction = 0; self.shortcuts = @[]; + self.audioCapture = native_sdk_audio_capture_create(NativeSdkAudioCaptureCallback, (__bridge void *)self); [self configureApplication]; NativeSdkLaunchLap("app_configured"); @@ -11576,6 +11618,11 @@ void native_sdk_appkit_destroy(native_sdk_appkit_host_t *host) { if (!host) { return; } + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + if (object.audioCapture) { + native_sdk_audio_capture_destroy(object.audioCapture); + object.audioCapture = NULL; + } CFBridgingRelease(host); } @@ -11689,6 +11736,32 @@ int native_sdk_appkit_audio_set_volume(native_sdk_appkit_host_t *host, double vo return [object audioSetVolume:volume]; } +int native_sdk_appkit_audio_capture_start(native_sdk_appkit_host_t *host, int system_audio, int microphone_kind, const char *microphone_id, size_t microphone_id_len, uint32_t sample_rate_hz, uint8_t channel_count, int exclude_current_process_audio, native_sdk_appkit_audio_capture_frame_push_t frame_push, void *frame_context, uint64_t frame_token) { + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + if (!object.audioCapture) return 6; + return native_sdk_audio_capture_start(object.audioCapture, system_audio, microphone_kind, microphone_id, microphone_id_len, sample_rate_hz, channel_count, exclude_current_process_audio, frame_push, frame_context, frame_token); +} + +void native_sdk_appkit_audio_capture_stop(native_sdk_appkit_host_t *host) { + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + native_sdk_audio_capture_stop(object.audioCapture); +} + +void native_sdk_appkit_microphone_devices(native_sdk_appkit_host_t *host) { + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + native_sdk_audio_capture_list_microphones(object.audioCapture); +} + +void native_sdk_appkit_audio_capture_access(native_sdk_appkit_host_t *host, int source, int action) { + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + native_sdk_audio_capture_access(object.audioCapture, source, action); +} + +void native_sdk_appkit_observe_microphone_devices(native_sdk_appkit_host_t *host, int enabled) { + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + native_sdk_audio_capture_observe_microphones(object.audioCapture, enabled); +} + int native_sdk_appkit_video_load(native_sdk_appkit_host_t *host, const char *path, size_t path_len, uint64_t token, native_sdk_appkit_video_sink_push_t push_fn, void *push_context) { NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; NSString *path_string = [[NSString alloc] initWithBytes:path length:path_len encoding:NSUTF8StringEncoding]; diff --git a/src/platform/macos/audio_capture.h b/src/platform/macos/audio_capture.h new file mode 100644 index 000000000..0f35fa758 --- /dev/null +++ b/src/platform/macos/audio_capture.h @@ -0,0 +1,58 @@ +#ifndef NATIVE_SDK_AUDIO_CAPTURE_H +#define NATIVE_SDK_AUDIO_CAPTURE_H + +#include +#include + +typedef struct native_sdk_audio_capture native_sdk_audio_capture_t; + +typedef enum { + NATIVE_SDK_AUDIO_CAPTURE_EVENT_CAPTURE = 0, + NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICE = 1, + NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICES_CHANGED = 2, + NATIVE_SDK_AUDIO_CAPTURE_EVENT_ACCESS = 3, +} native_sdk_audio_capture_event_kind_t; + +typedef struct { + native_sdk_audio_capture_event_kind_t kind; + int state; + int reason; + uint32_t sample_rate_hz; + uint8_t channel_count; + const char *device_id; + size_t device_id_len; + const char *device_name; + size_t device_name_len; + int device_is_default; + uint32_t device_index; + uint32_t device_total; + int access_source; + int access_status; + int restart_required; +} native_sdk_audio_capture_event_t; + +typedef void (*native_sdk_audio_capture_callback_t)(void *context, const native_sdk_audio_capture_event_t *event); +typedef int (*native_sdk_audio_capture_frame_callback_t)(void *context, uint64_t token, + uint64_t frame_offset, uint32_t frame_count, + const uint8_t *system_pcm, size_t system_pcm_len, + const uint8_t *microphone_pcm, size_t microphone_pcm_len, + uint32_t system_gap_frames, uint32_t microphone_gap_frames); + +native_sdk_audio_capture_t *native_sdk_audio_capture_create(native_sdk_audio_capture_callback_t callback, void *context); +void native_sdk_audio_capture_destroy(native_sdk_audio_capture_t *capture); + +/* Start results: 0 accepted; 1 invalid options; 2 already active; + * 4 permission required; 5 device missing; 6 unavailable. */ +int native_sdk_audio_capture_start(native_sdk_audio_capture_t *capture, + int system_audio, int microphone_kind, + const char *microphone_id, size_t microphone_id_len, + uint32_t sample_rate_hz, uint8_t channel_count, + int exclude_current_process_audio, + native_sdk_audio_capture_frame_callback_t frame_callback, + void *frame_context, uint64_t frame_token); +void native_sdk_audio_capture_stop(native_sdk_audio_capture_t *capture); +void native_sdk_audio_capture_list_microphones(native_sdk_audio_capture_t *capture); +void native_sdk_audio_capture_access(native_sdk_audio_capture_t *capture, int source, int action); +void native_sdk_audio_capture_observe_microphones(native_sdk_audio_capture_t *capture, int enabled); + +#endif diff --git a/src/platform/macos/audio_capture.m b/src/platform/macos/audio_capture.m new file mode 100644 index 000000000..ce91768a4 --- /dev/null +++ b/src/platform/macos/audio_capture.m @@ -0,0 +1,575 @@ +#import "audio_capture.h" + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Weverything" +#import +#import +#import +#import +#import +#pragma clang diagnostic pop +#include +#include +#include + +/* Ordinals mirror platform/types.zig. Readable is runtime-generated. */ +enum { NS_CAPTURE_STARTED = 0, NS_CAPTURE_READABLE = 1, NS_CAPTURE_STOPPED = 2, NS_CAPTURE_FAILED = 3, NS_CAPTURE_REJECTED = 4 }; +enum { + NS_REASON_NONE = 0, NS_REASON_INVALID_OPTIONS = 1, NS_REASON_PERMISSION_MISSING = 2, + NS_REASON_PERMISSION_REQUIRED = 3, NS_REASON_ALREADY_RECORDING = 4, + NS_REASON_DEVICE_NOT_FOUND = 5, NS_REASON_DEVICE_DISCONNECTED = 6, + NS_REASON_CAPTURE_FAILED = 7, NS_REASON_NO_AUDIO = 8, + NS_REASON_CONSUMER_TOO_SLOW = 9, NS_REASON_DISCARDED = 10, NS_REASON_UNSUPPORTED = 11, +}; +enum { NS_DEVICE = 0, NS_DEVICES_COMPLETED = 1, NS_DEVICES_FAILED = 2, NS_DEVICES_REJECTED = 3 }; +enum { + NS_ACCESS_AUTHORIZED = 0, NS_ACCESS_NOT_AUTHORIZED = 1, NS_ACCESS_NOT_DETERMINED = 2, + NS_ACCESS_DENIED = 3, NS_ACCESS_RESTRICTED = 4, NS_ACCESS_UNAVAILABLE = 5, +}; + +API_AVAILABLE(macos(15.0)) +@interface NativeSdkAudioBlock : NSObject +@property(nonatomic, strong) NSMutableData *systemPCM; +@property(nonatomic, strong) NSMutableData *microphonePCM; +@property(nonatomic, strong) NSMutableIndexSet *systemFrames; +@property(nonatomic, strong) NSMutableIndexSet *microphoneFrames; +- (instancetype)initWithByteCount:(NSUInteger)byteCount; +@end + +@implementation NativeSdkAudioBlock +- (instancetype)initWithByteCount:(NSUInteger)byteCount { + self = [super init]; + if (!self) return nil; + _systemPCM = [NSMutableData dataWithLength:byteCount]; + _microphonePCM = [NSMutableData dataWithLength:byteCount]; + _systemFrames = [NSMutableIndexSet indexSet]; + _microphoneFrames = [NSMutableIndexSet indexSet]; + return self; +} +@end + +API_AVAILABLE(macos(15.0)) +@interface NativeSdkAudioCapture : NSObject +@property(nonatomic, assign) native_sdk_audio_capture_callback_t callback; +@property(nonatomic, assign) void *callbackContext; +@property(nonatomic, assign) native_sdk_audio_capture_frame_callback_t frameCallback; +@property(nonatomic, assign) void *frameContext; +@property(nonatomic, assign) uint64_t frameToken; +@property(nonatomic, strong) dispatch_queue_t sampleQueue; +@property(nonatomic, strong) SCStream *screenStream; +@property(nonatomic, strong) AVCaptureSession *microphoneSession; +@property(nonatomic, strong) AVCaptureAudioDataOutput *microphoneOutput; +@property(nonatomic, strong) NSString *selectedMicrophoneID; +@property(nonatomic, assign) uint32_t sampleRate; +@property(nonatomic, assign) uint8_t channelCount; +@property(nonatomic, assign) BOOL systemAudio; +@property(nonatomic, assign) BOOL microphoneAudio; +@property(nonatomic, assign) BOOL active; +@property(nonatomic, assign) BOOL terminalEmitted; +@property(nonatomic, assign) BOOL observingDevices; +@property(nonatomic, assign) BOOL hasBasePTS; +@property(nonatomic, assign) CMTime basePTS; +@property(nonatomic, assign) uint64_t systemWatermark; +@property(nonatomic, assign) uint64_t microphoneWatermark; +@property(nonatomic, assign) uint64_t nextEmitFrame; +@property(nonatomic, assign) uint64_t acceptedBlocks; +@property(nonatomic, strong) NSMutableDictionary *blocks; +@property(nonatomic, strong) id connectedObserver; +@property(nonatomic, strong) id disconnectedObserver; +@property(nonatomic, strong) dispatch_semaphore_t finalizationSemaphore; +- (int)startSystemAudio:(BOOL)systemAudio microphoneKind:(int)microphoneKind microphoneID:(NSString *)microphoneID sampleRate:(uint32_t)sampleRate channels:(uint8_t)channels excludeCurrentProcessAudio:(BOOL)exclude frameCallback:(native_sdk_audio_capture_frame_callback_t)frameCallback frameContext:(void *)frameContext frameToken:(uint64_t)frameToken; +- (int)drainReadyBlocks:(BOOL)final; +- (void)stopCapture; +- (void)finishWithState:(int)state reason:(int)reason; +@end + +static NSArray *NativeSdkMicrophones(void) API_AVAILABLE(macos(15.0)); +static NSArray *NativeSdkMicrophones(void) { + AVCaptureDeviceDiscoverySession *session = [AVCaptureDeviceDiscoverySession + discoverySessionWithDeviceTypes:@[ AVCaptureDeviceTypeMicrophone ] + mediaType:AVMediaTypeAudio position:AVCaptureDevicePositionUnspecified]; + return session.devices; +} + +static AVCaptureDevice *NativeSdkMicrophone(NSString *identifier, BOOL useDefault) API_AVAILABLE(macos(15.0)); +static AVCaptureDevice *NativeSdkMicrophone(NSString *identifier, BOOL useDefault) { + if (useDefault) return [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; + for (AVCaptureDevice *device in NativeSdkMicrophones()) { + if ([device.uniqueID isEqualToString:identifier]) return device; + } + return nil; +} + +@implementation NativeSdkAudioCapture + +- (instancetype)initWithCallback:(native_sdk_audio_capture_callback_t)callback context:(void *)context { + self = [super init]; + if (!self) return nil; + _callback = callback; + _callbackContext = context; + _sampleQueue = dispatch_queue_create("dev.native-sdk.audio-capture", DISPATCH_QUEUE_SERIAL); + _blocks = [NSMutableDictionary dictionary]; + return self; +} + +- (void)dealloc { + [self stopDeviceObservers]; + if (_screenStream) [_screenStream stopCaptureWithCompletionHandler:nil]; + if (_microphoneSession.running) [_microphoneSession stopRunning]; +} + +- (void)emit:(native_sdk_audio_capture_event_t)event { + if ([NSThread isMainThread]) { + native_sdk_audio_capture_callback_t callback = self.callback; + if (callback) callback(self.callbackContext, &event); + return; + } + dispatch_async(dispatch_get_main_queue(), ^{ + native_sdk_audio_capture_callback_t callback = self.callback; + if (callback) callback(self.callbackContext, &event); + }); +} + +- (void)emitCaptureState:(int)state reason:(int)reason { + native_sdk_audio_capture_event_t event = { + .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_CAPTURE, + .state = state, + .reason = reason, + .sample_rate_hz = self.sampleRate, + .channel_count = self.channelCount, + }; + [self emit:event]; +} + +- (void)startDeviceObservers { + if (self.connectedObserver || self.disconnectedObserver) return; + __weak NativeSdkAudioCapture *weakSelf = self; + NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; + self.connectedObserver = [center addObserverForName:AVCaptureDeviceWasConnectedNotification object:nil queue:[NSOperationQueue mainQueue] usingBlock:^(NSNotification *note) { + (void)note; + NativeSdkAudioCapture *strongSelf = weakSelf; + if (!strongSelf) return; + native_sdk_audio_capture_event_t event = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICES_CHANGED }; + [strongSelf emit:event]; + }]; + self.disconnectedObserver = [center addObserverForName:AVCaptureDeviceWasDisconnectedNotification object:nil queue:[NSOperationQueue mainQueue] usingBlock:^(NSNotification *note) { + NativeSdkAudioCapture *strongSelf = weakSelf; + if (!strongSelf) return; + AVCaptureDevice *device = note.object; + if (strongSelf.active && strongSelf.selectedMicrophoneID.length > 0 && [device.uniqueID isEqualToString:strongSelf.selectedMicrophoneID]) { + [strongSelf finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_DEVICE_DISCONNECTED]; + } + native_sdk_audio_capture_event_t event = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICES_CHANGED }; + [strongSelf emit:event]; + }]; +} + +- (void)stopDeviceObservers { + NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; + if (self.connectedObserver) [center removeObserver:self.connectedObserver]; + if (self.disconnectedObserver) [center removeObserver:self.disconnectedObserver]; + self.connectedObserver = nil; + self.disconnectedObserver = nil; +} + +- (NSUInteger)blockFrames { return (NSUInteger)self.sampleRate * 20 / 1000; } +- (NSUInteger)bytesPerFrame { return (NSUInteger)self.channelCount * sizeof(int16_t); } + +- (NativeSdkAudioBlock *)blockAtIndex:(uint64_t)index create:(BOOL)create { + NSNumber *key = @(index); + NativeSdkAudioBlock *block = self.blocks[key]; + if (!block && create) { + block = [[NativeSdkAudioBlock alloc] initWithByteCount:self.blockFrames * self.bytesPerFrame]; + self.blocks[key] = block; + } + return block; +} + +- (void)storePCM:(const uint8_t *)bytes frames:(NSUInteger)frames atFrame:(uint64_t)startFrame microphone:(BOOL)microphone { + /* A source may deliver a late buffer after the paired interval was + already published with a zero-filled gap. Never recreate those old + blocks: paired offsets are monotonic and published frames are final. */ + if (startFrame + frames <= self.nextEmitFrame) return; + if (startFrame < self.nextEmitFrame) { + NSUInteger skip = (NSUInteger)(self.nextEmitFrame - startFrame); + bytes += skip * self.bytesPerFrame; + frames -= skip; + startFrame = self.nextEmitFrame; + } + NSUInteger consumed = 0; + const NSUInteger blockFrames = self.blockFrames; + const NSUInteger bytesPerFrame = self.bytesPerFrame; + while (consumed < frames) { + uint64_t absoluteFrame = startFrame + consumed; + uint64_t blockIndex = absoluteFrame / blockFrames; + NSUInteger inBlock = (NSUInteger)(absoluteFrame % blockFrames); + NSUInteger take = MIN(frames - consumed, blockFrames - inBlock); + NativeSdkAudioBlock *block = [self blockAtIndex:blockIndex create:YES]; + NSMutableData *target = microphone ? block.microphonePCM : block.systemPCM; + memcpy((uint8_t *)target.mutableBytes + inBlock * bytesPerFrame, bytes + consumed * bytesPerFrame, take * bytesPerFrame); + NSMutableIndexSet *coverage = microphone ? block.microphoneFrames : block.systemFrames; + [coverage addIndexesInRange:NSMakeRange(inBlock, take)]; + consumed += take; + } + uint64_t endFrame = startFrame + frames; + if (microphone) self.microphoneWatermark = MAX(self.microphoneWatermark, endFrame); + else self.systemWatermark = MAX(self.systemWatermark, endFrame); + int result = [self drainReadyBlocks:NO]; + if (result == 1) [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CONSUMER_TOO_SLOW]; + else if (result == 2) [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; +} + +- (int)drainReadyBlocks:(BOOL)final { + const NSUInteger blockFrames = self.blockFrames; + const NSUInteger blockBytes = blockFrames * self.bytesPerFrame; + /* Bound the native aligner as well as the runtime ring. If one source + pauses while the other advances, publish the older interval with a + counted zero-filled gap instead of accumulating blocks indefinitely. */ + const uint64_t alignmentLead = (uint64_t)blockFrames * 2; + while (self.active || final) { + uint64_t end = self.nextEmitFrame + blockFrames; + BOOL ready = final + ? end <= MAX(self.systemWatermark, self.microphoneWatermark) + blockFrames - 1 + : (!self.systemAudio || self.systemWatermark >= end || self.microphoneWatermark >= end + alignmentLead) && + (!self.microphoneAudio || self.microphoneWatermark >= end || self.systemWatermark >= end + alignmentLead); + if (!ready) break; + uint64_t blockIndex = self.nextEmitFrame / blockFrames; + NativeSdkAudioBlock *block = [self blockAtIndex:blockIndex create:YES]; + uint32_t systemGaps = self.systemAudio ? (uint32_t)(blockFrames - block.systemFrames.count) : 0; + uint32_t microphoneGaps = self.microphoneAudio ? (uint32_t)(blockFrames - block.microphoneFrames.count) : 0; + int result = self.frameCallback ? self.frameCallback( + self.frameContext, self.frameToken, self.nextEmitFrame, (uint32_t)blockFrames, + self.systemAudio ? block.systemPCM.bytes : NULL, self.systemAudio ? blockBytes : 0, + self.microphoneAudio ? block.microphonePCM.bytes : NULL, self.microphoneAudio ? blockBytes : 0, + systemGaps, microphoneGaps) : 2; + [self.blocks removeObjectForKey:@(blockIndex)]; + if (result != 0) return result; + self.acceptedBlocks += 1; + self.nextEmitFrame = end; + } + return 0; +} + +- (int)startSystemAudio:(BOOL)systemAudio microphoneKind:(int)microphoneKind microphoneID:(NSString *)microphoneID sampleRate:(uint32_t)sampleRate channels:(uint8_t)channels excludeCurrentProcessAudio:(BOOL)exclude frameCallback:(native_sdk_audio_capture_frame_callback_t)frameCallback frameContext:(void *)frameContext frameToken:(uint64_t)frameToken { + if (self.active) return 2; + if (@available(macOS 15.0, *)) {} else { return 6; } + if ((!systemAudio && microphoneKind == 0) || (channels != 1 && channels != 2) || !frameCallback) return 1; + if (sampleRate != 16000 && sampleRate != 24000 && sampleRate != 44100 && sampleRate != 48000) return 1; + if (systemAudio && !CGPreflightScreenCaptureAccess()) return 4; + if (microphoneKind != 0 && [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeAudio] != AVAuthorizationStatusAuthorized) return 4; + AVCaptureDevice *device = nil; + if (microphoneKind != 0) { + device = NativeSdkMicrophone(microphoneID, microphoneKind == 1); + if (!device) return 5; + } + self.selectedMicrophoneID = device.uniqueID; + self.sampleRate = sampleRate; + self.channelCount = channels; + self.systemAudio = systemAudio; + self.microphoneAudio = microphoneKind != 0; + self.frameCallback = frameCallback; + self.frameContext = frameContext; + self.frameToken = frameToken; + self.active = YES; + self.terminalEmitted = NO; + self.hasBasePTS = NO; + self.systemWatermark = 0; + self.microphoneWatermark = 0; + self.nextEmitFrame = 0; + self.acceptedBlocks = 0; + [self.blocks removeAllObjects]; + self.finalizationSemaphore = dispatch_semaphore_create(0); + [self startDeviceObservers]; + + if (systemAudio) { + [SCShareableContent getShareableContentExcludingDesktopWindows:YES onScreenWindowsOnly:YES completionHandler:^(SCShareableContent *content, NSError *error) { + if (!self.active) return; + if (error || content.displays.count == 0) { [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; return; } + NSArray *excluded = @[]; + if (exclude) { + NSString *bundleID = NSBundle.mainBundle.bundleIdentifier; + if (bundleID.length > 0) { + NSPredicate *predicate = [NSPredicate predicateWithBlock:^BOOL(SCRunningApplication *application, NSDictionary *bindings) { + (void)bindings; + return [application.bundleIdentifier isEqualToString:bundleID]; + }]; + excluded = [content.applications filteredArrayUsingPredicate:predicate]; + } + } + SCContentFilter *filter = [[SCContentFilter alloc] initWithDisplay:content.displays.firstObject excludingApplications:excluded exceptingWindows:@[]]; + SCStreamConfiguration *configuration = [SCStreamConfiguration new]; + configuration.width = 2; + configuration.height = 2; + configuration.minimumFrameInterval = CMTimeMake(1, 1); + configuration.showsCursor = NO; + configuration.capturesAudio = YES; + configuration.sampleRate = sampleRate; + configuration.channelCount = channels; + configuration.excludesCurrentProcessAudio = exclude; + if (microphoneKind != 0) { + configuration.captureMicrophone = YES; + configuration.microphoneCaptureDeviceID = device.uniqueID; + } + SCStream *stream = [[SCStream alloc] initWithFilter:filter configuration:configuration delegate:self]; + NSError *addError = nil; + if (![stream addStreamOutput:self type:SCStreamOutputTypeAudio sampleHandlerQueue:self.sampleQueue error:&addError] || + (microphoneKind != 0 && ![stream addStreamOutput:self type:SCStreamOutputTypeMicrophone sampleHandlerQueue:self.sampleQueue error:&addError])) { + [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; + return; + } + self.screenStream = stream; + [stream startCaptureWithCompletionHandler:^(NSError *startError) { + if (startError) [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; + else if (self.active && !self.terminalEmitted) [self emitCaptureState:NS_CAPTURE_STARTED reason:NS_REASON_NONE]; + }]; + }]; + return 0; + } + + dispatch_async(self.sampleQueue, ^{ + NSError *inputError = nil; + AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&inputError]; + if (!input) { [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_DEVICE_NOT_FOUND]; return; } + AVCaptureSession *session = [AVCaptureSession new]; + AVCaptureAudioDataOutput *output = [AVCaptureAudioDataOutput new]; + [output setSampleBufferDelegate:self queue:self.sampleQueue]; + [session beginConfiguration]; + if (![session canAddInput:input] || ![session canAddOutput:output]) { + [session commitConfiguration]; + [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; + return; + } + [session addInput:input]; + [session addOutput:output]; + [session commitConfiguration]; + self.microphoneSession = session; + self.microphoneOutput = output; + [session startRunning]; + if (!session.running) [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; + else [self emitCaptureState:NS_CAPTURE_STARTED reason:NS_REASON_NONE]; + }); + return 0; +} + +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { + (void)stream; + (void)error; + if (self.active && !self.terminalEmitted) [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; +} + +- (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer ofType:(SCStreamOutputType)type { + (void)stream; + [self consumeSampleBuffer:sampleBuffer microphone:(type == SCStreamOutputTypeMicrophone)]; +} + +- (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { + (void)output; + (void)connection; + [self consumeSampleBuffer:sampleBuffer microphone:YES]; +} + +- (void)consumeSampleBuffer:(CMSampleBufferRef)sampleBuffer microphone:(BOOL)microphone { + if (!self.active || !CMSampleBufferDataIsReady(sampleBuffer)) return; + CMAudioFormatDescriptionRef description = CMSampleBufferGetFormatDescription(sampleBuffer); + const AudioStreamBasicDescription *asbd = description ? CMAudioFormatDescriptionGetStreamBasicDescription(description) : NULL; + if (!asbd) return; + size_t listSize = 0; + if (CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, &listSize, NULL, 0, NULL, NULL, 0, NULL) != noErr || listSize == 0) return; + AudioBufferList *list = malloc(listSize); + if (!list) { [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; return; } + CMBlockBufferRef block = NULL; + if (CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, NULL, list, listSize, NULL, NULL, 0, &block) != noErr) { + free(list); + return; + } + AVAudioFormat *inputFormat = [[AVAudioFormat alloc] initWithStreamDescription:asbd]; + AVAudioPCMBuffer *input = [[AVAudioPCMBuffer alloc] initWithPCMFormat:inputFormat bufferListNoCopy:list deallocator:^(const AudioBufferList *bufferList) { + (void)bufferList; + if (block) CFRelease(block); + free(list); + }]; + if (!input) { if (block) CFRelease(block); free(list); return; } + input.frameLength = (AVAudioFrameCount)CMSampleBufferGetNumSamples(sampleBuffer); + AVAudioFormat *outputFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 sampleRate:self.sampleRate channels:self.channelCount interleaved:YES]; + AVAudioConverter *converter = [[AVAudioConverter alloc] initFromFormat:inputFormat toFormat:outputFormat]; + if (!converter) return; + AVAudioFrameCount capacity = (AVAudioFrameCount)ceil((double)input.frameLength * self.sampleRate / inputFormat.sampleRate) + 32; + AVAudioPCMBuffer *converted = [[AVAudioPCMBuffer alloc] initWithPCMFormat:outputFormat frameCapacity:capacity]; + __block BOOL supplied = NO; + NSError *conversionError = nil; + AVAudioConverterOutputStatus status = [converter convertToBuffer:converted error:&conversionError withInputFromBlock:^AVAudioBuffer *(AVAudioPacketCount requested, AVAudioConverterInputStatus *inputStatus) { + (void)requested; + if (supplied) { *inputStatus = AVAudioConverterInputStatus_EndOfStream; return nil; } + supplied = YES; + *inputStatus = AVAudioConverterInputStatus_HaveData; + return input; + }]; + if (status == AVAudioConverterOutputStatus_Error || converted.frameLength == 0) return; + CMTime pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); + if (!self.hasBasePTS) { self.basePTS = pts; self.hasBasePTS = YES; } + double offsetSeconds = CMTimeGetSeconds(CMTimeSubtract(pts, self.basePTS)); + uint64_t startFrame = isfinite(offsetSeconds) && offsetSeconds > 0 ? (uint64_t)llround(offsetSeconds * self.sampleRate) : 0; + const AudioBufferList *convertedList = converted.audioBufferList; + if (!convertedList || convertedList->mNumberBuffers == 0 || !convertedList->mBuffers[0].mData) return; + [self storePCM:convertedList->mBuffers[0].mData frames:converted.frameLength atFrame:startFrame microphone:microphone]; +} + +- (void)stopCapture { + if (!self.active || self.terminalEmitted) return; + [self finishWithState:NS_CAPTURE_STOPPED reason:NS_REASON_NONE]; +} + +- (void)finishWithState:(int)state reason:(int)reason { + @synchronized (self) { + if (self.terminalEmitted) return; + self.terminalEmitted = YES; + self.active = NO; + } + dispatch_async(self.sampleQueue, ^{ + SCStream *stream = self.screenStream; + self.screenStream = nil; + if (stream) [stream stopCaptureWithCompletionHandler:nil]; + AVCaptureSession *session = self.microphoneSession; + self.microphoneSession = nil; + self.microphoneOutput = nil; + if (session.running) [session stopRunning]; + int drainResult = [self drainReadyBlocks:YES]; + int terminalState = state; + int terminalReason = reason; + if (drainResult == 1) { + terminalState = NS_CAPTURE_FAILED; + terminalReason = NS_REASON_CONSUMER_TOO_SLOW; + } else if (drainResult == 2 && terminalReason == NS_REASON_NONE) { + terminalState = NS_CAPTURE_FAILED; + terminalReason = NS_REASON_CAPTURE_FAILED; + } else if (terminalReason == NS_REASON_NONE && self.acceptedBlocks == 0) { + terminalState = NS_CAPTURE_FAILED; + terminalReason = NS_REASON_NO_AUDIO; + } + [self emitCaptureState:terminalState reason:terminalReason]; + self.selectedMicrophoneID = nil; + self.frameCallback = NULL; + self.frameContext = NULL; + [self.blocks removeAllObjects]; + if (!self.observingDevices) [self stopDeviceObservers]; + dispatch_semaphore_t semaphore = self.finalizationSemaphore; + self.finalizationSemaphore = nil; + if (semaphore) dispatch_semaphore_signal(semaphore); + }); +} + +@end + +struct native_sdk_audio_capture { void *object; }; + +native_sdk_audio_capture_t *native_sdk_audio_capture_create(native_sdk_audio_capture_callback_t callback, void *context) { + native_sdk_audio_capture_t *handle = calloc(1, sizeof(*handle)); + if (!handle) return NULL; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = [[NativeSdkAudioCapture alloc] initWithCallback:callback context:context]; + if (!object) { free(handle); return NULL; } + handle->object = (__bridge_retained void *)object; + return handle; + } + free(handle); + return NULL; +} + +void native_sdk_audio_capture_destroy(native_sdk_audio_capture_t *capture) { + if (!capture) return; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = (__bridge_transfer NativeSdkAudioCapture *)capture->object; + object.callback = NULL; + object.callbackContext = NULL; + dispatch_semaphore_t semaphore = object.finalizationSemaphore; + [object stopCapture]; + if (semaphore) (void)dispatch_semaphore_wait(semaphore, dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC)); + } + free(capture); +} + +int native_sdk_audio_capture_start(native_sdk_audio_capture_t *capture, int system_audio, int microphone_kind, const char *microphone_id, size_t microphone_id_len, uint32_t sample_rate_hz, uint8_t channel_count, int exclude_current_process_audio, native_sdk_audio_capture_frame_callback_t frame_callback, void *frame_context, uint64_t frame_token) { + if (!capture || !capture->object) return 6; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = (__bridge NativeSdkAudioCapture *)capture->object; + NSString *deviceID = [[NSString alloc] initWithBytes:microphone_id length:microphone_id_len encoding:NSUTF8StringEncoding] ?: @""; + return [object startSystemAudio:(system_audio != 0) microphoneKind:microphone_kind microphoneID:deviceID sampleRate:sample_rate_hz channels:channel_count excludeCurrentProcessAudio:(exclude_current_process_audio != 0) frameCallback:frame_callback frameContext:frame_context frameToken:frame_token]; + } + return 6; +} + +void native_sdk_audio_capture_stop(native_sdk_audio_capture_t *capture) { + if (!capture || !capture->object) return; + if (@available(macOS 15.0, *)) [(__bridge NativeSdkAudioCapture *)capture->object stopCapture]; +} + +void native_sdk_audio_capture_list_microphones(native_sdk_audio_capture_t *capture) { + if (!capture || !capture->object) return; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = (__bridge NativeSdkAudioCapture *)capture->object; + dispatch_async(dispatch_get_main_queue(), ^{ + NSArray *devices = NativeSdkMicrophones(); + AVCaptureDevice *defaultDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; + uint32_t total = (uint32_t)MIN((NSUInteger)UINT32_MAX, devices.count); + [devices enumerateObjectsUsingBlock:^(AVCaptureDevice *device, NSUInteger index, BOOL *stop) { + (void)stop; + const char *identifier = device.uniqueID.UTF8String ?: ""; + const char *name = device.localizedName.UTF8String ?: ""; + native_sdk_audio_capture_event_t event = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICE, .state = NS_DEVICE, + .device_id = identifier, .device_id_len = strlen(identifier), .device_name = name, .device_name_len = strlen(name), + .device_is_default = [device.uniqueID isEqualToString:defaultDevice.uniqueID] ? 1 : 0, .device_index = (uint32_t)index, .device_total = total }; + [object emit:event]; + }]; + native_sdk_audio_capture_event_t completed = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICE, .state = NS_DEVICES_COMPLETED, .device_index = total, .device_total = total }; + [object emit:completed]; + }); + } +} + +void native_sdk_audio_capture_access(native_sdk_audio_capture_t *capture, int source, int action) { + if (!capture || !capture->object) return; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = (__bridge NativeSdkAudioCapture *)capture->object; + dispatch_async(dispatch_get_main_queue(), ^{ + if (source == 0) { + BOOL before = CGPreflightScreenCaptureAccess(); + BOOL granted = before; + if (action == 1 && !before) granted = CGRequestScreenCaptureAccess(); + BOOL after = CGPreflightScreenCaptureAccess(); + int status = (before || after || granted) ? NS_ACCESS_AUTHORIZED : NS_ACCESS_NOT_AUTHORIZED; + native_sdk_audio_capture_event_t event = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_ACCESS, .access_source = source, + .access_status = status, .restart_required = (granted && !after) ? 1 : 0 }; + [object emit:event]; + return; + } + AVAuthorizationStatus auth = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeAudio]; + void (^emitStatus)(AVAuthorizationStatus) = ^(AVAuthorizationStatus value) { + int status = NS_ACCESS_DENIED; + switch (value) { + case AVAuthorizationStatusAuthorized: status = NS_ACCESS_AUTHORIZED; break; + case AVAuthorizationStatusNotDetermined: status = NS_ACCESS_NOT_DETERMINED; break; + case AVAuthorizationStatusRestricted: status = NS_ACCESS_RESTRICTED; break; + case AVAuthorizationStatusDenied: default: status = NS_ACCESS_DENIED; break; + } + native_sdk_audio_capture_event_t event = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_ACCESS, .access_source = source, .access_status = status }; + [object emit:event]; + }; + if (action == 1 && auth == AVAuthorizationStatusNotDetermined) { + [AVCaptureDevice requestAccessForMediaType:AVMediaTypeAudio completionHandler:^(BOOL granted) { + (void)granted; + emitStatus([AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeAudio]); + }]; + } else emitStatus(auth); + }); + } +} + +void native_sdk_audio_capture_observe_microphones(native_sdk_audio_capture_t *capture, int enabled) { + if (!capture || !capture->object) return; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = (__bridge NativeSdkAudioCapture *)capture->object; + object.observingDevices = enabled != 0; + if (enabled) [object startDeviceObservers]; + else if (!object.active) [object stopDeviceObservers]; + } +} diff --git a/src/platform/macos/cef_host.mm b/src/platform/macos/cef_host.mm index 1d5a2c86e..d6e9f3c33 100644 --- a/src/platform/macos/cef_host.mm +++ b/src/platform/macos/cef_host.mm @@ -2651,6 +2651,18 @@ int native_sdk_appkit_audio_set_volume(native_sdk_appkit_host_t *host, double vo return 0; } +int native_sdk_appkit_audio_capture_start(native_sdk_appkit_host_t *host, int system_audio, int microphone_kind, const char *microphone_id, size_t microphone_id_len, uint32_t sample_rate_hz, uint8_t channel_count, int exclude_current_process_audio, native_sdk_appkit_audio_capture_frame_push_t frame_push, void *frame_context, uint64_t frame_token) { + (void)host; (void)system_audio; (void)microphone_kind; + (void)microphone_id; (void)microphone_id_len; (void)sample_rate_hz; (void)channel_count; + (void)exclude_current_process_audio; (void)frame_push; (void)frame_context; (void)frame_token; + return 6; +} + +void native_sdk_appkit_audio_capture_stop(native_sdk_appkit_host_t *host) { (void)host; } +void native_sdk_appkit_microphone_devices(native_sdk_appkit_host_t *host) { (void)host; } +void native_sdk_appkit_audio_capture_access(native_sdk_appkit_host_t *host, int source, int action) { (void)host; (void)source; (void)action; } +void native_sdk_appkit_observe_microphone_devices(native_sdk_appkit_host_t *host, int enabled) { (void)host; (void)enabled; } + /* Video playback lives in the system-engine AppKit host (AVFoundation). * The Chromium host reports the feature unsupported and the Zig side * refuses before calling, so these exist only to satisfy the shared C diff --git a/src/platform/macos/root.zig b/src/platform/macos/root.zig index eb3815f70..32753c63e 100644 --- a/src/platform/macos/root.zig +++ b/src/platform/macos/root.zig @@ -2,6 +2,19 @@ const std = @import("std"); const builtin = @import("builtin"); const geometry = @import("geometry"); const platform_mod = @import("../root.zig"); + +fn isMacOS15OrNewer() bool { + // This backend is imported into every desktop target's platform tests. + // Keep the Darwin-only sysctl out of Linux and Windows compilation. + if (comptime builtin.os.tag != .macos) return false; + var version_buf: [64]u8 = undefined; + var version_len: usize = version_buf.len; + if (std.posix.system.sysctlbyname("kern.osproductversion", &version_buf, &version_len, null, 0) != 0) return false; + const version = std.mem.sliceTo(version_buf[0..@min(version_len, version_buf.len)], 0); + const dot = std.mem.indexOfScalar(u8, version, '.') orelse version.len; + const major = std.fmt.parseUnsigned(u16, version[0..dot], 10) catch return false; + return major >= 15; +} const policy_values = @import("../policy_values.zig"); const security = @import("../../security/root.zig"); // The packaging pipeline's one-image icon machinery: dev runs borrow its @@ -42,6 +55,10 @@ const AppKitEventKind = enum(c_int) { audio = 20, video = 21, view_focused = 22, + audio_capture = 23, + microphone_device = 24, + microphone_devices_changed = 25, + audio_capture_access = 26, }; const AppKitEvent = extern struct { @@ -139,6 +156,21 @@ const AppKitEvent = extern struct { /// sink's pixel budget. Zeros on every other kind. video_width: u64, video_height: u64, + audio_capture_state: c_int, + audio_capture_reason: c_int, + audio_capture_sample_rate_hz: u32, + audio_capture_channel_count: u8, + microphone_device_state: c_int, + microphone_device_id: [*]const u8, + microphone_device_id_len: usize, + microphone_device_name: [*]const u8, + microphone_device_name_len: usize, + microphone_device_is_default: c_int, + microphone_device_index: u32, + microphone_device_total: u32, + audio_capture_access_source: c_int, + audio_capture_access_status: c_int, + audio_capture_restart_required: c_int, }; const AppKitCallback = *const fn (context: ?*anyopaque, event: *const AppKitEvent) callconv(.c) void; @@ -149,6 +181,7 @@ const AppKitBridgeCallback = *const fn (context: ?*anyopaque, window_id: u64, we /// receiving claim was released (the host stops its frame timer), /// anything else one dropped frame. const AppKitVideoSinkPush = *const fn (context: ?*anyopaque, width: usize, height: usize, pixels: [*c]const u8, len: usize) callconv(.c) c_int; +const AppKitAudioCaptureFramePush = *const fn (context: ?*anyopaque, token: u64, frame_offset: u64, frame_count: u32, system_pcm: [*c]const u8, system_pcm_len: usize, microphone_pcm: [*c]const u8, microphone_pcm_len: usize, system_gap_frames: u32, microphone_gap_frames: u32) callconv(.c) c_int; const shortcut_modifier_primary: u32 = 1 << 0; const shortcut_modifier_command: u32 = 1 << 1; @@ -205,6 +238,11 @@ extern fn native_sdk_appkit_audio_pause(host: *AppKitHost) c_int; extern fn native_sdk_appkit_audio_stop(host: *AppKitHost) c_int; extern fn native_sdk_appkit_audio_seek(host: *AppKitHost, position_ms: u64) c_int; extern fn native_sdk_appkit_audio_set_volume(host: *AppKitHost, volume: f64) c_int; +extern fn native_sdk_appkit_audio_capture_start(host: *AppKitHost, system_audio: c_int, microphone_kind: c_int, microphone_id: [*]const u8, microphone_id_len: usize, sample_rate_hz: u32, channel_count: u8, exclude_current_process_audio: c_int, frame_push: AppKitAudioCaptureFramePush, frame_context: ?*anyopaque, frame_token: u64) c_int; +extern fn native_sdk_appkit_audio_capture_stop(host: *AppKitHost) void; +extern fn native_sdk_appkit_microphone_devices(host: *AppKitHost) void; +extern fn native_sdk_appkit_audio_capture_access(host: *AppKitHost, source: c_int, action: c_int) void; +extern fn native_sdk_appkit_observe_microphone_devices(host: *AppKitHost, enabled: c_int) void; extern fn native_sdk_appkit_video_load(host: *AppKitHost, path: [*]const u8, path_len: usize, token: u64, push_fn: AppKitVideoSinkPush, push_context: ?*anyopaque) c_int; extern fn native_sdk_appkit_video_load_url(host: *AppKitHost, url: [*]const u8, url_len: usize, token: u64, push_fn: AppKitVideoSinkPush, push_context: ?*anyopaque) c_int; extern fn native_sdk_appkit_video_play(host: *AppKitHost) c_int; @@ -587,6 +625,8 @@ pub const MacPlatform = struct { /// every push happens on the main thread (the host's frame pump is /// a run-loop timer), so a plain field is race-free. video_sink: platform_mod.VideoFrameSink = .{}, + audio_capture_sink: ?platform_mod.AudioCaptureSink = null, + audio_capture_token: u64 = 0, pub fn init(title: []const u8, size: geometry.SizeF) Error!MacPlatform { return initWithEngine(title, size, .system); @@ -750,6 +790,11 @@ pub const MacPlatform = struct { .audio_stop_fn = audioStop, .audio_seek_fn = audioSeek, .audio_set_volume_fn = audioSetVolume, + .audio_capture_start_fn = audioCaptureStart, + .audio_capture_stop_fn = audioCaptureStop, + .microphone_devices_fn = microphoneDevices, + .audio_capture_access_fn = audioCaptureAccess, + .microphone_devices_observe_fn = observeMicrophoneDevices, .video_load_fn = videoLoad, .video_load_url_fn = videoLoadUrl, .video_play_fn = videoPlay, @@ -822,6 +867,10 @@ pub const MacPlatform = struct { .audio_streaming, .audio_spectrum, => self.web_engine == .system, + .system_audio_capture, + .microphone_capture, + .microphone_device_enumeration, + => self.web_engine == .system and isMacOS15OrNewer(), // AVFoundation video ships in the AppKit host only (one // AVPlayer whose AVPlayerItemVideoOutput frames feed the // media-surface sink); the CEF host stubs the C ABI and @@ -1007,6 +1056,26 @@ fn appkitCallback(context: ?*anyopaque, event: *const AppKitEvent) callconv(.c) .width = event.video_width, .height = event.video_height, } }), + .audio_capture => state.emit(.{ .audio_capture = .{ + .state = audioCaptureStateFromInt(event.audio_capture_state), + .reason = audioCaptureReasonFromInt(event.audio_capture_reason), + .sample_rate_hz = event.audio_capture_sample_rate_hz, + .channel_count = event.audio_capture_channel_count, + } }), + .microphone_device => state.emit(.{ .microphone_device = .{ + .state = microphoneDeviceStateFromInt(event.microphone_device_state), + .id = appKitEventBytes(event.microphone_device_id, event.microphone_device_id_len), + .name = appKitEventBytes(event.microphone_device_name, event.microphone_device_name_len), + .is_default = event.microphone_device_is_default != 0, + .index = event.microphone_device_index, + .total = event.microphone_device_total, + } }), + .microphone_devices_changed => state.emit(.microphone_devices_changed), + .audio_capture_access => state.emit(.{ .audio_capture_access = .{ + .source = if (event.audio_capture_access_source == 1) .microphone else .system_audio, + .status = audioCaptureAccessStatusFromInt(event.audio_capture_access_status), + .restart_required = event.audio_capture_restart_required != 0, + } }), .widget_accessibility_action => if (widgetAccessibilityActionFromInt(event.widget_action)) |action| { state.emit(.{ .widget_accessibility_action = .{ .window_id = event.window_id, @@ -1062,6 +1131,53 @@ fn videoEventKindFromInt(value: c_int) platform_mod.VideoEventKind { }; } +fn audioCaptureStateFromInt(value: c_int) platform_mod.AudioCaptureEventState { + return switch (value) { + 0 => .started, + 1 => .readable, + 2 => .stopped, + 3 => .failed, + else => .rejected, + }; +} + +fn audioCaptureReasonFromInt(value: c_int) platform_mod.AudioCaptureEventReason { + return switch (value) { + 0 => .none, + 1 => .invalid_options, + 2 => .permission_missing, + 3 => .permission_required, + 4 => .already_recording, + 5 => .device_not_found, + 6 => .device_disconnected, + 7 => .capture_failed, + 8 => .no_audio, + 9 => .consumer_too_slow, + 10 => .discarded, + else => .unsupported, + }; +} + +fn microphoneDeviceStateFromInt(value: c_int) platform_mod.MicrophoneDeviceEventState { + return switch (value) { + 0 => .device, + 1 => .completed, + 2 => .failed, + else => .rejected, + }; +} + +fn audioCaptureAccessStatusFromInt(value: c_int) platform_mod.AudioCaptureAccessStatus { + return switch (value) { + 0 => .authorized, + 1 => .not_authorized, + 2 => .not_determined, + 3 => .denied, + 4 => .restricted, + else => .unavailable, + }; +} + fn gpuSurfaceInputEventFromAppKitEvent(event: *const AppKitEvent) platform_mod.GpuSurfaceInputEvent { return .{ .window_id = event.window_id, @@ -1531,6 +1647,89 @@ fn audioSetVolume(context: ?*anyopaque, volume: f32) anyerror!void { _ = native_sdk_appkit_audio_set_volume(self.host, volume); } +fn audioCaptureStart(context: ?*anyopaque, config: platform_mod.AudioCaptureConfig, sink: platform_mod.AudioCaptureSink) anyerror!void { + const self: *MacPlatform = @ptrCast(@alignCast(context.?)); + if (self.web_engine != .system or !isMacOS15OrNewer()) return error.UnsupportedService; + if (config.system_audio and !security.hasPermission(self.app_info.permissions, security.permission_system_audio)) return error.PermissionMissing; + if (config.microphone != .none and !security.hasPermission(self.app_info.permissions, security.permission_microphone)) return error.PermissionMissing; + self.audio_capture_token +%= 1; + if (self.audio_capture_token == 0) self.audio_capture_token = 1; + self.audio_capture_sink = sink; + errdefer { + self.audio_capture_sink = null; + self.audio_capture_token = 0; + } + const result = native_sdk_appkit_audio_capture_start( + self.host, + @intFromBool(config.system_audio), + @intFromEnum(config.microphone), + config.microphone_device_id.ptr, + config.microphone_device_id.len, + config.sample_rate_hz, + config.channel_count, + @intFromBool(config.exclude_current_process_audio), + nativeSdkAudioCapturePush, + self, + self.audio_capture_token, + ); + return switch (result) { + 0 => {}, + 1 => error.InvalidAudioCaptureOptions, + 2 => error.AudioCaptureAlreadyActive, + 4 => error.AudioCapturePermissionRequired, + 5 => error.MicrophoneDeviceNotFound, + else => error.UnsupportedService, + }; +} + +fn nativeSdkAudioCapturePush(context: ?*anyopaque, token: u64, frame_offset: u64, frame_count: u32, system_pcm: [*c]const u8, system_pcm_len: usize, microphone_pcm: [*c]const u8, microphone_pcm_len: usize, system_gap_frames: u32, microphone_gap_frames: u32) callconv(.c) c_int { + const self: *MacPlatform = @ptrCast(@alignCast(context orelse return 2)); + if (token != self.audio_capture_token) return 2; + const sink = self.audio_capture_sink orelse return 2; + const system_bytes: []const u8 = if (system_pcm_len == 0 or system_pcm == null) &.{} else system_pcm[0..system_pcm_len]; + const microphone_bytes: []const u8 = if (microphone_pcm_len == 0 or microphone_pcm == null) &.{} else microphone_pcm[0..microphone_pcm_len]; + return switch (sink.push(.{ + .frame_offset = frame_offset, + .frame_count = frame_count, + .system_pcm = system_bytes, + .microphone_pcm = microphone_bytes, + .system_gap_frames = system_gap_frames, + .microphone_gap_frames = microphone_gap_frames, + })) { + .accepted => 0, + .full => 1, + .closed => 2, + }; +} + +fn audioCaptureStop(context: ?*anyopaque) anyerror!void { + const self: *MacPlatform = @ptrCast(@alignCast(context.?)); + if (self.web_engine != .system) return error.UnsupportedService; + native_sdk_appkit_audio_capture_stop(self.host); +} + +fn microphoneDevices(context: ?*anyopaque) anyerror!void { + const self: *MacPlatform = @ptrCast(@alignCast(context.?)); + if (self.web_engine != .system or !isMacOS15OrNewer()) return error.UnsupportedService; + if (!security.hasPermission(self.app_info.permissions, security.permission_microphone)) return error.PermissionMissing; + native_sdk_appkit_microphone_devices(self.host); +} + +fn audioCaptureAccess(context: ?*anyopaque, source: platform_mod.AudioCaptureAccessSource, action: platform_mod.AudioCaptureAccessAction) anyerror!void { + const self: *MacPlatform = @ptrCast(@alignCast(context.?)); + if (self.web_engine != .system or !isMacOS15OrNewer()) return error.UnsupportedService; + const permission = if (source == .system_audio) security.permission_system_audio else security.permission_microphone; + if (!security.hasPermission(self.app_info.permissions, permission)) return error.PermissionMissing; + native_sdk_appkit_audio_capture_access(self.host, @intFromEnum(source), @intFromEnum(action)); +} + +fn observeMicrophoneDevices(context: ?*anyopaque, enabled: bool) anyerror!void { + const self: *MacPlatform = @ptrCast(@alignCast(context.?)); + if (self.web_engine != .system or !isMacOS15OrNewer()) return error.UnsupportedService; + if (!security.hasPermission(self.app_info.permissions, security.permission_microphone)) return error.PermissionMissing; + native_sdk_appkit_observe_microphone_devices(self.host, @intFromBool(enabled)); +} + /// The C-callable bridge for `VideoFrameSink.push`: the sink's `push_fn` /// is a Zig-calling-convention error-union fn the host cannot invoke, so /// the host is handed this trampoline (context = the `MacPlatform`) and diff --git a/src/platform/null_platform.zig b/src/platform/null_platform.zig index 68cfd814b..6ef74f9e9 100644 --- a/src/platform/null_platform.zig +++ b/src/platform/null_platform.zig @@ -282,6 +282,24 @@ pub const NullAudio = struct { } }; +pub const NullAudioCapture = struct { + active: bool = false, + started_pending: bool = false, + sink: ?types.AudioCaptureSink = null, + system_audio: bool = false, + microphone: types.MicrophoneSelectionKind = .none, + microphone_device_id_storage: [types.max_microphone_device_id_bytes]u8 = undefined, + microphone_device_id_len: usize = 0, + sample_rate_hz: u32 = 48_000, + channel_count: u8 = 2, + exclude_current_process_audio: bool = true, + buffer_duration_ms: u32 = types.default_audio_capture_buffer_duration_ms, + + pub fn microphoneDeviceId(self: *const NullAudioCapture) []const u8 { + return self.microphone_device_id_storage[0..self.microphone_device_id_len]; + } +}; + pub const NullPlatform = struct { surface_value: Surface = .{}, web_engine: WebEngine = .system, @@ -625,6 +643,18 @@ pub const NullPlatform = struct { audio_stop_count: usize = 0, audio_seek_count: usize = 0, audio_volume_count: usize = 0, + audio_capture: bool = true, + microphone_device_enumeration: bool = true, + capture: NullAudioCapture = .{}, + audio_capture_start_count: usize = 0, + audio_capture_stop_count: usize = 0, + microphone_devices_count: usize = 0, + microphone_devices_observing: bool = false, + default_microphone_index: u8 = 0, + microphone_connected: [2]bool = .{ true, true }, + system_audio_access: types.AudioCaptureAccessStatus = .authorized, + microphone_access: types.AudioCaptureAccessStatus = .authorized, + access_pending: ?types.AudioCaptureAccessEvent = null, /// Whether this modeled host has a video decoder. On by default (the /// fake below stands in for AVFoundation); tests modelling a staged /// host (Windows/Linux today) set it false BEFORE `platform()` so @@ -851,6 +881,11 @@ pub const NullPlatform = struct { .audio_stop_fn = if (self.audio_playback) audioStop else null, .audio_seek_fn = if (self.audio_playback) audioSeek else null, .audio_set_volume_fn = if (self.audio_playback) audioSetVolume else null, + .audio_capture_start_fn = if (self.audio_capture) audioCaptureStart else null, + .audio_capture_stop_fn = if (self.audio_capture) audioCaptureStop else null, + .microphone_devices_fn = if (self.microphone_device_enumeration) microphoneDevices else null, + .audio_capture_access_fn = if (self.audio_capture) audioCaptureAccess else null, + .microphone_devices_observe_fn = if (self.microphone_device_enumeration) observeMicrophoneDevices else null, .video_load_fn = if (self.video_playback) videoLoad else null, .video_load_url_fn = if (self.video_playback) videoLoadUrl else null, .video_play_fn = if (self.video_playback) videoPlay else null, @@ -915,6 +950,8 @@ pub const NullPlatform = struct { .audio_playback => self.audio_playback, .audio_streaming => self.audio_playback and self.audio_streaming, .audio_spectrum => self.audio_playback and self.audio_spectrum, + .system_audio_capture, .microphone_capture => self.audio_capture, + .microphone_device_enumeration => self.microphone_device_enumeration, .video_playback => self.video_playback, }; } @@ -1667,6 +1704,139 @@ pub const NullPlatform = struct { self.audio.volume = volume; } + fn audioCaptureStart(context: ?*anyopaque, config: types.AudioCaptureConfig, sink: types.AudioCaptureSink) anyerror!void { + const self: *NullPlatform = @ptrCast(@alignCast(context.?)); + if (self.capture.active) return error.AudioCaptureAlreadyActive; + if ((!config.system_audio and config.microphone == .none) or + config.microphone_device_id.len > self.capture.microphone_device_id_storage.len) + return error.InvalidAudioCaptureOptions; + if (config.system_audio and self.system_audio_access != .authorized) return error.AudioCapturePermissionRequired; + if (config.microphone != .none and self.microphone_access != .authorized) return error.AudioCapturePermissionRequired; + var resolved_microphone_id = config.microphone_device_id; + if (config.microphone == .default) { + if (self.default_microphone_index >= self.microphone_connected.len or !self.microphone_connected[self.default_microphone_index]) return error.MicrophoneDeviceNotFound; + resolved_microphone_id = if (self.default_microphone_index == 0) "default-mic" else "usb-mic"; + } else if (config.microphone == .device_id) { + const index: ?usize = if (std.mem.eql(u8, config.microphone_device_id, "default-mic")) 0 else if (std.mem.eql(u8, config.microphone_device_id, "usb-mic")) 1 else null; + if (index == null or !self.microphone_connected[index.?]) return error.MicrophoneDeviceNotFound; + } + self.audio_capture_start_count += 1; + self.capture = .{ + .active = true, + .started_pending = true, + .sink = sink, + .system_audio = config.system_audio, + .microphone = config.microphone, + .sample_rate_hz = config.sample_rate_hz, + .channel_count = config.channel_count, + .exclude_current_process_audio = config.exclude_current_process_audio, + .buffer_duration_ms = config.buffer_duration_ms, + }; + @memcpy(self.capture.microphone_device_id_storage[0..resolved_microphone_id.len], resolved_microphone_id); + self.capture.microphone_device_id_len = resolved_microphone_id.len; + } + + fn audioCaptureStop(context: ?*anyopaque) anyerror!void { + const self: *NullPlatform = @ptrCast(@alignCast(context.?)); + if (!self.capture.active) return; + self.audio_capture_stop_count += 1; + self.capture.active = false; + } + + fn microphoneDevices(context: ?*anyopaque) anyerror!void { + const self: *NullPlatform = @ptrCast(@alignCast(context.?)); + self.microphone_devices_count += 1; + } + + fn audioCaptureAccess(context: ?*anyopaque, source: types.AudioCaptureAccessSource, action: types.AudioCaptureAccessAction) anyerror!void { + const self: *NullPlatform = @ptrCast(@alignCast(context.?)); + _ = action; + self.access_pending = .{ + .source = source, + .status = switch (source) { + .system_audio => self.system_audio_access, + .microphone => self.microphone_access, + }, + }; + } + + fn observeMicrophoneDevices(context: ?*anyopaque, enabled: bool) anyerror!void { + const self: *NullPlatform = @ptrCast(@alignCast(context.?)); + self.microphone_devices_observing = enabled; + } + + pub fn takeAudioCaptureStarted(self: *NullPlatform) ?Event { + if (!self.capture.started_pending) return null; + self.capture.started_pending = false; + return .{ .audio_capture = .{ + .state = .started, + .sample_rate_hz = self.capture.sample_rate_hz, + .channel_count = self.capture.channel_count, + .capacity_frames = self.capture.sample_rate_hz * self.capture.buffer_duration_ms / 1_000, + } }; + } + + pub fn completeAudioCapture(self: *NullPlatform) ?Event { + if (self.capture.active) self.capture.active = false; + return .{ .audio_capture = .{ + .state = .stopped, + .sample_rate_hz = self.capture.sample_rate_hz, + .channel_count = self.capture.channel_count, + } }; + } + + pub fn pushAudioCapturePair(self: *NullPlatform, pair: types.AudioCaptureFramePair) types.AudioCapturePushResult { + if (!self.capture.active) return .closed; + return if (self.capture.sink) |sink| sink.push(pair) else .closed; + } + + pub fn microphoneDeviceEvent(self: *const NullPlatform, index: u32) Event { + const total: u32 = @as(u32, @intFromBool(self.microphone_connected[0])) + @as(u32, @intFromBool(self.microphone_connected[1])); + var emitted: u32 = 0; + for (self.microphone_connected, 0..) |connected, device_index| { + if (!connected) continue; + if (emitted == index) return .{ .microphone_device = .{ + .state = .device, + .id = if (device_index == 0) "default-mic" else "usb-mic", + .name = if (device_index == 0) "Default Microphone" else "USB Microphone", + .is_default = device_index == self.default_microphone_index, + .index = emitted, + .total = total, + } }; + emitted += 1; + } + return .{ .microphone_device = .{ .state = .completed, .index = total, .total = total } }; + } + + pub fn setDefaultMicrophone(self: *NullPlatform, index: u8) !void { + if (index >= self.microphone_connected.len or !self.microphone_connected[index]) return error.MicrophoneDeviceNotFound; + self.default_microphone_index = index; + } + + pub fn disconnectMicrophone(self: *NullPlatform, id: []const u8) ?Event { + const index: usize = if (std.mem.eql(u8, id, "default-mic")) 0 else if (std.mem.eql(u8, id, "usb-mic")) 1 else return null; + self.microphone_connected[index] = false; + if (!self.capture.active or !std.mem.eql(u8, self.capture.microphoneDeviceId(), id)) return self.microphoneDevicesChanged(); + self.capture.active = false; + return .{ .audio_capture = .{ + .state = .failed, + .reason = .device_disconnected, + .sample_rate_hz = self.capture.sample_rate_hz, + .channel_count = self.capture.channel_count, + } }; + } + + pub fn takeAudioCaptureAccess(self: *NullPlatform) ?Event { + const access = self.access_pending orelse return null; + self.access_pending = null; + return .{ .audio_capture_access = access }; + } + + pub fn microphoneDevicesChanged(self: *NullPlatform) ?Event { + if (!self.microphone_devices_observing) return null; + return .microphone_devices_changed; + } + fn audioUrlHash(url: []const u8) u64 { return std.hash.Wyhash.hash(0, url); } diff --git a/src/platform/root.zig b/src/platform/root.zig index f88976698..2912a491d 100644 --- a/src/platform/root.zig +++ b/src/platform/root.zig @@ -134,6 +134,27 @@ pub const AudioEvent = types.AudioEvent; pub const AudioEventKind = types.AudioEventKind; pub const AudioLoadResolution = types.AudioLoadResolution; pub const max_audio_path_bytes = types.max_audio_path_bytes; +pub const max_microphone_device_id_bytes = types.max_microphone_device_id_bytes; +pub const max_microphone_device_name_bytes = types.max_microphone_device_name_bytes; +pub const default_audio_capture_buffer_duration_ms = types.default_audio_capture_buffer_duration_ms; +pub const min_audio_capture_buffer_duration_ms = types.min_audio_capture_buffer_duration_ms; +pub const max_audio_capture_buffer_duration_ms = types.max_audio_capture_buffer_duration_ms; +pub const audio_capture_block_duration_ms = types.audio_capture_block_duration_ms; +pub const audio_capture_max_read_duration_ms = types.audio_capture_max_read_duration_ms; +pub const MicrophoneSelectionKind = types.MicrophoneSelectionKind; +pub const AudioCaptureConfig = types.AudioCaptureConfig; +pub const AudioCaptureEventState = types.AudioCaptureEventState; +pub const AudioCaptureEventReason = types.AudioCaptureEventReason; +pub const AudioCaptureEvent = types.AudioCaptureEvent; +pub const AudioCaptureFramePair = types.AudioCaptureFramePair; +pub const AudioCapturePushResult = types.AudioCapturePushResult; +pub const AudioCaptureSink = types.AudioCaptureSink; +pub const MicrophoneDeviceEventState = types.MicrophoneDeviceEventState; +pub const MicrophoneDeviceEvent = types.MicrophoneDeviceEvent; +pub const AudioCaptureAccessSource = types.AudioCaptureAccessSource; +pub const AudioCaptureAccessAction = types.AudioCaptureAccessAction; +pub const AudioCaptureAccessStatus = types.AudioCaptureAccessStatus; +pub const AudioCaptureAccessEvent = types.AudioCaptureAccessEvent; pub const audio_spectrum_band_count = types.audio_spectrum_band_count; pub const audio_spectrum_floor_db = types.audio_spectrum_floor_db; pub const VideoEvent = types.VideoEvent; diff --git a/src/platform/types.zig b/src/platform/types.zig index 9fc185f2e..757cd6b8d 100644 --- a/src/platform/types.zig +++ b/src/platform/types.zig @@ -158,6 +158,12 @@ pub const PlatformFeature = enum { /// keeps emitting; the null platform models the rule through its /// windows' modeled occlusion so the suites can pin it. audio_spectrum, + /// Capture system output as a reliable pull-based PCM stream. + system_audio_capture, + /// Capture a default or explicitly selected microphone as PCM. + microphone_capture, + /// Enumerate connected microphones and observe list invalidations. + microphone_device_enumeration, /// The `close_policy = .hide` window shape: the host can intercept /// the user's close affordance, keep the window alive off the /// glass, and re-show it later (`show_window_fn`, tray actions, the @@ -1208,6 +1214,9 @@ pub const AppInfo = struct { /// folds this into its `window_hide_on_close` answer; macOS ignores /// it (the Dock reopen path always exists). declares_tray: bool = false, + /// Manifest permission grants, threaded to native services so APIs + /// that bypass the web bridge still enforce the same declaration. + permissions: []const []const u8 = &.{}, window_title: []const u8 = "", bundle_id: []const u8 = "dev.native_sdk.app", icon_path: []const u8 = "", @@ -1452,6 +1461,138 @@ pub const AudioLoadResolution = enum(u8) { stream, }; +pub const max_microphone_device_id_bytes: usize = 512; +pub const max_microphone_device_name_bytes: usize = 512; +pub const default_audio_capture_buffer_duration_ms: u32 = 5_000; +pub const min_audio_capture_buffer_duration_ms: u32 = 1_000; +pub const max_audio_capture_buffer_duration_ms: u32 = 30_000; +pub const audio_capture_block_duration_ms: u32 = 20; +pub const audio_capture_max_read_duration_ms: u32 = 100; + +pub const MicrophoneSelectionKind = enum(u8) { + none, + default, + device_id, +}; + +pub const AudioCaptureConfig = struct { + system_audio: bool = false, + microphone: MicrophoneSelectionKind = .none, + microphone_device_id: []const u8 = &.{}, + sample_rate_hz: u32 = 48_000, + channel_count: u8 = 2, + exclude_current_process_audio: bool = true, + buffer_duration_ms: u32 = default_audio_capture_buffer_duration_ms, +}; + +pub const AudioCaptureEventState = enum(u8) { + started, + readable, + stopped, + failed, + rejected, +}; + +pub const AudioCaptureEventReason = enum(u8) { + none, + invalid_options, + permission_missing, + permission_required, + already_recording, + device_not_found, + device_disconnected, + capture_failed, + no_audio, + consumer_too_slow, + discarded, + unsupported, +}; + +pub const AudioCaptureEvent = struct { + state: AudioCaptureEventState, + reason: AudioCaptureEventReason = .none, + sample_rate_hz: u32 = 0, + channel_count: u8 = 0, + available_frames: u32 = 0, + capacity_frames: u32 = 0, + frames_produced: u64 = 0, +}; + +/// One normalized, timestamp-aligned pair produced by a platform capture +/// backend. Both PCM slices are signed 16-bit little-endian and cover the +/// same `frame_count` interval. An inactive source uses an empty slice; +/// SDK-inserted silence is counted separately from delivered zero samples. +pub const AudioCaptureFramePair = struct { + frame_offset: u64, + frame_count: u32, + system_pcm: []const u8 = &.{}, + microphone_pcm: []const u8 = &.{}, + system_gap_frames: u32 = 0, + microphone_gap_frames: u32 = 0, +}; + +pub const AudioCapturePushResult = enum(u8) { + accepted, + full, + closed, +}; + +/// Cross-thread producer seam handed to a platform capture backend. The +/// callback copies synchronously; the borrowed slices need only remain valid +/// for the duration of `push`. A full answer is terminal backpressure: the +/// platform must stop capture and report `consumer_too_slow`. +pub const AudioCaptureSink = struct { + context: *anyopaque, + generation: u64, + push_fn: *const fn (context: *anyopaque, generation: u64, pair: AudioCaptureFramePair) AudioCapturePushResult, + + pub fn push(self: AudioCaptureSink, pair: AudioCaptureFramePair) AudioCapturePushResult { + return self.push_fn(self.context, self.generation, pair); + } +}; + +pub const MicrophoneDeviceEventState = enum(u8) { + device, + completed, + failed, + rejected, +}; + +/// Device strings are borrowed for the duration of EventHandler dispatch. +pub const MicrophoneDeviceEvent = struct { + state: MicrophoneDeviceEventState, + id: []const u8 = &.{}, + name: []const u8 = &.{}, + is_default: bool = false, + index: u32 = 0, + total: u32 = 0, +}; + +pub const AudioCaptureAccessSource = enum(u8) { + system_audio, + microphone, +}; + +pub const AudioCaptureAccessAction = enum(u8) { + status, + request, +}; + +pub const AudioCaptureAccessStatus = enum(u8) { + authorized, + not_authorized, + not_determined, + denied, + restricted, + unavailable, +}; + +pub const AudioCaptureAccessEvent = struct { + source: AudioCaptureAccessSource, + status: AudioCaptureAccessStatus, + restart_required: bool = false, +}; + /// Longest video source string (local path or URL) `videoLoad`/ /// `videoLoadUrl` accepts; longer strings are rejected with /// `error.VideoPathTooLarge` before the platform is asked. @@ -2255,6 +2396,10 @@ pub const Event = union(enum) { /// Audio player reports: load acknowledgment, coarse position ticks /// while playing, one completion at natural end, async failures. audio: AudioEvent, + audio_capture: AudioCaptureEvent, + microphone_device: MicrophoneDeviceEvent, + microphone_devices_changed, + audio_capture_access: AudioCaptureAccessEvent, /// Video player reports — the same shape, plus the stream's decoded /// dimensions on `.loaded`. Pixels never ride here. video: VideoEvent, @@ -2286,6 +2431,10 @@ pub const Event = union(enum) { .context_menu_action => "context_menu_action", .widget_accessibility_action => "widget_accessibility_action", .audio => "audio", + .audio_capture => "audio_capture", + .microphone_device => "microphone_device", + .microphone_devices_changed => "microphone_devices_changed", + .audio_capture_access => "audio_capture_access", .video => "video", }; } @@ -2461,6 +2610,11 @@ pub const PlatformServices = struct { audio_seek_fn: ?*const fn (context: ?*anyopaque, position_ms: u64) anyerror!void = null, /// Set the player volume, `0.0` (silent) through `1.0` (full). audio_set_volume_fn: ?*const fn (context: ?*anyopaque, volume: f32) anyerror!void = null, + audio_capture_start_fn: ?*const fn (context: ?*anyopaque, config: AudioCaptureConfig, sink: AudioCaptureSink) anyerror!void = null, + audio_capture_stop_fn: ?*const fn (context: ?*anyopaque) anyerror!void = null, + microphone_devices_fn: ?*const fn (context: ?*anyopaque) anyerror!void = null, + audio_capture_access_fn: ?*const fn (context: ?*anyopaque, source: AudioCaptureAccessSource, action: AudioCaptureAccessAction) anyerror!void = null, + microphone_devices_observe_fn: ?*const fn (context: ?*anyopaque, enabled: bool) anyerror!void = null, /// Load a local video file into THE app's single video player, /// leaving it PAUSED at position zero (transport is a separate /// verb, exactly like audio). Loading replaces whatever was loaded @@ -3064,6 +3218,31 @@ pub const PlatformServices = struct { return volume_fn(self.context, volume); } + pub fn audioCaptureStart(self: PlatformServices, config: AudioCaptureConfig, sink: AudioCaptureSink) anyerror!void { + const start_fn = self.audio_capture_start_fn orelse return error.UnsupportedService; + return start_fn(self.context, config, sink); + } + + pub fn audioCaptureStop(self: PlatformServices) anyerror!void { + const stop_fn = self.audio_capture_stop_fn orelse return error.UnsupportedService; + return stop_fn(self.context); + } + + pub fn microphoneDevices(self: PlatformServices) anyerror!void { + const list_fn = self.microphone_devices_fn orelse return error.UnsupportedService; + return list_fn(self.context); + } + + pub fn audioCaptureAccess(self: PlatformServices, source: AudioCaptureAccessSource, action: AudioCaptureAccessAction) anyerror!void { + const access_fn = self.audio_capture_access_fn orelse return error.UnsupportedService; + return access_fn(self.context, source, action); + } + + pub fn observeMicrophoneDevices(self: PlatformServices, enabled: bool) anyerror!void { + const observe_fn = self.microphone_devices_observe_fn orelse return error.UnsupportedService; + return observe_fn(self.context, enabled); + } + /// Load a local video file into the app's single video player (see /// `video_load_fn`). Platforms without video playback answer /// `error.UnsupportedService`; bad arguments are rejected here @@ -3303,6 +3482,9 @@ fn defaultSupportsFeature(services: PlatformServices, feature: PlatformFeature) // cannot see it; platforms that analyze answer through their own // `supports_fn` (like file_drops and gpu_surfaces above). .audio_spectrum => false, + .system_audio_capture => services.audio_capture_start_fn != null, + .microphone_capture => services.audio_capture_start_fn != null, + .microphone_device_enumeration => services.microphone_devices_fn != null, // Hide-on-close is host close-delegate behavior, not a service // verb: hosts that implement it answer through their own // `supports_fn`. The generic floor is honest refusal. diff --git a/src/platform/windows/root.zig b/src/platform/windows/root.zig index 34455422d..99c36b9c6 100644 --- a/src/platform/windows/root.zig +++ b/src/platform/windows/root.zig @@ -486,6 +486,7 @@ pub const WindowsPlatform = struct { .audio_playback, .audio_streaming, => self.web_engine == .system, + .system_audio_capture, .microphone_capture, .microphone_device_enumeration => false, // close_policy .hide: WM_CLOSE hides (ShowWindow SW_HIDE), // the window stays in the host map, and the tray is the // ONLY re-show affordance — SW_HIDE removes the taskbar diff --git a/src/primitives/app_manifest/root.zig b/src/primitives/app_manifest/root.zig index edd7cbd45..81184e6f2 100644 --- a/src/primitives/app_manifest/root.zig +++ b/src/primitives/app_manifest/root.zig @@ -28,6 +28,7 @@ pub const max_file_associations = types.max_file_associations; pub const max_file_association_extensions = types.max_file_association_extensions; pub const max_file_association_mime_types = types.max_file_association_mime_types; pub const max_url_schemes = types.max_url_schemes; +pub const max_privacy_usage_bytes = types.max_privacy_usage_bytes; pub const Platform = types.Platform; pub const PackageKind = types.PackageKind; pub const WebEngine = types.WebEngine; @@ -36,6 +37,7 @@ pub const CefConfig = types.CefConfig; pub const IconPurpose = types.IconPurpose; pub const PermissionKind = types.PermissionKind; pub const Permission = types.Permission; +pub const PrivacyUsage = types.PrivacyUsage; pub const CapabilityKind = types.CapabilityKind; pub const Capability = types.Capability; pub const AppIdentity = types.AppIdentity; @@ -101,6 +103,7 @@ pub const validateName = validation.validateName; pub const validateUrl = validation.validateUrl; pub const validateIcons = validation.validateIcons; pub const validatePermissions = validation.validatePermissions; +pub const validatePrivacy = validation.validatePrivacy; pub const validateCapabilities = validation.validateCapabilities; pub const validateBridge = validation.validateBridge; pub const validateFrontend = validation.validateFrontend; diff --git a/src/primitives/app_manifest/tests.zig b/src/primitives/app_manifest/tests.zig index 3a2c6a4d9..05457c6e1 100644 --- a/src/primitives/app_manifest/tests.zig +++ b/src/primitives/app_manifest/tests.zig @@ -30,6 +30,7 @@ const CefConfig = types.CefConfig; const IconPurpose = types.IconPurpose; const PermissionKind = types.PermissionKind; const Permission = types.Permission; +const PrivacyUsage = types.PrivacyUsage; const CapabilityKind = types.CapabilityKind; const Capability = types.Capability; const AppIdentity = types.AppIdentity; @@ -87,6 +88,7 @@ const validateDescription = validation.validateDescription; const validateUrl = validation.validateUrl; const validateIcons = validation.validateIcons; const validatePermissions = validation.validatePermissions; +const validatePrivacy = validation.validatePrivacy; const validateCapabilities = validation.validateCapabilities; const validateBridge = validation.validateBridge; const validateFrontend = validation.validateFrontend; @@ -690,6 +692,29 @@ test "permission validation catches duplicates" { try std.testing.expectError(error.InvalidName, validatePermissions(&.{.{ .custom = "bad/name" }})); } +test "audio permissions require valid privacy purpose strings" { + try validatePrivacy(.{ + .microphone_usage = "Record your voice in meeting notes.", + .system_audio_usage = "Record meeting audio for transcription.", + }, &.{ .microphone, .system_audio }); + + try std.testing.expectError(error.InvalidPrivacyUsage, validatePrivacy(.{}, &.{.microphone})); + try std.testing.expectError(error.InvalidPrivacyUsage, validatePrivacy(.{}, &.{.system_audio})); + try std.testing.expectError(error.InvalidPrivacyUsage, validatePrivacy(.{ .microphone_usage = " " }, &.{.microphone})); + try std.testing.expectError(error.InvalidPrivacyUsage, validatePrivacy(.{ .system_audio_usage = "bad\nvalue" }, &.{.system_audio})); + + const manifest: Manifest = .{ + .identity = .{ .id = "com.example.recorder", .name = "recorder" }, + .version = .{ .major = 1, .minor = 0, .patch = 0 }, + .permissions = &.{ .microphone, .system_audio }, + .privacy = PrivacyUsage{ + .microphone_usage = "Record your voice in meeting notes.", + .system_audio_usage = "Record meeting audio for transcription.", + }, + }; + try validateManifest(manifest); +} + test "platform validation catches duplicates and invalid overrides" { try validatePlatforms(&.{ .{ .platform = .macos, .id_override = "com.example.app.macos" }, .{ .platform = .linux } }); diff --git a/src/primitives/app_manifest/types.zig b/src/primitives/app_manifest/types.zig index 21bc79bcc..5d8f57bfa 100644 --- a/src/primitives/app_manifest/types.zig +++ b/src/primitives/app_manifest/types.zig @@ -8,6 +8,7 @@ pub const ValidationError = error{ InvalidDimension, DuplicateIcon, DuplicatePermission, + InvalidPrivacyUsage, DuplicateCapability, DuplicateBridgeCommand, DuplicateCommand, @@ -61,6 +62,7 @@ pub const max_file_association_mime_types: usize = 32; pub const max_url_schemes: usize = 32; /// Cap for the identity `description` — one sentence, not a README. pub const max_description_bytes: usize = 256; +pub const max_privacy_usage_bytes: usize = 1024; pub const Platform = enum { macos, @@ -113,6 +115,7 @@ pub const PermissionKind = enum { filesystem, camera, microphone, + system_audio, location, notifications, clipboard, @@ -129,6 +132,7 @@ pub const Permission = union(PermissionKind) { filesystem: void, camera: void, microphone: void, + system_audio: void, location: void, notifications: void, clipboard: void, @@ -209,6 +213,15 @@ pub const AppIdentity = struct { homepage: ?[]const u8 = null, }; +/// Human-facing purpose strings copied into platform privacy metadata. +/// They are separate from permission declarations: permissions say what an +/// app may use, while these strings explain that use to the person granting +/// operating-system access. +pub const PrivacyUsage = struct { + microphone_usage: ?[]const u8 = null, + system_audio_usage: ?[]const u8 = null, +}; + pub const Version = struct { major: u32, minor: u32, @@ -630,6 +643,7 @@ pub const Manifest = struct { version: Version, icons: []const Icon = &.{}, permissions: []const Permission = &.{}, + privacy: PrivacyUsage = .{}, capabilities: []const Capability = &.{}, bridge: BridgeConfig = .{}, frontend: ?FrontendConfig = null, diff --git a/src/primitives/app_manifest/validation.zig b/src/primitives/app_manifest/validation.zig index a218f1348..71915fef4 100644 --- a/src/primitives/app_manifest/validation.zig +++ b/src/primitives/app_manifest/validation.zig @@ -23,6 +23,7 @@ const max_file_associations = types.max_file_associations; const max_file_association_extensions = types.max_file_association_extensions; const max_file_association_mime_types = types.max_file_association_mime_types; const max_url_schemes = types.max_url_schemes; +const max_privacy_usage_bytes = types.max_privacy_usage_bytes; const Platform = types.Platform; const PackageKind = types.PackageKind; const WebEngine = types.WebEngine; @@ -30,6 +31,7 @@ const CefConfig = types.CefConfig; const IconPurpose = types.IconPurpose; const PermissionKind = types.PermissionKind; const Permission = types.Permission; +const PrivacyUsage = types.PrivacyUsage; const CapabilityKind = types.CapabilityKind; const Capability = types.Capability; const AppIdentity = types.AppIdentity; @@ -73,6 +75,7 @@ pub fn validateManifest(manifest: Manifest) ValidationError!void { try validateVersion(manifest.version); try validateIcons(manifest.icons); try validatePermissions(manifest.permissions); + try validatePrivacy(manifest.privacy, manifest.permissions); try validateCapabilities(manifest.capabilities); try validateBridge(manifest.bridge); if (manifest.frontend) |frontend| try validateFrontend(frontend); @@ -532,6 +535,24 @@ pub fn validatePermissions(permissions: []const Permission) ValidationError!void } } +pub fn validatePrivacy(privacy: PrivacyUsage, permissions: []const Permission) ValidationError!void { + if (privacy.microphone_usage) |usage| try validatePrivacyUsage(usage); + if (privacy.system_audio_usage) |usage| try validatePrivacyUsage(usage); + + for (permissions) |permission| switch (permission) { + .microphone => if (privacy.microphone_usage == null) return error.InvalidPrivacyUsage, + .system_audio => if (privacy.system_audio_usage == null) return error.InvalidPrivacyUsage, + else => {}, + }; +} + +fn validatePrivacyUsage(usage: []const u8) ValidationError!void { + if (usage.len == 0 or usage.len > max_privacy_usage_bytes or std.mem.trim(u8, usage, " ").len == 0) return error.InvalidPrivacyUsage; + for (usage) |ch| { + if (ch < 0x20 or ch == 0x7f) return error.InvalidPrivacyUsage; + } +} + pub fn validateCapabilities(capabilities: []const Capability) ValidationError!void { for (capabilities, 0..) |capability, i| { if (capability == .custom) try validateName(capability.custom); diff --git a/src/root.zig b/src/root.zig index 8b057ea75..223b844e5 100644 --- a/src/root.zig +++ b/src/root.zig @@ -75,6 +75,19 @@ pub const max_effect_timers = runtime.max_effect_timers; pub const EffectAudio = runtime.EffectAudio; pub const EffectAudioEventKind = runtime.EffectAudioEventKind; pub const EffectAudioSource = runtime.EffectAudioSource; +pub const EffectAudioCapture = runtime.EffectAudioCapture; +pub const EffectAudioCaptureState = runtime.EffectAudioCaptureState; +pub const EffectAudioCaptureReason = runtime.EffectAudioCaptureReason; +pub const EffectAudioCaptureRead = runtime.EffectAudioCaptureRead; +pub const EffectAudioCaptureReadState = runtime.EffectAudioCaptureReadState; +pub const EffectAudioCaptureReadReason = runtime.EffectAudioCaptureReadReason; +pub const EffectMicrophoneDevice = runtime.EffectMicrophoneDevice; +pub const EffectMicrophoneDeviceState = runtime.EffectMicrophoneDeviceState; +pub const EffectAudioCaptureAccess = runtime.EffectAudioCaptureAccess; +pub const EffectAudioCaptureAccessSource = runtime.EffectAudioCaptureAccessSource; +pub const EffectAudioCaptureAccessAction = runtime.EffectAudioCaptureAccessAction; +pub const EffectAudioCaptureAccessStatus = runtime.EffectAudioCaptureAccessStatus; +pub const MicrophoneSelection = runtime.MicrophoneSelection; pub const audioCachePath = runtime.audioCachePath; pub const max_effect_audio_path_bytes = runtime.max_effect_audio_path_bytes; pub const EffectVideo = runtime.EffectVideo; diff --git a/src/runtime/api.zig b/src/runtime/api.zig index 770098ef1..29a9a4ddd 100644 --- a/src/runtime/api.zig +++ b/src/runtime/api.zig @@ -331,6 +331,10 @@ pub const Event = union(enum) { /// tick, completion, failure): the ui-app layer routes it back /// through `Effects.takeAudioMsg` into the app's `on_event` Msg. audio: platform.AudioEvent, + audio_capture: platform.AudioCaptureEvent, + microphone_device: platform.MicrophoneDeviceEvent, + microphone_devices_changed, + audio_capture_access: platform.AudioCaptureAccessEvent, /// A platform video player report (load acknowledgment with /// dimensions, position tick, completion, failure): routed back /// through `Effects.takeVideoMsg` into the app's `on_event` Msg. @@ -366,6 +370,10 @@ pub const Event = union(enum) { .timer => "timer", .effects_wake => "effects_wake", .audio => "audio", + .audio_capture => "audio_capture", + .microphone_device => "microphone_device", + .microphone_devices_changed => "microphone_devices_changed", + .audio_capture_access => "audio_capture_access", .video => "video", .files_dropped => "files_dropped", .gpu_surface_frame => "gpu_surface_frame", diff --git a/src/runtime/bridge_payload.zig b/src/runtime/bridge_payload.zig index 0ac11d31d..e9a797f40 100644 --- a/src/runtime/bridge_payload.zig +++ b/src/runtime/bridge_payload.zig @@ -125,6 +125,9 @@ pub fn platformFeatureFromString(value: []const u8) ?platform.PlatformFeature { if (std.mem.eql(u8, value, "audioPlayback")) return .audio_playback; if (std.mem.eql(u8, value, "audioStreaming")) return .audio_streaming; if (std.mem.eql(u8, value, "audioSpectrum")) return .audio_spectrum; + if (std.mem.eql(u8, value, "systemAudioCapture")) return .system_audio_capture; + if (std.mem.eql(u8, value, "microphoneCapture")) return .microphone_capture; + if (std.mem.eql(u8, value, "microphoneDeviceEnumeration")) return .microphone_device_enumeration; if (std.mem.eql(u8, value, "windowHideOnClose")) return .window_hide_on_close; if (std.mem.eql(u8, value, "videoPlayback")) return .video_playback; return null; diff --git a/src/runtime/effects.zig b/src/runtime/effects.zig index 2ab31881f..d00f4ef11 100644 --- a/src/runtime/effects.zig +++ b/src/runtime/effects.zig @@ -651,6 +651,81 @@ pub const EffectAudioSource = enum(u8) { stream, }; +pub const EffectAudioCaptureState = platform.AudioCaptureEventState; +pub const EffectAudioCaptureReason = platform.AudioCaptureEventReason; +pub const EffectMicrophoneDeviceState = platform.MicrophoneDeviceEventState; +pub const EffectAudioCaptureAccessSource = platform.AudioCaptureAccessSource; +pub const EffectAudioCaptureAccessAction = platform.AudioCaptureAccessAction; +pub const EffectAudioCaptureAccessStatus = platform.AudioCaptureAccessStatus; + +pub const EffectAudioCapture = struct { + key: u64, + state: EffectAudioCaptureState, + reason: EffectAudioCaptureReason = .none, + sample_rate_hz: u32 = 0, + channel_count: u8 = 0, + available_frames: u32 = 0, + capacity_frames: u32 = 0, + frames_produced: u64 = 0, +}; + +pub const EffectAudioCaptureReadState = enum(u8) { + chunk, + empty, + ended, + rejected, +}; + +pub const EffectAudioCaptureReadReason = enum(u8) { + none, + invalid_options, + not_recording, + read_in_progress, +}; + +/// One borrowed, timestamp-aligned read from the reliable capture stream. +/// Both PCM slices are signed 16-bit little-endian and cover the same frame +/// interval. Copy a slice during update only when retaining it in the model. +pub const EffectAudioCaptureRead = struct { + key: u64, + state: EffectAudioCaptureReadState, + reason: EffectAudioCaptureReadReason = .none, + sequence: u64 = 0, + frame_offset: u64 = 0, + frames: u32 = 0, + system_pcm: []const u8 = &.{}, + microphone_pcm: []const u8 = &.{}, + system_gap_frames: u32 = 0, + microphone_gap_frames: u32 = 0, + remaining_frames: u32 = 0, + end_of_stream: bool = false, +}; + +/// Strings borrow platform event storage and must be copied before the +/// update callback returns when an app wants to retain them in its model. +pub const EffectMicrophoneDevice = struct { + key: u64, + state: EffectMicrophoneDeviceState, + id: []const u8 = &.{}, + name: []const u8 = &.{}, + is_default: bool = false, + index: u32 = 0, + total: u32 = 0, +}; + +pub const EffectAudioCaptureAccess = struct { + key: u64, + source: EffectAudioCaptureAccessSource, + status: EffectAudioCaptureAccessStatus, + restart_required: bool = false, +}; + +pub const MicrophoneSelection = union(enum) { + none, + default, + device_id: []const u8, +}; + /// Longest video source string (path or url) `loadVideo` accepts, /// mirroring the platform bound. Longer strings deliver exactly one /// `.rejected` video event Msg. @@ -1743,6 +1818,15 @@ pub const EffectResultKind = enum(u8) { /// opposite of the channel records' inline argument. Exit records /// ride `code`/`exit_reason` plus the pty fields. pty = 15, + /// One application-consumed paired audio-capture read. The two PCM + /// fields are concatenated in `payload`; `audio_capture_system_len` + /// splits them without introducing a dynamic object-array ABI. + audio_capture_read = 16, + /// The runtime-generated coalesced `.readable` lifecycle event. Native + /// start/terminal events already ride the platform-event journal, but + /// readiness is produced by the runtime ring and therefore needs this + /// effect record to recreate the pull trigger during replay. + audio_capture = 17, }; /// Journaled wall-clock reads buffered for replay (`Effects.wallMs`). @@ -1887,6 +1971,23 @@ pub const EffectResultRecord = struct { /// when it moves `payload` out of line (the image convention). pty_blob_hash: [effect_image_blob_hash_len]u8 = @splat(0), pty_blob_len: u64 = 0, + audio_capture_read_state: EffectAudioCaptureReadState = .empty, + audio_capture_read_reason: EffectAudioCaptureReadReason = .none, + audio_capture_sequence: u64 = 0, + audio_capture_frame_offset: u64 = 0, + audio_capture_frames: u32 = 0, + audio_capture_system_len: u32 = 0, + audio_capture_system_gap_frames: u32 = 0, + audio_capture_microphone_gap_frames: u32 = 0, + audio_capture_remaining_frames: u32 = 0, + audio_capture_end_of_stream: bool = false, + audio_capture_state: EffectAudioCaptureState = .readable, + audio_capture_reason: EffectAudioCaptureReason = .none, + audio_capture_sample_rate_hz: u32 = 0, + audio_capture_channel_count: u8 = 0, + audio_capture_available_frames: u32 = 0, + audio_capture_capacity_frames: u32 = 0, + audio_capture_frames_produced: u64 = 0, }; /// Type-erased sink the drain reports every delivered result to while a @@ -2253,6 +2354,193 @@ const WorkerThread = if (io_threaded_supported) std.Thread else void; /// context and its buffer, `deinit` releases the executor io. const process_allocator: std.mem.Allocator = std.heap.page_allocator; +const max_audio_capture_read_frames: usize = 48_000 * platform.audio_capture_max_read_duration_ms / 1_000; +const max_audio_capture_read_pcm_bytes: usize = max_audio_capture_read_frames * 2 * 2; + +const AudioCaptureBlock = struct { + frame_offset: u64 = 0, + frame_count: u32 = 0, + system_gap_frames: u32 = 0, + microphone_gap_frames: u32 = 0, +}; + +const AudioCaptureReadBatch = struct { + frame_offset: u64 = 0, + frames: u32 = 0, + system_gap_frames: u32 = 0, + microphone_gap_frames: u32 = 0, +}; + +const AudioCaptureOccupancy = struct { + available_frames: u32 = 0, + capacity_frames: u32 = 0, + frames_produced: u64 = 0, +}; + +const AudioCaptureRing = struct { + blocks: []AudioCaptureBlock, + system_pcm: []u8, + microphone_pcm: []u8, + block_head: usize = 0, + block_len: usize = 0, + frame_head: usize = 0, + available_frames: usize = 0, + capacity_frames: usize, + bytes_per_frame: usize, + system_audio: bool, + microphone_audio: bool, + frames_produced: u64 = 0, + + fn init(allocator: std.mem.Allocator, config: platform.AudioCaptureConfig) !*AudioCaptureRing { + const block_frames = @as(usize, config.sample_rate_hz) * platform.audio_capture_block_duration_ms / 1_000; + const capacity_frames = @as(usize, config.sample_rate_hz) * config.buffer_duration_ms / 1_000; + const block_capacity = (capacity_frames + block_frames - 1) / block_frames; + const bytes_per_frame = @as(usize, config.channel_count) * 2; + const ring = try allocator.create(AudioCaptureRing); + errdefer allocator.destroy(ring); + const blocks = try allocator.alloc(AudioCaptureBlock, block_capacity); + errdefer allocator.free(blocks); + const system_pcm: []u8 = if (config.system_audio) + try allocator.alloc(u8, capacity_frames * bytes_per_frame) + else + @constCast(&.{}); + errdefer if (system_pcm.len > 0) allocator.free(system_pcm); + const microphone_pcm: []u8 = if (config.microphone != .none) + try allocator.alloc(u8, capacity_frames * bytes_per_frame) + else + @constCast(&.{}); + errdefer if (microphone_pcm.len > 0) allocator.free(microphone_pcm); + ring.* = .{ + .blocks = blocks, + .system_pcm = system_pcm, + .microphone_pcm = microphone_pcm, + .capacity_frames = capacity_frames, + .bytes_per_frame = bytes_per_frame, + .system_audio = config.system_audio, + .microphone_audio = config.microphone != .none, + }; + return ring; + } + + fn deinit(ring: *AudioCaptureRing, allocator: std.mem.Allocator) void { + allocator.free(ring.blocks); + if (ring.system_pcm.len > 0) allocator.free(ring.system_pcm); + if (ring.microphone_pcm.len > 0) allocator.free(ring.microphone_pcm); + allocator.destroy(ring); + } + + fn copyInto(storage: []u8, frame_index: usize, bytes_per_frame: usize, source: []const u8) void { + if (storage.len == 0 or source.len == 0) return; + const byte_index = frame_index * bytes_per_frame; + const first = @min(source.len, storage.len - byte_index); + @memcpy(storage[byte_index .. byte_index + first], source[0..first]); + if (first < source.len) @memcpy(storage[0 .. source.len - first], source[first..]); + } + + fn copyOut(storage: []const u8, frame_index: usize, bytes_per_frame: usize, frame_count: usize, destination: []u8) void { + if (storage.len == 0 or destination.len == 0) return; + const byte_index = frame_index * bytes_per_frame; + const byte_count = frame_count * bytes_per_frame; + const first = @min(byte_count, storage.len - byte_index); + @memcpy(destination[0..first], storage[byte_index .. byte_index + first]); + if (first < byte_count) @memcpy(destination[first..byte_count], storage[0 .. byte_count - first]); + } + + fn push(ring: *AudioCaptureRing, pair: platform.AudioCaptureFramePair) platform.AudioCapturePushResult { + const frames: usize = pair.frame_count; + if (frames == 0 or pair.system_gap_frames > pair.frame_count or pair.microphone_gap_frames > pair.frame_count) return .closed; + if (pair.frame_offset != ring.frames_produced) return .closed; + const expected_bytes = frames * ring.bytes_per_frame; + if ((ring.system_audio and pair.system_pcm.len != expected_bytes) or + (!ring.system_audio and pair.system_pcm.len != 0) or + (ring.microphone_audio and pair.microphone_pcm.len != expected_bytes) or + (!ring.microphone_audio and pair.microphone_pcm.len != 0)) return .closed; + if (frames > ring.capacity_frames - ring.available_frames or ring.block_len == ring.blocks.len) return .full; + const tail_frame = (ring.frame_head + ring.available_frames) % ring.capacity_frames; + copyInto(ring.system_pcm, tail_frame, ring.bytes_per_frame, pair.system_pcm); + copyInto(ring.microphone_pcm, tail_frame, ring.bytes_per_frame, pair.microphone_pcm); + const block_index = (ring.block_head + ring.block_len) % ring.blocks.len; + ring.blocks[block_index] = .{ + .frame_offset = pair.frame_offset, + .frame_count = pair.frame_count, + .system_gap_frames = pair.system_gap_frames, + .microphone_gap_frames = pair.microphone_gap_frames, + }; + ring.block_len += 1; + ring.available_frames += frames; + ring.frames_produced = @max(ring.frames_produced, pair.frame_offset + pair.frame_count); + return .accepted; + } + + fn read(ring: *AudioCaptureRing, max_frames: usize, system_destination: []u8, microphone_destination: []u8) AudioCaptureReadBatch { + var batch: AudioCaptureReadBatch = .{}; + while (ring.block_len > 0) { + const block = ring.blocks[ring.block_head]; + if (batch.frames > 0 and @as(usize, batch.frames) + block.frame_count > max_frames) break; + if (batch.frames == 0 and block.frame_count > max_frames) break; + if (batch.frames == 0) batch.frame_offset = block.frame_offset; + const output_byte_offset = @as(usize, batch.frames) * ring.bytes_per_frame; + const block_bytes = @as(usize, block.frame_count) * ring.bytes_per_frame; + copyOut(ring.system_pcm, ring.frame_head, ring.bytes_per_frame, block.frame_count, system_destination[output_byte_offset .. output_byte_offset + if (ring.system_audio) block_bytes else 0]); + copyOut(ring.microphone_pcm, ring.frame_head, ring.bytes_per_frame, block.frame_count, microphone_destination[output_byte_offset .. output_byte_offset + if (ring.microphone_audio) block_bytes else 0]); + batch.frames += block.frame_count; + batch.system_gap_frames += block.system_gap_frames; + batch.microphone_gap_frames += block.microphone_gap_frames; + ring.frame_head = (ring.frame_head + block.frame_count) % ring.capacity_frames; + ring.available_frames -= block.frame_count; + ring.block_head = (ring.block_head + 1) % ring.blocks.len; + ring.block_len -= 1; + } + return batch; + } +}; + +const AudioCaptureOwnerRefs = struct { + seq: *std.atomic.Value(u64), + pending: *std.atomic.Value(usize), +}; + +/// Process-lifetime producer header. The platform may finish an in-flight +/// callback after stop or teardown, so its sink never points at allocator- +/// owned ring storage directly. The generation/open gate is checked while +/// holding the same mutex used to detach and free that ring. +const AudioCaptureShared = struct { + mutex: SpinMutex = .{}, + open: bool = false, + accepting: bool = false, + generation: u64 = 0, + ring: ?*AudioCaptureRing = null, + owner: ?AudioCaptureOwnerRefs = null, + readable_pending: bool = false, + readable_seq: u64 = 0, + wake: ChannelWake = .{}, +}; + +fn audioCapturePush(context: *anyopaque, generation: u64, pair: platform.AudioCaptureFramePair) platform.AudioCapturePushResult { + const shared: *AudioCaptureShared = @ptrCast(@alignCast(context)); + var wake = false; + shared.mutex.lock(); + if (!shared.open or !shared.accepting or shared.generation != generation) { + shared.mutex.unlock(); + return .closed; + } + const ring = shared.ring.?; + const was_empty = ring.available_frames == 0; + const result = ring.push(pair); + if (result == .full) shared.accepting = false; + if (result == .accepted and was_empty and !shared.readable_pending) { + if (shared.owner) |owner| { + shared.readable_seq = owner.seq.fetchAdd(1, .seq_cst); + _ = owner.pending.fetchAdd(1, .seq_cst); + shared.readable_pending = true; + wake = true; + } + } + shared.mutex.unlock(); + if (wake) ChannelHandle.requestHostWake(&shared.wake, generation); + return result; +} + /// Everything a real file worker's BLOCKING phase may touch, held /// out-of-line from the channel on its own heap block. File I/O is the /// one effect teardown cannot always interrupt (a write to a FIFO with @@ -2403,6 +2691,11 @@ pub fn Effects(comptime Msg: type) type { pub const ClipboardMsgFn = *const fn (result: EffectClipboardResult) Msg; pub const TimerMsgFn = *const fn (timer: EffectTimer) Msg; pub const AudioMsgFn = *const fn (event: EffectAudio) Msg; + pub const AudioCaptureMsgFn = *const fn (event: EffectAudioCapture) Msg; + pub const AudioCaptureReadMsgFn = *const fn (event: EffectAudioCaptureRead) Msg; + pub const MicrophoneDeviceMsgFn = *const fn (event: EffectMicrophoneDevice) Msg; + pub const AudioCaptureAccessMsgFn = *const fn (event: EffectAudioCaptureAccess) Msg; + pub const MicrophoneDevicesChangedMsgFn = *const fn () Msg; pub const VideoMsgFn = *const fn (event: EffectVideo) Msg; pub const HostMsgFn = *const fn (result: EffectHostResult) Msg; pub const ImageMsgFn = *const fn (result: EffectImageResult) Msg; @@ -2492,6 +2785,46 @@ pub fn Effects(comptime Msg: type) type { }.make; } + pub fn audioCaptureMsg(comptime tag: std.meta.Tag(Msg)) AudioCaptureMsgFn { + return struct { + fn make(event: EffectAudioCapture) Msg { + return @unionInit(Msg, @tagName(tag), event); + } + }.make; + } + + pub fn audioCaptureReadMsg(comptime tag: std.meta.Tag(Msg)) AudioCaptureReadMsgFn { + return struct { + fn make(event: EffectAudioCaptureRead) Msg { + return @unionInit(Msg, @tagName(tag), event); + } + }.make; + } + + pub fn microphoneDeviceMsg(comptime tag: std.meta.Tag(Msg)) MicrophoneDeviceMsgFn { + return struct { + fn make(event: EffectMicrophoneDevice) Msg { + return @unionInit(Msg, @tagName(tag), event); + } + }.make; + } + + pub fn audioCaptureAccessMsg(comptime tag: std.meta.Tag(Msg)) AudioCaptureAccessMsgFn { + return struct { + fn make(event: EffectAudioCaptureAccess) Msg { + return @unionInit(Msg, @tagName(tag), event); + } + }.make; + } + + pub fn microphoneDevicesChangedMsg(comptime tag: std.meta.Tag(Msg)) MicrophoneDevicesChangedMsgFn { + return struct { + fn make() Msg { + return @unionInit(Msg, @tagName(tag), {}); + } + }.make; + } + /// Comptime Msg constructor for `on_event` of video playback: /// `videoMsg(.video_event)` builds /// `Msg{ .video_event = event }` — the variant's payload type @@ -2794,6 +3127,40 @@ pub fn Effects(comptime Msg: type) type { on_event: ?AudioMsgFn = null, }; + pub const StartAudioCaptureOptions = struct { + /// App-owned route key. Only one capture may be active. + key: u64, + system_audio: bool = false, + microphone: MicrophoneSelection = .none, + sample_rate_hz: u32 = 48_000, + channel_count: u8 = 2, + exclude_current_process_audio: bool = true, + /// Runtime ring duration. Accepted PCM is never overwritten; + /// a full ring terminates capture with `consumer_too_slow`. + buffer_duration_ms: u32 = platform.default_audio_capture_buffer_duration_ms, + on_event: ?AudioCaptureMsgFn = null, + }; + + pub const ReadAudioCaptureOptions = struct { + key: u64, + /// Must hold at least one 20 ms block and at most 100 ms at + /// the capture's configured sample rate. + max_frames: u32, + on_read: ?AudioCaptureReadMsgFn = null, + }; + + pub const ListMicrophoneDevicesOptions = struct { + key: u64, + on_event: ?MicrophoneDeviceMsgFn = null, + }; + + pub const AudioCaptureAccessOptions = struct { + key: u64, + source: EffectAudioCaptureAccessSource, + action: EffectAudioCaptureAccessAction = .status, + on_event: ?AudioCaptureAccessMsgFn = null, + }; + pub const LoadImageOptions = struct { /// The ImageId the decoded pixels register under — model- /// owned, chosen by the app, exactly the id `image`/`avatar` @@ -3132,6 +3499,38 @@ pub fn Effects(comptime Msg: type) type { } }; + const AudioCaptureChannel = struct { + active: bool = false, + accepting: bool = false, + sealed: bool = false, + fake: bool = false, + key: u64 = 0, + generation: u64 = 0, + sample_rate_hz: u32 = 0, + channel_count: u8 = 0, + system_audio: bool = false, + microphone_audio: bool = false, + read_pending: bool = false, + pending_read_fn: ?AudioCaptureReadMsgFn = null, + release_after_delivery: bool = false, + read_sequence: u64 = 0, + on_event: ?AudioCaptureMsgFn = null, + system_scratch: [max_audio_capture_read_pcm_bytes]u8 = undefined, + microphone_scratch: [max_audio_capture_read_pcm_bytes]u8 = undefined, + }; + + const MicrophoneDeviceQuery = struct { + active: bool = false, + key: u64 = 0, + on_event: ?MicrophoneDeviceMsgFn = null, + }; + + const AudioCaptureAccessQuery = struct { + active: bool = false, + key: u64 = 0, + on_event: ?AudioCaptureAccessMsgFn = null, + }; + /// Playback state the automation snapshot exposes: honest — it /// reports what the platform has told us, not what the UI wishes. pub const AudioSnapshot = struct { @@ -3436,6 +3835,11 @@ pub fn Effects(comptime Msg: type) type { /// `takeAudioMsg`. Non-resolving entries (rejections and /// synchronous failures) are fully formed at enqueue. audio: struct { event: EffectAudio, audio_fn: ?AudioMsgFn, resolve: bool }, + audio_capture: struct { event: EffectAudioCapture, capture_fn: ?AudioCaptureMsgFn }, + audio_capture_read: struct { event: EffectAudioCaptureRead, read_fn: ?AudioCaptureReadMsgFn }, + microphone_device: struct { event: EffectMicrophoneDevice, device_fn: ?MicrophoneDeviceMsgFn }, + audio_capture_access: struct { event: EffectAudioCaptureAccess, access_fn: ?AudioCaptureAccessMsgFn }, + microphone_devices_changed: struct { changed_fn: ?MicrophoneDevicesChangedMsgFn }, /// The audio entry's shape for the video channel, staged /// in the non-lossy `pending_videos` (see `PendingVideo`) /// and taking this union shape only at drain time. @@ -3493,6 +3897,7 @@ pub fn Effects(comptime Msg: type) type { // EffectAudio carries no drop counter either; the // next position tick supersedes a lost one. .audio => {}, + .audio_capture, .audio_capture_read, .microphone_device, .audio_capture_access, .microphone_devices_changed => {}, // Video events never enter the ring (they stage in // the non-lossy `pending_videos`): a loop-side // `.rejected`/`.failed` is its load's only @@ -3532,6 +3937,7 @@ pub fn Effects(comptime Msg: type) type { .clipboard => |entry| entry.result.dropped_before, .timer => 0, .audio => 0, + .audio_capture, .audio_capture_read, .microphone_device, .audio_capture_access, .microphone_devices_changed => 0, .pty => 0, .host => 0, // Never in the ring; see `addDropped`. @@ -4254,6 +4660,16 @@ pub fn Effects(comptime Msg: type) type { /// The single audio playback channel (see `AudioChannel`). /// Loop-thread only, like the timer table. audio: AudioChannel = .{}, + audio_capture: AudioCaptureChannel = .{}, + /// Process-lifetime producer header reused across capture generations. + audio_capture_shared: ?*AudioCaptureShared = null, + audio_capture_generation: u64 = 0, + audio_capture_seq: std.atomic.Value(u64) = std.atomic.Value(u64).init(0), + audio_capture_pending_count: std.atomic.Value(usize) = std.atomic.Value(usize).init(0), + audio_capture_journal_scratch: [max_audio_capture_read_pcm_bytes * 2]u8 = undefined, + microphone_device_query: MicrophoneDeviceQuery = .{}, + audio_capture_access_query: AudioCaptureAccessQuery = .{}, + microphone_devices_changed_fn: ?MicrophoneDevicesChangedMsgFn = null, /// The single video playback channel (see `VideoChannel`). video: VideoChannel = .{}, /// Monotonic per-load video token mint (see @@ -4485,6 +4901,26 @@ pub fn Effects(comptime Msg: type) type { if (self.services) |services| services.audioStop() catch {}; } self.audio = .{}; + if (self.audio_capture.active and !self.audio_capture.fake) { + if (self.services) |services| services.audioCaptureStop() catch {}; + } + if (self.microphone_devices_changed_fn != null) { + if (self.services) |services| services.observeMicrophoneDevices(false) catch {}; + } + self.releaseAudioCaptureStream(); + // `releaseAudioCaptureStream` performs the non-blocking revoke + // needed by ordinary mid-life closes. Teardown additionally + // fences an already-running host wake before the services and + // platform it captured can be destroyed. + if (self.audio_capture_shared) |shared| { + if (!quiesceChannelWake(&shared.wake, self.channel_wake_join_deadline_ms)) { + self.abandoned_channel_wakes += 1; + if (self.services) |services| services.noteChannelWakeAbandoned(); + } + } + self.microphone_device_query = .{}; + self.audio_capture_access_query = .{}; + self.microphone_devices_changed_fn = null; // Stop the platform video player (best effort), release the // media-surface claim, and clear the channel. if (self.video.active and !self.video.fake) { @@ -7824,6 +8260,474 @@ pub fn Effects(comptime Msg: type) type { services.audioSetVolume(clamped) catch {}; } + pub fn startAudioCapture(self: *Self, options: StartAudioCaptureOptions) void { + var microphone_kind: platform.MicrophoneSelectionKind = .none; + var microphone_id: []const u8 = &.{}; + switch (options.microphone) { + .none => {}, + .default => microphone_kind = .default, + .device_id => |id| { + microphone_kind = .device_id; + microphone_id = id; + }, + } + const valid_rate = options.sample_rate_hz == 16_000 or options.sample_rate_hz == 24_000 or + options.sample_rate_hz == 44_100 or options.sample_rate_hz == 48_000; + const rejected = (!options.system_audio and microphone_kind == .none) or + (microphone_kind == .device_id and (microphone_id.len == 0 or microphone_id.len > platform.max_microphone_device_id_bytes)) or + !valid_rate or (options.channel_count != 1 and options.channel_count != 2) or + options.buffer_duration_ms < platform.min_audio_capture_buffer_duration_ms or + options.buffer_duration_ms > platform.max_audio_capture_buffer_duration_ms; + if (rejected) { + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = options.key, + .state = .rejected, + .reason = .invalid_options, + }, .capture_fn = options.on_event } }); + return; + } + if (self.audio_capture.active) { + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = options.key, + .state = .rejected, + .reason = .already_recording, + }, .capture_fn = options.on_event } }); + return; + } + const config: platform.AudioCaptureConfig = .{ + .system_audio = options.system_audio, + .microphone = microphone_kind, + .microphone_device_id = microphone_id, + .sample_rate_hz = options.sample_rate_hz, + .channel_count = options.channel_count, + .exclude_current_process_audio = options.exclude_current_process_audio, + .buffer_duration_ms = options.buffer_duration_ms, + }; + const ring = AudioCaptureRing.init(self.allocator, config) catch { + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = options.key, + .state = .rejected, + .reason = .capture_failed, + }, .capture_fn = options.on_event } }); + return; + }; + const shared = self.audio_capture_shared orelse create: { + const allocated = process_allocator.create(AudioCaptureShared) catch { + ring.deinit(self.allocator); + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = options.key, + .state = .rejected, + .reason = .capture_failed, + }, .capture_fn = options.on_event } }); + return; + }; + allocated.* = .{}; + self.audio_capture_shared = allocated; + break :create allocated; + }; + self.audio_capture_generation +%= 1; + if (self.audio_capture_generation == 0) self.audio_capture_generation = 1; + const generation = self.audio_capture_generation; + shared.mutex.lock(); + shared.open = true; + shared.accepting = true; + shared.generation = generation; + shared.ring = ring; + shared.owner = .{ .seq = &self.audio_capture_seq, .pending = &self.audio_capture_pending_count }; + shared.readable_pending = false; + shared.mutex.unlock(); + shared.wake.mutex.lock(); + shared.wake.generation = generation; + shared.wake.services = &self.wake_services; + shared.wake.pending.store(false, .seq_cst); + shared.wake.mutex.unlock(); + self.audio_capture = .{ + .active = true, + .accepting = true, + .fake = self.executor == .fake, + .key = options.key, + .generation = generation, + .sample_rate_hz = options.sample_rate_hz, + .channel_count = options.channel_count, + .system_audio = options.system_audio, + .microphone_audio = microphone_kind != .none, + .on_event = options.on_event, + }; + if (self.audio_capture.fake) { + // Session replay parks the request until the recorded + // platform events arrive. Ordinary fake-executor tests + // keep their deterministic synthetic lifecycle. + if (self.replay) return; + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = options.key, + .state = .started, + .sample_rate_hz = options.sample_rate_hz, + .channel_count = options.channel_count, + .capacity_frames = @intCast(ring.capacity_frames), + }, .capture_fn = options.on_event } }); + return; + } + const services = self.services orelse return self.rejectAudioCapture(.unsupported); + services.audioCaptureStart(config, .{ + .context = shared, + .generation = generation, + .push_fn = audioCapturePush, + }) catch |err| return self.rejectAudioCapture(switch (err) { + error.UnsupportedService => .unsupported, + error.PermissionMissing => .permission_missing, + error.AudioCapturePermissionRequired => .permission_required, + error.AudioCaptureAlreadyActive => .already_recording, + error.MicrophoneDeviceNotFound => .device_not_found, + error.InvalidAudioCaptureOptions => .invalid_options, + else => .capture_failed, + }); + } + + pub fn stopAudioCapture(self: *Self) void { + if (!self.audio_capture.active or self.audio_capture.sealed) return; + const key = self.audio_capture.key; + const on_event = self.audio_capture.on_event; + self.audio_capture.accepting = false; + self.closeAudioCaptureProducer(); + if (self.audio_capture.fake) { + // The recorded terminal event is authoritative during + // replay; keep the parked route alive until it arrives. + if (self.replay) return; + self.audio_capture.sealed = true; + const occupancy = self.audioCaptureOccupancy(); + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = key, + .state = .stopped, + .sample_rate_hz = self.audio_capture.sample_rate_hz, + .channel_count = self.audio_capture.channel_count, + .available_frames = occupancy.available_frames, + .capacity_frames = occupancy.capacity_frames, + .frames_produced = occupancy.frames_produced, + }, .capture_fn = on_event } }); + return; + } + const services = self.services orelse return self.failAudioCapture(.unsupported); + services.audioCaptureStop() catch return self.failAudioCapture(.capture_failed); + } + + pub fn readAudioCapture(self: *Self, options: ReadAudioCaptureOptions) void { + if (!self.audio_capture.active or self.audio_capture.key != options.key) { + self.deliverPending(.{ .audio_capture_read = .{ .event = .{ + .key = options.key, + .state = .rejected, + .reason = .not_recording, + }, .read_fn = options.on_read } }); + return; + } + if (self.audio_capture.read_pending) { + self.deliverPending(.{ .audio_capture_read = .{ .event = .{ + .key = options.key, + .state = .rejected, + .reason = .read_in_progress, + }, .read_fn = options.on_read } }); + return; + } + const block_frames = self.audio_capture.sample_rate_hz * platform.audio_capture_block_duration_ms / 1_000; + const max_frames = self.audio_capture.sample_rate_hz * platform.audio_capture_max_read_duration_ms / 1_000; + if (options.max_frames < block_frames or options.max_frames > max_frames) { + self.deliverPending(.{ .audio_capture_read = .{ .event = .{ + .key = options.key, + .state = .rejected, + .reason = .invalid_options, + }, .read_fn = options.on_read } }); + return; + } + if (self.replay) { + self.audio_capture.read_pending = true; + self.audio_capture.pending_read_fn = options.on_read; + return; + } + const shared = self.audio_capture_shared.?; + shared.mutex.lock(); + const ring = shared.ring.?; + const batch = ring.read(options.max_frames, &self.audio_capture.system_scratch, &self.audio_capture.microphone_scratch); + const remaining: u32 = @intCast(ring.available_frames); + if (ring.available_frames == 0 and shared.readable_pending) { + shared.readable_pending = false; + if (shared.owner) |owner| _ = owner.pending.fetchSub(1, .seq_cst); + } + const sealed = self.audio_capture.sealed; + const system_bytes = @as(usize, batch.frames) * @as(usize, self.audio_capture.channel_count) * 2; + const microphone_bytes = system_bytes; + const event: EffectAudioCaptureRead = .{ + .key = options.key, + .state = if (batch.frames > 0) .chunk else if (sealed) .ended else .empty, + .sequence = self.audio_capture.read_sequence, + .frame_offset = batch.frame_offset, + .frames = batch.frames, + .system_pcm = if (self.audio_capture.system_audio) self.audio_capture.system_scratch[0..system_bytes] else &.{}, + .microphone_pcm = if (self.audio_capture.microphone_audio) self.audio_capture.microphone_scratch[0..microphone_bytes] else &.{}, + .system_gap_frames = batch.system_gap_frames, + .microphone_gap_frames = batch.microphone_gap_frames, + .remaining_frames = remaining, + .end_of_stream = sealed and remaining == 0, + }; + shared.mutex.unlock(); + self.audio_capture.read_sequence +%= 1; + self.audio_capture.read_pending = true; + self.audio_capture.pending_read_fn = options.on_read; + self.deliverPending(.{ .audio_capture_read = .{ .event = event, .read_fn = options.on_read } }); + } + + pub fn discardAudioCapture(self: *Self) void { + if (!self.audio_capture.active) return; + const key = self.audio_capture.key; + const on_event = self.audio_capture.on_event; + if (!self.audio_capture.fake and !self.audio_capture.sealed) { + if (self.services) |services| services.audioCaptureStop() catch {}; + } + self.releaseAudioCaptureStream(); + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = key, + .state = .stopped, + .reason = .discarded, + }, .capture_fn = on_event } }); + } + + pub fn listMicrophoneDevices(self: *Self, options: ListMicrophoneDevicesOptions) void { + if (self.microphone_device_query.active) { + self.deliverPending(.{ .microphone_device = .{ .event = .{ + .key = options.key, + .state = .rejected, + }, .device_fn = options.on_event } }); + return; + } + self.microphone_device_query = .{ .active = true, .key = options.key, .on_event = options.on_event }; + if (self.executor == .fake) { + if (self.replay) return; + self.deliverPending(.{ .microphone_device = .{ .event = .{ .key = options.key, .state = .device, .id = "default-mic", .name = "Default Microphone", .is_default = true, .index = 0, .total = 2 }, .device_fn = options.on_event } }); + self.deliverPending(.{ .microphone_device = .{ .event = .{ .key = options.key, .state = .device, .id = "usb-mic", .name = "USB Microphone", .index = 1, .total = 2 }, .device_fn = options.on_event } }); + self.deliverPending(.{ .microphone_device = .{ .event = .{ .key = options.key, .state = .completed, .index = 2, .total = 2 }, .device_fn = options.on_event } }); + self.microphone_device_query = .{}; + return; + } + const services = self.services orelse return self.finishMicrophoneDevices(.rejected); + services.microphoneDevices() catch |err| return self.finishMicrophoneDevices(if (err == error.UnsupportedService or err == error.PermissionMissing) .rejected else .failed); + } + + pub fn audioCaptureAccess(self: *Self, options: AudioCaptureAccessOptions) void { + if (self.audio_capture_access_query.active) { + self.deliverPending(.{ .audio_capture_access = .{ .event = .{ + .key = options.key, + .source = options.source, + .status = .unavailable, + }, .access_fn = options.on_event } }); + return; + } + self.audio_capture_access_query = .{ .active = true, .key = options.key, .on_event = options.on_event }; + if (self.executor == .fake) { + if (self.replay) return; + self.deliverPending(.{ .audio_capture_access = .{ .event = .{ + .key = options.key, + .source = options.source, + .status = .authorized, + }, .access_fn = options.on_event } }); + self.audio_capture_access_query = .{}; + return; + } + const services = self.services orelse return self.failAudioCaptureAccess(options.source); + services.audioCaptureAccess(options.source, options.action) catch return self.failAudioCaptureAccess(options.source); + } + + pub fn observeMicrophoneDevices(self: *Self, on_change: ?MicrophoneDevicesChangedMsgFn) void { + self.microphone_devices_changed_fn = on_change; + if (self.executor == .fake) return; + const services = self.services orelse return; + services.observeMicrophoneDevices(on_change != null) catch {}; + } + + pub fn takeAudioCaptureMsg(self: *Self, event: platform.AudioCaptureEvent) ?Msg { + if (!self.audio_capture.active) return null; + // As with audio/video playback, the journaled effect record is + // the app-visible truth during replay. The raw platform record + // lacks runtime-ring occupancy after reads and would otherwise + // double-deliver with different counters. + if (self.replay) return null; + const key = self.audio_capture.key; + const event_fn = self.audio_capture.on_event; + var available_frames: u32 = event.available_frames; + var capacity_frames: u32 = event.capacity_frames; + var frames_produced: u64 = event.frames_produced; + if (self.audio_capture_shared) |shared| { + shared.mutex.lock(); + if (shared.generation == self.audio_capture.generation) { + if (shared.ring) |ring| { + available_frames = @intCast(ring.available_frames); + capacity_frames = @intCast(ring.capacity_frames); + frames_produced = ring.frames_produced; + } + if (event.state == .stopped or event.state == .failed or event.state == .rejected) { + shared.accepting = false; + // The terminal lifecycle event carries the final + // occupancy. It supersedes a coalesced readable + // wake so no lifecycle event follows a terminal. + if (shared.readable_pending) { + shared.readable_pending = false; + if (shared.owner) |owner| _ = owner.pending.fetchSub(1, .seq_cst); + } + } + } + shared.mutex.unlock(); + } + if (event.state == .stopped or event.state == .failed or event.state == .rejected) { + self.audio_capture.accepting = false; + self.audio_capture.sealed = true; + } + const map = event_fn orelse return null; + const resolved: EffectAudioCapture = .{ + .key = key, + .state = event.state, + .reason = event.reason, + .sample_rate_hz = if (event.sample_rate_hz != 0) event.sample_rate_hz else self.audio_capture.sample_rate_hz, + .channel_count = if (event.channel_count != 0) event.channel_count else self.audio_capture.channel_count, + .available_frames = available_frames, + .capacity_frames = capacity_frames, + .frames_produced = frames_produced, + }; + self.journalNote(.{ + .kind = .audio_capture, + .key = resolved.key, + .audio_capture_state = resolved.state, + .audio_capture_reason = resolved.reason, + .audio_capture_sample_rate_hz = resolved.sample_rate_hz, + .audio_capture_channel_count = resolved.channel_count, + .audio_capture_available_frames = resolved.available_frames, + .audio_capture_capacity_frames = resolved.capacity_frames, + .audio_capture_frames_produced = resolved.frames_produced, + }); + return map(resolved); + } + + /// Deterministic fake-executor producer seam used by tests. Replay + /// never calls it: journaled read records are the only PCM source. + pub fn feedAudioCaptureFrames(self: *Self, pair: platform.AudioCaptureFramePair) platform.AudioCapturePushResult { + if (!self.audio_capture.active or self.replay) return .closed; + const shared = self.audio_capture_shared orelse return .closed; + const result = audioCapturePush(shared, self.audio_capture.generation, pair); + if (result == .full) { + self.audio_capture.accepting = false; + self.audio_capture.sealed = true; + self.closeAudioCaptureProducer(); + const occupancy = self.audioCaptureOccupancy(); + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = self.audio_capture.key, + .state = .failed, + .reason = .consumer_too_slow, + .sample_rate_hz = self.audio_capture.sample_rate_hz, + .channel_count = self.audio_capture.channel_count, + .available_frames = occupancy.available_frames, + .capacity_frames = occupancy.capacity_frames, + .frames_produced = occupancy.frames_produced, + }, .capture_fn = self.audio_capture.on_event } }); + } + return result; + } + + pub fn feedAudioCaptureRead(self: *Self, event: EffectAudioCaptureRead) !void { + if (!self.replay or !self.audio_capture.active or !self.audio_capture.read_pending or event.key != self.audio_capture.key) return error.EffectNotFound; + if (event.system_pcm.len > self.audio_capture.system_scratch.len or event.microphone_pcm.len > self.audio_capture.microphone_scratch.len) return error.EffectNotFound; + if (event.sequence != self.audio_capture.read_sequence) return error.EffectNotFound; + const bytes_per_frame = @as(usize, self.audio_capture.channel_count) * 2; + const expected_bytes = @as(usize, event.frames) * bytes_per_frame; + if ((self.audio_capture.system_audio and event.system_pcm.len != expected_bytes) or + (!self.audio_capture.system_audio and event.system_pcm.len != 0) or + (self.audio_capture.microphone_audio and event.microphone_pcm.len != expected_bytes) or + (!self.audio_capture.microphone_audio and event.microphone_pcm.len != 0)) return error.EffectNotFound; + if ((event.state == .chunk) != (event.frames > 0)) return error.EffectNotFound; + if (event.end_of_stream and !self.audio_capture.sealed) return error.EffectNotFound; + @memcpy(self.audio_capture.system_scratch[0..event.system_pcm.len], event.system_pcm); + @memcpy(self.audio_capture.microphone_scratch[0..event.microphone_pcm.len], event.microphone_pcm); + var copied = event; + copied.system_pcm = self.audio_capture.system_scratch[0..event.system_pcm.len]; + copied.microphone_pcm = self.audio_capture.microphone_scratch[0..event.microphone_pcm.len]; + self.audio_capture.read_sequence +%= 1; + self.deliverPending(.{ .audio_capture_read = .{ .event = copied, .read_fn = self.audio_capture.pending_read_fn } }); + } + + pub fn feedAudioCaptureEvent(self: *Self, event: EffectAudioCapture) !void { + if (!self.replay or !self.audio_capture.active or event.key != self.audio_capture.key) return error.EffectNotFound; + if (event.state == .stopped or event.state == .failed or event.state == .rejected) { + self.audio_capture.accepting = false; + self.audio_capture.sealed = true; + self.closeAudioCaptureProducer(); + } + self.deliverPending(.{ .audio_capture = .{ .event = event, .capture_fn = self.audio_capture.on_event } }); + } + + fn takeAudioCaptureReadableMsg(self: *Self, before: u64) ?Msg { + if (!self.audio_capture.active) return null; + const shared = self.audio_capture_shared orelse return null; + shared.mutex.lock(); + if (!shared.open or shared.generation != self.audio_capture.generation or + !shared.readable_pending or shared.readable_seq >= before) + { + shared.mutex.unlock(); + return null; + } + shared.readable_pending = false; + if (shared.owner) |owner| _ = owner.pending.fetchSub(1, .seq_cst); + const ring = shared.ring.?; + const event: EffectAudioCapture = .{ + .key = self.audio_capture.key, + .state = .readable, + .sample_rate_hz = self.audio_capture.sample_rate_hz, + .channel_count = self.audio_capture.channel_count, + .available_frames = @intCast(ring.available_frames), + .capacity_frames = @intCast(ring.capacity_frames), + .frames_produced = ring.frames_produced, + }; + shared.mutex.unlock(); + const map = self.audio_capture.on_event orelse return null; + self.journalNote(.{ + .kind = .audio_capture, + .key = event.key, + .audio_capture_state = event.state, + .audio_capture_reason = event.reason, + .audio_capture_sample_rate_hz = event.sample_rate_hz, + .audio_capture_channel_count = event.channel_count, + .audio_capture_available_frames = event.available_frames, + .audio_capture_capacity_frames = event.capacity_frames, + .audio_capture_frames_produced = event.frames_produced, + }); + return map(event); + } + + pub fn takeMicrophoneDeviceMsg(self: *Self, event: platform.MicrophoneDeviceEvent) ?Msg { + if (!self.microphone_device_query.active) return null; + const key = self.microphone_device_query.key; + const event_fn = self.microphone_device_query.on_event; + if (event.state != .device) self.microphone_device_query = .{}; + const map = event_fn orelse return null; + return map(.{ + .key = key, + .state = event.state, + .id = event.id, + .name = event.name, + .is_default = event.is_default, + .index = event.index, + .total = event.total, + }); + } + + pub fn takeAudioCaptureAccessMsg(self: *Self, event: platform.AudioCaptureAccessEvent) ?Msg { + if (!self.audio_capture_access_query.active) return null; + const key = self.audio_capture_access_query.key; + const event_fn = self.audio_capture_access_query.on_event; + self.audio_capture_access_query = .{}; + const map = event_fn orelse return null; + return map(.{ .key = key, .source = event.source, .status = event.status, .restart_required = event.restart_required }); + } + + pub fn takeMicrophoneDevicesChangedMsg(self: *Self) ?Msg { + const map = self.microphone_devices_changed_fn orelse return null; + return map(); + } + /// Route a platform audio event back into an `on_event` Msg for /// the active channel, updating the playback mirrors on the way. /// Null when the channel is idle (a straggler after `stopAudio`) @@ -8758,6 +9662,7 @@ pub fn Effects(comptime Msg: type) type { self.pending_video_len > 0 or self.pending_pty_len > 0 or self.pending_staged_len > 0 or + self.audio_capture_pending_count.load(.seq_cst) > 0 or self.channel_pending_count.load(.seq_cst) > 0 or self.pty_pending_count.load(.seq_cst) > 0 or self.queue_count.load(.seq_cst) > 0; @@ -8802,6 +9707,9 @@ pub fn Effects(comptime Msg: type) type { /// the journal records exactly what delivered, so the /// causality cut only needs the stamp, not the bytes. pty_before: u64, + /// Coalesced audio-capture readable notices stamped before the + /// pass. PCM itself remains in the reliable ring until read. + audio_capture_before: u64, }; /// Snapshot the completion backlog at the start of one drain @@ -8835,6 +9743,11 @@ pub fn Effects(comptime Msg: type) type { shared.wake.pending.store(false, .seq_cst); shared.wake.mutex.unlock(); } + if (self.audio_capture_shared) |shared| { + shared.wake.mutex.lock(); + shared.wake.pending.store(false, .seq_cst); + shared.wake.mutex.unlock(); + } return .{ .pending_before = self.pending_seq, .queue_budget = self.queue_count.load(.acquire), @@ -8843,6 +9756,7 @@ pub fn Effects(comptime Msg: type) type { // before the clear above is inside this snapshot. .channel_before = self.channel_seq.load(.seq_cst), .pty_before = self.pty_seq.load(.seq_cst), + .audio_capture_before = self.audio_capture_seq.load(.seq_cst), }; } @@ -8860,6 +9774,7 @@ pub fn Effects(comptime Msg: type) type { .queue_budget = std.math.maxInt(usize), .channel_before = std.math.maxInt(u64), .pty_before = std.math.maxInt(u64), + .audio_capture_before = std.math.maxInt(u64), }; return self.takeMsgWithin(&unbounded); } @@ -8868,6 +9783,7 @@ pub fn Effects(comptime Msg: type) type { /// `boundary` deliver; anything produced after the snapshot /// waits for the wake its producer already nudged. pub fn takeMsgWithin(self: *Self, boundary: *DrainBoundary) ?Msg { + if (self.audio_capture.release_after_delivery) self.releaseAudioCaptureStream(); self.reclaimSlots(); while (true) { if (self.takePendingMsg(boundary.pending_before)) |pending| { @@ -8974,6 +9890,48 @@ pub fn Effects(comptime Msg: type) type { }); return event_fn(event); }, + .audio_capture => |entry| { + const event_fn = entry.capture_fn orelse continue; + return event_fn(entry.event); + }, + .audio_capture_read => |entry| { + self.audio_capture.read_pending = false; + self.audio_capture.pending_read_fn = null; + if (entry.event.end_of_stream) self.audio_capture.release_after_delivery = true; + const system_len = entry.event.system_pcm.len; + const microphone_len = entry.event.microphone_pcm.len; + @memcpy(self.audio_capture_journal_scratch[0..system_len], entry.event.system_pcm); + @memcpy(self.audio_capture_journal_scratch[system_len .. system_len + microphone_len], entry.event.microphone_pcm); + self.journalNote(.{ + .kind = .audio_capture_read, + .key = entry.event.key, + .payload = self.audio_capture_journal_scratch[0 .. system_len + microphone_len], + .audio_capture_read_state = entry.event.state, + .audio_capture_read_reason = entry.event.reason, + .audio_capture_sequence = entry.event.sequence, + .audio_capture_frame_offset = entry.event.frame_offset, + .audio_capture_frames = entry.event.frames, + .audio_capture_system_len = @intCast(system_len), + .audio_capture_system_gap_frames = entry.event.system_gap_frames, + .audio_capture_microphone_gap_frames = entry.event.microphone_gap_frames, + .audio_capture_remaining_frames = entry.event.remaining_frames, + .audio_capture_end_of_stream = entry.event.end_of_stream, + }); + const event_fn = entry.read_fn orelse continue; + return event_fn(entry.event); + }, + .microphone_device => |entry| { + const event_fn = entry.device_fn orelse continue; + return event_fn(entry.event); + }, + .audio_capture_access => |entry| { + const event_fn = entry.access_fn orelse continue; + return event_fn(entry.event); + }, + .microphone_devices_changed => |entry| { + const event_fn = entry.changed_fn orelse continue; + return event_fn(); + }, .video => |entry| { var event = entry.event; const video_fn = entry.video_fn; @@ -9146,6 +10104,7 @@ pub fn Effects(comptime Msg: type) type { }, } } + if (self.takeAudioCaptureReadableMsg(boundary.audio_capture_before)) |msg| return msg; if (self.takeChannelStagedMsg(boundary.channel_before)) |msg| return msg; if (self.takePtyStagedMsg(boundary.pty_before)) |msg| return msg; if (boundary.queue_budget == 0) return null; @@ -11250,6 +12209,110 @@ pub fn Effects(comptime Msg: type) type { self.deliverLoopAudio(.{ .key = key, .kind = .failed }, on_event); } + fn failAudioCapture(self: *Self, reason: EffectAudioCaptureReason) void { + const key = self.audio_capture.key; + const on_event = self.audio_capture.on_event; + self.audio_capture.accepting = false; + self.audio_capture.sealed = true; + self.closeAudioCaptureProducer(); + const occupancy = self.audioCaptureOccupancy(); + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = key, + .state = .failed, + .reason = reason, + .sample_rate_hz = self.audio_capture.sample_rate_hz, + .channel_count = self.audio_capture.channel_count, + .available_frames = occupancy.available_frames, + .capacity_frames = occupancy.capacity_frames, + .frames_produced = occupancy.frames_produced, + }, .capture_fn = on_event } }); + } + + fn audioCaptureOccupancy(self: *Self) AudioCaptureOccupancy { + const shared = self.audio_capture_shared orelse return .{}; + shared.mutex.lock(); + defer shared.mutex.unlock(); + if (shared.generation != self.audio_capture.generation) return .{}; + const ring = shared.ring orelse return .{}; + return .{ + .available_frames = @intCast(ring.available_frames), + .capacity_frames = @intCast(ring.capacity_frames), + .frames_produced = ring.frames_produced, + }; + } + + /// Stop producer admission and retire a redundant coalesced + /// readability wake. A stop/failure lifecycle event reports the + /// final occupancy and remains the last event on that route. + fn closeAudioCaptureProducer(self: *Self) void { + if (self.audio_capture_shared) |shared| { + shared.mutex.lock(); + if (shared.generation == self.audio_capture.generation) { + shared.accepting = false; + if (shared.readable_pending) { + shared.readable_pending = false; + if (shared.owner) |owner| _ = owner.pending.fetchSub(1, .seq_cst); + } + } + shared.mutex.unlock(); + } + } + + fn rejectAudioCapture(self: *Self, reason: EffectAudioCaptureReason) void { + const key = self.audio_capture.key; + const on_event = self.audio_capture.on_event; + self.releaseAudioCaptureStream(); + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = key, + .state = .rejected, + .reason = reason, + }, .capture_fn = on_event } }); + } + + fn releaseAudioCaptureStream(self: *Self) void { + if (!self.audio_capture.active) return; + var ring: ?*AudioCaptureRing = null; + if (self.audio_capture_shared) |shared| { + shared.mutex.lock(); + if (shared.generation == self.audio_capture.generation) { + shared.open = false; + shared.accepting = false; + if (shared.readable_pending) { + shared.readable_pending = false; + if (shared.owner) |owner| _ = owner.pending.fetchSub(1, .seq_cst); + } + shared.owner = null; + ring = shared.ring; + shared.ring = null; + } + shared.mutex.unlock(); + revokeChannelWake(&shared.wake); + } + if (ring) |value| value.deinit(self.allocator); + self.audio_capture = .{}; + } + + fn finishMicrophoneDevices(self: *Self, state: EffectMicrophoneDeviceState) void { + const key = self.microphone_device_query.key; + const on_event = self.microphone_device_query.on_event; + self.microphone_device_query = .{}; + self.deliverPending(.{ .microphone_device = .{ .event = .{ + .key = key, + .state = state, + }, .device_fn = on_event } }); + } + + fn failAudioCaptureAccess(self: *Self, source: EffectAudioCaptureAccessSource) void { + const key = self.audio_capture_access_query.key; + const on_event = self.audio_capture_access_query.on_event; + self.audio_capture_access_query = .{}; + self.deliverPending(.{ .audio_capture_access = .{ .event = .{ + .key = key, + .source = source, + .status = .unavailable, + }, .access_fn = on_event } }); + } + /// Queue an audio event Msg produced on the loop thread /// (rejections and synchronous failures) for the next drain. fn deliverLoopAudio(self: *Self, event: EffectAudio, audio_fn: ?AudioMsgFn) void { diff --git a/src/runtime/effects_audio_tests.zig b/src/runtime/effects_audio_tests.zig index 87a3f5b0d..d25ce54ea 100644 --- a/src/runtime/effects_audio_tests.zig +++ b/src/runtime/effects_audio_tests.zig @@ -621,6 +621,207 @@ fn effectsCachePathNoExt(buffer: []u8) ![]const u8 { return effects_mod.audioCachePath(buffer, "/tmp/caches/app", "https://music.example.test/stream?id=42"); } +const CaptureMsg = union(enum) { + capture: effects_mod.EffectAudioCapture, + read: effects_mod.EffectAudioCaptureRead, + device: effects_mod.EffectMicrophoneDevice, + access: effects_mod.EffectAudioCaptureAccess, + devices_changed, +}; +const CaptureFx = effects_mod.Effects(CaptureMsg); + +test "audio capture fake covers source combinations validation and duplicate starts" { + var fx = CaptureFx.init(std.testing.allocator); + defer fx.deinit(); + fx.executor = .fake; + + fx.startAudioCapture(.{ .key = 1, .system_audio = true, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.started, fx.takeMsg().?.capture.state); + fx.startAudioCapture(.{ .key = 2, .microphone = .default, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + const duplicate = fx.takeMsg().?.capture; + try std.testing.expectEqual(@as(u64, 2), duplicate.key); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.rejected, duplicate.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.already_recording, duplicate.reason); + const system_pcm: [3_840]u8 = @splat(7); + try std.testing.expectEqual(platform.AudioCapturePushResult.accepted, fx.feedAudioCaptureFrames(.{ + .frame_offset = 0, + .frame_count = 960, + .system_pcm = &system_pcm, + .system_gap_frames = 12, + })); + const readable = fx.takeMsg().?.capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.readable, readable.state); + try std.testing.expectEqual(@as(u32, 960), readable.available_frames); + fx.readAudioCapture(.{ .key = 1, .max_frames = 4_800, .on_read = CaptureFx.audioCaptureReadMsg(.read) }); + const chunk = fx.takeMsg().?.read; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReadState.chunk, chunk.state); + try std.testing.expectEqual(@as(u32, 960), chunk.frames); + try std.testing.expectEqual(@as(usize, 3_840), chunk.system_pcm.len); + try std.testing.expectEqual(@as(u32, 12), chunk.system_gap_frames); + try std.testing.expectEqual(@as(usize, 0), chunk.microphone_pcm.len); + fx.discardAudioCapture(); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.discarded, fx.takeMsg().?.capture.reason); + + fx.startAudioCapture(.{ .key = 3, .microphone = .default, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.started, fx.takeMsg().?.capture.state); + fx.discardAudioCapture(); + _ = fx.takeMsg(); + + fx.startAudioCapture(.{ .key = 4, .system_audio = true, .microphone = .{ .device_id = "usb-mic" }, .sample_rate_hz = 44_100, .channel_count = 1, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.started, fx.takeMsg().?.capture.state); + fx.discardAudioCapture(); + _ = fx.takeMsg(); + + fx.startAudioCapture(.{ .key = 5, .system_audio = true, .sample_rate_hz = 12_345, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + const invalid = fx.takeMsg().?.capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.rejected, invalid.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.invalid_options, invalid.reason); +} + +test "audio capture fake lists fixed device records and reports access" { + var fx = CaptureFx.init(std.testing.allocator); + defer fx.deinit(); + fx.executor = .fake; + + fx.listMicrophoneDevices(.{ .key = 7, .on_event = CaptureFx.microphoneDeviceMsg(.device) }); + const first = fx.takeMsg().?.device; + const second = fx.takeMsg().?.device; + const completed = fx.takeMsg().?.device; + try std.testing.expectEqual(effects_mod.EffectMicrophoneDeviceState.device, first.state); + try std.testing.expectEqualStrings("default-mic", first.id); + try std.testing.expect(first.is_default); + try std.testing.expectEqualStrings("usb-mic", second.id); + try std.testing.expectEqual(effects_mod.EffectMicrophoneDeviceState.completed, completed.state); + try std.testing.expectEqual(@as(u32, 2), completed.total); + + fx.audioCaptureAccess(.{ .key = 8, .source = .microphone, .action = .status, .on_event = CaptureFx.audioCaptureAccessMsg(.access) }); + const access = fx.takeMsg().?.access; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureAccessStatus.authorized, access.status); + + fx.observeMicrophoneDevices(CaptureFx.microphoneDevicesChangedMsg(.devices_changed)); + try std.testing.expect(fx.takeMicrophoneDevicesChangedMsg().? == .devices_changed); +} + +test "null audio capture resolves defaults rejects missing devices and preserves partial disconnects" { + var null_platform: platform.NullPlatform = .{}; + var platform_value = null_platform.platform(); + var fx = CaptureFx.init(std.testing.allocator); + defer fx.deinit(); + fx.bindServices(&platform_value.services); + + try null_platform.setDefaultMicrophone(1); + fx.startAudioCapture(.{ .key = 10, .microphone = .default, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + try std.testing.expectEqualStrings("usb-mic", null_platform.capture.microphoneDeviceId()); + const started_event = null_platform.takeAudioCaptureStarted().?.audio_capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.started, fx.takeAudioCaptureMsg(started_event).?.capture.state); + const microphone_pcm: [3_840]u8 = @splat(3); + try std.testing.expectEqual(platform.AudioCapturePushResult.accepted, null_platform.pushAudioCapturePair(.{ + .frame_offset = 0, + .frame_count = 960, + .microphone_pcm = µphone_pcm, + })); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.readable, fx.takeMsg().?.capture.state); + const disconnected_event = null_platform.disconnectMicrophone("usb-mic").?.audio_capture; + const disconnected = fx.takeAudioCaptureMsg(disconnected_event).?.capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.failed, disconnected.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.device_disconnected, disconnected.reason); + try std.testing.expectEqual(@as(u32, 960), disconnected.available_frames); + fx.readAudioCapture(.{ .key = 10, .max_frames = 4_800, .on_read = CaptureFx.audioCaptureReadMsg(.read) }); + const retained = fx.takeMsg().?.read; + try std.testing.expect(retained.end_of_stream); + try std.testing.expectEqual(@as(u32, 960), retained.frames); + _ = fx.takeMsg(); + + fx.startAudioCapture(.{ .key = 11, .microphone = .{ .device_id = "missing" }, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + const missing = fx.takeMsg().?.capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.rejected, missing.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.device_not_found, missing.reason); + + null_platform.system_audio_access = .not_authorized; + fx.startAudioCapture(.{ .key = 13, .system_audio = true, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + const permission = fx.takeMsg().?.capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.rejected, permission.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.permission_required, permission.reason); +} + +test "reliable audio capture stops at capacity and preserves every accepted frame for draining" { + var fx = CaptureFx.init(std.testing.allocator); + defer fx.deinit(); + fx.executor = .fake; + + fx.startAudioCapture(.{ + .key = 30, + .system_audio = true, + .sample_rate_hz = 16_000, + .channel_count = 1, + .buffer_duration_ms = 1_000, + .on_event = CaptureFx.audioCaptureMsg(.capture), + }); + _ = fx.takeMsg(); + + const pcm: [640]u8 = @splat(0x5a); + for (0..50) |index| { + try std.testing.expectEqual(platform.AudioCapturePushResult.accepted, fx.feedAudioCaptureFrames(.{ + .frame_offset = index * 320, + .frame_count = 320, + .system_pcm = &pcm, + })); + } + try std.testing.expectEqual(platform.AudioCapturePushResult.full, fx.feedAudioCaptureFrames(.{ + .frame_offset = 16_000, + .frame_count = 320, + .system_pcm = &pcm, + })); + const terminal = fx.takeMsg().?.capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.failed, terminal.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.consumer_too_slow, terminal.reason); + try std.testing.expectEqual(@as(u32, 16_000), terminal.available_frames); + + var drained: u32 = 0; + while (drained < 16_000) { + fx.readAudioCapture(.{ .key = 30, .max_frames = 1_600, .on_read = CaptureFx.audioCaptureReadMsg(.read) }); + const read = fx.takeMsg().?.read; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReadState.chunk, read.state); + try std.testing.expectEqual(@as(usize, read.frames) * 2, read.system_pcm.len); + drained += read.frames; + if (drained < 16_000) try std.testing.expect(!read.end_of_stream); + } + try std.testing.expectEqual(@as(u32, 16_000), drained); + // The terminal chunk releases the retained stream on the next drain. + try std.testing.expect(fx.takeMsg() == null); + fx.readAudioCapture(.{ .key = 30, .max_frames = 1_600, .on_read = CaptureFx.audioCaptureReadMsg(.read) }); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReadReason.not_recording, fx.takeMsg().?.read.reason); +} + +test "null microphone listing access and device-change observation are deterministic" { + var null_platform: platform.NullPlatform = .{}; + var platform_value = null_platform.platform(); + var fx = CaptureFx.init(std.testing.allocator); + defer fx.deinit(); + fx.bindServices(&platform_value.services); + + try null_platform.setDefaultMicrophone(1); + fx.listMicrophoneDevices(.{ .key = 20, .on_event = CaptureFx.microphoneDeviceMsg(.device) }); + try std.testing.expectEqual(@as(usize, 1), null_platform.microphone_devices_count); + const first = fx.takeMicrophoneDeviceMsg(null_platform.microphoneDeviceEvent(0).microphone_device).?.device; + const second = fx.takeMicrophoneDeviceMsg(null_platform.microphoneDeviceEvent(1).microphone_device).?.device; + const completed = fx.takeMicrophoneDeviceMsg(null_platform.microphoneDeviceEvent(2).microphone_device).?.device; + try std.testing.expect(!first.is_default); + try std.testing.expect(second.is_default); + try std.testing.expectEqual(effects_mod.EffectMicrophoneDeviceState.completed, completed.state); + + null_platform.microphone_access = .denied; + fx.audioCaptureAccess(.{ .key = 21, .source = .microphone, .action = .status, .on_event = CaptureFx.audioCaptureAccessMsg(.access) }); + const access_event = null_platform.takeAudioCaptureAccess().?.audio_capture_access; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureAccessStatus.denied, fx.takeAudioCaptureAccessMsg(access_event).?.access.status); + + fx.observeMicrophoneDevices(CaptureFx.microphoneDevicesChangedMsg(.devices_changed)); + try std.testing.expect(null_platform.microphone_devices_observing); + try std.testing.expect(fx.takeMicrophoneDevicesChangedMsg().? == .devices_changed); + fx.observeMicrophoneDevices(null); + try std.testing.expect(!null_platform.microphone_devices_observing); +} + test "quit while playing: the stop hook silences audio on the live platform, and app deinit after platform teardown answers inert" { // The desktop runner's exit ordering, replayed exactly: main defers // app deinit FIRST and calls the runner, whose own defers destroy diff --git a/src/runtime/effects_video_tests.zig b/src/runtime/effects_video_tests.zig index 574d5adf3..f1aa16799 100644 --- a/src/runtime/effects_video_tests.zig +++ b/src/runtime/effects_video_tests.zig @@ -1978,8 +1978,12 @@ test "a replaced load's terminal keeps its own identity under replay" { /// Overwrite the `video_width` field of the first journaled `.video` /// record carrying `kind`, in place. Per `journal.encodeEffect` the /// width sits 26 bytes before the pty trailer (width, height, token, -/// source, handled), and the pty fields (kind 1, signal 4, drops 4, -/// blob hash 16, blob len 8 = 33 bytes) end the payload. +/// source, handled). The pty fields occupy 33 bytes and the reliable +/// audio-capture fields appended after them occupy 62 bytes. +const effect_audio_capture_trailer_len: usize = 62; +const effect_pty_trailer_len: usize = 33; +const effect_video_fields_len: usize = 45; + fn patchFirstVideoWidth(bytes: []u8, kind: effects_mod.EffectVideoEventKind, width: u64) bool { var pos: usize = journal.preamble_len; while (bytes.len - pos >= 5) { @@ -1990,7 +1994,7 @@ fn patchFirstVideoWidth(bytes: []u8, kind: effects_mod.EffectVideoEventKind, wid if (record_kind != @intFromEnum(journal.RecordKind.effect)) continue; const record = journal.decodeEffect(payload) catch continue; if (record.kind != .video or record.video_kind != kind) continue; - std.mem.writeInt(u64, payload[payload.len - 59 ..][0..8], width, .little); + std.mem.writeInt(u64, payload[payload.len - (26 + effect_pty_trailer_len + effect_audio_capture_trailer_len) ..][0..8], width, .little); return true; } return false; @@ -2082,7 +2086,7 @@ fn patchFirstVideoKind(bytes: []u8, kind: effects_mod.EffectVideoEventKind) bool if (record_kind != @intFromEnum(journal.RecordKind.effect)) continue; const record = journal.decodeEffect(payload) catch continue; if (record.kind != .video) continue; - payload[payload.len - 78] = @intFromEnum(kind); + payload[payload.len - (effect_video_fields_len + effect_pty_trailer_len + effect_audio_capture_trailer_len)] = @intFromEnum(kind); return true; } return false; @@ -2104,7 +2108,7 @@ fn patchFirstVideoLoadKind(bytes: []u8, kind: effects_mod.EffectVideoEventKind) if (record_kind != @intFromEnum(journal.RecordKind.effect)) continue; const record = journal.decodeEffect(payload) catch continue; if (record.kind != .video_load) continue; - payload[payload.len - 78] = @intFromEnum(kind); + payload[payload.len - (effect_video_fields_len + effect_pty_trailer_len + effect_audio_capture_trailer_len)] = @intFromEnum(kind); return true; } return false; @@ -2350,10 +2354,11 @@ test "a video load record claiming a non-outcome kind refuses replay as damage" /// Overwrite the `video_position_ms` field of the first journaled /// video effect record, in place. Per `journal.encodeEffect` the video -/// fields are the 45 bytes before the 33-byte pty trailer — video_kind +/// fields are the 45 bytes before the 33-byte pty and 62-byte audio +/// capture trailers — video_kind /// (1), position (8), duration (8), playing (1), buffering (1), width /// (8), height (8), token (8), source (1), handled (1) — so the -/// position lives 77 bytes from the end. +/// position lives one byte after video_kind. /// Framing and every other field stay valid: only replay's damage gate /// can catch the value. Returns whether a record was damaged. fn patchFirstVideoPosition(bytes: []u8, position: u64) bool { @@ -2366,7 +2371,7 @@ fn patchFirstVideoPosition(bytes: []u8, position: u64) bool { if (kind != @intFromEnum(journal.RecordKind.effect)) continue; const record = journal.decodeEffect(payload) catch continue; if (record.kind != .video) continue; - std.mem.writeInt(u64, payload[payload.len - 77 ..][0..8], position, .little); + std.mem.writeInt(u64, payload[payload.len - (effect_video_fields_len - 1 + effect_pty_trailer_len + effect_audio_capture_trailer_len) ..][0..8], position, .little); return true; } return false; diff --git a/src/runtime/flow.zig b/src/runtime/flow.zig index 68ba0741a..4dc3e40bb 100644 --- a/src/runtime/flow.zig +++ b/src/runtime/flow.zig @@ -450,6 +450,10 @@ pub fn RuntimeFlow(comptime Runtime: type) type { .audio => |audio_event| { try dispatchEvent(self, app, .{ .audio = audio_event }); }, + .audio_capture => |capture_event| try dispatchEvent(self, app, .{ .audio_capture = capture_event }), + .microphone_device => |device_event| try dispatchEvent(self, app, .{ .microphone_device = device_event }), + .microphone_devices_changed => try dispatchEvent(self, app, .microphone_devices_changed), + .audio_capture_access => |access_event| try dispatchEvent(self, app, .{ .audio_capture_access = access_event }), .video => |video_event| { try dispatchEvent(self, app, .{ .video = video_event }); }, @@ -518,6 +522,10 @@ pub fn RuntimeFlow(comptime Runtime: type) type { .timer => {}, .effects_wake => {}, .audio => {}, + .audio_capture => {}, + .microphone_device => {}, + .microphone_devices_changed => {}, + .audio_capture_access => {}, .video => {}, .files_dropped => {}, .gpu_surface_frame => {}, diff --git a/src/runtime/root.zig b/src/runtime/root.zig index e5c0a4d57..a6c6be523 100644 --- a/src/runtime/root.zig +++ b/src/runtime/root.zig @@ -96,6 +96,19 @@ pub const effect_timer_platform_id_base = runtime_effects.effect_timer_platform_ pub const EffectAudio = runtime_effects.EffectAudio; pub const EffectAudioEventKind = runtime_effects.EffectAudioEventKind; pub const EffectAudioSource = runtime_effects.EffectAudioSource; +pub const EffectAudioCapture = runtime_effects.EffectAudioCapture; +pub const EffectAudioCaptureState = runtime_effects.EffectAudioCaptureState; +pub const EffectAudioCaptureReason = runtime_effects.EffectAudioCaptureReason; +pub const EffectAudioCaptureRead = runtime_effects.EffectAudioCaptureRead; +pub const EffectAudioCaptureReadState = runtime_effects.EffectAudioCaptureReadState; +pub const EffectAudioCaptureReadReason = runtime_effects.EffectAudioCaptureReadReason; +pub const EffectMicrophoneDevice = runtime_effects.EffectMicrophoneDevice; +pub const EffectMicrophoneDeviceState = runtime_effects.EffectMicrophoneDeviceState; +pub const EffectAudioCaptureAccess = runtime_effects.EffectAudioCaptureAccess; +pub const EffectAudioCaptureAccessSource = runtime_effects.EffectAudioCaptureAccessSource; +pub const EffectAudioCaptureAccessAction = runtime_effects.EffectAudioCaptureAccessAction; +pub const EffectAudioCaptureAccessStatus = runtime_effects.EffectAudioCaptureAccessStatus; +pub const MicrophoneSelection = runtime_effects.MicrophoneSelection; pub const audioCachePath = runtime_effects.audioCachePath; pub const max_effect_audio_path_bytes = runtime_effects.max_effect_audio_path_bytes; pub const EffectVideo = runtime_effects.EffectVideo; diff --git a/src/runtime/session_journal.zig b/src/runtime/session_journal.zig index fdb2e9cf6..a961c0bc5 100644 --- a/src/runtime/session_journal.zig +++ b/src/runtime/session_journal.zig @@ -186,6 +186,9 @@ fn formatLayoutDescription(comptime epoch: u32) []const u8 { "menu_command=" ++ layout_fingerprint.describe(platform.MenuCommandEvent) ++ "\n" ++ "timer=" ++ layout_fingerprint.describe(platform.TimerEvent) ++ "\n" ++ "audio=" ++ layout_fingerprint.describe(platform.AudioEvent) ++ "\n" ++ + "audio_capture=" ++ layout_fingerprint.describe(platform.AudioCaptureEvent) ++ "\n" ++ + "microphone_device=" ++ layout_fingerprint.describe(platform.MicrophoneDeviceEvent) ++ "\n" ++ + "audio_capture_access=" ++ layout_fingerprint.describe(platform.AudioCaptureAccessEvent) ++ "\n" ++ "video=" ++ layout_fingerprint.describe(platform.VideoEvent) ++ "\n" ++ "files_dropped=" ++ layout_fingerprint.describe(platform.FileDropEvent) ++ "\n" ++ // gpu_surface_frame journals a deliberate SUBSET of a @@ -464,6 +467,10 @@ const EventTag = enum(u8) { audio = 24, video = 25, view_focused = 26, + audio_capture = 27, + microphone_device = 28, + microphone_devices_changed = 29, + audio_capture_access = 30, }; // The bit assignments below are hand-written wire layout: they are @@ -622,6 +629,32 @@ pub fn encodeEvent(event: platform.Event, buffer: []u8) JournalError![]const u8 try cursor.writeBool(audio.buffering); try cursor.writeBytes(&audio.bands); }, + .audio_capture => |capture| { + try cursor.writeEnum(EventTag.audio_capture); + try cursor.writeEnum(capture.state); + try cursor.writeEnum(capture.reason); + try cursor.writeInt(u32, capture.sample_rate_hz); + try cursor.writeInt(u8, capture.channel_count); + try cursor.writeInt(u32, capture.available_frames); + try cursor.writeInt(u32, capture.capacity_frames); + try cursor.writeInt(u64, capture.frames_produced); + }, + .microphone_device => |device| { + try cursor.writeEnum(EventTag.microphone_device); + try cursor.writeEnum(device.state); + try cursor.writeStr(device.id); + try cursor.writeStr(device.name); + try cursor.writeBool(device.is_default); + try cursor.writeInt(u32, device.index); + try cursor.writeInt(u32, device.total); + }, + .microphone_devices_changed => try cursor.writeEnum(EventTag.microphone_devices_changed), + .audio_capture_access => |access| { + try cursor.writeEnum(EventTag.audio_capture_access); + try cursor.writeEnum(access.source); + try cursor.writeEnum(access.status); + try cursor.writeBool(access.restart_required); + }, // Recorded for stream fidelity like `.audio`. On replay the // journaled video EFFECT records are the Msg source; the // platform events steer only the channel MIRRORS (the house @@ -849,6 +882,29 @@ pub fn decodeEvent(bytes: []const u8, storage: *EventDecodeStorage) JournalError @memcpy(&decoded.bands, try cursor.readBytes(decoded.bands.len)); break :blk .{ .audio = decoded }; }, + .audio_capture => .{ .audio_capture = .{ + .state = try cursor.readEnum(platform.AudioCaptureEventState), + .reason = try cursor.readEnum(platform.AudioCaptureEventReason), + .sample_rate_hz = try cursor.readInt(u32), + .channel_count = try cursor.readInt(u8), + .available_frames = try cursor.readInt(u32), + .capacity_frames = try cursor.readInt(u32), + .frames_produced = try cursor.readInt(u64), + } }, + .microphone_device => .{ .microphone_device = .{ + .state = try cursor.readEnum(platform.MicrophoneDeviceEventState), + .id = try cursor.readStr(), + .name = try cursor.readStr(), + .is_default = try cursor.readBool(), + .index = try cursor.readInt(u32), + .total = try cursor.readInt(u32), + } }, + .microphone_devices_changed => .microphone_devices_changed, + .audio_capture_access => .{ .audio_capture_access = .{ + .source = try cursor.readEnum(platform.AudioCaptureAccessSource), + .status = try cursor.readEnum(platform.AudioCaptureAccessStatus), + .restart_required = try cursor.readBool(), + } }, .video => blk: { const kind = try cursor.readEnum(platform.VideoEventKind); break :blk .{ .video = .{ @@ -1068,6 +1124,27 @@ pub fn encodeEffect(record: EffectResultRecord, buffer: []u8) JournalError![]con try cursor.writeInt(u32, record.pty_dropped_writes); try cursor.writeBytes(&record.pty_blob_hash); try cursor.writeInt(u64, record.pty_blob_len); + // Reliable paired audio-capture reads. PCM rides `payload` inline; + // the system length splits it from the microphone bytes. + try cursor.writeEnum(record.audio_capture_read_state); + try cursor.writeEnum(record.audio_capture_read_reason); + try cursor.writeInt(u64, record.audio_capture_sequence); + try cursor.writeInt(u64, record.audio_capture_frame_offset); + try cursor.writeInt(u32, record.audio_capture_frames); + try cursor.writeInt(u32, record.audio_capture_system_len); + try cursor.writeInt(u32, record.audio_capture_system_gap_frames); + try cursor.writeInt(u32, record.audio_capture_microphone_gap_frames); + try cursor.writeInt(u32, record.audio_capture_remaining_frames); + try cursor.writeBool(record.audio_capture_end_of_stream); + // Runtime-generated audio-capture readiness. Platform start and + // terminal lifecycle events already use the platform-event codec. + try cursor.writeEnum(record.audio_capture_state); + try cursor.writeEnum(record.audio_capture_reason); + try cursor.writeInt(u32, record.audio_capture_sample_rate_hz); + try cursor.writeByte(record.audio_capture_channel_count); + try cursor.writeInt(u32, record.audio_capture_available_frames); + try cursor.writeInt(u32, record.audio_capture_capacity_frames); + try cursor.writeInt(u64, record.audio_capture_frames_produced); return buffer[0..cursor.len]; } @@ -1126,6 +1203,23 @@ pub fn decodeEffect(bytes: []const u8) JournalError!EffectResultRecord { record.pty_dropped_writes = try cursor.readInt(u32); @memcpy(&record.pty_blob_hash, try cursor.readBytes(record.pty_blob_hash.len)); record.pty_blob_len = try cursor.readInt(u64); + record.audio_capture_read_state = try cursor.readEnum(runtime_effects.EffectAudioCaptureReadState); + record.audio_capture_read_reason = try cursor.readEnum(runtime_effects.EffectAudioCaptureReadReason); + record.audio_capture_sequence = try cursor.readInt(u64); + record.audio_capture_frame_offset = try cursor.readInt(u64); + record.audio_capture_frames = try cursor.readInt(u32); + record.audio_capture_system_len = try cursor.readInt(u32); + record.audio_capture_system_gap_frames = try cursor.readInt(u32); + record.audio_capture_microphone_gap_frames = try cursor.readInt(u32); + record.audio_capture_remaining_frames = try cursor.readInt(u32); + record.audio_capture_end_of_stream = try cursor.readBool(); + record.audio_capture_state = try cursor.readEnum(runtime_effects.EffectAudioCaptureState); + record.audio_capture_reason = try cursor.readEnum(runtime_effects.EffectAudioCaptureReason); + record.audio_capture_sample_rate_hz = try cursor.readInt(u32); + record.audio_capture_channel_count = try cursor.readByte(); + record.audio_capture_available_frames = try cursor.readInt(u32); + record.audio_capture_capacity_frames = try cursor.readInt(u32); + record.audio_capture_frames_produced = try cursor.readInt(u64); if (!cursor.done()) return error.JournalCorrupt; return record; } @@ -1483,6 +1577,51 @@ test "event codec round-trips every payload variant" { try testing.expectEqual(@as(u64, 1280), decoded.video.width); try testing.expectEqual(@as(u64, 720), decoded.video.height); } + { + const decoded = try roundTripEvent(.{ .audio_capture = .{ + .state = .failed, + .reason = .device_disconnected, + .sample_rate_hz = 48_000, + .channel_count = 2, + .available_frames = 960, + .capacity_frames = 240_000, + .frames_produced = 4_800, + } }); + try testing.expectEqual(platform.AudioCaptureEventState.failed, decoded.audio_capture.state); + try testing.expectEqual(platform.AudioCaptureEventReason.device_disconnected, decoded.audio_capture.reason); + try testing.expectEqual(@as(u32, 48_000), decoded.audio_capture.sample_rate_hz); + try testing.expectEqual(@as(u8, 2), decoded.audio_capture.channel_count); + try testing.expectEqual(@as(u32, 960), decoded.audio_capture.available_frames); + try testing.expectEqual(@as(u32, 240_000), decoded.audio_capture.capacity_frames); + try testing.expectEqual(@as(u64, 4_800), decoded.audio_capture.frames_produced); + } + { + const decoded = try roundTripEvent(.{ .microphone_device = .{ + .state = .device, + .id = "usb-mic", + .name = "USB Microphone", + .is_default = true, + .index = 1, + .total = 2, + } }); + try testing.expectEqual(platform.MicrophoneDeviceEventState.device, decoded.microphone_device.state); + try testing.expectEqualStrings("usb-mic", decoded.microphone_device.id); + try testing.expectEqualStrings("USB Microphone", decoded.microphone_device.name); + try testing.expect(decoded.microphone_device.is_default); + try testing.expectEqual(@as(u32, 2), decoded.microphone_device.total); + } + { + const changed = try roundTripEvent(.microphone_devices_changed); + try testing.expect(changed == .microphone_devices_changed); + const decoded = try roundTripEvent(.{ .audio_capture_access = .{ + .source = .system_audio, + .status = .authorized, + .restart_required = true, + } }); + try testing.expectEqual(platform.AudioCaptureAccessSource.system_audio, decoded.audio_capture_access.source); + try testing.expectEqual(platform.AudioCaptureAccessStatus.authorized, decoded.audio_capture_access.status); + try testing.expect(decoded.audio_capture_access.restart_required); + } { const paths = [_][]const u8{ "/tmp/a.txt", "/tmp/b.txt" }; const decoded = try roundTripEvent(.{ .files_dropped = .{ @@ -1724,6 +1863,67 @@ test "effect codec round-trips payloads and outcomes" { try testing.expect(video_decoded.video_buffering); try testing.expectEqual(@as(u64, 1280), video_decoded.video_width); try testing.expectEqual(@as(u64, 720), video_decoded.video_height); + + const capture_encoded = try encodeEffect(.{ + .kind = .audio_capture_read, + .key = 73, + .payload = "sys\x00mic\x00", + .audio_capture_read_state = .chunk, + .audio_capture_sequence = 4, + .audio_capture_frame_offset = 960, + .audio_capture_frames = 2, + .audio_capture_system_len = 4, + .audio_capture_system_gap_frames = 1, + .audio_capture_microphone_gap_frames = 0, + .audio_capture_remaining_frames = 958, + .audio_capture_end_of_stream = false, + }, &buffer); + const capture_decoded = try decodeEffect(capture_encoded); + try testing.expectEqual(runtime_effects.EffectResultKind.audio_capture_read, capture_decoded.kind); + try testing.expectEqual(@as(u64, 73), capture_decoded.key); + try testing.expectEqualStrings("sys\x00mic\x00", capture_decoded.payload); + try testing.expectEqual(runtime_effects.EffectAudioCaptureReadState.chunk, capture_decoded.audio_capture_read_state); + try testing.expectEqual(@as(u64, 4), capture_decoded.audio_capture_sequence); + try testing.expectEqual(@as(u64, 960), capture_decoded.audio_capture_frame_offset); + try testing.expectEqual(@as(u32, 2), capture_decoded.audio_capture_frames); + try testing.expectEqual(@as(u32, 4), capture_decoded.audio_capture_system_len); + try testing.expectEqual(@as(u32, 1), capture_decoded.audio_capture_system_gap_frames); + try testing.expectEqual(@as(u32, 958), capture_decoded.audio_capture_remaining_frames); + + const readable_encoded = try encodeEffect(.{ + .kind = .audio_capture, + .key = 73, + .audio_capture_state = .readable, + .audio_capture_sample_rate_hz = 48_000, + .audio_capture_channel_count = 2, + .audio_capture_available_frames = 960, + .audio_capture_capacity_frames = 240_000, + .audio_capture_frames_produced = 960, + }, &buffer); + const readable_decoded = try decodeEffect(readable_encoded); + try testing.expectEqual(runtime_effects.EffectResultKind.audio_capture, readable_decoded.kind); + try testing.expectEqual(runtime_effects.EffectAudioCaptureState.readable, readable_decoded.audio_capture_state); + try testing.expectEqual(@as(u32, 48_000), readable_decoded.audio_capture_sample_rate_hz); + try testing.expectEqual(@as(u8, 2), readable_decoded.audio_capture_channel_count); + try testing.expectEqual(@as(u32, 960), readable_decoded.audio_capture_available_frames); + try testing.expectEqual(@as(u32, 240_000), readable_decoded.audio_capture_capacity_frames); + try testing.expectEqual(@as(u64, 960), readable_decoded.audio_capture_frames_produced); + + const terminal_encoded = try encodeEffect(.{ + .kind = .audio_capture, + .key = 73, + .audio_capture_state = .failed, + .audio_capture_reason = .consumer_too_slow, + .audio_capture_sample_rate_hz = 48_000, + .audio_capture_channel_count = 2, + .audio_capture_available_frames = 240_000, + .audio_capture_capacity_frames = 240_000, + .audio_capture_frames_produced = 240_000, + }, &buffer); + const terminal_decoded = try decodeEffect(terminal_encoded); + try testing.expectEqual(runtime_effects.EffectAudioCaptureState.failed, terminal_decoded.audio_capture_state); + try testing.expectEqual(runtime_effects.EffectAudioCaptureReason.consumer_too_slow, terminal_decoded.audio_capture_reason); + try testing.expectEqual(@as(u32, 240_000), terminal_decoded.audio_capture_available_frames); } test "header, checkpoint, screenshot, and end codecs round-trip" { diff --git a/src/runtime/session_replay.zig b/src/runtime/session_replay.zig index b1a03a3a7..3c9bf6eec 100644 --- a/src/runtime/session_replay.zig +++ b/src/runtime/session_replay.zig @@ -30,6 +30,7 @@ const std = @import("std"); const canvas = @import("canvas"); const canvas_limits = @import("canvas_limits.zig"); const automation_protocol = @import("../automation/protocol.zig"); +const platform = @import("../platform/root.zig"); const core = @import("core.zig"); const journal = @import("session_journal.zig"); const runtime_effects = @import("effects.zig"); @@ -268,6 +269,20 @@ pub fn replaySession( ); return error.ReplayDamagedRecord; } + if (effect.kind == .audio_capture_read and audioCaptureReadRecordDamaged(effect)) { + std.debug.print( + "replay refused after event {d}: audio capture read for key {d} has an impossible state, PCM split, frame count, or gap count - a live reliable stream only records bounded paired s16le chunks and canonical empty/ended/rejected records; re-record the session\n", + .{ report.events_replayed, effect.key }, + ); + return error.ReplayDamagedRecord; + } + if (effect.kind == .audio_capture and audioCaptureRecordDamaged(effect)) { + std.debug.print( + "replay refused after event {d}: audio capture lifecycle for key {d} has a state, reason, format, or buffer counters the runtime ring cannot emit; re-record the session\n", + .{ report.events_replayed, effect.key }, + ); + return error.ReplayDamagedRecord; + } // Provenance consistency, gated BEFORE the regeneration // skip below: a `.rejected` stamped onto a delivered // record (nonzero token) would be skipped there and its @@ -595,6 +610,61 @@ fn audioScalarsDamaged(record: journal.EffectResultRecord) bool { record.audio_duration_ms >= max_exact; } +fn audioCaptureReadRecordDamaged(record: journal.EffectResultRecord) bool { + if (record.audio_capture_system_len > record.payload.len) return true; + const system_len: usize = record.audio_capture_system_len; + const microphone_len = record.payload.len - system_len; + const frames: usize = record.audio_capture_frames; + const max_frames = @as(usize, 48_000) * platform.audio_capture_max_read_duration_ms / 1_000; + const max_remaining = @as(usize, 48_000) * platform.max_audio_capture_buffer_duration_ms / 1_000; + if (frames > max_frames or @as(usize, record.audio_capture_remaining_frames) > max_remaining) return true; + if (record.audio_capture_system_gap_frames > frames or record.audio_capture_microphone_gap_frames > frames) return true; + if (record.audio_capture_sequence >= runtime_effects.max_effect_video_scalar_exclusive or + record.audio_capture_frame_offset >= runtime_effects.max_effect_video_scalar_exclusive or + record.audio_capture_frame_offset +| record.audio_capture_frames >= runtime_effects.max_effect_video_scalar_exclusive) return true; + const valid_source_len = struct { + fn check(len: usize, frame_count: usize) bool { + return len == 0 or len == frame_count * 2 or len == frame_count * 4; + } + }.check; + return switch (record.audio_capture_read_state) { + .chunk => record.audio_capture_read_reason != .none or frames == 0 or + (!valid_source_len(system_len, frames) or !valid_source_len(microphone_len, frames)) or + (system_len == 0 and microphone_len == 0) or + (record.audio_capture_end_of_stream and record.audio_capture_remaining_frames != 0), + .empty => record.audio_capture_read_reason != .none or frames != 0 or record.payload.len != 0 or + record.audio_capture_system_gap_frames != 0 or record.audio_capture_microphone_gap_frames != 0 or + record.audio_capture_end_of_stream, + .ended => record.audio_capture_read_reason != .none or frames != 0 or record.payload.len != 0 or + record.audio_capture_system_gap_frames != 0 or record.audio_capture_microphone_gap_frames != 0 or + record.audio_capture_remaining_frames != 0 or !record.audio_capture_end_of_stream, + .rejected => record.audio_capture_read_reason == .none or frames != 0 or record.payload.len != 0 or + record.audio_capture_system_gap_frames != 0 or record.audio_capture_microphone_gap_frames != 0 or + record.audio_capture_remaining_frames != 0 or record.audio_capture_end_of_stream, + }; +} + +fn audioCaptureRecordDamaged(record: journal.EffectResultRecord) bool { + const valid_rate = record.audio_capture_sample_rate_hz == 16_000 or + record.audio_capture_sample_rate_hz == 24_000 or + record.audio_capture_sample_rate_hz == 44_100 or + record.audio_capture_sample_rate_hz == 48_000; + if (!valid_rate or (record.audio_capture_channel_count != 1 and record.audio_capture_channel_count != 2)) return true; + const min_capacity = @as(u64, record.audio_capture_sample_rate_hz) * platform.min_audio_capture_buffer_duration_ms / 1_000; + const max_capacity = @as(u64, record.audio_capture_sample_rate_hz) * platform.max_audio_capture_buffer_duration_ms / 1_000; + if (record.audio_capture_available_frames > record.audio_capture_capacity_frames or + record.audio_capture_capacity_frames < min_capacity or + record.audio_capture_capacity_frames > max_capacity or + record.audio_capture_frames_produced < record.audio_capture_available_frames or + record.audio_capture_frames_produced >= runtime_effects.max_effect_video_scalar_exclusive) return true; + return switch (record.audio_capture_state) { + .started => record.audio_capture_reason != .none, + .readable => record.audio_capture_reason != .none or record.audio_capture_available_frames == 0, + .stopped => record.audio_capture_reason != .none, + .failed, .rejected => record.audio_capture_reason == .none, + }; +} + /// A `.video_load` record's `video_kind` is the load's OUTCOME, and the /// recorder writes exactly two values: `.loaded` on a resolved cascade /// and `.failed` on a synchronous refusal. Any other kind steers @@ -707,7 +777,8 @@ fn effectRegeneratesUnderReplay(record: journal.EffectResultRecord) bool { // Launch-env deliveries are exactly what must NOT regenerate: // the recorded values feed the replayed envMsgs dispatch so the // replay launch's environment is never consulted. - .line, .clock, .env => false, + .audio_capture_read => record.audio_capture_read_state == .rejected, + .line, .clock, .env, .audio_capture => false, }; } diff --git a/src/runtime/session_tests.zig b/src/runtime/session_tests.zig index a5e94e238..398c01978 100644 --- a/src/runtime/session_tests.zig +++ b/src/runtime/session_tests.zig @@ -633,6 +633,234 @@ test "a native-only session records and replays like a web-layer one" { try std.testing.expectEqual(recorded.fingerprint, replayed.fingerprint); } +const AudioCaptureSessionModel = struct { + capture_events: u32 = 0, + capture_state: u8 = 0, + capture_reason: u8 = 0, + sample_rate_hz: u32 = 0, + channel_count: u8 = 0, + capture_available_frames: u32 = 0, + capture_capacity_frames: u32 = 0, + capture_frames_produced: u64 = 0, + read_events: u32 = 0, + read_frames: u32 = 0, + system_pcm_hash: u64 = 0, + microphone_pcm_hash: u64 = 0, + system_gap_frames: u32 = 0, + microphone_gap_frames: u32 = 0, + end_of_stream: bool = false, + microphone_records: u32 = 0, + microphone_listing_completed: bool = false, + microphone_access: effects_mod.EffectAudioCaptureAccessStatus = .unavailable, + restart_required: bool = false, + device_changes: u32 = 0, +}; + +const AudioCaptureSessionMsg = union(enum) { + start, + read, + stop, + list_microphones, + check_access, + capture: effects_mod.EffectAudioCapture, + capture_read: effects_mod.EffectAudioCaptureRead, + microphone: effects_mod.EffectMicrophoneDevice, + access: effects_mod.EffectAudioCaptureAccess, + microphones_changed, +}; + +const AudioCaptureSessionApp = ui_app_mod.UiApp(AudioCaptureSessionModel, AudioCaptureSessionMsg); + +fn audioCaptureSessionBoot(_: *AudioCaptureSessionModel, fx: *AudioCaptureSessionApp.Effects) void { + fx.observeMicrophoneDevices(AudioCaptureSessionApp.Effects.microphoneDevicesChangedMsg(.microphones_changed)); +} + +fn audioCaptureSessionUpdate(model: *AudioCaptureSessionModel, msg: AudioCaptureSessionMsg, fx: *AudioCaptureSessionApp.Effects) void { + switch (msg) { + .start => fx.startAudioCapture(.{ + .key = 41, + .system_audio = true, + .microphone = .default, + .on_event = AudioCaptureSessionApp.Effects.audioCaptureMsg(.capture), + }), + .stop => fx.stopAudioCapture(), + .read => fx.readAudioCapture(.{ + .key = 41, + .max_frames = 4_800, + .on_read = AudioCaptureSessionApp.Effects.audioCaptureReadMsg(.capture_read), + }), + .list_microphones => fx.listMicrophoneDevices(.{ + .key = 42, + .on_event = AudioCaptureSessionApp.Effects.microphoneDeviceMsg(.microphone), + }), + .check_access => fx.audioCaptureAccess(.{ + .key = 43, + .source = .microphone, + .on_event = AudioCaptureSessionApp.Effects.audioCaptureAccessMsg(.access), + }), + .capture => |event| { + model.capture_events += 1; + model.capture_state = @intFromEnum(event.state) + 1; + model.capture_reason = @intFromEnum(event.reason); + model.sample_rate_hz = event.sample_rate_hz; + model.channel_count = event.channel_count; + model.capture_available_frames = event.available_frames; + model.capture_capacity_frames = event.capacity_frames; + model.capture_frames_produced = event.frames_produced; + }, + .capture_read => |event| { + model.read_events += 1; + model.read_frames += event.frames; + if (event.frames > 0) { + model.system_pcm_hash = std.hash.Wyhash.hash(0, event.system_pcm); + model.microphone_pcm_hash = std.hash.Wyhash.hash(0, event.microphone_pcm); + } + model.system_gap_frames += event.system_gap_frames; + model.microphone_gap_frames += event.microphone_gap_frames; + model.end_of_stream = event.end_of_stream; + }, + .microphone => |event| switch (event.state) { + .device => model.microphone_records += 1, + .completed => model.microphone_listing_completed = true, + .failed, .rejected => {}, + }, + .access => |event| { + model.microphone_access = event.status; + model.restart_required = event.restart_required; + }, + .microphones_changed => model.device_changes += 1, + } +} + +fn audioCaptureSessionView(ui: *AudioCaptureSessionApp.Ui, model: *const AudioCaptureSessionModel) AudioCaptureSessionApp.Ui.Node { + return ui.text(.{}, ui.fmt("capture {d} · microphones {d}", .{ model.capture_events, model.microphone_records })); +} + +fn audioCaptureSessionCommand(name: []const u8) ?AudioCaptureSessionMsg { + if (std.mem.eql(u8, name, "capture.start")) return .start; + if (std.mem.eql(u8, name, "capture.read")) return .read; + if (std.mem.eql(u8, name, "capture.stop")) return .stop; + if (std.mem.eql(u8, name, "capture.microphones")) return .list_microphones; + if (std.mem.eql(u8, name, "capture.access")) return .check_access; + return null; +} + +fn audioCaptureSessionOptions() AudioCaptureSessionApp.Options { + return .{ + .name = "audio-capture-session", + .scene = session_scene, + .canvas_label = canvas_label, + .init_fx = audioCaptureSessionBoot, + .update_fx = audioCaptureSessionUpdate, + .view = audioCaptureSessionView, + .on_command = audioCaptureSessionCommand, + }; +} + +test "audio capture device and access platform events replay without hardware side effects" { + const gpa = std.testing.allocator; + const buffer = try std.heap.page_allocator.create(JournalBuffer); + defer std.heap.page_allocator.destroy(buffer); + buffer.len = 0; + + const recorder = try std.heap.page_allocator.create(session_record.SessionRecorder); + defer std.heap.page_allocator.destroy(recorder); + recorder.* = session_record.SessionRecorder.init(buffer.sink()); + recorder.begin(.{ .platform_name = "test", .app_name = "audio-capture-session", .window_width = 400, .window_height = 300 }); + + const record_harness = try core.TestHarness().create(gpa, .{ .size = geometry.SizeF.init(400, 300) }); + defer record_harness.destroy(gpa); + record_harness.null_platform.gpu_surfaces = true; + record_harness.runtime.options.session_recorder = recorder; + + const recorded_app = try gpa.create(AudioCaptureSessionApp); + defer gpa.destroy(recorded_app); + recorded_app.* = AudioCaptureSessionApp.init(std.heap.page_allocator, .{}, audioCaptureSessionOptions()); + defer recorded_app.deinit(); + const app = recorded_app.app(); + + try record_harness.start(app); + try record_harness.runtime.dispatchPlatformEvent(app, .{ .gpu_surface_frame = .{ + .label = canvas_label, + .size = geometry.SizeF.init(400, 300), + .scale_factor = 2, + .frame_index = 1, + .timestamp_ns = 1_000_000, + } }); + try record_harness.runtime.dispatchPlatformEvent(app, .frame_requested); + + try record_harness.runtime.dispatchPlatformEvent(app, .{ .menu_command = .{ .name = "capture.start", .window_id = 1 } }); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.takeAudioCaptureStarted().?); + const system_pcm: [3_840]u8 = @splat(0x21); + const microphone_pcm: [3_840]u8 = @splat(0x42); + try std.testing.expectEqual(platform.AudioCapturePushResult.accepted, record_harness.null_platform.pushAudioCapturePair(.{ + .frame_offset = 0, + .frame_count = 960, + .system_pcm = &system_pcm, + .microphone_pcm = µphone_pcm, + .system_gap_frames = 7, + .microphone_gap_frames = 11, + })); + try record_harness.runtime.dispatchPlatformEvent(app, .wake); + try record_harness.runtime.dispatchPlatformEvent(app, .{ .menu_command = .{ .name = "capture.read", .window_id = 1 } }); + try record_harness.runtime.dispatchPlatformEvent(app, .wake); + try record_harness.runtime.dispatchPlatformEvent(app, .{ .menu_command = .{ .name = "capture.stop", .window_id = 1 } }); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.completeAudioCapture().?); + try record_harness.runtime.dispatchPlatformEvent(app, .{ .menu_command = .{ .name = "capture.read", .window_id = 1 } }); + try record_harness.runtime.dispatchPlatformEvent(app, .wake); + + try record_harness.runtime.dispatchPlatformEvent(app, .{ .menu_command = .{ .name = "capture.microphones", .window_id = 1 } }); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.microphoneDeviceEvent(0)); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.microphoneDeviceEvent(1)); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.microphoneDeviceEvent(2)); + + try record_harness.runtime.dispatchPlatformEvent(app, .{ .menu_command = .{ .name = "capture.access", .window_id = 1 } }); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.takeAudioCaptureAccess().?); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.microphoneDevicesChanged().?); + try record_harness.runtime.dispatchPlatformEvent(app, .frame_requested); + + recorder.finish(); + try std.testing.expect(!recorder.failed); + const recorded_model = recorded_app.model; + const recorded_fingerprint = record_harness.runtime.sessionStateFingerprint(); + try std.testing.expectEqual(@as(u32, 3), recorded_model.capture_events); + try std.testing.expectEqual(@as(u32, 48_000), recorded_model.sample_rate_hz); + try std.testing.expectEqual(@as(u8, 2), recorded_model.channel_count); + try std.testing.expectEqual(@as(u32, 0), recorded_model.capture_available_frames); + try std.testing.expectEqual(@as(u32, 240_000), recorded_model.capture_capacity_frames); + try std.testing.expectEqual(@as(u64, 960), recorded_model.capture_frames_produced); + try std.testing.expectEqual(@as(u32, 2), recorded_model.read_events); + try std.testing.expectEqual(@as(u32, 960), recorded_model.read_frames); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, &system_pcm), recorded_model.system_pcm_hash); + try std.testing.expectEqual(std.hash.Wyhash.hash(0, µphone_pcm), recorded_model.microphone_pcm_hash); + try std.testing.expectEqual(@as(u32, 7), recorded_model.system_gap_frames); + try std.testing.expectEqual(@as(u32, 11), recorded_model.microphone_gap_frames); + try std.testing.expect(recorded_model.end_of_stream); + try std.testing.expectEqual(@as(u32, 2), recorded_model.microphone_records); + try std.testing.expect(recorded_model.microphone_listing_completed); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureAccessStatus.authorized, recorded_model.microphone_access); + try std.testing.expectEqual(@as(u32, 1), recorded_model.device_changes); + + const replay_harness = try core.TestHarness().create(gpa, .{ .size = geometry.SizeF.init(400, 300) }); + defer replay_harness.destroy(gpa); + replay_harness.null_platform.gpu_surfaces = true; + const replayed_app = try gpa.create(AudioCaptureSessionApp); + defer gpa.destroy(replayed_app); + replayed_app.* = AudioCaptureSessionApp.init(std.heap.page_allocator, .{}, audioCaptureSessionOptions()); + defer replayed_app.deinit(); + + const report = try session_replay.replaySession(&replay_harness.runtime, replayed_app.app(), buffer.journalBytes(), .{ + .verify = true, + .require_same_platform = false, + }); + try std.testing.expect(report.ok()); + try std.testing.expectEqual(@as(usize, 0), replay_harness.null_platform.audio_capture_start_count); + try std.testing.expectEqual(@as(usize, 0), replay_harness.null_platform.audio_capture_stop_count); + try std.testing.expectEqual(@as(usize, 0), replay_harness.null_platform.microphone_devices_count); + try std.testing.expectEqualDeep(recorded_model, replayed_app.model); + try std.testing.expectEqual(recorded_fingerprint, replay_harness.runtime.sessionStateFingerprint()); +} + test "accessibility actions journal once and replay without double-dispatch" { // A journaled `widget_accessibility_action` re-runs its verb on // replay, and the verb synthesizes REAL platform events (press its @@ -2900,10 +3128,11 @@ test "a journal referencing blobs refuses to replay without its blob store" { /// fields — video_kind (1), position (8), duration (8), playing (1), /// buffering (1), width (8), height (8), token (8), source (1), /// handled (1) — then the pty fields — pty_kind (1), pty_signal (4), -/// pty_dropped_writes (4), pty_blob_hash (16), pty_blob_len (8). The -/// image fields the damage helpers below patch sit immediately before -/// it. -const effect_post_image_trailer_len: usize = 83; +/// pty_dropped_writes (4), pty_blob_hash (16), pty_blob_len (8) — then +/// the reliable audio-read fields (39) and the runtime-generated audio +/// readiness fields (23). The image fields the damage +/// helpers below patch sit immediately before it. +const effect_post_image_trailer_len: usize = 83 + 39 + 23; /// Zero the `image_blob_len` field — the last eight bytes before the /// post-image trailer of the effect payload, see `journal.encodeEffect` diff --git a/src/runtime/ts_core_host.zig b/src/runtime/ts_core_host.zig index 33562b992..c86114699 100644 --- a/src/runtime/ts_core_host.zig +++ b/src/runtime/ts_core_host.zig @@ -1,6 +1,6 @@ //! The native host consumer for transpiled app cores: bridges the //! versioned command/subscription wire format a transpiled core emits -//! (`packages/core/rt/rt.zig`, `cmd_format_version` 3) onto the +//! (`packages/core/rt/rt.zig`, `cmd_format_version` 5) onto the //! real effect engine (`effects.zig`). The transpiler's output is a //! pure Model/Msg/update core whose effects are INERT BYTES — this //! module is the one place those bytes become engine calls, so the @@ -390,6 +390,10 @@ pub const spawn_key_base: u64 = 0x5453_5350_0000_0000; /// ("TSAU"). Audio keys are their own engine namespace and one player /// is the whole surface, so one constant key is the honest shape. pub const audio_key_base: u64 = 0x5453_4155_0000_0000; +pub const audio_capture_key_base: u64 = 0x5453_4143_0000_0000; +pub const audio_capture_read_key_base: u64 = 0x5453_4152_0000_0000; +pub const microphone_devices_key_base: u64 = 0x5453_4D44_0000_0000; +pub const audio_capture_access_key_base: u64 = 0x5453_4141_0000_0000; /// The engine-key namespace of the bridge's single video playback /// channel ("TSVI") — the audio key's twin, except the low byte @@ -544,6 +548,20 @@ pub fn TsCoreHost(comptime core: type) type { } }; + const RoutedStreamEntry = AudioEntry; + + const AudioCaptureReadEntry = struct { + used: bool = false, + key_len: usize = 0, + key: [max_wire_key_bytes]u8 = undefined, + event_tag: u8 = 0, + effect_key: u64 = 0, + + fn wireKey(entry: *const AudioCaptureReadEntry) []const u8 { + return entry.key[0..entry.key_len]; + } + }; + /// The single video stream entry — the audio entry's exact /// shape (one player is the whole engine surface). Non-retiring: /// video_ctl `stop` closes it, a new video_load re-keys and @@ -621,6 +639,11 @@ pub fn TsCoreHost(comptime core: type) type { var delays: [runtime_effects.max_effect_timers]DelayEntry = @splat(.{}); var streams: [runtime_effects.max_effects]StreamEntry = @splat(.{}); var audio_entry: AudioEntry = .{}; + var audio_capture_entries: [runtime_effects.max_effects]RoutedStreamEntry = @splat(.{}); + var audio_capture_read_entries: [runtime_effects.max_effects]AudioCaptureReadEntry = @splat(.{}); + var microphone_device_entries: [runtime_effects.max_effects]RoutedStreamEntry = @splat(.{}); + var audio_capture_access_entries: [runtime_effects.max_effects]RoutedStreamEntry = @splat(.{}); + var microphone_devices_changed_tag: ?u8 = null; var video_entry: VideoEntry = .{}; var images: [runtime_effects.max_effects]ImageEntry = @splat(.{}); var channels: [runtime_effects.max_effect_channels]ChannelEntry = @splat(.{}); @@ -695,6 +718,11 @@ pub fn TsCoreHost(comptime core: type) type { delays = @splat(.{}); streams = @splat(.{}); audio_entry = .{}; + audio_capture_entries = @splat(.{}); + audio_capture_read_entries = @splat(.{}); + microphone_device_entries = @splat(.{}); + audio_capture_access_entries = @splat(.{}); + microphone_devices_changed_tag = null; video_entry = .{}; images = @splat(.{}); channels = @splat(.{}); @@ -848,7 +876,7 @@ pub fn TsCoreHost(comptime core: type) type { // ------------------------------------------------- command walk - /// Walk one command value (v3 wire format; batch is plain + /// Walk one command value (v5 wire format; batch is plain /// concatenation, the empty slice is `Cmd.none`). fn runCmd( fx: *Fx, @@ -1217,6 +1245,104 @@ pub fn TsCoreHost(comptime core: type) type { // retiring the entry in ptyEventMsg. if (findPty(key)) |index| fx.ptyKill(pty_key_base + index); }, + // audio_capture_start [op][key][event][flags][mic kind] + // [sample rate u32][channels][buffer duration u32][device id long] + 0x1D => { + const key = takeShortBytes(cmd, &at); + const event_tag = takeByte(cmd, &at); + const flags = takeByte(cmd, &at); + const microphone_kind = takeByte(cmd, &at); + const rate_bytes = takeBytes(cmd, &at, 4); + const sample_rate = std.mem.readInt(u32, rate_bytes[0..4], .little); + const channels_count = takeByte(cmd, &at); + const buffer_bytes = takeBytes(cmd, &at, 4); + const buffer_duration_ms = std.mem.readInt(u32, buffer_bytes[0..4], .little); + const device_id = takeLongBytes(cmd, &at); + const index = allocRoutedStreamEntry(&audio_capture_entries, key, event_tag) orelse + @panic("ts core host: more pending audio capture starts than the effects table can route"); + const microphone: runtime_effects.MicrophoneSelection = switch (microphone_kind) { + 0 => .none, + 1 => .default, + 2 => .{ .device_id = device_id }, + else => @panic("ts core host: invalid microphone selection wire value"), + }; + fx.startAudioCapture(.{ + .key = audio_capture_key_base + index, + .system_audio = (flags & 1) != 0, + .microphone = microphone, + .sample_rate_hz = sample_rate, + .channel_count = channels_count, + .exclude_current_process_audio = (flags & 2) != 0, + .buffer_duration_ms = buffer_duration_ms, + .on_event = audioCaptureEventMsg, + }); + }, + // audio_capture_stop [op][key] + 0x1E => { + const key = takeShortBytes(cmd, &at); + for (&audio_capture_entries) |*entry| { + if (entry.used and std.mem.eql(u8, entry.wireKey(), key)) { + fx.stopAudioCapture(); + break; + } + } + }, + // microphone_devices [op][key][event] + 0x1F => { + const key = takeShortBytes(cmd, &at); + const event_tag = takeByte(cmd, &at); + const index = allocRoutedStreamEntry(µphone_device_entries, key, event_tag) orelse + @panic("ts core host: more pending microphone device queries than the effects table can route"); + fx.listMicrophoneDevices(.{ .key = microphone_devices_key_base + index, .on_event = microphoneDeviceEventMsg }); + }, + // audio_capture_access [op][key][event][source][action] + 0x20 => { + const key = takeShortBytes(cmd, &at); + const event_tag = takeByte(cmd, &at); + const source_byte = takeByte(cmd, &at); + const action_byte = takeByte(cmd, &at); + const index = allocRoutedStreamEntry(&audio_capture_access_entries, key, event_tag) orelse + @panic("ts core host: more pending audio access queries than the effects table can route"); + fx.audioCaptureAccess(.{ + .key = audio_capture_access_key_base + index, + .source = if (source_byte == 1) .microphone else .system_audio, + .action = if (action_byte == 1) .request else .status, + .on_event = audioCaptureAccessEventMsg, + }); + }, + // audio_capture_read [op][key][event][max frames u32] + 0x21 => { + const key = takeShortBytes(cmd, &at); + const event_tag = takeByte(cmd, &at); + const frame_bytes = takeBytes(cmd, &at, 4); + const max_frames = std.mem.readInt(u32, frame_bytes[0..4], .little); + var effect_key: ?u64 = null; + for (&audio_capture_entries, 0..) |*entry, index| { + if (entry.used and std.mem.eql(u8, entry.wireKey(), key)) { + effect_key = audio_capture_key_base + index; + break; + } + } + const read_index = allocAudioCaptureReadEntry(key, event_tag, effect_key orelse audio_capture_read_key_base) orelse + @panic("ts core host: more pending audio capture reads than the effects table can route"); + if (effect_key == null) audio_capture_read_entries[read_index].effect_key = audio_capture_read_key_base + read_index; + fx.readAudioCapture(.{ + .key = audio_capture_read_entries[read_index].effect_key, + .max_frames = max_frames, + .on_read = audioCaptureReadEventMsg, + }); + }, + // audio_capture_discard [op][key] + 0x22 => { + const key = takeShortBytes(cmd, &at); + for (&audio_capture_entries) |*entry| { + if (entry.used and std.mem.eql(u8, entry.wireKey(), key)) { + fx.discardAudioCapture(); + entry.used = false; + break; + } + } + }, else => @panic("ts core host: unknown command wire record - the core and this runtime disagree on cmd_format_version"), } } @@ -1225,6 +1351,31 @@ pub fn TsCoreHost(comptime core: type) type { /// The shared routed-op head: [key_len][key][ok_tag][err_tag]. const RoutedHead = struct { key: []const u8, ok_tag: u8, err_tag: u8 }; + fn allocRoutedStreamEntry(entries: []RoutedStreamEntry, key: []const u8, event_tag: u8) ?usize { + for (entries, 0..) |*entry, index| { + if (entry.used) continue; + entry.used = true; + entry.key_len = key.len; + @memcpy(entry.key[0..key.len], key); + entry.event_tag = event_tag; + return index; + } + return null; + } + + fn allocAudioCaptureReadEntry(key: []const u8, event_tag: u8, effect_key: u64) ?usize { + for (&audio_capture_read_entries, 0..) |*entry, index| { + if (entry.used) continue; + entry.used = true; + entry.key_len = key.len; + @memcpy(entry.key[0..key.len], key); + entry.event_tag = event_tag; + entry.effect_key = effect_key; + return index; + } + return null; + } + fn takeRoutedHead(cmd: []const u8, at: *usize) RoutedHead { const key = takeShortBytes(cmd, at); const ok_tag = takeByte(cmd, at); @@ -1473,6 +1624,62 @@ pub fn TsCoreHost(comptime core: type) type { return msgFromTagAudio(audio_entry.event_tag, event); } + fn audioCaptureEventMsg(event: runtime_effects.EffectAudioCapture) Msg { + if (event.key < audio_capture_key_base) @panic("ts core host: audio capture event outside bridge namespace"); + const index = event.key - audio_capture_key_base; + if (index >= audio_capture_entries.len or !audio_capture_entries[index].used) @panic("ts core host: audio capture event has no routed command"); + const entry = &audio_capture_entries[index]; + const tag = entry.event_tag; + const key = entry.wireKey(); + const msg = msgFromTagAudioCapture(tag, key, event); + return msg; + } + + fn audioCaptureReadEventMsg(event: runtime_effects.EffectAudioCaptureRead) Msg { + for (&audio_capture_read_entries) |*entry| { + if (!entry.used or entry.effect_key != event.key) continue; + const msg = msgFromTagAudioCaptureRead(entry.event_tag, entry.wireKey(), event); + entry.used = false; + if (event.end_of_stream) { + for (&audio_capture_entries, 0..) |*capture, index| { + if (capture.used and audio_capture_key_base + index == event.key) { + capture.used = false; + break; + } + } + } + return msg; + } + @panic("ts core host: audio capture read event has no routed command"); + } + + fn microphoneDeviceEventMsg(event: runtime_effects.EffectMicrophoneDevice) Msg { + if (event.key < microphone_devices_key_base) @panic("ts core host: microphone event outside bridge namespace"); + const index = event.key - microphone_devices_key_base; + if (index >= microphone_device_entries.len or !microphone_device_entries[index].used) @panic("ts core host: microphone event has no routed command"); + const entry = µphone_device_entries[index]; + const tag = entry.event_tag; + const key = entry.wireKey(); + const msg = msgFromTagMicrophoneDevice(tag, key, event); + if (event.state != .device) entry.used = false; + return msg; + } + + fn audioCaptureAccessEventMsg(event: runtime_effects.EffectAudioCaptureAccess) Msg { + if (event.key < audio_capture_access_key_base) @panic("ts core host: audio access event outside bridge namespace"); + const index = event.key - audio_capture_access_key_base; + if (index >= audio_capture_access_entries.len or !audio_capture_access_entries[index].used) @panic("ts core host: audio access event has no routed command"); + const entry = &audio_capture_access_entries[index]; + const msg = msgFromTagAudioCaptureAccess(entry.event_tag, entry.wireKey(), event); + entry.used = false; + return msg; + } + + fn microphoneDevicesChangedMsg() Msg { + return msgFromTagVoid(microphone_devices_changed_tag orelse + @panic("ts core host: microphone device invalidation arrived without a subscription")); + } + // ------------------------------------------------- video stream /// The video_ctl record: drive the single playback channel, @@ -2119,9 +2326,17 @@ pub fn TsCoreHost(comptime core: type) type { if (comptime !has_subscriptions) return; const subs = core.subscriptions(model_root); var seen = [_]bool{false} ** timers.len; + var microphone_devices_seen = false; + var next_microphone_devices_tag: u8 = 0; var at: usize = 0; while (at < subs.len) { const op = takeByte(subs, &at); + if (op == 0x02) { + if (microphone_devices_seen) @panic("ts core host: duplicate Sub.microphoneDevicesChanged descriptor"); + microphone_devices_seen = true; + next_microphone_devices_tag = takeByte(subs, &at); + continue; + } if (op != 0x01) { @panic("ts core host: unknown subscription wire record - the core and this runtime disagree on cmd_format_version"); } @@ -2180,6 +2395,14 @@ pub fn TsCoreHost(comptime core: type) type { fx.cancelTimer(timer_key_base + index); } } + const microphone_devices_was_active = microphone_devices_changed_tag != null; + if (microphone_devices_seen) { + microphone_devices_changed_tag = next_microphone_devices_tag; + if (!microphone_devices_was_active) fx.observeMicrophoneDevices(microphoneDevicesChangedMsg); + } else if (microphone_devices_was_active) { + microphone_devices_changed_tag = null; + fx.observeMicrophoneDevices(null); + } } fn freeTimerIndex() ?usize { @@ -2397,6 +2620,212 @@ pub fn TsCoreHost(comptime core: type) type { @panic("ts core host: an audio event names a Msg tag outside the union"); } + fn enumValueNamed(comptime E: type, name: []const u8) E { + inline for (@typeInfo(E).@"enum".fields) |field| { + if (std.mem.eql(u8, field.name, name)) return @enumFromInt(field.value); + } + @panic("ts core host: event enum member missing from routed Msg arm"); + } + + fn copyFrameBytes(bytes: []const u8) []const u8 { + if (bytes.len == 0) return ""; + const copy = core.rt.frameAlloc(u8, bytes.len); + @memcpy(copy, bytes); + return copy; + } + + fn audioCaptureArmShape(comptime T: type) bool { + const info = @typeInfo(T); + if (info != .@"struct" or info.@"struct".fields.len != 8) return false; + var ok = true; + for (info.@"struct".fields) |field| { + if (std.mem.eql(u8, field.name, "key")) { + if (field.type != []const u8) ok = false; + } else if (std.mem.eql(u8, field.name, "state") or std.mem.eql(u8, field.name, "reason")) { + if (@typeInfo(field.type) != .@"enum") ok = false; + } else if (std.mem.eql(u8, field.name, "sampleRate") or std.mem.eql(u8, field.name, "channels") or + std.mem.eql(u8, field.name, "availableFrames") or std.mem.eql(u8, field.name, "capacityFrames") or + std.mem.eql(u8, field.name, "framesProduced")) + { + if (field.type != i64 and field.type != u64 and field.type != f64) ok = false; + } else ok = false; + } + return ok; + } + + fn msgFromTagAudioCapture(tag: u8, wire_key: []const u8, event: runtime_effects.EffectAudioCapture) Msg { + inline for (msg_arms, 0..) |arm, index| if (tag == index) { + if (comptime audioCaptureArmShape(arm.type)) { + var payload: arm.type = undefined; + inline for (@typeInfo(arm.type).@"struct".fields) |field| { + if (comptime std.mem.eql(u8, field.name, "key")) { + @field(payload, field.name) = copyFrameBytes(wire_key); + } else if (comptime std.mem.eql(u8, field.name, "state")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.state)); + } else if (comptime std.mem.eql(u8, field.name, "reason")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.reason)); + } else if (comptime std.mem.eql(u8, field.name, "sampleRate")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.sample_rate_hz) else @intCast(event.sample_rate_hz); + } else if (comptime std.mem.eql(u8, field.name, "channels")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.channel_count) else @intCast(event.channel_count); + } else if (comptime std.mem.eql(u8, field.name, "availableFrames")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.available_frames) else @intCast(event.available_frames); + } else if (comptime std.mem.eql(u8, field.name, "capacityFrames")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.capacity_frames) else @intCast(event.capacity_frames); + } else if (comptime std.mem.eql(u8, field.name, "framesProduced")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.frames_produced) else @intCast(event.frames_produced); + } else unreachable; + } + return @unionInit(Msg, arm.name, payload); + } + @panic("ts core host: invalid audio capture event arm shape"); + }; + @panic("ts core host: audio capture event tag outside Msg union"); + } + + fn audioCaptureReadArmShape(comptime T: type) bool { + const info = @typeInfo(T); + if (info != .@"struct" or info.@"struct".fields.len != 12) return false; + var ok = true; + for (info.@"struct".fields) |field| { + if (std.mem.eql(u8, field.name, "key") or std.mem.eql(u8, field.name, "systemPcm") or std.mem.eql(u8, field.name, "microphonePcm")) { + if (field.type != []const u8) ok = false; + } else if (std.mem.eql(u8, field.name, "state") or std.mem.eql(u8, field.name, "reason")) { + if (@typeInfo(field.type) != .@"enum") ok = false; + } else if (std.mem.eql(u8, field.name, "endOfStream")) { + if (field.type != bool) ok = false; + } else if (std.mem.eql(u8, field.name, "sequence") or std.mem.eql(u8, field.name, "frameOffset") or + std.mem.eql(u8, field.name, "frames") or std.mem.eql(u8, field.name, "systemGapFrames") or + std.mem.eql(u8, field.name, "microphoneGapFrames") or std.mem.eql(u8, field.name, "remainingFrames")) + { + if (field.type != i64 and field.type != u64 and field.type != f64) ok = false; + } else ok = false; + } + return ok; + } + + fn msgFromTagAudioCaptureRead(tag: u8, wire_key: []const u8, event: runtime_effects.EffectAudioCaptureRead) Msg { + inline for (msg_arms, 0..) |arm, index| if (tag == index) { + if (comptime audioCaptureReadArmShape(arm.type)) { + var payload: arm.type = undefined; + inline for (@typeInfo(arm.type).@"struct".fields) |field| { + if (comptime std.mem.eql(u8, field.name, "key")) { + @field(payload, field.name) = copyFrameBytes(wire_key); + } else if (comptime std.mem.eql(u8, field.name, "state")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.state)); + } else if (comptime std.mem.eql(u8, field.name, "reason")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.reason)); + } else if (comptime std.mem.eql(u8, field.name, "systemPcm")) { + @field(payload, field.name) = copyFrameBytes(event.system_pcm); + } else if (comptime std.mem.eql(u8, field.name, "microphonePcm")) { + @field(payload, field.name) = copyFrameBytes(event.microphone_pcm); + } else if (comptime std.mem.eql(u8, field.name, "endOfStream")) { + @field(payload, field.name) = event.end_of_stream; + } else if (comptime std.mem.eql(u8, field.name, "sequence")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.sequence) else @intCast(event.sequence); + } else if (comptime std.mem.eql(u8, field.name, "frameOffset")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.frame_offset) else @intCast(event.frame_offset); + } else if (comptime std.mem.eql(u8, field.name, "frames")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.frames) else @intCast(event.frames); + } else if (comptime std.mem.eql(u8, field.name, "systemGapFrames")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.system_gap_frames) else @intCast(event.system_gap_frames); + } else if (comptime std.mem.eql(u8, field.name, "microphoneGapFrames")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.microphone_gap_frames) else @intCast(event.microphone_gap_frames); + } else if (comptime std.mem.eql(u8, field.name, "remainingFrames")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.remaining_frames) else @intCast(event.remaining_frames); + } else unreachable; + } + return @unionInit(Msg, arm.name, payload); + } + @panic("ts core host: invalid audio capture read event arm shape"); + }; + @panic("ts core host: audio capture read event tag outside Msg union"); + } + + fn microphoneDeviceArmShape(comptime T: type) bool { + const info = @typeInfo(T); + if (info != .@"struct" or info.@"struct".fields.len != 7) return false; + var ok = true; + for (info.@"struct".fields) |field| { + if (std.mem.eql(u8, field.name, "key") or std.mem.eql(u8, field.name, "id") or std.mem.eql(u8, field.name, "name")) { + if (field.type != []const u8) ok = false; + } else if (std.mem.eql(u8, field.name, "state")) { + if (@typeInfo(field.type) != .@"enum") ok = false; + } else if (std.mem.eql(u8, field.name, "isDefault")) { + if (field.type != bool) ok = false; + } else if (std.mem.eql(u8, field.name, "index") or std.mem.eql(u8, field.name, "total")) { + if (field.type != i64 and field.type != u64 and field.type != f64) ok = false; + } else ok = false; + } + return ok; + } + + fn msgFromTagMicrophoneDevice(tag: u8, wire_key: []const u8, event: runtime_effects.EffectMicrophoneDevice) Msg { + inline for (msg_arms, 0..) |arm, index| if (tag == index) { + if (comptime microphoneDeviceArmShape(arm.type)) { + var payload: arm.type = undefined; + inline for (@typeInfo(arm.type).@"struct".fields) |field| { + if (comptime std.mem.eql(u8, field.name, "key")) { + @field(payload, field.name) = copyFrameBytes(wire_key); + } else if (comptime std.mem.eql(u8, field.name, "state")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.state)); + } else if (comptime std.mem.eql(u8, field.name, "id")) { + @field(payload, field.name) = copyFrameBytes(event.id); + } else if (comptime std.mem.eql(u8, field.name, "name")) { + @field(payload, field.name) = copyFrameBytes(event.name); + } else if (comptime std.mem.eql(u8, field.name, "isDefault")) { + @field(payload, field.name) = event.is_default; + } else if (comptime std.mem.eql(u8, field.name, "index")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.index) else @intCast(event.index); + } else { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.total) else @intCast(event.total); + } + } + return @unionInit(Msg, arm.name, payload); + } + @panic("ts core host: invalid microphone device event arm shape"); + }; + @panic("ts core host: microphone device event tag outside Msg union"); + } + + fn audioCaptureAccessArmShape(comptime T: type) bool { + const info = @typeInfo(T); + if (info != .@"struct" or info.@"struct".fields.len != 4) return false; + var ok = true; + for (info.@"struct".fields) |field| { + if (std.mem.eql(u8, field.name, "key")) { + if (field.type != []const u8) ok = false; + } else if (std.mem.eql(u8, field.name, "source") or std.mem.eql(u8, field.name, "status")) { + if (@typeInfo(field.type) != .@"enum") ok = false; + } else if (std.mem.eql(u8, field.name, "restartRequired")) { + if (field.type != bool) ok = false; + } else ok = false; + } + return ok; + } + + fn msgFromTagAudioCaptureAccess(tag: u8, wire_key: []const u8, event: runtime_effects.EffectAudioCaptureAccess) Msg { + inline for (msg_arms, 0..) |arm, index| if (tag == index) { + if (comptime audioCaptureAccessArmShape(arm.type)) { + var payload: arm.type = undefined; + inline for (@typeInfo(arm.type).@"struct".fields) |field| { + if (comptime std.mem.eql(u8, field.name, "key")) { + @field(payload, field.name) = copyFrameBytes(wire_key); + } else if (comptime std.mem.eql(u8, field.name, "source")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.source)); + } else if (comptime std.mem.eql(u8, field.name, "status")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.status)); + } else { + @field(payload, field.name) = event.restart_required; + } + } + return @unionInit(Msg, arm.name, payload); + } + @panic("ts core host: invalid audio capture access event arm shape"); + }; + @panic("ts core host: audio capture access event tag outside Msg union"); + } + /// Whether an arm payload struct is the video event record: the /// seven SDK-fixed fields, matched by NAME — `state` (any enum; /// its members are matched by member name at delivery), diff --git a/src/runtime/ts_core_host_tests.zig b/src/runtime/ts_core_host_tests.zig index bf55b77b5..3ca89e12e 100644 --- a/src/runtime/ts_core_host_tests.zig +++ b/src/runtime/ts_core_host_tests.zig @@ -83,6 +83,13 @@ const mini_core = struct { /// carries TWO name-matched unions). pub const PtyState = enum { exit, output }; pub const PtyReason = enum { cancelled, exited, rejected, spawn_failed, signaled }; + pub const CaptureState = enum { rejected, failed, stopped, readable, started }; + pub const CaptureReason = enum { unsupported, discarded, consumer_too_slow, no_audio, capture_failed, device_disconnected, device_not_found, already_recording, permission_required, permission_missing, invalid_options, none }; + pub const CaptureReadState = enum { rejected, ended, empty, chunk }; + pub const CaptureReadReason = enum { read_in_progress, not_recording, invalid_options, none }; + pub const DeviceState = enum { rejected, failed, completed, device }; + pub const AccessSource = enum { microphone, system_audio }; + pub const AccessStatus = enum { unavailable, restricted, denied, not_determined, not_authorized, authorized }; pub const Model = struct { polling: bool, @@ -145,6 +152,36 @@ const mini_core = struct { // Unsigned-class mirrors (u64-classed arm routing). ustamp_ms: u64, ucode: u64, + mic_watch: bool, + capture_state: CaptureState, + capture_reason: CaptureReason, + capture_sample_rate: f64, + capture_channels: f64, + capture_available: f64, + capture_capacity: f64, + capture_produced: f64, + capture_events: i64, + capture_read_state: CaptureReadState, + capture_read_reason: CaptureReadReason, + capture_read_sequence: f64, + capture_read_offset: f64, + capture_read_frames: f64, + capture_system_pcm: []const u8, + capture_microphone_pcm: []const u8, + capture_system_gaps: f64, + capture_microphone_gaps: f64, + capture_remaining: f64, + capture_end: bool, + capture_read_events: i64, + device_state: DeviceState, + device_total: f64, + device_default: bool, + device_events: i64, + access_source: AccessSource, + access_status: AccessStatus, + access_restart: bool, + access_events: i64, + device_change_events: i64, }; pub const Msg = union(enum) { @@ -289,6 +326,53 @@ const mini_core = struct { uget, // 81: fetch "uget" -> ufetched/failed ufetched: struct { status: u64, body: []const u8 }, // 82: fetch ok // record with a u64-classed number field + start_capture, // 83: combined capture -> capture_evt + stop_capture, // 84: stop the active capture key + list_mics, // 85: enumerate microphone records -> device_evt + capture_evt: struct { // 86 + key: []const u8, + state: CaptureState, + reason: CaptureReason, + sampleRate: f64, + channels: f64, + availableFrames: f64, + capacityFrames: f64, + framesProduced: f64, + }, + device_evt: struct { // 87 + key: []const u8, + state: DeviceState, + id: []const u8, + name: []const u8, + isDefault: bool, + index: f64, + total: f64, + }, + capture_access, // 88: microphone permission request -> access_evt + access_evt: struct { // 89 + key: []const u8, + source: AccessSource, + status: AccessStatus, + restartRequired: bool, + }, + toggle_mic_watch, // 90: subscription on/off + devices_changed, // 91: no-payload subscription event + read_capture, // 92: read paired PCM from "meeting" + discard_capture, // 93: release the retained stream + capture_read_evt: struct { // 94 + key: []const u8, + state: CaptureReadState, + reason: CaptureReadReason, + sequence: f64, + frameOffset: f64, + frames: f64, + systemPcm: []const u8, + microphonePcm: []const u8, + systemGapFrames: f64, + microphoneGapFrames: f64, + remainingFrames: f64, + endOfStream: bool, + }, }; pub const InitResult = struct { model: *const Model, cmd: []const u8 }; @@ -353,6 +437,36 @@ const mini_core = struct { .video2_events = 0, .ustamp_ms = 0, .ucode = 0, + .mic_watch = false, + .capture_state = .rejected, + .capture_reason = .none, + .capture_sample_rate = 0, + .capture_channels = 0, + .capture_available = 0, + .capture_capacity = 0, + .capture_produced = 0, + .capture_events = 0, + .capture_read_state = .empty, + .capture_read_reason = .none, + .capture_read_sequence = 0, + .capture_read_offset = 0, + .capture_read_frames = 0, + .capture_system_pcm = "", + .capture_microphone_pcm = "", + .capture_system_gaps = 0, + .capture_microphone_gaps = 0, + .capture_remaining = 0, + .capture_end = false, + .capture_read_events = 0, + .device_state = .completed, + .device_total = 0, + .device_default = false, + .device_events = 0, + .access_source = .microphone, + .access_status = .unavailable, + .access_restart = false, + .access_events = 0, + .device_change_events = 0, }), .cmd = cmdRequest("status.read", "status", 7, 8, "boot"), }; @@ -640,6 +754,66 @@ const mini_core = struct { @memcpy(out[first.len..], second); return .{ .model = model, .cmd = out }; }, + .start_capture => return .{ .model = model, .cmd = cmdAudioCaptureStart("meeting", 86, true, 2, "usb-mic", 44_100, 1, true, 5_000) }, + .stop_capture => return .{ .model = model, .cmd = cmdKeyOnly(0x1E, "meeting") }, + .read_capture => return .{ .model = model, .cmd = cmdAudioCaptureRead("meeting", 94, 882) }, + .discard_capture => return .{ .model = model, .cmd = cmdKeyOnly(0x22, "meeting") }, + .list_mics => return .{ .model = model, .cmd = cmdRoutedKey(0x1F, "mics", 87) }, + .capture_evt => |event| { + const out = frameCreate(model.*); + out.capture_state = event.state; + out.capture_reason = event.reason; + out.capture_sample_rate = event.sampleRate; + out.capture_channels = event.channels; + out.capture_available = event.availableFrames; + out.capture_capacity = event.capacityFrames; + out.capture_produced = event.framesProduced; + out.capture_events = model.capture_events + 1; + return .{ .model = out, .cmd = "" }; + }, + .capture_read_evt => |event| { + const out = frameCreate(model.*); + out.capture_read_state = event.state; + out.capture_read_reason = event.reason; + out.capture_read_sequence = event.sequence; + out.capture_read_offset = event.frameOffset; + out.capture_read_frames = event.frames; + out.capture_system_pcm = event.systemPcm; + out.capture_microphone_pcm = event.microphonePcm; + out.capture_system_gaps = event.systemGapFrames; + out.capture_microphone_gaps = event.microphoneGapFrames; + out.capture_remaining = event.remainingFrames; + out.capture_end = event.endOfStream; + out.capture_read_events = model.capture_read_events + 1; + return .{ .model = out, .cmd = "" }; + }, + .device_evt => |event| { + const out = frameCreate(model.*); + out.device_state = event.state; + out.device_total = event.total; + out.device_default = event.isDefault; + out.device_events = model.device_events + 1; + return .{ .model = out, .cmd = "" }; + }, + .capture_access => return .{ .model = model, .cmd = cmdAudioCaptureAccess("access", 89, 1, 1) }, + .access_evt => |event| { + const out = frameCreate(model.*); + out.access_source = event.source; + out.access_status = event.status; + out.access_restart = event.restartRequired; + out.access_events = model.access_events + 1; + return .{ .model = out, .cmd = "" }; + }, + .toggle_mic_watch => { + const out = frameCreate(model.*); + out.mic_watch = !model.mic_watch; + return .{ .model = out, .cmd = "" }; + }, + .devices_changed => { + const out = frameCreate(model.*); + out.device_change_events = model.device_change_events + 1; + return .{ .model = out, .cmd = "" }; + }, } } @@ -651,6 +825,7 @@ const mini_core = struct { } pub fn subscriptions(model: *const Model) []const u8 { + if (model.mic_watch) return subMicrophoneDevicesChanged(91); if (!model.polling) return ""; return subTimer("tick", if (model.fast) 40 else 100, 9); } @@ -852,6 +1027,61 @@ const mini_core = struct { return out; } + fn cmdAudioCaptureStart(key: []const u8, event_tag: u8, system_audio: bool, microphone_kind: u8, microphone_id: []const u8, sample_rate: u32, channels: u8, exclude_current_process_audio: bool, buffer_duration_ms: u32) []const u8 { + const out = rt.frameAlloc(u8, 2 + key.len + 12 + 4 + microphone_id.len); + out[0] = 0x1D; + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + var off: usize = 2 + key.len; + out[off] = event_tag; + out[off + 1] = @as(u8, @intFromBool(system_audio)) | (@as(u8, @intFromBool(exclude_current_process_audio)) << 1); + out[off + 2] = microphone_kind; + std.mem.writeInt(u32, out[off + 3 ..][0..4], sample_rate, .little); + out[off + 7] = channels; + std.mem.writeInt(u32, out[off + 8 ..][0..4], buffer_duration_ms, .little); + off += 12; + _ = writeLongBytes(out, off, microphone_id); + return out; + } + + fn cmdKeyOnly(op: u8, key: []const u8) []const u8 { + const out = rt.frameAlloc(u8, 2 + key.len); + out[0] = op; + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + return out; + } + + fn cmdRoutedKey(op: u8, key: []const u8, event_tag: u8) []const u8 { + const out = rt.frameAlloc(u8, 3 + key.len); + out[0] = op; + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + out[2 + key.len] = event_tag; + return out; + } + + fn cmdAudioCaptureAccess(key: []const u8, event_tag: u8, source: u8, action: u8) []const u8 { + const out = rt.frameAlloc(u8, 5 + key.len); + out[0] = 0x20; + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + out[2 + key.len] = event_tag; + out[3 + key.len] = source; + out[4 + key.len] = action; + return out; + } + + fn cmdAudioCaptureRead(key: []const u8, event_tag: u8, max_frames: u32) []const u8 { + const out = rt.frameAlloc(u8, 3 + key.len + 4); + out[0] = 0x21; + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + out[2 + key.len] = event_tag; + std.mem.writeInt(u32, out[3 + key.len ..][0..4], max_frames, .little); + return out; + } + fn cmdVideoLoad(key: []const u8, event_tag: u8, surface: f64, video_path: []const u8, url: []const u8, flags: u8) []const u8 { const out = rt.frameAlloc(u8, 2 + key.len + 1 + 8 + 4 + video_path.len + 4 + url.len + 1); out[0] = 0x17; @@ -976,6 +1206,13 @@ const mini_core = struct { out[2 + key.len + 8] = msg_tag; return out; } + + fn subMicrophoneDevicesChanged(msg_tag: u8) []const u8 { + const out = rt.frameAlloc(u8, 2); + out[0] = 0x02; + out[1] = msg_tag; + return out; + } }; const Host = ts_core_host.TsCoreHost(mini_core); @@ -1815,6 +2052,84 @@ test "audio_ctl verbs drive the engine channel, gated by the wire key" { try std.testing.expectError(error.EffectNotFound, fx.feedAudioEvent(.position, 50_000, 183_000, true)); } +test "audio capture commands route paired PCM and retain the sealed stream until its final read" { + const fx = freshChannel(); + defer fx.deinit(); + Host.init(fx); + + Host.dispatch(fx, .start_capture); + Host.drain(fx); + try std.testing.expectEqual(mini_core.CaptureState.started, Host.model().capture_state); + try std.testing.expectEqual(mini_core.CaptureReason.none, Host.model().capture_reason); + try std.testing.expectEqual(@as(i64, 1), Host.model().capture_events); + + const system_pcm: [1_764]u8 = @splat(0x21); + const microphone_pcm: [1_764]u8 = @splat(0x42); + try std.testing.expectEqual(.accepted, fx.feedAudioCaptureFrames(.{ + .frame_offset = 0, + .frame_count = 882, + .system_pcm = &system_pcm, + .microphone_pcm = µphone_pcm, + .system_gap_frames = 7, + .microphone_gap_frames = 11, + })); + Host.drain(fx); + try std.testing.expectEqual(mini_core.CaptureState.readable, Host.model().capture_state); + + Host.dispatch(fx, .read_capture); + Host.drain(fx); + try std.testing.expectEqual(mini_core.CaptureReadState.chunk, Host.model().capture_read_state); + try std.testing.expectEqual(mini_core.CaptureReadReason.none, Host.model().capture_read_reason); + try std.testing.expectEqual(@as(f64, 882), Host.model().capture_read_frames); + try std.testing.expectEqualSlices(u8, &system_pcm, Host.model().capture_system_pcm); + try std.testing.expectEqualSlices(u8, µphone_pcm, Host.model().capture_microphone_pcm); + try std.testing.expectEqual(@as(f64, 7), Host.model().capture_system_gaps); + try std.testing.expectEqual(@as(f64, 11), Host.model().capture_microphone_gaps); + try std.testing.expect(!Host.model().capture_end); + + Host.dispatch(fx, .stop_capture); + Host.drain(fx); + try std.testing.expectEqual(mini_core.CaptureState.stopped, Host.model().capture_state); + try std.testing.expectEqual(@as(f64, 44_100), Host.model().capture_sample_rate); + try std.testing.expectEqual(@as(f64, 1), Host.model().capture_channels); + try std.testing.expectEqual(@as(i64, 3), Host.model().capture_events); + + Host.dispatch(fx, .read_capture); + Host.drain(fx); + try std.testing.expectEqual(mini_core.CaptureReadState.ended, Host.model().capture_read_state); + try std.testing.expect(Host.model().capture_end); + try std.testing.expectEqual(@as(i64, 2), Host.model().capture_read_events); + + // The final read retires the retained bridge entry. Discarding the old + // wire key is consequently an idempotent no-op. + Host.dispatch(fx, .discard_capture); +} + +test "microphone listing access and changed subscription route through the TS host" { + const fx = freshChannel(); + defer fx.deinit(); + Host.init(fx); + + Host.dispatch(fx, .list_mics); + Host.drain(fx); + try std.testing.expectEqual(@as(i64, 3), Host.model().device_events); + try std.testing.expectEqual(mini_core.DeviceState.completed, Host.model().device_state); + try std.testing.expectEqual(@as(f64, 2), Host.model().device_total); + + Host.dispatch(fx, .capture_access); + Host.drain(fx); + try std.testing.expectEqual(@as(i64, 1), Host.model().access_events); + try std.testing.expectEqual(mini_core.AccessSource.microphone, Host.model().access_source); + try std.testing.expectEqual(mini_core.AccessStatus.authorized, Host.model().access_status); + + Host.dispatch(fx, .toggle_mic_watch); + const changed = fx.takeMicrophoneDevicesChangedMsg() orelse return error.TestExpectedMsg; + Host.dispatch(fx, changed); + try std.testing.expectEqual(@as(i64, 1), Host.model().device_change_events); + Host.dispatch(fx, .toggle_mic_watch); + try std.testing.expect(fx.takeMicrophoneDevicesChangedMsg() == null); +} + test "a replacing audio_play re-keys the stream and the url source decodes whole" { const fx = freshChannel(); defer fx.deinit(); diff --git a/src/runtime/ui_app.zig b/src/runtime/ui_app.zig index ebe93644c..72180d0fb 100644 --- a/src/runtime/ui_app.zig +++ b/src/runtime/ui_app.zig @@ -1520,6 +1520,34 @@ pub fn UiAppWithFeatures(comptime ModelT: type, comptime MsgT: type, comptime fe // the record precedes the event whose dispatch // issues the load (see `pushReplayVideoSource`). .video_load => self.effects.pushReplayVideoSource(record.key, record.video_token, record.video_source, record.video_kind == .failed), + .audio_capture_read => { + const system_len: usize = @intCast(record.audio_capture_system_len); + if (system_len > record.payload.len) return error.ReplayDamagedRecord; + try self.effects.feedAudioCaptureRead(.{ + .key = record.key, + .state = record.audio_capture_read_state, + .reason = record.audio_capture_read_reason, + .sequence = record.audio_capture_sequence, + .frame_offset = record.audio_capture_frame_offset, + .frames = record.audio_capture_frames, + .system_pcm = record.payload[0..system_len], + .microphone_pcm = record.payload[system_len..], + .system_gap_frames = record.audio_capture_system_gap_frames, + .microphone_gap_frames = record.audio_capture_microphone_gap_frames, + .remaining_frames = record.audio_capture_remaining_frames, + .end_of_stream = record.audio_capture_end_of_stream, + }); + }, + .audio_capture => try self.effects.feedAudioCaptureEvent(.{ + .key = record.key, + .state = record.audio_capture_state, + .reason = record.audio_capture_reason, + .sample_rate_hz = record.audio_capture_sample_rate_hz, + .channel_count = record.audio_capture_channel_count, + .available_frames = record.audio_capture_available_frames, + .capacity_frames = record.audio_capture_capacity_frames, + .frames_produced = record.audio_capture_frames_produced, + }), .timer => {}, }, .finish => { @@ -3968,6 +3996,18 @@ pub fn UiAppWithFeatures(comptime ModelT: type, comptime MsgT: type, comptime fe .audio => |audio_event| if (self.effects.takeAudioMsg(audio_event)) |msg| { try self.dispatch(runtime, self.canvas_window_id, msg); }, + .audio_capture => |capture_event| if (self.effects.takeAudioCaptureMsg(capture_event)) |msg| { + try self.dispatch(runtime, self.canvas_window_id, msg); + }, + .microphone_device => |device_event| if (self.effects.takeMicrophoneDeviceMsg(device_event)) |msg| { + try self.dispatch(runtime, self.canvas_window_id, msg); + }, + .audio_capture_access => |access_event| if (self.effects.takeAudioCaptureAccessMsg(access_event)) |msg| { + try self.dispatch(runtime, self.canvas_window_id, msg); + }, + .microphone_devices_changed => if (self.effects.takeMicrophoneDevicesChangedMsg()) |msg| { + try self.dispatch(runtime, self.canvas_window_id, msg); + }, // Platform video reports route the same way: through // the effects channel into the app's `on_event` Msg, // journaled at the delivery boundary. Without an app diff --git a/src/security/root.zig b/src/security/root.zig index 29f73df50..9d1574126 100644 --- a/src/security/root.zig +++ b/src/security/root.zig @@ -9,6 +9,8 @@ pub const permission_clipboard = "clipboard"; pub const permission_network = "network"; pub const permission_notifications = "notifications"; pub const permission_credentials = "credentials"; +pub const permission_microphone = "microphone"; +pub const permission_system_audio = "system_audio"; pub const ExternalLinkAction = enum(c_int) { deny = 0, diff --git a/src/tooling/manifest.zig b/src/tooling/manifest.zig index e668d5de7..d80583256 100644 --- a/src/tooling/manifest.zig +++ b/src/tooling/manifest.zig @@ -21,6 +21,7 @@ pub const Metadata = struct { icons: []const []const u8 = &.{}, platforms: []const []const u8 = &.{}, permissions: []const []const u8 = &.{}, + privacy: PrivacyMetadata = .{}, capabilities: []const []const u8 = &.{}, bridge_commands: []const BridgeCommandMetadata = &.{}, web_engine: []const u8 = "system", @@ -71,6 +72,8 @@ pub const Metadata = struct { if (self.platforms.len > 0) allocator.free(self.platforms); for (self.permissions) |value| allocator.free(value); if (self.permissions.len > 0) allocator.free(self.permissions); + if (self.privacy.microphone_usage) |value| allocator.free(value); + if (self.privacy.system_audio_usage) |value| allocator.free(value); for (self.capabilities) |value| allocator.free(value); if (self.capabilities.len > 0) allocator.free(self.capabilities); for (self.bridge_commands) |command| { @@ -184,6 +187,11 @@ pub const Metadata = struct { } }; +pub const PrivacyMetadata = struct { + microphone_usage: ?[]const u8 = null, + system_audio_usage: ?[]const u8 = null, +}; + pub const BridgeCommandMetadata = struct { name: []const u8, permissions: []const []const u8 = &.{}, @@ -437,6 +445,10 @@ pub fn validateFile(allocator: std.mem.Allocator, io: std.Io, path: []const u8) .identity = .{ .id = metadata.id, .name = metadata.name, .display_name = metadata.display_name, .description = metadata.description }, .version = parseVersion(metadata.version) catch return .{ .ok = false, .message = "app.zon version is invalid" }, .permissions = permissions, + .privacy = .{ + .microphone_usage = metadata.privacy.microphone_usage, + .system_audio_usage = metadata.privacy.system_audio_usage, + }, .capabilities = capabilities, .bridge = .{ .commands = bridge_commands }, .frontend = frontend, @@ -510,6 +522,10 @@ pub fn parseText(allocator: std.mem.Allocator, source: []const u8) !Metadata { .icons = try duplicateStringList(allocator, raw.icons), .platforms = try duplicateStringList(allocator, raw.platforms), .permissions = try duplicateStringList(allocator, raw.permissions), + .privacy = .{ + .microphone_usage = try duplicateOptionalString(allocator, raw.privacy.microphone_usage), + .system_audio_usage = try duplicateOptionalString(allocator, raw.privacy.system_audio_usage), + }, .capabilities = try duplicateStringList(allocator, raw.capabilities), .bridge_commands = try convertRawBridgeCommands(allocator, raw.bridge.commands), .web_engine = try allocator.dupe(u8, raw.web_engine), @@ -1095,6 +1111,7 @@ fn parsePermission(value: []const u8) app_manifest.Permission { if (std.mem.eql(u8, value, "filesystem")) return .filesystem; if (std.mem.eql(u8, value, "camera")) return .camera; if (std.mem.eql(u8, value, "microphone")) return .microphone; + if (std.mem.eql(u8, value, "system_audio")) return .system_audio; if (std.mem.eql(u8, value, "location")) return .location; if (std.mem.eql(u8, value, "notifications")) return .notifications; if (std.mem.eql(u8, value, "clipboard")) return .clipboard; @@ -1641,6 +1658,31 @@ test "manifest metadata parser reads identity version and lists" { }); } +test "manifest metadata parser reads audio privacy and permissions" { + const metadata = try parseText(std.testing.allocator, + \\.{ + \\ .id = "com.example.recorder", + \\ .name = "recorder", + \\ .version = "1.0.0", + \\ .permissions = .{ "filesystem", "microphone", "system_audio" }, + \\ .privacy = .{ + \\ .microphone_usage = "Record your voice.", + \\ .system_audio_usage = "Record meeting audio.", + \\ }, + \\} + ); + defer metadata.deinit(std.testing.allocator); + + try std.testing.expectEqualStrings("microphone", metadata.permissions[1]); + try std.testing.expectEqualStrings("system_audio", metadata.permissions[2]); + try std.testing.expectEqualStrings("Record your voice.", metadata.privacy.microphone_usage.?); + try std.testing.expectEqualStrings("Record meeting audio.", metadata.privacy.system_audio_usage.?); + const permissions = try parsePermissions(std.testing.allocator, metadata.permissions); + defer std.testing.allocator.free(permissions); + try std.testing.expectEqual(app_manifest.PermissionKind.microphone, permissions[1].kind()); + try std.testing.expectEqual(app_manifest.PermissionKind.system_audio, permissions[2].kind()); +} + test "manifest metadata parser reads structured security policy" { const metadata = try parseText(std.testing.allocator, \\.{ diff --git a/src/tooling/package.zig b/src/tooling/package.zig index 02f78969f..c804ab818 100644 --- a/src/tooling/package.zig +++ b/src/tooling/package.zig @@ -672,6 +672,8 @@ fn macosInfoPlist(allocator: std.mem.Allocator, metadata: manifest_tool.Metadata // dev runs pass to the panel directly. const about_line = try macosAboutLine(allocator, metadata); defer allocator.free(about_line); + const privacy_entries = try macosPrivacyEntries(allocator, metadata); + defer allocator.free(privacy_entries); // CFBundleName is the SHORT user-visible name — the application // menu's title next to the Apple menu reads it — while // CFBundleDisplayName serves the Finder and longer surfaces. Both @@ -702,11 +704,31 @@ fn macosInfoPlist(allocator: std.mem.Allocator, metadata: manifest_tool.Metadata \\ {s} \\ CFBundleVersion \\ {s} - \\{s}{s}{s} + \\{s}{s}{s}{s} \\ \\ \\ - , .{ bundle_id, display_name, display_name, executable, icon, version, version, about_line, document_types, url_types }); + , .{ bundle_id, display_name, display_name, executable, icon, version, version, about_line, privacy_entries, document_types, url_types }); +} + +fn macosPrivacyEntries(allocator: std.mem.Allocator, metadata: manifest_tool.Metadata) ![]const u8 { + var out: std.ArrayList(u8) = .empty; + defer out.deinit(allocator); + if (metadata.privacy.microphone_usage) |usage| { + const escaped = try xmlEscapeAlloc(allocator, usage); + defer allocator.free(escaped); + const entry = try std.fmt.allocPrint(allocator, " NSMicrophoneUsageDescription\n {s}\n", .{escaped}); + defer allocator.free(entry); + try out.appendSlice(allocator, entry); + } + if (metadata.privacy.system_audio_usage) |usage| { + const escaped = try xmlEscapeAlloc(allocator, usage); + defer allocator.free(escaped); + const entry = try std.fmt.allocPrint(allocator, " NSScreenCaptureUsageDescription\n {s}\n NSAudioCaptureUsageDescription\n {s}\n", .{ escaped, escaped }); + defer allocator.free(entry); + try out.appendSlice(allocator, entry); + } + return out.toOwnedSlice(allocator); } /// The optional NSHumanReadableCopyright entry (with trailing newline) @@ -2297,6 +2319,25 @@ test "plist template includes identity executable and version" { try std.testing.expect(std.mem.indexOf(u8, bare_plist, "NSHumanReadableCopyright") == null); } +test "plist template emits audio capture purpose keys" { + const metadata: manifest_tool.Metadata = .{ + .id = "dev.example.recorder", + .name = "recorder", + .version = "1.2.3", + .privacy = .{ + .microphone_usage = "Record voice & commentary.", + .system_audio_usage = "Record meeting