From f4faec6027eb3afc3d3f72204f2be6b8b9aa485a Mon Sep 17 00:00:00 2001 From: "Yifeng \"Evan\" Wang" <7312949+doodlewind@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:48:11 +0800 Subject: [PATCH 1/2] Sound: audio.play in @threert/gameplay, played by the engine on every platform - packages/gameplay/src/audio.ts: `audio.play(sound, { at, gain, pitch, loop, key, falloff })`, `stop`, `move`, `preload`, posted on the host channel (`audio.*`). A sound is a file of the game's package. - renderer/src/audio (the `audio` feature, on by default): decoding by Symphonia (WAV, Ogg Vorbis, MP3, FLAC), placing against the listener (the frame's camera: falloff / distance, equal-power pan to 80 %, a low pass far and behind), mixing (48 voices, 8 copies of a file, ramps, fades, a soft limiter) and the device: cpal on macOS, iOS and Windows, ALSA on Linux and AAudio on Android loaded at run time (no sound library linked). Apps hand it the posts and the camera; the device opens on the first sound, closes with the game and in the background. - `threert-render play` plays a game's sound; the three.js view plays it on Web Audio by the same numbers (tools/three/audio.ts, held to mix.rs by a test). - docs/RUNTIME.md "Sound": the contract. Co-Authored-By: Claude Opus 5.5 --- docs/RUNTIME.md | 63 +++- packages/gameplay/README.md | 1 + packages/gameplay/src/audio.ts | 103 ++++++ packages/gameplay/src/host.ts | 4 +- packages/gameplay/src/index.ts | 2 + packages/gameplay/test/audio.test.ts | 44 +++ renderer/Cargo.lock | 410 +++++++++++++++++++++++- renderer/Cargo.toml | 16 +- renderer/src/audio/decode.rs | 113 +++++++ renderer/src/audio/mix.rs | 453 +++++++++++++++++++++++++++ renderer/src/audio/mod.rs | 336 ++++++++++++++++++++ renderer/src/audio/out.rs | 332 ++++++++++++++++++++ renderer/src/lib.rs | 2 + renderer/src/view.rs | 24 +- tools/three/audio.ts | 234 ++++++++++++++ tools/three/test/audio.test.ts | 25 ++ tools/three/view.ts | 24 +- 17 files changed, 2159 insertions(+), 27 deletions(-) create mode 100644 packages/gameplay/src/audio.ts create mode 100644 packages/gameplay/test/audio.test.ts create mode 100644 renderer/src/audio/decode.rs create mode 100644 renderer/src/audio/mix.rs create mode 100644 renderer/src/audio/mod.rs create mode 100644 renderer/src/audio/out.rs create mode 100644 tools/three/audio.ts create mode 100644 tools/three/test/audio.test.ts diff --git a/docs/RUNTIME.md b/docs/RUNTIME.md index 5864991..4696f35 100644 --- a/docs/RUNTIME.md +++ b/docs/RUNTIME.md @@ -387,6 +387,7 @@ Messages a game posts: | `net.close` | `id`, `code?`, `reason?` | closes it (1000 unless `code` is 3000–4999); answers `net.closed` | | `net.fetch` | `id`, `url`, `method?`, `headers?`, `body?` | one HTTP request (`GET` unless `method`); answers `net.fetched` | | `hud` | `hud` | draws the HUD it describes ("HUD", below). `hud.set` posts it; a game does not post it by hand | +| `audio.*` | below | plays sound ("Sound", below), with the engine's `audio` module. `audio.play` and the rest post them | Messages an app sends: @@ -418,11 +419,11 @@ a binary WebSocket frame is dropped. Which addresses a game may reach is the app The apps in this repository: -| app | input | `net.*` | `launch` | HUD | -| --- | --- | --- | --- | --- | -| `threert-render play` (`view.rs`) | keys by physical position, mouse buttons, wheel; the pointer captured for `"look"` (a click; Esc lets go, Esc again quits) | WebSockets and HTTP on a thread each (`src/net.rs`: tungstenite and ureq on rustls; text frames of at most 1 MiB, bodies of at most 4 MiB, 16 sockets, 32 requests at once), any address | `--launch '{"room":"K7Q4MZ"}'` | not drawn; `--hud-log` prints each description | -| `threert-render live` (`live/session.rs`) | whatever the driver sends | none: the driver is the app | `--launch` | on stdout with every other post | -| the three.js view (`tools/three/view.ts`, `host.ts`, `hud.ts`) | keyboard, mouse, the Pointer Lock API | `WebSocket`, `fetch` (the browser's rules: same-origin policy, CORS) | the page's query (`?room=K7Q4MZ&name=Ada`) | DOM over the canvas | +| app | input | `net.*` | `launch` | HUD | sound | +| --- | --- | --- | --- | --- | --- | +| `threert-render play` (`view.rs`) | keys by physical position, mouse buttons, wheel; the pointer captured for `"look"` (a click; Esc lets go, Esc again quits) | WebSockets and HTTP on a thread each (`src/net.rs`: tungstenite and ureq on rustls; text frames of at most 1 MiB, bodies of at most 4 MiB, 16 sockets, 32 requests at once), any address | `--launch '{"room":"K7Q4MZ"}'` | not drawn; `--hud-log` prints each description | played (`audio` feature) | +| `threert-render live` (`live/session.rs`) | whatever the driver sends | none: the driver is the app | `--launch` | on stdout with every other post | on stdout: the driver's | +| the three.js view (`tools/three/view.ts`, `host.ts`, `hud.ts`) | keyboard, mouse, the Pointer Lock API | `WebSocket`, `fetch` (the browser's rules: same-origin policy, CORS) | the page's query (`?room=K7Q4MZ&name=Ada`) | DOM over the canvas | Web Audio by the same rules (`audio.ts`) | - `live` speaks the channel as JSON lines, so a game's use of it is testable without a window: on stdin `{"cmd":"host","messages":[{"t":"net.opened","id":"room"}, ...]}`, on stdout one line `{"host":[...]}` with what @@ -553,6 +554,58 @@ hud.flash("damage", 1.2); // the player was hit from 1.2 rad clockwise of - **Text is text.** An app shows a game's strings as they are, never as markup. - A scene built anew in the same runtime posts its first description whatever the last one was. +### Sound + +Unlike the HUD, sound is not left to each app: a shot is to sound the same in ThreeRT for macOS, on a Steam Deck and +on an Android handheld. The game names a file and a place (`packages/gameplay/src/audio.ts`); the engine's library +decodes, places and mixes it (`renderer/src/audio`, the `audio` feature, on by default and in the clients), and an +app only hands it the messages and the camera. The device is the one part that differs a platform. + +```ts +import { audio } from "@threert/gameplay"; + +setup() { audio.preload(["assets/sounds/shot.ogg"]); ... } // decoded ahead, off the frame +frame(f) { + if (fired) audio.play("assets/sounds/shot.ogg", { at: muzzle, gain: 0.9, pitch: 0.97 + 0.06 * rng(), falloff: 8 }); + audio.play("assets/sounds/tick.wav"); // no `at`: in the player's head + audio.play("assets/sounds/fire.ogg", { at: brazier, loop: true, key: "brazier" }); // then audio.move / audio.stop +} +``` + +- **Messages** (script → app; nothing comes back): + + | `t` | fields | | + | --- | --- | --- | + | `audio.play` | `sound`, `at?` `[x, y, z]`, `gain?` (0..4, 1), `pitch?` (0.25..4, 1), `loop?`, `key?`, `falloff?` (m, 2) | starts a sound | + | `audio.stop` | `key?`, `fade?` (s, 0.05) | fades out the sounds of `key`, or all of them | + | `audio.move` | `key`, `at` | moves the sounds of `key` | + | `audio.preload` | `sounds` | decodes these files on a thread of their own | +- **Files.** A sound is a file of the game, named as the scene names its files (`assets/sounds/shot.ogg`, relative to + the package; nothing outside it): WAV (PCM or float), Ogg Vorbis, MP3, FLAC, at any rate, one or two channels, at + most ten minutes. A package carries everything under `assets/`. Decoded once (by Symphonia: the same samples on + every platform), kept until the game closes; a file that is missing or unreadable is said once in the log and then + plays nothing. +- **The listener** is the camera the frame was drawn from: its position, forward and right. A game sends nothing for + it. +- **Placing** (`renderer/src/audio/mix.rs`): a sound at `at` keeps its level out to `falloff` metres, then falls as + falloff / distance (−6 dB a doubling) and fades out between 100 and 120 m. It is panned by its direction from the + listener's forward, with equal power and never more than 80 % to one side, and comes back towards the middle within + `falloff`. Far and behind it is duller: a one-pole low pass from 18 kHz, divided by 1 + distance / 40 m, halved + behind. Without `at` a sound is in the middle at its own gain. +- **Limits:** 32 plays a frame (more are dropped), 48 voices at once (the quietest one-shot goes), 8 of one file (its + oldest goes). Levels ramp over each block; a stop fades. The app's volume, then a soft limiter above −2 dB. +- **Presentation only.** Play sounds from the frame script and from host listeners, never from a handler: handlers + are replayed and verified, and a sound is not state. A node with no app (a test, a render node) drops the posts. +- **Apps:** each message the game posts goes to `audio::Audio::post` (it answers false for what is not `audio.*`), and + once a frame `Audio::listener(camera)`. The device opens on the first sound and closes with the game (`close`) or + while the app is in the background (`set_paused`; loops go on where they were, one-shots in between are dropped); + `set_volume` is the app's own control. Devices (`renderer/src/audio/out.rs`): Core Audio and WASAPI by cpal, ALSA's + `default` on Linux and AAudio on Android, both loaded at run time, so a Linux or Android build links no sound + library and is silent, not broken, where there is none. `threert-render play` plays a game's sound the same way. + The three.js view plays it on Web Audio by the same numbers (`tools/three/audio.ts`, held to mix.rs by + `tools/three/test/audio.test.ts`; its low pass is a biquad, the one difference), resumed on the page's first key or + click as browsers require. The engine's library in a browser (wasm32) has no `audio` module. + ## The runtime's shape: Rust owns the world, the script engine is a guest Decided by the user on 2026-11-10, with the measurements below. diff --git a/packages/gameplay/README.md b/packages/gameplay/README.md index a020c8e..0019248 100644 --- a/packages/gameplay/README.md +++ b/packages/gameplay/README.md @@ -62,6 +62,7 @@ export default defineGame({ - **Deterministic and replayable.** Events are logged with what they read and wrote; `replayLog` and `verifyLog` rebuild a world from its log, and seeded randomness (`seededRandom`) gives the same result on every machine. - **World handlers** (`on`) reach the engine's own world: physics, a ray cast into the scene, objects added and removed as an event arrives. - **Input, a HUD and the app.** A game with `pointer: "look"` reads the keys and the mouse every frame (`f.input.move`, `f.input.look`, `f.input.pressed("Space")`); `hud.set(...)` describes a crosshair, figures, bars and a scoreboard for the app to draw; `host.post` / `host.on` reach what the app owns (a WebSocket, an HTTP request, how the game was opened). +- **Sound.** `audio.play("assets/sounds/shot.ogg", { at: muzzle, pitch: 1.04 })` plays a file of the game at a place in the world (or, without `at`, in the player's head); the engine decodes, places and mixes it the same way on every platform, with the camera as the ears. `audio.preload`, `audio.stop(key)` and `audio.move(key, at)` for the rest. How scene code runs in the engine is in [RUNTIME.md](https://github.com/meshy-dev/ThreeRT/blob/main/docs/RUNTIME.md). diff --git a/packages/gameplay/src/audio.ts b/packages/gameplay/src/audio.ts new file mode 100644 index 0000000..6b11dc0 --- /dev/null +++ b/packages/gameplay/src/audio.ts @@ -0,0 +1,103 @@ +// Sound: what the game plays, from its package's files, at a place in the world or in the player's head. The engine +// decodes, places and mixes it (renderer/src/audio): the same code in every app that runs the game, so a shot sounds +// the same in ThreeRT for macOS, on a Steam Deck and on an Android handheld. `audio.play(...)` posts +// `{ t: "audio.play", ... }` on the host channel (host.ts); docs/RUNTIME.md "Sound" is the contract. +// +// setup() { +// audio.preload(["assets/sounds/shot.ogg", "assets/sounds/step-1.ogg"]); // decoded ahead, off the frame +// ... +// }, +// frame(f) { +// if (fired) audio.play("assets/sounds/shot.ogg", { at: muzzle, pitch: 0.97 + 0.06 * rng() }); +// if (f.input.pressed("Escape")) audio.play("assets/sounds/ui-tick.ogg"); // no `at`: in the head +// audio.play("assets/sounds/fire.ogg", { at: brazier, loop: true, key: "brazier" }); // once; then +// audio.move("brazier", brazier); audio.stop("brazier", 0.5); +// } +// +// - A sound is a file of the game (WAV, Ogg Vorbis, MP3 or FLAC), named as the scene names its files +// ("assets/sounds/shot.ogg"); a package carries everything under assets/. It is decoded once and kept while the game +// is open: when first played, or ahead after `preload` (do that in setup for what plays in the first seconds). +// - The ears are the camera the frame is drawn from: nothing to send. `at` is in metres, world space; the sound keeps +// its level out to `falloff` metres (2 by default) and falls as falloff / distance beyond, panned by its direction. +// - `gain` (0 to 4, 1 by default) and `pitch` (0.25 to 4: a playback rate, 1.1 is a little higher and shorter) vary a +// sound that repeats: a step, a shot. +// - Sound is presentation: play it from the frame script and from host listeners, never from a handler (handlers are +// replayed and verified; a sound is not state). An app that plays no sound ignores the messages, and the game goes +// on the same. +// - At most 32 plays a frame, 48 voices at once, 8 of one sound (the oldest goes); the rest is dropped. + +import { host } from "./host.ts"; + +/** A point: `[x, y, z]` or anything with x, y, z (a Vector3). */ +export type AudioPoint = readonly [number, number, number] | { x: number; y: number; z: number }; + +export interface PlayOptions { + /** Where in the world, metres. Without it the sound is in the player's head (the middle, its own level). */ + at?: AudioPoint; + /** 0 to 4; 1 by default. */ + gain?: number; + /** Playback rate, 0.25 to 4; 1 by default. */ + pitch?: number; + /** Plays until stopped (`key` names it for `stop` and `move`). */ + loop?: boolean; + /** The name `stop` and `move` reach this sound by (several sounds may share one). */ + key?: string; + /** Metres out to which the sound keeps its level (2 by default): larger for a loud one, a gunshot, an engine. */ + falloff?: number; +} + +/** The messages, as posted. */ +export type AudioPost = + | { t: "audio.play"; sound: string; at?: [number, number, number]; gain?: number; pitch?: number; loop?: boolean; key?: string; falloff?: number } + | { t: "audio.stop"; key?: string; fade?: number } + | { t: "audio.move"; key: string; at: [number, number, number] } + | { t: "audio.preload"; sounds: string[] }; + +export interface Audio { + /** Plays `sound` (a file of the game). */ + play(sound: string, options?: PlayOptions): void; + /** Fades out the sounds of `key` (every sound without one) over `fade` seconds (0.05 by default). */ + stop(key?: string, fade?: number): void; + /** Moves the sounds of `key` (a loop that follows something). */ + move(key: string, at: AudioPoint): void; + /** Decodes these files ahead, off the frame. */ + preload(sounds: readonly string[]): void; +} + +const round = (v: number) => Math.round(v * 1000) / 1000; + +function point(p: AudioPoint): [number, number, number] { + const [x, y, z] = Array.isArray(p) ? (p as readonly number[]) : [(p as { x: number }).x, (p as { y: number }).y, (p as { z: number }).z]; + if (![x, y, z].every((v) => typeof v === "number" && Number.isFinite(v))) throw new Error(`audio: not a point: ${JSON.stringify(p)}`); + return [round(x!), round(y!), round(z!)]; +} + +function name(sound: unknown): string { + if (typeof sound !== "string" || sound === "") throw new Error(`audio: a sound is a file's path, not ${JSON.stringify(sound)}`); + return sound; +} + +export const audio: Audio = { + play(sound, o = {}) { + const m: Extract = { t: "audio.play", sound: name(sound) }; + if (o.at !== undefined) m.at = point(o.at); + if (o.gain !== undefined && o.gain !== 1) m.gain = round(o.gain); + if (o.pitch !== undefined && o.pitch !== 1) m.pitch = round(o.pitch); + if (o.loop) m.loop = true; + if (o.key !== undefined) m.key = String(o.key); + if (o.falloff !== undefined) m.falloff = round(o.falloff); + host.post(m); + }, + stop(key, fade) { + const m: Extract = { t: "audio.stop" }; + if (key !== undefined) m.key = String(key); + if (fade !== undefined) m.fade = round(fade); + host.post(m); + }, + move(key, at) { + host.post({ t: "audio.move", key: String(key), at: point(at) }); + }, + preload(sounds) { + host.post({ t: "audio.preload", sounds: sounds.map(name) }); + }, +}; diff --git a/packages/gameplay/src/host.ts b/packages/gameplay/src/host.ts index 1f6b491..97d03b0 100644 --- a/packages/gameplay/src/host.ts +++ b/packages/gameplay/src/host.ts @@ -21,8 +21,8 @@ // - The same object is `globalThis.__threert_host`, for a library that does not import this package. // // The message families an app may implement are in docs/RUNTIME.md ("The host channel"): `net.*` (sockets and HTTP -// the app owns), `hud` (hud.ts), `launch` (how the game was opened), and `input` (input.ts), which the node host -// takes itself. +// the app owns), `hud` (hud.ts), `audio.*` (audio.ts: played by the engine's mixer), `launch` (how the game was +// opened), and `input` (input.ts), which the node host takes itself. import type { WorldContext } from "./world.ts"; diff --git a/packages/gameplay/src/index.ts b/packages/gameplay/src/index.ts index e9866f9..d1261cb 100644 --- a/packages/gameplay/src/index.ts +++ b/packages/gameplay/src/index.ts @@ -8,6 +8,8 @@ export { host } from "./host.ts"; export type { Host, HostEvents, HostListener, HostMessage, LaunchParams, NetPost } from "./host.ts"; export { InputState } from "./input.ts"; export type { Input, InputMessage } from "./input.ts"; +export { audio } from "./audio.ts"; +export type { Audio, AudioPoint, AudioPost, PlayOptions } from "./audio.ts"; export { hud } from "./hud.ts"; export type { Hud, HudAnchor, HudBar, HudCrosshair, HudFlash, HudItem, HudLine, HudList, HudMessage, HudState, HudTable, HudText, HudTone } from "./hud.ts"; export { runWorldHandler } from "./world.ts"; diff --git a/packages/gameplay/test/audio.test.ts b/packages/gameplay/test/audio.test.ts new file mode 100644 index 0000000..ff216e4 --- /dev/null +++ b/packages/gameplay/test/audio.test.ts @@ -0,0 +1,44 @@ +// Sound (src/audio.ts): what the game asks for reaches the app's queue as the messages docs/RUNTIME.md "Sound" +// describes, points as arrays, defaults left out, and a sound that is not a path refused. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { Vector3 } from "@threert/scene"; +import { audio } from "../src/index.ts"; + +function app(): { posts: { t: string; [k: string]: unknown }[]; close(): void } { + const posts: { t: string }[] = []; + const g = globalThis as { __host_post?: (text: string) => unknown }; + g.__host_post = (text) => void posts.push(JSON.parse(text)); + return { posts, close: () => void delete g.__host_post }; +} + +test("play, stop, move and preload post their messages", () => { + const a = app(); + try { + audio.preload(["assets/sounds/shot.ogg"]); + audio.play("assets/sounds/shot.ogg", { at: new Vector3(1, 2.00049, -3), gain: 0.8, pitch: 1, falloff: 6 }); + audio.play("assets/sounds/tick.wav"); + audio.play("assets/sounds/fire.ogg", { at: [0, 1, 0], loop: true, key: "fire" }); + audio.move("fire", { x: 1, y: 1, z: 0 }); + audio.stop("fire", 0.5); + audio.stop(); + assert.deepEqual(a.posts, [ + { t: "audio.preload", sounds: ["assets/sounds/shot.ogg"] }, + { t: "audio.play", sound: "assets/sounds/shot.ogg", at: [1, 2, -3], gain: 0.8, falloff: 6 }, + { t: "audio.play", sound: "assets/sounds/tick.wav" }, + { t: "audio.play", sound: "assets/sounds/fire.ogg", at: [0, 1, 0], loop: true, key: "fire" }, + { t: "audio.move", key: "fire", at: [1, 1, 0] }, + { t: "audio.stop", key: "fire", fade: 0.5 }, + { t: "audio.stop" }, + ]); + } finally { + a.close(); + } +}); + +test("a sound is a path and a point is three finite numbers", () => { + assert.throws(() => audio.play(""), /a sound is a file's path/); + assert.throws(() => audio.play("a.ogg", { at: [0, Number.NaN, 0] }), /not a point/); + // Without an app nothing listens, and nothing throws. + audio.play("a.ogg"); +}); diff --git a/renderer/Cargo.lock b/renderer/Cargo.lock index c53a925..81b7abd 100644 --- a/renderer/Cargo.lock +++ b/renderer/Cargo.lock @@ -52,6 +52,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" +dependencies = [ + "alsa-sys", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android-activity" version = "0.6.1" @@ -61,7 +83,7 @@ dependencies = [ "android-properties", "bitflags 2.13.2", "cc", - "jni", + "jni 0.22.4", "libc", "log", "ndk", @@ -364,6 +386,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.5" @@ -478,6 +506,56 @@ dependencies = [ "libc", ] +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "coreaudio-rs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "cpal" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8942da362c0f0d895d7cac616263f2f9424edc5687364dfd1d25ef7eba506d7" +dependencies = [ + "alsa", + "coreaudio-rs", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive 0.4.2", + "num-traits", + "objc2 0.6.5", + "objc2-audio-toolbox", + "objc2-avf-audio", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -561,6 +639,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.11.1" @@ -678,6 +762,20 @@ dependencies = [ "log", ] +[[package]] +name = "encoding_rs" +version = "0.8.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + [[package]] name = "equivalent" version = "1.0.3" @@ -710,6 +808,12 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fdeflate" version = "0.3.7" @@ -1127,6 +1231,22 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -1409,6 +1529,15 @@ dependencies = [ "logos-codegen", ] +[[package]] +name = "mach2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1b95cd5421ec55b445b5ae102f5ea0e768de1f82bd3001e11f426c269c3aea" +dependencies = [ + "libc", +] + [[package]] name = "matrixmultiply" version = "0.3.11" @@ -1464,6 +1593,12 @@ dependencies = [ "pxfm", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "naga" version = "30.0.1" @@ -1583,6 +1718,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num-derive" version = "0.5.1" @@ -1687,6 +1833,31 @@ dependencies = [ "objc2-quartz-core 0.2.2", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2 0.6.5", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.2.2" @@ -1711,6 +1882,29 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2 0.6.5", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", +] + [[package]] name = "objc2-core-data" version = "0.2.2" @@ -1730,7 +1924,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.13.2", + "block2 0.6.2", "dispatch2", + "libc", "objc2 0.6.5", ] @@ -1796,6 +1992,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ "bitflags 2.13.2", + "block2 0.6.2", + "libc", "objc2 0.6.5", "objc2-core-foundation", ] @@ -2018,7 +2216,7 @@ dependencies = [ "hashbrown 0.17.1", "indexmap", "log", - "num-derive", + "num-derive 0.5.1", "num-traits", "ordered-float", "rayon", @@ -2810,6 +3008,127 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symphonia" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73" +dependencies = [ + "log", + "symphonia-core", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-utils-xiph" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16" +dependencies = [ + "symphonia-core", + "symphonia-metadata", +] + [[package]] name = "syn" version = "2.0.119" @@ -2887,11 +3206,13 @@ version = "0.1.0" dependencies = [ "anyhow", "bytemuck", + "cpal", "glam 0.30.10", "gltf", "half", "image", "libc", + "libloading", "naga", "oidn", "physics-core", @@ -2902,6 +3223,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "symphonia", "tungstenite", "ureq", "voxel-core", @@ -3717,13 +4039,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -3732,7 +4063,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -3744,20 +4075,35 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -3769,18 +4115,36 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -3793,24 +4157,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/renderer/Cargo.toml b/renderer/Cargo.toml index add0a69..9b03272 100644 --- a/renderer/Cargo.toml +++ b/renderer/Cargo.toml @@ -50,6 +50,16 @@ tungstenite = { version = "0.30", optional = true, default-features = false, fea ureq = { version = "3", optional = true, default-features = false, features = ["rustls", "gzip"] } rustls = { version = "0.23", optional = true, default-features = false, features = ["ring", "std"] } +# Sound (the `audio` feature): the decoders, pure Rust (WAV, Ogg Vorbis, MP3, FLAC). +symphonia = { version = "0.5", optional = true, default-features = false, features = ["wav", "pcm", "ogg", "vorbis", "mp3", "flac"] } + +# The sound device (the `audio` feature, src/audio/out.rs): Core Audio and WASAPI through cpal; ALSA and AAudio are +# loaded at run time (no sound library to link on Linux or Android). +[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "windows"))'.dependencies] +cpal = { version = "0.17", optional = true } +[target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies] +libloading = { version = "0.8", optional = true } + # The performance cores' count (src/physics.rs: the physics' threads). [target.'cfg(target_os = "macos")'.dependencies] libc = "0.2" @@ -74,7 +84,11 @@ name = "physics_scene" crate-type = ["cdylib"] [features] -default = ["net"] +default = ["net", "audio"] +# Sound for games (src/audio: the `audio.*` messages of the host channel, docs/RUNTIME.md "Sound"): decoding, placing +# and mixing in Rust on every platform; the device by cpal (macOS, iOS, Windows), ALSA (Linux) or AAudio (Android), +# the last two loaded at run time. The clients turn it on with the library (`features = ["audio"]`). +audio = ["dep:symphonia", "dep:cpal", "dep:libloading"] # WebSockets and HTTP in `threert-render play` (above). Without it the window answers every `net.*` request with a # failure that says so. net = ["dep:tungstenite", "dep:ureq", "dep:rustls"] diff --git a/renderer/src/audio/decode.rs b/renderer/src/audio/decode.rs new file mode 100644 index 0000000..640cf90 --- /dev/null +++ b/renderer/src/audio/decode.rs @@ -0,0 +1,113 @@ +//! A sound file to frames: WAV (PCM, float), Ogg Vorbis, MP3 and FLAC, by Symphonia (pure Rust: the same decoder on +//! every platform, so a sound is the same samples everywhere). One or two channels are kept (more: the first two); +//! at most ten minutes. + +use super::mix::Sound; +use anyhow::{anyhow, bail, Context, Result}; +use symphonia::core::audio::SampleBuffer; +use symphonia::core::codecs::DecoderOptions; +use symphonia::core::errors::Error as SymErr; +use symphonia::core::formats::FormatOptions; +use symphonia::core::io::MediaSourceStream; +use symphonia::core::meta::MetadataOptions; +use symphonia::core::probe::Hint; + +const MAX_SECONDS: usize = 600; + +/// The file extensions that are sounds. +pub const EXTENSIONS: [&str; 6] = ["wav", "ogg", "oga", "mp3", "flac", "wave"]; + +pub fn decode(name: &str, bytes: Vec) -> Result { + let mut hint = Hint::new(); + if let Some(ext) = name.rsplit('.').next() { + hint.with_extension(&ext.to_ascii_lowercase()); + } + let mss = MediaSourceStream::new(Box::new(std::io::Cursor::new(bytes)), Default::default()); + let probed = symphonia::default::get_probe().format(&hint, mss, &FormatOptions::default(), &MetadataOptions::default()).with_context(|| format!("{name}: not a sound this engine reads (WAV, Ogg Vorbis, MP3, FLAC)"))?; + let mut format = probed.format; + let track = format.default_track().ok_or_else(|| anyhow!("{name}: no audio track"))?; + let track_id = track.id; + let rate = track.codec_params.sample_rate.ok_or_else(|| anyhow!("{name}: no sample rate"))?; + let mut decoder = symphonia::default::get_codecs().make(&track.codec_params, &DecoderOptions::default()).with_context(|| format!("{name}: its codec"))?; + let mut channels = 0usize; + let mut data: Vec = Vec::new(); + let mut buf: Option> = None; + loop { + let packet = match format.next_packet() { + Ok(p) => p, + Err(SymErr::IoError(e)) if e.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(SymErr::ResetRequired) => break, + Err(e) => return Err(anyhow!("{name}: {e}")), + }; + if packet.track_id() != track_id { + continue; + } + let decoded = match decoder.decode(&packet) { + Ok(d) => d, + // (A damaged packet is skipped, as players do.) + Err(SymErr::DecodeError(_)) => continue, + Err(e) => return Err(anyhow!("{name}: {e}")), + }; + let spec = *decoded.spec(); + let n = spec.channels.count(); + if channels == 0 { + channels = n.min(2); + } + let b = buf.get_or_insert_with(|| SampleBuffer::new(decoded.capacity() as u64, spec)); + if b.capacity() < decoded.capacity() * n { + *b = SampleBuffer::new(decoded.capacity() as u64, spec); + } + b.copy_interleaved_ref(decoded); + for frame in b.samples().chunks(n) { + data.extend_from_slice(&frame[..channels]); + } + if data.len() / channels.max(1) > MAX_SECONDS * rate as usize { + bail!("{name}: longer than {} minutes", MAX_SECONDS / 60); + } + } + if channels == 0 || data.is_empty() { + bail!("{name}: no samples"); + } + Ok(Sound { name: name.to_string(), rate, channels, data }) +} + +/// A 16-bit PCM WAV of `samples` (mono), for tests. +#[cfg(test)] +pub(crate) fn tests_wav(rate: u32, samples: &[i16]) -> Vec { + let mut v = Vec::new(); + let data = (samples.len() * 2) as u32; + v.extend_from_slice(b"RIFF"); + v.extend_from_slice(&(36 + data).to_le_bytes()); + v.extend_from_slice(b"WAVEfmt "); + v.extend_from_slice(&16u32.to_le_bytes()); + v.extend_from_slice(&1u16.to_le_bytes()); + v.extend_from_slice(&1u16.to_le_bytes()); + v.extend_from_slice(&rate.to_le_bytes()); + v.extend_from_slice(&(rate * 2).to_le_bytes()); + v.extend_from_slice(&2u16.to_le_bytes()); + v.extend_from_slice(&16u16.to_le_bytes()); + v.extend_from_slice(b"data"); + v.extend_from_slice(&data.to_le_bytes()); + for s in samples { + v.extend_from_slice(&s.to_le_bytes()); + } + v +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_a_wav() { + let s = decode("a.wav", tests_wav(22_050, &[0, 16384, -16384, 32767])).unwrap(); + assert_eq!((s.rate, s.channels, s.frames()), (22_050, 1, 4)); + assert!((s.data[1] - 0.5).abs() < 1e-3 && (s.data[2] + 0.5).abs() < 1e-3); + } + + #[test] + fn says_what_is_not_a_sound() { + let e = decode("a.ogg", b"not a sound".to_vec()).unwrap_err(); + assert!(format!("{e:#}").contains("a.ogg"), "{e:#}"); + } +} diff --git a/renderer/src/audio/mix.rs b/renderer/src/audio/mix.rs new file mode 100644 index 0000000..287d3c3 --- /dev/null +++ b/renderer/src/audio/mix.rs @@ -0,0 +1,453 @@ +//! The mixer: every voice a game plays, placed against the listener and summed into the device's frames. It runs in the +//! output's callback (out.rs) and takes what the app asks of it as commands over a channel, never a lock: the app's +//! thread and the device's never wait for each other. +//! +//! How a sound is heard (docs/RUNTIME.md "Sound", the contract every app that plays sound keeps): +//! +//! - **Distance:** a sound at `at` keeps its level out to `falloff` metres (2 by default), then falls as falloff / +//! distance (−6 dB a doubling: 20 m from a falloff of 2 is −20 dB) and fades out between 100 and 120 m. +//! - **Direction:** panned by the angle from the listener's forward (equal power, never more than 80 % to one side: +//! both ears hear a shot); within `falloff` it comes back towards the middle. Behind the listener and far away it is +//! duller (a low pass from 18 kHz down with distance, halved behind). +//! - **Head-relative:** a sound without `at` (a UI tick, the player's own breath) is in the middle at its gain. +//! - **Voices:** 48 at once; a ninth copy of one sound ends its oldest; past 48 the quietest one-shot goes. Changes in +//! level are ramped over the block (no clicks); a stop fades (50 ms unless the game says). +//! - **Master:** the app's volume, then a soft limiter above −2 dB. + +use glam::Vec3; +use std::collections::VecDeque; +use std::f32::consts::{FRAC_PI_4, TAU}; +use std::sync::mpsc::Receiver; +use std::sync::Arc; + +pub const MAX_VOICES: usize = 48; +pub const MAX_COPIES: usize = 8; +pub const DEFAULT_FALLOFF: f32 = 2.0; +pub const FADE_FROM: f32 = 100.0; +pub const SILENT_AT: f32 = 120.0; +const MAX_PAN: f32 = 0.8; +const DEFAULT_STOP_FADE: f32 = 0.05; + +/// A decoded sound: interleaved f32 frames at its own rate, one or two channels. +#[derive(Debug)] +pub struct Sound { + pub name: String, + pub rate: u32, + pub channels: usize, + pub data: Vec, +} + +impl Sound { + pub fn frames(&self) -> usize { + self.data.len() / self.channels.max(1) + } + pub fn seconds(&self) -> f32 { + self.frames() as f32 / self.rate.max(1) as f32 + } +} + +/// Where the ears are: the camera the frame was drawn from. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Listener { + pub pos: Vec3, + pub forward: Vec3, + pub right: Vec3, +} + +impl Default for Listener { + fn default() -> Self { + Listener { pos: Vec3::ZERO, forward: Vec3::NEG_Z, right: Vec3::X } + } +} + +/// A sound to start. +#[derive(Clone, Debug)] +pub struct Play { + pub sound: Arc, + pub at: Option, + pub gain: f32, + pub pitch: f32, + pub looped: bool, + pub key: Option, + pub falloff: f32, +} + +/// What the app asks of the mixer. +#[derive(Debug)] +pub enum Cmd { + Play(Play), + /// Fades out the voices of `key` (all of them: None), over seconds. + Stop(Option, f32), + /// Moves the voices of `key`. + Move(String, Vec3), + Listener(Listener), + Volume(f32), + /// Paused: the device gets silence and voices stand still (the app in the background). + Paused(bool), +} + +/// Gain and pan of a sound at `at` for `l` (the contract above), and the low pass's cutoff in Hz. +pub fn place(l: &Listener, at: Vec3, falloff: f32) -> (f32, f32, f32) { + let falloff = falloff.max(0.05); + let to = at - l.pos; + let d = to.length(); + let mut gain = if d <= falloff { 1.0 } else { falloff / d }; + if d > FADE_FROM { + gain *= ((SILENT_AT - d) / (SILENT_AT - FADE_FROM)).clamp(0.0, 1.0); + } + if d < 1e-4 { + return (gain, 0.0, 18_000.0); + } + let dir = to / d; + let x = dir.dot(l.right); + let z = dir.dot(l.forward); + let pan = (x * (d / falloff).min(1.0)).clamp(-1.0, 1.0) * MAX_PAN; + let behind = (-z).max(0.0); + let cutoff = 18_000.0 / (1.0 + d / 40.0) * (1.0 - 0.5 * behind); + (gain, pan, cutoff) +} + +/// Equal-power gains of a pan in −1..1, the middle at 1 on both sides. +pub fn pan_gains(pan: f32) -> (f32, f32) { + let th = (pan.clamp(-1.0, 1.0) + 1.0) * FRAC_PI_4; + (th.cos() * std::f32::consts::SQRT_2, th.sin() * std::f32::consts::SQRT_2) +} + +struct Voice { + play: Play, + /// Position in the sound, frames (fractional: pitch and rate). + pos: f64, + step: f64, + /// The gains the last block ended at (each block ramps from them to its own). + cur: (f32, f32), + /// Fade: its level now, and how much it changes a frame (negative: stopping). + fade: f32, + fade_step: f32, + /// Low pass state per channel, and its coefficient now. + lp: (f32, f32), + lp_a: f32, + /// What it is heard at now (for choosing what goes when there are too many). + level: f32, + age: u64, + started: bool, +} + +pub struct Mixer { + rate: u32, + rx: Receiver, + voices: Vec, + listener: Listener, + volume: f32, + paused: bool, + ages: u64, + /// Plays that came in while paused wait, at most a few (a paused game starts nothing of note). + held: VecDeque, +} + +impl Mixer { + pub fn new(rate: u32, rx: Receiver) -> Mixer { + Mixer { rate: rate.max(8000), rx, voices: Vec::with_capacity(MAX_VOICES + 1), listener: Listener::default(), volume: 1.0, paused: false, ages: 0, held: VecDeque::new() } + } + + pub fn voices(&self) -> usize { + self.voices.len() + } + + /// The device's rate (it may change when the device is opened again): the voices keep their pitch. + pub fn set_rate(&mut self, rate: u32) { + self.rate = rate.max(8000); + for v in &mut self.voices { + v.step = v.play.sound.rate as f64 / self.rate as f64 * v.play.pitch.clamp(0.25, 4.0) as f64; + } + } + + fn command(&mut self, c: Cmd) { + match c { + Cmd::Play(p) => { + if self.paused { + if self.held.len() < 8 && p.looped { + self.held.push_back(p); + } + } else { + self.start(p); + } + } + Cmd::Stop(key, secs) => { + let step = 1.0 / (secs.max(0.0) * self.rate as f32).max(1.0); + for v in &mut self.voices { + if key.is_none() || v.play.key == key { + v.fade_step = -step; + } + } + self.held.retain(|p| key.is_some() && p.key != key); + } + Cmd::Move(key, at) => { + for v in &mut self.voices { + if v.play.key.as_deref() == Some(key.as_str()) { + v.play.at = Some(at); + } + } + } + Cmd::Listener(l) => self.listener = l, + Cmd::Volume(v) => self.volume = v.clamp(0.0, 1.0), + Cmd::Paused(p) => { + self.paused = p; + if !p { + while let Some(h) = self.held.pop_front() { + self.start(h); + } + } + } + } + } + + fn start(&mut self, p: Play) { + if p.sound.frames() == 0 { + return; + } + // A ninth copy of one sound: its oldest goes. Too many in all: the quietest one-shot. + let copies: Vec = (0..self.voices.len()).filter(|&i| Arc::ptr_eq(&self.voices[i].play.sound, &p.sound) && self.voices[i].fade_step >= 0.0).collect(); + if copies.len() >= MAX_COPIES { + let oldest = copies.into_iter().min_by_key(|&i| self.voices[i].age).unwrap(); + self.voices.swap_remove(oldest); + } + if self.voices.len() >= MAX_VOICES { + let quiet = (0..self.voices.len()).filter(|&i| !self.voices[i].play.looped).min_by(|&a, &b| self.voices[a].level.total_cmp(&self.voices[b].level)); + match quiet { + Some(i) => { + self.voices.swap_remove(i); + } + None => return, + } + } + let step = p.sound.rate as f64 / self.rate as f64 * p.pitch.clamp(0.25, 4.0) as f64; + self.ages += 1; + self.voices.push(Voice { play: p, pos: 0.0, step, cur: (0.0, 0.0), fade: 1.0, fade_step: 0.0, lp: (0.0, 0.0), lp_a: 1.0, level: 1.0, age: self.ages, started: false }); + } + + /// Fills `out` (interleaved, `channels` a frame): the first two channels are left and right, a mono device gets + /// their mean, any others silence. + pub fn render(&mut self, out: &mut [f32], channels: usize) { + while let Ok(c) = self.rx.try_recv() { + self.command(c); + } + out.fill(0.0); + let channels = channels.max(1); + let frames = out.len() / channels; + if self.paused || frames == 0 { + return; + } + let rate = self.rate as f32; + let l = self.listener; + let n = frames as f32; + for v in &mut self.voices { + // Where it is heard this block. + let (g, pan, cutoff) = match v.play.at { + Some(at) => place(&l, at, v.play.falloff), + None => (1.0, 0.0, 18_000.0), + }; + let (pl, pr) = pan_gains(pan); + let gain = v.play.gain.clamp(0.0, 4.0) * g; + let target = (gain * pl, gain * pr); + let a = 1.0 - (-TAU * cutoff.min(rate * 0.45) / rate).exp(); + if !v.started { + // (The first block starts where it is: no ramp up from silence, no low pass's settling.) + v.cur = target; + v.lp_a = a; + v.started = true; + } + v.level = gain * v.fade; + let (dl, dr) = ((target.0 - v.cur.0) / n, (target.1 - v.cur.1) / n); + let da = (a - v.lp_a) / n; + let s = &v.play.sound; + let sc = s.channels.max(1); + let len = s.frames(); + let mut done = false; + for f in 0..frames { + let i = v.pos as usize; + if i >= len { + if v.play.looped && len > 0 { + v.pos -= len as f64; + continue_loop(&mut v.pos, len); + } else { + done = true; + break; + } + } + let i = v.pos as usize; + let t = (v.pos - i as f64) as f32; + let j = if i + 1 < len { i + 1 } else if v.play.looped { 0 } else { i }; + let (xl, xr) = if sc == 1 { + let x = s.data[i] + (s.data[j] - s.data[i]) * t; + (x, x) + } else { + let (a0, b0) = (s.data[i * sc], s.data[i * sc + 1]); + let (a1, b1) = (s.data[j * sc], s.data[j * sc + 1]); + (a0 + (a1 - a0) * t, b0 + (b1 - b0) * t) + }; + v.lp.0 += v.lp_a * (xl - v.lp.0); + v.lp.1 += v.lp_a * (xr - v.lp.1); + v.lp_a += da; + v.cur.0 += dl; + v.cur.1 += dr; + if v.fade_step != 0.0 { + v.fade = (v.fade + v.fade_step).clamp(0.0, 1.0); + if v.fade <= 0.0 { + done = true; + break; + } + } + let k = f * channels; + let (ol, or) = (v.lp.0 * v.cur.0 * v.fade, v.lp.1 * v.cur.1 * v.fade); + if channels == 1 { + out[k] += 0.5 * (ol + or); + } else { + out[k] += ol; + out[k + 1] += or; + } + v.pos += v.step; + } + v.cur = target; + v.lp_a = a; + if done { + v.fade = 0.0; + v.fade_step = -1.0; + } + } + self.voices.retain(|v| v.fade > 0.0); + // The app's volume, then the limiter: transparent under −2 dB (0.8), a soft knee above it, never past 1. + let vol = self.volume; + for x in out.iter_mut() { + let y = *x * vol; + let a = y.abs(); + *x = if a <= 0.8 { y } else { y.signum() * (0.8 + 0.2 * ((a - 0.8) / 0.2).tanh()) }; + } + } +} + +fn continue_loop(pos: &mut f64, len: usize) { + while *pos >= len as f64 { + *pos -= len as f64; + } + if *pos < 0.0 { + *pos = 0.0; + } +} + +/// How long a stop fades when the game does not say. +pub fn stop_fade(secs: Option) -> f32 { + secs.map_or(DEFAULT_STOP_FADE, |s| s.clamp(0.0, 10.0) as f32) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::mpsc::channel; + + fn tone(frames: usize, rate: u32) -> Arc { + Arc::new(Sound { name: "t".into(), rate, channels: 1, data: (0..frames).map(|i| (i as f32 * 0.05).sin() * 0.5).collect() }) + } + + fn play(s: &Arc, at: Option) -> Play { + Play { sound: s.clone(), at, gain: 1.0, pitch: 1.0, looped: false, key: None, falloff: DEFAULT_FALLOFF } + } + + fn energy(b: &[f32], ch: usize, c: usize) -> f32 { + b.iter().skip(c).step_by(ch).map(|x| x * x).sum() + } + + #[test] + fn distance_falls_as_falloff_over_distance_and_fades_out() { + let l = Listener::default(); + assert_eq!(place(&l, Vec3::new(0.0, 0.0, -1.5), 2.0).0, 1.0); + assert!((place(&l, Vec3::new(0.0, 0.0, -20.0), 2.0).0 - 0.1).abs() < 1e-6); + assert!((place(&l, Vec3::new(0.0, 0.0, -110.0), 2.0).0 - 2.0 / 110.0 * 0.5).abs() < 1e-6); + assert_eq!(place(&l, Vec3::new(0.0, 0.0, -130.0), 2.0).0, 0.0); + } + + #[test] + fn a_sound_to_the_right_is_louder_on_the_right_and_one_in_front_is_in_the_middle() { + let l = Listener::default(); + let (_, pan, _) = place(&l, Vec3::new(10.0, 0.0, 0.0), 2.0); + assert!((pan - MAX_PAN).abs() < 1e-6); + let (gl, gr) = pan_gains(pan); + assert!(gr > 3.0 * gl, "{gl} {gr}"); + let (_, front, _) = place(&l, Vec3::new(0.0, 0.0, -10.0), 2.0); + assert!(front.abs() < 1e-6); + let (a, b) = pan_gains(0.0); + assert!((a - 1.0).abs() < 1e-6 && (b - 1.0).abs() < 1e-6); + // Behind is duller than in front at the same distance. + assert!(place(&l, Vec3::new(0.0, 0.0, 10.0), 2.0).2 < place(&l, Vec3::new(0.0, 0.0, -10.0), 2.0).2); + } + + #[test] + fn mixes_places_and_ends_a_one_shot() { + let (tx, rx) = channel(); + let mut m = Mixer::new(48_000, rx); + let s = tone(4800, 48_000); + tx.send(Cmd::Play(play(&s, Some(Vec3::new(5.0, 0.0, 0.0))))).unwrap(); + let mut b = vec![0.0; 1024 * 2]; + m.render(&mut b, 2); + assert!(energy(&b, 2, 1) > 4.0 * energy(&b, 2, 0)); + for _ in 0..10 { + m.render(&mut b, 2); + } + assert_eq!(m.voices(), 0); + assert!(b.iter().all(|x| *x == 0.0)); + } + + #[test] + fn pitch_two_ends_in_half_the_time_and_a_loop_goes_on_until_stopped() { + let (tx, rx) = channel(); + let mut m = Mixer::new(48_000, rx); + let s = tone(4800, 48_000); + tx.send(Cmd::Play(Play { pitch: 2.0, ..play(&s, None) })).unwrap(); + let mut b = vec![0.0; 2400 * 2]; + m.render(&mut b, 2); + assert!(m.voices() == 0 || { m.render(&mut b, 2); m.voices() == 0 }); + tx.send(Cmd::Play(Play { looped: true, key: Some("amb".into()), ..play(&s, None) })).unwrap(); + for _ in 0..20 { + m.render(&mut b, 2); + } + assert_eq!(m.voices(), 1); + tx.send(Cmd::Stop(Some("amb".into()), 0.01)).unwrap(); + m.render(&mut b, 2); + assert_eq!(m.voices(), 0); + } + + #[test] + fn copies_and_voices_are_bounded_and_the_limiter_holds() { + let (tx, rx) = channel(); + let mut m = Mixer::new(48_000, rx); + let s = tone(48_000, 48_000); + for _ in 0..20 { + tx.send(Cmd::Play(Play { gain: 4.0, ..play(&s, None) })).unwrap(); + } + let mut b = vec![0.0; 256 * 2]; + m.render(&mut b, 2); + assert_eq!(m.voices(), MAX_COPIES); + assert!(b.iter().all(|x| x.abs() <= 1.0)); + for i in 0..60 { + tx.send(Cmd::Play(play(&tone(48_000 + i, 48_000), None))).unwrap(); + } + m.render(&mut b, 2); + assert_eq!(m.voices(), MAX_VOICES); + } + + #[test] + fn paused_is_silent_and_volume_scales() { + let (tx, rx) = channel(); + let mut m = Mixer::new(48_000, rx); + let s = tone(48_000, 48_000); + tx.send(Cmd::Play(Play { looped: true, key: Some("k".into()), ..play(&s, None) })).unwrap(); + let mut b = vec![0.0; 512]; + m.render(&mut b, 1); + let full = energy(&b, 1, 0); + tx.send(Cmd::Volume(0.5)).unwrap(); + m.render(&mut b, 1); + assert!(energy(&b, 1, 0) < full * 0.5); + tx.send(Cmd::Paused(true)).unwrap(); + m.render(&mut b, 1); + assert!(b.iter().all(|x| *x == 0.0)); + } +} diff --git a/renderer/src/audio/mod.rs b/renderer/src/audio/mod.rs new file mode 100644 index 0000000..907f329 --- /dev/null +++ b/renderer/src/audio/mod.rs @@ -0,0 +1,336 @@ +//! Sound for games: the `audio.*` messages of the host channel (docs/RUNTIME.md "Sound"; packages/gameplay +//! src/audio.ts), played by the engine. Every app that embeds the engine (`threert-render play`, the ThreeRT clients +//! on macOS, iOS, Linux, Android) hands the game's posts and its camera to an [`Audio`]; the decoding (decode.rs), the +//! placing and the mixing (mix.rs) are this module's, so a game sounds the same wherever it runs. Only the device +//! differs a platform (out.rs). +//! +//! ```text +//! game: audio.play("assets/sounds/shot.ogg", { at: muzzle, gain: 0.9, pitch: 1.03 }) +//! -> { t: "audio.play", sound: "assets/sounds/shot.ogg", at: [x, y, z], gain: 0.9, pitch: 1.03 } +//! app: audio.post(&message) each message the game posted +//! audio.listener(&camera) once a frame: the ears are the camera the frame was drawn from +//! audio.take_log() what to tell the developer (a sound that is missing, the device) +//! ``` +//! +//! A sound is a file of the game's package (the path is the package's, as the scene's files are), decoded once and +//! kept until the game closes: in the background after `audio.preload`, otherwise when first played. The device opens +//! when the first sound plays and closes with the game (`close`) or while the app is in the background (`set_paused`); +//! where none opens the game goes on, silent, and the log says why once. + +mod decode; +mod mix; +mod out; + +pub use decode::{decode, EXTENSIONS}; +pub use mix::{pan_gains, place, Listener, Sound, DEFAULT_FALLOFF, MAX_COPIES, MAX_VOICES}; + +use glam::Vec3; +use mix::{Cmd, Mixer, Play}; +use serde_json::Value; +use std::collections::HashMap; +use std::path::{Component, Path, PathBuf}; +use std::sync::mpsc::{channel, Receiver, Sender}; +use std::sync::{Arc, Mutex}; + +/// Plays a frame may start (more are dropped: a shotgun's pellets on a wall are one sound each). +pub const MAX_PLAYS_A_FRAME: usize = 32; + +enum Device { + Closed, + Opening(out::Output), + Open(#[allow(dead_code)] out::Output), + /// It did not open: not tried again until the next game. + Failed, +} + +pub struct Audio { + tx: Sender, + mixer: Arc>, + device: Device, + root: Option, + sounds: HashMap>>, + decoded_tx: Sender<(String, Result)>, + decoded: Receiver<(String, Result)>, + listener: Listener, + volume: f32, + paused: bool, + plays: usize, + log: Vec<(&'static str, String)>, + /// What the device was opened as, for the app to show (None: not open). + pub device_name: Option, +} + +impl Default for Audio { + fn default() -> Self { + Self::new() + } +} + +impl Audio { + /// No device yet: it opens when a game plays its first sound. + pub fn new() -> Audio { + let (tx, rx) = channel(); + let (decoded_tx, decoded) = channel(); + Audio { + tx, + mixer: Arc::new(Mutex::new(Mixer::new(48_000, rx))), + device: Device::Closed, + root: None, + sounds: HashMap::new(), + decoded_tx, + decoded, + listener: Listener::default(), + volume: 1.0, + paused: false, + plays: 0, + log: vec![], + device_name: None, + } + } + + /// A game opened: its files are under `root` (the package's asset root). What the last one played stops. + pub fn open(&mut self, root: &Path) { + self.close(); + self.root = Some(root.to_path_buf()); + } + + /// The game closed: its sounds stop, are forgotten, and the device closes. + pub fn close(&mut self) { + let _ = self.tx.send(Cmd::Stop(None, 0.0)); + self.sounds.clear(); + self.root = None; + if !matches!(self.device, Device::Closed) { + // (The mixer drops the stopped voices on its next block; the device goes now.) + self.device = Device::Closed; + self.device_name = None; + if let Ok(mut m) = self.mixer.lock() { + let mut scratch = [0.0f32; 2]; + m.render(&mut scratch, 2); + } + } + } + + /// The app's volume, 0 to 1 (0: muted; the device stays as it is). + pub fn set_volume(&mut self, v: f32) { + let v = if v.is_finite() { v.clamp(0.0, 1.0) } else { 1.0 }; + if v != self.volume { + self.volume = v; + let _ = self.tx.send(Cmd::Volume(v)); + } + } + + pub fn volume(&self) -> f32 { + self.volume + } + + /// The app went to the background (true) or came back: the device is let go of and opened again; looping sounds + /// go on where they were, one-shots that came in between are dropped. + pub fn set_paused(&mut self, paused: bool) { + if paused == self.paused { + return; + } + self.paused = paused; + let _ = self.tx.send(Cmd::Paused(paused)); + if paused { + if matches!(self.device, Device::Open(_) | Device::Opening(_)) { + self.device = Device::Closed; + self.device_name = None; + } + } else if self.root.is_some() && self.mixer.lock().map(|m| m.voices() > 0).unwrap_or(false) { + self.start_device(); + } + } + + /// Where the ears are: the camera the frame was drawn from (position, forward, right). Once a frame, after the + /// game's messages of that frame. + pub fn listener(&mut self, pos: Vec3, forward: Vec3, right: Vec3) { + self.plays = 0; + let l = Listener { pos, forward: forward.normalize_or(Vec3::NEG_Z), right: right.normalize_or(Vec3::X) }; + if l != self.listener { + self.listener = l; + let _ = self.tx.send(Cmd::Listener(l)); + } + self.poll(); + } + + /// What happened that the developer should know, since the last call: (level, text). + pub fn take_log(&mut self) -> Vec<(&'static str, String)> { + self.poll(); + std::mem::take(&mut self.log) + } + + /// One message the game posted. Not an `audio.*` one: false, not this module's. + pub fn post(&mut self, m: &Value) -> bool { + let Some(t) = m["t"].as_str() else { return false }; + match t { + "audio.play" => self.play(m), + "audio.stop" => { + let key = m["key"].as_str().map(String::from); + let _ = self.tx.send(Cmd::Stop(key, mix::stop_fade(m["fade"].as_f64()))); + } + "audio.move" => { + if let (Some(key), Some(at)) = (m["key"].as_str(), vec3(&m["at"])) { + let _ = self.tx.send(Cmd::Move(key.to_string(), at)); + } + } + "audio.preload" => { + for s in m["sounds"].as_array().into_iter().flatten().filter_map(Value::as_str) { + self.preload(s); + } + } + _ => return t.starts_with("audio."), + } + true + } + + fn play(&mut self, m: &Value) { + let Some(name) = m["sound"].as_str() else { + self.log.push(("warn", "audio.play without a sound".into())); + return; + }; + if self.plays >= MAX_PLAYS_A_FRAME { + return; + } + self.plays += 1; + let Some(sound) = self.sound(name) else { return }; + let num = |k: &str, d: f32| m[k].as_f64().map(|v| v as f32).filter(|v| v.is_finite()).unwrap_or(d); + let play = Play { + sound, + at: vec3(&m["at"]), + gain: num("gain", 1.0).clamp(0.0, 4.0), + pitch: num("pitch", 1.0).clamp(0.25, 4.0), + looped: m["loop"].as_bool().unwrap_or(false), + key: m["key"].as_str().map(String::from), + falloff: num("falloff", DEFAULT_FALLOFF).clamp(0.05, 1000.0), + }; + if !self.paused && matches!(self.device, Device::Closed) { + self.start_device(); + } + let _ = self.tx.send(Cmd::Play(play)); + } + + fn start_device(&mut self) { + match out::open(self.mixer.clone()) { + Ok(o) => self.device = Device::Opening(o), + Err(e) => { + self.log.push(("warn", format!("no sound: {e}"))); + self.device = Device::Failed; + } + } + } + + /// The decoded sound `name`, decoded now if it is not yet (None: missing or unreadable, said once). + fn sound(&mut self, name: &str) -> Option> { + self.poll(); + if let Some(s) = self.sounds.get(name) { + return s.clone(); + } + let r = self.path(name).and_then(|p| std::fs::read(&p).map_err(|e| format!("{name}: {e}"))).and_then(|b| decode(name, b).map_err(|e| format!("{e:#}"))); + let s = match r { + Ok(s) => Some(Arc::new(s)), + Err(e) => { + self.log.push(("warn", format!("sound {e}"))); + None + } + }; + self.sounds.insert(name.to_string(), s.clone()); + s + } + + /// Decodes `name` on a thread of its own, unless it is decoded or on its way. + fn preload(&mut self, name: &str) { + if self.sounds.contains_key(name) { + return; + } + let path = match self.path(name) { + Ok(p) => p, + Err(e) => { + self.log.push(("warn", format!("sound {e}"))); + self.sounds.insert(name.to_string(), None); + return; + } + }; + // (On its way: an entry is made when it arrives; a play before that decodes it itself.) + let (tx, n) = (self.decoded_tx.clone(), name.to_string()); + std::thread::spawn(move || { + let r = std::fs::read(&path).map_err(|e| format!("{n}: {e}")).and_then(|b| decode(&n, b).map_err(|e| format!("{e:#}"))); + let _ = tx.send((n, r)); + }); + } + + fn path(&self, name: &str) -> Result { + let root = self.root.as_ref().ok_or_else(|| format!("{name}: no game is open"))?; + let rel = Path::new(name.trim_start_matches("./")); + if name.is_empty() || !rel.components().all(|c| matches!(c, Component::Normal(_))) { + return Err(format!("{name}: a sound is a path inside the game's package")); + } + Ok(root.join(rel)) + } + + fn poll(&mut self) { + while let Ok((name, r)) = self.decoded.try_recv() { + if self.root.is_none() || self.sounds.contains_key(&name) { + continue; + } + let s = match r { + Ok(s) => Some(Arc::new(s)), + Err(e) => { + self.log.push(("warn", format!("sound {e}"))); + None + } + }; + self.sounds.insert(name, s); + } + if let Device::Opening(o) = &self.device { + match o.ready.try_recv() { + Ok(Ok((rate, channels, name))) => { + self.log.push(("info", format!("sound: {name}, {rate} Hz, {channels} channels"))); + self.device_name = Some(name); + if let Device::Opening(o) = std::mem::replace(&mut self.device, Device::Closed) { + self.device = Device::Open(o); + } + } + Ok(Err(e)) => { + self.log.push(("warn", format!("no sound: {e}"))); + self.device = Device::Failed; + } + Err(_) => {} + } + } + } +} + +fn vec3(v: &Value) -> Option { + let a = v.as_array()?; + if a.len() != 3 { + return None; + } + let f = |i: usize| a[i].as_f64().map(|x| x as f32).filter(|x| x.is_finite()); + Some(Vec3::new(f(0)?, f(1)?, f(2)?)) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn plays_a_package_sound_and_refuses_paths_outside_it() { + let dir = std::env::temp_dir().join(format!("threert-audio-{}", std::process::id())); + std::fs::create_dir_all(dir.join("assets")).unwrap(); + let samples: Vec = (0..2205).map(|i| ((i as f32 * 0.1).sin() * 8000.0) as i16).collect(); + std::fs::write(dir.join("assets/a.wav"), decode::tests_wav(22_050, &samples)).unwrap(); + let mut a = Audio::new(); + a.open(&dir); + assert!(a.sound("assets/a.wav").is_some()); + assert!(a.sound("../etc/passwd").is_none()); + assert!(a.sound("assets/missing.wav").is_none()); + let log = a.take_log(); + assert_eq!(log.len(), 2, "{log:?}"); + assert!(a.post(&json!({ "t": "audio.stop" }))); + assert!(!a.post(&json!({ "t": "hud" }))); + a.close(); + assert!(a.sounds.is_empty()); + let _ = std::fs::remove_dir_all(dir); + } +} diff --git a/renderer/src/audio/out.rs b/renderer/src/audio/out.rs new file mode 100644 index 0000000..4d995b4 --- /dev/null +++ b/renderer/src/audio/out.rs @@ -0,0 +1,332 @@ +//! The sound device: a thread of its own opens the system's default output and lets the mixer fill it. Only this part +//! differs between platforms, and it does nothing but move the mixer's frames: +//! +//! - macOS, iOS, Windows: cpal (Core Audio, WASAPI); +//! - Linux (the Steam Deck among them): ALSA's `default` device, the library loaded at run time (libasound.so.2: on +//! SteamOS and every desktop it is PipeWire's or PulseAudio's), so a program built for Linux links no sound library +//! and runs, silent, where there is none; +//! - Android: AAudio (libaaudio.so, Android 8 and later), loaded at run time too. +//! +//! The stream is opened again when the device goes (headphones pulled, the default output changed), with the same +//! mixer. Elsewhere, or when no device opens, `open` says why and the app plays nothing. + +use super::mix::Mixer; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{channel, Receiver, Sender}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::Duration; + +pub struct Output { + stop: Arc, + thread: Option>, + /// The device thread's report: once, when the stream runs (frames a second, channels, the device's name) or did + /// not open. + pub ready: Receiver>, +} + +impl Drop for Output { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + if let Some(t) = self.thread.take() { + t.thread().unpark(); + let _ = t.join(); + } + } +} + +/// What a backend reports once its stream runs. +type Ready = Sender>; + +/// Opens the default output for `mixer` (its rate is set to the device's) on a thread of its own, without waiting +/// for it: `ready` says how it went. Dropping the output closes the device; the mixer and its voices stay. +pub fn open(mixer: Arc>) -> std::io::Result { + let stop = Arc::new(AtomicBool::new(false)); + let (tx, ready) = channel(); + let st = stop.clone(); + let thread = std::thread::Builder::new().name("threert-audio".into()).spawn(move || run(mixer, st, tx))?; + Ok(Output { stop, thread: Some(thread), ready }) +} + +/// The device thread: opens, runs until stopped, opens again after the device went (at most once a second). +fn run(mixer: Arc>, stop: Arc, ready: Ready) { + let mut ready = Some(ready); + while !stop.load(Ordering::SeqCst) { + let lost = Arc::new(AtomicBool::new(false)); + if let Err(e) = backend::stream(&mixer, &stop, &lost, &mut ready) { + if let Some(r) = ready.take() { + let _ = r.send(Err(e)); + return; + } + } + if !stop.load(Ordering::SeqCst) { + std::thread::park_timeout(Duration::from_secs(1)); + } + } +} + +/// Waits until the app stops the output or the stream reports the device gone. +#[allow(dead_code)] +fn wait(stop: &AtomicBool, lost: &AtomicBool) { + while !stop.load(Ordering::SeqCst) && !lost.load(Ordering::SeqCst) { + std::thread::park_timeout(Duration::from_millis(250)); + } +} + +/// The mixer's frames as f32, `channels` a frame (a buffer reused from call to call). +#[allow(dead_code)] +fn fill(mixer: &Mutex, out: &mut [f32], channels: usize) { + match mixer.lock() { + Ok(mut m) => m.render(out, channels), + Err(_) => out.fill(0.0), + } +} + +#[cfg(any(target_os = "macos", target_os = "ios", target_os = "windows"))] +mod backend { + use super::*; + use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; + use cpal::{FromSample, SizedSample}; + + pub fn stream(mixer: &Arc>, stop: &AtomicBool, lost: &Arc, ready: &mut Option) -> Result<(), String> { + let host = cpal::default_host(); + let device = host.default_output_device().ok_or("no sound output device")?; + let name = device.description().map(|d| d.name().to_string()).unwrap_or_else(|_| "the default output".into()); + let config = device.default_output_config().map_err(|e| format!("{name}: {e}"))?; + let (rate, channels) = (config.sample_rate(), config.channels() as usize); + mixer.lock().map_err(|_| "mixer")?.set_rate(rate); + let sc: cpal::StreamConfig = config.config(); + let s = match config.sample_format() { + cpal::SampleFormat::F32 => build::(&device, &sc, mixer, lost), + cpal::SampleFormat::I16 => build::(&device, &sc, mixer, lost), + cpal::SampleFormat::I32 => build::(&device, &sc, mixer, lost), + cpal::SampleFormat::U16 => build::(&device, &sc, mixer, lost), + f => return Err(format!("{name}: sample format {f} not handled")), + } + .map_err(|e| format!("{name}: {e}"))?; + s.play().map_err(|e| format!("{name}: {e}"))?; + if let Some(r) = ready.take() { + let _ = r.send(Ok((rate, channels, name))); + } + wait(stop, lost); + Ok(()) + } + + fn build>(device: &cpal::Device, sc: &cpal::StreamConfig, mixer: &Arc>, lost: &Arc) -> Result { + let ch = sc.channels as usize; + let m = mixer.clone(); + let l = lost.clone(); + let mut buf: Vec = Vec::new(); + device.build_output_stream( + sc, + move |data: &mut [T], _: &cpal::OutputCallbackInfo| { + buf.resize(data.len(), 0.0); + fill(&m, &mut buf, ch); + for (d, s) in data.iter_mut().zip(&buf) { + *d = T::from_sample(*s); + } + }, + move |e| { + // (The device went, or another error: the stream is made again.) + if matches!(e, cpal::StreamError::DeviceNotAvailable) || !matches!(e, cpal::StreamError::BufferUnderrun) { + l.store(true, Ordering::SeqCst); + } + }, + None, + ) + } +} + +#[cfg(all(target_os = "linux", not(target_os = "android")))] +mod backend { + use super::*; + use libloading::{Library, Symbol}; + use std::ffi::{c_char, c_int, c_long, c_uint, c_ulong, c_void, CStr}; + + const RATE: u32 = 48_000; + const CHANNELS: usize = 2; + /// Frames a write: 10 ms. + const PERIOD: usize = 480; + const SND_PCM_STREAM_PLAYBACK: c_int = 0; + const SND_PCM_FORMAT_S16_LE: c_int = 2; + const SND_PCM_ACCESS_RW_INTERLEAVED: c_int = 3; + + type Open = unsafe extern "C" fn(*mut *mut c_void, *const c_char, c_int, c_int) -> c_int; + type SetParams = unsafe extern "C" fn(*mut c_void, c_int, c_int, c_uint, c_uint, c_int, c_uint) -> c_int; + type WriteI = unsafe extern "C" fn(*mut c_void, *const c_void, c_ulong) -> c_long; + type Recover = unsafe extern "C" fn(*mut c_void, c_int, c_int) -> c_int; + type Close = unsafe extern "C" fn(*mut c_void) -> c_int; + type StrError = unsafe extern "C" fn(c_int) -> *const c_char; + + pub fn stream(mixer: &Arc>, stop: &AtomicBool, _lost: &Arc, ready: &mut Option) -> Result<(), String> { + // SAFETY: libasound's documented C API, called as its headers declare it; the handle is used on this thread + // only and closed before the library goes. + unsafe { + let lib = Library::new("libasound.so.2").map_err(|e| format!("no ALSA library (libasound.so.2): {e}"))?; + let sym = |n: &[u8]| -> Result<*const (), String> { lib.get::<*const ()>(n).map(|s: Symbol<*const ()>| *s).map_err(|e| format!("ALSA: {e}")) }; + let open: Open = std::mem::transmute(sym(b"snd_pcm_open\0")?); + let set_params: SetParams = std::mem::transmute(sym(b"snd_pcm_set_params\0")?); + let writei: WriteI = std::mem::transmute(sym(b"snd_pcm_writei\0")?); + let recover: Recover = std::mem::transmute(sym(b"snd_pcm_recover\0")?); + let close: Close = std::mem::transmute(sym(b"snd_pcm_close\0")?); + let strerror: StrError = std::mem::transmute(sym(b"snd_strerror\0")?); + let err = |c: c_int| CStr::from_ptr(strerror(c)).to_string_lossy().into_owned(); + let mut pcm: *mut c_void = std::ptr::null_mut(); + let r = open(&mut pcm, c"default".as_ptr(), SND_PCM_STREAM_PLAYBACK, 0); + if r < 0 { + return Err(format!("ALSA default device: {}", err(r))); + } + // 48 kHz stereo 16-bit, resampled by ALSA when the device is otherwise, 40 ms of latency. + let r = set_params(pcm, SND_PCM_FORMAT_S16_LE, SND_PCM_ACCESS_RW_INTERLEAVED, CHANNELS as c_uint, RATE, 1, 40_000); + if r < 0 { + close(pcm); + return Err(format!("ALSA default device: {}", err(r))); + } + mixer.lock().map_err(|_| "mixer")?.set_rate(RATE); + if let Some(r) = ready.take() { + let _ = r.send(Ok((RATE, CHANNELS, "ALSA default".into()))); + } + let mut f = vec![0.0f32; PERIOD * CHANNELS]; + let mut s = vec![0i16; PERIOD * CHANNELS]; + let mut failures = 0; + while !stop.load(Ordering::SeqCst) { + fill(mixer, &mut f, CHANNELS); + for (d, x) in s.iter_mut().zip(&f) { + *d = (x.clamp(-1.0, 1.0) * 32767.0) as i16; + } + let mut off = 0usize; + while off < PERIOD && !stop.load(Ordering::SeqCst) { + let n = writei(pcm, s[off * CHANNELS..].as_ptr() as *const c_void, (PERIOD - off) as c_ulong); + if n < 0 { + // An underrun or a suspend is recovered; anything else is the device gone: open it again. + if recover(pcm, n as c_int, 1) < 0 { + failures += 1; + break; + } + } else { + off += n as usize; + } + } + if failures > 0 { + break; + } + } + close(pcm); + Ok(()) + } + } +} + +#[cfg(target_os = "android")] +mod backend { + use super::*; + use libloading::Library; + use std::ffi::c_void; + + const AAUDIO_FORMAT_PCM_FLOAT: i32 = 2; + const AAUDIO_PERFORMANCE_MODE_LOW_LATENCY: i32 = 12; + const AAUDIO_USAGE_GAME: i32 = 14; + const AAUDIO_CALLBACK_RESULT_CONTINUE: i32 = 0; + + type Builder = *mut c_void; + type Stream = *mut c_void; + type DataCb = extern "C" fn(Stream, *mut c_void, *mut c_void, i32) -> i32; + type ErrorCb = extern "C" fn(Stream, *mut c_void, i32); + + /// What the callbacks get: the mixer, the channels, and the flag that says the device went. + struct User { + mixer: Arc>, + channels: usize, + lost: Arc, + } + + extern "C" fn data(_s: Stream, user: *mut c_void, audio: *mut c_void, frames: i32) -> i32 { + // SAFETY: `user` is the `User` this stream was opened with, alive until the stream is closed; `audio` is + // `frames` frames of f32 of `channels` each, AAudio's buffer for this call. + let u = unsafe { &*(user as *const User) }; + let out = unsafe { std::slice::from_raw_parts_mut(audio as *mut f32, frames.max(0) as usize * u.channels) }; + fill(&u.mixer, out, u.channels); + AAUDIO_CALLBACK_RESULT_CONTINUE + } + + extern "C" fn error(_s: Stream, user: *mut c_void, _e: i32) { + // SAFETY: as in `data`. + let u = unsafe { &*(user as *const User) }; + u.lost.store(true, Ordering::SeqCst); + } + + pub fn stream(mixer: &Arc>, stop: &AtomicBool, lost: &Arc, ready: &mut Option) -> Result<(), String> { + // SAFETY: AAudio's documented C API (aaudio/AAudio.h, API 26), called as declared; the stream is closed (no + // callback runs after `AAudioStream_close` returns) before `user` and the library go. + unsafe { + let lib = Library::new("libaaudio.so").map_err(|e| format!("no AAudio (libaaudio.so): {e}"))?; + macro_rules! f { + ($n:literal, $t:ty) => { + *lib.get::<$t>(concat!($n, "\0").as_bytes()).map_err(|e| format!("AAudio: {e}"))? + }; + } + let create = f!("AAudio_createStreamBuilder", unsafe extern "C" fn(*mut Builder) -> i32); + let set_format = f!("AAudioStreamBuilder_setFormat", unsafe extern "C" fn(Builder, i32)); + let set_channels = f!("AAudioStreamBuilder_setChannelCount", unsafe extern "C" fn(Builder, i32)); + let set_perf = f!("AAudioStreamBuilder_setPerformanceMode", unsafe extern "C" fn(Builder, i32)); + let set_data = f!("AAudioStreamBuilder_setDataCallback", unsafe extern "C" fn(Builder, DataCb, *mut c_void)); + let set_error = f!("AAudioStreamBuilder_setErrorCallback", unsafe extern "C" fn(Builder, ErrorCb, *mut c_void)); + let open = f!("AAudioStreamBuilder_openStream", unsafe extern "C" fn(Builder, *mut Stream) -> i32); + let delete = f!("AAudioStreamBuilder_delete", unsafe extern "C" fn(Builder) -> i32); + let rate_of = f!("AAudioStream_getSampleRate", unsafe extern "C" fn(Stream) -> i32); + let start = f!("AAudioStream_requestStart", unsafe extern "C" fn(Stream) -> i32); + let close = f!("AAudioStream_close", unsafe extern "C" fn(Stream) -> i32); + // (Android 9: usage "game"; on 8 the symbol is missing and the default, media, is kept.) + let set_usage = lib.get::(b"AAudioStreamBuilder_setUsage\0").ok().map(|s| *s); + + const CHANNELS: usize = 2; + let user = Box::into_raw(Box::new(User { mixer: mixer.clone(), channels: CHANNELS, lost: lost.clone() })); + let mut b: Builder = std::ptr::null_mut(); + let r = create(&mut b); + if r != 0 { + drop(Box::from_raw(user)); + return Err(format!("AAudio: builder ({r})")); + } + set_format(b, AAUDIO_FORMAT_PCM_FLOAT); + set_channels(b, CHANNELS as i32); + set_perf(b, AAUDIO_PERFORMANCE_MODE_LOW_LATENCY); + if let Some(u) = set_usage { + u(b, AAUDIO_USAGE_GAME); + } + set_data(b, data, user as *mut c_void); + set_error(b, error, user as *mut c_void); + let mut s: Stream = std::ptr::null_mut(); + let r = open(b, &mut s); + delete(b); + if r != 0 { + drop(Box::from_raw(user)); + return Err(format!("AAudio: no output stream ({r})")); + } + let rate = rate_of(s).max(8000) as u32; + if let Ok(mut m) = mixer.lock() { + m.set_rate(rate); + } + let r = start(s); + if r != 0 { + close(s); + drop(Box::from_raw(user)); + return Err(format!("AAudio: the stream did not start ({r})")); + } + if let Some(r) = ready.take() { + let _ = r.send(Ok((rate, CHANNELS, "AAudio".into()))); + } + wait(stop, lost); + close(s); + drop(Box::from_raw(user)); + Ok(()) + } + } +} + +#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "windows", target_os = "linux", target_os = "android")))] +mod backend { + use super::*; + pub fn stream(_: &Arc>, _: &AtomicBool, _: &Arc, _: &mut Option) -> Result<(), String> { + Err("no sound output on this platform".into()) + } +} diff --git a/renderer/src/lib.rs b/renderer/src/lib.rs index c39ea82..b731de8 100644 --- a/renderer/src/lib.rs +++ b/renderer/src/lib.rs @@ -2,6 +2,8 @@ //! `threert-render` CLI and by the native playground client). pub mod api; +#[cfg(all(feature = "audio", not(target_arch = "wasm32")))] +pub mod audio; pub mod bvh; #[cfg(not(target_arch = "wasm32"))] pub mod compare; diff --git a/renderer/src/view.rs b/renderer/src/view.rs index f3fca51..bb9d080 100644 --- a/renderer/src/view.rs +++ b/renderer/src/view.rs @@ -11,7 +11,8 @@ //! Esc. With `pointer: "look"` a click captures the pointer (its motion is the game's look), Esc lets go of it, and //! Esc again quits. The window is the app of the host channel: `--launch '{"room":"K7Q4MZ"}'` is the `launch` //! message, `--sensitivity 1.5` scales the look, `--hud-log` prints the HUD the game describes (the window draws -//! none), and the game's `net.*` messages are the binary's sockets and HTTP (src/net.rs). +//! none), the game's `net.*` messages are the binary's sockets and HTTP (src/net.rs), and its `audio.*` messages are +//! played (src/audio, the `audio` feature: the ears are the camera of each frame). //! //! Controls: left-drag orbit · right/shift-drag pan · wheel zoom · WASD/QE fly · R reset camera · 1/2/4/8 cap spp, //! 0 adaptive · P snapshot · Esc quit. Scripted over stdio: @@ -111,6 +112,9 @@ struct State { input: Option, /// `pointer: "look"`: a click captures the pointer. look: bool, + /// The game's sound (the `audio.*` messages; the ears are the camera of the frame). + #[cfg(feature = "audio")] + audio: crate::audio::Audio, } impl State { @@ -274,6 +278,12 @@ impl App { game, input: pointer.is_some().then(InputTracker::new), look: pointer.as_deref() == Some("look"), + #[cfg(feature = "audio")] + audio: { + let mut au = crate::audio::Audio::new(); + au.open(&a.assets); + au + }, }); Ok(()) } @@ -552,6 +562,10 @@ fn frame(s: &mut State, cmds: &mpsc::Receiver, services: &mut dyn HostServi // Game to app: what the game posted in this frame's entries (events among them). let posts = s.game.take_posts(); for p in &posts { + #[cfg(feature = "audio")] + if s.audio.post(p) { + continue; + } services.post(p); if args.hud_log && p["t"] == "hud" { eprintln!("hud: {}", p["hud"]); @@ -566,6 +580,14 @@ fn frame(s: &mut State, cmds: &mpsc::Receiver, services: &mut dyn HostServi follow_game_camera(s, cam); s.core.set_camera(s.orbit.camera(s.tan)); } + #[cfg(feature = "audio")] + { + let c = s.core.cam; + s.audio.listener(c.pos, c.fwd, c.right); + for (level, text) in s.audio.take_log() { + eprintln!("audio: {level}: {text}"); + } + } let tex = match s.surface.get_current_texture() { wgpu::CurrentSurfaceTexture::Success(t) | wgpu::CurrentSurfaceTexture::Suboptimal(t) => t, diff --git a/tools/three/audio.ts b/tools/three/audio.ts new file mode 100644 index 0000000..c0c89ec --- /dev/null +++ b/tools/three/audio.ts @@ -0,0 +1,234 @@ +/// +// The `audio.*` messages in a page (docs/RUNTIME.md "Sound"), on Web Audio: what the engine's library does natively +// (renderer/src/audio), kept to its numbers so a game sounds the same in a browser. Each voice is a buffer source, +// a low pass, and a gain for each ear; once a frame the listener (the camera) places every voice by the same rules as +// mix.rs `place` and `pan_gains`. The output goes through the same limiter. +// +// A browser plays nothing before the person has touched the page: the context resumes on the first key or click. +// Files are fetched from the page's root (`/assets/sounds/shot.ogg`), as the scene's are (engine.ts). +import type { HostMessage } from "@threert/gameplay"; + +export const DEFAULT_FALLOFF = 2; +const FADE_FROM = 100, SILENT_AT = 120, MAX_PAN = 0.8, MAX_VOICES = 48, MAX_COPIES = 8, MAX_PLAYS_A_FRAME = 32; + +type V3 = [number, number, number]; +const sub = (a: V3, b: V3): V3 => [a[0] - b[0], a[1] - b[1], a[2] - b[2]]; +const dot = (a: V3, b: V3) => a[0] * b[0] + a[1] * b[1] + a[2] * b[2]; + +/** Gain, pan and the low pass's cutoff of a sound at `at` for the listener: mix.rs `place`, number for number. */ +export function place(pos: V3, forward: V3, right: V3, at: V3, falloff: number): [number, number, number] { + falloff = Math.max(falloff, 0.05); + const to = sub(at, pos); + const d = Math.hypot(...to); + let gain = d <= falloff ? 1 : falloff / d; + if (d > FADE_FROM) gain *= Math.min(1, Math.max(0, (SILENT_AT - d) / (SILENT_AT - FADE_FROM))); + if (d < 1e-4) return [gain, 0, 18000]; + const dir: V3 = [to[0] / d, to[1] / d, to[2] / d]; + const pan = Math.min(1, Math.max(-1, dot(dir, right) * Math.min(1, d / falloff))) * MAX_PAN; + const behind = Math.max(0, -dot(dir, forward)); + return [gain, pan, (18000 / (1 + d / 40)) * (1 - 0.5 * behind)]; +} + +/** Equal-power gains of a pan in −1..1, the middle at 1 on both sides: mix.rs `pan_gains`. */ +export function panGains(pan: number): [number, number] { + const th = (Math.min(1, Math.max(-1, pan)) + 1) * (Math.PI / 4); + return [Math.cos(th) * Math.SQRT2, Math.sin(th) * Math.SQRT2]; +} + +interface Voice { + sound: string; + src: AudioBufferSourceNode; + lp: BiquadFilterNode; + left: GainNode; + right: GainNode; + fade: GainNode; + at?: V3; + gain: number; + falloff: number; + key?: string; + looped: boolean; + level: number; + age: number; + ended: boolean; +} + +export class PageAudio { + private ctx: AudioContext | null = null; + private master: GainNode | null = null; + private readonly buffers = new Map>(); + private readonly ready = new Map(); + private voices: Voice[] = []; + private listener: { pos: V3; forward: V3; right: V3 } = { pos: [0, 0, 0], forward: [0, 0, -1], right: [1, 0, 0] }; + private plays = 0; + private ages = 0; + private readonly off: (() => void)[] = []; + private readonly base: string; + + constructor(base = "/") { + this.base = base; + const resume = () => void this.context().resume(); + for (const t of ["keydown", "pointerdown"] as const) { + window.addEventListener(t, resume, { capture: true }); + this.off.push(() => window.removeEventListener(t, resume, { capture: true })); + } + } + + private context(): AudioContext { + if (!this.ctx) { + this.ctx = new AudioContext({ latencyHint: "interactive" }); + // The limiter of mix.rs: transparent under 0.8, a soft knee to 1. + const shaper = this.ctx.createWaveShaper(); + const n = 4097, curve = new Float32Array(n); + for (let i = 0; i < n; i++) { + const x = (i / (n - 1)) * 4 - 2, a = Math.abs(x); + curve[i] = a <= 0.8 ? x : Math.sign(x) * (0.8 + 0.2 * Math.tanh((a - 0.8) / 0.2)); + } + shaper.curve = curve; + this.master = this.ctx.createGain(); + this.master.connect(shaper).connect(this.ctx.destination); + } + return this.ctx; + } + + /** The app's volume, 0..1. */ + set volume(v: number) { + if (this.master) this.master.gain.value = Math.min(1, Math.max(0, v)); + } + + /** A message the game posted: false when it is not `audio.*`. */ + post(m: HostMessage): boolean { + if (typeof m.t !== "string" || !m.t.startsWith("audio.")) return false; + if (m.t === "audio.play") { + this.play(m); + } else if (m.t === "audio.stop") { + this.stop(typeof m.key === "string" ? m.key : undefined, typeof m.fade === "number" ? m.fade : 0.05); + } else if (m.t === "audio.move" && typeof m.key === "string" && Array.isArray(m.at)) { + for (const v of this.voices) if (v.key === m.key) v.at = m.at as V3; + } else if (m.t === "audio.preload" && Array.isArray(m.sounds)) { + for (const s of m.sounds) if (typeof s === "string") void this.load(s); + } + return true; + } + + private load(sound: string): Promise { + let p = this.buffers.get(sound); + if (!p) { + if (!sound || sound.split("/").some((c) => c === ".." || c === "") || sound.startsWith("/")) { + console.warn(`sound ${sound}: a sound is a path inside the game's package`); + p = Promise.resolve(null); + } else { + p = fetch(this.base + sound) + .then((r) => (r.ok ? r.arrayBuffer() : Promise.reject(new Error(`${r.status}`)))) + .then((b) => this.context().decodeAudioData(b)) + .catch((e) => (console.warn(`sound ${sound}: ${e instanceof Error ? e.message : e}`), null)); + } + p.then((b) => this.ready.set(sound, b)); + this.buffers.set(sound, p); + } + return p; + } + + private play(m: HostMessage) { + if (typeof m.sound !== "string" || this.plays >= MAX_PLAYS_A_FRAME) return; + this.plays++; + const num = (v: unknown, d: number, lo: number, hi: number) => (typeof v === "number" && Number.isFinite(v) ? Math.min(hi, Math.max(lo, v)) : d); + const o = { + at: Array.isArray(m.at) && m.at.length === 3 ? (m.at as V3) : undefined, + gain: num(m.gain, 1, 0, 4), + pitch: num(m.pitch, 1, 0.25, 4), + looped: m.loop === true, + key: typeof m.key === "string" ? m.key : undefined, + falloff: num(m.falloff, DEFAULT_FALLOFF, 0.05, 1000), + }; + const buffer = this.ready.get(m.sound); + // (Not decoded yet: it starts when it is, if that is soon; a one-shot that would be late is dropped.) + if (buffer === undefined) { + const asked = performance.now(); + void this.load(m.sound).then((b) => b && (o.looped || performance.now() - asked < 150) && this.start(m.sound as string, b, o)); + return; + } + if (buffer) this.start(m.sound, buffer, o); + } + + private start(sound: string, buffer: AudioBuffer, o: { at?: V3; gain: number; pitch: number; looped: boolean; key?: string; falloff: number }) { + const ctx = this.context(); + const copies = this.voices.filter((v) => v.sound === sound && !v.ended); + if (copies.length >= MAX_COPIES) this.end(copies.reduce((a, b) => (a.age < b.age ? a : b)), 0); + if (this.voices.length >= MAX_VOICES) { + const quiet = this.voices.filter((v) => !v.looped).sort((a, b) => a.level - b.level)[0]; + if (!quiet) return; + this.end(quiet, 0); + } + const src = ctx.createBufferSource(); + src.buffer = buffer; + src.loop = o.looped; + src.playbackRate.value = o.pitch; + const lp = ctx.createBiquadFilter(); + lp.type = "lowpass"; + lp.Q.value = 0.5; + const left = ctx.createGain(), right = ctx.createGain(), fade = ctx.createGain(); + const merge = ctx.createChannelMerger(2); + src.connect(lp); + if (buffer.numberOfChannels > 1) { + const split = ctx.createChannelSplitter(2); + lp.connect(split); + split.connect(left, 0); + split.connect(right, 1); + } else { + lp.connect(left); + lp.connect(right); + } + left.connect(merge, 0, 0); + right.connect(merge, 0, 1); + merge.connect(fade).connect(this.master!); + const v: Voice = { sound, src, lp, left, right, fade, at: o.at, gain: o.gain, falloff: o.falloff, key: o.key, looped: o.looped, level: o.gain, age: ++this.ages, ended: false }; + this.apply(v, true); + src.onended = () => { + v.ended = true; + this.voices = this.voices.filter((x) => x !== v); + merge.disconnect(); + }; + src.start(); + this.voices.push(v); + } + + private end(v: Voice, fade: number) { + if (v.ended) return; + v.ended = true; + const t = this.ctx!.currentTime; + v.fade.gain.setValueAtTime(v.fade.gain.value, t); + v.fade.gain.linearRampToValueAtTime(0, t + Math.max(fade, 0.005)); + v.src.stop(t + Math.max(fade, 0.005)); + } + + stop(key?: string, fade = 0.05) { + for (const v of this.voices) if (key === undefined || v.key === key) this.end(v, fade); + } + + /** Where it is heard from this frame on (`now`: without a ramp). */ + private apply(v: Voice, now = false) { + const l = this.listener; + const [g, pan, cutoff] = v.at ? place(l.pos, l.forward, l.right, v.at, v.falloff) : [1, 0, 18000]; + const [pl, pr] = panGains(pan); + v.level = v.gain * g; + const t = this.ctx!.currentTime; + const set = (p: AudioParam, x: number) => (now ? p.setValueAtTime(x, t) : p.setTargetAtTime(x, t, 0.01)); + set(v.left.gain, v.gain * g * pl); + set(v.right.gain, v.gain * g * pr); + set(v.lp.frequency, Math.min(cutoff, this.ctx!.sampleRate * 0.45)); + } + + /** Once a frame, after the game's messages: the ears are the camera (position, forward, right). */ + setListener(pos: V3, forward: V3, right: V3) { + this.plays = 0; + this.listener = { pos, forward, right }; + if (this.ctx) for (const v of this.voices) if (v.at) this.apply(v); + } + + close() { + this.stop(undefined, 0); + for (const f of this.off.splice(0)) f(); + void this.ctx?.close(); + this.ctx = null; + } +} diff --git a/tools/three/test/audio.test.ts b/tools/three/test/audio.test.ts new file mode 100644 index 0000000..3455f62 --- /dev/null +++ b/tools/three/test/audio.test.ts @@ -0,0 +1,25 @@ +// The page's placing (tools/three/audio.ts) is the native mixer's (renderer/src/audio/mix.rs `place`, `pan_gains`): +// the same figures for the same points, so a game sounds the same in a browser. The numbers are mix.rs's tests'. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { panGains, place } from "../audio.ts"; + +const L = { pos: [0, 0, 0] as [number, number, number], forward: [0, 0, -1] as [number, number, number], right: [1, 0, 0] as [number, number, number] }; +const at = (x: number, y: number, z: number, falloff = 2) => place(L.pos, L.forward, L.right, [x, y, z], falloff); + +test("distance: level out to the falloff, then falloff / distance, silent past 120 m", () => { + assert.equal(at(0, 0, -1.5)[0], 1); + assert.ok(Math.abs(at(0, 0, -20)[0] - 0.1) < 1e-9); + assert.ok(Math.abs(at(0, 0, -110)[0] - (2 / 110) * 0.5) < 1e-9); + assert.equal(at(0, 0, -130)[0], 0); +}); + +test("direction: at most 80 % to a side, the middle at 1 on both, duller behind", () => { + assert.ok(Math.abs(at(10, 0, 0)[1] - 0.8) < 1e-9); + const [l, r] = panGains(0.8); + assert.ok(r > 3 * l); + const [a, b] = panGains(0); + assert.ok(Math.abs(a - 1) < 1e-9 && Math.abs(b - 1) < 1e-9); + assert.ok(at(0, 0, 10)[2] < at(0, 0, -10)[2]); + assert.ok(Math.abs(at(0, 0, -10)[2] - 18000 / 1.25) < 1e-6); +}); diff --git a/tools/three/view.ts b/tools/three/view.ts index d7f5637..34f01ca 100644 --- a/tools/three/view.ts +++ b/tools/three/view.ts @@ -10,12 +10,13 @@ // // A game is played, not only looked at: its frame script runs every frame, its `keys` run their handlers, a click is // its `click` event (the object under it, the view ray), and the page is the app of its host channel -// (docs/RUNTIME.md "The host channel"; tools/three/host.ts, hud.ts): `net.*` on the browser's WebSocket and fetch, -// `launch` from the page's query (`?room=K7Q4MZ&name=Ada`), the HUD drawn as DOM over the canvas. A game that reads -// input (`pointer` in its definition) has the keys, the pointer and the camera to itself: no orbit controls, and -// with `pointer: "look"` a click takes the pointer lock (Esc lets go). With `physics=` the page -// loads the engine's own physics (tools/three/physics.ts; tools/three/open.ts builds it and adds the parameter), so -// bodies move and the physics queries answer as in the renderer. +// (docs/RUNTIME.md "The host channel"; tools/three/host.ts, hud.ts, audio.ts): `net.*` on the browser's WebSocket and +// fetch, `audio.*` on Web Audio by the native mixer's rules, `launch` from the page's query (`?room=K7Q4MZ&name=Ada`), +// the HUD drawn as DOM over the canvas. A game that reads input (`pointer` in its definition) has the keys, the pointer +// and the camera to itself: no orbit controls, and with `pointer: "look"` a click takes the pointer lock (Esc lets go). +// With `physics=` the page loads the engine's own physics (tools/three/physics.ts; +// tools/three/open.ts builds it and adds the parameter), so bodies move and the physics queries answer as in the +// renderer. // // URL parameters: ev100 (auto-exposure cameras), autoLights (the adapter's cap; 0 = none), width / height (a fixed // canvas instead of the window), check (after 120 frames publish `window.__result`: the overlay text and the frame as a @@ -30,6 +31,7 @@ import { RenderPass } from "three/addons/postprocessing/RenderPass.js"; import { ShaderPass } from "three/addons/postprocessing/ShaderPass.js"; import { defineGame, installNodeHost, type GameDefinition, type HostMessage, type Hud } from "@threert/gameplay"; import { ThreeEngine } from "./engine.ts"; +import { PageAudio } from "./audio.ts"; import { launchMessage, PageInput, PageNet } from "./host.ts"; import { HudOverlay } from "./hud.ts"; import { VignetteShader } from "./page.ts"; @@ -65,11 +67,12 @@ export async function run(mod: SceneModule): Promise { const posts: HostMessage[] = []; (globalThis as { __host_post?: (text: string) => void }).__host_post = (text) => void posts.push(JSON.parse(text) as HostMessage); const net = new PageNet(); + const sound = new PageAudio(); let hud: HudOverlay | null = null; const flush = () => { for (const m of posts.splice(0)) { if (m.t === "hud") (hud ??= new HudOverlay()).set((m.hud ?? {}) as Hud); - else net.post(m); + else if (!sound.post(m)) net.post(m); } }; // The engine's physics, when the page was opened with it. @@ -227,6 +230,13 @@ export async function run(mod: SceneModule): Promise { } } controls.update(); + // The ears are the camera the frame is drawn from. + { + const c = scenePass.camera as THREE.Camera; + const p = c.getWorldPosition(new THREE.Vector3()), f = c.getWorldDirection(new THREE.Vector3()); + const r = new THREE.Vector3().crossVectors(f, c.up).normalize(); + sound.setListener([p.x, p.y, p.z], [f.x, f.y, f.z], [r.x, r.y, r.z]); + } let qy: WebGLQuery | null = null; if (ext && pending.length < 4) { qy = gl.createQuery(); From 9327b9263d30fade13eb0c484ba47802a1d962da Mon Sep 17 00:00:00 2001 From: "Yifeng \"Evan\" Wang" <7312949+doodlewind@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:59:54 +0800 Subject: [PATCH 2/2] Sound in a page: files relative to the page, as a package hosted under a path needs Co-Authored-By: Claude Opus 5.5 --- tools/three/audio.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/three/audio.ts b/tools/three/audio.ts index c0c89ec..d5e4e3e 100644 --- a/tools/three/audio.ts +++ b/tools/three/audio.ts @@ -5,7 +5,7 @@ // mix.rs `place` and `pan_gains`. The output goes through the same limiter. // // A browser plays nothing before the person has touched the page: the context resumes on the first key or click. -// Files are fetched from the page's root (`/assets/sounds/shot.ogg`), as the scene's are (engine.ts). +// Files are fetched relative to the page (`assets/sounds/shot.ogg`), so a package hosted under a path finds them. import type { HostMessage } from "@threert/gameplay"; export const DEFAULT_FALLOFF = 2; @@ -64,7 +64,7 @@ export class PageAudio { private readonly off: (() => void)[] = []; private readonly base: string; - constructor(base = "/") { + constructor(base = "") { this.base = base; const resume = () => void this.context().resume(); for (const t of ["keydown", "pointerdown"] as const) {