From 7525bfe56e895325c7e4ad2558c60a2a7c22f1ba Mon Sep 17 00:00:00 2001 From: Piggidragon Date: Thu, 5 Nov 2026 15:51:48 +0100 Subject: [PATCH 1/4] Keep abbreviations and ordinals inside a spoken sentence, cut long ones at a comma The voice reply was cut at German abbreviations such as "bzw." or "ca." and at "am 3. November". Those no longer end a phrase. A sentence that runs past about 220 characters is now cut at its last comma, semicolon or colon before falling back to a space. Co-Authored-By: Claude Sonnet 5.5 --- core/web/src/voice.ts | 32 ++++++++++++++++++++++---- docs/guide/voice.md | 6 +++++ tests/speech-display.test.mts | 43 ++++++++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/core/web/src/voice.ts b/core/web/src/voice.ts index 6daafd13e..0a6070193 100644 --- a/core/web/src/voice.ts +++ b/core/web/src/voice.ts @@ -58,8 +58,29 @@ export class StreamingSpeech { } } +// Past this many characters without a sentence end, a long sentence is cut where a reader pauses. +const LONG_SENTENCE = 220; +// A comma that early would leave a stub, so such a sentence is cut at the space instead. +const MIN_SOFT_CUT = 40; +// Titles and common German and English abbreviations; a word that only ends like one is not caught. +// A single letter covers initials and the pieces of "z. B.", "d. h.", "u. a." and "o. ä.". +const ABBREVIATION = /(?= 240 && /\s/.test(text[i]))) return i + 1; + // Avoid abbreviation, initial, decimal and ordinal breaks; the next sentence or completion flushes them. + // An ordinal still waiting for its next word is held back the same way. + const abbreviation = text[i] === '.' && (ABBREVIATION.test(text.slice(0, i + 1)) || ordinalDot(text, i) !== false); + if ((boundary && !abbreviation) || text[i] === '\n' || text[i] === '—') return i + 1; + if (i >= LONG_SENTENCE && /\s/.test(text[i])) return soft >= MIN_SOFT_CUT ? soft : i + 1; } return end; } diff --git a/docs/guide/voice.md b/docs/guide/voice.md index b8855560b..318861c1b 100644 --- a/docs/guide/voice.md +++ b/docs/guide/voice.md @@ -565,6 +565,12 @@ assistant text arrives, without waiting for the full reply. The browser buffers each spoken sentence or bounded phrase before playing it as one continuous buffer. +A reply is cut at sentence ends, not at common abbreviations such as "z. B.", +"bzw.", "ca.", "e.g." or "Dr.", and not after a day or ordinal number as in "am +3. November". A sentence that runs past about 220 characters without an end is cut +at its last comma, semicolon or colon, or at a space if it has none, so the pause +falls where a reader would pause. + This avoids interruptions within words when speech is generated slower than playback. The full text response does not need to finish. diff --git a/tests/speech-display.test.mts b/tests/speech-display.test.mts index 2f7493fd2..8c3d3bbd3 100644 --- a/tests/speech-display.test.mts +++ b/tests/speech-display.test.mts @@ -1,6 +1,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { displaySpeechText, speechChunks } from '../core/web/src/voice.js'; +import { displaySpeechText, speechChunks, StreamingSpeech } from '../core/web/src/voice.js'; import { buildTranscript } from '../core/web/src/transcript.js'; test('speech cues stay in TTS input but disappear from display including partial streamed tags', () => { const text = '(laugh) Hello. (cough) Excuse me. (clears throat) Ready. (sigh) Yes.'; @@ -20,3 +20,44 @@ test('only responses to audio-marked requests get display filtering', () => { assert.equal(items[1].audio, true); assert.equal(items[3].audio, false); assert.equal(items[1].text, '(laugh) Hi.'); }); + +/** What the TTS is sent for `text` as one reply, delivered whole or `step` characters at a time. */ +function phrases(text: string, step = 0): string[] { + const speech = new StreamingSpeech(0), spoken: string[] = []; + const reply = (part: string, done: boolean) => [{ kind: 'assistant' as const, id: 'a1', text: part, thinking: '', done }]; + if (step) for (let n = step; n < text.length; n += step) spoken.push(...speech.observe(reply(text.slice(0, n), false))); + spoken.push(...speech.observe(reply(text, true))); + return spoken; +} +test('German and English abbreviations and ordinals do not end a spoken phrase', () => { + for (const step of [0, 5]) { + const sentences = [ + 'Das kostet ca. 20 Euro, bzw. etwa so viel wie vorher.', + 'Wir brauchen z. B. Äpfel und z.B. Birnen, d. h. Obst, Nüsse u. a. Dinge.', + 'Siehe Nr. 5 und vgl. Abb. 3, ggf. evtl. inkl. Versand.', + 'Das Treffen ist am 3. November um zehn Uhr, danach gibt es Essen.', + 'Er kam am 1. dann am 2. und wieder am 3. mit dem Zug an.', + 'Mr. Smith and Dr. Meyer said so, e.g. in the first half of the year.', + ]; + assert.deepEqual(phrases(sentences.join(' '), step), sentences); + } +}); +test('words that only end like an abbreviation, and real sentence ends, still cut', () => { + const text = 'Das ist ein Gebiet. Er mag die Oma. Wir haben 3 Äpfel. Danach gehen wir heim.'; + assert.deepEqual(phrases(text), ['Das ist ein Gebiet.', 'Er mag die Oma.', 'Wir haben 3 Äpfel.', 'Danach gehen wir heim.']); + assert.deepEqual(phrases('I have 3 apples. Then I leave now. Is that really so? Yes, it is so.', 4), ['I have 3 apples.', 'Then I leave now.', 'Is that really so?', 'Yes, it is so.']); +}); +test('a long sentence is cut at its last comma before 220 characters', () => { + const head = 'Das ist ein sehr langer Satz mit vielen Wörtern, der immer weiter geht und kein Ende findet, weil er einfach nicht aufhört'; + const text = `${head}, während wir noch mehr Wörter hinzufügen um die Grenze zu überschreiten, und noch ein paar mehr Wörter am Ende ohne Punkt`; + const cut = text.indexOf('überschreiten,') + 'überschreiten,'.length; + assert.ok(cut < 220 && text.indexOf(' ', 220) > cut); + for (const step of [0, 9]) assert.deepEqual(phrases(text, step), [text.slice(0, cut), text.slice(cut + 1)]); + // Without any comma or colon it falls back to the first space past the limit, as before. + const plain = 'wort '.repeat(60).trim(); + const space = plain.indexOf(' ', 220); + assert.deepEqual(phrases(plain), [plain.slice(0, space), plain.slice(space + 1)]); + // A semicolon and a colon pause like a comma; a colon inside a time does not. + const colon = `${'wort '.repeat(30)}um 12:30 Uhr sagte er: ${'noch '.repeat(30)}`.trim(); + assert.equal(phrases(colon)[0], `${'wort '.repeat(30)}um 12:30 Uhr sagte er:`); +}); From b032587bdd06eecd7b799dc3f96f35872d5c12eb Mon Sep 17 00:00:00 2001 From: Piggidragon Date: Thu, 5 Nov 2026 15:55:23 +0100 Subject: [PATCH 2/4] Fix what the first review of the speech cuts found Co-Authored-By: Claude Sonnet 5.5 --- core/web/src/voice.ts | 13 +++++++++---- docs/guide/voice.md | 13 +++++++------ tests/speech-display.test.mts | 5 +++++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/core/web/src/voice.ts b/core/web/src/voice.ts index 0a6070193..e9fabf4f8 100644 --- a/core/web/src/voice.ts +++ b/core/web/src/voice.ts @@ -64,15 +64,20 @@ const LONG_SENTENCE = 220; const MIN_SOFT_CUT = 40; // Titles and common German and English abbreviations; a word that only ends like one is not caught. // A single letter covers initials and the pieces of "z. B.", "d. h.", "u. a." and "o. ä.". -const ABBREVIATION = /(? { const text = 'Das ist ein Gebiet. Er mag die Oma. Wir haben 3 Äpfel. Danach gehen wir heim.'; assert.deepEqual(phrases(text), ['Das ist ein Gebiet.', 'Er mag die Oma.', 'Wir haben 3 Äpfel.', 'Danach gehen wir heim.']); + // Words that are real words or units are no abbreviation; a number before a capital word outside an ordinal slot ends the sentence. + const ends = ['Die Latenz ist 20 ms.', 'Er aß eine reife fig.', 'Ich bin heute genau 25.', 'Morgen gehen wir heim.']; + assert.deepEqual(phrases(ends.join(' ')), ends); + assert.deepEqual(phrases(ends.join(' '), 5), ends); assert.deepEqual(phrases('I have 3 apples. Then I leave now. Is that really so? Yes, it is so.', 4), ['I have 3 apples.', 'Then I leave now.', 'Is that really so?', 'Yes, it is so.']); }); test('a long sentence is cut at its last comma before 220 characters', () => { From 6593570a207d62abe4642f648d725f75aad253ec Mon Sep 17 00:00:00 2001 From: Piggidragon Date: Thu, 5 Nov 2026 16:36:42 +0100 Subject: [PATCH 3/4] Play speech at the rate the speech server speaks at, not at 24 kHz alone A WAV is read at the rate its header gives (8 to 192 kHz, mono or stereo, 16-bit integer or 32-bit float, also the extensible form) with a checked parse. Raw PCM takes its rate from the server's x-sample-rate header or the rate= parameter of its content type, else from a new setting of Voice servers, Sample rate of raw audio (24000 by default). Each phrase carries its rate to the browser, which makes the buffers, the replay and the time stretch at that rate. A provider that says nothing about its rate is still taken to be at 24 kHz. Also write the speech cuts' regexes without a lookbehind, which Safari before 16.4 cannot read. Co-Authored-By: Claude Sonnet 5.5 --- core/server/src/api/voice.ts | 59 ++++++------ core/server/src/wav.ts | 87 ++++++++++++++++++ core/server/test/voice-providers.test.mjs | 29 +++++- core/web/src/components/VoiceControl.tsx | 28 +++--- core/web/src/pcm-stream.ts | 31 +++++-- core/web/src/samples.ts | 3 + core/web/src/time-stretch.ts | 92 +++++++++++-------- core/web/src/voice.ts | 7 +- docs/guide/extension-ui.md | 18 ++-- docs/guide/upgrading.md | 2 +- docs/guide/voice-servers.md | 31 ++++--- docs/guide/voice.md | 25 ++++- docs/reference/api.md | 6 +- extensions/voice-servers/src/de.ts | 4 +- .../voice-servers/src/server/provider.ts | 19 ++-- .../voice-servers/src/server/requests.ts | 2 +- extensions/voice-servers/src/server/store.ts | 20 +++- .../voice-servers/src/web/VoiceServers.tsx | 5 +- extensions/voice-servers/src/web/api.ts | 2 + .../test/browser/voice-servers.spec.mts | 11 ++- .../voice-servers/test/provider.test.mts | 74 +++++++++++++-- extensions/voice-servers/test/routes.test.mts | 14 +++ packages/extension-api/server.d.ts | 9 +- tests/browser/voice.spec.mts | 46 ++++++++++ tests/fixtures/extensions/speaker/server.js | 73 +++++++++++---- tests/pcm-stream.test.mts | 69 +++++++++++++- tests/time-stretch.test.mts | 42 +++++++++ tests/voice.test.mts | 84 +++++++++++++++-- 28 files changed, 713 insertions(+), 179 deletions(-) create mode 100644 core/server/src/wav.ts diff --git a/core/server/src/api/voice.ts b/core/server/src/api/voice.ts index ec259cc90..5769b2007 100644 --- a/core/server/src/api/voice.ts +++ b/core/server/src/api/voice.ts @@ -6,6 +6,7 @@ import { getSession, getSetting, putSetting, setVoiceSpeaks } from "../db.js"; import { agentFor } from "../project-agent.js"; import { setVoiceChooser, voiceProviders, within } from "../extension-host.js"; import type { ProvidedVoice, VoiceProvider } from "@pithagoras/extension-api/server"; +import { DEFAULT_SAMPLE_RATE, MAX_SAMPLE_RATE, MIN_SAMPLE_RATE, playableRate, wavAudio } from "../wav.js"; const DEFAULT_VAD = { positiveSpeechThreshold: 0.65, negativeSpeechThreshold: 0.35, minSpeechMs: 256, preSpeechPadMs: 320, redemptionMs: 1000 }; export interface VoiceConfig { @@ -83,31 +84,23 @@ function withInstructions wav.length || wav.readUInt16LE(at + 8) !== 1 || wav.readUInt16LE(at + 10) !== 1 || wav.readUInt32LE(at + 12) !== 24000 || wav.readUInt16LE(at + 22) !== 16) - throw new Error(`Expected mono 24 kHz 16-bit audio from ${label}`); - format = true; - } - if (id === "data") { - // Samples are only meaningful once the fmt chunk has described them. - if (!format) throw new Error(`Expected mono 24 kHz 16-bit audio from ${label}`); - // A writer that does not know the length up front leaves a placeholder - // size behind. The response is fully buffered, so its end is the truth. - const end = size && at + 8 + size <= wav.length ? at + 8 + size : wav.length; - const pcm = wav.subarray(at + 8, end); - if (!pcm.length || pcm.length % 2) throw new Error(`${label} returned invalid PCM audio`); - return pcm; - } - at += 8 + size + (size % 2); - } - throw new Error(`${label} returned no audio data`); +/** The largest WAV answer read whole: minutes of speech at the highest rate, far more than a phrase of 600 characters. */ +const MAX_WAV_BYTES = 64 * 1024 * 1024; +/** + * The rate of raw PCM, which has no header of its own: what the source says in an `x-sample-rate` header, else in a + * `rate=` parameter of the content type, else 24 kHz, as a source that says nothing has always been played. Throws + * where it is not a rate voice mode plays, or the PCM is not mono. + */ +function pcmRate(headers: Headers, label: string): number { + const type = headers.get("content-type") ?? ""; + const param = (name: string) => new RegExp(`;\\s*${name}\\s*=\\s*"?([^";\\s]*)`, "i").exec(type)?.[1]; + const channels = param("channels"); + if (channels !== undefined && channels !== "1") throw new Error(`${label} answered audio with ${channels} channels: voice mode plays mono`); + const said = headers.get("x-sample-rate") ?? param("rate"); + if (said === null || said === undefined) return DEFAULT_SAMPLE_RATE; + const rate = /^\d{1,7}$/.test(said.trim()) ? Number(said) : NaN; + if (!playableRate(rate)) throw new Error(`${label} answered audio at ${said} Hz: voice mode plays ${MIN_SAMPLE_RATE} to ${MAX_SAMPLE_RATE} Hz`); + return rate; } /** Voice mode listens and speaks through an extension that is not running, or offers nothing now: said, rather than anything else used. */ class SourceAbsent extends Error { @@ -325,18 +318,20 @@ export function voiceRouter(): Router { answer = given; if (!answer.ok) throw new Error(`${label} answered HTTP ${answer.status}`); const type = answer.headers.get("content-type") ?? ""; - const headers = { "Content-Type": "audio/pcm", "X-Sample-Rate": "24000", "X-Sample-Format": "s16le", "Cache-Control": "no-store", "X-Accel-Buffering": "no" }; - // A WAV is read whole and checked, and its samples sent on. + const headers = { "Content-Type": "audio/pcm", "X-Sample-Format": "s16le", "Cache-Control": "no-store", "X-Accel-Buffering": "no" }; + // A WAV is read whole and checked, and its samples sent on at the rate its header gives. if (/^audio\/(wav|x-wav|wave|vnd\.wave)\b/.test(type)) { - const pcm = wavPcm(Buffer.from(await answer.arrayBuffer()), label); - return res.set({ ...headers, "Server-Timing": `tts_headers;dur=${(performance.now() - started).toFixed(1)}` }).send(pcm); + if (Number(answer.headers.get("content-length")) > MAX_WAV_BYTES) throw new Error(`${label} answered more than ${MAX_WAV_BYTES >> 20} MB of audio`); + const wav = Buffer.from(await answer.arrayBuffer()); + if (wav.length > MAX_WAV_BYTES) throw new Error(`${label} answered more than ${MAX_WAV_BYTES >> 20} MB of audio`); + const { rate, pcm } = wavAudio(wav, label); + return res.set({ ...headers, "X-Sample-Rate": String(rate), "Server-Timing": `tts_headers;dur=${(performance.now() - started).toFixed(1)}` }).send(pcm); } if (!/^audio\/pcm\b/.test(type)) throw new Error(`${label} answered ${type || "no type"}: voice mode plays audio/pcm or audio/wav`); // PCM is passed on as it comes, checked before anything of it is sent. - const rate = answer.headers.get("x-sample-rate"); - if (rate !== null && rate !== "24000") throw new Error(`${label} answered audio at ${rate} Hz: voice mode plays 24 kHz`); + const rate = pcmRate(answer.headers, label); if (!answer.body) throw new Error(`${label} gave no audio`); - res.set({ ...headers, "Server-Timing": `tts_headers;dur=${(performance.now() - started).toFixed(1)}` }); + res.set({ ...headers, "X-Sample-Rate": String(rate), "Server-Timing": `tts_headers;dur=${(performance.now() - started).toFixed(1)}` }); res.flushHeaders(); const reader = answer.body.getReader(); let bytes = 0; diff --git a/core/server/src/wav.ts b/core/server/src/wav.ts new file mode 100644 index 000000000..211588392 --- /dev/null +++ b/core/server/src/wav.ts @@ -0,0 +1,87 @@ +/** The sample rates voice mode plays, in Hz: what a speech server may answer at. */ +export const MIN_SAMPLE_RATE = 8000; +export const MAX_SAMPLE_RATE = 192000; +/** The rate of speech that does not say its own: raw audio from a source that names none. */ +export const DEFAULT_SAMPLE_RATE = 24000; + +/** WAVE_FORMAT_* codes of the fmt chunk. */ +const FORMAT_PCM = 1, FORMAT_FLOAT = 3, FORMAT_EXTENSIBLE = 0xfffe; + +/** Mono 16-bit little-endian samples and the rate they are played at. */ +export interface Audio { rate: number; pcm: Buffer } + +/** Whether a rate is one voice mode plays: whole hertz from 8 to 192 kHz. */ +export const playableRate = (rate: number) => Number.isInteger(rate) && rate >= MIN_SAMPLE_RATE && rate <= MAX_SAMPLE_RATE; + +/** What a fmt chunk says, as words for an error: "stereo 24-bit integer audio at 44100 Hz". */ +function described(tag: number, channels: number, bits: number, rate: number): string { + const kind = tag === FORMAT_PCM ? "integer" : tag === FORMAT_FLOAT ? "float" : `format ${tag}`; + const layout = channels === 1 ? "mono" : channels === 2 ? "stereo" : `${channels}-channel`; + return `${layout} ${bits}-bit ${kind} audio at ${rate} Hz`; +} + +/** One sample of a frame as a float in [-1, 1], from the bytes at `at`. */ +type Reader = (wav: Buffer, at: number) => number; +const int16: Reader = (wav, at) => wav.readInt16LE(at) / 32768; +const float32: Reader = (wav, at) => wav.readFloatLE(at); + +/** + * The samples of a RIFF/WAVE buffer as mono 16-bit little-endian PCM, with the rate the header gives. `label` is what + * answered with it. Read: 16-bit integer and 32-bit float samples, mono or stereo (mixed to mono), also as + * WAVE_FORMAT_EXTENSIBLE, at 8 to 192 kHz. Anything else is refused, with what it was in the message. + */ +export function wavAudio(wav: Buffer, label: string): Audio { + if (wav.length < 12 || wav.toString("ascii", 0, 4) !== "RIFF" || wav.toString("ascii", 8, 12) !== "WAVE") + throw new Error(`${label} returned invalid WAV audio`); + let format: { tag: number; channels: number; bits: number; rate: number; frame: number } | undefined; + for (let at = 12; at + 8 <= wav.length;) { + const id = wav.toString("ascii", at, at + 4), size = wav.readUInt32LE(at + 4), body = at + 8; + if (id === "fmt ") { + if (size < 16 || body + size > wav.length) throw new Error(`${label} returned a WAV file with a broken fmt chunk`); + let tag = wav.readUInt16LE(body); + const channels = wav.readUInt16LE(body + 2), rate = wav.readUInt32LE(body + 4), frame = wav.readUInt16LE(body + 12), bits = wav.readUInt16LE(body + 14); + // The extensible form keeps the format in the first two bytes of its sub-format GUID. + if (tag === FORMAT_EXTENSIBLE) { + if (size < 40) throw new Error(`${label} returned a WAV file with a broken fmt chunk`); + tag = wav.readUInt16LE(body + 24); + } + const got = described(tag, channels, bits, rate); + const readable = (tag === FORMAT_PCM && bits === 16) || (tag === FORMAT_FLOAT && bits === 32); + if (!readable || channels < 1 || channels > 2) + throw new Error(`Expected mono or stereo 16-bit integer or 32-bit float audio from ${label}, got ${got}`); + if (!playableRate(rate)) throw new Error(`Expected audio from ${MIN_SAMPLE_RATE} to ${MAX_SAMPLE_RATE} Hz from ${label}, got ${got}`); + // A frame is one sample of each channel, and its size is what the data is cut into. + if (frame !== channels * bits / 8) throw new Error(`${label} returned a WAV file whose fmt chunk does not agree with itself (${got}, ${frame} bytes a frame)`); + format = { tag, channels, bits, rate, frame }; + } + if (id === "data") { + // Samples are only meaningful once the fmt chunk has described them. + if (!format) throw new Error(`${label} returned a WAV file whose samples come before its fmt chunk`); + // A writer that does not know the length up front leaves a placeholder + // size behind. The response is fully buffered, so its end is the truth. + const end = size && body + size <= wav.length ? body + size : wav.length; + const data = wav.subarray(body, end); + if (!data.length || data.length % format.frame) throw new Error(`${label} returned invalid PCM audio`); + return { rate: format.rate, pcm: mono(data, format) }; + } + at = body + size + (size % 2); + } + throw new Error(`${label} returned no audio data`); +} + +/** `data`, frames of `format`, as mono 16-bit little-endian samples: the same bytes where they already are. */ +function mono(data: Buffer, format: { tag: number; channels: number; bits: number; frame: number }): Buffer { + if (format.tag === FORMAT_PCM && format.channels === 1) return data; + const read = format.tag === FORMAT_FLOAT ? float32 : int16; + const step = format.frame / format.channels, frames = data.length / format.frame; + const out = Buffer.alloc(frames * 2); + for (let i = 0; i < frames; i++) { + let sum = 0; + for (let c = 0; c < format.channels; c++) sum += read(data, i * format.frame + c * step); + const value = sum / format.channels; + // A float outside [-1, 1] clips, and one that is not a number is silence. + const clipped = Number.isNaN(value) ? 0 : Math.max(-1, Math.min(1, value)); + out.writeInt16LE(Math.round(clipped < 0 ? clipped * 32768 : clipped * 32767), i * 2); + } + return out; +} diff --git a/core/server/test/voice-providers.test.mjs b/core/server/test/voice-providers.test.mjs index 6dcf7d9b3..7153320fe 100644 --- a/core/server/test/voice-providers.test.mjs +++ b/core/server/test/voice-providers.test.mjs @@ -158,10 +158,35 @@ test("speech is passed on as 24 kHz PCM, as the source streams it or from the WA assert.equal((await asked()).cancelled, 0, "nothing played is cancelled"); }); +test("speech at another rate than 24 kHz is played at its own: a WAV says it in its header, PCM in a header, in its content type, or not at all", async () => { + const played = async (answer, rate, samples) => { + const got = await speak(); + assert.equal(got.status, 200, answer); + assert.deepEqual([got.headers.get("content-type"), got.headers.get("x-sample-rate"), got.headers.get("x-sample-format")], ["audio/pcm", String(rate), "s16le"], answer); + assert.deepEqual([...new Uint8Array(await got.arrayBuffer())], samples, answer); + }; + for (const [answer, rate, samples] of [ + // The WAVs: mono 16-bit as it is; stereo float mixed to mono (half of full scale, then silence). + ["wav44k", 44100, [0, 0, 255, 127]], + ["wav22k", 22050, [0, 0, 255, 127]], + ["wav48kStereoFloat", 48000, [0, 64, 0, 0]], + // The raw PCM: a header, a content type, and nothing, which is 24 kHz as it always was. + ["pcm44k", 44100, [0, 0, 255, 127]], + ["pcm48k", 48000, [0, 0, 255, 127]], + ["pcmType", 44100, [0, 0, 255, 127]], + ["pcmBare", 24000, [0, 0, 255, 127]], + ]) { + await set({ answer }); + await played(answer, rate, samples); + } + await set({ answer: "pcm" }); +}); + test("what voice mode cannot play is a 502 that names the source, and what it answered is let go of", async () => { for (const [answer, error, cancels] of [ - ["pcm48k", "Speaker answered audio at 48000 Hz: voice mode plays 24 kHz", 1], - ["wav22k", "Expected mono 24 kHz 16-bit audio from Speaker", 1], + ["pcm300k", "Speaker answered audio at 300000 Hz: voice mode plays 8000 to 192000 Hz", 1], + ["pcmStereo", "Speaker answered audio with 2 channels: voice mode plays mono", 1], + ["wav24bit", "Expected mono or stereo 16-bit integer or 32-bit float audio from Speaker, got mono 24-bit integer audio at 44100 Hz", 1], ["text", "Speaker answered text/plain: voice mode plays audio/pcm or audio/wav", 1], ["notOk", "Speaker answered HTTP 500", 1], ["notResponse", "Speaker gave no audio", 0], diff --git a/core/web/src/components/VoiceControl.tsx b/core/web/src/components/VoiceControl.tsx index a5aebb0de..ac370b6df 100644 --- a/core/web/src/components/VoiceControl.tsx +++ b/core/web/src/components/VoiceControl.tsx @@ -13,9 +13,9 @@ import { api, json, type PortalEvent, type PromptOptions } from "../api"; import { micError } from "../mic-error"; import type { Item } from "../transcript"; import { LiveTranscription } from "../live-transcription"; -import { preparePcmSpeech, readPcmStream, playAudioBuffer, bufferOf } from "../pcm-stream"; +import { preparePcmSpeech, readPcmStream, playAudioBuffer, bufferOf, sampleRateOf } from "../pcm-stream"; import { stretchInSteps } from "../time-stretch"; -import { joinSamples } from "../samples"; +import { DEFAULT_SAMPLE_RATE, joinSamples } from "../samples"; import { asksToRepeat, couldAskToRepeat } from "../voice-commands"; import { isImage, pending, type Attachment } from "../attachments"; import { VOICE_RATES } from "./VoiceSettings"; @@ -186,7 +186,7 @@ export function VoiceControl({ sessionId, folder, items, running, work, onSend, }; const maxTurn = useRef>(); // The spoken error line, asked for as voice mode started, in the chat's own voice: its audio once it is there, or nothing. - const errorLine = useRef<{ ready: Promise } | null>(null); + const errorLine = useRef<{ ready: Promise<{ bytes: ArrayBuffer; sampleRate: number } | undefined> } | null>(null); useEffect(() => { if (!error || error !== fadingError || enabled || starting) return; @@ -377,7 +377,7 @@ export function VoiceControl({ sessionId, folder, items, running, work, onSend, const kept = phrase.stretched; // One cut short by an earlier Repeat is worked out again. if (kept?.rate === rate && (kept.ready || !kept.signal.aborted)) return kept.samples; - const stretched: NonNullable = { rate, signal: controller.signal, samples: stretchInSteps(joinSamples(phrase.samples), rate, controller.signal) }; + const stretched: NonNullable = { rate, signal: controller.signal, samples: stretchInSteps(joinSamples(phrase.samples), rate, controller.signal, phrase.sampleRate) }; stretched.samples.then(() => { stretched.ready = true; }, () => {}); phrase.stretched = stretched; return stretched.samples; @@ -401,14 +401,16 @@ export function VoiceControl({ sessionId, folder, items, running, work, onSend, */ const prepareErrorLine = (version: number) => { const current = () => mounted.current && epoch.current === version; - const fetchLine = async (): Promise => { + const fetchLine = async () => { for (let attempt = 0; ; attempt++) { try { const response = await fetch(`/api/sessions/${sessionId}/voice/speech`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text: t("An error occurred.") }), }); - if (response.ok && response.headers.get("content-type")?.startsWith("audio/pcm")) return await response.arrayBuffer(); + const sampleRate = sampleRateOf(response.headers); + // Kept with the rate it was spoken at, as it is played from the bytes alone. + if (response.ok && response.headers.get("content-type")?.startsWith("audio/pcm") && sampleRate) return { bytes: await response.arrayBuffer(), sampleRate }; } catch {} if (attempt >= ERROR_LINE_WAITS_MS.length || !current()) return undefined; await new Promise(resolve => setTimeout(resolve, ERROR_LINE_WAITS_MS[attempt])); @@ -427,11 +429,11 @@ export function VoiceControl({ sessionId, folder, items, running, work, onSend, // It may still be on its way, as the speech model is often just coming up when voice mode begins. const line = errorLine.current; let timer: ReturnType | undefined; - const bytes = line && await Promise.race([line.ready, new Promise(resolve => { timer = setTimeout(() => resolve(undefined), ERROR_LINE_WAIT_MS); })]); + const kept = line && await Promise.race([line.ready, new Promise(resolve => { timer = setTimeout(() => resolve(undefined), ERROR_LINE_WAIT_MS); })]); clearTimeout(timer); signal.throwIfAborted(); - if (!bytes) throw new Error(t("The spoken error line is not available, so the error is shown in writing only. The speech model was not ready when voice mode started.")); - response = new Response(bytes.slice(0), { headers: { "content-type": "audio/pcm", "x-sample-rate": "24000" } }); + if (!kept) throw new Error(t("The spoken error line is not available, so the error is shown in writing only. The speech model was not ready when voice mode started.")); + response = new Response(kept.bytes.slice(0), { headers: { "content-type": "audio/pcm", "x-sample-rate": String(kept.sampleRate) } }); } else { response = await fetch(`/api/sessions/${sessionId}/voice/speech`, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -448,9 +450,11 @@ export function VoiceControl({ sessionId, folder, items, running, work, onSend, let buffer: AudioBuffer | undefined; let stream: Awaited> | undefined; // Kept for Repeat once it is played; a phrase prepared and then cancelled was never heard. - const phrase = { samples: [] as Float32Array[], sampleRate: 24000 }; - const options = { rate: speed.current, ...(kind === 'reply' ? { record: (samples: Float32Array) => { phrase.samples.push(samples); } } : {}) }; - if (!response.headers.get("content-type")?.startsWith("audio/pcm") || response.headers.get("x-sample-rate") !== "24000" || !body) throw new Error(t("Unsupported speech stream")); + // Made at the rate the speech server speaks at: the audio context resamples it to its output. + const sampleRate = sampleRateOf(response.headers); + const phrase = { samples: [] as Float32Array[], sampleRate: sampleRate ?? DEFAULT_SAMPLE_RATE }; + const options = { rate: speed.current, sampleRate: phrase.sampleRate, ...(kind === 'reply' ? { record: (samples: Float32Array) => { phrase.samples.push(samples); } } : {}) }; + if (!response.headers.get("content-type")?.startsWith("audio/pcm") || !sampleRate || !body) throw new Error(t("Unsupported speech stream")); // Played as it arrives, unless the pipeline speaks one phrase after another. if (!sequential.current) stream = await preparePcmSpeech(body, audio, signal, options); else buffer = await readPcmStream(body, audio, signal, options); diff --git a/core/web/src/pcm-stream.ts b/core/web/src/pcm-stream.ts index ef545348c..ede67307f 100644 --- a/core/web/src/pcm-stream.ts +++ b/core/web/src/pcm-stream.ts @@ -1,14 +1,25 @@ import { t } from "./i18n"; -import { joinSamples } from "./samples"; +import { DEFAULT_SAMPLE_RATE, joinSamples } from "./samples"; import { TimeStretch } from "./time-stretch"; +/** The sample rates the portal passes on, in Hz (core/server/src/wav.ts): the audio context resamples to its output. */ +const MIN_SAMPLE_RATE = 8000, MAX_SAMPLE_RATE = 192000; + +/** The sample rate a phrase's answer says (`x-sample-rate`), or none where it says none a phrase can be played at. */ +export function sampleRateOf(headers: Headers): number | undefined { + const said = headers.get("x-sample-rate"); + const rate = said !== null && /^\d{1,7}$/.test(said) ? Number(said) : NaN; + return rate >= MIN_SAMPLE_RATE && rate <= MAX_SAMPLE_RATE ? rate : undefined; +} + /** * How a phrase is played: `rate` makes it faster without raising the voice - * (see time-stretch.ts), and `record` is given the phrase's samples as they + * (see time-stretch.ts), `sampleRate` is the rate its samples were made at + * (24 kHz where left out), and `record` is given the phrase's samples as they * were spoken, before any of that, so a reply can be played again later at * whatever speed is chosen then. */ -export interface SpeechOptions { rate?: number; record?: (samples: Float32Array) => void } +export interface SpeechOptions { rate?: number; sampleRate?: number; record?: (samples: Float32Array) => void } const samplesOf = (bytes: Uint8Array, count: number) => { const samples = new Float32Array(count), view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); @@ -40,8 +51,8 @@ function pcmDecoder() { /** Whatever engine made the audio: nothing here knows which one it was. */ const incompleteAudio = () => new Error(t("Voice mode's source returned incomplete audio")); -/** A buffer holding `samples`, or none for none: a zero-length AudioBuffer is an error. */ -export function bufferOf(audio: BaseAudioContext, samples: Float32Array, sampleRate = 24000): AudioBuffer | undefined { +/** A buffer holding `samples` made at `sampleRate`, or none for none: a zero-length AudioBuffer is an error. */ +export function bufferOf(audio: BaseAudioContext, samples: Float32Array, sampleRate = DEFAULT_SAMPLE_RATE): AudioBuffer | undefined { if (!samples.length) return undefined; const buffer = audio.createBuffer(1, samples.length, sampleRate); buffer.getChannelData(0).set(samples); @@ -56,7 +67,7 @@ export async function readPcmStream( const reader = body.getReader(); // The stretch is worked on each chunk as it arrives, so the whole buffer // does not have to wait for one long stretch pass over everything. - const stretcher = new TimeStretch(options.rate ?? 1); + const stretcher = new TimeStretch(options.rate ?? 1, options.sampleRate); const chunks: Uint8Array[] = []; const stretched: Float32Array[] = []; const decoder = pcmDecoder(); @@ -82,7 +93,7 @@ export async function readPcmStream( let offset = 0; for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.length; } options.record?.(samplesOf(bytes, length / 2)); - const buffer = bufferOf(audio, joinSamples(stretched)); + const buffer = bufferOf(audio, joinSamples(stretched), options.sampleRate); if (!buffer) throw incompleteAudio(); signal.throwIfAborted(); return buffer; @@ -104,7 +115,7 @@ export async function playAudioBuffer(buffer: AudioBuffer, audio: AudioContext, /** Start from a small PCM cushion while the producer continues generating. */ export async function preparePcmSpeech(body: ReadableStream, audio: AudioContext, signal: AbortSignal, options: SpeechOptions = {}) { const reader = body.getReader(); - const stretcher = new TimeStretch(options.rate ?? 1); + const stretcher = new TimeStretch(options.rate ?? 1, options.sampleRate); let spoken = 0; const pending: AudioBuffer[] = []; const sources = new Set(); @@ -142,14 +153,14 @@ export async function preparePcmSpeech(body: ReadableStream, audio: const samples = decoder.push(value); if (!samples) continue; spoken += samples.length; options.record?.(samples); - const buffer = bufferOf(audio, stretcher.push(samples)); + const buffer = bufferOf(audio, stretcher.push(samples), options.sampleRate); if (!buffer) continue; pending.push(buffer); buffered += buffer.duration; if (buffered >= 0.65) ready(); pump(); } if (!spoken || decoder.cut) throw incompleteAudio(); - const rest = bufferOf(audio, stretcher.flush()); + const rest = bufferOf(audio, stretcher.flush(), options.sampleRate); if (rest) { pending.push(rest); buffered += rest.duration; } finished = true; ready(); pump(); } catch (error) { rejectReady(error); failPlay?.(error); throw error; } diff --git a/core/web/src/samples.ts b/core/web/src/samples.ts index 915d8a8ad..3b0024283 100644 --- a/core/web/src/samples.ts +++ b/core/web/src/samples.ts @@ -1,3 +1,6 @@ +/** The rate speech is made at where nothing says another: voice mode played 24 kHz alone before it was told its rate. */ +export const DEFAULT_SAMPLE_RATE = 24000; + /** Pieces of audio laid end to end, as one new array. */ export function joinSamples(pieces: Float32Array[]): Float32Array { const joined = new Float32Array(pieces.reduce((n, p) => n + p.length, 0)); diff --git a/core/web/src/time-stretch.ts b/core/web/src/time-stretch.ts index 3d63e545a..500e8139b 100644 --- a/core/web/src/time-stretch.ts +++ b/core/web/src/time-stretch.ts @@ -6,34 +6,42 @@ * WSOLA instead: the output is built from overlapping, windowed pieces of the * input, taken further apart than they are laid down, and each piece is * nudged to where it lines up best with the one before it so that the - * waveform stays continuous. The pieces are short (40 ms at 24 kHz), which is - * what keeps a voice's pitch; the nudge is what keeps it from buzzing. + * waveform stays continuous. The pieces are short (40 ms, whatever the sample + * rate), which is what keeps a voice's pitch; the nudge is what keeps it from + * buzzing. * * It streams: samples go in as they arrive from the speech service and * finished output comes out behind them, so a streamed reply starts about as * early as it did. `flush` gives what is left at the end. */ -import { joinSamples } from "./samples"; +import { DEFAULT_SAMPLE_RATE, joinSamples } from "./samples"; -const FRAME = 960; -const HOP = FRAME / 2; -/** How far a piece may move to line up: 10 ms either way. */ -const TOLERANCE = 240; -/** Samples compared when lining up, every other one: enough to find the period, cheap enough for a phone. */ -const COMPARE = HOP; +/** The sizes of the pieces, in samples, for one sample rate: 40 ms a piece, laid down every 20, lined up within 10 either way. */ +function sizes(sampleRate: number) { + const hop = Math.round(sampleRate * 0.02); + // Half a frame is what a window is overlapped by, so the frame is two of them. + return { frame: hop * 2, hop, tolerance: Math.round(sampleRate * 0.01) }; +} /** A periodic Hann window: two of them half a frame apart add up to exactly one. */ -const WINDOW = (() => { - const w = new Float32Array(FRAME); - for (let i = 0; i < FRAME; i++) w[i] = 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / FRAME); +const windows = new Map(); +function windowOf(frame: number): Float32Array { + let w = windows.get(frame); + if (!w) { + w = new Float32Array(frame); + for (let i = 0; i < frame; i++) w[i] = 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / frame); + windows.set(frame, w); + } return w; -})(); +} export class TimeStretch { + private readonly size: { frame: number; hop: number; tolerance: number }; + private readonly window: Float32Array; /** Input so far, after half a frame of silence (so the first piece does not fade in). */ - private input = new Float32Array(HOP); - private length = HOP; + private input: Float32Array; + private length: number; /** Real samples taken in, and output given out, for the length the whole comes to. */ private taken = 0; private given = 0; @@ -42,12 +50,20 @@ export class TimeStretch { /** Where the last piece was taken from, in `input`. */ private previous = 0; /** The half-finished half-frame at the end of the output, still to be overlapped. */ - private tail = new Float32Array(HOP); + private tail: Float32Array; /** The first half-frame of output is the silence put in front: it is not given out. */ - private skip = HOP; + private skip: number; - constructor(readonly rate: number) { + /** `rate` times as fast, for samples made at `sampleRate` Hz. */ + constructor(readonly rate: number, sampleRate = DEFAULT_SAMPLE_RATE) { if (!(rate > 0)) throw new Error("The rate must be above zero"); + if (!(sampleRate > 0)) throw new Error("The sample rate must be above zero"); + this.size = sizes(sampleRate); + this.window = windowOf(this.size.frame); + this.input = new Float32Array(this.size.hop); + this.length = this.size.hop; + this.tail = new Float32Array(this.size.hop); + this.skip = this.size.hop; } private append(samples: Float32Array) { @@ -63,12 +79,13 @@ export class TimeStretch { /** Where the next piece lines up best with what the last one would have gone on to. */ private align(ideal: number, limit: number): number { if (this.frame === 0) return 0; - const natural = this.previous + HOP; + const { hop, tolerance } = this.size; + const natural = this.previous + hop; let best = ideal, score = -Infinity; - const from = Math.max(0, ideal - TOLERANCE), to = Math.min(limit, ideal + TOLERANCE); + const from = Math.max(0, ideal - tolerance), to = Math.min(limit, ideal + tolerance); for (let at = from; at <= to; at++) { let sum = 0; - for (let i = 0; i < COMPARE; i += 2) sum += this.input[natural + i] * this.input[at + i]; + for (let i = 0; i < hop; i += 2) sum += this.input[natural + i] * this.input[at + i]; if (sum > score) { score = sum; best = at; } } return best; @@ -77,16 +94,17 @@ export class TimeStretch { /** Lay down every piece the input so far is enough for. */ private run(end: boolean): Float32Array { const out: Float32Array[] = []; + const { frame, hop, tolerance } = this.size; while (true) { - const ideal = Math.round(this.frame * HOP * this.rate); + const ideal = Math.round(this.frame * hop * this.rate); // Past the end of what there is to read, with the silence at the end counted. - if (end ? ideal >= this.length : ideal + TOLERANCE + FRAME > this.length || this.previous + HOP + FRAME > this.length) break; - const at = this.align(ideal, Math.max(0, Math.min(this.length - FRAME, ideal + TOLERANCE))); - const piece = new Float32Array(FRAME); - for (let i = 0; i < FRAME; i++) piece[i] = (this.input[at + i] ?? 0) * WINDOW[i]; - const done = new Float32Array(HOP); - for (let i = 0; i < HOP; i++) done[i] = this.tail[i] + piece[i]; - this.tail = piece.slice(HOP); + if (end ? ideal >= this.length : ideal + tolerance + frame > this.length || this.previous + hop + frame > this.length) break; + const at = this.align(ideal, Math.max(0, Math.min(this.length - frame, ideal + tolerance))); + const piece = new Float32Array(frame); + for (let i = 0; i < frame; i++) piece[i] = (this.input[at + i] ?? 0) * this.window[i]; + const done = new Float32Array(hop); + for (let i = 0; i < hop; i++) done[i] = this.tail[i] + piece[i]; + this.tail = piece.slice(hop); this.previous = at; this.frame++; out.push(done); @@ -116,28 +134,28 @@ export class TimeStretch { flush(): Float32Array { if (this.rate === 1) return new Float32Array(0); // Silence after the end, so the last pieces have something to overlap with. - this.append(new Float32Array(FRAME + TOLERANCE + HOP)); + this.append(new Float32Array(this.size.frame + this.size.tolerance + this.size.hop)); return this.run(true); } } -/** One whole recording, made `rate` times as fast. */ -export function stretch(samples: Float32Array, rate: number): Float32Array { +/** One whole recording made at `sampleRate` Hz, made `rate` times as fast. */ +export function stretch(samples: Float32Array, rate: number, sampleRate = DEFAULT_SAMPLE_RATE): Float32Array { if (rate === 1) return samples; - const s = new TimeStretch(rate); + const s = new TimeStretch(rate, sampleRate); return joinSamples([s.push(samples), s.flush()]); } -/** Half a second at 24 kHz: a few milliseconds of work at a time. */ -const STEP = 12000; +/** Half a second of samples: a few milliseconds of work at a time. */ +const stepOf = (sampleRate: number) => Math.round(sampleRate / 2); /** * `stretch` a little at a time, letting the page carry on in between: a * whole reply in one go holds up the orb and the microphone for a moment. */ -export async function stretchInSteps(samples: Float32Array, rate: number, signal: AbortSignal): Promise { +export async function stretchInSteps(samples: Float32Array, rate: number, signal: AbortSignal, sampleRate = DEFAULT_SAMPLE_RATE): Promise { if (rate === 1) return samples; - const s = new TimeStretch(rate), out: Float32Array[] = []; + const s = new TimeStretch(rate, sampleRate), out: Float32Array[] = [], STEP = stepOf(sampleRate); for (let at = 0; at < samples.length; at += STEP) { out.push(s.push(samples.subarray(at, at + STEP))); await new Promise(resolve => setTimeout(resolve, 0)); diff --git a/core/web/src/voice.ts b/core/web/src/voice.ts index e9fabf4f8..75233395b 100644 --- a/core/web/src/voice.ts +++ b/core/web/src/voice.ts @@ -64,9 +64,10 @@ const LONG_SENTENCE = 220; const MIN_SOFT_CUT = 40; // Titles and common German and English abbreviations; a word that only ends like one is not caught. // A single letter covers initials and the pieces of "z. B.", "d. h.", "u. a." and "o. ä.". -const ABBREVIATION = /(? = { "Empty: the server detects it": "Leer: Der Server erkennt sie", "Exact words in the recording": "Genauer Wortlaut der Aufnahme", "Fixed voices, such as Kokoro": "Feste Stimmen, etwa Kokoro", + "In Hz, from 8000 to 192000. Used when the speech server streams raw audio, as Breeze does, and does not say its rate. Audio in a WAV file says its own.": "In Hz, von 8000 bis 192000. Wird verwendet, wenn der Sprachserver rohes Audio streamt, wie Breeze, und seine Rate nicht nennt. Audio in einer WAV-Datei nennt seine eigene.", "In use for voice mode": "Wird für den Sprachmodus verwendet", "Input language": "Eingabesprache", "Loading…": "Wird geladen…", @@ -44,6 +45,7 @@ export const de: Record = { "Read voices from the server": "Stimmen vom Server lesen", "Reference clone": "Referenzklon", "Reference recording": "Referenzaufnahme", + "Sample rate of raw audio": "Abtastrate von rohem Audio", "Save": "Speichern", "Save new voice": "Neue Stimme speichern", "Save voice": "Stimme speichern", @@ -55,7 +57,7 @@ export const de: Record = { "Sent as the voice service sends Chatterbox in audio.cpp: the clone's recording goes with each phrase, spoken in the input language above, which Chatterbox needs.": "Gesendet, wie der Sprachdienst Chatterbox in audio.cpp anspricht: Die Aufnahme des Klons geht mit jedem Satz mit, gesprochen in der Eingabesprache oben, die Chatterbox braucht.", "Set a speech recognition address first": "Zuerst eine Adresse für die Spracherkennung festlegen", "Speech engine": "Sprach-Engine", - "Speech recognition and speech synthesis on servers you run, spoken to as OpenAI's audio API is. The speech server has to answer 24 kHz mono WAV.": "Spracherkennung und Sprachausgabe auf Servern, die du betreibst, angesprochen wie die Audio-API von OpenAI. Der Sprachserver muss 24-kHz-Mono-WAV liefern.", + "Speech recognition and speech synthesis on servers you run, spoken to as OpenAI's audio API is. The speech server answers a WAV file, mono or stereo, at 8 to 192 kHz, or raw PCM.": "Spracherkennung und Sprachausgabe auf Servern, die du betreibst, angesprochen wie die Audio-API von OpenAI. Der Sprachserver antwortet mit einer WAV-Datei, Mono oder Stereo, mit 8 bis 192 kHz, oder mit rohem PCM.", "Speech recognition model": "Modell der Spracherkennung", "Speech recognition URL": "URL der Spracherkennung", "Speech synthesis model": "Modell der Sprachsynthese", diff --git a/extensions/voice-servers/src/server/provider.ts b/extensions/voice-servers/src/server/provider.ts index 660ff6457..25ec99029 100644 --- a/extensions/voice-servers/src/server/provider.ts +++ b/extensions/voice-servers/src/server/provider.ts @@ -2,7 +2,7 @@ import { setTimeout as delay } from "node:timers/promises"; import type { VoiceProvider } from "@pithagoras/extension-api/server"; import { offered, serverOf, speechRequest, type SpeechRequest } from "./requests"; import { storedAt, storeOnServer } from "./server-store"; -import { settings } from "./store"; +import { RAW_RATE_DEFAULT, settings } from "./store"; import { findVoice } from "./voices"; /** @@ -63,8 +63,10 @@ async function post(url: string, request: SpeechRequest, retryBusy: boolean, sig /** * One phrase, as the speech server says it in `voice` where it is one of the voices offered, else in the voice chosen * on the page, else in its own. A WAV is asked for, which says its own rate, and its answer is labelled so whatever the - * server calls it: voice mode reads it whole and plays it where it is mono 24 kHz 16-bit, and refuses it otherwise. - * Breeze streams instead, raw 24 kHz PCM (audio.cpp calls it application/octet-stream), passed on as it comes. + * server calls it: voice mode reads it whole and plays it at that rate where it is mono or stereo, 16-bit integer or + * 32-bit float, and refuses it otherwise. Breeze streams instead, raw PCM (audio.cpp calls it application/octet-stream), + * passed on as it comes. So is the answer of any server that says `audio/pcm`. Raw PCM has no header: its rate is the one + * the server says (an `x-sample-rate` header, or a `rate=` parameter of the content type), else the one set for raw audio. * * A clone spoken with by Chatterbox-TTS-Server is stored there first where it is not stored at this address yet, and * again where the server answers that it does not have it (404), as after it was installed anew. @@ -85,18 +87,15 @@ async function speak(text: string, { voice, signal }: { voice?: string; signal: upstream = await send(); } if (!upstream.ok) throw new Error(`The speech server answered HTTP ${upstream.status}`); - if (!request.pcm) return new Response(upstream.body, { headers: { "content-type": "audio/wav" } }); const type = upstream.headers.get("content-type"); + if (!request.pcm && !type?.startsWith("audio/pcm")) return new Response(upstream.body, { headers: { "content-type": "audio/wav" } }); if (!type?.startsWith("audio/pcm") && !type?.startsWith("application/octet-stream")) { await upstream.body?.cancel(); throw new Error(`The speech server answered ${type || "no type"}, not the PCM Breeze streams`); } - const rate = upstream.headers.get("x-sample-rate"); - if (rate && rate !== "24000") { - await upstream.body?.cancel(); - throw new Error(`The speech server answered audio at ${rate} Hz: voice mode plays 24 kHz`); - } - return new Response(upstream.body, { headers: { "content-type": "audio/pcm", "x-sample-rate": "24000" } }); + // Voice mode checks that it is a rate it plays. + const rate = upstream.headers.get("x-sample-rate") ?? /;\s*rate\s*=\s*"?([^";\s]*)/i.exec(type)?.[1] ?? String(s.rawSampleRate ?? RAW_RATE_DEFAULT); + return new Response(upstream.body, { headers: { "content-type": "audio/pcm", "x-sample-rate": rate } }); } export const provider: VoiceProvider = { label: "Voice servers", state, transcribe, speak }; diff --git a/extensions/voice-servers/src/server/requests.ts b/extensions/voice-servers/src/server/requests.ts index cea90dfb6..8dc87b8c6 100644 --- a/extensions/voice-servers/src/server/requests.ts +++ b/extensions/voice-servers/src/server/requests.ts @@ -45,7 +45,7 @@ export function offered(s: Pick ({ id, name })); } -/** A request for one phrase: a JSON body or a form, and whether it is answered with raw 24 kHz PCM as it speaks rather than a WAV. */ +/** A request for one phrase: a JSON body or a form, and whether it is answered with raw PCM as it speaks rather than a WAV. */ export type SpeechRequest = { json: Record; pcm: boolean } | { form: FormData; pcm: boolean }; /** A clone's recording, as audio.cpp takes it inline. */ diff --git a/extensions/voice-servers/src/server/store.ts b/extensions/voice-servers/src/server/store.ts index 09421656b..121c56fe0 100644 --- a/extensions/voice-servers/src/server/store.ts +++ b/extensions/voice-servers/src/server/store.ts @@ -27,6 +27,11 @@ export interface Settings { ttsUrl?: string; /** Sent as `model` with each phrase. */ ttsModel?: string; + /** + * The rate in Hz of the raw audio the speech server streams, which has no header to say it: used where its answer does + * not say (RAW_RATE_DEFAULT where left out). A WAV says its own. + */ + rawSampleRate?: number; /** What an agent's voice is chosen from: typed in, or read from the server. */ voices?: ServerVoice[]; /** @@ -86,6 +91,18 @@ export function saveSettings(next: Settings): void { kept = next; } +/** The rates voice mode plays, in Hz, and the one raw audio is taken to be at where nothing says. */ +export const RAW_RATE_MIN = 8000, RAW_RATE_MAX = 192000, RAW_RATE_DEFAULT = 24000; + +/** The sample rate of raw audio as it was sent, a number or text of digits: whole hertz in the range voice mode plays; undefined where empty. */ +function checkRawRate(given: unknown): number | undefined { + if (given === undefined || given === null || given === "") return undefined; + const rate = typeof given === "number" ? given : typeof given === "string" && /^\s*\d{1,7}\s*$/.test(given) ? Number(given) : NaN; + if (!Number.isInteger(rate) || rate < RAW_RATE_MIN || rate > RAW_RATE_MAX) + throw new Error(`The sample rate of raw audio is a whole number of hertz from ${RAW_RATE_MIN} to ${RAW_RATE_MAX}, or empty for ${RAW_RATE_DEFAULT}`); + return rate; +} + /** A field as it was sent; empty where it was not. */ function said(label: string, given: unknown): string { if (given === undefined) return ""; @@ -148,7 +165,8 @@ export function checkSettings(value: Record | undefined): Setti voices: listed, voice, }; - const checked: Settings = { ...(Object.fromEntries(Object.entries(next).filter(([, v]) => v.length)) as Settings), ...(customVoices ? { customVoices } : {}), ...(speechServer ? { speechServer } : {}) }; + const rawSampleRate = checkRawRate(value?.rawSampleRate); + const checked: Settings = { ...(Object.fromEntries(Object.entries(next).filter(([, v]) => v.length)) as Settings), ...(rawSampleRate ? { rawSampleRate } : {}), ...(customVoices ? { customVoices } : {}), ...(speechServer ? { speechServer } : {}) }; checkCustom(checked); return checked; } diff --git a/extensions/voice-servers/src/web/VoiceServers.tsx b/extensions/voice-servers/src/web/VoiceServers.tsx index 53f070b8e..f2d570a82 100644 --- a/extensions/voice-servers/src/web/VoiceServers.tsx +++ b/extensions/voice-servers/src/web/VoiceServers.tsx @@ -12,6 +12,7 @@ interface Form { dictationModel: string; ttsUrl: string; ttsModel: string; + rawSampleRate: string; /** One voice per line: its id, or `id = Name`. */ voices: string; voice: string; @@ -66,6 +67,7 @@ const formOf = (saved: ServerSettings): Form => ({ dictationModel: saved.dictationModel ?? "", ttsUrl: saved.ttsUrl ?? "", ttsModel: saved.ttsModel ?? "", + rawSampleRate: String(saved.rawSampleRate ?? ""), voices: (saved.voices ?? []).map(lineOf).join("\n"), voice: saved.voice ?? "", }); @@ -165,7 +167,7 @@ export function VoiceServers() {
{failure && setFailure(null)}>{failure}} {/* The page's head, drawn by Settings, has the extension's name and mark already. */} -

{t("Speech recognition and speech synthesis on servers you run, spoken to as OpenAI's audio API is. The speech server has to answer 24 kHz mono WAV.")}

+

{t("Speech recognition and speech synthesis on servers you run, spoken to as OpenAI's audio API is. The speech server answers a WAV file, mono or stereo, at 8 to 192 kHz, or raw PCM.")}

{saved.chosen ? ( <> @@ -191,6 +193,7 @@ export function VoiceServers() { {field("dictationModel", t("Dictation model"), t("Empty uses the speech recognition model above"), t("The model the microphone button of the message box transcribes with. Voice mode keeps using the one above, so a conversation can have the quick model and dictation the accurate one."), true)} {field("ttsUrl", t("Speech synthesis URL"), "http://127.0.0.1:8880/v1/audio/speech", undefined, true)} {field("ttsModel", t("Speech synthesis model"), "kokoro")} + {field("rawSampleRate", t("Sample rate of raw audio"), "24000", t("In Hz, from 8000 to 192000. Used when the speech server streams raw audio, as Breeze does, and does not say its rate. Audio in a WAV file says its own."), true)}
{t("Voice")} diff --git a/extensions/voice-servers/src/web/api.ts b/extensions/voice-servers/src/web/api.ts index b84e4336a..3c07a75af 100644 --- a/extensions/voice-servers/src/web/api.ts +++ b/extensions/voice-servers/src/web/api.ts @@ -33,6 +33,8 @@ export interface ServerSettings { dictationModel?: string; ttsUrl?: string; ttsModel?: string; + /** Hz: a number as saved, or as typed on the page. */ + rawSampleRate?: number | string; voices?: ServerVoice[]; voice?: string; } diff --git a/extensions/voice-servers/test/browser/voice-servers.spec.mts b/extensions/voice-servers/test/browser/voice-servers.spec.mts index ec3e2f3c7..f471ddf60 100644 --- a/extensions/voice-servers/test/browser/voice-servers.spec.mts +++ b/extensions/voice-servers/test/browser/voice-servers.spec.mts @@ -90,7 +90,7 @@ test('the addresses, models and voices are typed in and saved together, and Esca const dialog = await open(page); const save = dialog.getByRole('button', { name: 'Save', exact: true }); await expect(save).toBeDisabled(); - await expect(dialog.getByText('The speech server has to answer 24 kHz mono WAV.', { exact: false })).toBeVisible(); + await expect(dialog.getByText('The speech server answers a WAV file, mono or stereo, at 8 to 192 kHz, or raw PCM.', { exact: false })).toBeVisible(); await dialog.getByLabel('Speech recognition URL').fill('http://127.0.0.1:8178/inference'); await dialog.getByLabel('Speech recognition model').fill('whisper-1'); await dialog.getByLabel('Input language').fill('de'); @@ -98,6 +98,10 @@ test('the addresses, models and voices are typed in and saved together, and Esca await dialog.getByLabel('Dictation model').fill('accurate'); await dialog.getByLabel('Speech synthesis URL').fill('http://127.0.0.1:8880/v1/audio/speech'); await dialog.getByLabel('Speech synthesis model').fill('kokoro'); + // Empty, it is 24000: the field says so, and says what it is for. + await expect(dialog.getByLabel('Sample rate of raw audio')).toHaveAttribute('placeholder', '24000'); + await expect(dialog.getByText('Used when the speech server streams raw audio', { exact: false })).toBeVisible(); + await dialog.getByLabel('Sample rate of raw audio').fill('44100'); await dialog.getByRole('textbox', { name: 'Voices' }).fill('af_heart\nbm_george = George\n\n'); // Custom voices are off: their switch, and nothing else of them. await expect(dialog.getByRole('switch', { name: 'Custom voices' })).not.toBeChecked(); @@ -122,7 +126,7 @@ test('the addresses, models and voices are typed in and saved together, and Esca body: { sttUrl: 'http://127.0.0.1:8178/inference', sttModel: 'whisper-1', language: 'de', dictationUrl: 'http://127.0.0.1:8179/inference', dictationModel: 'accurate', - ttsUrl: 'http://127.0.0.1:8880/v1/audio/speech', ttsModel: 'kokoro', + ttsUrl: 'http://127.0.0.1:8880/v1/audio/speech', ttsModel: 'kokoro', rawSampleRate: '44100', voices: [{ id: 'af_heart', name: 'af_heart' }, { id: 'bm_george', name: 'George' }], voice: 'bm_george', }, }]); @@ -301,7 +305,8 @@ test.describe('in a browser set to German', () => { await portal(page, { sttUrl: 'http://127.0.0.1:8178/inference' }); await page.goto('/settings/x/pithagoras-voice-servers'); const dialog = page.getByRole('dialog', { name: 'Einstellungen' }); - await expect(dialog.getByText('Der Sprachserver muss 24-kHz-Mono-WAV liefern.', { exact: false })).toBeVisible(); + await expect(dialog.getByText('Der Sprachserver antwortet mit einer WAV-Datei, Mono oder Stereo, mit 8 bis 192 kHz, oder mit rohem PCM.', { exact: false })).toBeVisible(); + await expect(dialog.getByLabel('Abtastrate von rohem Audio')).toHaveAttribute('placeholder', '24000'); await expect(dialog.getByLabel('URL der Spracherkennung')).toHaveValue('http://127.0.0.1:8178/inference'); await expect(dialog.getByRole('button', { name: 'Für den Sprachmodus verwenden' })).toBeEnabled(); await expect(dialog.getByRole('button', { name: 'Stimmen vom Server lesen' })).toBeDisabled(); diff --git a/extensions/voice-servers/test/provider.test.mts b/extensions/voice-servers/test/provider.test.mts index 15c859776..ba6dd8e95 100644 --- a/extensions/voice-servers/test/provider.test.mts +++ b/extensions/voice-servers/test/provider.test.mts @@ -128,7 +128,7 @@ test("a phrase is sent as OpenAI's speech takes it, asking for WAV, and the answ await assert.rejects(provider.speak("Hello", { signal }), { message: "No speech synthesis address is set" }); }); -test("voice mode plays a 24 kHz WAV, and refuses one at another rate", async () => { +test("voice mode plays a WAV at the rate it says, 24 kHz or not, and refuses one it cannot read", async () => { answering(); await put({ sttUrl: RECOGNITION, ttsUrl: SPEECH }); const played = await speech("Hello"); @@ -136,9 +136,20 @@ test("voice mode plays a 24 kHz WAV, and refuses one at another rate", async () assert.equal(played.headers.get("content-type"), "audio/pcm"); assert.equal(played.headers.get("x-sample-rate"), "24000"); assert.deepEqual(Buffer.from(await played.arrayBuffer()), SAMPLES); - upstream.answers.set("/v1/audio/speech", { type: "audio/wav", body: wav(SAMPLES, 22050) }); + // Another rate is played at that rate, with the setting for raw audio beside the point. + await put({ sttUrl: RECOGNITION, ttsUrl: SPEECH, rawSampleRate: 16000 }); + for (const rate of [22050, 44100, 48000]) { + upstream.answers.set("/v1/audio/speech", { type: "audio/wav", body: wav(SAMPLES, rate) }); + const other = await speech("Hello"); + assert.deepEqual([other.status, other.headers.get("content-type"), other.headers.get("x-sample-rate")], [200, "audio/pcm", String(rate)]); + assert.deepEqual(Buffer.from(await other.arrayBuffer()), SAMPLES); + } + upstream.answers.set("/v1/audio/speech", { type: "audio/wav", body: Buffer.from("not a recording") }); const refused = await speech("Hello"); - assert.deepEqual([refused.status, await refused.json()], [502, { error: "Expected mono 24 kHz 16-bit audio from Voice servers" }]); + assert.deepEqual([refused.status, await refused.json()], [502, { error: "Voice servers returned invalid WAV audio" }]); + upstream.answers.set("/v1/audio/speech", { type: "audio/wav", body: wav(SAMPLES, 4000) }); + const slow = await speech("Hello"); + assert.deepEqual([slow.status, (await slow.json()).error], [502, "Expected audio from 8000 to 192000 Hz from Voice servers, got mono 16-bit integer audio at 4000 Hz"]); }); test("a server that fails, or answers without a transcript, is said so", async () => { @@ -289,7 +300,60 @@ test("Breeze-TTS's own server is sent its form: the text, the description, guida assert.deepEqual(await formOf((await sent()).last), { text: "Hello there", cfg_scale: "4", seed: "42" }); }); -test("Breeze busy with a phrase before is asked again until it answers, and an answer that is not 24 kHz PCM is refused", async () => { +test("raw PCM has no header: its rate is the one the server says in a header or its content type, else the one set for raw audio, else 24 kHz", async () => { + await custom(true); + await put({ sttUrl: RECOGNITION, ttsUrl: SPEECH, ttsModel: "breeze" }); + await server("breeze-audiocpp"); + const signal = new AbortController().signal; + /** The rate the PCM voice mode is given is labelled with, for what the speech server answered. */ + const rateOf = async (answer: { type: string; headers?: Record }) => { + upstream.answers.set("/v1/audio/speech", { ...answer, body: SAMPLES }); + const spoken = await provider.speak("Hello", { signal }); + assert.equal(spoken.headers.get("content-type"), "audio/pcm"); + assert.deepEqual(Buffer.from(await spoken.arrayBuffer()), SAMPLES); + return spoken.headers.get("x-sample-rate"); + }; + const octets = { type: "application/octet-stream" }; + // Nothing said, nothing set: 24 kHz, as audio.cpp's Breeze speaks. + assert.equal(await rateOf(octets), "24000"); + assert.equal(await rateOf({ type: "audio/pcm" }), "24000"); + // The setting is what raw audio is at where the server says nothing. + await put({ sttUrl: RECOGNITION, ttsUrl: SPEECH, ttsModel: "breeze", rawSampleRate: 44100 }); + assert.equal(await rateOf(octets), "44100"); + assert.equal(await rateOf({ type: "audio/pcm" }), "44100"); + // What the server says wins over it: a header first, then a parameter of the type. + assert.equal(await rateOf({ type: "audio/pcm", headers: { "x-sample-rate": "48000" } }), "48000"); + assert.equal(await rateOf({ type: "audio/pcm;rate=22050" }), "22050"); + assert.equal(await rateOf({ type: 'audio/pcm; channels=1; rate="16000"' }), "16000"); + assert.equal(await rateOf({ type: "application/octet-stream;rate=32000", headers: { "x-sample-rate": "8000" } }), "8000"); + // Breeze's own server is the same. + await server("breeze-tts"); + assert.equal(await rateOf(octets), "44100"); + assert.equal(await rateOf({ type: "audio/pcm", headers: { "x-sample-rate": "48000" } }), "48000"); + // A fixed-voice server that answers raw PCM, though a WAV was asked for, is played too. + await server("fixed"); + assert.equal(await rateOf({ type: "audio/pcm", headers: { "x-sample-rate": "48000" } }), "48000"); + assert.equal(await rateOf({ type: "audio/pcm" }), "44100"); +}); + +test("raw PCM reaches voice mode at its rate: streamed as 44.1 kHz from the server, or from the setting, and a rate that is not played is refused", async () => { + await custom(true); + await put({ sttUrl: RECOGNITION, ttsUrl: SPEECH, ttsModel: "breeze", rawSampleRate: 44100 }); + await server("breeze-audiocpp"); + const rateOf = async (answer: { type: string; headers?: Record }) => { + upstream.answers.set("/v1/audio/speech", { ...answer, body: SAMPLES }); + const played = await speech("Hello"); + return [played.status, played.headers.get("content-type"), played.headers.get("x-sample-rate"), Buffer.from(await played.arrayBuffer())]; + }; + assert.deepEqual(await rateOf({ type: "application/octet-stream" }), [200, "audio/pcm", "44100", SAMPLES]); + assert.deepEqual(await rateOf({ type: "audio/pcm", headers: { "x-sample-rate": "48000" } }), [200, "audio/pcm", "48000", SAMPLES]); + assert.deepEqual(await rateOf({ type: "audio/pcm;rate=22050" }), [200, "audio/pcm", "22050", SAMPLES]); + upstream.answers.set("/v1/audio/speech", { type: "audio/pcm", headers: { "x-sample-rate": "300000" }, body: SAMPLES }); + const tooFast = await speech("Hello"); + assert.deepEqual([tooFast.status, (await tooFast.json()).error], [502, "Voice servers answered audio at 300000 Hz: voice mode plays 8000 to 192000 Hz"]); +}); + +test("Breeze busy with a phrase before is asked again until it answers, and an answer that is not PCM is refused", async () => { await custom(true); await put({ sttUrl: RECOGNITION, ttsUrl: SPEECH, ttsModel: "breeze" }); const signal = new AbortController().signal; @@ -308,8 +372,6 @@ test("Breeze busy with a phrase before is asked again until it answers, and an a await put({ sttUrl: RECOGNITION, ttsUrl: SPEECH, ttsModel: "breeze" }); upstream.answers.set("/v1/audio/speech", { type: "audio/wav", body: wav(SAMPLES, 24000) }); await assert.rejects(provider.speak("Hello", { signal }), { message: "The speech server answered audio/wav, not the PCM Breeze streams" }); - upstream.answers.set("/v1/audio/speech", { type: "audio/pcm", headers: { "x-sample-rate": "48000" }, body: SAMPLES }); - await assert.rejects(provider.speak("Hello", { signal }), { message: "The speech server answered audio at 48000 Hz: voice mode plays 24 kHz" }); upstream.answers.set("/v1/audio/speech", { status: 400, body: JSON.stringify({ error: "speech streaming requires a model configured with mode=streaming" }) }); await assert.rejects(provider.speak("Hello", { signal }), { message: "The speech server answered HTTP 400" }); // A server with fixed voices is asked once: a 409 there is its answer. diff --git a/extensions/voice-servers/test/routes.test.mts b/extensions/voice-servers/test/routes.test.mts index a44827ee5..3a8494c3e 100644 --- a/extensions/voice-servers/test/routes.test.mts +++ b/extensions/voice-servers/test/routes.test.mts @@ -34,6 +34,7 @@ test("the settings are kept for their owner alone, each left out while it is emp dictationModel: " accurate ", ttsUrl: "https://speech.example.test/v1/audio/speech", ttsModel: "kokoro", + rawSampleRate: " 44100 ", voices: [{ id: "af_heart" }, { id: "bm_george", name: "George" }, { id: "am_adam", name: "" }], voice: "bm_george", }); @@ -43,6 +44,7 @@ test("the settings are kept for their owner alone, each left out while it is emp dictationModel: "accurate", ttsUrl: "https://speech.example.test/v1/audio/speech", ttsModel: "kokoro", + rawSampleRate: 44100, voices: [{ id: "af_heart", name: "af_heart" }, { id: "bm_george", name: "George" }, { id: "am_adam", name: "am_adam" }], voice: "bm_george", }; @@ -64,6 +66,7 @@ test("each setting is checked, and a save refused changes nothing", async () => const language = "The input language is a code of two or three small letters, such as en, or empty for the server to detect it"; const list = "The voices are a list of at most 500"; const id = "A voice's id is 1–200 characters, with no control characters"; + const rate = "The sample rate of raw audio is a whole number of hertz from 8000 to 192000, or empty for 24000"; for (const [settings, error] of [ [{ sttUrl: "ftp://example.test/inference" }, url("The speech recognition URL")], [{ sttUrl: "not an address" }, url("The speech recognition URL")], @@ -73,6 +76,12 @@ test("each setting is checked, and a save refused changes nothing", async () => [{ sttModel: "a model" }, model("The speech recognition model")], [{ dictationModel: "m".repeat(101) }, model("The dictation model")], [{ ttsModel: "kokoro/v1" }, model("The speech synthesis model")], + [{ rawSampleRate: 7999 }, rate], + [{ rawSampleRate: 192001 }, rate], + [{ rawSampleRate: 44100.5 }, rate], + [{ rawSampleRate: "fast" }, rate], + [{ rawSampleRate: "44100 Hz" }, rate], + [{ rawSampleRate: true }, rate], [{ language: "EN" }, language], [{ language: "auto" }, language], [{ voices: "af_heart" }, list], @@ -88,6 +97,11 @@ test("each setting is checked, and a save refused changes nothing", async () => assert.deepEqual(await said(await put(settings)), [400, error], JSON.stringify(settings).slice(0, 100)); } assert.deepEqual(await shown(), { sttUrl: RECOGNITION, chosen: false }); + // The sample rate of raw audio, as a number or as typed, at both ends of what is played; empty is left out. + for (const [given, kept] of [[8000, 8000], ["192000", 192000], ["", undefined], [null, undefined]] as const) { + assert.equal((await put({ sttUrl: RECOGNITION, rawSampleRate: given })).status, 200); + assert.equal((await shown()).rawSampleRate, kept, String(given)); + } // As many as 500 voices, of up to 200 characters each. assert.equal((await put({ voices: [...Array.from({ length: 499 }, (_, n) => ({ id: `voice-${n}` })), { id: "v".repeat(200) }] })).status, 200); }); diff --git a/packages/extension-api/server.d.ts b/packages/extension-api/server.d.ts index cdecf676f..94ec15b7f 100644 --- a/packages/extension-api/server.d.ts +++ b/packages/extension-api/server.d.ts @@ -513,10 +513,11 @@ export interface VoiceProvider { transcribe(wav: Buffer, options: { purpose: "voice" | "dictation"; signal: AbortSignal }): Promise; /** * One phrase of at most 600 characters, spoken in `voice`: an id from `state().voices`; undefined, or an id it no - * longer has, is its own choice. Answers an ok Response whose body is `audio/pcm` (mono, 24 kHz, signed 16-bit - * little-endian, passed on to the page as it arrives; an `x-sample-rate` header, where there is one, says 24000) or - * `audio/wav` (mono, 24 kHz, 16-bit PCM, read whole and checked). `signal` as for `transcribe`. Throws with what the - * page should say. + * longer has, is its own choice. Answers an ok Response whose body is `audio/pcm` (mono, signed 16-bit little-endian, + * passed on to the page as it arrives; its rate in Hz, 8000 to 192000, is the `x-sample-rate` header or a `rate=` + * parameter of the content type, 24000 where it says none) or `audio/wav` (8 to 192 kHz by its header, mono or + * stereo, 16-bit integer or 32-bit float samples, read whole and checked). The page plays a phrase at its own rate. + * `signal` as for `transcribe`. Throws with what the page should say. */ speak(text: string, options: { voice?: string; signal: AbortSignal }): Promise; /** diff --git a/tests/browser/voice.spec.mts b/tests/browser/voice.spec.mts index f49186e87..bb11eead5 100644 --- a/tests/browser/voice.spec.mts +++ b/tests/browser/voice.spec.mts @@ -193,6 +193,52 @@ for (const pipelineMode of ['parallel', 'sequential'] as const) { }); } +for (const pipelineMode of ['parallel', 'sequential'] as const) { + test(`a phrase the portal answers at 44.1 kHz is buffered and stretched at 44.1 kHz in the ${pipelineMode} pipeline, and one that names no usable rate is refused`, async ({ page }) => { + // One second of a 220 Hz tone at 44.1 kHz, spoken 1.5 times as fast; every buffer made is counted with its rate. + const pcm = Buffer.alloc(44100 * 2); + for (let i = 0; i < pcm.length / 2; i++) pcm.writeInt16LE(Math.round(Math.sin(i * 2 * Math.PI * 220 / 44100) * 5000), i * 2); + await page.addInitScript(() => { + localStorage.setItem('voiceRate', '1.5'); + const w = window as any; + w.buffers = [] as [number, number][]; + const create = BaseAudioContext.prototype.createBuffer; + BaseAudioContext.prototype.createBuffer = function (channels: number, length: number, sampleRate: number) { w.buffers.push([length, sampleRate]); return create.call(this, channels, length, sampleRate); }; + }); + let rate: string | undefined = '44100'; + await page.route('**/api/voice', route => route.fulfill({ json: { enabled: true, pipelineMode } })); + await page.route('**/voice/speech', route => { + // The spoken error line, asked for as voice mode starts, is not a phrase of the reply. + const body = route.request().postDataJSON().text === 'An error occurred.' ? Buffer.alloc(4800) : pcm; + return route.fulfill({ body, headers: { 'content-type': 'audio/pcm', ...(rate ? { 'x-sample-rate': rate } : {}) } }); + }); + await page.goto('/tests/voice.html'); + await page.getByRole('button', { name: 'Turn on hands-free voice' }).click(); + await expect(page.getByRole('status')).toHaveText('Listening', { timeout: 25000 }); + await page.getByRole('button', { name: 'Stream reply', exact: true }).click(); + await page.getByRole('button', { name: 'Finish reply', exact: true }).click(); + await expect(page.locator('.voice-orb')).toHaveAttribute('data-mode', 'output'); + // The reply is two phrases. All the buffers are made at the phrase's rate, and together as long as the phrases made faster. + const phrases = () => page.evaluate(() => ((window as any).buffers as [number, number][]).filter(([length]) => length > 1000)); + const seconds = async () => (await phrases()).reduce((n, [length, sampleRate]) => n + length / sampleRate, 0); + await expect.poll(seconds, { timeout: 10000 }).toBeGreaterThan(2 / 1.5 - 0.01); + expect(await seconds()).toBeLessThan(2 / 1.5 + 0.01); + expect((await phrases()).map(([, sampleRate]) => sampleRate)).toEqual((await phrases()).map(() => 44100)); + await page.getByRole('button', { name: 'End voice mode' }).click(); + // No rate said, or one that is not played, is no speech to play. + for (rate of [undefined, '300000', 'fast']) { + await page.evaluate(() => { (window as any).buffers = []; }); + await page.getByRole('button', { name: 'Turn on hands-free voice' }).click(); + await expect(page.getByRole('status')).toHaveText('Listening', { timeout: 25000 }); + await page.getByRole('button', { name: 'Stream reply', exact: true }).click(); + await page.getByRole('button', { name: 'Finish reply', exact: true }).click(); + await expect(page.getByText('Unsupported speech stream')).toBeVisible(); + expect((await page.evaluate(() => (window as any).buffers as [number, number][])).filter(([length]) => length > 1000)).toEqual([]); + await page.getByRole('button', { name: 'End voice mode' }).click(); + } + }); +} + test('slow PCM chunks become one uninterrupted buffer, including split samples', async ({ page }) => { await page.goto('/tests/voice.html'); const result = await page.evaluate(async () => { diff --git a/tests/fixtures/extensions/speaker/server.js b/tests/fixtures/extensions/speaker/server.js index dd0871f4c..2e7ceab0f 100644 --- a/tests/fixtures/extensions/speaker/server.js +++ b/tests/fixtures/extensions/speaker/server.js @@ -9,8 +9,13 @@ * whose body was cancelled * PUT /api/x/test-speaker/settings { ready?, speaks?, voices?, answer?, stateThrows?, stateAnswers?, fails?, hangs? }: * whether `state` answers, whether it speaks, the voices it offers, what `speak` - * answers ("pcm", "pcm48k", "wav", "wav22k", "text", "notOk", "notResponse", - * "throws", or "hangs", which `transcribe` does too), whether `state` throws, what + * answers ("pcm": raw PCM that says 24000 Hz, "pcmBare": raw PCM that says nothing of its rate, + * "pcm44k" and "pcm48k": raw PCM that says its rate in a header, "pcmType": + * raw PCM at 44100 Hz that says so in its content type, streamed in two pieces, + * "pcm300k": raw PCM at a rate no one plays, "pcmStereo": raw PCM that says it + * has two channels, "wav": 24 kHz, "wav22k", "wav44k", "wav48kStereoFloat" and + * "wav24bit" WAVs, "text", "notOk", "notResponse", "throws", or "hangs", which + * `transcribe` does too), whether `state` throws, what * it answers in place of its own (null for its own), and whether `use(true)` fails * or never settles * POST /api/x/test-speaker/choose { chosen }: `portal.voice.choose` @@ -24,22 +29,37 @@ /** Two samples of mono 16-bit PCM. */ const PCM = Buffer.from([0, 0, 255, 127]); -/** A phrase as a speech engine that answers with a WAV gives it: mono, 16-bit, two samples, at `rate`. */ -function wav(rate) { +/** + * A phrase as a speech engine that answers with a WAV gives it: two samples at `rate`, mono 16-bit integers; with + * `float`, 32-bit floats (one full-scale sample and one silent); with `channels` 2, the second channel silent in the + * first sample and equal to the first in the second; with `bits` 24, 24-bit integers, which voice mode does not read. + */ +function wav({ rate, float = false, channels = 1, bits = float ? 32 : 16 }) { + const samples = float ? [1, 0] : [0, 32767]; + const sample = (value) => { + const bytes = Buffer.alloc(bits / 8); + if (float) bytes.writeFloatLE(value); + else if (bits === 16) bytes.writeInt16LE(value); + else bytes.writeIntLE(value << 8, 0, 3); + return bytes; + }; + const frames = samples.map((value, i) => Buffer.concat(channels === 1 ? [sample(value)] : [sample(value), sample(i === 0 ? 0 : value)])); + const data = Buffer.concat(frames); + const frame = (bits / 8) * channels; const header = Buffer.alloc(44); header.write("RIFF"); - header.writeUInt32LE(36 + PCM.length, 4); + header.writeUInt32LE(36 + data.length, 4); header.write("WAVEfmt ", 8); header.writeUInt32LE(16, 16); - header.writeUInt16LE(1, 20); - header.writeUInt16LE(1, 22); + header.writeUInt16LE(float ? 3 : 1, 20); + header.writeUInt16LE(channels, 22); header.writeUInt32LE(rate, 24); - header.writeUInt32LE(rate * 2, 28); - header.writeUInt16LE(2, 32); - header.writeUInt16LE(16, 34); + header.writeUInt32LE(rate * frame, 28); + header.writeUInt16LE(frame, 32); + header.writeUInt16LE(bits, 34); header.write("data", 36); - header.writeUInt32LE(PCM.length, 40); - return Buffer.concat([header, PCM]); + header.writeUInt32LE(data.length, 40); + return Buffer.concat([header, data]); } export default function (portal) { @@ -56,11 +76,11 @@ export default function (portal) { fails: false, hangs: false, }; - /** A body whose cancelling is counted. */ - const body = (bytes) => { + /** A body, in one piece or the pieces given, whose cancelling is counted. */ + const body = (bytes, ...more) => { const stream = new ReadableStream({ start(controller) { - controller.enqueue(new Uint8Array(bytes)); + for (const piece of [bytes, ...more]) controller.enqueue(new Uint8Array(piece)); controller.close(); }, }); @@ -91,16 +111,33 @@ export default function (portal) { async speak(text, { voice, signal }) { watch(signal); asked.spoken.push({ text, voice: voice ?? null }); - const audio = (type, bytes, more = {}) => new Response(body(bytes), { headers: { "content-type": type, ...more } }); + const audio = (type, bytes, more = {}, ...rest) => new Response(body(bytes, ...rest), { headers: { "content-type": type, ...more } }); switch (settings.answer) { case "pcm": return audio("audio/pcm", PCM, { "x-sample-rate": "24000" }); + case "pcmBare": + return audio("audio/pcm", PCM); + case "pcm44k": + return audio("audio/pcm", PCM, { "x-sample-rate": "44100" }); case "pcm48k": return audio("audio/pcm", PCM, { "x-sample-rate": "48000" }); + case "pcmType": + // Streamed, in two pieces, the second cutting the first sample of a pair short. + return audio("audio/pcm;rate=44100", PCM.subarray(0, 3), {}, PCM.subarray(3)); + case "pcm300k": + return audio("audio/pcm", PCM, { "x-sample-rate": "300000" }); + case "pcmStereo": + return audio("audio/pcm;rate=44100;channels=2", PCM); case "wav": - return audio("audio/wav", wav(24000)); + return audio("audio/wav", wav({ rate: 24000 })); case "wav22k": - return audio("audio/wav", wav(22050)); + return audio("audio/wav", wav({ rate: 22050 })); + case "wav44k": + return audio("audio/wav", wav({ rate: 44100 })); + case "wav48kStereoFloat": + return audio("audio/wav", wav({ rate: 48000, float: true, channels: 2 })); + case "wav24bit": + return audio("audio/wav", wav({ rate: 44100, bits: 24 })); case "text": return audio("text/plain", Buffer.from("not audio")); case "notOk": diff --git a/tests/pcm-stream.test.mts b/tests/pcm-stream.test.mts index 4cb5e8c4f..868acee71 100644 --- a/tests/pcm-stream.test.mts +++ b/tests/pcm-stream.test.mts @@ -1,14 +1,15 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { preparePcmSpeech, readPcmStream } from '../core/web/src/pcm-stream.js'; +import { bufferOf, preparePcmSpeech, readPcmStream, sampleRateOf } from '../core/web/src/pcm-stream.js'; import { stretch } from '../core/web/src/time-stretch.js'; import { addLocale, setLanguage } from '../core/web/src/i18n.js'; const tick = () => new Promise(resolve => setImmediate(resolve)); function audio() { - const sources: any[] = []; - return { sources, context: { + const sources: any[] = [], rates: number[] = []; + return { sources, rates, context: { currentTime: 1, createBuffer: (_channels: number, count: number, rate: number) => { + rates.push(rate); const data = new Float32Array(count); return { duration: count / rate, getChannelData: () => data }; }, @@ -108,3 +109,65 @@ test('barge-in cancels both streaming reads and scheduled audio', async () => { await assert.rejects(speech.completed, /interrupted/); assert.equal(cancelled, true); assert.ok(sources.every(s => s.stopped)); }); + +const bytesOfSamples = (samples: Float32Array) => { + const bytes = new Uint8Array(samples.length * 2), view = new DataView(bytes.buffer); + samples.forEach((v, i) => view.setInt16(i * 2, Math.round(v * 32767), true)); + return bytes; +}; +test('a phrase says its sample rate in x-sample-rate, and only one the portal passes on counts', () => { + const said = (value?: string) => sampleRateOf(new Headers(value === undefined ? {} : { 'x-sample-rate': value })); + assert.deepEqual([said('24000'), said('44100'), said('8000'), said('192000')], [24000, 44100, 8000, 192000]); + for (const bad of [undefined, '', '7999', '192001', '44100.5', '-44100', 'fast', '4e4', '0x5000', '44100 Hz']) assert.equal(said(bad), undefined, String(bad)); +}); +test('a buffer is made at the rate of the phrase, whatever the audio context plays at', () => { + const { context, rates } = audio(); + assert.equal(bufferOf(context, new Float32Array(441), 44100)!.duration, 0.01); + assert.equal(bufferOf(context, new Float32Array(240))!.duration, 0.01); + assert.deepEqual(rates, [44100, 24000]); +}); +test('44.1 kHz PCM is buffered at 44.1 kHz, stretched at the length and pitch it has there', async () => { + const sampleRate = 44100, hz = 45; + const samples = Float32Array.from({ length: sampleRate * 2 }, (_, i) => 0.5 * Math.sin(2 * Math.PI * hz * i / sampleRate)); + const bytes = bytesOfSamples(samples); + const whole = new Float32Array(bytes.length / 2).map((_, i) => new DataView(bytes.buffer).getInt16(i * 2, true) / 32768); + for (const rate of [1, 1.5]) { + const { context, rates } = audio(); + const recorded: Float32Array[] = []; + // In pieces cut inside a sample, as a network does. + const body = new ReadableStream({ start(c) { for (let at = 0; at < bytes.length; at += 4001) c.enqueue(bytes.subarray(at, at + 4001)); c.close(); } }); + const buffer = await readPcmStream(body, context, new AbortController().signal, { rate, sampleRate, record: s => recorded.push(s) }); + assert.deepEqual([...new Set(rates)], [sampleRate], `${rate}: made at the phrase's rate`); + // Recorded as it was spoken, for a replay at the phrase's rate. + assert.equal(recorded[0].length, samples.length); + const out = buffer.getChannelData(0); + assert.ok(Math.abs(out.length - samples.length / rate) <= 1, `${rate}: ${out.length}`); + assert.ok(Math.abs(buffer.duration - 2 / rate) < 1e-3, `${rate}: ${buffer.duration} s`); + const expected = stretch(whole, rate, sampleRate); + assert.equal(out.length, expected.length); + assert.ok(out.every((v, i) => Math.abs(v - expected[i]) < 1e-6)); + const middle = out.subarray(4410, out.length - 4410); + let crossings = 0; + for (let i = 1; i < middle.length; i++) if ((middle[i - 1] < 0) !== (middle[i] < 0)) crossings++; + assert.ok(Math.abs(crossings / 2 / (middle.length / sampleRate) - hz) < 1.5, `${rate}: pitch`); + } +}); +test('44.1 kHz PCM that streams is scheduled in buffers of 44.1 kHz, back to back', async () => { + const { context, sources, rates } = audio(); const abort = new AbortController(); + let writer!: ReadableStreamDefaultController; + const body = new ReadableStream({ start(c) { writer = c; } }); + // 0.7 s are enough to start; each buffer is as long as the piece that made it. + const ready = preparePcmSpeech(body, context, abort.signal, { sampleRate: 44100 }); + writer.enqueue(new Uint8Array(61740)); + const speech = await ready; + const playing = speech.play({} as AudioNode, () => {}); + writer.enqueue(new Uint8Array(44100)); await tick(); + assert.equal(sources.length, 2); + assert.ok(Math.abs(sources[0].buffer.duration - 0.7) < 1e-9, `${sources[0].buffer.duration} s`); + assert.ok(Math.abs(sources[1].buffer.duration - 0.5) < 1e-9, `${sources[1].buffer.duration} s`); + assert.equal(sources[1].at, sources[0].at + sources[0].buffer.duration); + writer.close(); await speech.completed; + for (const source of sources) source.onended(); + await playing; + assert.deepEqual([...new Set(rates)], [44100]); +}); diff --git a/tests/time-stretch.test.mts b/tests/time-stretch.test.mts index 564a5812f..624110f51 100644 --- a/tests/time-stretch.test.mts +++ b/tests/time-stretch.test.mts @@ -64,3 +64,45 @@ test('stretching a little at a time gives what stretching it at once does', asyn const controller = new AbortController(); controller.abort(); await assert.rejects(stretchInSteps(input, 1.5, controller.signal)); }); + +/** A tone made at any sample rate, and its pitch worked out at that rate. */ +const toneAt = (sampleRate: number, seconds: number, hz: number) => Float32Array.from({ length: Math.round(seconds * sampleRate) }, (_, i) => 0.5 * Math.sin(2 * Math.PI * hz * i / sampleRate)); +const pitchAt = (samples: Float32Array, sampleRate: number) => { + let crossings = 0; + for (let i = 1; i < samples.length; i++) if ((samples[i - 1] < 0) !== (samples[i] < 0)) crossings++; + return crossings / 2 / (samples.length / sampleRate); +}; + +test('speech made at 44.1 or 48 kHz is shorter by the rate and keeps its pitch, low voices too', () => { + for (const sampleRate of [22050, 44100, 48000]) { + // 40 and 50 Hz are a deep voice's lowest tones: their period is longer than the pieces' nudge was at 24 kHz's sizes. + for (const hz of [40, 50, 220]) { + const input = toneAt(sampleRate, 2, hz); + for (const rate of [1.5, 2]) { + const out = stretch(input, rate, sampleRate); + assert.ok(Math.abs(out.length - input.length / rate) <= 1, `${sampleRate} Hz, ${hz} Hz, ${rate}: ${out.length}`); + const middle = out.subarray(Math.round(sampleRate / 10), out.length - Math.round(sampleRate / 10)); + assert.ok(Math.abs(pitchAt(middle, sampleRate) - hz) < 1.5, `${sampleRate} Hz, ${hz} Hz, ${rate}: ${pitchAt(middle, sampleRate)} Hz`); + assert.ok(Math.abs(rms(middle) - rms(input)) < 0.03, `${sampleRate} Hz, ${hz} Hz, ${rate}: rms ${rms(middle)}`); + } + } + } +}); + +test('the pieces are as long in time at every sample rate, so streamed in parts it comes out as all at once', async () => { + const sampleRate = 44100, input = toneAt(sampleRate, 1.3, 180); + const whole = stretch(input, 1.5, sampleRate); + const s = new TimeStretch(1.5, sampleRate); + const parts: Float32Array[] = []; + for (let i = 0; i < input.length; i += 2111) parts.push(s.push(input.subarray(i, i + 2111))); + parts.push(s.flush()); + const streamed = new Float32Array(parts.reduce((n, p) => n + p.length, 0)); + let offset = 0; + for (const p of parts) { streamed.set(p, offset); offset += p.length; } + assert.equal(streamed.length, whole.length); + assert.ok(streamed.every((v, i) => Math.abs(v - whole[i]) < 1e-6)); + const stepped = await stretchInSteps(input, 1.5, new AbortController().signal, sampleRate); + assert.equal(stepped.length, whole.length); + assert.ok(stepped.every((v, i) => Math.abs(v - whole[i]) < 1e-6)); + assert.throws(() => new TimeStretch(1.5, 0), /sample rate/); +}); diff --git a/tests/voice.test.mts b/tests/voice.test.mts index 87a015893..35beb2dc9 100644 --- a/tests/voice.test.mts +++ b/tests/voice.test.mts @@ -5,7 +5,8 @@ import express from 'express'; import { speechChunks } from '../core/web/src/voice.js'; import { inProcessHome } from "./helpers.mts"; inProcessHome('pithagoras-voice-'); -const { voiceRouter, wavPcm, validateConfig } = await import('../core/server/src/api/voice.js'); +const { voiceRouter, validateConfig } = await import('../core/server/src/api/voice.js'); +const { wavAudio } = await import('../core/server/src/wav.js'); const host = await import('../core/server/src/extension-host.js'); const { getDb, putSetting, getSkipThinkingProviders, getVoiceInstructions, getVoiceModel } = await import('../core/server/src/db.js'); const { DEFAULT_SKIP_THINKING_PROVIDERS, DEFAULT_VOICE_INSTRUCTIONS } = await import('../core/server/src/pi/voice-first.js'); @@ -69,21 +70,86 @@ test('voice conversations are on unless a save switches them off, with true or f test('a WAV whose data chunk carries a placeholder size keeps its samples', () => { const placeheld = wav(Buffer.from([0, 0, 255, 127])); - assert.deepEqual(wavPcm(placeheld, 'Speaker'), Buffer.from([0, 0, 255, 127])); + assert.deepEqual(wavAudio(placeheld, 'Speaker'), { rate: 24000, pcm: Buffer.from([0, 0, 255, 127]) }); for (const size of [0, 0xffffffff]) { const placeholder = Buffer.from(placeheld); placeholder.writeUInt32LE(size, 40); - assert.deepEqual(wavPcm(placeholder, 'Speaker'), Buffer.from([0, 0, 255, 127])); + assert.deepEqual(wavAudio(placeholder, 'Speaker').pcm, Buffer.from([0, 0, 255, 127])); } // A truncated fmt chunk is reported as unsupported audio, not as a read past the end. const truncated = Buffer.concat([Buffer.from('RIFF\u0000\u0000\u0000\u0000WAVE', 'ascii'), Buffer.alloc(40), Buffer.from('fmt ', 'ascii'), Buffer.alloc(8)]); - assert.throws(() => wavPcm(truncated, 'Speaker'), /Expected mono 24 kHz 16-bit audio from Speaker/); - // Samples ahead of their fmt chunk, or in a non-PCM encoding, are refused. + assert.throws(() => wavAudio(truncated, 'Speaker'), /Speaker returned a WAV file with a broken fmt chunk/); + // Samples ahead of their fmt chunk, or in an encoding that is not read, are refused. const dataFirst = Buffer.concat([placeheld.subarray(0, 12), placeheld.subarray(36), placeheld.subarray(12, 36)]); - assert.throws(() => wavPcm(dataFirst, 'Speaker'), /Expected mono 24 kHz 16-bit audio/); - const float = Buffer.from(placeheld); - float.writeUInt16LE(3, 20); - assert.throws(() => wavPcm(float, 'Speaker'), /Expected mono 24 kHz 16-bit audio/); + assert.throws(() => wavAudio(dataFirst, 'Speaker'), /Speaker returned a WAV file whose samples come before its fmt chunk/); + const alaw = Buffer.from(placeheld); + alaw.writeUInt16LE(6, 20); + assert.throws(() => wavAudio(alaw, 'Speaker'), /Expected mono or stereo 16-bit integer or 32-bit float audio from Speaker, got mono 16-bit format 6 audio at 24000 Hz/); +}); + +/** A WAV of any of the shapes a speech server answers in; `data` is the samples as they are, `fmt` the fields to override. */ +function shaped(data: Buffer, fmt: { tag?: number; channels?: number; rate?: number; bits?: number; extensible?: number; frame?: number } = {}): Buffer { + const { tag = 1, channels = 1, rate = 24000, bits = 16, extensible, frame = channels * bits / 8 } = fmt; + const chunk = Buffer.alloc(extensible === undefined ? 16 : 40); + chunk.writeUInt16LE(extensible === undefined ? tag : 0xfffe, 0); chunk.writeUInt16LE(channels, 2); chunk.writeUInt32LE(rate, 4); + chunk.writeUInt32LE((rate * frame) >>> 0, 8); chunk.writeUInt16LE(frame, 12); chunk.writeUInt16LE(bits, 14); + if (extensible !== undefined) { chunk.writeUInt16LE(22, 16); chunk.writeUInt16LE(bits, 18); chunk.writeUInt16LE(extensible, 24); } + const head = (id: string, size: number) => { const h = Buffer.alloc(8); h.write(id); h.writeUInt32LE(size, 4); return h; }; + const body = Buffer.concat([Buffer.from('WAVE'), head('fmt ', chunk.length), chunk, head('data', data.length), data]); + return Buffer.concat([Buffer.from('RIFF'), (() => { const n = Buffer.alloc(4); n.writeUInt32LE(body.length); return n; })(), body]); +} +const floats = (...values: number[]) => { const b = Buffer.alloc(values.length * 4); values.forEach((v, i) => b.writeFloatLE(v, i * 4)); return b; }; +const ints = (...values: number[]) => { const b = Buffer.alloc(values.length * 2); values.forEach((v, i) => b.writeInt16LE(v, i * 2)); return b; }; +const samplesOf = (pcm: Buffer) => Array.from({ length: pcm.length / 2 }, (_, i) => pcm.readInt16LE(i * 2)); + +test('a WAV is read at the rate its header gives, from 8 to 192 kHz', () => { + for (const rate of [8000, 22050, 44100, 48000, 96000, 192000]) { + assert.deepEqual(wavAudio(shaped(ints(100, -100), { rate }), 'Speaker'), { rate, pcm: ints(100, -100) }, String(rate)); + } + for (const rate of [7999, 192001, 0, 4294967295]) { + assert.throws(() => wavAudio(shaped(ints(1, 2), { rate }), 'Speaker'), /Expected audio from 8000 to 192000 Hz from Speaker, got mono 16-bit integer audio at/, String(rate)); + } +}); + +test('stereo is mixed to mono, and 32-bit float is made 16-bit integer, also in the extensible form', () => { + // Pairs: (1000, 3000) mixes to 2000; (-32768, 32767) to 0 (-0.5 rounded). + assert.deepEqual(samplesOf(wavAudio(shaped(ints(1000, 3000, -32768, 32767), { channels: 2, rate: 44100 }), 'Speaker').pcm), [2000, 0]); + // Float: full scale, silence, a negative half, and values past full scale or not a number. + const float = wavAudio(shaped(floats(1, 0, -0.5, 2, -3, NaN), { tag: 3, bits: 32, rate: 48000 }), 'Speaker'); + assert.equal(float.rate, 48000); + assert.deepEqual(samplesOf(float.pcm), [32767, 0, -16384, 32767, -32768, 0]); + const stereoFloat = wavAudio(shaped(floats(1, -1, 0.5, 0.5), { tag: 3, bits: 32, channels: 2, rate: 44100 }), 'Speaker'); + assert.deepEqual([stereoFloat.rate, samplesOf(stereoFloat.pcm)], [44100, [0, 16384]]); + // WAVE_FORMAT_EXTENSIBLE names the format in its sub-format: PCM or float. + assert.deepEqual(wavAudio(shaped(ints(5, -5), { extensible: 1, rate: 44100 }), 'Speaker'), { rate: 44100, pcm: ints(5, -5) }); + assert.deepEqual(samplesOf(wavAudio(shaped(floats(0.5), { extensible: 3, bits: 32 }), 'Speaker').pcm), [16384]); +}); + +test('a WAV that cannot be read is refused with what it was', () => { + const refused = (wav: Buffer, message: RegExp) => assert.throws(() => wavAudio(wav, 'Speaker'), message); + refused(shaped(Buffer.alloc(9), { bits: 24 }), /got mono 24-bit integer audio at 24000 Hz/); + refused(shaped(Buffer.alloc(8), { bits: 8 }), /got mono 8-bit integer audio at 24000 Hz/); + refused(shaped(Buffer.alloc(8), { tag: 3, bits: 64 }), /got mono 64-bit float audio at 24000 Hz/); + refused(shaped(Buffer.alloc(12), { channels: 6 }), /got 6-channel 16-bit integer audio at 24000 Hz/); + refused(shaped(Buffer.alloc(4), { channels: 0 }), /got 0-channel 16-bit integer audio/); + refused(shaped(ints(1, 2), { extensible: 6 }), /got mono 16-bit format 6 audio/); + // A fmt chunk that disagrees with itself, one cut short, and extensible without its sub-format. + refused(shaped(ints(1, 2), { frame: 4 }), /does not agree with itself \(mono 16-bit integer audio at 24000 Hz, 4 bytes a frame\)/); + refused(shaped(ints(1, 2)).subarray(0, 30), /broken fmt chunk/); + const short = shaped(ints(1, 2), { extensible: 1 }); + short.writeUInt32LE(16, 16); + refused(short, /broken fmt chunk/); + // Samples that stop inside a frame, none, and a file that is no WAV. + refused(shaped(Buffer.alloc(5), { channels: 2 }), /Speaker returned invalid PCM audio/); + refused(shaped(Buffer.alloc(0)).subarray(0, 36), /Speaker returned no audio data/); + refused(Buffer.from('RIFF....WAVX'), /Speaker returned invalid WAV audio/); + refused(Buffer.from('not a wav'), /Speaker returned invalid WAV audio/); + // A chunk that claims more than the file holds does not carry the reader past its end. + const long = shaped(ints(1, 2)); + long.writeUInt32LE(0xfffffff0, 16); + refused(long, /broken fmt chunk/); + const skipped = Buffer.concat([Buffer.from('RIFF\0\0\0\0WAVE', 'ascii'), Buffer.from('junk'), Buffer.from([0xf0, 0xff, 0xff, 0xff])]); + refused(skipped, /Speaker returned no audio data/); }); test('recognition alone: nothing is spoken, and dictation still has its transcription', async () => { From fddc17d122399c7438833e430ea4506d21ad627b Mon Sep 17 00:00:00 2001 From: Piggidragon Date: Thu, 5 Nov 2026 16:48:43 +0100 Subject: [PATCH 4/4] Fix what the first review of the speech rates found Voice servers refuses raw PCM that says it has more than one channel, as the core does for a direct provider, instead of dropping the parameter and playing it as mono at half speed. A WAV answer is counted as it is read and let go of at 64 MB, also where the source says no length. The time stretch compares and searches more coarsely at higher rates (then refines), so the work for a second of speech stays the same at 192 kHz as at 24 kHz. Co-Authored-By: Claude Opus 5.5 --- core/server/src/api/voice.ts | 26 +++++++++++-- core/server/test/voice-providers.test.mjs | 1 + core/web/src/time-stretch.ts | 38 +++++++++++++------ docs/guide/voice.md | 1 + .../voice-servers/src/server/provider.ts | 6 +++ .../voice-servers/test/provider.test.mts | 7 ++++ tests/fixtures/extensions/speaker/server.js | 12 +++++- tests/time-stretch.test.mts | 18 +++++++++ 8 files changed, 94 insertions(+), 15 deletions(-) diff --git a/core/server/src/api/voice.ts b/core/server/src/api/voice.ts index 5769b2007..f7cdf9fe6 100644 --- a/core/server/src/api/voice.ts +++ b/core/server/src/api/voice.ts @@ -86,6 +86,28 @@ function withInstructions { + const tooMuch = () => new Error(`${label} answered more than ${MAX_WAV_BYTES >> 20} MB of audio`); + if (Number(answer.headers.get("content-length")) > MAX_WAV_BYTES) throw tooMuch(); + if (!answer.body) return Buffer.alloc(0); + const reader = answer.body.getReader(); + const pieces: Uint8Array[] = []; + let bytes = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + bytes += value.length; + if (bytes > MAX_WAV_BYTES) throw tooMuch(); + pieces.push(value); + } + } finally { await reader.cancel().catch(() => {}); reader.releaseLock(); } + return Buffer.concat(pieces); +} /** * The rate of raw PCM, which has no header of its own: what the source says in an `x-sample-rate` header, else in a * `rate=` parameter of the content type, else 24 kHz, as a source that says nothing has always been played. Throws @@ -321,9 +343,7 @@ export function voiceRouter(): Router { const headers = { "Content-Type": "audio/pcm", "X-Sample-Format": "s16le", "Cache-Control": "no-store", "X-Accel-Buffering": "no" }; // A WAV is read whole and checked, and its samples sent on at the rate its header gives. if (/^audio\/(wav|x-wav|wave|vnd\.wave)\b/.test(type)) { - if (Number(answer.headers.get("content-length")) > MAX_WAV_BYTES) throw new Error(`${label} answered more than ${MAX_WAV_BYTES >> 20} MB of audio`); - const wav = Buffer.from(await answer.arrayBuffer()); - if (wav.length > MAX_WAV_BYTES) throw new Error(`${label} answered more than ${MAX_WAV_BYTES >> 20} MB of audio`); + const wav = await readWav(answer, label); const { rate, pcm } = wavAudio(wav, label); return res.set({ ...headers, "X-Sample-Rate": String(rate), "Server-Timing": `tts_headers;dur=${(performance.now() - started).toFixed(1)}` }).send(pcm); } diff --git a/core/server/test/voice-providers.test.mjs b/core/server/test/voice-providers.test.mjs index 7153320fe..b2b416ad8 100644 --- a/core/server/test/voice-providers.test.mjs +++ b/core/server/test/voice-providers.test.mjs @@ -187,6 +187,7 @@ test("what voice mode cannot play is a 502 that names the source, and what it an ["pcm300k", "Speaker answered audio at 300000 Hz: voice mode plays 8000 to 192000 Hz", 1], ["pcmStereo", "Speaker answered audio with 2 channels: voice mode plays mono", 1], ["wav24bit", "Expected mono or stereo 16-bit integer or 32-bit float audio from Speaker, got mono 24-bit integer audio at 44100 Hz", 1], + ["wavEndless", "Speaker answered more than 64 MB of audio", 1], ["text", "Speaker answered text/plain: voice mode plays audio/pcm or audio/wav", 1], ["notOk", "Speaker answered HTTP 500", 1], ["notResponse", "Speaker gave no audio", 0], diff --git a/core/web/src/time-stretch.ts b/core/web/src/time-stretch.ts index 500e8139b..8cc196dd5 100644 --- a/core/web/src/time-stretch.ts +++ b/core/web/src/time-stretch.ts @@ -17,11 +17,16 @@ import { DEFAULT_SAMPLE_RATE, joinSamples } from "./samples"; -/** The sizes of the pieces, in samples, for one sample rate: 40 ms a piece, laid down every 20, lined up within 10 either way. */ +/** + * The sizes of the pieces, in samples, for one sample rate: 40 ms a piece, laid down every 20, lined up within 10 either + * way. Lining up compares samples `compare` apart and tries positions `search` apart (then every one around the best), + * which keeps the work for a second of speech the same at any rate: 24 kHz is searched as it always was. + */ function sizes(sampleRate: number) { const hop = Math.round(sampleRate * 0.02); + const search = Math.max(1, Math.round(sampleRate / 24000)); // Half a frame is what a window is overlapped by, so the frame is two of them. - return { frame: hop * 2, hop, tolerance: Math.round(sampleRate * 0.01) }; + return { frame: hop * 2, hop, tolerance: Math.round(sampleRate * 0.01), search, compare: 2 * search }; } /** A periodic Hann window: two of them half a frame apart add up to exactly one. */ @@ -37,7 +42,7 @@ function windowOf(frame: number): Float32Array { } export class TimeStretch { - private readonly size: { frame: number; hop: number; tolerance: number }; + private readonly size: ReturnType; private readonly window: Float32Array; /** Input so far, after half a frame of silence (so the first piece does not fade in). */ private input: Float32Array; @@ -76,19 +81,30 @@ export class TimeStretch { this.length += samples.length; } + /** How well a piece taken from `at` goes on from what the last one would have gone on to. */ + private overlap(natural: number, at: number): number { + let sum = 0; + for (let i = 0; i < this.size.hop; i += this.size.compare) sum += this.input[natural + i] * this.input[at + i]; + return sum; + } + /** Where the next piece lines up best with what the last one would have gone on to. */ private align(ideal: number, limit: number): number { if (this.frame === 0) return 0; - const { hop, tolerance } = this.size; + const { hop, tolerance, search } = this.size; const natural = this.previous + hop; - let best = ideal, score = -Infinity; const from = Math.max(0, ideal - tolerance), to = Math.min(limit, ideal + tolerance); - for (let at = from; at <= to; at++) { - let sum = 0; - for (let i = 0; i < hop; i += 2) sum += this.input[natural + i] * this.input[at + i]; - if (sum > score) { score = sum; best = at; } - } - return best; + const bestOf = (first: number, last: number, step: number, fallback: number) => { + let best = fallback, score = -Infinity; + for (let at = first; at <= last; at += step) { + const sum = this.overlap(natural, at); + if (sum > score) { score = sum; best = at; } + } + return best; + }; + // Coarse over the whole range, then every position a step either way of the best. + const coarse = bestOf(from, to, search, ideal); + return search === 1 ? coarse : bestOf(Math.max(from, coarse - search + 1), Math.min(to, coarse + search - 1), 1, coarse); } /** Lay down every piece the input so far is enough for. */ diff --git a/docs/guide/voice.md b/docs/guide/voice.md index bc0311b23..33625f35c 100644 --- a/docs/guide/voice.md +++ b/docs/guide/voice.md @@ -568,6 +568,7 @@ These are the formats a speech server may answer in: samples or more than two channels say, is refused, and voice mode names what the server sent, as in *Expected mono or stereo 16-bit integer or 32-bit float audio from Voice servers, got mono 24-bit integer audio at 44100 Hz*. + A WAV is read whole, up to 64 MB; a server that sends more is let go of. - **Raw PCM** (mono, signed 16-bit little-endian), streamed. It has no header, so its rate is the one the server says, in an `x-sample-rate` header or a `rate=` parameter of its content type (`audio/pcm;rate=44100`). A server that diff --git a/extensions/voice-servers/src/server/provider.ts b/extensions/voice-servers/src/server/provider.ts index 25ec99029..97c929f87 100644 --- a/extensions/voice-servers/src/server/provider.ts +++ b/extensions/voice-servers/src/server/provider.ts @@ -93,6 +93,12 @@ async function speak(text: string, { voice, signal }: { voice?: string; signal: await upstream.body?.cancel(); throw new Error(`The speech server answered ${type || "no type"}, not the PCM Breeze streams`); } + // Only the rate is passed on, so the core cannot see a channel count of the type: mono is checked here. + const channels = /;\s*channels\s*=\s*"?([^";\s]*)/i.exec(type)?.[1]; + if (channels !== undefined && channels !== "1") { + await upstream.body?.cancel(); + throw new Error(`The speech server answered audio with ${channels} channels: voice mode plays mono`); + } // Voice mode checks that it is a rate it plays. const rate = upstream.headers.get("x-sample-rate") ?? /;\s*rate\s*=\s*"?([^";\s]*)/i.exec(type)?.[1] ?? String(s.rawSampleRate ?? RAW_RATE_DEFAULT); return new Response(upstream.body, { headers: { "content-type": "audio/pcm", "x-sample-rate": rate } }); diff --git a/extensions/voice-servers/test/provider.test.mts b/extensions/voice-servers/test/provider.test.mts index ba6dd8e95..0b9e2f819 100644 --- a/extensions/voice-servers/test/provider.test.mts +++ b/extensions/voice-servers/test/provider.test.mts @@ -351,6 +351,13 @@ test("raw PCM reaches voice mode at its rate: streamed as 44.1 kHz from the serv upstream.answers.set("/v1/audio/speech", { type: "audio/pcm", headers: { "x-sample-rate": "300000" }, body: SAMPLES }); const tooFast = await speech("Hello"); assert.deepEqual([tooFast.status, (await tooFast.json()).error], [502, "Voice servers answered audio at 300000 Hz: voice mode plays 8000 to 192000 Hz"]); + // Stereo raw PCM would play as mono at twice the length: the channel count is not passed on, so it is refused here. + for (const type of ["audio/pcm;rate=44100;channels=2", 'audio/pcm; channels="2"']) { + upstream.answers.set("/v1/audio/speech", { type, body: SAMPLES }); + const stereo = await speech("Hello"); + assert.deepEqual([stereo.status, (await stereo.json()).error], [502, "The speech server answered audio with 2 channels: voice mode plays mono"], type); + } + assert.deepEqual(await rateOf({ type: "audio/pcm;rate=44100;channels=1" }), [200, "audio/pcm", "44100", SAMPLES]); }); test("Breeze busy with a phrase before is asked again until it answers, and an answer that is not PCM is refused", async () => { diff --git a/tests/fixtures/extensions/speaker/server.js b/tests/fixtures/extensions/speaker/server.js index 2e7ceab0f..39617addd 100644 --- a/tests/fixtures/extensions/speaker/server.js +++ b/tests/fixtures/extensions/speaker/server.js @@ -14,7 +14,7 @@ * raw PCM at 44100 Hz that says so in its content type, streamed in two pieces, * "pcm300k": raw PCM at a rate no one plays, "pcmStereo": raw PCM that says it * has two channels, "wav": 24 kHz, "wav22k", "wav44k", "wav48kStereoFloat" and - * "wav24bit" WAVs, "text", "notOk", "notResponse", "throws", or "hangs", which + * "wav24bit" WAVs, "wavEndless": a WAV that streams on without end and without a length, "text", "notOk", "notResponse", "throws", or "hangs", which * `transcribe` does too), whether `state` throws, what * it answers in place of its own (null for its own), and whether `use(true)` fails * or never settles @@ -138,6 +138,16 @@ export default function (portal) { return audio("audio/wav", wav({ rate: 48000, float: true, channels: 2 })); case "wav24bit": return audio("audio/wav", wav({ rate: 44100, bits: 24 })); + case "wavEndless": { + // A WAV body that goes on for ever and says no length, one megabyte at a time, read only as it is asked for. + const stream = new ReadableStream({ pull: (controller) => controller.enqueue(new Uint8Array(1 << 20)) }); + const cancel = stream.cancel.bind(stream); + stream.cancel = (reason) => { + asked.cancelled++; + return cancel(reason); + }; + return new Response(stream, { headers: { "content-type": "audio/wav" } }); + } case "text": return audio("text/plain", Buffer.from("not audio")); case "notOk": diff --git a/tests/time-stretch.test.mts b/tests/time-stretch.test.mts index 624110f51..739e27f76 100644 --- a/tests/time-stretch.test.mts +++ b/tests/time-stretch.test.mts @@ -106,3 +106,21 @@ test('the pieces are as long in time at every sample rate, so streamed in parts assert.ok(stepped.every((v, i) => Math.abs(v - whole[i]) < 1e-6)); assert.throws(() => new TimeStretch(1.5, 0), /sample rate/); }); + +test('the work for a second of speech does not grow with the sample rate', () => { + /** The fastest of a few runs, so a busy machine does not tip it. */ + const best = (sampleRate: number) => { + const input = toneAt(sampleRate, 2, 120); + let fastest = Infinity; + for (let run = 0; run < 3; run++) { + const start = performance.now(); + stretch(input, 1.5, sampleRate); + fastest = Math.min(fastest, performance.now() - start); + } + return fastest; + }; + best(24000); + // Compared at every sample of a piece, 192 kHz takes 64 times as long as 24 kHz; held to a few times it keeps up on a phone. + const ratio = best(192000) / best(24000); + assert.ok(ratio < 8, `192 kHz takes ${ratio.toFixed(1)} times as long as 24 kHz`); +});