Voice: keep abbreviations inside a spoken sentence, and play speech at the speech server's own sample rate

core/server/src/api/voice.ts

@@ -6,6 +6,7 @@ import { getSession, getSetting, putSetting, setVoiceSpeaks } from "../db.js";
 import { agentFor } from "../project-agent.js";
 import { setVoiceChooser, voiceProviders, within } from "../extension-host.js";
 import type { ProvidedVoice, VoiceProvider } from "@pithagoras/extension-api/server";
+import { DEFAULT_SAMPLE_RATE, MAX_SAMPLE_RATE, MIN_SAMPLE_RATE, playableRate, wavAudio } from "../wav.js";
 
 const DEFAULT_VAD = { positiveSpeechThreshold: 0.65, negativeSpeechThreshold: 0.35, minSpeechMs: 256, preSpeechPadMs: 320, redemptionMs: 1000 };
 export interface VoiceConfig {
@@ -83,31 +84,45 @@ function withInstructions<T extends { responseInstructions?: string; skipThinkin
   return { ...value, responseInstructions: voiceInstructions(value.responseInstructions), defaultResponseInstructions: DEFAULT_VOICE_INSTRUCTIONS, responseInstructionsOff: !voiceRulesOn(),
     skipThinkingProviders: value.skipThinkingProviders ?? [...DEFAULT_SKIP_THINKING_PROVIDERS], defaultSkipThinkingProviders: [...DEFAULT_SKIP_THINKING_PROVIDERS] };
 }
-/** The samples of a RIFF/WAVE buffer, checked to be what the player expects. `label` is what answered with it. */
-export function wavPcm(wav: Buffer, label: string): Buffer {
-  if (wav.length < 44 || wav.toString("ascii", 0, 4) !== "RIFF" || wav.toString("ascii", 8, 12) !== "WAVE")
-    throw new Error(`${label} returned invalid WAV audio`);
-  let format = false;
-  for (let at = 12; at + 8 <= wav.length;) {
-    const id = wav.toString("ascii", at, at + 4), size = wav.readUInt32LE(at + 4);
-    if (id === "fmt ") {
-      if (size < 16 || at + 24 > wav.length || wav.readUInt16LE(at + 8) !== 1 || wav.readUInt16LE(at + 10) !== 1 || wav.readUInt32LE(at + 12) !== 24000 || wav.readUInt16LE(at + 22) !== 16)
-        throw new Error(`Expected mono 24 kHz 16-bit audio from ${label}`);
-      format = true;
-    }
-    if (id === "data") {
-      // Samples are only meaningful once the fmt chunk has described them.
-      if (!format) throw new Error(`Expected mono 24 kHz 16-bit audio from ${label}`);
-      // A writer that does not know the length up front leaves a placeholder
-      // size behind. The response is fully buffered, so its end is the truth.
-      const end = size && at + 8 + size <= wav.length ? at + 8 + size : wav.length;
-      const pcm = wav.subarray(at + 8, end);
-      if (!pcm.length || pcm.length % 2) throw new Error(`${label} returned invalid PCM audio`);
-      return pcm;
+/** The largest WAV answer read whole: minutes of speech at the highest rate, far more than a phrase of 600 characters. */
+const MAX_WAV_BYTES = 64 * 1024 * 1024;
+/**
+ * A WAV answer read whole, but counted as it comes: a source that streams on without end (and says no length) is let go
+ * of at the limit, not buffered until the portal runs out of memory.
+ */
+async function readWav(answer: Response, label: string): Promise<Buffer> {
+  const tooMuch = () => new Error(`${label} answered more than ${MAX_WAV_BYTES >> 20} MB of audio`);
+  if (Number(answer.headers.get("content-length")) > MAX_WAV_BYTES) throw tooMuch();
+  if (!answer.body) return Buffer.alloc(0);
+  const reader = answer.body.getReader();
+  const pieces: Uint8Array[] = [];
+  let bytes = 0;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      bytes += value.length;
+      if (bytes > MAX_WAV_BYTES) throw tooMuch();
+      pieces.push(value);
     }
-    at += 8 + size + (size % 2);
-  }
-  throw new Error(`${label} returned no audio data`);
+  } finally { await reader.cancel().catch(() => {}); reader.releaseLock(); }
+  return Buffer.concat(pieces);
+}
+/**
+ * The rate of raw PCM, which has no header of its own: what the source says in an `x-sample-rate` header, else in a
+ * `rate=` parameter of the content type, else 24 kHz, as a source that says nothing has always been played. Throws
+ * where it is not a rate voice mode plays, or the PCM is not mono.
+ */
+function pcmRate(headers: Headers, label: string): number {
+  const type = headers.get("content-type") ?? "";
+  const param = (name: string) => new RegExp(`;\\s*${name}\\s*=\\s*"?([^";\\s]*)`, "i").exec(type)?.[1];
+  const channels = param("channels");
+  if (channels !== undefined && channels !== "1") throw new Error(`${label} answered audio with ${channels} channels: voice mode plays mono`);
+  const said = headers.get("x-sample-rate") ?? param("rate");
+  if (said === null || said === undefined) return DEFAULT_SAMPLE_RATE;
+  const rate = /^\d{1,7}$/.test(said.trim()) ? Number(said) : NaN;
+  if (!playableRate(rate)) throw new Error(`${label} answered audio at ${said} Hz: voice mode plays ${MIN_SAMPLE_RATE} to ${MAX_SAMPLE_RATE} Hz`);
+  return rate;
 }
 /** Voice mode listens and speaks through an extension that is not running, or offers nothing now: said, rather than anything else used. */
 class SourceAbsent extends Error {
@@ -325,18 +340,18 @@ export function voiceRouter(): Router {
       answer = given;
       if (!answer.ok) throw new Error(`${label} answered HTTP ${answer.status}`);
       const type = answer.headers.get("content-type") ?? "";
-      const headers = { "Content-Type": "audio/pcm", "X-Sample-Rate": "24000", "X-Sample-Format": "s16le", "Cache-Control": "no-store", "X-Accel-Buffering": "no" };
-      // A WAV is read whole and checked, and its samples sent on.
+      const headers = { "Content-Type": "audio/pcm", "X-Sample-Format": "s16le", "Cache-Control": "no-store", "X-Accel-Buffering": "no" };
+      // A WAV is read whole and checked, and its samples sent on at the rate its header gives.
       if (/^audio\/(wav|x-wav|wave|vnd\.wave)\b/.test(type)) {
-        const pcm = wavPcm(Buffer.from(await answer.arrayBuffer()), label);
-        return res.set({ ...headers, "Server-Timing": `tts_headers;dur=${(performance.now() - started).toFixed(1)}` }).send(pcm);
+        const wav = await readWav(answer, label);
+        const { rate, pcm } = wavAudio(wav, label);
+        return res.set({ ...headers, "X-Sample-Rate": String(rate), "Server-Timing": `tts_headers;dur=${(performance.now() - started).toFixed(1)}` }).send(pcm);
       }
       if (!/^audio\/pcm\b/.test(type)) throw new Error(`${label} answered ${type || "no type"}: voice mode plays audio/pcm or audio/wav`);
       // PCM is passed on as it comes, checked before anything of it is sent.
-      const rate = answer.headers.get("x-sample-rate");
-      if (rate !== null && rate !== "24000") throw new Error(`${label} answered audio at ${rate} Hz: voice mode plays 24 kHz`);
+      const rate = pcmRate(answer.headers, label);
       if (!answer.body) throw new Error(`${label} gave no audio`);
-      res.set({ ...headers, "Server-Timing": `tts_headers;dur=${(performance.now() - started).toFixed(1)}` });
+      res.set({ ...headers, "X-Sample-Rate": String(rate), "Server-Timing": `tts_headers;dur=${(performance.now() - started).toFixed(1)}` });
       res.flushHeaders();
       const reader = answer.body.getReader();
       let bytes = 0;

core/server/src/wav.ts

@@ -0,0 +1,87 @@
+/** The sample rates voice mode plays, in Hz: what a speech server may answer at. */
+export const MIN_SAMPLE_RATE = 8000;
+export const MAX_SAMPLE_RATE = 192000;
+/** The rate of speech that does not say its own: raw audio from a source that names none. */
+export const DEFAULT_SAMPLE_RATE = 24000;
+
+/** WAVE_FORMAT_* codes of the fmt chunk. */
+const FORMAT_PCM = 1, FORMAT_FLOAT = 3, FORMAT_EXTENSIBLE = 0xfffe;
+
+/** Mono 16-bit little-endian samples and the rate they are played at. */
+export interface Audio { rate: number; pcm: Buffer }
+
+/** Whether a rate is one voice mode plays: whole hertz from 8 to 192 kHz. */
+export const playableRate = (rate: number) => Number.isInteger(rate) && rate >= MIN_SAMPLE_RATE && rate <= MAX_SAMPLE_RATE;
+
+/** What a fmt chunk says, as words for an error: "stereo 24-bit integer audio at 44100 Hz". */
+function described(tag: number, channels: number, bits: number, rate: number): string {
+  const kind = tag === FORMAT_PCM ? "integer" : tag === FORMAT_FLOAT ? "float" : `format ${tag}`;
+  const layout = channels === 1 ? "mono" : channels === 2 ? "stereo" : `${channels}-channel`;
+  return `${layout} ${bits}-bit ${kind} audio at ${rate} Hz`;
+}
+
+/** One sample of a frame as a float in [-1, 1], from the bytes at `at`. */
+type Reader = (wav: Buffer, at: number) => number;
+const int16: Reader = (wav, at) => wav.readInt16LE(at) / 32768;
+const float32: Reader = (wav, at) => wav.readFloatLE(at);
+
+/**
+ * The samples of a RIFF/WAVE buffer as mono 16-bit little-endian PCM, with the rate the header gives. `label` is what
+ * answered with it. Read: 16-bit integer and 32-bit float samples, mono or stereo (mixed to mono), also as
+ * WAVE_FORMAT_EXTENSIBLE, at 8 to 192 kHz. Anything else is refused, with what it was in the message.
+ */
+export function wavAudio(wav: Buffer, label: string): Audio {
+  if (wav.length < 12 || wav.toString("ascii", 0, 4) !== "RIFF" || wav.toString("ascii", 8, 12) !== "WAVE")
+    throw new Error(`${label} returned invalid WAV audio`);
+  let format: { tag: number; channels: number; bits: number; rate: number; frame: number } | undefined;
+  for (let at = 12; at + 8 <= wav.length;) {
+    const id = wav.toString("ascii", at, at + 4), size = wav.readUInt32LE(at + 4), body = at + 8;
+    if (id === "fmt ") {
+      if (size < 16 || body + size > wav.length) throw new Error(`${label} returned a WAV file with a broken fmt chunk`);
+      let tag = wav.readUInt16LE(body);
+      const channels = wav.readUInt16LE(body + 2), rate = wav.readUInt32LE(body + 4), frame = wav.readUInt16LE(body + 12), bits = wav.readUInt16LE(body + 14);
+      // The extensible form keeps the format in the first two bytes of its sub-format GUID.
+      if (tag === FORMAT_EXTENSIBLE) {
+        if (size < 40) throw new Error(`${label} returned a WAV file with a broken fmt chunk`);
+        tag = wav.readUInt16LE(body + 24);
+      }
+      const got = described(tag, channels, bits, rate);
+      const readable = (tag === FORMAT_PCM && bits === 16) || (tag === FORMAT_FLOAT && bits === 32);
+      if (!readable || channels < 1 || channels > 2)
+        throw new Error(`Expected mono or stereo 16-bit integer or 32-bit float audio from ${label}, got ${got}`);
+      if (!playableRate(rate)) throw new Error(`Expected audio from ${MIN_SAMPLE_RATE} to ${MAX_SAMPLE_RATE} Hz from ${label}, got ${got}`);
+      // A frame is one sample of each channel, and its size is what the data is cut into.
+      if (frame !== channels * bits / 8) throw new Error(`${label} returned a WAV file whose fmt chunk does not agree with itself (${got}, ${frame} bytes a frame)`);
+      format = { tag, channels, bits, rate, frame };
+    }
+    if (id === "data") {
+      // Samples are only meaningful once the fmt chunk has described them.
+      if (!format) throw new Error(`${label} returned a WAV file whose samples come before its fmt chunk`);
+      // A writer that does not know the length up front leaves a placeholder
+      // size behind. The response is fully buffered, so its end is the truth.
+      const end = size && body + size <= wav.length ? body + size : wav.length;
+      const data = wav.subarray(body, end);
+      if (!data.length || data.length % format.frame) throw new Error(`${label} returned invalid PCM audio`);
+      return { rate: format.rate, pcm: mono(data, format) };
+    }
+    at = body + size + (size % 2);
+  }
+  throw new Error(`${label} returned no audio data`);
+}
+
+/** `data`, frames of `format`, as mono 16-bit little-endian samples: the same bytes where they already are. */
+function mono(data: Buffer, format: { tag: number; channels: number; bits: number; frame: number }): Buffer {
+  if (format.tag === FORMAT_PCM && format.channels === 1) return data;
+  const read = format.tag === FORMAT_FLOAT ? float32 : int16;
+  const step = format.frame / format.channels, frames = data.length / format.frame;
+  const out = Buffer.alloc(frames * 2);
+  for (let i = 0; i < frames; i++) {
+    let sum = 0;
+    for (let c = 0; c < format.channels; c++) sum += read(data, i * format.frame + c * step);
+    const value = sum / format.channels;
+    // A float outside [-1, 1] clips, and one that is not a number is silence.
+    const clipped = Number.isNaN(value) ? 0 : Math.max(-1, Math.min(1, value));
+    out.writeInt16LE(Math.round(clipped < 0 ? clipped * 32768 : clipped * 32767), i * 2);
+  }
+  return out;
+}

core/server/test/voice-providers.test.mjs

@@ -158,10 +158,36 @@ test("speech is passed on as 24 kHz PCM, as the source streams it or from the WA
   assert.equal((await asked()).cancelled, 0, "nothing played is cancelled");
 });
 
+test("speech at another rate than 24 kHz is played at its own: a WAV says it in its header, PCM in a header, in its content type, or not at all", async () => {
+  const played = async (answer, rate, samples) => {
+    const got = await speak();
+    assert.equal(got.status, 200, answer);
+    assert.deepEqual([got.headers.get("content-type"), got.headers.get("x-sample-rate"), got.headers.get("x-sample-format")], ["audio/pcm", String(rate), "s16le"], answer);
+    assert.deepEqual([...new Uint8Array(await got.arrayBuffer())], samples, answer);
+  };
+  for (const [answer, rate, samples] of [
+    // The WAVs: mono 16-bit as it is; stereo float mixed to mono (half of full scale, then silence).
+    ["wav44k", 44100, [0, 0, 255, 127]],
+    ["wav22k", 22050, [0, 0, 255, 127]],
+    ["wav48kStereoFloat", 48000, [0, 64, 0, 0]],
+    // The raw PCM: a header, a content type, and nothing, which is 24 kHz as it always was.
+    ["pcm44k", 44100, [0, 0, 255, 127]],
+    ["pcm48k", 48000, [0, 0, 255, 127]],
+    ["pcmType", 44100, [0, 0, 255, 127]],
+    ["pcmBare", 24000, [0, 0, 255, 127]],
+  ]) {
+    await set({ answer });
+    await played(answer, rate, samples);
+  }
+  await set({ answer: "pcm" });
+});
+
 test("what voice mode cannot play is a 502 that names the source, and what it answered is let go of", async () => {
   for (const [answer, error, cancels] of [
-    ["pcm48k", "Speaker answered audio at 48000 Hz: voice mode plays 24 kHz", 1],
-    ["wav22k", "Expected mono 24 kHz 16-bit audio from Speaker", 1],
+    ["pcm300k", "Speaker answered audio at 300000 Hz: voice mode plays 8000 to 192000 Hz", 1],
+    ["pcmStereo", "Speaker answered audio with 2 channels: voice mode plays mono", 1],
+    ["wav24bit", "Expected mono or stereo 16-bit integer or 32-bit float audio from Speaker, got mono 24-bit integer audio at 44100 Hz", 1],
+    ["wavEndless", "Speaker answered more than 64 MB of audio", 1],
     ["text", "Speaker answered text/plain: voice mode plays audio/pcm or audio/wav", 1],
     ["notOk", "Speaker answered HTTP 500", 1],
     ["notResponse", "Speaker gave no audio", 0],