From c9ee81ecc317b217c369cb4d8e552431ce5c2530 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Tue, 29 Sep 2026 22:22:15 -0700 Subject: [PATCH] feat(web): voice mode overlay talks to the bench's worker with browser speech (CL-9568) --- apps/web/src/app.css | 155 ++++++++++++++++++++ apps/web/src/chat/composer.tsx | 16 ++- apps/web/src/lib/icons/index.tsx | 3 + apps/web/src/pages/workbench-page.tsx | 15 ++ apps/web/src/voice/use-speech.ts | 197 ++++++++++++++++++++++++++ apps/web/src/voice/voice-overlay.tsx | 165 +++++++++++++++++++++ 6 files changed, 550 insertions(+), 1 deletion(-) create mode 100644 apps/web/src/voice/use-speech.ts create mode 100644 apps/web/src/voice/voice-overlay.tsx diff --git a/apps/web/src/app.css b/apps/web/src/app.css index 91fb1c339..ac952cca1 100644 --- a/apps/web/src/app.css +++ b/apps/web/src/app.css @@ -4535,3 +4535,158 @@ tr.insights-row-clickable:hover { width: 0.9rem; height: 0.9rem; } + +/* Voice mode over the thread. */ +.chat-composer-voice { + display: inline-flex; + align-items: center; + justify-content: center; + width: 2rem; + height: 2rem; + border: 0; + border-radius: 999px; + background: var(--hover); + color: var(--ink); + cursor: pointer; +} +.chat-composer-voice:hover { + background: var(--press); +} +.voice { + position: absolute; + inset: 0; + z-index: 25; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + background: color-mix(in srgb, var(--surface) 94%, transparent); + backdrop-filter: blur(16px); +} +.voice-orb { + position: relative; + width: 168px; + height: 168px; + display: grid; + place-items: center; +} +.voice-orb::before, +.voice-orb::after { + content: ""; + position: absolute; + inset: 0; + border-radius: 50%; + background: radial-gradient( + closest-side, + color-mix(in srgb, var(--action) 26%, transparent), + transparent + ); + transform: scale(calc(0.8 + var(--lvl, 0) * 0.5)); + transition: transform 90ms linear; +} +.voice-orb::after { + inset: 24px; + background: radial-gradient( + closest-side, + color-mix(in srgb, var(--action) 40%, transparent), + transparent + ); + transform: scale(calc(0.85 + var(--lvl, 0) * 0.3)); +} +.voice-orb > * { + position: relative; + z-index: 1; +} +.voice-state { + margin-top: 20px; + display: flex; + align-items: center; + gap: 8px; + font-size: 14px; + font-weight: 700; + color: var(--ink); +} +.voice-state svg { + color: var(--action); +} +.voice-pulse { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--action); +} +.voice-cap { + margin-top: 12px; + max-width: 52ch; + min-height: 3em; + padding: 0 24px; + text-align: center; + font-size: 17px; + line-height: 1.5; + color: var(--ink-2); + text-wrap: balance; +} +.voice-cap-hint { + color: var(--ink-3); +} +.voice-controls { + position: absolute; + bottom: 32px; + display: flex; + align-items: center; + gap: 16px; +} +.voice-btn { + width: 48px; + height: 48px; + border-radius: 50%; + border: 0; + display: grid; + place-items: center; + background: var(--hover); + color: var(--ink); + cursor: pointer; + transition: + transform 160ms var(--ease-out), + background-color 150ms ease; +} +.voice-btn:hover { + background: var(--press); +} +.voice-btn:active { + transform: scale(0.94); +} +.voice-btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} +.voice-btn[aria-pressed="true"] { + background: var(--press); + box-shadow: inset 0 0 0 1.5px var(--line-strong); +} +.voice-mic { + width: 64px; + height: 64px; + background: var(--primary); + color: var(--primary-ink); +} +.voice-mic:hover { + background: var(--primary-hover); +} +.voice-mic.muted { + background: var(--danger-wash); + color: var(--danger); +} +.voice-end { + background: var(--danger); + color: var(--danger-ink); +} +.voice-end:hover { + background: var(--danger-hover); +} +.voice-hint { + position: absolute; + bottom: 104px; + font-size: 12.5px; + color: var(--ink-3); +} diff --git a/apps/web/src/chat/composer.tsx b/apps/web/src/chat/composer.tsx index b15030484..fe4011475 100644 --- a/apps/web/src/chat/composer.tsx +++ b/apps/web/src/chat/composer.tsx @@ -1,7 +1,7 @@ // No Stop action: neither `threads-api.ts` nor the hub API expose a way to // cancel a running turn yet. import { useCommandPaletteNavigation } from "@corbits/react-ui"; -import { ArrowUp, CircleNotch } from "@/lib/icons"; +import { ArrowUp, CircleNotch, Microphone } from "@/lib/icons"; import { useRef, useState } from "react"; import { activeMention, applyMention, matchMentionQuery, type ActiveMention } from "./mentions"; @@ -19,6 +19,7 @@ export function Composer({ disabled, mentionables = [], onSend, + onVoice, }: { readonly placeholder: string; readonly busy: boolean; @@ -26,6 +27,8 @@ export function Composer({ /** Agents an `@` token can address; empty disables the popover. */ readonly mentionables?: readonly ComposerMention[]; readonly onSend: (text: string) => void; + /** Opens voice mode; the button is absent without it. */ + readonly onVoice?: () => void; }) { const [text, setText] = useState(""); const [mention, setMention] = useState(null); @@ -137,6 +140,17 @@ export function Composer({ }} />
+ {onVoice === undefined ? null : ( + + )}
@@ -298,6 +305,14 @@ function Workbench({ workbenchTenantId }: { readonly workbenchTenantId: string } /> )} + {voiceOpen ? ( + send.mutate({ content: text })} + onEnd={endVoice} + /> + ) : null} ; +}; +type Recognition = { + continuous: boolean; + interimResults: boolean; + lang: string; + onresult: ((event: RecognitionEvent) => void) | null; + onerror: ((event: { readonly error: string }) => void) | null; + onend: (() => void) | null; + start(): void; + stop(): void; + abort(): void; +}; +type RecognitionCtor = new () => Recognition; + +function recognitionCtor(): RecognitionCtor | undefined { + const w = window as unknown as { + SpeechRecognition?: RecognitionCtor; + webkitSpeechRecognition?: RecognitionCtor; + }; + return w.SpeechRecognition ?? w.webkitSpeechRecognition; +} + +export function speechSupported(): boolean { + return ( + typeof window !== "undefined" && recognitionCtor() !== undefined && "speechSynthesis" in window + ); +} + +export function useSpeech({ onFinal }: { readonly onFinal: (text: string) => void }) { + const [state, setState] = useState(() => + speechSupported() ? "idle" : "unsupported", + ); + const [interim, setInterim] = useState(""); + const [level, setLevel] = useState(0); + const [muted, setMutedState] = useState(false); + + const onFinalRef = useRef(onFinal); + onFinalRef.current = onFinal; + const recognition = useRef(null); + const stream = useRef(null); + const active = useRef(false); + const mutedRef = useRef(false); + const speaking = useRef(false); + + const listen = useCallback(() => { + const rec = recognition.current; + if (rec === null || !active.current || mutedRef.current || speaking.current) return; + try { + rec.start(); + setState("listening"); + } catch (cause) { + // start() throws when a session is already running; that is the goal. + reportError(cause, { operation: "voice_recognition_start" }); + } + }, []); + + const start = useCallback(async () => { + const Ctor = recognitionCtor(); + if (Ctor === undefined || !("speechSynthesis" in window)) { + setState("unsupported"); + return; + } + try { + stream.current = await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch (cause) { + reportError(cause, { operation: "voice_microphone_access" }); + setState("denied"); + return; + } + active.current = true; + const rec = new Ctor(); + rec.continuous = true; + rec.interimResults = true; + rec.lang = navigator.language; + rec.onresult = (event) => { + let pending = ""; + for (let i = event.resultIndex; i < event.results.length; i += 1) { + const result = event.results[i]; + if (result === undefined) continue; + if (result.isFinal) { + const text = result[0].transcript.trim(); + if (text !== "") onFinalRef.current(text); + } else { + pending += result[0].transcript; + } + } + setInterim(pending); + }; + rec.onerror = (event) => { + if (event.error === "not-allowed" || event.error === "service-not-allowed") { + active.current = false; + setState("denied"); + } + }; + // Chrome ends a continuous session on silence; keep it going. + rec.onend = () => { + setInterim(""); + listen(); + }; + recognition.current = rec; + listen(); + }, [listen]); + + const stop = useCallback(() => { + active.current = false; + const rec = recognition.current; + recognition.current = null; + if (rec !== null) { + rec.onend = null; + rec.abort(); + } + window.speechSynthesis.cancel(); + speaking.current = false; + stream.current?.getTracks().forEach((track) => track.stop()); + stream.current = null; + setInterim(""); + setLevel(0); + setState((prev) => (prev === "unsupported" || prev === "denied" ? prev : "idle")); + }, []); + + const setMuted = useCallback( + (next: boolean) => { + mutedRef.current = next; + setMutedState(next); + stream.current?.getAudioTracks().forEach((track) => { + track.enabled = !next; + }); + if (next) recognition.current?.abort(); + else listen(); + }, + [listen], + ); + + const speak = useCallback( + (text: string) => { + if (!active.current || text.trim() === "") return; + speaking.current = true; + recognition.current?.abort(); + setState("speaking"); + const utterance = new SpeechSynthesisUtterance(text); + const done = () => { + speaking.current = window.speechSynthesis.speaking; + if (!speaking.current) listen(); + }; + utterance.onend = done; + utterance.onerror = done; + window.speechSynthesis.speak(utterance); + }, + [listen], + ); + + // Mic level for the orb glow, 0..1. + useEffect(() => { + if (state === "idle" || state === "unsupported" || state === "denied") return; + const current = stream.current; + if (current === null) return; + const ctx = new AudioContext(); + const analyser = ctx.createAnalyser(); + analyser.fftSize = 256; + ctx.createMediaStreamSource(current).connect(analyser); + const data = new Uint8Array(analyser.frequencyBinCount); + let frame = 0; + let last = 0; + const tick = () => { + analyser.getByteTimeDomainData(data); + let peak = 0; + for (const v of data) peak = Math.max(peak, Math.abs(v - 128)); + const next = Math.min(1, (peak / 128) * 2); + if (Math.abs(next - last) > 0.03) { + last = next; + setLevel(next); + } + frame = requestAnimationFrame(tick); + }; + tick(); + return () => { + cancelAnimationFrame(frame); + void ctx.close(); + }; + // Rebuild only when the stream appears (idle -> active), not on listen/speak flips. + }, [state === "idle" || state === "unsupported" || state === "denied"]); + + useEffect(() => stop, [stop]); + + return { state, interim, level, muted, start, stop, setMuted, speak }; +} diff --git a/apps/web/src/voice/voice-overlay.tsx b/apps/web/src/voice/voice-overlay.tsx new file mode 100644 index 000000000..80fd29c5a --- /dev/null +++ b/apps/web/src/voice/voice-overlay.tsx @@ -0,0 +1,165 @@ +// Voice mode: covers the thread, sends each final utterance as an ordinary +// message, and speaks the worker's replies. Browser speech only. +import { useEffect, useRef, useState, type CSSProperties } from "react"; + +import { CorbitAvatar, avatarColorForPrincipal } from "@/chat/avatar"; +import { Captions, Microphone, MicrophoneSlash, SpeakerHigh, X } from "@/lib/icons"; +import type { WorkbenchMessage } from "@/chat/threads-api"; +import { useSpeech } from "./use-speech"; + +// Replies are markdown; speech should not read the syntax aloud. +function spoken(markdown: string): string { + return markdown + .replace(/```[\s\S]*?```/g, " ") + .replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/[`*_#>~|]/g, "") + .replace(/\s+/g, " ") + .trim(); +} + +export function VoiceOverlay({ + worker, + messages, + onSend, + onEnd, +}: { + readonly worker: { readonly id: string; readonly name: string } | undefined; + readonly messages: readonly WorkbenchMessage[]; + readonly onSend: (text: string) => void; + readonly onEnd: () => void; +}) { + const [captions, setCaptions] = useState(true); + const [heard, setHeard] = useState(""); + const [reply, setReply] = useState(""); + const speech = useSpeech({ + onFinal: (text) => { + setHeard(text); + setReply(""); + onSend(text); + }, + }); + const { start, speak, setMuted } = speech; + + // Only replies that arrive while voice is open get spoken. + const seen = useRef | null>(null); + if (seen.current === null) seen.current = new Set(messages.map((message) => message.id)); + useEffect(() => { + const fresh = messages.filter((m) => m.author === "other" && !seen.current?.has(m.id)); + for (const message of messages) seen.current?.add(message.id); + if (fresh.length === 0) return; + const text = spoken(fresh.map((m) => m.body).join("\n")); + setReply(text); + speak(text); + }, [messages, speak]); + + useEffect(() => { + void start(); + }, [start]); + + useEffect(() => { + const onKey = (event: KeyboardEvent) => { + if (event.key === "Escape") { + event.preventDefault(); + onEnd(); + } + if (event.key === " " && !(event.target instanceof HTMLButtonElement)) { + event.preventDefault(); + setMuted(!speech.muted); + } + }; + document.addEventListener("keydown", onKey); + return () => document.removeEventListener("keydown", onKey); + }, [onEnd, setMuted, speech.muted]); + + const name = worker?.name ?? "the worker"; + const blocked = speech.state === "unsupported" || speech.state === "denied"; + const stateLine = speech.muted + ? "Muted" + : speech.state === "speaking" + ? `${name} is speaking` + : "Listening…"; + const caption = + speech.state === "speaking" ? reply : speech.interim !== "" ? speech.interim : heard; + + return ( +
+
+ +
+ {blocked ? ( +

+ {speech.state === "unsupported" + ? "Voice needs speech recognition, which this browser doesn't offer. Try Chrome, Edge, or Safari." + : "Microphone access was blocked. Allow it in your browser's site settings, then try again."} +

+ ) : ( + <> +
+ {speech.muted ? ( +
+
+ {caption === "" ? ( + Say anything to {name}. + ) : ( + caption + )} +
+
Space to mute · Esc to end
+ + )} +
+ + + +
+
+ ); +}