From 655230d9304502015bbd0640ef198e7ff61f554e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Tue, 20 Oct 2026 15:12:03 +0200 Subject: [PATCH 1/6] feat(agents): add OpenAI Realtime SDK example (Next.js) Ported from elevenlabs/elevenlabs-dx#2577. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + agents/nextjs/openai-realtime/PROMPT.md | 46 ++ agents/nextjs/openai-realtime/README.md | 41 ++ .../openai-realtime/example/.env.example | 1 + .../nextjs/openai-realtime/example/README.md | 41 ++ .../example/app/api/token/route.ts | 33 ++ .../openai-realtime/example/app/favicon.ico | Bin 0 -> 25931 bytes .../openai-realtime/example/app/globals.css | 126 ++++ .../openai-realtime/example/app/layout.tsx | 34 ++ .../openai-realtime/example/app/page.tsx | 153 +++++ .../openai-realtime/example/components.json | 23 + .../example/components/ui/live-waveform.tsx | 560 ++++++++++++++++++ .../openai-realtime/example/eslint.config.mjs | 18 + .../example/hooks/use-realtime-session.ts | 421 +++++++++++++ .../example/lib/realtime-session.ts | 27 + .../openai-realtime/example/lib/utils.ts | 6 + .../openai-realtime/example/next.config.ts | 13 + .../openai-realtime/example/package.json | 36 ++ .../example/postcss.config.mjs | 7 + .../openai-realtime/example/public/file.svg | 1 + .../openai-realtime/example/public/globe.svg | 1 + .../openai-realtime/example/public/next.svg | 1 + .../openai-realtime/example/public/vercel.svg | 1 + .../openai-realtime/example/public/window.svg | 1 + .../openai-realtime/example/tsconfig.json | 34 ++ agents/nextjs/openai-realtime/setup.sh | 43 ++ 26 files changed, 1669 insertions(+) create mode 100644 agents/nextjs/openai-realtime/PROMPT.md create mode 100644 agents/nextjs/openai-realtime/README.md create mode 100644 agents/nextjs/openai-realtime/example/.env.example create mode 100644 agents/nextjs/openai-realtime/example/README.md create mode 100644 agents/nextjs/openai-realtime/example/app/api/token/route.ts create mode 100644 agents/nextjs/openai-realtime/example/app/favicon.ico create mode 100644 agents/nextjs/openai-realtime/example/app/globals.css create mode 100644 agents/nextjs/openai-realtime/example/app/layout.tsx create mode 100644 agents/nextjs/openai-realtime/example/app/page.tsx create mode 100644 agents/nextjs/openai-realtime/example/components.json create mode 100644 agents/nextjs/openai-realtime/example/components/ui/live-waveform.tsx create mode 100644 agents/nextjs/openai-realtime/example/eslint.config.mjs create mode 100644 agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts create mode 100644 agents/nextjs/openai-realtime/example/lib/realtime-session.ts create mode 100644 agents/nextjs/openai-realtime/example/lib/utils.ts create mode 100644 agents/nextjs/openai-realtime/example/next.config.ts create mode 100644 agents/nextjs/openai-realtime/example/package.json create mode 100644 agents/nextjs/openai-realtime/example/postcss.config.mjs create mode 100644 agents/nextjs/openai-realtime/example/public/file.svg create mode 100644 agents/nextjs/openai-realtime/example/public/globe.svg create mode 100644 agents/nextjs/openai-realtime/example/public/next.svg create mode 100644 agents/nextjs/openai-realtime/example/public/vercel.svg create mode 100644 agents/nextjs/openai-realtime/example/public/window.svg create mode 100644 agents/nextjs/openai-realtime/example/tsconfig.json create mode 100755 agents/nextjs/openai-realtime/setup.sh diff --git a/README.md b/README.md index 84b5aa03..a56ba175 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Shared base templates live in `templates/` (Expo, Next.js, Python, TypeScript). - [Speech Engine Quickstart (Next.js)](speech-engine/nextjs/quickstart/example/README.md) — Add voice to your own LLM-backed chat agent with a Speech Engine WebSocket server and Next.js client. - [Real-Time Voice Agent (Next.js)](agents/nextjs/quickstart/example/README.md) — Live voice conversations with the ElevenLabs Agents Platform using the React Agents SDK. - [Voice Agent Guardrails Demo (Next.js)](agents/nextjs/guardrails/example/README.md) — Demonstrate custom guardrails and the `guardrail_triggered` client event in a live voice agent. +- [OpenAI Realtime SDK with ElevenLabs (Next.js)](agents/nextjs/openai-realtime/example/README.md) — Talk to ElevenLabs Realtime using OpenAI's Realtime SDK pointed at the ElevenLabs OpenAI-compatible endpoint. - [Voice Isolator (Next.js)](voice-isolator/nextjs/quickstart/example/README.md) — Record your voice in the browser and remove background noise with the Voice Isolator API. - [Dubbing Recorder (Next.js)](dubbing/nextjs/quickstart/example/README.md) — Record your voice in the browser, dub it into another language, and play or download the result. diff --git a/agents/nextjs/openai-realtime/PROMPT.md b/agents/nextjs/openai-realtime/PROMPT.md new file mode 100644 index 00000000..f749337a --- /dev/null +++ b/agents/nextjs/openai-realtime/PROMPT.md @@ -0,0 +1,46 @@ +Before writing any code, invoke the `/agents` skill to learn the correct ElevenLabs SDK patterns. + +Build a voice demo that talks to ElevenLabs Realtime through OpenAI's Realtime SDK (`@openai/agents-realtime`), pointed at the ElevenLabs OpenAI-compatible endpoint instead of `api.openai.com`. + +## 1. `package.json` + +- Add `@elevenlabs/elevenlabs-js`, `@openai/agents-realtime`, and `wavtools` dependencies. + +## 2. `lib/realtime-session.ts` + +Shared constants used by both the server route and the browser. + +- Export `REALTIME_URL` (`wss://api.elevenlabs.io/v1/convai/realtime`), `REALTIME_MODEL` (`eleven_realtime_v1_mini`), `REALTIME_VOICE` (`brian`), `AUDIO_SAMPLE_RATE` (`24000`), and short assistant instructions. +- Export a `sessionConfig` typed as `ElevenLabs.OpenAiSessionConfigInput`: `type: "realtime"`, the model and instructions, `outputModalities: ["audio"]`, `audio.input` with PCM format at 24kHz and `semantic_vad` turn detection, and `audio.output` with PCM format and the voice. +- Use the GA session shape (`outputModalities`, nested `audio.input`/`audio.output`). Do not use the deprecated top-level `modalities` or `voice` fields. + +## 3. `app/api/token/route.ts` + +Secure GET endpoint that mints a short-lived realtime client secret. Never expose `ELEVENLABS_API_KEY` to the client. + +- Use `client.conversationalAi.realtime.createClientSecret({ session: sessionConfig, expiresAfter: { anchor: "created_at", seconds: 600 } })`. +- Return only `{ value }` to the browser. +- Return a JSON `{ error }` with a sensible status when the key is missing or the request fails. + +## 4. `hooks/use-realtime-session.ts` + +Client hook that owns the realtime session. + +- Fetch `/api/token`, then create a `RealtimeAgent` and a `RealtimeSession` with an `OpenAIRealtimeWebSocket` transport using `REALTIME_URL`, the model, and the GA config (`outputModalities`, `audio.output.voice`). Connect with `session.connect({ apiKey: value })`. +- The SDK only wires audio for WebRTC. Capture and play PCM16 at 24kHz with `wavtools` (`WavRecorder`, `WavStreamPlayer`). Import `wavtools` dynamically inside `start` because it touches browser APIs at module load. +- Stream mic audio continuously with `session.sendAudio`; `wavtools` types `data.mono` as `Int16Array` but it is already an `ArrayBuffer` at runtime, so pass it through as-is. +- Play `audio` events via `add16BitPCM`, and interrupt playback on `audio_interrupted`. +- Track user speaking (`input_audio_buffer.speech_started/stopped`), agent speaking (`audio_start`/`audio_stopped`), response in flight (`response.created`/`response.done`), and the negotiated session (`session.created`/`session.updated`) from `transport_event`. +- Build the transcript from raw transport events: stream assistant text from `response.output_audio_transcript.delta`, take user text from `conversation.item.input_audio_transcription.completed`, finalize on `.done` or `response.done`, and order turns by `previous_item_id`. +- Keep a capped log of transport events, skipping the high-frequency audio and transcript deltas. +- Tear down recorder, player, and session on stop or error. + +## 5. `app/page.tsx` + +Minimal voice-first page. + +- A Start/Stop toggle and connection status. +- While connected, show independent indicators for user speaking, agent speaking, and responding, plus the negotiated model, voice, and turn detection. +- A running transcript with a streaming caret for in-progress assistant turns and a "Transcribing…" placeholder for pending user turns. +- A collapsible list of recent transport events. +- Show errors inline and allow reconnecting. diff --git a/agents/nextjs/openai-realtime/README.md b/agents/nextjs/openai-realtime/README.md new file mode 100644 index 00000000..dcac97a6 --- /dev/null +++ b/agents/nextjs/openai-realtime/README.md @@ -0,0 +1,41 @@ +# OpenAI Realtime SDK with ElevenLabs (Next.js) + +Hold a voice conversation with ElevenLabs Realtime using OpenAI's Realtime SDK (`@openai/agents-realtime`), pointed at the ElevenLabs OpenAI-compatible endpoint instead of `api.openai.com`. See the [OpenAI Realtime API compatibility guide](https://elevenlabs.io/docs/eleven-agents/guides/openai-realtime-compatibility). + +## Setup + +1. Copy the environment file and add your API key: + + ```bash + cp .env.example .env.local + ``` + + Then set: + - `ELEVENLABS_API_KEY` + +2. Install dependencies: + + ```bash + pnpm install + ``` + +## Run + +```bash +pnpm run dev +``` + +Open [http://localhost:3000](http://localhost:3000) in your browser. + +## Usage + +- Click **Start** and allow microphone access when prompted. +- Talk; the agent replies by voice and the transcript fills in as you go. +- Expand **Transport events** to see the raw OpenAI Realtime events exchanged with ElevenLabs. +- Click **Stop** to end the session. + +## How it works + +- `app/api/token/route.ts` mints a short-lived client secret with `client.conversationalAi.realtime.createClientSecret`, so `ELEVENLABS_API_KEY` never reaches the browser. +- The browser passes that secret to `RealtimeSession.connect()` and connects `OpenAIRealtimeWebSocket` to `wss://api.elevenlabs.io/v1/convai/realtime`. +- The SDK only wires audio automatically for WebRTC, so the page captures and plays PCM16 audio at 24kHz with [`wavtools`](https://www.npmjs.com/package/wavtools). diff --git a/agents/nextjs/openai-realtime/example/.env.example b/agents/nextjs/openai-realtime/example/.env.example new file mode 100644 index 00000000..4c49a949 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/.env.example @@ -0,0 +1 @@ +ELEVENLABS_API_KEY= diff --git a/agents/nextjs/openai-realtime/example/README.md b/agents/nextjs/openai-realtime/example/README.md new file mode 100644 index 00000000..dcac97a6 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/README.md @@ -0,0 +1,41 @@ +# OpenAI Realtime SDK with ElevenLabs (Next.js) + +Hold a voice conversation with ElevenLabs Realtime using OpenAI's Realtime SDK (`@openai/agents-realtime`), pointed at the ElevenLabs OpenAI-compatible endpoint instead of `api.openai.com`. See the [OpenAI Realtime API compatibility guide](https://elevenlabs.io/docs/eleven-agents/guides/openai-realtime-compatibility). + +## Setup + +1. Copy the environment file and add your API key: + + ```bash + cp .env.example .env.local + ``` + + Then set: + - `ELEVENLABS_API_KEY` + +2. Install dependencies: + + ```bash + pnpm install + ``` + +## Run + +```bash +pnpm run dev +``` + +Open [http://localhost:3000](http://localhost:3000) in your browser. + +## Usage + +- Click **Start** and allow microphone access when prompted. +- Talk; the agent replies by voice and the transcript fills in as you go. +- Expand **Transport events** to see the raw OpenAI Realtime events exchanged with ElevenLabs. +- Click **Stop** to end the session. + +## How it works + +- `app/api/token/route.ts` mints a short-lived client secret with `client.conversationalAi.realtime.createClientSecret`, so `ELEVENLABS_API_KEY` never reaches the browser. +- The browser passes that secret to `RealtimeSession.connect()` and connects `OpenAIRealtimeWebSocket` to `wss://api.elevenlabs.io/v1/convai/realtime`. +- The SDK only wires audio automatically for WebRTC, so the page captures and plays PCM16 audio at 24kHz with [`wavtools`](https://www.npmjs.com/package/wavtools). diff --git a/agents/nextjs/openai-realtime/example/app/api/token/route.ts b/agents/nextjs/openai-realtime/example/app/api/token/route.ts new file mode 100644 index 00000000..48f7e51d --- /dev/null +++ b/agents/nextjs/openai-realtime/example/app/api/token/route.ts @@ -0,0 +1,33 @@ +import { ElevenLabsClient, ElevenLabsError } from "@elevenlabs/elevenlabs-js"; +import { NextResponse } from "next/server"; + +import { sessionConfig } from "@/lib/realtime-session"; + +export async function GET() { + const apiKey = process.env.ELEVENLABS_API_KEY?.trim(); + if (!apiKey) { + return NextResponse.json( + { error: "Server misconfiguration: ELEVENLABS_API_KEY is not set." }, + { status: 500 } + ); + } + + try { + const client = new ElevenLabsClient({ apiKey }); + const secret = await client.conversationalAi.realtime.createClientSecret({ + session: sessionConfig, + expiresAfter: { anchor: "created_at", seconds: 600 }, + }); + return NextResponse.json({ value: secret.value }); + } catch (err) { + const status = + err instanceof ElevenLabsError && err.statusCode ? err.statusCode : 502; + return NextResponse.json( + { + error: + err instanceof Error ? err.message : "Failed to mint realtime token.", + }, + { status: status >= 400 && status < 600 ? status : 502 } + ); + } +} diff --git a/agents/nextjs/openai-realtime/example/app/favicon.ico b/agents/nextjs/openai-realtime/example/app/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..718d6fea4835ec2d246af9800eddb7ffb276240c GIT binary patch literal 25931 zcmeHv30#a{`}aL_*G&7qml|y<+KVaDM2m#dVr!KsA!#An?kSQM(q<_dDNCpjEux83 zLb9Z^XxbDl(w>%i@8hT6>)&Gu{h#Oeyszu?xtw#Zb1mO{pgX9699l+Qppw7jXaYf~-84xW z)w4x8?=youko|}Vr~(D$UXIbiXABHh`p1?nn8Po~fxRJv}|0e(BPs|G`(TT%kKVJAdg5*Z|x0leQq0 zkdUBvb#>9F()jo|T~kx@OM8$9wzs~t2l;K=woNssA3l6|sx2r3+kdfVW@e^8e*E}v zA1y5{bRi+3Z`uD3{F7LgFJDdvm;nJilkzDku>BwXH(8ItVCXk*-lSJnR?-2UN%hJ){&rlvg`CDTj z)Bzo!3v7Ou#83zEDEFcKt(f1E0~=rqeEbTnMvWR#{+9pg%7G8y>u1OVRUSoox-ovF z2Ydma(;=YuBY(eI|04{hXzZD6_f(v~H;C~y5=DhAC{MMS>2fm~1H_t2$56pc$NH8( z5bH|<)71dV-_oCHIrzrT`2s-5w_+2CM0$95I6X8p^r!gHp+j_gd;9O<1~CEQQGS8) zS9Qh3#p&JM-G8rHekNmKVewU;pJRcTAog68KYo^dRo}(M>36U4Us zfgYWSiHZL3;lpWT=zNAW>Dh#mB!_@Lg%$ms8N-;aPqMn+C2HqZgz&9~Eu z4|Kp<`$q)Uw1R?y(~S>ePdonHxpV1#eSP1B;Ogo+-Pk}6#0GsZZ5!||ev2MGdh}_m z{DeR7?0-1^zVs&`AV6Vt;r3`I`OI_wgs*w=eO%_#7Kepl{B@xiyCANc(l zzIyd4y|c6PXWq9-|KM8(zIk8LPk(>a)zyFWjhT!$HJ$qX1vo@d25W<fvZQ2zUz5WRc(UnFMKHwe1| zWmlB1qdbiA(C0jmnV<}GfbKtmcu^2*P^O?MBLZKt|As~ge8&AAO~2K@zbXelK|4T<{|y4`raF{=72kC2Kn(L4YyenWgrPiv z@^mr$t{#X5VuIMeL!7Ab6_kG$&#&5p*Z{+?5U|TZ`B!7llpVmp@skYz&n^8QfPJzL z0G6K_OJM9x+Wu2gfN45phANGt{7=C>i34CV{Xqlx(fWpeAoj^N0Biu`w+MVcCUyU* zDZuzO0>4Z6fbu^T_arWW5n!E45vX8N=bxTVeFoep_G#VmNlQzAI_KTIc{6>c+04vr zx@W}zE5JNSU>!THJ{J=cqjz+4{L4A{Ob9$ZJ*S1?Ggg3klFp!+Y1@K+pK1DqI|_gq z5ZDXVpge8-cs!o|;K73#YXZ3AShj50wBvuq3NTOZ`M&qtjj#GOFfgExjg8Gn8>Vq5 z`85n+9|!iLCZF5$HJ$Iu($dm?8~-ofu}tEc+-pyke=3!im#6pk_Wo8IA|fJwD&~~F zc16osQ)EBo58U7XDuMexaPRjU@h8tXe%S{fA0NH3vGJFhuyyO!Uyl2^&EOpX{9As0 zWj+P>{@}jxH)8|r;2HdupP!vie{sJ28b&bo!8`D^x}TE$%zXNb^X1p@0PJ86`dZyj z%ce7*{^oo+6%&~I!8hQy-vQ7E)0t0ybH4l%KltWOo~8cO`T=157JqL(oq_rC%ea&4 z2NcTJe-HgFjNg-gZ$6!Y`SMHrlj}Etf7?r!zQTPPSv}{so2e>Fjs1{gzk~LGeesX%r(Lh6rbhSo_n)@@G-FTQy93;l#E)hgP@d_SGvyCp0~o(Y;Ee8{ zdVUDbHm5`2taPUOY^MAGOw*>=s7=Gst=D+p+2yON!0%Hk` zz5mAhyT4lS*T3LS^WSxUy86q&GnoHxzQ6vm8)VS}_zuqG?+3td68_x;etQAdu@sc6 zQJ&5|4(I?~3d-QOAODHpZ=hlSg(lBZ!JZWCtHHSj`0Wh93-Uk)_S%zsJ~aD>{`A0~ z9{AG(e|q3g5B%wYKRxiL2Y$8(4w6bzchKuloQW#e&S3n+P- z8!ds-%f;TJ1>)v)##>gd{PdS2Oc3VaR`fr=`O8QIO(6(N!A?pr5C#6fc~Ge@N%Vvu zaoAX2&(a6eWy_q&UwOhU)|P3J0Qc%OdhzW=F4D|pt0E4osw;%<%Dn58hAWD^XnZD= z>9~H(3bmLtxpF?a7su6J7M*x1By7YSUbxGi)Ot0P77`}P3{)&5Un{KD?`-e?r21!4vTTnN(4Y6Lin?UkSM z`MXCTC1@4A4~mvz%Rh2&EwY))LeoT=*`tMoqcEXI>TZU9WTP#l?uFv+@Dn~b(>xh2 z;>B?;Tz2SR&KVb>vGiBSB`@U7VIWFSo=LDSb9F{GF^DbmWAfpms8Sx9OX4CnBJca3 zlj9(x!dIjN?OG1X4l*imJNvRCk}F%!?SOfiOq5y^mZW)jFL@a|r-@d#f7 z2gmU8L3IZq0ynIws=}~m^#@&C%J6QFo~Mo4V`>v7MI-_!EBMMtb%_M&kvAaN)@ZVw z+`toz&WG#HkWDjnZE!6nk{e-oFdL^$YnbOCN}JC&{$#$O27@|Tn-skXr)2ml2~O!5 zX+gYoxhoc7qoU?C^3~&!U?kRFtnSEecWuH0B0OvLodgUAi}8p1 zrO6RSXHH}DMc$&|?D004DiOVMHV8kXCP@7NKB zgaZq^^O<7PoKEp72kby@W0Z!Y*Ay{&vfg#C&gG@YVR9g?FEocMUi1gSN$+V+ayF45{a zuDZDTN}mS|;BO%gEf}pjBfN2-gIrU#G5~cucA;dokXW89%>AyXJJI z9X4UlIWA|ZYHgbI z5?oFk@A=Ik7lrEQPDH!H+b`7_Y~aDb_qa=B2^Y&Ow41cU=4WDd40dp5(QS-WMN-=Y z9g;6_-JdNU;|6cPwf$ak*aJIcwL@1n$#l~zi{c{EW?T;DaW*E8DYq?Umtz{nJ&w-M zEMyTDrC&9K$d|kZe2#ws6)L=7K+{ zQw{XnV6UC$6-rW0emqm8wJoeZK)wJIcV?dST}Z;G0Arq{dVDu0&4kd%N!3F1*;*pW zR&qUiFzK=@44#QGw7k1`3t_d8&*kBV->O##t|tonFc2YWrL7_eqg+=+k;!F-`^b8> z#KWCE8%u4k@EprxqiV$VmmtiWxDLgnGu$Vs<8rppV5EajBXL4nyyZM$SWVm!wnCj-B!Wjqj5-5dNXukI2$$|Bu3Lrw}z65Lc=1G z^-#WuQOj$hwNGG?*CM_TO8Bg-1+qc>J7k5c51U8g?ZU5n?HYor;~JIjoWH-G>AoUP ztrWWLbRNqIjW#RT*WqZgPJXU7C)VaW5}MiijYbABmzoru6EmQ*N8cVK7a3|aOB#O& zBl8JY2WKfmj;h#Q!pN%9o@VNLv{OUL?rixHwOZuvX7{IJ{(EdPpuVFoQqIOa7giLVkBOKL@^smUA!tZ1CKRK}#SSM)iQHk)*R~?M!qkCruaS!#oIL1c z?J;U~&FfH#*98^G?i}pA{ z9Jg36t4=%6mhY(quYq*vSxptes9qy|7xSlH?G=S@>u>Ebe;|LVhs~@+06N<4CViBk zUiY$thvX;>Tby6z9Y1edAMQaiH zm^r3v#$Q#2T=X>bsY#D%s!bhs^M9PMAcHbCc0FMHV{u-dwlL;a1eJ63v5U*?Q_8JO zT#50!RD619#j_Uf))0ooADz~*9&lN!bBDRUgE>Vud-i5ck%vT=r^yD*^?Mp@Q^v+V zG#-?gKlr}Eeqifb{|So?HM&g91P8|av8hQoCmQXkd?7wIJwb z_^v8bbg`SAn{I*4bH$u(RZ6*xUhuA~hc=8czK8SHEKTzSxgbwi~9(OqJB&gwb^l4+m`k*Q;_?>Y-APi1{k zAHQ)P)G)f|AyjSgcCFps)Fh6Bca*Xznq36!pV6Az&m{O8$wGFD? zY&O*3*J0;_EqM#jh6^gMQKpXV?#1?>$ml1xvh8nSN>-?H=V;nJIwB07YX$e6vLxH( zqYwQ>qxwR(i4f)DLd)-$P>T-no_c!LsN@)8`e;W@)-Hj0>nJ-}Kla4-ZdPJzI&Mce zv)V_j;(3ERN3_@I$N<^|4Lf`B;8n+bX@bHbcZTopEmDI*Jfl)-pFDvo6svPRoo@(x z);_{lY<;);XzT`dBFpRmGrr}z5u1=pC^S-{ce6iXQlLGcItwJ^mZx{m$&DA_oEZ)B{_bYPq-HA zcH8WGoBG(aBU_j)vEy+_71T34@4dmSg!|M8Vf92Zj6WH7Q7t#OHQqWgFE3ARt+%!T z?oLovLVlnf?2c7pTc)~cc^($_8nyKwsN`RA-23ed3sdj(ys%pjjM+9JrctL;dy8a( z@en&CQmnV(()bu|Y%G1-4a(6x{aLytn$T-;(&{QIJB9vMox11U-1HpD@d(QkaJdEb zG{)+6Dos_L+O3NpWo^=gR?evp|CqEG?L&Ut#D*KLaRFOgOEK(Kq1@!EGcTfo+%A&I z=dLbB+d$u{sh?u)xP{PF8L%;YPPW53+@{>5W=Jt#wQpN;0_HYdw1{ksf_XhO4#2F= zyPx6Lx2<92L-;L5PD`zn6zwIH`Jk($?Qw({erA$^bC;q33hv!d!>%wRhj# zal^hk+WGNg;rJtb-EB(?czvOM=H7dl=vblBwAv>}%1@{}mnpUznfq1cE^sgsL0*4I zJ##!*B?=vI_OEVis5o+_IwMIRrpQyT_Sq~ZU%oY7c5JMIADzpD!Upz9h@iWg_>>~j zOLS;wp^i$-E?4<_cp?RiS%Rd?i;f*mOz=~(&3lo<=@(nR!_Rqiprh@weZlL!t#NCc zO!QTcInq|%#>OVgobj{~ixEUec`E25zJ~*DofsQdzIa@5^nOXj2T;8O`l--(QyU^$t?TGY^7#&FQ+2SS3B#qK*k3`ye?8jUYSajE5iBbJls75CCc(m3dk{t?- zopcER9{Z?TC)mk~gpi^kbbu>b-+a{m#8-y2^p$ka4n60w;Sc2}HMf<8JUvhCL0B&Btk)T`ctE$*qNW8L$`7!r^9T+>=<=2qaq-;ll2{`{Rg zc5a0ZUI$oG&j-qVOuKa=*v4aY#IsoM+1|c4Z)<}lEDvy;5huB@1RJPquU2U*U-;gu z=En2m+qjBzR#DEJDO`WU)hdd{Vj%^0V*KoyZ|5lzV87&g_j~NCjwv0uQVqXOb*QrQ zy|Qn`hxx(58c70$E;L(X0uZZ72M1!6oeg)(cdKO ze0gDaTz+ohR-#d)NbAH4x{I(21yjwvBQfmpLu$)|m{XolbgF!pmsqJ#D}(ylp6uC> z{bqtcI#hT#HW=wl7>p!38sKsJ`r8}lt-q%Keqy%u(xk=yiIJiUw6|5IvkS+#?JTBl z8H5(Q?l#wzazujH!8o>1xtn8#_w+397*_cy8!pQGP%K(Ga3pAjsaTbbXJlQF_+m+-UpUUent@xM zg%jqLUExj~o^vQ3Gl*>wh=_gOr2*|U64_iXb+-111aH}$TjeajM+I20xw(((>fej-@CIz4S1pi$(#}P7`4({6QS2CaQS4NPENDp>sAqD z$bH4KGzXGffkJ7R>V>)>tC)uax{UsN*dbeNC*v}#8Y#OWYwL4t$ePR?VTyIs!wea+ z5Urmc)X|^`MG~*dS6pGSbU+gPJoq*^a=_>$n4|P^w$sMBBy@f*Z^Jg6?n5?oId6f{ z$LW4M|4m502z0t7g<#Bx%X;9<=)smFolV&(V^(7Cv2-sxbxopQ!)*#ZRhTBpx1)Fc zNm1T%bONzv6@#|dz(w02AH8OXe>kQ#1FMCzO}2J_mST)+ExmBr9cva-@?;wnmWMOk z{3_~EX_xadgJGv&H@zK_8{(x84`}+c?oSBX*Ge3VdfTt&F}yCpFP?CpW+BE^cWY0^ zb&uBN!Ja3UzYHK-CTyA5=L zEMW{l3Usky#ly=7px648W31UNV@K)&Ub&zP1c7%)`{);I4b0Q<)B}3;NMG2JH=X$U zfIW4)4n9ZM`-yRj67I)YSLDK)qfUJ_ij}a#aZN~9EXrh8eZY2&=uY%2N0UFF7<~%M zsB8=erOWZ>Ct_#^tHZ|*q`H;A)5;ycw*IcmVxi8_0Xk}aJA^ath+E;xg!x+As(M#0=)3!NJR6H&9+zd#iP(m0PIW8$ z1Y^VX`>jm`W!=WpF*{ioM?C9`yOR>@0q=u7o>BP-eSHqCgMDj!2anwH?s%i2p+Q7D zzszIf5XJpE)IG4;d_(La-xenmF(tgAxK`Y4sQ}BSJEPs6N_U2vI{8=0C_F?@7<(G; zo$~G=8p+076G;`}>{MQ>t>7cm=zGtfbdDXm6||jUU|?X?CaE?(<6bKDYKeHlz}DA8 zXT={X=yp_R;HfJ9h%?eWvQ!dRgz&Su*JfNt!Wu>|XfU&68iRikRrHRW|ZxzRR^`eIGt zIeiDgVS>IeExKVRWW8-=A=yA`}`)ZkWBrZD`hpWIxBGkh&f#ijr449~m`j6{4jiJ*C!oVA8ZC?$1RM#K(_b zL9TW)kN*Y4%^-qPpMP7d4)o?Nk#>aoYHT(*g)qmRUb?**F@pnNiy6Fv9rEiUqD(^O zzyS?nBrX63BTRYduaG(0VVG2yJRe%o&rVrLjbxTaAFTd8s;<<@Qs>u(<193R8>}2_ zuwp{7;H2a*X7_jryzriZXMg?bTuegABb^87@SsKkr2)0Gyiax8KQWstw^v#ix45EVrcEhr>!NMhprl$InQMzjSFH54x5k9qHc`@9uKQzvL4ihcq{^B zPrVR=o_ic%Y>6&rMN)hTZsI7I<3&`#(nl+3y3ys9A~&^=4?PL&nd8)`OfG#n zwAMN$1&>K++c{^|7<4P=2y(B{jJsQ0a#U;HTo4ZmWZYvI{+s;Td{Yzem%0*k#)vjpB zia;J&>}ICate44SFYY3vEelqStQWFihx%^vQ@Do(sOy7yR2@WNv7Y9I^yL=nZr3mb zXKV5t@=?-Sk|b{XMhA7ZGB@2hqsx}4xwCW!in#C zI@}scZlr3-NFJ@NFaJlhyfcw{k^vvtGl`N9xSo**rDW4S}i zM9{fMPWo%4wYDG~BZ18BD+}h|GQKc-g^{++3MY>}W_uq7jGHx{mwE9fZiPCoxN$+7 zrODGGJrOkcPQUB(FD5aoS4g~7#6NR^ma7-!>mHuJfY5kTe6PpNNKC9GGRiu^L31uG z$7v`*JknQHsYB!Tm_W{a32TM099djW%5e+j0Ve_ct}IM>XLF1Ap+YvcrLV=|CKo6S zb+9Nl3_YdKP6%Cxy@6TxZ>;4&nTneadr z_ES90ydCev)LV!dN=#(*f}|ZORFdvkYBni^aLbUk>BajeWIOcmHP#8S)*2U~QKI%S zyrLmtPqb&TphJ;>yAxri#;{uyk`JJqODDw%(Z=2`1uc}br^V%>j!gS)D*q*f_-qf8&D;W1dJgQMlaH5er zN2U<%Smb7==vE}dDI8K7cKz!vs^73o9f>2sgiTzWcwY|BMYHH5%Vn7#kiw&eItCqa zIkR2~Q}>X=Ar8W|^Ms41Fm8o6IB2_j60eOeBB1Br!boW7JnoeX6Gs)?7rW0^5psc- zjS16yb>dFn>KPOF;imD}e!enuIniFzv}n$m2#gCCv4jM#ArwlzZ$7@9&XkFxZ4n!V zj3dyiwW4Ki2QG{@i>yuZXQizw_OkZI^-3otXC{!(lUpJF33gI60ak;Uqitp74|B6I zgg{b=Iz}WkhCGj1M=hu4#Aw173YxIVbISaoc z-nLZC*6Tgivd5V`K%GxhBsp@SUU60-rfc$=wb>zdJzXS&-5(NRRodFk;Kxk!S(O(a0e7oY=E( zAyS;Ow?6Q&XA+cnkCb{28_1N8H#?J!*$MmIwLq^*T_9-z^&UE@A(z9oGYtFy6EZef LrJugUA?W`A8`#=m literal 0 HcmV?d00001 diff --git a/agents/nextjs/openai-realtime/example/app/globals.css b/agents/nextjs/openai-realtime/example/app/globals.css new file mode 100644 index 00000000..d767ad63 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/app/globals.css @@ -0,0 +1,126 @@ +@import "tailwindcss"; +@import "tw-animate-css"; +@import "shadcn/tailwind.css"; + +@custom-variant dark (&:is(.dark *)); + +@theme inline { + --color-background: var(--background); + --color-foreground: var(--foreground); + --font-sans: var(--font-geist-sans); + --font-mono: var(--font-geist-mono); + --color-sidebar-ring: var(--sidebar-ring); + --color-sidebar-border: var(--sidebar-border); + --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); + --color-sidebar-accent: var(--sidebar-accent); + --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); + --color-sidebar-primary: var(--sidebar-primary); + --color-sidebar-foreground: var(--sidebar-foreground); + --color-sidebar: var(--sidebar); + --color-chart-5: var(--chart-5); + --color-chart-4: var(--chart-4); + --color-chart-3: var(--chart-3); + --color-chart-2: var(--chart-2); + --color-chart-1: var(--chart-1); + --color-ring: var(--ring); + --color-input: var(--input); + --color-border: var(--border); + --color-destructive: var(--destructive); + --color-accent-foreground: var(--accent-foreground); + --color-accent: var(--accent); + --color-muted-foreground: var(--muted-foreground); + --color-muted: var(--muted); + --color-secondary-foreground: var(--secondary-foreground); + --color-secondary: var(--secondary); + --color-primary-foreground: var(--primary-foreground); + --color-primary: var(--primary); + --color-popover-foreground: var(--popover-foreground); + --color-popover: var(--popover); + --color-card-foreground: var(--card-foreground); + --color-card: var(--card); + --radius-sm: calc(var(--radius) - 4px); + --radius-md: calc(var(--radius) - 2px); + --radius-lg: var(--radius); + --radius-xl: calc(var(--radius) + 4px); + --radius-2xl: calc(var(--radius) + 8px); + --radius-3xl: calc(var(--radius) + 12px); + --radius-4xl: calc(var(--radius) + 16px); +} + +:root { + --radius: 0.625rem; + --background: oklch(1 0 0); + --foreground: oklch(0.145 0 0); + --card: oklch(1 0 0); + --card-foreground: oklch(0.145 0 0); + --popover: oklch(1 0 0); + --popover-foreground: oklch(0.145 0 0); + --primary: oklch(0.205 0 0); + --primary-foreground: oklch(0.985 0 0); + --secondary: oklch(0.97 0 0); + --secondary-foreground: oklch(0.205 0 0); + --muted: oklch(0.97 0 0); + --muted-foreground: oklch(0.556 0 0); + --accent: oklch(0.97 0 0); + --accent-foreground: oklch(0.205 0 0); + --destructive: oklch(0.577 0.245 27.325); + --border: oklch(0.922 0 0); + --input: oklch(0.922 0 0); + --ring: oklch(0.708 0 0); + --chart-1: oklch(0.646 0.222 41.116); + --chart-2: oklch(0.6 0.118 184.704); + --chart-3: oklch(0.398 0.07 227.392); + --chart-4: oklch(0.828 0.189 84.429); + --chart-5: oklch(0.769 0.188 70.08); + --sidebar: oklch(0.985 0 0); + --sidebar-foreground: oklch(0.145 0 0); + --sidebar-primary: oklch(0.205 0 0); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.97 0 0); + --sidebar-accent-foreground: oklch(0.205 0 0); + --sidebar-border: oklch(0.922 0 0); + --sidebar-ring: oklch(0.708 0 0); +} + +.dark { + --background: oklch(0.145 0 0); + --foreground: oklch(0.985 0 0); + --card: oklch(0.205 0 0); + --card-foreground: oklch(0.985 0 0); + --popover: oklch(0.205 0 0); + --popover-foreground: oklch(0.985 0 0); + --primary: oklch(0.922 0 0); + --primary-foreground: oklch(0.205 0 0); + --secondary: oklch(0.269 0 0); + --secondary-foreground: oklch(0.985 0 0); + --muted: oklch(0.269 0 0); + --muted-foreground: oklch(0.708 0 0); + --accent: oklch(0.269 0 0); + --accent-foreground: oklch(0.985 0 0); + --destructive: oklch(0.704 0.191 22.216); + --border: oklch(1 0 0 / 10%); + --input: oklch(1 0 0 / 15%); + --ring: oklch(0.556 0 0); + --chart-1: oklch(0.488 0.243 264.376); + --chart-2: oklch(0.696 0.17 162.48); + --chart-3: oklch(0.769 0.188 70.08); + --chart-4: oklch(0.627 0.265 303.9); + --chart-5: oklch(0.645 0.246 16.439); + --sidebar: oklch(0.205 0 0); + --sidebar-foreground: oklch(0.985 0 0); + --sidebar-primary: oklch(0.488 0.243 264.376); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.269 0 0); + --sidebar-accent-foreground: oklch(0.985 0 0); + --sidebar-border: oklch(1 0 0 / 10%); + --sidebar-ring: oklch(0.556 0 0); +} + +@layer base { + * { + @apply border-border outline-ring/50; + } + body { + @apply bg-background text-foreground; + } +} diff --git a/agents/nextjs/openai-realtime/example/app/layout.tsx b/agents/nextjs/openai-realtime/example/app/layout.tsx new file mode 100644 index 00000000..f7fa87eb --- /dev/null +++ b/agents/nextjs/openai-realtime/example/app/layout.tsx @@ -0,0 +1,34 @@ +import type { Metadata } from "next"; +import { Geist, Geist_Mono } from "next/font/google"; +import "./globals.css"; + +const geistSans = Geist({ + variable: "--font-geist-sans", + subsets: ["latin"], +}); + +const geistMono = Geist_Mono({ + variable: "--font-geist-mono", + subsets: ["latin"], +}); + +export const metadata: Metadata = { + title: "Create Next App", + description: "Generated by create next app", +}; + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + + {children} + + + ); +} diff --git a/agents/nextjs/openai-realtime/example/app/page.tsx b/agents/nextjs/openai-realtime/example/app/page.tsx new file mode 100644 index 00000000..00e7cec2 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/app/page.tsx @@ -0,0 +1,153 @@ +"use client"; + +import { + type RealtimeEventLogEntry, + type SessionInfo, + useRealtimeSession, +} from "@/hooks/use-realtime-session"; + +export default function Home() { + const { + status, + sessionInfo, + transcript, + isUserSpeaking, + isAssistantSpeaking, + isResponding, + responseStatus, + events, + error, + start, + stop, + } = useRealtimeSession(); + + const isConnected = status === "connected"; + const isConnecting = status === "connecting"; + + return ( +
+
+
+

+ OpenAI Realtime SDK +

+

+ Talk to ElevenLabs Realtime using OpenAI's Realtime SDK. +

+
+ +
+
+ + + {isConnected + ? "Connected" + : isConnecting + ? "Connecting…" + : "Disconnected"} + +
+ + {isConnected ? ( +
+ + + + {responseStatus && responseStatus !== "in_progress" ? ( + + Last response: {responseStatus} + + ) : null} +
+ ) : null} + + {isConnected && sessionInfo ? ( + + ) : null} + + {error ?

{error}

: null} + +
+

Transcript

+
    + {transcript.length === 0 ? ( +
  • No messages yet.
    • + ) : ( + transcript.map(entry => ( +
  • + + {entry.role === "user" ? "You" : "Agent"} + +

    + {entry.text || + (entry.role === "user" ? "Transcribing…" : "…")} + {entry.text && !entry.isFinal ? ( + + ▍ + + ) : null} +

    +
    • + )) + )} +
+
+ + {events.length > 0 ? : null} +
+
+
+ ); +} + +function Indicator({ active, label }: { active: boolean; label: string }) { + return ( + + {active ? "● " : "○ "} + {label} + + ); +} + +function SessionInfoLine({ info }: { info: SessionInfo }) { + const parts = [ + info.model, + info.voice ?? "default voice", + info.turnDetection ?? "vad", + ]; + return

{parts.join(" · ")}

; +} + +function EventLog({ events }: { events: RealtimeEventLogEntry[] }) { + return ( +
+ + Transport events ({events.length}) + +
    + {events.toReversed().map(event => ( +
  • + + {new Date(event.at).toLocaleTimeString()} + + {event.type} + {event.summary ? ( + {event.summary} + ) : null} +
    • + ))} +
+
+ ); +} diff --git a/agents/nextjs/openai-realtime/example/components.json b/agents/nextjs/openai-realtime/example/components.json new file mode 100644 index 00000000..f87021ee --- /dev/null +++ b/agents/nextjs/openai-realtime/example/components.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "new-york", + "rsc": true, + "tsx": true, + "tailwind": { + "config": "", + "css": "app/globals.css", + "baseColor": "neutral", + "cssVariables": true, + "prefix": "" + }, + "iconLibrary": "lucide", + "rtl": false, + "aliases": { + "components": "@/components", + "utils": "@/lib/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" + }, + "registries": {} +} diff --git a/agents/nextjs/openai-realtime/example/components/ui/live-waveform.tsx b/agents/nextjs/openai-realtime/example/components/ui/live-waveform.tsx new file mode 100644 index 00000000..d3533506 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/components/ui/live-waveform.tsx @@ -0,0 +1,560 @@ +"use client"; + +import { useEffect, useRef, type HTMLAttributes } from "react"; + +import { cn } from "@/lib/utils"; + +export type LiveWaveformProps = HTMLAttributes & { + active?: boolean; + processing?: boolean; + deviceId?: string; + barWidth?: number; + barHeight?: number; + barGap?: number; + barRadius?: number; + barColor?: string; + fadeEdges?: boolean; + fadeWidth?: number; + height?: string | number; + sensitivity?: number; + smoothingTimeConstant?: number; + fftSize?: number; + historySize?: number; + updateRate?: number; + mode?: "scrolling" | "static"; + onError?: (error: Error) => void; + onStreamReady?: (stream: MediaStream) => void; + onStreamEnd?: () => void; +}; + +export const LiveWaveform = ({ + active = false, + processing = false, + deviceId, + barWidth = 3, + barGap = 1, + barRadius = 1.5, + barColor, + fadeEdges = true, + fadeWidth = 24, + barHeight: baseBarHeight = 4, + height = 64, + sensitivity = 1, + smoothingTimeConstant = 0.8, + fftSize = 256, + historySize = 60, + updateRate = 30, + mode = "static", + onError, + onStreamReady, + onStreamEnd, + className, + ...props +}: LiveWaveformProps) => { + const canvasRef = useRef(null); + const containerRef = useRef(null); + const historyRef = useRef([]); + const analyserRef = useRef(null); + const audioContextRef = useRef(null); + const streamRef = useRef(null); + const animationRef = useRef(0); + const lastUpdateRef = useRef(0); + const processingAnimationRef = useRef(null); + const lastActiveDataRef = useRef([]); + const transitionProgressRef = useRef(0); + const staticBarsRef = useRef([]); + const needsRedrawRef = useRef(true); + const gradientCacheRef = useRef(null); + const lastWidthRef = useRef(0); + + const heightStyle = typeof height === "number" ? `${height}px` : height; + + // Handle canvas resizing + useEffect(() => { + const canvas = canvasRef.current; + const container = containerRef.current; + if (!canvas || !container) return; + + const resizeObserver = new ResizeObserver(() => { + const rect = container.getBoundingClientRect(); + const dpr = window.devicePixelRatio || 1; + + canvas.width = rect.width * dpr; + canvas.height = rect.height * dpr; + canvas.style.width = `${rect.width}px`; + canvas.style.height = `${rect.height}px`; + + const ctx = canvas.getContext("2d"); + if (ctx) { + ctx.scale(dpr, dpr); + } + + gradientCacheRef.current = null; + lastWidthRef.current = rect.width; + needsRedrawRef.current = true; + }); + + resizeObserver.observe(container); + return () => resizeObserver.disconnect(); + }, []); + + useEffect(() => { + if (processing && !active) { + let time = 0; + transitionProgressRef.current = 0; + + const animateProcessing = () => { + time += 0.03; + transitionProgressRef.current = Math.min( + 1, + transitionProgressRef.current + 0.02 + ); + + const processingData = []; + const barCount = Math.floor( + (containerRef.current?.getBoundingClientRect().width || 200) / + (barWidth + barGap) + ); + + if (mode === "static") { + const halfCount = Math.floor(barCount / 2); + + for (let i = 0; i < barCount; i++) { + const normalizedPosition = (i - halfCount) / halfCount; + const centerWeight = 1 - Math.abs(normalizedPosition) * 0.4; + + const wave1 = Math.sin(time * 1.5 + normalizedPosition * 3) * 0.25; + const wave2 = Math.sin(time * 0.8 - normalizedPosition * 2) * 0.2; + const wave3 = Math.cos(time * 2 + normalizedPosition) * 0.15; + const combinedWave = wave1 + wave2 + wave3; + const processingValue = (0.2 + combinedWave) * centerWeight; + + let finalValue = processingValue; + if ( + lastActiveDataRef.current.length > 0 && + transitionProgressRef.current < 1 + ) { + const lastDataIndex = Math.min( + i, + lastActiveDataRef.current.length - 1 + ); + const lastValue = lastActiveDataRef.current[lastDataIndex] || 0; + finalValue = + lastValue * (1 - transitionProgressRef.current) + + processingValue * transitionProgressRef.current; + } + + processingData.push(Math.max(0.05, Math.min(1, finalValue))); + } + } else { + for (let i = 0; i < barCount; i++) { + const normalizedPosition = (i - barCount / 2) / (barCount / 2); + const centerWeight = 1 - Math.abs(normalizedPosition) * 0.4; + + const wave1 = Math.sin(time * 1.5 + i * 0.15) * 0.25; + const wave2 = Math.sin(time * 0.8 - i * 0.1) * 0.2; + const wave3 = Math.cos(time * 2 + i * 0.05) * 0.15; + const combinedWave = wave1 + wave2 + wave3; + const processingValue = (0.2 + combinedWave) * centerWeight; + + let finalValue = processingValue; + if ( + lastActiveDataRef.current.length > 0 && + transitionProgressRef.current < 1 + ) { + const lastDataIndex = Math.floor( + (i / barCount) * lastActiveDataRef.current.length + ); + const lastValue = lastActiveDataRef.current[lastDataIndex] || 0; + finalValue = + lastValue * (1 - transitionProgressRef.current) + + processingValue * transitionProgressRef.current; + } + + processingData.push(Math.max(0.05, Math.min(1, finalValue))); + } + } + + if (mode === "static") { + staticBarsRef.current = processingData; + } else { + historyRef.current = processingData; + } + + needsRedrawRef.current = true; + processingAnimationRef.current = + requestAnimationFrame(animateProcessing); + }; + + animateProcessing(); + + return () => { + if (processingAnimationRef.current) { + cancelAnimationFrame(processingAnimationRef.current); + } + }; + } else if (!active && !processing) { + const hasData = + mode === "static" + ? staticBarsRef.current.length > 0 + : historyRef.current.length > 0; + + if (hasData) { + let fadeProgress = 0; + const fadeToIdle = () => { + fadeProgress += 0.03; + if (fadeProgress < 1) { + if (mode === "static") { + staticBarsRef.current = staticBarsRef.current.map( + value => value * (1 - fadeProgress) + ); + } else { + historyRef.current = historyRef.current.map( + value => value * (1 - fadeProgress) + ); + } + needsRedrawRef.current = true; + requestAnimationFrame(fadeToIdle); + } else { + if (mode === "static") { + staticBarsRef.current = []; + } else { + historyRef.current = []; + } + } + }; + fadeToIdle(); + } + } + }, [processing, active, barWidth, barGap, mode]); + + // Handle microphone setup and teardown + useEffect(() => { + if (!active) { + if (streamRef.current) { + streamRef.current.getTracks().forEach(track => track.stop()); + streamRef.current = null; + onStreamEnd?.(); + } + if ( + audioContextRef.current && + audioContextRef.current.state !== "closed" + ) { + audioContextRef.current.close(); + audioContextRef.current = null; + } + if (animationRef.current) { + cancelAnimationFrame(animationRef.current); + animationRef.current = 0; + } + return; + } + + const setupMicrophone = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: deviceId + ? { + deviceId: { exact: deviceId }, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + } + : { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + streamRef.current = stream; + onStreamReady?.(stream); + + const AudioContextConstructor = + window.AudioContext || + (window as unknown as { webkitAudioContext: typeof AudioContext }) + .webkitAudioContext; + const audioContext = new AudioContextConstructor(); + const analyser = audioContext.createAnalyser(); + analyser.fftSize = fftSize; + analyser.smoothingTimeConstant = smoothingTimeConstant; + + const source = audioContext.createMediaStreamSource(stream); + source.connect(analyser); + + audioContextRef.current = audioContext; + analyserRef.current = analyser; + + // Clear history when starting + historyRef.current = []; + } catch (error) { + onError?.(error as Error); + } + }; + + setupMicrophone(); + + return () => { + if (streamRef.current) { + streamRef.current.getTracks().forEach(track => track.stop()); + streamRef.current = null; + onStreamEnd?.(); + } + if ( + audioContextRef.current && + audioContextRef.current.state !== "closed" + ) { + audioContextRef.current.close(); + audioContextRef.current = null; + } + if (animationRef.current) { + cancelAnimationFrame(animationRef.current); + animationRef.current = 0; + } + }; + }, [ + active, + deviceId, + fftSize, + smoothingTimeConstant, + onError, + onStreamReady, + onStreamEnd, + ]); + + // Animation loop + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas) return; + + const ctx = canvas.getContext("2d"); + if (!ctx) return; + + let rafId: number; + + const animate = (currentTime: number) => { + // Render waveform + const rect = canvas.getBoundingClientRect(); + + // Update audio data if active + if (active && currentTime - lastUpdateRef.current > updateRate) { + lastUpdateRef.current = currentTime; + + if (analyserRef.current) { + const dataArray = new Uint8Array( + analyserRef.current.frequencyBinCount + ); + analyserRef.current.getByteFrequencyData(dataArray); + + if (mode === "static") { + // For static mode, update bars in place + const startFreq = Math.floor(dataArray.length * 0.05); + const endFreq = Math.floor(dataArray.length * 0.4); + const relevantData = dataArray.slice(startFreq, endFreq); + + const barCount = Math.floor(rect.width / (barWidth + barGap)); + const halfCount = Math.floor(barCount / 2); + const newBars: number[] = []; + + // Mirror the data for symmetric display + for (let i = halfCount - 1; i >= 0; i--) { + const dataIndex = Math.floor( + (i / halfCount) * relevantData.length + ); + const value = Math.min( + 1, + (relevantData[dataIndex] / 255) * sensitivity + ); + newBars.push(Math.max(0.05, value)); + } + + for (let i = 0; i < halfCount; i++) { + const dataIndex = Math.floor( + (i / halfCount) * relevantData.length + ); + const value = Math.min( + 1, + (relevantData[dataIndex] / 255) * sensitivity + ); + newBars.push(Math.max(0.05, value)); + } + + staticBarsRef.current = newBars; + lastActiveDataRef.current = newBars; + } else { + // Scrolling mode - original behavior + let sum = 0; + const startFreq = Math.floor(dataArray.length * 0.05); + const endFreq = Math.floor(dataArray.length * 0.4); + const relevantData = dataArray.slice(startFreq, endFreq); + + for (let i = 0; i < relevantData.length; i++) { + sum += relevantData[i]; + } + const average = (sum / relevantData.length / 255) * sensitivity; + + // Add to history + historyRef.current.push(Math.min(1, Math.max(0.05, average))); + lastActiveDataRef.current = [...historyRef.current]; + + // Maintain history size + if (historyRef.current.length > historySize) { + historyRef.current.shift(); + } + } + needsRedrawRef.current = true; + } + } + + // Only redraw if needed + if (!needsRedrawRef.current && !active) { + rafId = requestAnimationFrame(animate); + return; + } + + needsRedrawRef.current = active; + ctx.clearRect(0, 0, rect.width, rect.height); + + const computedBarColor = + barColor || + (() => { + const style = getComputedStyle(canvas); + // Try to get the computed color value directly + const color = style.color; + return color || "#000"; + })(); + + const step = barWidth + barGap; + const barCount = Math.floor(rect.width / step); + const centerY = rect.height / 2; + + // Draw bars based on mode + if (mode === "static") { + // Static mode - bars in fixed positions + const dataToRender = processing + ? staticBarsRef.current + : active + ? staticBarsRef.current + : staticBarsRef.current.length > 0 + ? staticBarsRef.current + : []; + + for (let i = 0; i < barCount && i < dataToRender.length; i++) { + const value = dataToRender[i] || 0.1; + const x = i * step; + const barHeight = Math.max(baseBarHeight, value * rect.height * 0.8); + const y = centerY - barHeight / 2; + + ctx.fillStyle = computedBarColor; + ctx.globalAlpha = 0.4 + value * 0.6; + + if (barRadius > 0) { + ctx.beginPath(); + ctx.roundRect(x, y, barWidth, barHeight, barRadius); + ctx.fill(); + } else { + ctx.fillRect(x, y, barWidth, barHeight); + } + } + } else { + // Scrolling mode - original behavior + for (let i = 0; i < barCount && i < historyRef.current.length; i++) { + const dataIndex = historyRef.current.length - 1 - i; + const value = historyRef.current[dataIndex] || 0.1; + const x = rect.width - (i + 1) * step; + const barHeight = Math.max(baseBarHeight, value * rect.height * 0.8); + const y = centerY - barHeight / 2; + + ctx.fillStyle = computedBarColor; + ctx.globalAlpha = 0.4 + value * 0.6; + + if (barRadius > 0) { + ctx.beginPath(); + ctx.roundRect(x, y, barWidth, barHeight, barRadius); + ctx.fill(); + } else { + ctx.fillRect(x, y, barWidth, barHeight); + } + } + } + + // Apply edge fading + if (fadeEdges && fadeWidth > 0 && rect.width > 0) { + // Cache gradient if width hasn't changed + if (!gradientCacheRef.current || lastWidthRef.current !== rect.width) { + const gradient = ctx.createLinearGradient(0, 0, rect.width, 0); + const fadePercent = Math.min(0.3, fadeWidth / rect.width); + + // destination-out: removes destination where source alpha is high + // We want: fade edges out, keep center solid + // Left edge: start opaque (1) = remove, fade to transparent (0) = keep + gradient.addColorStop(0, "rgba(255,255,255,1)"); + gradient.addColorStop(fadePercent, "rgba(255,255,255,0)"); + // Center stays transparent = keep everything + gradient.addColorStop(1 - fadePercent, "rgba(255,255,255,0)"); + // Right edge: fade from transparent (0) = keep to opaque (1) = remove + gradient.addColorStop(1, "rgba(255,255,255,1)"); + + gradientCacheRef.current = gradient; + lastWidthRef.current = rect.width; + } + + ctx.globalCompositeOperation = "destination-out"; + ctx.fillStyle = gradientCacheRef.current; + ctx.fillRect(0, 0, rect.width, rect.height); + ctx.globalCompositeOperation = "source-over"; + } + + ctx.globalAlpha = 1; + + rafId = requestAnimationFrame(animate); + }; + + rafId = requestAnimationFrame(animate); + + return () => { + if (rafId) { + cancelAnimationFrame(rafId); + } + }; + }, [ + active, + processing, + sensitivity, + updateRate, + historySize, + barWidth, + baseBarHeight, + barGap, + barRadius, + barColor, + fadeEdges, + fadeWidth, + mode, + ]); + + return ( +
+ {!active && !processing && ( +
+ )} +
+ ); +}; diff --git a/agents/nextjs/openai-realtime/example/eslint.config.mjs b/agents/nextjs/openai-realtime/example/eslint.config.mjs new file mode 100644 index 00000000..05e726d1 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/eslint.config.mjs @@ -0,0 +1,18 @@ +import { defineConfig, globalIgnores } from "eslint/config"; +import nextVitals from "eslint-config-next/core-web-vitals"; +import nextTs from "eslint-config-next/typescript"; + +const eslintConfig = defineConfig([ + ...nextVitals, + ...nextTs, + // Override default ignores of eslint-config-next. + globalIgnores([ + // Default ignores of eslint-config-next: + ".next/**", + "out/**", + "build/**", + "next-env.d.ts", + ]), +]); + +export default eslintConfig; diff --git a/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts new file mode 100644 index 00000000..b4d8ee97 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts @@ -0,0 +1,421 @@ +"use client"; + +import { + OpenAIRealtimeWebSocket, + RealtimeAgent, + RealtimeSession, + type TransportEvent, +} from "@openai/agents-realtime"; +import { useCallback, useRef, useState } from "react"; +import type { WavRecorder, WavStreamPlayer } from "wavtools"; + +import { + AGENT_INSTRUCTIONS, + AUDIO_SAMPLE_RATE, + REALTIME_MODEL, + REALTIME_URL, + REALTIME_VOICE, +} from "@/lib/realtime-session"; + +export type SessionStatus = "disconnected" | "connecting" | "connected"; + +export type ResponseStatus = + "in_progress" | "completed" | "cancelled" | "incomplete" | "failed"; + +const RESPONSE_STATUSES = new Set([ + "in_progress", + "completed", + "cancelled", + "incomplete", + "failed", +]); + +export type TranscriptEntry = { + id: string; + role: "user" | "assistant"; + text: string; + isFinal: boolean; + previousItemId?: string | null; +}; + +export type SessionInfo = { + model: string; + voice: string | null; + turnDetection: string | null; +}; + +export type RealtimeEventLogEntry = { + id: number; + type: string; + at: number; + summary?: string; +}; + +const NOISY_EVENT_TYPES = new Set([ + "response.output_audio.delta", + "response.output_audio_transcript.delta", +]); + +const TRANSCRIPT_EVENT_TYPES = new Set([ + "conversation.item.added", + "conversation.item.done", + "conversation.item.input_audio_transcription.completed", + "response.output_audio_transcript.delta", + "response.output_audio_transcript.done", + "response.done", +]); + +const MAX_EVENT_LOG = 100; + +export function useRealtimeSession() { + const [status, setStatus] = useState("disconnected"); + const [sessionInfo, setSessionInfo] = useState(null); + const [transcript, setTranscript] = useState([]); + const [isUserSpeaking, setIsUserSpeaking] = useState(false); + const [isAssistantSpeaking, setIsAssistantSpeaking] = useState(false); + const [isResponding, setIsResponding] = useState(false); + const [responseStatus, setResponseStatus] = useState( + null + ); + const [events, setEvents] = useState([]); + const [error, setError] = useState(null); + + const sessionRef = useRef(null); + const recorderRef = useRef(null); + const playerRef = useRef(null); + const eventIdRef = useRef(0); + + const logEvent = useCallback((type: string, summary?: string) => { + const entry: RealtimeEventLogEntry = { + id: (eventIdRef.current += 1), + type, + at: Date.now(), + summary, + }; + setEvents(prev => [...prev.slice(-(MAX_EVENT_LOG - 1)), entry]); + }, []); + + const teardown = useCallback(async () => { + try { + await recorderRef.current?.quit(); + } catch {} + recorderRef.current = null; + + try { + playerRef.current?.interrupt(); + } catch {} + playerRef.current = null; + + try { + sessionRef.current?.close(); + } catch {} + sessionRef.current = null; + + setIsUserSpeaking(false); + setIsAssistantSpeaking(false); + setIsResponding(false); + }, []); + + const handleTransportEvent = useCallback( + (event: TransportEvent) => { + const { type } = event; + switch (type) { + case "input_audio_buffer.speech_started": + setIsUserSpeaking(true); + break; + case "input_audio_buffer.speech_stopped": + setIsUserSpeaking(false); + break; + case "session.created": + case "session.updated": { + const info = readSessionInfo(event); + if (info) setSessionInfo(info); + break; + } + case "response.created": + setIsResponding(true); + setResponseStatus("in_progress"); + break; + case "response.done": + setIsResponding(false); + setResponseStatus(readResponseStatus(event)); + break; + } + + if (TRANSCRIPT_EVENT_TYPES.has(type)) { + setTranscript(prev => { + const next = updateTranscript(prev, type, event); + return next === prev ? prev : orderByPreviousItemId(next); + }); + } + + if (!NOISY_EVENT_TYPES.has(type)) { + logEvent(type, summarizeEvent(type, event)); + } + }, + [logEvent] + ); + + const start = useCallback(async () => { + if (sessionRef.current) return; + setError(null); + setTranscript([]); + setSessionInfo(null); + setEvents([]); + setResponseStatus(null); + eventIdRef.current = 0; + setStatus("connecting"); + + try { + const tokenResponse = await fetch("/api/token"); + const body: { value?: string; error?: string } = await tokenResponse + .json() + .catch(() => ({})); + if (!tokenResponse.ok || !body.value) { + throw new Error( + body.error ?? `Token request failed (${tokenResponse.status}).` + ); + } + + // The SDK only wires audio for WebRTC, so capture and play PCM16 ourselves over WebSocket. + const { WavRecorder, WavStreamPlayer } = await import("wavtools"); + const recorder = new WavRecorder({ sampleRate: AUDIO_SAMPLE_RATE }); + const player = new WavStreamPlayer({ sampleRate: AUDIO_SAMPLE_RATE }); + await recorder.begin(); + await player.connect(); + recorderRef.current = recorder; + playerRef.current = player; + + const agent = new RealtimeAgent({ + name: "ElevenLabs agent", + instructions: AGENT_INSTRUCTIONS, + }); + const session = new RealtimeSession(agent, { + transport: new OpenAIRealtimeWebSocket({ url: REALTIME_URL }), + model: REALTIME_MODEL, + config: { + outputModalities: ["audio"], + audio: { output: { voice: REALTIME_VOICE } }, + }, + }); + sessionRef.current = session; + + session.on("audio", event => { + player.add16BitPCM(event.data, event.responseId); + }); + session.on("audio_start", () => setIsAssistantSpeaking(true)); + session.on("audio_interrupted", () => { + player.interrupt(); + setIsAssistantSpeaking(false); + }); + session.on("audio_stopped", () => setIsAssistantSpeaking(false)); + session.on("error", event => setError(extractErrorMessage(event))); + session.on("transport_event", handleTransportEvent); + + await session.connect({ apiKey: body.value }); + + // wavtools types `mono` as Int16Array, but at runtime it is already the PCM16 ArrayBuffer. + await recorder.record(data => { + session.sendAudio(data.mono as unknown as ArrayBuffer); + }); + + setStatus("connected"); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + await teardown(); + setStatus("disconnected"); + } + }, [handleTransportEvent, teardown]); + + const stop = useCallback(async () => { + await teardown(); + setStatus("disconnected"); + }, [teardown]); + + return { + status, + sessionInfo, + transcript, + isUserSpeaking, + isAssistantSpeaking, + isResponding, + responseStatus, + events, + error, + start, + stop, + }; +} + +function updateTranscript( + prev: TranscriptEntry[], + type: string, + event: TransportEvent +): TranscriptEntry[] { + const obj = asObject(event); + switch (type) { + case "conversation.item.added": + case "conversation.item.done": { + const item = getObject(obj, "item"); + const id = getString(item, "id"); + const role = getString(item, "role"); + if (!id || (role !== "user" && role !== "assistant")) return prev; + const previousItemId = getString(obj, "previous_item_id"); + return upsertTranscript(prev, id, role, entry => ({ + previousItemId: previousItemId ?? entry.previousItemId, + })); + } + case "conversation.item.input_audio_transcription.completed": { + const id = getString(obj, "item_id"); + if (!id) return prev; + const text = getString(obj, "transcript") ?? ""; + return upsertTranscript(prev, id, "user", () => ({ + text, + isFinal: true, + })); + } + case "response.output_audio_transcript.delta": { + const id = getString(obj, "item_id"); + if (!id) return prev; + const delta = getString(obj, "delta") ?? ""; + return upsertTranscript(prev, id, "assistant", entry => ({ + text: entry.text + delta, + })); + } + case "response.output_audio_transcript.done": { + const id = getString(obj, "item_id"); + if (!id) return prev; + const text = getString(obj, "transcript") ?? ""; + return upsertTranscript(prev, id, "assistant", () => ({ + text, + isFinal: true, + })); + } + // An interrupted response can end without a transcript.done event. + case "response.done": { + if (!prev.some(e => e.role === "assistant" && !e.isFinal)) return prev; + return prev.map(e => + e.role === "assistant" && !e.isFinal ? { ...e, isFinal: true } : e + ); + } + default: + return prev; + } +} + +// Events can arrive out of order, so follow the server's previous_item_id chain. +function orderByPreviousItemId(entries: TranscriptEntry[]): TranscriptEntry[] { + const ids = new Set(entries.map(entry => entry.id)); + const successorOf = new Map( + entries + .filter(entry => entry.previousItemId) + .map(entry => [entry.previousItemId, entry] as const) + ); + + const ordered: TranscriptEntry[] = []; + const seen = new Set(); + let cursor = entries.find( + entry => !entry.previousItemId || !ids.has(entry.previousItemId) + ); + while (cursor && !seen.has(cursor.id)) { + seen.add(cursor.id); + ordered.push(cursor); + cursor = successorOf.get(cursor.id); + } + return [...ordered, ...entries.filter(entry => !seen.has(entry.id))]; +} + +function upsertTranscript( + prev: TranscriptEntry[], + id: string, + role: TranscriptEntry["role"], + updater: (entry: TranscriptEntry) => Partial +): TranscriptEntry[] { + const index = prev.findIndex(entry => entry.id === id); + if (index === -1) { + const base: TranscriptEntry = { id, role, text: "", isFinal: false }; + return [...prev, { ...base, ...updater(base) }]; + } + return prev.map((entry, i) => + i === index ? { ...entry, ...updater(entry) } : entry + ); +} + +function readSessionInfo(event: unknown): SessionInfo | null { + const session = getObject(asObject(event), "session"); + if (!session) return null; + const audio = getObject(session, "audio"); + return { + model: getString(session, "model") ?? "unknown", + voice: getString(getObject(audio, "output"), "voice"), + turnDetection: getString( + getObject(getObject(audio, "input"), "turn_detection"), + "type" + ), + }; +} + +function readResponseStatus(event: unknown): ResponseStatus { + const status = getString(getObject(asObject(event), "response"), "status"); + return isResponseStatus(status) ? status : "completed"; +} + +function isResponseStatus(value: string | null): value is ResponseStatus { + return value !== null && RESPONSE_STATUSES.has(value); +} + +function summarizeEvent(type: string, event: unknown): string | undefined { + switch (type) { + case "session.created": + case "session.updated": { + const info = readSessionInfo(event); + return info ? `${info.model} · ${info.voice ?? "default"}` : undefined; + } + case "conversation.item.input_audio_transcription.completed": + case "response.output_audio_transcript.done": + return getString(asObject(event), "transcript") ?? undefined; + case "response.done": + return readResponseStatus(event); + default: + return undefined; + } +} + +function extractErrorMessage(value: unknown, depth = 0): string { + if (value == null) return "Unknown error"; + if (typeof value === "string") return value; + if (value instanceof Error) return value.message; + const obj = asObject(value); + if (!obj) return String(value); + const message = getString(obj, "message") ?? getString(obj, "reason"); + if (message) return message; + if ("error" in obj && depth < 5) { + return extractErrorMessage(obj.error, depth + 1); + } + try { + return JSON.stringify(value); + } catch { + return "Unknown error"; + } +} + +function asObject(value: unknown): Record | null { + return typeof value === "object" && value !== null + ? (value as Record) + : null; +} + +function getObject( + obj: Record | null, + key: string +): Record | null { + return asObject(obj?.[key]); +} + +function getString( + obj: Record | null, + key: string +): string | null { + const value = obj?.[key]; + return typeof value === "string" ? value : null; +} diff --git a/agents/nextjs/openai-realtime/example/lib/realtime-session.ts b/agents/nextjs/openai-realtime/example/lib/realtime-session.ts new file mode 100644 index 00000000..365c9c71 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/lib/realtime-session.ts @@ -0,0 +1,27 @@ +import type { ElevenLabs } from "@elevenlabs/elevenlabs-js"; + +export const REALTIME_URL = "wss://api.elevenlabs.io/v1/convai/realtime"; +export const REALTIME_MODEL = "eleven_realtime_v1_mini"; +export const REALTIME_VOICE = "brian"; +export const AUDIO_SAMPLE_RATE = 24000; + +export const AGENT_INSTRUCTIONS = + "You are a helpful, friendly voice assistant. Keep your replies concise and conversational."; + +export const sessionConfig: ElevenLabs.OpenAiSessionConfigInput = { + type: "realtime", + model: REALTIME_MODEL, + instructions: AGENT_INSTRUCTIONS, + outputModalities: ["audio"], + audio: { + input: { + format: { type: "audio/pcm", rate: AUDIO_SAMPLE_RATE }, + turnDetection: { type: "semantic_vad", eagerness: "medium" }, + }, + output: { + format: { type: "audio/pcm" }, + voice: REALTIME_VOICE, + }, + }, + temperature: 0.7, +}; diff --git a/agents/nextjs/openai-realtime/example/lib/utils.ts b/agents/nextjs/openai-realtime/example/lib/utils.ts new file mode 100644 index 00000000..a5ef1935 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/lib/utils.ts @@ -0,0 +1,6 @@ +import { clsx, type ClassValue } from "clsx"; +import { twMerge } from "tailwind-merge"; + +export function cn(...inputs: ClassValue[]) { + return twMerge(clsx(inputs)); +} diff --git a/agents/nextjs/openai-realtime/example/next.config.ts b/agents/nextjs/openai-realtime/example/next.config.ts new file mode 100644 index 00000000..fff67474 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/next.config.ts @@ -0,0 +1,13 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { NextConfig } from "next"; + +const projectRoot = path.dirname(fileURLToPath(import.meta.url)); + +const nextConfig: NextConfig = { + turbopack: { + root: projectRoot, + }, +}; + +export default nextConfig; diff --git a/agents/nextjs/openai-realtime/example/package.json b/agents/nextjs/openai-realtime/example/package.json new file mode 100644 index 00000000..5c96eb54 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/package.json @@ -0,0 +1,36 @@ +{ + "name": "agents-openai-realtime-demo", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "eslint" + }, + "dependencies": { + "class-variance-authority": "^0.7.1", + "clsx": "^2.1.1", + "lucide-react": "^0.575.0", + "next": "16.1.6", + "radix-ui": "^1.4.3", + "react": "19.2.3", + "react-dom": "19.2.3", + "tailwind-merge": "^3.5.0", + "@elevenlabs/elevenlabs-js": "^2.74.0", + "@openai/agents-realtime": "^0.20.0", + "wavtools": "^0.1.5" + }, + "devDependencies": { + "@tailwindcss/postcss": "^4", + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "eslint": "^9", + "eslint-config-next": "16.1.6", + "shadcn": "^3.8.5", + "tailwindcss": "^4", + "tw-animate-css": "^1.4.0", + "typescript": "^5" + } +} diff --git a/agents/nextjs/openai-realtime/example/postcss.config.mjs b/agents/nextjs/openai-realtime/example/postcss.config.mjs new file mode 100644 index 00000000..61e36849 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/postcss.config.mjs @@ -0,0 +1,7 @@ +const config = { + plugins: { + "@tailwindcss/postcss": {}, + }, +}; + +export default config; diff --git a/agents/nextjs/openai-realtime/example/public/file.svg b/agents/nextjs/openai-realtime/example/public/file.svg new file mode 100644 index 00000000..004145cd --- /dev/null +++ b/agents/nextjs/openai-realtime/example/public/file.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/agents/nextjs/openai-realtime/example/public/globe.svg b/agents/nextjs/openai-realtime/example/public/globe.svg new file mode 100644 index 00000000..567f17b0 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/public/globe.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/agents/nextjs/openai-realtime/example/public/next.svg b/agents/nextjs/openai-realtime/example/public/next.svg new file mode 100644 index 00000000..5174b28c --- /dev/null +++ b/agents/nextjs/openai-realtime/example/public/next.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/agents/nextjs/openai-realtime/example/public/vercel.svg b/agents/nextjs/openai-realtime/example/public/vercel.svg new file mode 100644 index 00000000..77053960 --- /dev/null +++ b/agents/nextjs/openai-realtime/example/public/vercel.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/agents/nextjs/openai-realtime/example/public/window.svg b/agents/nextjs/openai-realtime/example/public/window.svg new file mode 100644 index 00000000..b2b2a44f --- /dev/null +++ b/agents/nextjs/openai-realtime/example/public/window.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/agents/nextjs/openai-realtime/example/tsconfig.json b/agents/nextjs/openai-realtime/example/tsconfig.json new file mode 100644 index 00000000..3a13f90a --- /dev/null +++ b/agents/nextjs/openai-realtime/example/tsconfig.json @@ -0,0 +1,34 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "react-jsx", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + } + }, + "include": [ + "next-env.d.ts", + "**/*.ts", + "**/*.tsx", + ".next/types/**/*.ts", + ".next/dev/types/**/*.ts", + "**/*.mts" + ], + "exclude": ["node_modules"] +} diff --git a/agents/nextjs/openai-realtime/setup.sh b/agents/nextjs/openai-realtime/setup.sh new file mode 100755 index 00000000..ecb53e3a --- /dev/null +++ b/agents/nextjs/openai-realtime/setup.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$DIR/../../.." && pwd)" +cd "$DIR" + +# Clean example/ but preserve node_modules for speed +if [ -d example ]; then + find example -mindepth 1 -maxdepth 1 ! -name node_modules ! -name .next -exec rm -rf {} + +fi +mkdir -p example + +rsync -a \ + --exclude node_modules --exclude .next \ + --exclude pnpm-lock.yaml --exclude package-lock.json \ + --exclude example \ + "$REPO_ROOT/templates/nextjs/" example/ + +if [ -f README.md ]; then + cp README.md example/README.md +fi + +cd example +export ELEVENLABS_VER=$(npm view @elevenlabs/elevenlabs-js version) +export OPENAI_REALTIME_VER=$(npm view @openai/agents-realtime version) +export WAVTOOLS_VER=$(npm view wavtools version) +node -e " + const pkg = JSON.parse(require('fs').readFileSync('package.json', 'utf8')); + pkg.name = 'agents-openai-realtime-demo'; + pkg.dependencies['@elevenlabs/elevenlabs-js'] = '^' + process.env.ELEVENLABS_VER; + pkg.dependencies['@openai/agents-realtime'] = '^' + process.env.OPENAI_REALTIME_VER; + pkg.dependencies['wavtools'] = '^' + process.env.WAVTOOLS_VER; + require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n'); +" + +mkdir -p app/api/token + +if [ -f "$DIR/.env" ]; then + cp "$DIR/.env" .env.local +fi + +pnpm install --config.confirmModulesPurge=false From aa60c004cddc50cb50c6e318f1dd2447a2b30478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Tue, 20 Oct 2026 15:18:47 +0200 Subject: [PATCH 2/6] style: format with the repo's pinned prettier Co-Authored-By: Claude Opus 5.5 --- .../openai-realtime/example/hooks/use-realtime-session.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts index b4d8ee97..e7ec23ea 100644 --- a/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts +++ b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts @@ -20,7 +20,11 @@ import { export type SessionStatus = "disconnected" | "connecting" | "connected"; export type ResponseStatus = - "in_progress" | "completed" | "cancelled" | "incomplete" | "failed"; + | "in_progress" + | "completed" + | "cancelled" + | "incomplete" + | "failed"; const RESPONSE_STATUSES = new Set([ "in_progress", From fb392a7d38cb2de4d9d07fd7eefb24f8dd289214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Tue, 20 Oct 2026 15:29:01 +0200 Subject: [PATCH 3/6] fix(agents): clean up realtime audio and handle server disconnects - Close the AudioContexts wavtools leaves open, so repeated Start/Stop cycles don't accumulate them. - Tear down when the transport disconnects, so the mic stops and the UI leaves "Connected". - Connect the player before the token fetch, so Safari creates its AudioContext within the click. Co-Authored-By: Claude Opus 5.5 --- agents/nextjs/openai-realtime/PROMPT.md | 6 ++- .../example/hooks/use-realtime-session.ts | 50 +++++++++++++------ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/agents/nextjs/openai-realtime/PROMPT.md b/agents/nextjs/openai-realtime/PROMPT.md index f749337a..c7bf4b17 100644 --- a/agents/nextjs/openai-realtime/PROMPT.md +++ b/agents/nextjs/openai-realtime/PROMPT.md @@ -27,13 +27,15 @@ Secure GET endpoint that mints a short-lived realtime client secret. Never expos Client hook that owns the realtime session. - Fetch `/api/token`, then create a `RealtimeAgent` and a `RealtimeSession` with an `OpenAIRealtimeWebSocket` transport using `REALTIME_URL`, the model, and the GA config (`outputModalities`, `audio.output.voice`). Connect with `session.connect({ apiKey: value })`. -- The SDK only wires audio for WebRTC. Capture and play PCM16 at 24kHz with `wavtools` (`WavRecorder`, `WavStreamPlayer`). Import `wavtools` dynamically inside `start` because it touches browser APIs at module load. +- The SDK only wires audio for WebRTC. Capture and play PCM16 at 24kHz with `wavtools` (`WavRecorder`, `WavStreamPlayer`). +- Call `player.connect()` before any other `await` in `start` (including the token fetch), so Safari creates the playback `AudioContext` within the click's user activation. - Stream mic audio continuously with `session.sendAudio`; `wavtools` types `data.mono` as `Int16Array` but it is already an `ArrayBuffer` at runtime, so pass it through as-is. - Play `audio` events via `add16BitPCM`, and interrupt playback on `audio_interrupted`. - Track user speaking (`input_audio_buffer.speech_started/stopped`), agent speaking (`audio_start`/`audio_stopped`), response in flight (`response.created`/`response.done`), and the negotiated session (`session.created`/`session.updated`) from `transport_event`. - Build the transcript from raw transport events: stream assistant text from `response.output_audio_transcript.delta`, take user text from `conversation.item.input_audio_transcription.completed`, finalize on `.done` or `response.done`, and order turns by `previous_item_id`. - Keep a capped log of transport events, skipping the high-frequency audio and transcript deltas. -- Tear down recorder, player, and session on stop or error. +- Tear down recorder, player, and session on stop, on a failed start, and when the transport reports `connection_change` as `disconnected`. Do not tear down on every `error` event; many are non-fatal. +- `wavtools` never closes its `AudioContext`s; close both the player's (`player.context`) and the recorder's (`recorder.source.context`) on teardown so repeated sessions don't leak them. ## 5. `app/page.tsx` diff --git a/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts index e7ec23ea..233e30eb 100644 --- a/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts +++ b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts @@ -7,7 +7,7 @@ import { type TransportEvent, } from "@openai/agents-realtime"; import { useCallback, useRef, useState } from "react"; -import type { WavRecorder, WavStreamPlayer } from "wavtools"; +import { WavRecorder, WavStreamPlayer } from "wavtools"; import { AGENT_INSTRUCTIONS, @@ -100,20 +100,26 @@ export function useRealtimeSession() { }, []); const teardown = useCallback(async () => { + const recorder = recorderRef.current; + recorderRef.current = null; + const recorderContext: unknown = recorder?.source?.context; try { - await recorderRef.current?.quit(); + await recorder?.quit(); } catch {} - recorderRef.current = null; + await closeAudioContext(recorderContext); + const player = playerRef.current; + playerRef.current = null; try { - playerRef.current?.interrupt(); + await player?.interrupt(); } catch {} - playerRef.current = null; + await closeAudioContext(player?.context); + const session = sessionRef.current; + sessionRef.current = null; try { - sessionRef.current?.close(); + session?.close(); } catch {} - sessionRef.current = null; setIsUserSpeaking(false); setIsAssistantSpeaking(false); @@ -171,6 +177,15 @@ export function useRealtimeSession() { setStatus("connecting"); try { + // The SDK only wires audio for WebRTC, so capture and play PCM16 ourselves over WebSocket. + const player = new WavStreamPlayer({ sampleRate: AUDIO_SAMPLE_RATE }); + const recorder = new WavRecorder({ sampleRate: AUDIO_SAMPLE_RATE }); + playerRef.current = player; + recorderRef.current = recorder; + // Connect the player before any other await so Safari creates its AudioContext within the click. + await player.connect(); + await recorder.begin(); + const tokenResponse = await fetch("/api/token"); const body: { value?: string; error?: string } = await tokenResponse .json() @@ -181,15 +196,6 @@ export function useRealtimeSession() { ); } - // The SDK only wires audio for WebRTC, so capture and play PCM16 ourselves over WebSocket. - const { WavRecorder, WavStreamPlayer } = await import("wavtools"); - const recorder = new WavRecorder({ sampleRate: AUDIO_SAMPLE_RATE }); - const player = new WavStreamPlayer({ sampleRate: AUDIO_SAMPLE_RATE }); - await recorder.begin(); - await player.connect(); - recorderRef.current = recorder; - playerRef.current = player; - const agent = new RealtimeAgent({ name: "ElevenLabs agent", instructions: AGENT_INSTRUCTIONS, @@ -215,6 +221,11 @@ export function useRealtimeSession() { session.on("audio_stopped", () => setIsAssistantSpeaking(false)); session.on("error", event => setError(extractErrorMessage(event))); session.on("transport_event", handleTransportEvent); + session.transport.on("connection_change", connectionStatus => { + if (connectionStatus !== "disconnected") return; + if (sessionRef.current !== session) return; + void teardown().then(() => setStatus("disconnected")); + }); await session.connect({ apiKey: body.value }); @@ -251,6 +262,13 @@ export function useRealtimeSession() { }; } +async function closeAudioContext(context: unknown) { + if (!(context instanceof AudioContext) || context.state === "closed") return; + try { + await context.close(); + } catch {} +} + function updateTranscript( prev: TranscriptEntry[], type: string, From dd4374ec8f4c3ecb04eb4ae80e57f5242cd586be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Tue, 20 Oct 2026 15:56:48 +0200 Subject: [PATCH 4/6] fix(agents): detach realtime refs before teardown awaits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A disconnect-triggered teardown read sessionRef after awaiting audio cleanup, so a Stop → Start during that window could close the new session and leave the UI showing disconnected. Co-Authored-By: Claude Opus 5.5 --- .../example/hooks/use-realtime-session.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts index 233e30eb..7f7d9bcd 100644 --- a/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts +++ b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts @@ -100,30 +100,31 @@ export function useRealtimeSession() { }, []); const teardown = useCallback(async () => { + // Detach before the first await so a concurrent start() can't have its refs torn down. const recorder = recorderRef.current; + const player = playerRef.current; + const session = sessionRef.current; recorderRef.current = null; + playerRef.current = null; + sessionRef.current = null; + setIsUserSpeaking(false); + setIsAssistantSpeaking(false); + setIsResponding(false); + const recorderContext: unknown = recorder?.source?.context; try { await recorder?.quit(); } catch {} await closeAudioContext(recorderContext); - const player = playerRef.current; - playerRef.current = null; try { await player?.interrupt(); } catch {} await closeAudioContext(player?.context); - const session = sessionRef.current; - sessionRef.current = null; try { session?.close(); } catch {} - - setIsUserSpeaking(false); - setIsAssistantSpeaking(false); - setIsResponding(false); }, []); const handleTransportEvent = useCallback( @@ -224,7 +225,8 @@ export function useRealtimeSession() { session.transport.on("connection_change", connectionStatus => { if (connectionStatus !== "disconnected") return; if (sessionRef.current !== session) return; - void teardown().then(() => setStatus("disconnected")); + setStatus("disconnected"); + void teardown(); }); await session.connect({ apiKey: body.value }); From 7649ff0f2220ca19e407fd751faa68071191256b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Tue, 20 Oct 2026 16:28:31 +0200 Subject: [PATCH 5/6] fix(agents): ignore stale realtime start attempts A disconnect during session.connect or recorder.record let the in-flight start() mark the UI connected, or tear down a newer session from its catch. Co-Authored-By: Claude Opus 5.5 --- .../example/hooks/use-realtime-session.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts index 7f7d9bcd..2905a20f 100644 --- a/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts +++ b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts @@ -88,6 +88,7 @@ export function useRealtimeSession() { const recorderRef = useRef(null); const playerRef = useRef(null); const eventIdRef = useRef(0); + const attemptRef = useRef(0); const logEvent = useCallback((type: string, summary?: string) => { const entry: RealtimeEventLogEntry = { @@ -100,7 +101,8 @@ export function useRealtimeSession() { }, []); const teardown = useCallback(async () => { - // Detach before the first await so a concurrent start() can't have its refs torn down. + // Detach and invalidate any in-flight start() before the first await. + attemptRef.current += 1; const recorder = recorderRef.current; const player = playerRef.current; const session = sessionRef.current; @@ -169,6 +171,8 @@ export function useRealtimeSession() { const start = useCallback(async () => { if (sessionRef.current) return; + const attempt = (attemptRef.current += 1); + const isStale = () => attemptRef.current !== attempt; setError(null); setTranscript([]); setSessionInfo(null); @@ -230,14 +234,17 @@ export function useRealtimeSession() { }); await session.connect({ apiKey: body.value }); + if (isStale()) return; // wavtools types `mono` as Int16Array, but at runtime it is already the PCM16 ArrayBuffer. await recorder.record(data => { session.sendAudio(data.mono as unknown as ArrayBuffer); }); + if (isStale()) return; setStatus("connected"); } catch (err) { + if (isStale()) return; setError(err instanceof Error ? err.message : String(err)); await teardown(); setStatus("disconnected"); From e6006e0d55ee86a4a2f60b64959f000363eafff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Tue, 20 Oct 2026 17:50:25 +0200 Subject: [PATCH 6/6] fix(agents): surface realtime connect errors after disconnect A socket closed before open rejects session.connect() right after emitting "disconnected", so the torn-down attempt skipped setError. Errors are now hidden only when a newer start() has begun. Co-Authored-By: Claude Opus 5.5 --- .../example/hooks/use-realtime-session.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts index 2905a20f..5daec8ca 100644 --- a/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts +++ b/agents/nextjs/openai-realtime/example/hooks/use-realtime-session.ts @@ -88,7 +88,8 @@ export function useRealtimeSession() { const recorderRef = useRef(null); const playerRef = useRef(null); const eventIdRef = useRef(0); - const attemptRef = useRef(0); + const latestStartRef = useRef(0); + const activeStartRef = useRef(0); const logEvent = useCallback((type: string, summary?: string) => { const entry: RealtimeEventLogEntry = { @@ -102,7 +103,7 @@ export function useRealtimeSession() { const teardown = useCallback(async () => { // Detach and invalidate any in-flight start() before the first await. - attemptRef.current += 1; + activeStartRef.current = 0; const recorder = recorderRef.current; const player = playerRef.current; const session = sessionRef.current; @@ -171,8 +172,9 @@ export function useRealtimeSession() { const start = useCallback(async () => { if (sessionRef.current) return; - const attempt = (attemptRef.current += 1); - const isStale = () => attemptRef.current !== attempt; + const id = (latestStartRef.current += 1); + activeStartRef.current = id; + const isStale = () => activeStartRef.current !== id; setError(null); setTranscript([]); setSessionInfo(null); @@ -244,8 +246,10 @@ export function useRealtimeSession() { setStatus("connected"); } catch (err) { + if (latestStartRef.current === id) { + setError(err instanceof Error ? err.message : String(err)); + } if (isStale()) return; - setError(err instanceof Error ? err.message : String(err)); await teardown(); setStatus("disconnected"); }