diff --git a/.changeset/fuzzy-pandas-greet.md b/.changeset/fuzzy-pandas-greet.md new file mode 100644 index 000000000..1bddbef65 --- /dev/null +++ b/.changeset/fuzzy-pandas-greet.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': minor +--- + +Resolve transcript-only AMD greetings at the endpointing backstop, preserve timeout transcripts, and allow `null` to force session LLM or STT reuse. diff --git a/agents/etc/agents.api.md b/agents/etc/agents.api.md index c97be80ae..21c581fbb 100644 --- a/agents/etc/agents.api.md +++ b/agents/etc/agents.api.md @@ -858,14 +858,14 @@ export interface AMDOptions { humanSpeechThresholdMs?: number; // (undocumented) interruptOnMachine?: boolean; - llm?: LLM | string; + llm?: LLM | string | null; machineSilenceThresholdMs?: number; // Warning: (ae-unresolved-link) The @link reference could not be resolved: The package "@livekit/agents" does not have an export "DEFAULT_MAX_ENDPOINTING_DELAY_MS" maxEndpointingDelayMs?: number; noSpeechTimeoutMs?: number; participantIdentity?: string; prompt?: string; - stt?: STT | string; + stt?: STT | string | null; suppressCompatibilityWarning?: boolean; waitUntilFinished?: boolean; } diff --git a/agents/src/voice/amd.test.ts b/agents/src/voice/amd.test.ts index 2a2fa5461..05d61e090 100644 --- a/agents/src/voice/amd.test.ts +++ b/agents/src/voice/amd.test.ts @@ -88,6 +88,29 @@ class StaticLLM extends LLM { } } +class SequentialLLM extends LLM { + constructor(private readonly responses: string[]) { + super(); + } + + label(): string { + return 'sequential-llm'; + } + + chat(): LLMStream { + const response = this.responses.shift(); + if (response === undefined) throw new Error('no response configured'); + return { + async *[Symbol.asyncIterator](): AsyncGenerator { + yield { + id: 'sequential', + delta: { role: 'assistant', content: response }, + }; + }, + } as unknown as LLMStream; + } +} + class MockSession extends EventEmitter { llm?: LLM; pauseReplyAuthorization = vi.fn(); @@ -494,7 +517,7 @@ describe('AMD', () => { expect(setAmd).toHaveBeenCalledWith(null); }); - it('should fall back to session.llm when no cloud creds are available', async () => { + it('should inherit session models when cloud inference is unavailable', async () => { vi.stubEnv('LIVEKIT_URL', ''); try { const session = new MockSession(); @@ -503,6 +526,10 @@ describe('AMD', () => { session.llm = llm; const amd = new AMD(asAgentSession(session), { detectionTimeoutMs: 50 }); + expect((amd as unknown as { llm: LLM }).llm).toBe(llm); + expect((amd as unknown as { stt?: STT }).stt).toBeUndefined(); + expect((amd as unknown as { source: string }).source).toBe('stt'); + const promise = amd.execute(); pushTranscript(amd, 'Hello?'); await expect(promise).resolves.toMatchObject({ @@ -514,6 +541,93 @@ describe('AMD', () => { } }); + it('should reuse session models when null is explicit on cloud', () => { + vi.stubEnv('LIVEKIT_URL', 'wss://test.livekit.cloud'); + vi.stubEnv('LIVEKIT_API_KEY', 'key'); + vi.stubEnv('LIVEKIT_API_SECRET', 'test-secret-that-is-at-least-32-bytes'); + const session = new MockSession(); + const llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN })); + session.llm = llm; + + const amd = new AMD(asAgentSession(session), { + llm: null, + stt: null, + suppressCompatibilityWarning: true, + }); + + expect((amd as unknown as { llm: LLM }).llm).toBe(llm); + expect((amd as unknown as { stt?: STT }).stt).toBeUndefined(); + expect((amd as unknown as { source: string }).source).toBe('stt'); + }); + + it('should auto-select cloud defaults when models are omitted', () => { + vi.stubEnv('LIVEKIT_URL', 'wss://test.livekit.cloud'); + vi.stubEnv('LIVEKIT_API_KEY', 'key'); + vi.stubEnv('LIVEKIT_API_SECRET', 'test-secret-that-is-at-least-32-bytes'); + const session = new MockSession(); + session.llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN })); + + const amd = new AMD(asAgentSession(session), { suppressCompatibilityWarning: true }); + + expect((amd as unknown as { llm: LLM }).llm.model).toBe('google/gemini-3.1-flash-lite'); + expect((amd as unknown as { stt?: STT }).stt?.model).toBe('cartesia/ink-whisper'); + expect((amd as unknown as { source: string }).source).toBe('amd_stt'); + }); + + it('should inherit a null LLM and use the omitted cloud STT default', () => { + vi.stubEnv('LIVEKIT_URL', 'wss://test.livekit.cloud'); + vi.stubEnv('LIVEKIT_API_KEY', 'key'); + vi.stubEnv('LIVEKIT_API_SECRET', 'test-secret-that-is-at-least-32-bytes'); + const session = new MockSession(); + const llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN })); + session.llm = llm; + + const amd = new AMD(asAgentSession(session), { + llm: null, + suppressCompatibilityWarning: true, + }); + + expect((amd as unknown as { llm: LLM }).llm).toBe(llm); + expect((amd as unknown as { stt?: STT }).stt?.model).toBe('cartesia/ink-whisper'); + expect((amd as unknown as { source: string }).source).toBe('amd_stt'); + }); + + it('should use the omitted cloud LLM default and inherit a null STT', () => { + vi.stubEnv('LIVEKIT_URL', 'wss://test.livekit.cloud'); + vi.stubEnv('LIVEKIT_API_KEY', 'key'); + vi.stubEnv('LIVEKIT_API_SECRET', 'test-secret-that-is-at-least-32-bytes'); + const session = new MockSession(); + session.llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN })); + + const amd = new AMD(asAgentSession(session), { + stt: null, + suppressCompatibilityWarning: true, + }); + + expect((amd as unknown as { llm: LLM }).llm.model).toBe('google/gemini-3.1-flash-lite'); + expect((amd as unknown as { stt?: STT }).stt).toBeUndefined(); + expect((amd as unknown as { source: string }).source).toBe('stt'); + }); + + it('should use explicit models instead of cloud defaults', () => { + vi.stubEnv('LIVEKIT_URL', 'wss://test.livekit.cloud'); + vi.stubEnv('LIVEKIT_API_KEY', 'key'); + vi.stubEnv('LIVEKIT_API_SECRET', 'test-secret-that-is-at-least-32-bytes'); + const session = new MockSession(); + session.llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN })); + const llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN })); + + const amd = new AMD(asAgentSession(session), { + llm, + stt: 'deepgram/nova-3', + suppressCompatibilityWarning: true, + }); + + expect((amd as unknown as { llm: LLM }).llm).toBe(llm); + expect((amd as unknown as { stt?: STT }).stt?.model).toBe('deepgram/nova-3'); + expect((amd as unknown as { source: string }).source).toBe('amd_stt'); + }); + it('should throw when no cloud creds and session has no compatible LLM', () => { vi.stubEnv('LIVEKIT_URL', ''); try { @@ -753,7 +867,30 @@ describe('AMD', () => { await expect(promise).resolves.toMatchObject({ category: AMDCategory.MACHINE_VM }); }, 5_000); - it('uses maxEndpointingDelay for transcripts without speech end', async () => { + it('uses maxEndpointingDelay to emit a human without VAD boundaries', async () => { + const session = new MockSession(); + const llm = new StaticLLM( + JSON.stringify({ category: AMDCategory.HUMAN, reason: 'live person' }), + ); + llm.on('error', () => {}); + const amd = new AMD(asAgentSession(session), { + llm, + maxEndpointingDelayMs: 30, + detectionTimeoutMs: 5_000, + suppressCompatibilityWarning: true, + }); + + const promise = amd.execute(); + await waitForListening(amd); + pushTranscript(amd, 'hello'); + + await expect(promise).resolves.toMatchObject({ + category: AMDCategory.HUMAN, + reason: 'live person', + }); + }, 5_000); + + it('uses maxEndpointingDelay to emit a machine without VAD boundaries', async () => { const session = new MockSession(); const llm = new StaticLLM( JSON.stringify({ category: AMDCategory.MACHINE_VM, reason: 'voicemail greeting' }), @@ -762,7 +899,7 @@ describe('AMD', () => { const amd = new AMD(asAgentSession(session), { llm, maxEndpointingDelayMs: 30, - detectionTimeoutMs: 80, + detectionTimeoutMs: 5_000, suppressCompatibilityWarning: true, }); @@ -773,6 +910,59 @@ describe('AMD', () => { await expect(promise).resolves.toMatchObject({ category: AMDCategory.MACHINE_VM }); }, 5_000); + it('waits for the latest transcript chunk before transcript EOT', async () => { + const session = new MockSession(); + const llm = new SequentialLLM([ + JSON.stringify({ category: AMDCategory.HUMAN, reason: 'partial greeting' }), + JSON.stringify({ category: AMDCategory.MACHINE_VM, reason: 'complete greeting' }), + ]); + llm.on('error', () => {}); + const amd = new AMD(asAgentSession(session), { + llm, + maxEndpointingDelayMs: 80, + detectionTimeoutMs: 5_000, + suppressCompatibilityWarning: true, + }); + + const promise = amd.execute(); + await waitForListening(amd); + pushTranscript(amd, 'hello'); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(await waitForPending(promise, 0)).toBe(false); + + pushTranscript(amd, "you've reached"); + expect(await waitForPending(promise, 50)).toBe(false); + + await expect(promise).resolves.toMatchObject({ + category: AMDCategory.MACHINE_VM, + transcript: "hello you've reached", + }); + }, 5_000); + + it('preserves the transcript on detection timeout', async () => { + const session = new MockSession(); + const llm = new StaticLLM( + JSON.stringify({ category: AMDCategory.UNCERTAIN, reason: 'not enough context' }), + ); + llm.on('error', () => {}); + const amd = new AMD(asAgentSession(session), { + llm, + detectionTimeoutMs: 80, + maxEndpointingDelayMs: 30, + suppressCompatibilityWarning: true, + }); + + const promise = amd.execute(); + await waitForListening(amd); + pushTranscript(amd, 'hello'); + + await expect(promise).resolves.toMatchObject({ + category: AMDCategory.UNCERTAIN, + reason: 'detection_timeout', + transcript: 'hello', + }); + }, 5_000); + it('subtracts already-elapsed silence from maxEndpointingDelay on speech end', async () => { const session = new MockSession(); const llm = new StaticLLM( diff --git a/agents/src/voice/amd.ts b/agents/src/voice/amd.ts index 519a4e7fe..d28c83ce0 100644 --- a/agents/src/voice/amd.ts +++ b/agents/src/voice/amd.ts @@ -61,21 +61,23 @@ export interface AMDOptions { * - `LLM` instance: used as-is (caller-owned; AMD will not close it). * - `string`: treated as a Cloud Inference model id (e.g. `'openai/gpt-4o-mini'`) * and an inference LLM is constructed (AMD-owned). + * - `null`: always reuse the session's LLM. * - `undefined` (default): auto-select — if LiveKit Cloud inference credentials * are available in the environment, uses `'google/gemini-3.1-flash-lite'` via * the inference gateway; otherwise falls back to the session's own LLM. */ - llm?: LLM | string; + llm?: LLM | string | null; /** * Dedicated STT used to transcribe call audio for AMD. * - `STT` instance: used as-is (caller-owned; AMD will not close it). * - `string`: treated as a Cloud Inference model id (e.g. `'cartesia/ink-whisper'`) * and an inference STT is constructed (AMD-owned). + * - `null`: always reuse the session's existing STT transcripts. * - `undefined` (default): auto-select — if LiveKit Cloud inference credentials * are available in the environment, uses `'cartesia/ink-whisper'` via the * inference gateway; otherwise reuses the session's existing STT transcripts. */ - stt?: STT | string; + stt?: STT | string | null; interruptOnMachine?: boolean; /** If no speech is heard within this window, settle as UNCERTAIN (not a machine, so no interrupt). */ noSpeechTimeoutMs?: number; @@ -784,8 +786,9 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) /** * Ref: python classifier.py `_try_emit_result` + `_can_emit` — releases a * verdict only when the silence gate is open AND, for everything except a - * confident human, the end-of-turn gate is open too. Humans release on - * silence alone so the agent can respond quickly. + * confident human, the end-of-turn gate is open too. When VAD misses speech + * end, EOT also opens the silence gate. Humans release on silence alone so + * the agent can respond quickly. */ private tryEmitResult(): void { if (!this.verdictResult || this.settled) { @@ -931,6 +934,10 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) private onEotReached(): void { if (this.settled) return; this.clearTimer('eot'); + if (this.speechActive || this.speechEndedAt === undefined) { + this.speechActive = false; + this.silenceReached = true; + } this.eotReached = true; this.tryEmitResult(); } @@ -1115,9 +1122,9 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) * Mirrors python `_resolve_classifier`. * - `LLM` instance: caller-owned, used as-is. * - string: construct a Cloud Inference LLM (AMD-owned). - * - `undefined`: fall back to `session.llm`. + * - `null` or `undefined`: fall back to `session.llm`. */ - private resolveLLM(option?: LLM | string): { llm: LLM; owned: boolean } { + private resolveLLM(option?: LLM | string | null): { llm: LLM; owned: boolean } { if (option instanceof LLM) { return { llm: option, owned: false }; } @@ -1138,9 +1145,9 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) * Mirrors python `_InferenceSTT(stt) if isinstance(stt, str) else stt`. * - `STT` instance: caller-owned. * - string: AMD-owned Cloud Inference STT. - * - `undefined`: listen to session-level STT events. + * - `null` or `undefined`: listen to session-level STT events. */ - private resolveSTT(option?: STT | string): { + private resolveSTT(option?: STT | string | null): { stt: STT | undefined; owned: boolean; } { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d8945e5f8..fcbe3957b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -629,7 +629,7 @@ importers: version: 3.1.3 openai: specifier: ^6.8.1 - version: 6.8.1(ws@8.21.1)(zod@4.3.6) + version: 6.8.1(ws@8.21.3)(zod@4.3.6) devDependencies: '@livekit/agents': specifier: workspace:* @@ -802,7 +802,7 @@ importers: version: 0.4.0 openai: specifier: ^6.8.1 - version: 6.8.1(ws@8.21.1)(zod@4.3.6) + version: 6.8.1(ws@8.21.3)(zod@4.3.6) devDependencies: '@livekit/agents': specifier: workspace:* @@ -1186,7 +1186,7 @@ importers: dependencies: openai: specifier: ^6.8.1 - version: 6.8.1(ws@8.21.1)(zod@4.3.6) + version: 6.8.1(ws@8.21.3)(zod@4.3.6) zod: specifier: ^3.25.76 || ^4.1.8 version: 4.3.6 @@ -8709,11 +8709,6 @@ snapshots: platform: 1.3.6 protobufjs: 7.6.5 - openai@6.8.1(ws@8.21.1)(zod@4.3.6): - optionalDependencies: - ws: 8.21.1 - zod: 4.3.6 - openai@6.8.1(ws@8.21.3)(zod@3.25.76): optionalDependencies: ws: 8.21.3