fix(amd): resolve transcript-only greetings at eot

.changeset/fuzzy-pandas-greet.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents': minor
+---
+
+Resolve transcript-only AMD greetings at the endpointing backstop, preserve timeout transcripts, and allow `null` to force session LLM or STT reuse.

agents/etc/agents.api.md

@@ -858,14 +858,14 @@ export interface AMDOptions {
     humanSpeechThresholdMs?: number;
     // (undocumented)
     interruptOnMachine?: boolean;
-    llm?: LLM | string;
+    llm?: LLM | string | null;
     machineSilenceThresholdMs?: number;
     // Warning: (ae-unresolved-link) The @link reference could not be resolved: The package "@livekit/agents" does not have an export "DEFAULT_MAX_ENDPOINTING_DELAY_MS"
     maxEndpointingDelayMs?: number;
     noSpeechTimeoutMs?: number;
     participantIdentity?: string;
     prompt?: string;
-    stt?: STT | string;
+    stt?: STT | string | null;
     suppressCompatibilityWarning?: boolean;
     waitUntilFinished?: boolean;
 }

agents/src/voice/amd.test.ts

@@ -88,6 +88,29 @@ class StaticLLM extends LLM {
   }
 }
 
+class SequentialLLM extends LLM {
+  constructor(private readonly responses: string[]) {
+    super();
+  }
+
+  label(): string {
+    return 'sequential-llm';
+  }
+
+  chat(): LLMStream {
+    const response = this.responses.shift();
+    if (response === undefined) throw new Error('no response configured');
+    return {
+      async *[Symbol.asyncIterator](): AsyncGenerator<ChatChunk> {
+        yield {
+          id: 'sequential',
+          delta: { role: 'assistant', content: response },
+        };
+      },
+    } as unknown as LLMStream;
+  }
+}
+
 class MockSession extends EventEmitter {
   llm?: LLM;
   pauseReplyAuthorization = vi.fn();
@@ -494,7 +517,7 @@ describe('AMD', () => {
     expect(setAmd).toHaveBeenCalledWith(null);
   });
 
-  it('should fall back to session.llm when no cloud creds are available', async () => {
+  it('should inherit session models when cloud inference is unavailable', async () => {
     vi.stubEnv('LIVEKIT_URL', '');
     try {
       const session = new MockSession();
@@ -503,6 +526,10 @@ describe('AMD', () => {
       session.llm = llm;
       const amd = new AMD(asAgentSession(session), { detectionTimeoutMs: 50 });
 
+      expect((amd as unknown as { llm: LLM }).llm).toBe(llm);
+      expect((amd as unknown as { stt?: STT }).stt).toBeUndefined();
+      expect((amd as unknown as { source: string }).source).toBe('stt');
+
       const promise = amd.execute();
       pushTranscript(amd, 'Hello?');
       await expect(promise).resolves.toMatchObject({
@@ -514,6 +541,93 @@ describe('AMD', () => {
     }
   });
 
+  it('should reuse session models when null is explicit on cloud', () => {
+    vi.stubEnv('LIVEKIT_URL', 'wss://test.livekit.cloud');
+    vi.stubEnv('LIVEKIT_API_KEY', 'key');
+    vi.stubEnv('LIVEKIT_API_SECRET', 'test-secret-that-is-at-least-32-bytes');
+    const session = new MockSession();
+    const llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN }));
+    session.llm = llm;
+
+    const amd = new AMD(asAgentSession(session), {
+      llm: null,
+      stt: null,
+      suppressCompatibilityWarning: true,
+    });
+
+    expect((amd as unknown as { llm: LLM }).llm).toBe(llm);
+    expect((amd as unknown as { stt?: STT }).stt).toBeUndefined();
+    expect((amd as unknown as { source: string }).source).toBe('stt');
+  });
+
+  it('should auto-select cloud defaults when models are omitted', () => {
+    vi.stubEnv('LIVEKIT_URL', 'wss://test.livekit.cloud');
+    vi.stubEnv('LIVEKIT_API_KEY', 'key');
+    vi.stubEnv('LIVEKIT_API_SECRET', 'test-secret-that-is-at-least-32-bytes');
+    const session = new MockSession();
+    session.llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN }));
+
+    const amd = new AMD(asAgentSession(session), { suppressCompatibilityWarning: true });
+
+    expect((amd as unknown as { llm: LLM }).llm.model).toBe('google/gemini-3.1-flash-lite');
+    expect((amd as unknown as { stt?: STT }).stt?.model).toBe('cartesia/ink-whisper');
+    expect((amd as unknown as { source: string }).source).toBe('amd_stt');
+  });
+
+  it('should inherit a null LLM and use the omitted cloud STT default', () => {
+    vi.stubEnv('LIVEKIT_URL', 'wss://test.livekit.cloud');
+    vi.stubEnv('LIVEKIT_API_KEY', 'key');
+    vi.stubEnv('LIVEKIT_API_SECRET', 'test-secret-that-is-at-least-32-bytes');
+    const session = new MockSession();
+    const llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN }));
+    session.llm = llm;
+
+    const amd = new AMD(asAgentSession(session), {
+      llm: null,
+      suppressCompatibilityWarning: true,
+    });
+
+    expect((amd as unknown as { llm: LLM }).llm).toBe(llm);
+    expect((amd as unknown as { stt?: STT }).stt?.model).toBe('cartesia/ink-whisper');
+    expect((amd as unknown as { source: string }).source).toBe('amd_stt');
+  });
+
+  it('should use the omitted cloud LLM default and inherit a null STT', () => {
+    vi.stubEnv('LIVEKIT_URL', 'wss://test.livekit.cloud');
+    vi.stubEnv('LIVEKIT_API_KEY', 'key');
+    vi.stubEnv('LIVEKIT_API_SECRET', 'test-secret-that-is-at-least-32-bytes');
+    const session = new MockSession();
+    session.llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN }));
+
+    const amd = new AMD(asAgentSession(session), {
+      stt: null,
+      suppressCompatibilityWarning: true,
+    });
+
+    expect((amd as unknown as { llm: LLM }).llm.model).toBe('google/gemini-3.1-flash-lite');
+    expect((amd as unknown as { stt?: STT }).stt).toBeUndefined();
+    expect((amd as unknown as { source: string }).source).toBe('stt');
+  });
+
+  it('should use explicit models instead of cloud defaults', () => {
+    vi.stubEnv('LIVEKIT_URL', 'wss://test.livekit.cloud');
+    vi.stubEnv('LIVEKIT_API_KEY', 'key');
+    vi.stubEnv('LIVEKIT_API_SECRET', 'test-secret-that-is-at-least-32-bytes');
+    const session = new MockSession();
+    session.llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN }));
+    const llm = new StaticLLM(JSON.stringify({ category: AMDCategory.HUMAN }));
+
+    const amd = new AMD(asAgentSession(session), {
+      llm,
+      stt: 'deepgram/nova-3',
+      suppressCompatibilityWarning: true,
+    });
+
+    expect((amd as unknown as { llm: LLM }).llm).toBe(llm);
+    expect((amd as unknown as { stt?: STT }).stt?.model).toBe('deepgram/nova-3');
+    expect((amd as unknown as { source: string }).source).toBe('amd_stt');
+  });
+
   it('should throw when no cloud creds and session has no compatible LLM', () => {
     vi.stubEnv('LIVEKIT_URL', '');
     try {
@@ -753,7 +867,30 @@ describe('AMD', () => {
     await expect(promise).resolves.toMatchObject({ category: AMDCategory.MACHINE_VM });
   }, 5_000);
 
-  it('uses maxEndpointingDelay for transcripts without speech end', async () => {
+  it('uses maxEndpointingDelay to emit a human without VAD boundaries', async () => {
+    const session = new MockSession();
+    const llm = new StaticLLM(
+      JSON.stringify({ category: AMDCategory.HUMAN, reason: 'live person' }),
+    );
+    llm.on('error', () => {});
+    const amd = new AMD(asAgentSession(session), {
+      llm,
+      maxEndpointingDelayMs: 30,
+      detectionTimeoutMs: 5_000,
+      suppressCompatibilityWarning: true,
+    });
+
+    const promise = amd.execute();
+    await waitForListening(amd);
+    pushTranscript(amd, 'hello');
+
+    await expect(promise).resolves.toMatchObject({
+      category: AMDCategory.HUMAN,
+      reason: 'live person',
+    });
+  }, 5_000);
+
+  it('uses maxEndpointingDelay to emit a machine without VAD boundaries', async () => {
     const session = new MockSession();
     const llm = new StaticLLM(
       JSON.stringify({ category: AMDCategory.MACHINE_VM, reason: 'voicemail greeting' }),
@@ -762,7 +899,7 @@ describe('AMD', () => {
     const amd = new AMD(asAgentSession(session), {
       llm,
       maxEndpointingDelayMs: 30,
-      detectionTimeoutMs: 80,
+      detectionTimeoutMs: 5_000,
       suppressCompatibilityWarning: true,
     });
 
@@ -773,6 +910,59 @@ describe('AMD', () => {
     await expect(promise).resolves.toMatchObject({ category: AMDCategory.MACHINE_VM });
   }, 5_000);
 
+  it('waits for the latest transcript chunk before transcript EOT', async () => {
+    const session = new MockSession();
+    const llm = new SequentialLLM([
+      JSON.stringify({ category: AMDCategory.HUMAN, reason: 'partial greeting' }),
+      JSON.stringify({ category: AMDCategory.MACHINE_VM, reason: 'complete greeting' }),
+    ]);
+    llm.on('error', () => {});
+    const amd = new AMD(asAgentSession(session), {
+      llm,
+      maxEndpointingDelayMs: 80,
+      detectionTimeoutMs: 5_000,
+      suppressCompatibilityWarning: true,
+    });
+
+    const promise = amd.execute();
+    await waitForListening(amd);
+    pushTranscript(amd, 'hello');
+    await new Promise((resolve) => setTimeout(resolve, 50));
+    expect(await waitForPending(promise, 0)).toBe(false);
+
+    pushTranscript(amd, "you've reached");
+    expect(await waitForPending(promise, 50)).toBe(false);
+
+    await expect(promise).resolves.toMatchObject({
+      category: AMDCategory.MACHINE_VM,
+      transcript: "hello you've reached",
+    });
+  }, 5_000);
+
+  it('preserves the transcript on detection timeout', async () => {
+    const session = new MockSession();
+    const llm = new StaticLLM(
+      JSON.stringify({ category: AMDCategory.UNCERTAIN, reason: 'not enough context' }),
+    );
+    llm.on('error', () => {});
+    const amd = new AMD(asAgentSession(session), {
+      llm,
+      detectionTimeoutMs: 80,
+      maxEndpointingDelayMs: 30,
+      suppressCompatibilityWarning: true,
+    });
+
+    const promise = amd.execute();
+    await waitForListening(amd);
+    pushTranscript(amd, 'hello');
+
+    await expect(promise).resolves.toMatchObject({
+      category: AMDCategory.UNCERTAIN,
+      reason: 'detection_timeout',
+      transcript: 'hello',
+    });
+  }, 5_000);
+
   it('subtracts already-elapsed silence from maxEndpointingDelay on speech end', async () => {
     const session = new MockSession();
     const llm = new StaticLLM(

agents/src/voice/amd.ts

@@ -61,21 +61,23 @@ export interface AMDOptions {
    * - `LLM` instance: used as-is (caller-owned; AMD will not close it).
    * - `string`: treated as a Cloud Inference model id (e.g. `'openai/gpt-4o-mini'`)
    *   and an inference LLM is constructed (AMD-owned).
+   * - `null`: always reuse the session's LLM.
    * - `undefined` (default): auto-select — if LiveKit Cloud inference credentials
    *   are available in the environment, uses `'google/gemini-3.1-flash-lite'` via
    *   the inference gateway; otherwise falls back to the session's own LLM.
    */
-  llm?: LLM | string;
+  llm?: LLM | string | null;
   /**
    * Dedicated STT used to transcribe call audio for AMD.
    * - `STT` instance: used as-is (caller-owned; AMD will not close it).
    * - `string`: treated as a Cloud Inference model id (e.g. `'cartesia/ink-whisper'`)
    *   and an inference STT is constructed (AMD-owned).
+   * - `null`: always reuse the session's existing STT transcripts.
    * - `undefined` (default): auto-select — if LiveKit Cloud inference credentials
    *   are available in the environment, uses `'cartesia/ink-whisper'` via the
    *   inference gateway; otherwise reuses the session's existing STT transcripts.
    */
-  stt?: STT | string;
+  stt?: STT | string | null;
   interruptOnMachine?: boolean;
   /** If no speech is heard within this window, settle as UNCERTAIN (not a machine, so no interrupt). */
   noSpeechTimeoutMs?: number;
@@ -784,8 +786,9 @@ export class AMD extends (EventEmitter as new () => TypedEmitter<AMDCallbacks>)
   /**
    * Ref: python classifier.py `_try_emit_result` + `_can_emit` — releases a
    * verdict only when the silence gate is open AND, for everything except a
-   * confident human, the end-of-turn gate is open too. Humans release on
-   * silence alone so the agent can respond quickly.
+   * confident human, the end-of-turn gate is open too. When VAD misses speech
+   * end, EOT also opens the silence gate. Humans release on silence alone so
+   * the agent can respond quickly.
    */
   private tryEmitResult(): void {
     if (!this.verdictResult || this.settled) {
@@ -931,6 +934,10 @@ export class AMD extends (EventEmitter as new () => TypedEmitter<AMDCallbacks>)
   private onEotReached(): void {
     if (this.settled) return;
     this.clearTimer('eot');
+    if (this.speechActive || this.speechEndedAt === undefined) {
+      this.speechActive = false;
+      this.silenceReached = true;
+    }
     this.eotReached = true;
     this.tryEmitResult();
   }
@@ -1115,9 +1122,9 @@ export class AMD extends (EventEmitter as new () => TypedEmitter<AMDCallbacks>)
    * Mirrors python `_resolve_classifier`.
    * - `LLM` instance: caller-owned, used as-is.
    * - string: construct a Cloud Inference LLM (AMD-owned).
-   * - `undefined`: fall back to `session.llm`.
+   * - `null` or `undefined`: fall back to `session.llm`.
    */
-  private resolveLLM(option?: LLM | string): { llm: LLM; owned: boolean } {
+  private resolveLLM(option?: LLM | string | null): { llm: LLM; owned: boolean } {
     if (option instanceof LLM) {
       return { llm: option, owned: false };
     }
@@ -1138,9 +1145,9 @@ export class AMD extends (EventEmitter as new () => TypedEmitter<AMDCallbacks>)
    * Mirrors python `_InferenceSTT(stt) if isinstance(stt, str) else stt`.
    * - `STT` instance: caller-owned.
    * - string: AMD-owned Cloud Inference STT.
-   * - `undefined`: listen to session-level STT events.
+   * - `null` or `undefined`: listen to session-level STT events.
    */
-  private resolveSTT(option?: STT | string): {
+  private resolveSTT(option?: STT | string | null): {
     stt: STT | undefined;
     owned: boolean;
   } {