diff --git a/.changeset/wise-cats-stream.md b/.changeset/wise-cats-stream.md new file mode 100644 index 000000000..31145009d --- /dev/null +++ b/.changeset/wise-cats-stream.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-assemblyai': patch +--- + +Add support for AssemblyAI's Universal-3.6 Pro streaming model. diff --git a/plugins/assemblyai/etc/agents-plugin-assemblyai.api.md b/plugins/assemblyai/etc/agents-plugin-assemblyai.api.md index 0b84ad682..be7729fe8 100644 --- a/plugins/assemblyai/etc/agents-plugin-assemblyai.api.md +++ b/plugins/assemblyai/etc/agents-plugin-assemblyai.api.md @@ -72,7 +72,7 @@ export type STTEncoding = 'pcm_s16le' | 'pcm_mulaw'; // Warning: (ae-missing-release-tag) "STTModels" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export type STTModels = 'universal-streaming-english' | 'universal-streaming-multilingual' | 'u3-rt-pro' | 'u3-rt-pro-beta-1' | 'universal-3-5-pro' | 'u3-pro'; +export type STTModels = 'universal-streaming-english' | 'universal-streaming-multilingual' | 'u3-rt-pro' | 'u3-rt-pro-beta-1' | 'universal-3-5-pro' | 'universal-3-6-pro' | 'u3-pro'; // Warning: (ae-missing-release-tag) "STTOptions" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -85,6 +85,7 @@ export interface STTOptions { // (undocumented) baseUrl: string; bufferSizeMs: number; + continuousPartials?: boolean; // (undocumented) domain?: string; // (undocumented) @@ -94,8 +95,10 @@ export interface STTOptions { // (undocumented) formatTurns?: boolean; inactivityTimeout?: number; + interruptionDelay?: number; // (undocumented) keytermsPrompt?: string[]; + languageCodes?: string | string[]; // (undocumented) languageDetection?: boolean; // (undocumented) diff --git a/plugins/assemblyai/src/models.ts b/plugins/assemblyai/src/models.ts index 00a655379..3b928a6fc 100644 --- a/plugins/assemblyai/src/models.ts +++ b/plugins/assemblyai/src/models.ts @@ -8,6 +8,7 @@ export type STTModels = | 'u3-rt-pro' | 'u3-rt-pro-beta-1' | 'universal-3-5-pro' + | 'universal-3-6-pro' // Deprecated alias — AssemblyAI maps this to `universal-3-5-pro`, but the // Python plugin emits a warning and rewrites it. Kept here so TS users don't // break if they already pass it. diff --git a/plugins/assemblyai/src/stt.test.ts b/plugins/assemblyai/src/stt.test.ts index f4872caf5..9b6be3ee5 100644 --- a/plugins/assemblyai/src/stt.test.ts +++ b/plugins/assemblyai/src/stt.test.ts @@ -50,6 +50,115 @@ describe('AssemblyAI options', () => { ).not.toThrow(); }); + it('accepts universal-3-6-pro', () => { + const stt = new STT({ apiKey: 'test-key', speechModel: 'universal-3-6-pro' }); + + expect(stt.model).toBe('universal-3-6-pro'); + }); + + it('accepts u3-pro parameters for universal-3-6-pro', async () => { + const { wss, baseUrl } = await startWebSocketServer(); + let requestUrl = ''; + + wss.on('connection', (_ws, req) => { + requestUrl = req.url ?? ''; + }); + + try { + const stream = new STT({ + apiKey: 'test-key', + baseUrl, + speechModel: 'universal-3-6-pro', + prompt: 'medical dictation', + agentContext: "The agent asked for the patient's name.", + previousContextNTurns: 10, + interruptionDelay: 300, + voiceFocus: 'near-field', + mode: 'max_accuracy', + languageCodes: ['en', 'es'], + }).stream(); + + await waitUntil(() => requestUrl !== ''); + stream.close(); + + const url = new URL(`ws://127.0.0.1${requestUrl}`); + expect(url.searchParams.get('speech_model')).toBe('universal-3-6-pro'); + expect(url.searchParams.get('prompt')).toBe('medical dictation'); + expect(url.searchParams.get('agent_context')).toBe("The agent asked for the patient's name."); + expect(url.searchParams.get('previous_context_n_turns')).toBe('10'); + expect(url.searchParams.get('interruption_delay')).toBe('300'); + expect(url.searchParams.get('voice_focus')).toBe('near-field'); + expect(url.searchParams.get('mode')).toBe('max_accuracy'); + expect(JSON.parse(url.searchParams.get('language_codes')!)).toEqual(['en', 'es']); + } finally { + await closeWebSocketServer(wss); + } + }); + + it('applies u3-pro connection defaults to universal-3-6-pro', async () => { + const { wss, baseUrl } = await startWebSocketServer(); + let requestUrl = ''; + + wss.on('connection', (_ws, req) => { + requestUrl = req.url ?? ''; + }); + + try { + const stream = new STT({ + apiKey: 'test-key', + baseUrl, + speechModel: 'universal-3-6-pro', + }).stream(); + + await waitUntil(() => requestUrl !== ''); + stream.close(); + + const url = new URL(`ws://127.0.0.1${requestUrl}`); + expect(url.searchParams.get('speech_model')).toBe('universal-3-6-pro'); + expect(url.searchParams.get('min_turn_silence')).toBe('100'); + expect(url.searchParams.get('max_turn_silence')).toBe('100'); + expect(url.searchParams.get('language_detection')).toBe('true'); + } finally { + await closeWebSocketServer(wss); + } + }); + + it('allows family-only options for every u3-pro family model', () => { + const models = [ + 'u3-rt-pro', + 'u3-rt-pro-beta-1', + 'universal-3-5-pro', + 'universal-3-6-pro', + ] as const; + + for (const speechModel of models) { + expect( + () => + new STT({ + apiKey: 'test-key', + speechModel, + voiceFocus: 'far-field', + mode: 'min_latency', + languageCodes: ['en', 'es'], + }), + ).not.toThrow(); + } + }); + + it('enables chat context by default for every u3-pro family model', () => { + const models = [ + 'u3-rt-pro', + 'u3-rt-pro-beta-1', + 'universal-3-5-pro', + 'universal-3-6-pro', + 'u3-pro', + ] as const; + + for (const speechModel of models) { + expect(new STT({ apiKey: 'test-key', speechModel }).capabilities.chatContext).toBe(true); + } + }); + it('requires a u3-rt-pro model for agentContext', () => { expect( () => diff --git a/plugins/assemblyai/src/stt.ts b/plugins/assemblyai/src/stt.ts index f2f0db277..8be318ddf 100644 --- a/plugins/assemblyai/src/stt.ts +++ b/plugins/assemblyai/src/stt.ts @@ -23,12 +23,59 @@ import { WebSocket } from 'ws'; import type { STTEncoding, STTModels, VoiceFocus } from './models.js'; // Speech models in the Universal-3 Pro family, which share the same parameter support. -const U3_PRO_MODELS = ['u3-rt-pro', 'u3-rt-pro-beta-1', 'universal-3-5-pro'] as const; +const U3_PRO_MODELS = [ + 'u3-rt-pro', + 'u3-rt-pro-beta-1', + 'universal-3-5-pro', + 'universal-3-6-pro', +] as const; + +const U3_PRO_ONLY_PARAMS = [ + 'prompt', + 'agentContext', + 'previousContextNTurns', + 'continuousPartials', + 'interruptionDelay', + 'voiceFocus', + 'voiceFocusThreshold', + 'mode', + 'languageCodes', +] as const; + +const MAX_LANGUAGE_CODES = 10; +const MAX_AGENT_CONTEXT_CHARS = 1750; function isU3ProModel(model: STTModels): boolean { return U3_PRO_MODELS.includes(model as (typeof U3_PRO_MODELS)[number]); } +function normalizeLanguageCodes(languageCodes: string | string[]): string[] { + const codes = + typeof languageCodes === 'string' ? (languageCodes ? [languageCodes] : []) : languageCodes; + const normalized = [ + ...new Set(codes.map((code) => normalizeLanguage(code).split('-')[0] as string)), + ]; + if (normalized.length > MAX_LANGUAGE_CODES) { + throw new Error( + `languageCodes accepts at most ${MAX_LANGUAGE_CODES} codes (got ${normalized.length} after normalization)`, + ); + } + if (normalized.includes('multi') && normalized.length > 1) { + throw new Error( + "'multi' routes to the unsteered multilingual model and cannot be combined with other language codes", + ); + } + return normalized; +} + +function validateAgentContext(agentContext: string | undefined): void { + if (agentContext !== undefined && agentContext.length > MAX_AGENT_CONTEXT_CHARS) { + throw new Error( + `agentContext exceeds maximum length of ${MAX_AGENT_CONTEXT_CHARS} characters (got ${agentContext.length})`, + ); + } +} + // AssemblyAI Universal-Streaming (v3) message envelope. All fields are optional // since we narrow on `type` before reading anything else. interface StreamEventMessage { @@ -92,6 +139,10 @@ export interface STTOptions { /** Maximum silence (ms) before end-of-turn is forced regardless of confidence. */ maxTurnSilence?: number; formatTurns?: boolean; + /** Emit additional partial transcripts during long turns. Universal-3 Pro only. */ + continuousPartials?: boolean; + /** Delay before the first early partial is emitted, in milliseconds. Universal-3 Pro only. */ + interruptionDelay?: number; keytermsPrompt?: string[]; /** Only supported with the Universal-3 Pro model family. */ prompt?: string; @@ -120,11 +171,13 @@ export interface STTOptions { * or `max_accuracy`. Explicit turn-silence values still take precedence over mode defaults. */ mode?: 'min_latency' | 'balanced' | 'max_accuracy'; + /** Languages to steer transcription toward. Universal-3 Pro only. */ + languageCodes?: string | string[]; /** * When the model supports it, let an `AgentSession` push each assistant reply into - * `agentContext` so it is carried into the model's conversation context. Defaults to false; - * set true to enable. Prior user turns are carried automatically by the model regardless of - * this flag. Ignored on models without context support. + * `agentContext` so it is carried into the model's conversation context. Defaults to true for + * Universal-3 Pro models; set false to disable. Prior user turns are carried automatically by + * the model regardless of this flag. Ignored on models without context support. */ agentContextCarryover?: boolean; baseUrl: string; @@ -156,6 +209,12 @@ export class STT extends stt.STT { } constructor(opts: Partial = {}) { + validateAgentContext(opts.agentContext); + if (opts.languageCodes !== undefined) { + const languageCodes = normalizeLanguageCodes(opts.languageCodes); + opts.languageCodes = languageCodes.length > 0 ? languageCodes : undefined; + } + // u3-rt-pro family — "u3-pro" is normalized below — and is opt-in via the user) const rawModel = opts.speechModel ?? defaultSTTOptions.speechModel; const supportsCarryover = isU3ProModel(rawModel) || rawModel === 'u3-pro'; @@ -169,7 +228,7 @@ export class STT extends stt.STT { interimResults: true, alignedTranscript: 'word', keyterms: true, - chatContext: (opts.agentContextCarryover ?? false) && supportsCarryover, + chatContext: (opts.agentContextCarryover ?? true) && supportsCarryover, }); if (opts.speechModel === 'u3-pro') { @@ -179,14 +238,7 @@ export class STT extends stt.STT { const speechModel = opts.speechModel ?? defaultSTTOptions.speechModel; if (!isU3ProModel(speechModel)) { - for (const param of [ - 'prompt', - 'agentContext', - 'previousContextNTurns', - 'voiceFocus', - 'voiceFocusThreshold', - 'mode', - ] as const) { + for (const param of U3_PRO_ONLY_PARAMS) { if (opts[param] !== undefined) { throw new Error( `The '${param}' parameter is only supported with the ${U3_PRO_MODELS.join(', ')} models.`, @@ -220,8 +272,19 @@ export class STT extends stt.STT { } updateOptions(opts: Partial) { + validateAgentContext(opts.agentContext); + // session keyterms so a user update doesn't drop them) const nextOpts = { ...opts }; + if (nextOpts.languageCodes !== undefined) { + const speechModel = nextOpts.speechModel ?? this.#opts.speechModel; + if (!isU3ProModel(speechModel)) { + throw new Error( + `The 'languageCodes' parameter is only supported with the ${U3_PRO_MODELS.join(', ')} models.`, + ); + } + nextOpts.languageCodes = normalizeLanguageCodes(nextOpts.languageCodes); + } if (nextOpts.keytermsPrompt !== undefined) { this.#userKeyterms = [...nextOpts.keytermsPrompt]; nextOpts.keytermsPrompt = [...new Set([...this.#userKeyterms, ...this.#sessionKeyterms])]; @@ -261,7 +324,7 @@ export class STT extends stt.STT { override _pushConversationItem(ev: ConversationItemAddedEvent): void { const chatItem = ev.item; if (chatItem instanceof ChatMessage && chatItem.role === 'assistant' && chatItem.textContent) { - this.updateOptions({ agentContext: chatItem.textContent }); + this.updateOptions({ agentContext: chatItem.textContent.slice(-MAX_AGENT_CONTEXT_CHARS) }); } } @@ -309,18 +372,35 @@ export class SpeechStream extends stt.SpeechStream { } updateOptions(opts: Partial) { + validateAgentContext(opts.agentContext); + if (opts.languageCodes !== undefined) { + if (!isU3ProModel(this.#opts.speechModel)) { + throw new Error( + `The 'languageCodes' parameter is only supported with the ${U3_PRO_MODELS.join(', ')} models.`, + ); + } + opts.languageCodes = normalizeLanguageCodes(opts.languageCodes); + } + this.#opts = { ...this.#opts, ...opts }; const configMsg: Record = { type: 'UpdateConfiguration' }; if (opts.prompt !== undefined) configMsg.prompt = opts.prompt; if (opts.agentContext !== undefined) configMsg.agent_context = opts.agentContext; if (opts.keytermsPrompt !== undefined) configMsg.keyterms_prompt = opts.keytermsPrompt; + if (opts.languageCodes !== undefined) configMsg.language_codes = opts.languageCodes; if (opts.maxTurnSilence !== undefined) configMsg.max_turn_silence = opts.maxTurnSilence; if (opts.minTurnSilence !== undefined) configMsg.min_turn_silence = opts.minTurnSilence; if (opts.endOfTurnConfidenceThreshold !== undefined) { configMsg.end_of_turn_confidence_threshold = opts.endOfTurnConfidenceThreshold; } if (opts.vadThreshold !== undefined) configMsg.vad_threshold = opts.vadThreshold; + if (opts.continuousPartials !== undefined) { + configMsg.continuous_partials = opts.continuousPartials; + } + if (opts.interruptionDelay !== undefined) { + configMsg.interruption_delay = opts.interruptionDelay; + } // Only send if any actual fields (besides `type`) were specified. if (Object.keys(configMsg).length > 1) { @@ -392,6 +472,8 @@ export class SpeechStream extends stt.SpeechStream { encoding: this.#opts.encoding, speech_model: this.#opts.speechModel, format_turns: this.#opts.formatTurns, + continuous_partials: this.#opts.continuousPartials, + interruption_delay: this.#opts.interruptionDelay, end_of_turn_confidence_threshold: this.#opts.endOfTurnConfidenceThreshold, min_turn_silence: minSilence, max_turn_silence: maxSilence, @@ -400,6 +482,10 @@ export class SpeechStream extends stt.SpeechStream { ? JSON.stringify(this.#opts.keytermsPrompt) : undefined, language_detection: languageDetection, + language_codes: + this.#opts.languageCodes !== undefined && this.#opts.languageCodes.length > 0 + ? JSON.stringify(this.#opts.languageCodes) + : undefined, inactivity_timeout: this.#opts.inactivityTimeout, prompt: this.#opts.prompt, agent_context: this.#opts.agentContext,