diff --git a/.changeset/expose-reasoning-tokens.md b/.changeset/expose-reasoning-tokens.md new file mode 100644 index 000000000..e5ef3f236 --- /dev/null +++ b/.changeset/expose-reasoning-tokens.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': minor +--- + +Expose LLM reasoning-token usage in metrics, model usage, logs, and traces. diff --git a/agents/etc/agents.api.md b/agents/etc/agents.api.md index c97be80ae..66f1b5024 100644 --- a/agents/etc/agents.api.md +++ b/agents/etc/agents.api.md @@ -1297,6 +1297,11 @@ const ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS = "gen_ai.usage.output_text_tokens"; // @public (undocumented) const ATTR_GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; +// Warning: (ae-missing-release-tag) "ATTR_GEN_AI_USAGE_REASONING_TOKENS" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +const ATTR_GEN_AI_USAGE_REASONING_TOKENS = "gen_ai.usage.reasoning_tokens"; + // Warning: (ae-missing-release-tag) "ATTR_INSTRUCTIONS" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -2519,6 +2524,7 @@ export interface CompletionUsage { promptCachedTokens: number; // (undocumented) promptTokens: number; + reasoningTokens?: number; serviceTier?: string; // (undocumented) totalTokens: number; @@ -4888,6 +4894,7 @@ export type LLMMetrics = { promptTokens: number; promptCachedTokens: number; cacheCreationTokens?: number; + reasoningTokens?: number; totalTokens: number; tokensPerSecond: number; speechId?: string; @@ -4919,6 +4926,7 @@ export type LLMModelUsage = { outputTokens: number; outputAudioTokens: number; outputTextTokens: number; + outputReasoningTokens?: number; sessionDurationMs: number; }; @@ -8027,6 +8035,7 @@ declare namespace traceTypes { ATTR_GEN_AI_USAGE_INPUT_CACHED_TOKENS, ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS, ATTR_GEN_AI_USAGE_OUTPUT_AUDIO_TOKENS, + ATTR_GEN_AI_USAGE_REASONING_TOKENS, EVENT_GEN_AI_SYSTEM_MESSAGE, EVENT_GEN_AI_USER_MESSAGE, EVENT_GEN_AI_ASSISTANT_MESSAGE, @@ -9159,8 +9168,8 @@ export const zipFunctionCallsAndOutputs: (event: FunctionToolsExecutedEvent) => // src/llm/chat_context.ts:76:3 - (ae-unresolved-link) The @link reference could not be resolved: The package "@livekit/agents" does not have an export "audio" // src/llm/tool_context.ts:702:3 - (ae-unresolved-link) The @link reference could not be resolved: The reference is ambiguous because "ToolFlag" has more than one declaration; you need to add a TSDoc member reference selector // src/llm/tool_context.ts:746:3 - (ae-unresolved-link) The @link reference could not be resolved: The reference is ambiguous because "ToolFlag" has more than one declaration; you need to add a TSDoc member reference selector -// src/metrics/base.ts:194:3 - (ae-forgotten-export) The symbol "RealtimeModelMetricsInputTokenDetails" needs to be exported by the entry point index.d.ts -// src/metrics/base.ts:198:3 - (ae-forgotten-export) The symbol "RealtimeModelMetricsOutputTokenDetails" needs to be exported by the entry point index.d.ts +// src/metrics/base.ts:199:3 - (ae-forgotten-export) The symbol "RealtimeModelMetricsInputTokenDetails" needs to be exported by the entry point index.d.ts +// src/metrics/base.ts:203:3 - (ae-forgotten-export) The symbol "RealtimeModelMetricsOutputTokenDetails" needs to be exported by the entry point index.d.ts // src/stt/stt.ts:358:3 - (ae-unresolved-link) The @link reference could not be resolved: The package "@livekit/agents" does not have an export "STT" // src/utils.ts:549:3 - (ae-unresolved-link) The @link reference could not be resolved: The package "@livekit/agents" does not have an export "cancelled" // src/voice/agent_session.ts:380:3 - (ae-unresolved-link) The @link reference could not be resolved: This type of declaration is not supported yet by the resolver diff --git a/agents/src/inference/llm.test.ts b/agents/src/inference/llm.test.ts index e6e4eca54..f1860509e 100644 --- a/agents/src/inference/llm.test.ts +++ b/agents/src/inference/llm.test.ts @@ -396,6 +396,40 @@ describe('inference.LLM X-LiveKit-Inference-Priority header', () => { }); }); +describe('inference.LLM reasoning usage', () => { + it('reports reasoning tokens from completion token details', async () => { + const chunks = await collectChatChunks([ + { + id: 'chatcmpl_test', + choices: [], + usage: { + completion_tokens: 40, + prompt_tokens: 7, + total_tokens: 47, + completion_tokens_details: { reasoning_tokens: 32 }, + }, + }, + ]); + + expect(chunks).toHaveLength(1); + expect(chunks[0]?.usage?.reasoningTokens).toBe(32); + expect(chunks[0]?.usage?.completionTokens).toBe(40); + expect(chunks[0]?.usage?.totalTokens).toBe(47); + }); + + it('defaults reasoning tokens to zero without completion token details', async () => { + const chunks = await collectChatChunks([ + { + id: 'chatcmpl_test', + choices: [], + usage: { completion_tokens: 0, prompt_tokens: 7, total_tokens: 0 }, + }, + ]); + + expect(chunks[0]?.usage?.reasoningTokens).toBe(0); + }); +}); + describe('inference.LLM streamed tool calls', () => { it('does not expose content alongside tool calls', async () => { const chunks = await collectChatChunks( diff --git a/agents/src/inference/llm.ts b/agents/src/inference/llm.ts index f94ca84d5..16f61fc15 100644 --- a/agents/src/inference/llm.ts +++ b/agents/src/inference/llm.ts @@ -542,6 +542,7 @@ export class LLMStream extends llm.LLMStream { completionTokens: usage.completion_tokens || 0, promptTokens: usage.prompt_tokens || 0, promptCachedTokens: usage.prompt_tokens_details?.cached_tokens || 0, + reasoningTokens: usage.completion_tokens_details?.reasoning_tokens || 0, totalTokens: usage.total_tokens || 0, }, }); diff --git a/agents/src/llm/llm.test.ts b/agents/src/llm/llm.test.ts index 272ae5da3..baa769f70 100644 --- a/agents/src/llm/llm.test.ts +++ b/agents/src/llm/llm.test.ts @@ -103,6 +103,31 @@ describe('LLMStream metrics', () => { expect(metrics.cacheCreationTokens).toBe(42); }); + + it('defaults reasoning tokens to zero', async () => { + const metrics = await collectMetrics(new MockLLM([])); + + expect(metrics.reasoningTokens).toBe(0); + }); + + it('carries reasoning tokens', async () => { + const metrics = await collectMetrics( + new MockLLM([ + { + id: '1', + usage: { + completionTokens: 100, + promptTokens: 20, + promptCachedTokens: 0, + reasoningTokens: 64, + totalTokens: 120, + }, + }, + ]), + ); + + expect(metrics.reasoningTokens).toBe(64); + }); }); describe('LLM prewarm lifecycle', () => { diff --git a/agents/src/llm/llm.ts b/agents/src/llm/llm.ts index 6270a130f..5950f33f5 100644 --- a/agents/src/llm/llm.ts +++ b/agents/src/llm/llm.ts @@ -31,6 +31,11 @@ export interface CompletionUsage { promptCachedTokens: number; /** Tokens used to write to the prompt cache. */ cacheCreationTokens?: number; + /** + * Completion tokens spent on hidden reasoning. Already included in `completionTokens`. + * Providers that do not report this separately leave it unset. + */ + reasoningTokens?: number; totalTokens: number; /** The service tier used for processing (e.g. 'default', 'priority', 'flex'). */ serviceTier?: string; @@ -350,6 +355,7 @@ export abstract class LLMStream implements AsyncIterableIterator { promptTokens: usage?.promptTokens || 0, promptCachedTokens: usage?.promptCachedTokens || 0, cacheCreationTokens: usage?.cacheCreationTokens || 0, + reasoningTokens: usage?.reasoningTokens || 0, totalTokens: usage?.totalTokens || 0, tokensPerSecond: (() => { if (durationMs <= 0) { @@ -370,6 +376,12 @@ export abstract class LLMStream implements AsyncIterableIterator { [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens, [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens, }); + if (metrics.reasoningTokens) { + this.#llmRequestSpan.setAttribute( + traceTypes.ATTR_GEN_AI_USAGE_REASONING_TOKENS, + metrics.reasoningTokens, + ); + } if (completionStartTime) { this.#llmRequestSpan.setAttribute( diff --git a/agents/src/metrics/base.ts b/agents/src/metrics/base.ts index 51ee04806..4b8cf392f 100644 --- a/agents/src/metrics/base.ts +++ b/agents/src/metrics/base.ts @@ -35,6 +35,11 @@ export type LLMMetrics = { promptCachedTokens: number; /** Tokens used to write to the prompt cache. Not all providers report this. */ cacheCreationTokens?: number; + /** + * Completion tokens spent on hidden reasoning. Already included in `completionTokens`. + * Not all providers report this separately. + */ + reasoningTokens?: number; totalTokens: number; tokensPerSecond: number; speechId?: string; diff --git a/agents/src/metrics/model_usage.test.ts b/agents/src/metrics/model_usage.test.ts index 841ed6771..b18ba7c3f 100644 --- a/agents/src/metrics/model_usage.test.ts +++ b/agents/src/metrics/model_usage.test.ts @@ -120,6 +120,37 @@ describe('model_usage', () => { expect((usage[0] as LLMModelUsage).inputCacheCreationTokens).toBe(50); }); + it('should aggregate reasoning tokens without adding them to output tokens', () => { + const metrics: LLMMetrics = { + type: 'llm_metrics', + label: 'test', + requestId: 'req1', + timestamp: Date.now(), + durationMs: 100, + ttftMs: 50, + cancelled: false, + completionTokens: 100, + promptTokens: 20, + promptCachedTokens: 0, + reasoningTokens: 64, + totalTokens: 120, + tokensPerSecond: 10, + }; + + collector.collect(metrics); + collector.collect({ + ...metrics, + completionTokens: 50, + reasoningTokens: 8, + totalTokens: 70, + }); + + const usage = collector.flatten(); + expect(usage).toHaveLength(1); + expect((usage[0] as LLMModelUsage).outputReasoningTokens).toBe(72); + expect((usage[0] as LLMModelUsage).outputTokens).toBe(150); + }); + it('should aggregate LLM metrics by provider and model', () => { const metrics1: LLMMetrics = { type: 'llm_metrics', diff --git a/agents/src/metrics/model_usage.ts b/agents/src/metrics/model_usage.ts index 8a8416db1..1a684ef81 100644 --- a/agents/src/metrics/model_usage.ts +++ b/agents/src/metrics/model_usage.ts @@ -41,6 +41,8 @@ export type LLMModelUsage = { outputAudioTokens: number; /** Output text tokens. */ outputTextTokens: number; + /** Output tokens spent on hidden reasoning. Already included in `outputTokens`. */ + outputReasoningTokens?: number; /** Total session connection duration in milliseconds (for session-based billing like xAI). */ sessionDurationMs: number; }; @@ -163,6 +165,7 @@ export class ModelUsageCollector { outputTokens: 0, outputAudioTokens: 0, outputTextTokens: 0, + outputReasoningTokens: 0, sessionDurationMs: 0, }; this.llmUsage.set(key, usage); @@ -247,6 +250,8 @@ export class ModelUsageCollector { usage.inputCacheCreationTokens = (usage.inputCacheCreationTokens ?? 0) + (metrics.cacheCreationTokens ?? 0); usage.outputTokens += metrics.completionTokens; + usage.outputReasoningTokens = + (usage.outputReasoningTokens ?? 0) + (metrics.reasoningTokens ?? 0); } else if (metrics.type === 'realtime_model_metrics') { const [provider, model] = this.extractProviderModel(metrics); const usage = this.getLLMUsage(provider, model); diff --git a/agents/src/metrics/utils.ts b/agents/src/metrics/utils.ts index f70f136c9..77c5f4042 100644 --- a/agents/src/metrics/utils.ts +++ b/agents/src/metrics/utils.ts @@ -18,6 +18,7 @@ export const logMetrics = (metrics: AgentMetrics) => { promptCachedTokens: metrics.promptCachedTokens, cacheCreationTokens: metrics.cacheCreationTokens ?? 0, outputTokens: metrics.completionTokens, + reasoningTokens: metrics.reasoningTokens ?? 0, tokensPerSecond: roundTwoDecimals(metrics.tokensPerSecond), }) .info('LLM metrics'); diff --git a/agents/src/telemetry/trace_types.test.ts b/agents/src/telemetry/trace_types.test.ts index b3053132c..ad6ee6106 100644 --- a/agents/src/telemetry/trace_types.test.ts +++ b/agents/src/telemetry/trace_types.test.ts @@ -173,6 +173,7 @@ const SAFE_KEYS = new Set([ 'gen_ai.usage.input_cached_tokens', 'gen_ai.usage.output_text_tokens', 'gen_ai.usage.output_audio_tokens', + 'gen_ai.usage.reasoning_tokens', 'gen_ai.system.message', 'gen_ai.user.message', 'gen_ai.assistant.message', diff --git a/agents/src/telemetry/trace_types.ts b/agents/src/telemetry/trace_types.ts index cb22e4b4a..dea855691 100644 --- a/agents/src/telemetry/trace_types.ts +++ b/agents/src/telemetry/trace_types.ts @@ -133,6 +133,7 @@ export const ATTR_GEN_AI_USAGE_INPUT_AUDIO_TOKENS = 'gen_ai.usage.input_audio_to export const ATTR_GEN_AI_USAGE_INPUT_CACHED_TOKENS = 'gen_ai.usage.input_cached_tokens'; export const ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS = 'gen_ai.usage.output_text_tokens'; export const ATTR_GEN_AI_USAGE_OUTPUT_AUDIO_TOKENS = 'gen_ai.usage.output_audio_tokens'; +export const ATTR_GEN_AI_USAGE_REASONING_TOKENS = 'gen_ai.usage.reasoning_tokens'; // OpenTelemetry GenAI event names (for structured logging) export const EVENT_GEN_AI_SYSTEM_MESSAGE = 'gen_ai.system.message';