feat(metrics): expose reasoning tokens in LLM metrics and traces

.changeset/expose-reasoning-tokens.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents': minor
+---
+
+Expose LLM reasoning-token usage in metrics, model usage, logs, and traces.

agents/etc/agents.api.md

@@ -1297,6 +1297,11 @@ const ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS = "gen_ai.usage.output_text_tokens";
 // @public (undocumented)
 const ATTR_GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens";
 
+// Warning: (ae-missing-release-tag) "ATTR_GEN_AI_USAGE_REASONING_TOKENS" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+const ATTR_GEN_AI_USAGE_REASONING_TOKENS = "gen_ai.usage.reasoning_tokens";
+
 // Warning: (ae-missing-release-tag) "ATTR_INSTRUCTIONS" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -2519,6 +2524,7 @@ export interface CompletionUsage {
     promptCachedTokens: number;
     // (undocumented)
     promptTokens: number;
+    reasoningTokens?: number;
     serviceTier?: string;
     // (undocumented)
     totalTokens: number;
@@ -4888,6 +4894,7 @@ export type LLMMetrics = {
     promptTokens: number;
     promptCachedTokens: number;
     cacheCreationTokens?: number;
+    reasoningTokens?: number;
     totalTokens: number;
     tokensPerSecond: number;
     speechId?: string;
@@ -4919,6 +4926,7 @@ export type LLMModelUsage = {
     outputTokens: number;
     outputAudioTokens: number;
     outputTextTokens: number;
+    outputReasoningTokens?: number;
     sessionDurationMs: number;
 };
 
@@ -8027,6 +8035,7 @@ declare namespace traceTypes {
         ATTR_GEN_AI_USAGE_INPUT_CACHED_TOKENS,
         ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS,
         ATTR_GEN_AI_USAGE_OUTPUT_AUDIO_TOKENS,
+        ATTR_GEN_AI_USAGE_REASONING_TOKENS,
         EVENT_GEN_AI_SYSTEM_MESSAGE,
         EVENT_GEN_AI_USER_MESSAGE,
         EVENT_GEN_AI_ASSISTANT_MESSAGE,
@@ -9159,8 +9168,8 @@ export const zipFunctionCallsAndOutputs: (event: FunctionToolsExecutedEvent) =>
 // src/llm/chat_context.ts:76:3 - (ae-unresolved-link) The @link reference could not be resolved: The package "@livekit/agents" does not have an export "audio"
 // src/llm/tool_context.ts:702:3 - (ae-unresolved-link) The @link reference could not be resolved: The reference is ambiguous because "ToolFlag" has more than one declaration; you need to add a TSDoc member reference selector
 // src/llm/tool_context.ts:746:3 - (ae-unresolved-link) The @link reference could not be resolved: The reference is ambiguous because "ToolFlag" has more than one declaration; you need to add a TSDoc member reference selector
-// src/metrics/base.ts:194:3 - (ae-forgotten-export) The symbol "RealtimeModelMetricsInputTokenDetails" needs to be exported by the entry point index.d.ts
-// src/metrics/base.ts:198:3 - (ae-forgotten-export) The symbol "RealtimeModelMetricsOutputTokenDetails" needs to be exported by the entry point index.d.ts
+// src/metrics/base.ts:199:3 - (ae-forgotten-export) The symbol "RealtimeModelMetricsInputTokenDetails" needs to be exported by the entry point index.d.ts
+// src/metrics/base.ts:203:3 - (ae-forgotten-export) The symbol "RealtimeModelMetricsOutputTokenDetails" needs to be exported by the entry point index.d.ts
 // src/stt/stt.ts:358:3 - (ae-unresolved-link) The @link reference could not be resolved: The package "@livekit/agents" does not have an export "STT"
 // src/utils.ts:549:3 - (ae-unresolved-link) The @link reference could not be resolved: The package "@livekit/agents" does not have an export "cancelled"
 // src/voice/agent_session.ts:380:3 - (ae-unresolved-link) The @link reference could not be resolved: This type of declaration is not supported yet by the resolver

agents/src/inference/llm.test.ts

@@ -396,6 +396,40 @@ describe('inference.LLM X-LiveKit-Inference-Priority header', () => {
   });
 });
 
+describe('inference.LLM reasoning usage', () => {
+  it('reports reasoning tokens from completion token details', async () => {
+    const chunks = await collectChatChunks([
+      {
+        id: 'chatcmpl_test',
+        choices: [],
+        usage: {
+          completion_tokens: 40,
+          prompt_tokens: 7,
+          total_tokens: 47,
+          completion_tokens_details: { reasoning_tokens: 32 },
+        },
+      },
+    ]);
+
+    expect(chunks).toHaveLength(1);
+    expect(chunks[0]?.usage?.reasoningTokens).toBe(32);
+    expect(chunks[0]?.usage?.completionTokens).toBe(40);
+    expect(chunks[0]?.usage?.totalTokens).toBe(47);
+  });
+
+  it('defaults reasoning tokens to zero without completion token details', async () => {
+    const chunks = await collectChatChunks([
+      {
+        id: 'chatcmpl_test',
+        choices: [],
+        usage: { completion_tokens: 0, prompt_tokens: 7, total_tokens: 0 },
+      },
+    ]);
+
+    expect(chunks[0]?.usage?.reasoningTokens).toBe(0);
+  });
+});
+
 describe('inference.LLM streamed tool calls', () => {
   it('does not expose content alongside tool calls', async () => {
     const chunks = await collectChatChunks(

agents/src/inference/llm.ts

@@ -542,6 +542,7 @@ export class LLMStream extends llm.LLMStream {
               completionTokens: usage.completion_tokens || 0,
               promptTokens: usage.prompt_tokens || 0,
               promptCachedTokens: usage.prompt_tokens_details?.cached_tokens || 0,
+              reasoningTokens: usage.completion_tokens_details?.reasoning_tokens || 0,
               totalTokens: usage.total_tokens || 0,
             },
           });

agents/src/llm/llm.test.ts

@@ -103,6 +103,31 @@ describe('LLMStream metrics', () => {
 
     expect(metrics.cacheCreationTokens).toBe(42);
   });
+
+  it('defaults reasoning tokens to zero', async () => {
+    const metrics = await collectMetrics(new MockLLM([]));
+
+    expect(metrics.reasoningTokens).toBe(0);
+  });
+
+  it('carries reasoning tokens', async () => {
+    const metrics = await collectMetrics(
+      new MockLLM([
+        {
+          id: '1',
+          usage: {
+            completionTokens: 100,
+            promptTokens: 20,
+            promptCachedTokens: 0,
+            reasoningTokens: 64,
+            totalTokens: 120,
+          },
+        },
+      ]),
+    );
+
+    expect(metrics.reasoningTokens).toBe(64);
+  });
 });
 
 describe('LLM prewarm lifecycle', () => {

agents/src/llm/llm.ts

@@ -31,6 +31,11 @@ export interface CompletionUsage {
   promptCachedTokens: number;
   /** Tokens used to write to the prompt cache. */
   cacheCreationTokens?: number;
+  /**
+   * Completion tokens spent on hidden reasoning. Already included in `completionTokens`.
+   * Providers that do not report this separately leave it unset.
+   */
+  reasoningTokens?: number;
   totalTokens: number;
   /** The service tier used for processing (e.g. 'default', 'priority', 'flex'). */
   serviceTier?: string;
@@ -350,6 +355,7 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
       promptTokens: usage?.promptTokens || 0,
       promptCachedTokens: usage?.promptCachedTokens || 0,
       cacheCreationTokens: usage?.cacheCreationTokens || 0,
+      reasoningTokens: usage?.reasoningTokens || 0,
       totalTokens: usage?.totalTokens || 0,
       tokensPerSecond: (() => {
         if (durationMs <= 0) {
@@ -370,6 +376,12 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
         [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,
         [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,
       });
+      if (metrics.reasoningTokens) {
+        this.#llmRequestSpan.setAttribute(
+          traceTypes.ATTR_GEN_AI_USAGE_REASONING_TOKENS,
+          metrics.reasoningTokens,
+        );
+      }
 
       if (completionStartTime) {
         this.#llmRequestSpan.setAttribute(

agents/src/metrics/base.ts

@@ -35,6 +35,11 @@ export type LLMMetrics = {
   promptCachedTokens: number;
   /** Tokens used to write to the prompt cache. Not all providers report this. */
   cacheCreationTokens?: number;
+  /**
+   * Completion tokens spent on hidden reasoning. Already included in `completionTokens`.
+   * Not all providers report this separately.
+   */
+  reasoningTokens?: number;
   totalTokens: number;
   tokensPerSecond: number;
   speechId?: string;

agents/src/metrics/model_usage.test.ts

@@ -120,6 +120,37 @@ describe('model_usage', () => {
         expect((usage[0] as LLMModelUsage).inputCacheCreationTokens).toBe(50);
       });
 
+      it('should aggregate reasoning tokens without adding them to output tokens', () => {
+        const metrics: LLMMetrics = {
+          type: 'llm_metrics',
+          label: 'test',
+          requestId: 'req1',
+          timestamp: Date.now(),
+          durationMs: 100,
+          ttftMs: 50,
+          cancelled: false,
+          completionTokens: 100,
+          promptTokens: 20,
+          promptCachedTokens: 0,
+          reasoningTokens: 64,
+          totalTokens: 120,
+          tokensPerSecond: 10,
+        };
+
+        collector.collect(metrics);
+        collector.collect({
+          ...metrics,
+          completionTokens: 50,
+          reasoningTokens: 8,
+          totalTokens: 70,
+        });
+
+        const usage = collector.flatten();
+        expect(usage).toHaveLength(1);
+        expect((usage[0] as LLMModelUsage).outputReasoningTokens).toBe(72);
+        expect((usage[0] as LLMModelUsage).outputTokens).toBe(150);
+      });
+
       it('should aggregate LLM metrics by provider and model', () => {
         const metrics1: LLMMetrics = {
           type: 'llm_metrics',

agents/src/metrics/model_usage.ts

@@ -41,6 +41,8 @@ export type LLMModelUsage = {
   outputAudioTokens: number;
   /** Output text tokens. */
   outputTextTokens: number;
+  /** Output tokens spent on hidden reasoning. Already included in `outputTokens`. */
+  outputReasoningTokens?: number;
   /** Total session connection duration in milliseconds (for session-based billing like xAI). */
   sessionDurationMs: number;
 };
@@ -163,6 +165,7 @@ export class ModelUsageCollector {
         outputTokens: 0,
         outputAudioTokens: 0,
         outputTextTokens: 0,
+        outputReasoningTokens: 0,
         sessionDurationMs: 0,
       };
       this.llmUsage.set(key, usage);
@@ -247,6 +250,8 @@ export class ModelUsageCollector {
       usage.inputCacheCreationTokens =
         (usage.inputCacheCreationTokens ?? 0) + (metrics.cacheCreationTokens ?? 0);
       usage.outputTokens += metrics.completionTokens;
+      usage.outputReasoningTokens =
+        (usage.outputReasoningTokens ?? 0) + (metrics.reasoningTokens ?? 0);
     } else if (metrics.type === 'realtime_model_metrics') {
       const [provider, model] = this.extractProviderModel(metrics);
       const usage = this.getLLMUsage(provider, model);

agents/src/metrics/utils.ts

@@ -18,6 +18,7 @@ export const logMetrics = (metrics: AgentMetrics) => {
         promptCachedTokens: metrics.promptCachedTokens,
         cacheCreationTokens: metrics.cacheCreationTokens ?? 0,
         outputTokens: metrics.completionTokens,
+        reasoningTokens: metrics.reasoningTokens ?? 0,
         tokensPerSecond: roundTwoDecimals(metrics.tokensPerSecond),
       })
       .info('LLM metrics');

agents/src/telemetry/trace_types.test.ts

@@ -173,6 +173,7 @@ const SAFE_KEYS = new Set([
   'gen_ai.usage.input_cached_tokens',
   'gen_ai.usage.output_text_tokens',
   'gen_ai.usage.output_audio_tokens',
+  'gen_ai.usage.reasoning_tokens',
   'gen_ai.system.message',
   'gen_ai.user.message',
   'gen_ai.assistant.message',

agents/src/telemetry/trace_types.ts

@@ -133,6 +133,7 @@ export const ATTR_GEN_AI_USAGE_INPUT_AUDIO_TOKENS = 'gen_ai.usage.input_audio_to
 export const ATTR_GEN_AI_USAGE_INPUT_CACHED_TOKENS = 'gen_ai.usage.input_cached_tokens';
 export const ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS = 'gen_ai.usage.output_text_tokens';
 export const ATTR_GEN_AI_USAGE_OUTPUT_AUDIO_TOKENS = 'gen_ai.usage.output_audio_tokens';
+export const ATTR_GEN_AI_USAGE_REASONING_TOKENS = 'gen_ai.usage.reasoning_tokens';
 
 // OpenTelemetry GenAI event names (for structured logging)
 export const EVENT_GEN_AI_SYSTEM_MESSAGE = 'gen_ai.system.message';