Skip to content

Add cached token fields to TokenUsage and parse OpenAI-compatible cache usage #259

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yaman-694 wants to merge 2 commits into
base: trunk
Choose a base branch
from
Open

Add cached token fields to TokenUsage and parse OpenAI-compatible cache usage #259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4382027
Add cached token fields to TokenUsage and parse OpenAI cached_tokens
yaman-694 Jul 14, 2026
969377a
Parse cache_write_tokens into cache creation token count
yaman-694 Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions src/Providers/OpenAiCompatibleImplementation/AbstractOpenAiCompatibleTextGenerationModel.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,11 @@
* @phpstan-type UsageData array{
* prompt_tokens?: int,
* completion_tokens?: int,
* total_tokens?: int
* total_tokens?: int,
* prompt_tokens_details?: array{
* cached_tokens?: int,
* cache_write_tokens?: int
* }
* }
* @phpstan-type ResponseData array{
* id?: string,
Expand Down Expand Up @@ -607,7 +611,10 @@ protected function parseResponseToGenerativeAiResult(Response $response): Genera
$tokenUsage = new TokenUsage(
$usage['prompt_tokens'] ?? 0,
$usage['completion_tokens'] ?? 0,
$usage['total_tokens'] ?? 0
$usage['total_tokens'] ?? 0,
null,
$usage['prompt_tokens_details']['cached_tokens'] ?? null,
$usage['prompt_tokens_details']['cache_write_tokens'] ?? null
);
} else {
$tokenUsage = new TokenUsage(0, 0, 0);
Expand Down
79 changes: 75 additions & 4 deletions src/Results/DTO/TokenUsage.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,18 @@
* Note that thought tokens are a subset of completion tokens, not additive.
* In other words: completionTokens - thoughtTokens = tokens of actual output content.
*
* Similarly, cached tokens are a subset of prompt tokens, not additive.
* In other words: promptTokens - cachedTokens = prompt tokens that were not served from a provider cache.
*
* @since 0.1.0
*
* @phpstan-type TokenUsageArrayShape array{
* promptTokens: int,
* completionTokens: int,
* totalTokens: int,
* thoughtTokens?: int
* thoughtTokens?: int,
* cachedTokens?: int,
* cacheCreationTokens?: int
* }
*
* @extends AbstractDataTransferObject<TokenUsageArrayShape>
Expand All @@ -32,6 +37,8 @@ class TokenUsage extends AbstractDataTransferObject
public const KEY_COMPLETION_TOKENS = 'completionTokens';
public const KEY_TOTAL_TOKENS = 'totalTokens';
public const KEY_THOUGHT_TOKENS = 'thoughtTokens';
public const KEY_CACHED_TOKENS = 'cachedTokens';
public const KEY_CACHE_CREATION_TOKENS = 'cacheCreationTokens';
/**
* @var int Number of tokens in the prompt.
*/
Expand All @@ -52,6 +59,16 @@ class TokenUsage extends AbstractDataTransferObject
*/
private ?int $thoughtTokens;

/**
* @var int|null Number of prompt tokens served from a provider cache, as a subset of prompt tokens.
*/
private ?int $cachedTokens;

/**
* @var int|null Number of tokens written to a provider cache while processing the request.
*/
private ?int $cacheCreationTokens;

/**
* Constructor.
*
Expand All @@ -61,13 +78,24 @@ class TokenUsage extends AbstractDataTransferObject
* @param int $completionTokens Number of tokens in the completion, including any thought tokens.
* @param int $totalTokens Total number of tokens used.
* @param int|null $thoughtTokens Number of tokens used for thinking, as a subset of completion tokens.
* @param int|null $cachedTokens Number of prompt tokens served from a provider cache, as a subset of prompt
* tokens.
* @param int|null $cacheCreationTokens Number of tokens written to a provider cache while processing the request.
*/
public function __construct(int $promptTokens, int $completionTokens, int $totalTokens, ?int $thoughtTokens = null)
{
public function __construct(
int $promptTokens,
int $completionTokens,
int $totalTokens,
?int $thoughtTokens = null,
?int $cachedTokens = null,
?int $cacheCreationTokens = null
) {
$this->promptTokens = $promptTokens;
$this->completionTokens = $completionTokens;
$this->totalTokens = $totalTokens;
$this->thoughtTokens = $thoughtTokens;
$this->cachedTokens = $cachedTokens;
$this->cacheCreationTokens = $cacheCreationTokens;
}

/**
Expand Down Expand Up @@ -118,6 +146,30 @@ public function getThoughtTokens(): ?int
return $this->thoughtTokens;
}

/**
* Gets the number of cached tokens, which is a subset of the prompt token count.
*
* @since n.e.x.t
*
* @return int|null The cached token count or null if not available.
*/
public function getCachedTokens(): ?int
{
return $this->cachedTokens;
}

/**
* Gets the number of tokens written to a provider cache while processing the request.
*
* @since n.e.x.t
*
* @return int|null The cache creation token count or null if not available.
*/
public function getCacheCreationTokens(): ?int
{
return $this->cacheCreationTokens;
}

/**
* {@inheritDoc}
*
Expand All @@ -144,6 +196,15 @@ public static function getJsonSchema(): array
'type' => 'integer',
'description' => 'Number of tokens used for thinking, as a subset of completion tokens.',
],
self::KEY_CACHED_TOKENS => [
'type' => 'integer',
'description' => 'Number of prompt tokens served from a provider cache, as a subset of prompt'
. ' tokens.',
],
self::KEY_CACHE_CREATION_TOKENS => [
'type' => 'integer',
'description' => 'Number of tokens written to a provider cache while processing the request.',
],
],
'required' => [self::KEY_PROMPT_TOKENS, self::KEY_COMPLETION_TOKENS, self::KEY_TOTAL_TOKENS],
];
Expand All @@ -168,6 +229,14 @@ public function toArray(): array
$data[self::KEY_THOUGHT_TOKENS] = $this->thoughtTokens;
}

if ($this->cachedTokens !== null) {
$data[self::KEY_CACHED_TOKENS] = $this->cachedTokens;
}

if ($this->cacheCreationTokens !== null) {
$data[self::KEY_CACHE_CREATION_TOKENS] = $this->cacheCreationTokens;
}

return $data;
}

Expand All @@ -188,7 +257,9 @@ public static function fromArray(array $array): self
$array[self::KEY_PROMPT_TOKENS],
$array[self::KEY_COMPLETION_TOKENS],
$array[self::KEY_TOTAL_TOKENS],
$array[self::KEY_THOUGHT_TOKENS] ?? null
$array[self::KEY_THOUGHT_TOKENS] ?? null,
$array[self::KEY_CACHED_TOKENS] ?? null,
$array[self::KEY_CACHE_CREATION_TOKENS] ?? null
);
}
}
99 changes: 99 additions & 0 deletions ...viders/OpenAiCompatibleImplementation/AbstractOpenAiCompatibleTextGenerationModelTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,105 @@ public function testGenerateTextResultSuccess(): void
$this->assertEquals(15, $result->getTokenUsage()->getTotalTokens());
}

/**
* Tests generateTextResult() surfaces cache token counts from prompt tokens details.
*
* @return void
*/
public function testGenerateTextResultParsesCachedTokens(): void
{
$prompt = [new Message(MessageRoleEnum::user(), [new MessagePart('Hello')])];
$response = new Response(
200,
[],
json_encode([
'id' => 'chatcmpl-123',
'choices' => [
[
'message' => [
'role' => 'assistant',
'content' => 'Hi there!',
],
'finish_reason' => 'stop',
],
],
'usage' => [
'prompt_tokens' => 2048,
'completion_tokens' => 5,
'total_tokens' => 2053,
'prompt_tokens_details' => [
'cached_tokens' => 1024,
'cache_write_tokens' => 512,
],
],
])
);

$this->mockRequestAuthentication
->expects($this->once())
->method('authenticateRequest')
->willReturnArgument(0);

$this->mockHttpTransporter
->expects($this->once())
->method('send')
->willReturn($response);

$model = $this->createModel();
$result = $model->generateTextResult($prompt);

$this->assertEquals(2048, $result->getTokenUsage()->getPromptTokens());
$this->assertEquals(1024, $result->getTokenUsage()->getCachedTokens());
$this->assertEquals(512, $result->getTokenUsage()->getCacheCreationTokens());
}

/**
* Tests generateTextResult() returns null cached tokens when the provider omits prompt tokens details.
*
* @return void
*/
public function testGenerateTextResultWithoutCachedTokens(): void
{
$prompt = [new Message(MessageRoleEnum::user(), [new MessagePart('Hello')])];
$response = new Response(
200,
[],
json_encode([
'id' => 'chatcmpl-123',
'choices' => [
[
'message' => [
'role' => 'assistant',
'content' => 'Hi there!',
],
'finish_reason' => 'stop',
],
],
'usage' => [
'prompt_tokens' => 10,
'completion_tokens' => 5,
'total_tokens' => 15,
],
])
);

$this->mockRequestAuthentication
->expects($this->once())
->method('authenticateRequest')
->willReturnArgument(0);

$this->mockHttpTransporter
->expects($this->once())
->method('send')
->willReturn($response);

$model = $this->createModel();
$result = $model->generateTextResult($prompt);

$this->assertNull($result->getTokenUsage()->getCachedTokens());
$this->assertNull($result->getTokenUsage()->getCacheCreationTokens());
}

/**
* Tests generateTextResult() method on API failure.
*
Expand Down
Loading
Loading