Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/docs/src/content/docs/reference/config-and-env.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ related:
| `AI_REASONING_LEVEL` | No | Main-agent reasoning override: `none`, `low`, `medium`, `high`, or `xhigh`. Unset by default. An explicit profile reasoning level takes precedence when routing is enabled. |
| `AI_FAST_MODEL` | No | Faster model for lightweight tasks and routing/classification passes before the main turn begins. Defaults to `openai/gpt-6-luna`. |
| `AI_GUARDIAN_MODEL` | No | Model for Guardian action review. Defaults to `openai/gpt-6-luna`. |
| `AI_DECISION_MODEL` | No | Decision model for passive reply routing in subscribed threads. Uses the AI SDK decision API. Defaults to `typesafe-ai/jev`. |
| `AI_HANDOFF_MODEL` | No | Deprecated profile setting. Creates `handoff`. Defaults to `anthropic/claude-opus-5.5` with high reasoning. |
| `AI_MODEL_PROFILES` | No | Deprecated JSON map of profile names to model IDs for env-only setup. Names must match `^[a-z][a-z0-9_-]*$`. |
| `AI_EMBEDDING_MODEL` | No | Embedding model for plugin-owned vector retrieval. Defaults to `openai/text-embedding-3-small`; memory v1 stores fixed 1536-dimensional vectors. |
Expand Down Expand Up @@ -270,6 +271,7 @@ const app = await createApp({
reasoningLevel: "high",
},
},
decisionModelId: "typesafe-ai/jev",
fastModelId: "openai/gpt-6-luna",
guardianModelId: "openai/gpt-6-luna",
embeddingModelId: "openai/text-embedding-3-small",
Expand Down
2 changes: 1 addition & 1 deletion packages/junior-evals/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"@sentry/junior-plugin-api": "workspace:*",
"@sentry/junior-sentry": "workspace:*",
"@sentry/junior-testing": "workspace:*",
"ai": "^6.0.190",
"ai": "^7.0.153",
"chat": "4.29.0",
"hono": "^4.12.27",
"msw": "^2.14.6",
Expand Down
2 changes: 1 addition & 1 deletion packages/junior-evals/src/fixture/web.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ export interface WebSearchResult {
url: string;
}

const SEARCH_REQUEST_URL = "https://ai-gateway.vercel.sh/v3/ai/language-model";
const SEARCH_REQUEST_URL = "https://ai-gateway.vercel.sh/v4/ai/language-model";
const SEARCH_TOOL_ID = "gateway.parallel_search";

let searchResults: WebSearchResult[] = [];
Expand Down
2 changes: 1 addition & 1 deletion packages/junior-evals/src/recording-rules.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ const RECORDING_RULES: RecordingRule[] = [
// image generation and web search before the proxy.
name: "model",
match: { method: "POST", url: "https://ai-gateway.vercel.sh/" },
// AI SDK gateway requests, such as `/v3/ai/language-model`, name the
// AI SDK gateway requests, such as `/v4/ai/language-model`, name the
// model in a header, not in the body.
keyHeaders: [
"ai-language-model-id",
Expand Down
4 changes: 2 additions & 2 deletions packages/junior/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@
},
"dependencies": {
"@agentclientprotocol/sdk": "1.3.0",
"@ai-sdk/gateway": "^3.0.119",
"@ai-sdk/gateway": "^4.0.126",
"@chat-adapter/slack": "4.29.0",
"@chat-adapter/state-memory": "4.29.0",
"@chat-adapter/state-redis": "4.29.0",
Expand All @@ -91,7 +91,7 @@
"@vercel/oidc": "^3.4.1",
"@vercel/queue": "^0.2.0",
"@vercel/sandbox": "2.8.0",
"ai": "^6.0.190",
"ai": "^7.0.153",
"better-auth": "^1.6.22",
"chat": "4.29.0",
"commander": "^14.0.3",
Expand Down
6 changes: 3 additions & 3 deletions packages/junior/src/chat/app/services.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { completeObject, completeText } from "@/chat/pi/client";
import { completeDecision, completeText } from "@/chat/pi/client";
import { executeAgentRun as executeAgentRunImpl } from "@/chat/agent";
import type { SandboxEgressTracePropagationConfig } from "@/chat/sandbox/egress/tracing";
import {
Expand Down Expand Up @@ -86,8 +86,8 @@ export function createJuniorRuntimeServices(
executeTurn: async (run, saveResult) =>
await executeTurn(agentRunner, run, saveResult),
subscribedReplyPolicy: createSubscribedReplyPolicy({
completeObject:
overrides.subscribedReplyPolicy?.completeObject ?? completeObject,
completeDecision:
overrides.subscribedReplyPolicy?.completeDecision ?? completeDecision,
}),
visionContext,
};
Expand Down
13 changes: 13 additions & 0 deletions packages/junior/src/chat/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ const DEFAULT_ASSISTANT_LOADING_MESSAGES = [
] as const;

export interface BotModelConfig {
/** Decision model used by the passive reply router. */
decisionModelId?: string;
/** Model used for embeddings. */
embeddingModelId?: string;
/** Model used for low-cost structured and utility calls. */
Expand All @@ -94,6 +96,7 @@ export interface BotModelConfig {
export interface BotConfig {
contextWindowTokens: number;
crossActorMidRunMode: CrossActorMidRunMode;
decisionModelId: string;
defaultProfile: ModelProfile;
embeddingModelId: string;
fastModelId: string;
Expand Down Expand Up @@ -261,6 +264,7 @@ const DEFAULT_HANDOFF_MODEL_ID = resolveGatewayModel(
DEFAULT_MODEL_PROFILES.handoff.modelId,
).id;
const DEFAULT_WEB_SEARCH_MODEL_ID = resolveGatewayModel("openai/gpt-6-luna").id;
const DEFAULT_DECISION_MODEL_ID = "openai/gpt-6-luna-decisions";
const DEFAULT_EMBEDDING_MODEL_ID = "openai/text-embedding-3-small";
const DEFAULT_IMAGE_GENERATION_MODEL_ID = "google/gemini-3-pro-image";
const DEFAULT_VISION_MODEL_ID = resolveGatewayModel("openai/gpt-5.6-sol").id;
Expand Down Expand Up @@ -502,6 +506,8 @@ function readBotConfig(
reasoningLevel === undefined
? undefined
: parseTurnReasoningLevel(reasoningLevel),
decisionModelId:
toOptionalTrimmed(env.AI_DECISION_MODEL) ?? DEFAULT_DECISION_MODEL_ID,
fastModelId,
guardianModelId,
imageGenerationModelId:
Expand Down Expand Up @@ -825,6 +831,13 @@ export interface SlackReactionConfig {

/** Apply model overrides from createApp(). */
export function setBotModelConfig(config: BotModelConfig): void {
if (config.decisionModelId !== undefined) {
botConfig.decisionModelId = requireModelId(
"decisionModelId",
config.decisionModelId,
toOptionalTrimmed,
);
}
if (config.embeddingModelId !== undefined) {
botConfig.embeddingModelId = requireModelId(
"embeddingModelId",
Expand Down
99 changes: 99 additions & 0 deletions packages/junior/src/chat/pi/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,13 @@ export {
} from "@/chat/pi/gateway-auth";
import {
embedMany,
experimental_decide,
Experimental_DecisionRefusalError,
generateObject,
NoObjectGeneratedError,
type Experimental_DecisionQuestion,
type Experimental_DecisionResult,
type Experimental_DecisionState,
type GenerateObjectResult,
type LanguageModelUsage,
} from "ai";
Expand Down Expand Up @@ -179,6 +184,7 @@ export const GEN_AI_SERVER_ADDRESS = "ai-gateway.vercel.sh";
export const GEN_AI_SERVER_PORT = 443;
const GEN_AI_OPERATION_CHAT = "chat" as const;
const GEN_AI_OPERATION_EMBEDDINGS = "embeddings" as const;
const GEN_AI_OPERATION_DECIDE = "decide" as const;
// AI Gateway input rates in USD per million tokens. Keep estimates
// model-specific so unlisted configured models remain explicitly unknown.
const EMBEDDING_INPUT_COST_USD_PER_MILLION_TOKENS: Readonly<
Expand All @@ -194,6 +200,25 @@ const EMBEDDING_INPUT_COST_USD_PER_MILLION_TOKENS: Readonly<
"voyage/voyage-3.5": 0.06,
"voyage/voyage-3.5-lite": 0.02,
};
// Decision models bill input tokens only.
const DECISION_INPUT_COST_USD_PER_MILLION_TOKENS: Readonly<
Record<string, number>
> = {
"openai/gpt-6-luna-decisions": 0.1,
"typesafe-ai/jev": 0.042,
};

function decisionCostUsd(
modelId: string,
inputTokens: number | undefined,
): number | undefined {
const costPerMillionTokens =
DECISION_INPUT_COST_USD_PER_MILLION_TOKENS[modelId];
return costPerMillionTokens === undefined || inputTokens === undefined
? undefined
: (inputTokens * costPerMillionTokens) / 1_000_000;
}

function embeddingCostUsd(
modelId: string,
inputTokens: number,
Expand Down Expand Up @@ -552,6 +577,80 @@ export async function completeObject<TSchema extends ZodTypeAny>(params: {
};
}

/** Answer typed decision questions with an AI Gateway decision model, such as Jev. */
export async function completeDecision<
const TQuestions extends Record<string, Experimental_DecisionQuestion>,
>(params: {
modelId: string;
state: Experimental_DecisionState;
questions: TQuestions;
signal?: AbortSignal;
metadata?: Record<string, unknown>;
promptName?: string;
}): Promise<{
answers: Experimental_DecisionResult<TQuestions>["answers"];
costUsd?: number;
}> {
const credential = await resolveGatewayCredential();
const provider = createGatewayProvider(
credential ? { apiKey: credential.token } : {},
);
try {
const result = await withSpan(
`${GEN_AI_OPERATION_DECIDE} ${params.modelId}`,
"gen_ai.decide",
logContextFromMetadata(params.modelId, params.metadata),
async (setSpanAttributes) => {
const result = await experimental_decide({
model: provider.decisionModel(params.modelId),
state: params.state,
questions: params.questions,
...(params.signal !== undefined
? { abortSignal: params.signal }
: undefined),
});
const costUsd = decisionCostUsd(
params.modelId,
result.usage.inputTokens,
);
setSpanAttributes(
extractGenAiUsageAttributes({
...result.usage,
...(costUsd !== undefined
? { cost: { input: costUsd, total: costUsd } }
: undefined),
}),
);
return { answers: result.answers, costUsd };
},
{
"gen_ai.provider.name": GEN_AI_PROVIDER_NAME,
"gen_ai.operation.name": GEN_AI_OPERATION_DECIDE,
"gen_ai.request.model": params.modelId,
...(params.promptName
? { "gen_ai.prompt.name": params.promptName }
: undefined),
"server.address": GEN_AI_SERVER_ADDRESS,
"server.port": GEN_AI_SERVER_PORT,
"gen_ai.provider.auth_mode": credential?.mode ?? "api_key",
},
);
return {
answers: result.answers,
...(result.costUsd !== undefined
? { costUsd: result.costUsd }
: undefined),
};
} catch (error) {
throw createProviderError(error, {
...(Experimental_DecisionRefusalError.isInstance(error)
? { kind: "invalid_response" }
: undefined),
modelId: params.modelId,
});
}
}

/** Generate text embeddings through the host-owned AI Gateway provider. */
export async function embedTexts(params: {
modelId: string;
Expand Down
Loading
Loading