Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion llm_inference/model_inference.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import logging
from typing import Dict, Any
from openai import OpenAI
import re
import tiktoken

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -194,6 +195,11 @@ def _get_provider(self, model_name: str) -> str:
"qwen/qwen3-next-80b-a3b-instruct": "openrouter",
"Qwen/Qwen3-Coder-Next": "openrouter",
"deepseek/deepseek-v4-flash": "openrouter",
"google/gemini-3.8-flash-reasoning-low": "openrouter",
"z-ai/glm-5.3-flash": "openrouter",
"openai/gpt-6-luna": "openrouter",
"xiaomi/mimo-v2.6-flash": "openrouter",
"google/gemma-4-31b-it": "openrouter",
"x-ai/grok-4.1-fast": "openrouter",
"mistralai/devstral-2512:free": "openrouter",
"meta-llama/llama-3.3-70b-instruct": "openrouter",
Expand Down Expand Up @@ -321,8 +327,20 @@ def _call_openrouter(self, model_name: str, prompt: str) -> Dict[str, Any]:
api_key=openrouter_api_key,
)

# "<model>-reasoning-<effort>" selects the same model with an explicit
# OpenRouter reasoning effort (e.g. google/gemini-3.8-flash-reasoning-low).
api_model, extra_body = model_name, None
match = re.fullmatch(
r"(.+)-reasoning-(none|minimal|low|medium|high)", model_name
)
if match:
api_model = match.group(1)
extra_body = {"reasoning": {"effort": match.group(2)}}

response = client.chat.completions.create(
model=model_name, messages=[{"role": "user", "content": prompt}]
model=api_model,
messages=[{"role": "user", "content": prompt}],
extra_body=extra_body,
)

usage = getattr(response, "usage", None)
Expand Down
16 changes: 16 additions & 0 deletions model_cost/model_cost.json
Original file line number Diff line number Diff line change
Expand Up @@ -386,5 +386,21 @@
"google/gemma-4-31b-it": {
"input_token_price_per_million": 0.08,
"output_token_price_per_million": 0.35
},
"google/gemini-3.8-flash-reasoning-low": {
"input_token_price_per_million": 0.75,
"output_token_price_per_million": 3.75
},
"z-ai/glm-5.3-flash": {
"input_token_price_per_million": 0.15,
"output_token_price_per_million": 0.5
},
"openai/gpt-6-luna": {
"input_token_price_per_million": 0.1,
"output_token_price_per_million": 0.5
},
"xiaomi/mimo-v2.6-flash": {
"input_token_price_per_million": 0.14,
"output_token_price_per_million": 0.28
}
}
14 changes: 14 additions & 0 deletions router_inference/config/argrouter.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"pipeline_params": {
"router_name": "argrouter",
"router_cls_name": "external_prediction_file",
"models": [
"google/gemini-3.8-flash-reasoning-low",
"google/gemma-4-31b-it",
"z-ai/glm-5.3-flash",
"openai/gpt-6-luna",
"xiaomi/mimo-v2.6-flash"
],
"description": "Routes on the question content only: the first and last paragraph of the prompt (instruction and answer-format text) are dropped and field labels / option letters stripped with generic patterns; no RouterArena config, template or dataset file is read. Per-model P(correct) from logistic regression + kNN on local bge-small embeddings of that content, expected cost from the billed token usage (reasoning included) of the nearest external training prompts; picks argmax P(correct) - lambda * expected cost. Calibration sources weighted equally. Trained only on held-out items from the same public source benchmarks, with every RouterArena item excluded."
}
}
Loading
Loading