Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions app/src/modelMeta.ts
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ export const MODEL_RELEASE_DATES: Record<string, string> = {
"qwen-3.7-max": "2026-05-19",
"qwen3.8-max": "2026-08-03",
inkling: "2026-07-15",
"muse-wonder": "2026-10-12",
};

export const PROVIDER_LABELS: Record<ProviderKey, string> = {
Expand Down
12 changes: 12 additions & 0 deletions policybench/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,12 @@
"qwen-3.7-max": "openrouter/qwen/qwen3.7-max",
"qwen3.8-max": "openrouter/qwen/qwen3.8-max",
"inkling": "openrouter/thinkingmachines/inkling",
# Meta Muse Wonder (2026-10-12). Meta's own Model API serves it as
# muse-wonder-1.0; PolicyBench holds no Meta key, and OpenRouter's one
# endpoint for the id is Meta itself (provider "Meta", serving
# meta/muse-wonder-20261012; openrouter.ai/api/v1/models/meta/
# muse-wonder/endpoints, read 2026-10-12).
"muse-wonder": "openrouter/meta/muse-wonder",
}

# Configured per-1M-token USD list prices, keyed by the predictions.csv.gz model
Expand Down Expand Up @@ -161,6 +167,12 @@
# inkling: $1 / $4.05 per 1M input/output tokens (OpenRouter model
# listing, 2026-08-03); litellm's price map does not yet cover the id.
"inkling": {"input": 1.0, "output": 4.05},
# muse-wonder: $4 / $20 per 1M input/output tokens, $0.20 cached input,
# one rate at every prompt length (Meta Model API pricing in Meta's
# launch post, dev.meta.ai/resources/blog/build-with-muse-wonder, and
# the OpenRouter listing, both read 2026-10-12); litellm's price map
# does not yet cover the id.
"muse-wonder": {"input": 4.0, "output": 20.0, "cache_read": 0.20},
# gemini-3.6-flash: $1.50 / $7.50 per 1M input/output tokens (Google
# launch pricing, 2026-07-21); litellm's price map does not yet cover it.
"gemini-3.6-flash": {"input": 1.5, "output": 7.5},
Expand Down
27 changes: 27 additions & 0 deletions policybench/model_cards.py
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,33 @@ class ModelCard:
"$1/$4.05 per 1M on OpenRouter."
),
),
"openrouter/meta/muse-wonder": ModelCard(
litellm_id="openrouter/meta/muse-wonder",
answer_contract="json",
thinking_budget=True,
expected_cost_per_scenario_usd=0.059,
notes=(
"Onboarded 2026-10-13, the day after Meta released it. OpenRouter "
"lists one endpoint, Meta's own (meta/muse-wonder-20261012), so "
"OpenRouter passes requests to Meta's API. That API accepts "
"tool_choice 'auto' only: a forced (named-function) or 'required' "
'tool choice returns a 400 (\'only "auto" is supported for '
"tool_choice'), so the row runs the JSON contract whole-scenario, "
"as Opus 5.5 and Sonnet 5.5 do. A request with no reasoning "
"parameter reasons at the provider default: on a one-question "
"probe, 1,913 and 3,089 of 1,991 and 3,181 completion tokens with "
"no tools, and 2,189 and 2,635 of 2,352 and 2,815 with tool_choice "
"auto; it returns a short reasoning summary and an encrypted "
"trace. No reasoning-suppression sensitivity run is needed: the "
"row already reasons on the JSON contract, as Opus 5.5's and "
"Sonnet 5.5's do. Gauntlet: without the reasoning budget the "
"3-variable JSON probe stopped at the token limit (384 tokens); "
"with it, 3/3 and 16/16 parsed on the JSON contract (2,402 and "
"2,736 completion tokens, 15s and 18s). The gauntlet's cost "
"estimate is $0.059 per scenario at $4/$20 per 1M with $0.20 "
"cache reads."
),
),
"openrouter/z-ai/glm-5.2": ModelCard(
litellm_id="openrouter/z-ai/glm-5.2",
answer_contract="json",
Expand Down
7 changes: 7 additions & 0 deletions policybench/paper_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -3093,6 +3093,13 @@ def top_failure_subtypes(self, n: int = 3) -> str:
# techcrunch.com 2026-07-15; weights on Hugging Face the same day
# under Apache 2.0 (Thinking Machines Lab's first from-scratch model)
"inkling": "2026-07-15",
# techcrunch.com 2026-10-12 ("Meta releases its first frontier model,
# Muse Wonder") and venturebeat.com the same day; on Meta's Model API
# (muse-wonder-1.0) and OpenRouter (meta/muse-wonder, listed
# 2026-10-12) from launch. Proprietary: no weights. Muse Wonder Full,
# released alongside it, is limited to Muse Code subscriptions and has
# no API, so the board cannot run it.
"muse-wonder": "2026-10-12",
# openrouter.ai/stealth/ox-alpha ("released on August 20, 2026" as a
# stealth preview; the 2026-09-01 table carried 08-21 from the listing's
# first observed day). After the row's run Z.ai identified the preview as
Expand Down
37 changes: 37 additions & 0 deletions tests/test_eval_no_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -3087,6 +3087,43 @@ def test_local_claude_registration_works_without_the_remote_cost_map():
assert all(out["models"].values()), out["models"]


def test_muse_wonder_routes_through_openrouter_at_metas_list_price():
"""Muse Wonder runs through OpenRouter, whose one endpoint for the id is
Meta's own, at Meta's list price: $4 / $20 per 1M input/output tokens
and $0.20 cached input, at every prompt length."""
assert MODELS["muse-wonder"] == "openrouter/meta/muse-wonder"
assert PRICE_OVERRIDES_PER_1M["muse-wonder"] == {
"input": 4.0,
"output": 20.0,
"cache_read": 0.20,
}


@pytest.mark.parametrize(
("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"),
[
(1_000_000, 0, 0, 4.0),
(0, 1_000_000, 0, 20.0),
(1_000_000, 1_000_000, 0, 24.0),
(1_000_000, 0, 1_000_000, 0.20),
(1_000_000, 0, 250_000, 3.05),
],
)
def test_muse_wonder_override_reconstructs_token_costs(
prompt_tokens, completion_tokens, cached_prompt_tokens, expected
):
reconstructed = _reconstruct_token_cost(
model_name="muse-wonder",
model_id="openrouter/meta/muse-wonder",
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
cached_prompt_tokens=cached_prompt_tokens,
)

assert reconstructed.usd == pytest.approx(expected)
assert reconstructed.is_estimated is False


def test_claude_haiku_5_5_reasons_by_default_but_not_under_the_forced_tool():
"""Haiku 5.5 thinks when the request omits the thinking parameter, so it
is a thinking-default Claude: provider-default thinking, the thinking
Expand Down
1 change: 1 addition & 0 deletions tests/test_model_cards.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
"openrouter/deepseek/deepseek-v4-pro-0813": ("tool", None, 2400, 98_304),
"openrouter/deepseek/deepseek-v4-flash-0731": ("tool", None, 2400, 98_304),
"openrouter/z-ai/glm-5.3": ("tool", None, 2400, 16_384),
"openrouter/meta/muse-wonder": ("json", None, 300, 16_384),
}


Expand Down
6 changes: 6 additions & 0 deletions tests/test_paper_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,12 @@ def test_claude_haiku_5_5_release_date_is_recorded():
assert MODEL_RELEASE_DATES["claude-haiku-5.5"] == "2026-10-07"


def test_muse_wonder_release_date_is_recorded():
"""Meta released Muse Wonder on 2026-10-12 (TechCrunch and VentureBeat,
both that day), on its Model API and OpenRouter from launch."""
assert MODEL_RELEASE_DATES["muse-wonder"] == "2026-10-12"


def test_app_release_dates_mirror_the_paper_registry():
"""app/src/modelMeta.ts copies MODEL_RELEASE_DATES (its comment says to
update both together); every runnable model has a date, and the two
Expand Down
Loading