From 8a81d234316ec7d5b2d948a56778ed558235aae5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 16:58:51 -0400 Subject: [PATCH 1/4] Add Meta Muse Wonder's roster entry, price and release date (card WIP) Meta released Muse Wonder on 2026-10-12 (TechCrunch, VentureBeat). Its API id on Meta's Model API is muse-wonder-1.0. PolicyBench holds no Meta key, and OpenRouter's only endpoint for meta/muse-wonder is Meta itself (provider "Meta", serving meta/muse-wonder-20261012), so the row runs through OpenRouter as Kimi, GLM, Qwen, MiniMax and Inkling do. Priced at Meta's list rate, $4 / $20 per 1M input/output tokens with $0.20 cached input and no long-context tier (Meta's launch post and the OpenRouter listing, read 2026-10-12). The serving card waits on the onboarding probes (forced tool vs auto reasoning, gauntlet, smoke). Co-Authored-By: Claude Opus 5.5 --- app/src/modelMeta.ts | 1 + policybench/config.py | 12 ++++++++++++ policybench/paper_results.py | 7 +++++++ tests/test_eval_no_tools.py | 37 ++++++++++++++++++++++++++++++++++++ tests/test_paper_results.py | 6 ++++++ 5 files changed, 63 insertions(+) diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts index 94d27faf..204312b6 100644 --- a/app/src/modelMeta.ts +++ b/app/src/modelMeta.ts @@ -169,6 +169,7 @@ export const MODEL_RELEASE_DATES: Record = { "qwen-3.7-max": "2026-05-19", "qwen3.8-max": "2026-08-03", inkling: "2026-07-15", + "muse-wonder": "2026-10-12", }; export const PROVIDER_LABELS: Record = { diff --git a/policybench/config.py b/policybench/config.py index ca471f6f..b829b466 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -107,6 +107,12 @@ "qwen-3.7-max": "openrouter/qwen/qwen3.7-max", "qwen3.8-max": "openrouter/qwen/qwen3.8-max", "inkling": "openrouter/thinkingmachines/inkling", + # Meta Muse Wonder (2026-10-12). Meta's own Model API serves it as + # muse-wonder-1.0; PolicyBench holds no Meta key, and OpenRouter's one + # endpoint for the id is Meta itself (provider "Meta", serving + # meta/muse-wonder-20261012; openrouter.ai/api/v1/models/meta/ + # muse-wonder/endpoints, read 2026-10-12). + "muse-wonder": "openrouter/meta/muse-wonder", } # Configured per-1M-token USD list prices, keyed by the predictions.csv.gz model @@ -161,6 +167,12 @@ # inkling: $1 / $4.05 per 1M input/output tokens (OpenRouter model # listing, 2026-08-03); litellm's price map does not yet cover the id. "inkling": {"input": 1.0, "output": 4.05}, + # muse-wonder: $4 / $20 per 1M input/output tokens, $0.20 cached input, + # one rate at every prompt length (Meta Model API pricing in Meta's + # launch post, dev.meta.ai/resources/blog/build-with-muse-wonder, and + # the OpenRouter listing, both read 2026-10-12); litellm's price map + # does not yet cover the id. + "muse-wonder": {"input": 4.0, "output": 20.0, "cache_read": 0.20}, # gemini-3.6-flash: $1.50 / $7.50 per 1M input/output tokens (Google # launch pricing, 2026-07-21); litellm's price map does not yet cover it. "gemini-3.6-flash": {"input": 1.5, "output": 7.5}, diff --git a/policybench/paper_results.py b/policybench/paper_results.py index 7375450c..026eda3d 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -3093,6 +3093,13 @@ def top_failure_subtypes(self, n: int = 3) -> str: # techcrunch.com 2026-07-15; weights on Hugging Face the same day # under Apache 2.0 (Thinking Machines Lab's first from-scratch model) "inkling": "2026-07-15", + # techcrunch.com 2026-10-12 ("Meta releases its first frontier model, + # Muse Wonder") and venturebeat.com the same day; on Meta's Model API + # (muse-wonder-1.0) and OpenRouter (meta/muse-wonder, listed + # 2026-10-12) from launch. Proprietary: no weights. Muse Wonder Full, + # released alongside it, is limited to Muse Code subscriptions and has + # no API, so the board cannot run it. + "muse-wonder": "2026-10-12", # openrouter.ai/stealth/ox-alpha ("released on August 20, 2026" as a # stealth preview; the 2026-09-01 table carried 08-21 from the listing's # first observed day). After the row's run Z.ai identified the preview as diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index e30980fb..1d16347e 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -3087,6 +3087,43 @@ def test_local_claude_registration_works_without_the_remote_cost_map(): assert all(out["models"].values()), out["models"] +def test_muse_wonder_routes_through_openrouter_at_metas_list_price(): + """Muse Wonder runs through OpenRouter, whose one endpoint for the id is + Meta's own, at Meta's list price: $4 / $20 per 1M input/output tokens + and $0.20 cached input, at every prompt length.""" + assert MODELS["muse-wonder"] == "openrouter/meta/muse-wonder" + assert PRICE_OVERRIDES_PER_1M["muse-wonder"] == { + "input": 4.0, + "output": 20.0, + "cache_read": 0.20, + } + + +@pytest.mark.parametrize( + ("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"), + [ + (1_000_000, 0, 0, 4.0), + (0, 1_000_000, 0, 20.0), + (1_000_000, 1_000_000, 0, 24.0), + (1_000_000, 0, 1_000_000, 0.20), + (1_000_000, 0, 250_000, 3.05), + ], +) +def test_muse_wonder_override_reconstructs_token_costs( + prompt_tokens, completion_tokens, cached_prompt_tokens, expected +): + reconstructed = _reconstruct_token_cost( + model_name="muse-wonder", + model_id="openrouter/meta/muse-wonder", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ) + + assert reconstructed.usd == pytest.approx(expected) + assert reconstructed.is_estimated is False + + def test_claude_haiku_5_5_reasons_by_default_but_not_under_the_forced_tool(): """Haiku 5.5 thinks when the request omits the thinking parameter, so it is a thinking-default Claude: provider-default thinking, the thinking diff --git a/tests/test_paper_results.py b/tests/test_paper_results.py index e8468798..801dc71e 100644 --- a/tests/test_paper_results.py +++ b/tests/test_paper_results.py @@ -73,6 +73,12 @@ def test_claude_haiku_5_5_release_date_is_recorded(): assert MODEL_RELEASE_DATES["claude-haiku-5.5"] == "2026-10-07" +def test_muse_wonder_release_date_is_recorded(): + """Meta released Muse Wonder on 2026-10-12 (TechCrunch and VentureBeat, + both that day), on its Model API and OpenRouter from launch.""" + assert MODEL_RELEASE_DATES["muse-wonder"] == "2026-10-12" + + def test_app_release_dates_mirror_the_paper_registry(): """app/src/modelMeta.ts copies MODEL_RELEASE_DATES (its comment says to update both together); every runnable model has a date, and the two From d087c7eab8f43cd6e654beeec69be8f8aca0c7a2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 22:49:37 -0400 Subject: [PATCH 2/4] Add Meta Muse Wonder's model card: the JSON contract, since Meta refuses a forced tool Meta's API, which OpenRouter's one endpoint for meta/muse-wonder passes requests to, accepts tool_choice "auto" only: a named-function or "required" choice returns 400. The board's rule is a forced answer tool unless the provider rejects it, so the row runs the JSON contract whole-scenario, as Opus 5.5 and Sonnet 5.5 do, reasoning at the provider default (a one-question probe used 1,913 to 3,089 reasoning tokens with no tools and 2,189 to 2,635 with tool_choice auto). With no forced tool there is no tool_choice auto sensitivity run. Gauntlet (2026-10-13): without the reasoning budget the 3-variable JSON probe stopped at the token limit; with it, 3/3 and 16/16 parsed (2,402 and 2,736 completion tokens, 15s and 18s), about $0.059 per scenario. A two-scenario smoke on the run path completed 2/2 with no missing prediction or explanation for $0.12 (Meta's reported cost equals the reconstruction from list prices). Co-Authored-By: Claude Opus 5.5 --- policybench/model_cards.py | 27 +++++++++++++++++++++++++++ tests/test_model_cards.py | 1 + 2 files changed, 28 insertions(+) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index b63de6be..c8e9f9db 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -574,6 +574,33 @@ class ModelCard: "$1/$4.05 per 1M on OpenRouter." ), ), + "openrouter/meta/muse-wonder": ModelCard( + litellm_id="openrouter/meta/muse-wonder", + answer_contract="json", + thinking_budget=True, + expected_cost_per_scenario_usd=0.059, + notes=( + "Onboarded 2026-10-13, the day after Meta released it. OpenRouter " + "lists one endpoint, Meta's own (meta/muse-wonder-20261012), so " + "OpenRouter passes requests to Meta's API. That API accepts " + "tool_choice 'auto' only: a forced (named-function) or 'required' " + "tool choice returns a 400 ('only \"auto\" is supported for " + "tool_choice'), so the row runs the JSON contract whole-scenario, " + "as Opus 5.5 and Sonnet 5.5 do. A request with no reasoning " + "parameter reasons at the provider default: on a one-question " + "probe, 1,913 and 3,089 of 1,991 and 3,181 completion tokens with " + "no tools, and 2,189 and 2,635 of 2,352 and 2,815 with tool_choice " + "auto; it returns a short reasoning summary and an encrypted " + "trace. With no forced tool there is no tool_choice auto " + "sensitivity run, which measures how much a forced tool " + "suppresses reasoning. Gauntlet: without the reasoning budget the " + "3-variable JSON probe stopped at the token limit (384 tokens); " + "with it, 3/3 and 16/16 parsed on the JSON contract (2,402 and " + "2,736 completion tokens, 15s and 18s). The gauntlet's cost " + "estimate is $0.059 per scenario at $4/$20 per 1M with $0.20 " + "cache reads." + ), + ), "openrouter/z-ai/glm-5.2": ModelCard( litellm_id="openrouter/z-ai/glm-5.2", answer_contract="json", diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index a7f28dd3..179a7343 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -68,6 +68,7 @@ "openrouter/deepseek/deepseek-v4-pro-0813": ("tool", None, 2400, 98_304), "openrouter/deepseek/deepseek-v4-flash-0731": ("tool", None, 2400, 98_304), "openrouter/z-ai/glm-5.3": ("tool", None, 2400, 16_384), + "openrouter/meta/muse-wonder": ("json", None, 300, 16_384), } From 812d538889247fa77ecd4b60076be2640fb04ac5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 23:05:36 -0400 Subject: [PATCH 3/4] Format the Muse Wonder card Co-Authored-By: Claude Opus 5.5 --- policybench/model_cards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index c8e9f9db..f919819b 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -584,7 +584,7 @@ class ModelCard: "lists one endpoint, Meta's own (meta/muse-wonder-20261012), so " "OpenRouter passes requests to Meta's API. That API accepts " "tool_choice 'auto' only: a forced (named-function) or 'required' " - "tool choice returns a 400 ('only \"auto\" is supported for " + 'tool choice returns a 400 (\'only "auto" is supported for ' "tool_choice'), so the row runs the JSON contract whole-scenario, " "as Opus 5.5 and Sonnet 5.5 do. A request with no reasoning " "parameter reasons at the provider default: on a one-question " From 925e5fc9f8932bc2474e6c6f6dca52102c8c18f7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 23:25:52 -0400 Subject: [PATCH 4/4] Describe why the card needs no reasoning-suppression sensitivity run (card review P3) Co-Authored-By: Claude Opus 5.5 --- policybench/model_cards.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index f919819b..e07bdd7f 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -591,9 +591,9 @@ class ModelCard: "probe, 1,913 and 3,089 of 1,991 and 3,181 completion tokens with " "no tools, and 2,189 and 2,635 of 2,352 and 2,815 with tool_choice " "auto; it returns a short reasoning summary and an encrypted " - "trace. With no forced tool there is no tool_choice auto " - "sensitivity run, which measures how much a forced tool " - "suppresses reasoning. Gauntlet: without the reasoning budget the " + "trace. No reasoning-suppression sensitivity run is needed: the " + "row already reasons on the JSON contract, as Opus 5.5's and " + "Sonnet 5.5's do. Gauntlet: without the reasoning budget the " "3-variable JSON probe stopped at the token limit (384 tokens); " "with it, 3/3 and 16/16 parsed on the JSON contract (2,402 and " "2,736 completion tokens, 15s and 18s). The gauntlet's cost "