From 8c400c9da70342dd0e9514792adfe52a1f1cc6e6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 13:57:07 -0400 Subject: [PATCH 1/3] Add Meta Muse Spark 1.3's model card: the JSON contract, since Meta refuses a forced tool Max ruled on 2026-10-13 (d1441) that Muse Spark 1.3 joins the board in the Gemini 4 Argon release. Meta released it on 2026-09-02 ("Introducing Muse Spark 1.3": available that day in Muse Code and the Meta Model API). It runs through OpenRouter, whose one endpoint for meta/muse-spark-1.3 is Meta's own (meta/muse-spark-1.3-20260902), as Muse Wonder does. Price: $1.25 / $4.25 per 1M input/output tokens and $0.15 cached input, Meta's standard-mode rate at every prompt length (Meta's rate card and the OpenRouter endpoint listing, read 2026-10-13). The contributor-mode listing is a different endpoint and is not the board's. Card: Meta's API accepts tool_choice "auto" only (a named-function or "required" choice returned 400 on each of three tries), so the row runs the JSON contract whole-scenario and reasons at the provider default (4,311 to 8,174 reasoning tokens on a one-question probe with no tools, 4,844 to 6,022 with tool_choice auto). With no forced tool there is no reasoning-suppression sensitivity run. Gauntlet: with no card the 3-variable JSON probe stopped at the 384-token limit; with the reasoning budget, 3/3 and 16/16 parsed (8,163 and 4,258 completion tokens, 106s and 58s), about $0.020 per scenario. A four-household smoke on the run path, with the board's two largest households, parsed 96/96 in one request each for $0.11 (Meta's reported cost equals the reconstruction from list prices). Co-Authored-By: Claude Opus 5.5 --- app/src/modelMeta.ts | 1 + policybench/config.py | 13 ++++++++++++ policybench/model_cards.py | 30 ++++++++++++++++++++++++++ policybench/paper_results.py | 5 +++++ tests/test_eval_no_tools.py | 41 ++++++++++++++++++++++++++++++++++++ tests/test_model_cards.py | 1 + tests/test_paper_results.py | 7 ++++++ 7 files changed, 98 insertions(+) diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts index d0f84fdc..79cebaeb 100644 --- a/app/src/modelMeta.ts +++ b/app/src/modelMeta.ts @@ -173,6 +173,7 @@ export const MODEL_RELEASE_DATES: Record = { "qwen3.8-max": "2026-08-03", inkling: "2026-07-15", "muse-wonder": "2026-10-12", + "muse-spark-1.3": "2026-09-02", }; export const PROVIDER_LABELS: Record = { diff --git a/policybench/config.py b/policybench/config.py index b829b466..64f16164 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -113,6 +113,13 @@ # meta/muse-wonder-20261012; openrouter.ai/api/v1/models/meta/ # muse-wonder/endpoints, read 2026-10-12). "muse-wonder": "openrouter/meta/muse-wonder", + # Meta Muse Spark 1.3 (2026-09-02). Like Muse Wonder it runs through + # OpenRouter, whose one endpoint for the id is Meta itself (provider + # "Meta", serving meta/muse-spark-1.3-20260902; openrouter.ai/api/v1/ + # models/meta/muse-spark-1.3/endpoints, read 2026-10-13). The cheaper + # meta/muse-spark-1.3-contributor listing is Meta's contributor mode, a + # different endpoint with its own rate card; the board runs standard mode. + "muse-spark-1.3": "openrouter/meta/muse-spark-1.3", } # Configured per-1M-token USD list prices, keyed by the predictions.csv.gz model @@ -173,6 +180,12 @@ # the OpenRouter listing, both read 2026-10-12); litellm's price map # does not yet cover the id. "muse-wonder": {"input": 4.0, "output": 20.0, "cache_read": 0.20}, + # muse-spark-1.3: $1.25 / $4.25 per 1M input/output tokens, $0.15 cached + # input, standard mode, one rate at every prompt length ("No long-context + # premium"; Meta's rate card, dev.meta.ai/docs/pricing-rate-limits, and + # the OpenRouter endpoint listing, both read 2026-10-13); litellm's price + # map is not relied on for the id. + "muse-spark-1.3": {"input": 1.25, "output": 4.25, "cache_read": 0.15}, # gemini-3.6-flash: $1.50 / $7.50 per 1M input/output tokens (Google # launch pricing, 2026-07-21); litellm's price map does not yet cover it. "gemini-3.6-flash": {"input": 1.5, "output": 7.5}, diff --git a/policybench/model_cards.py b/policybench/model_cards.py index e07bdd7f..a87d7087 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -601,6 +601,36 @@ class ModelCard: "cache reads." ), ), + "openrouter/meta/muse-spark-1.3": ModelCard( + litellm_id="openrouter/meta/muse-spark-1.3", + answer_contract="json", + thinking_budget=True, + expected_cost_per_scenario_usd=0.02, + notes=( + "Onboarded 2026-10-13; Meta released it on 2026-09-02. OpenRouter " + "lists one endpoint, Meta's own (meta/muse-spark-1.3-20260902), so " + "OpenRouter passes requests to Meta's API. As for Muse Wonder, " + "that API accepts tool_choice 'auto' only: a forced " + "(named-function) or 'required' tool choice returned a 400 on " + 'each of three tries (\'only "auto" is supported for ' + "tool_choice'), so the row runs the JSON contract whole-scenario. " + "A request with no reasoning parameter reasons at the provider " + "default: on a one-question probe, 4,311, 7,684 and 8,174 of " + "4,690, 8,073 and 8,586 completion tokens with no tools, and " + "4,844, 5,481 and 6,022 of 5,055, 5,667 and 6,266 with " + "tool_choice auto. No reasoning-suppression sensitivity run is " + "needed: the row already reasons on the JSON contract. Gauntlet: " + "without the reasoning budget the 3-variable JSON probe stopped at " + "the token limit (384 tokens); with it, 3/3 and 16/16 parsed on " + "the JSON contract (8,163 and 4,258 completion tokens, 106s and " + "58s), about $0.020 per scenario at $1.25/$4.25 per 1M with $0.15 " + "cache reads. A four-household smoke on the run path, with the " + "board's two largest households (28 and 32 outputs), parsed 96/96 " + "in one request each (43s to 93s, at most 7,115 completion " + "tokens), so the row keeps the thinking class's 300s timeout and " + "16,384-token starting budget." + ), + ), "openrouter/z-ai/glm-5.2": ModelCard( litellm_id="openrouter/z-ai/glm-5.2", answer_contract="json", diff --git a/policybench/paper_results.py b/policybench/paper_results.py index f2e43339..011a4574 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -3345,6 +3345,11 @@ def last_engine_upgrade(self) -> EngineUpgrade: # released alongside it, is limited to Muse Code subscriptions and has # no API, so the board cannot run it. "muse-wonder": "2026-10-12", + # research.meta.ai/blog/introducing-muse-spark-1-3 (2026-09-02: "available + # today in Muse Code and in Meta Model API"); datanorth.ai 2026-09-03; + # OpenRouter lists meta/muse-spark-1.3 from 2026-09-02. Proprietary: no + # weights. + "muse-spark-1.3": "2026-09-02", # openrouter.ai/stealth/ox-alpha ("released on August 20, 2026" as a # stealth preview; the 2026-09-01 table carried 08-21 from the listing's # first observed day). After the row's run Z.ai identified the preview as diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index 1d16347e..b72605a9 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -3087,6 +3087,47 @@ def test_local_claude_registration_works_without_the_remote_cost_map(): assert all(out["models"].values()), out["models"] +def test_muse_spark_1_3_routes_through_openrouter_at_metas_standard_rate(): + """Muse Spark 1.3 runs through OpenRouter, whose one endpoint for the id + is Meta's own, at Meta's standard-mode rate: $1.25 / $4.25 per 1M + input/output tokens and $0.15 cached input, at every prompt length. The + contributor-mode listing is a different endpoint and is not the board's.""" + assert MODELS["muse-spark-1.3"] == "openrouter/meta/muse-spark-1.3" + assert PRICE_OVERRIDES_PER_1M["muse-spark-1.3"] == { + "input": 1.25, + "output": 4.25, + "cache_read": 0.15, + } + + +@pytest.mark.parametrize( + ("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"), + [ + (1_000_000, 0, 0, 1.25), + (0, 1_000_000, 0, 4.25), + (1_000_000, 1_000_000, 0, 5.50), + (1_000_000, 0, 1_000_000, 0.15), + (1_000_000, 0, 250_000, 0.975), + # The smoke's largest household: 2,080 prompt and 6,966 completion + # tokens, which Meta billed at $0.0322. + (2_080, 6_966, 0, 0.0322055), + ], +) +def test_muse_spark_1_3_override_reconstructs_token_costs( + prompt_tokens, completion_tokens, cached_prompt_tokens, expected +): + reconstructed = _reconstruct_token_cost( + model_name="muse-spark-1.3", + model_id="openrouter/meta/muse-spark-1.3", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ) + + assert reconstructed.usd == pytest.approx(expected) + assert reconstructed.is_estimated is False + + def test_muse_wonder_routes_through_openrouter_at_metas_list_price(): """Muse Wonder runs through OpenRouter, whose one endpoint for the id is Meta's own, at Meta's list price: $4 / $20 per 1M input/output tokens diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index 179a7343..3ee8fae9 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -59,6 +59,7 @@ "openrouter/z-ai/glm-5.2": ("json", 3, 300, 16_384), "openrouter/minimax/minimax-m3": ("tool", None, 300, 16_384), "openrouter/qwen/qwen3.7-max": ("json", 3, 600, 16_384), + "openrouter/meta/muse-spark-1.3": ("json", None, 300, 16_384), "gemini/gemini-3.8-flash": ("tool", None, 300, 16_384), "gemini/gemini-3.5-flash-lite": ("tool", None, 300, 16_384), "gpt-6-astra": ("tool", None, 300, 16_384), diff --git a/tests/test_paper_results.py b/tests/test_paper_results.py index 46dac785..e8bc19b4 100644 --- a/tests/test_paper_results.py +++ b/tests/test_paper_results.py @@ -75,6 +75,13 @@ def test_claude_haiku_5_5_release_date_is_recorded(): assert MODEL_RELEASE_DATES["claude-haiku-5.5"] == "2026-10-07" +def test_muse_spark_1_3_release_date_is_recorded(): + """Meta's post "Introducing Muse Spark 1.3" is dated 2026-09-02 and says + the model is available that day in Muse Code and the Meta Model API; + OpenRouter lists meta/muse-spark-1.3 from the same day.""" + assert MODEL_RELEASE_DATES["muse-spark-1.3"] == "2026-09-02" + + def test_muse_wonder_release_date_is_recorded(): """Meta released Muse Wonder on 2026-10-12 (TechCrunch and VentureBeat, both that day), on its Model API and OpenRouter from launch.""" From eb0b819039cdd5f7895da3cefff2ef338f4df31f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 14:39:42 -0400 Subject: [PATCH 2/3] Correct Muse Spark 1.3's household-size claim and name Meta's rejection (card review P1, P3) P1: the card and commit 8c400c9d called the smoke's scenarios 099 and 119 (32 and 28 outputs) the board's two largest households. They are not: scenario_008 has 56 outputs, three households have 38 and two have 34. The card now gives the smoke's sizes (16, 20, 28 and 32 outputs) and rests the timeout and starting budget on the board run, which answered every household in one request, scenario_008 included: 1,984/1,984 parsed, no budget escalation, at most 96s and 9,871 completion tokens per household. The squash message will not repeat 8c400c9d's claim. The card also says that the board run's four smoke households replay the smoke's responses from the harness cache (identical requests), as the two smoke households of the Muse Wonder, Haiku 5.5 and Opus 5.5 rows do. P3: the notes now say "Meta rejects forced tool choices", the wording the disclosure test classifies as a provider rejection. Co-Authored-By: Claude Opus 5.5 --- policybench/model_cards.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index a87d7087..e969b756 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -610,9 +610,9 @@ class ModelCard: "Onboarded 2026-10-13; Meta released it on 2026-09-02. OpenRouter " "lists one endpoint, Meta's own (meta/muse-spark-1.3-20260902), so " "OpenRouter passes requests to Meta's API. As for Muse Wonder, " - "that API accepts tool_choice 'auto' only: a forced " - "(named-function) or 'required' tool choice returned a 400 on " - 'each of three tries (\'only "auto" is supported for ' + "that API accepts tool_choice 'auto' only, and Meta rejects forced " + "tool choices: a named-function or 'required' tool choice returned " + 'a 400 on each of three tries (\'only "auto" is supported for ' "tool_choice'), so the row runs the JSON contract whole-scenario. " "A request with no reasoning parameter reasons at the provider " "default: on a one-question probe, 4,311, 7,684 and 8,174 of " @@ -624,11 +624,15 @@ class ModelCard: "the token limit (384 tokens); with it, 3/3 and 16/16 parsed on " "the JSON contract (8,163 and 4,258 completion tokens, 106s and " "58s), about $0.020 per scenario at $1.25/$4.25 per 1M with $0.15 " - "cache reads. A four-household smoke on the run path, with the " - "board's two largest households (28 and 32 outputs), parsed 96/96 " - "in one request each (43s to 93s, at most 7,115 completion " - "tokens), so the row keeps the thinking class's 300s timeout and " - "16,384-token starting budget." + "cache reads. A four-household smoke on the run path (16, 20, 28 " + "and 32 outputs) parsed 96/96 in one request each. The board run " + "answered all 100 households in one request each, the largest " + "(scenario_008, 56 outputs) included: 1,984/1,984 parsed, no " + "budget escalation, at most 96s and 9,871 completion tokens per " + "household, so the row keeps the thinking class's 300s timeout " + "and 16,384-token starting budget. Its four smoke households " + "replay the smoke's responses from the harness cache (identical " + "requests), as the smoke households of earlier rows do." ), ), "openrouter/z-ai/glm-5.2": ModelCard( From a7fbf14ad23a4bab337a73a7b62fc4e4bd03f9f3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 15:19:23 -0400 Subject: [PATCH 3/3] Say which four board households replay the smoke (card review r2 P3) The reviewer's wording: the board run answered all 100 households in one request each, four of them replayed from the smoke. Co-Authored-By: Claude Opus 5.5 --- policybench/model_cards.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index e969b756..a7204a80 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -626,7 +626,8 @@ class ModelCard: "58s), about $0.020 per scenario at $1.25/$4.25 per 1M with $0.15 " "cache reads. A four-household smoke on the run path (16, 20, 28 " "and 32 outputs) parsed 96/96 in one request each. The board run " - "answered all 100 households in one request each, the largest " + "answered all 100 households in one request each (four replayed " + "from the smoke, below), the largest " "(scenario_008, 56 outputs) included: 1,984/1,984 parsed, no " "budget escalation, at most 96s and 9,871 completion tokens per " "household, so the row keeps the thinking class's 300s timeout "