diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts index d0f84fdc..79cebaeb 100644 --- a/app/src/modelMeta.ts +++ b/app/src/modelMeta.ts @@ -173,6 +173,7 @@ export const MODEL_RELEASE_DATES: Record = { "qwen3.8-max": "2026-08-03", inkling: "2026-07-15", "muse-wonder": "2026-10-12", + "muse-spark-1.3": "2026-09-02", }; export const PROVIDER_LABELS: Record = { diff --git a/policybench/config.py b/policybench/config.py index b829b466..64f16164 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -113,6 +113,13 @@ # meta/muse-wonder-20261012; openrouter.ai/api/v1/models/meta/ # muse-wonder/endpoints, read 2026-10-12). "muse-wonder": "openrouter/meta/muse-wonder", + # Meta Muse Spark 1.3 (2026-09-02). Like Muse Wonder it runs through + # OpenRouter, whose one endpoint for the id is Meta itself (provider + # "Meta", serving meta/muse-spark-1.3-20260902; openrouter.ai/api/v1/ + # models/meta/muse-spark-1.3/endpoints, read 2026-10-13). The cheaper + # meta/muse-spark-1.3-contributor listing is Meta's contributor mode, a + # different endpoint with its own rate card; the board runs standard mode. + "muse-spark-1.3": "openrouter/meta/muse-spark-1.3", } # Configured per-1M-token USD list prices, keyed by the predictions.csv.gz model @@ -173,6 +180,12 @@ # the OpenRouter listing, both read 2026-10-12); litellm's price map # does not yet cover the id. "muse-wonder": {"input": 4.0, "output": 20.0, "cache_read": 0.20}, + # muse-spark-1.3: $1.25 / $4.25 per 1M input/output tokens, $0.15 cached + # input, standard mode, one rate at every prompt length ("No long-context + # premium"; Meta's rate card, dev.meta.ai/docs/pricing-rate-limits, and + # the OpenRouter endpoint listing, both read 2026-10-13); litellm's price + # map is not relied on for the id. + "muse-spark-1.3": {"input": 1.25, "output": 4.25, "cache_read": 0.15}, # gemini-3.6-flash: $1.50 / $7.50 per 1M input/output tokens (Google # launch pricing, 2026-07-21); litellm's price map does not yet cover it. "gemini-3.6-flash": {"input": 1.5, "output": 7.5}, diff --git a/policybench/model_cards.py b/policybench/model_cards.py index e07bdd7f..a7204a80 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -601,6 +601,41 @@ class ModelCard: "cache reads." ), ), + "openrouter/meta/muse-spark-1.3": ModelCard( + litellm_id="openrouter/meta/muse-spark-1.3", + answer_contract="json", + thinking_budget=True, + expected_cost_per_scenario_usd=0.02, + notes=( + "Onboarded 2026-10-13; Meta released it on 2026-09-02. OpenRouter " + "lists one endpoint, Meta's own (meta/muse-spark-1.3-20260902), so " + "OpenRouter passes requests to Meta's API. As for Muse Wonder, " + "that API accepts tool_choice 'auto' only, and Meta rejects forced " + "tool choices: a named-function or 'required' tool choice returned " + 'a 400 on each of three tries (\'only "auto" is supported for ' + "tool_choice'), so the row runs the JSON contract whole-scenario. " + "A request with no reasoning parameter reasons at the provider " + "default: on a one-question probe, 4,311, 7,684 and 8,174 of " + "4,690, 8,073 and 8,586 completion tokens with no tools, and " + "4,844, 5,481 and 6,022 of 5,055, 5,667 and 6,266 with " + "tool_choice auto. No reasoning-suppression sensitivity run is " + "needed: the row already reasons on the JSON contract. Gauntlet: " + "without the reasoning budget the 3-variable JSON probe stopped at " + "the token limit (384 tokens); with it, 3/3 and 16/16 parsed on " + "the JSON contract (8,163 and 4,258 completion tokens, 106s and " + "58s), about $0.020 per scenario at $1.25/$4.25 per 1M with $0.15 " + "cache reads. A four-household smoke on the run path (16, 20, 28 " + "and 32 outputs) parsed 96/96 in one request each. The board run " + "answered all 100 households in one request each (four replayed " + "from the smoke, below), the largest " + "(scenario_008, 56 outputs) included: 1,984/1,984 parsed, no " + "budget escalation, at most 96s and 9,871 completion tokens per " + "household, so the row keeps the thinking class's 300s timeout " + "and 16,384-token starting budget. Its four smoke households " + "replay the smoke's responses from the harness cache (identical " + "requests), as the smoke households of earlier rows do." + ), + ), "openrouter/z-ai/glm-5.2": ModelCard( litellm_id="openrouter/z-ai/glm-5.2", answer_contract="json", diff --git a/policybench/paper_results.py b/policybench/paper_results.py index f2e43339..011a4574 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -3345,6 +3345,11 @@ def last_engine_upgrade(self) -> EngineUpgrade: # released alongside it, is limited to Muse Code subscriptions and has # no API, so the board cannot run it. "muse-wonder": "2026-10-12", + # research.meta.ai/blog/introducing-muse-spark-1-3 (2026-09-02: "available + # today in Muse Code and in Meta Model API"); datanorth.ai 2026-09-03; + # OpenRouter lists meta/muse-spark-1.3 from 2026-09-02. Proprietary: no + # weights. + "muse-spark-1.3": "2026-09-02", # openrouter.ai/stealth/ox-alpha ("released on August 20, 2026" as a # stealth preview; the 2026-09-01 table carried 08-21 from the listing's # first observed day). After the row's run Z.ai identified the preview as diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index 1d16347e..b72605a9 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -3087,6 +3087,47 @@ def test_local_claude_registration_works_without_the_remote_cost_map(): assert all(out["models"].values()), out["models"] +def test_muse_spark_1_3_routes_through_openrouter_at_metas_standard_rate(): + """Muse Spark 1.3 runs through OpenRouter, whose one endpoint for the id + is Meta's own, at Meta's standard-mode rate: $1.25 / $4.25 per 1M + input/output tokens and $0.15 cached input, at every prompt length. The + contributor-mode listing is a different endpoint and is not the board's.""" + assert MODELS["muse-spark-1.3"] == "openrouter/meta/muse-spark-1.3" + assert PRICE_OVERRIDES_PER_1M["muse-spark-1.3"] == { + "input": 1.25, + "output": 4.25, + "cache_read": 0.15, + } + + +@pytest.mark.parametrize( + ("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"), + [ + (1_000_000, 0, 0, 1.25), + (0, 1_000_000, 0, 4.25), + (1_000_000, 1_000_000, 0, 5.50), + (1_000_000, 0, 1_000_000, 0.15), + (1_000_000, 0, 250_000, 0.975), + # The smoke's largest household: 2,080 prompt and 6,966 completion + # tokens, which Meta billed at $0.0322. + (2_080, 6_966, 0, 0.0322055), + ], +) +def test_muse_spark_1_3_override_reconstructs_token_costs( + prompt_tokens, completion_tokens, cached_prompt_tokens, expected +): + reconstructed = _reconstruct_token_cost( + model_name="muse-spark-1.3", + model_id="openrouter/meta/muse-spark-1.3", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ) + + assert reconstructed.usd == pytest.approx(expected) + assert reconstructed.is_estimated is False + + def test_muse_wonder_routes_through_openrouter_at_metas_list_price(): """Muse Wonder runs through OpenRouter, whose one endpoint for the id is Meta's own, at Meta's list price: $4 / $20 per 1M input/output tokens diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index 179a7343..3ee8fae9 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -59,6 +59,7 @@ "openrouter/z-ai/glm-5.2": ("json", 3, 300, 16_384), "openrouter/minimax/minimax-m3": ("tool", None, 300, 16_384), "openrouter/qwen/qwen3.7-max": ("json", 3, 600, 16_384), + "openrouter/meta/muse-spark-1.3": ("json", None, 300, 16_384), "gemini/gemini-3.8-flash": ("tool", None, 300, 16_384), "gemini/gemini-3.5-flash-lite": ("tool", None, 300, 16_384), "gpt-6-astra": ("tool", None, 300, 16_384), diff --git a/tests/test_paper_results.py b/tests/test_paper_results.py index 46dac785..e8bc19b4 100644 --- a/tests/test_paper_results.py +++ b/tests/test_paper_results.py @@ -75,6 +75,13 @@ def test_claude_haiku_5_5_release_date_is_recorded(): assert MODEL_RELEASE_DATES["claude-haiku-5.5"] == "2026-10-07" +def test_muse_spark_1_3_release_date_is_recorded(): + """Meta's post "Introducing Muse Spark 1.3" is dated 2026-09-02 and says + the model is available that day in Muse Code and the Meta Model API; + OpenRouter lists meta/muse-spark-1.3 from the same day.""" + assert MODEL_RELEASE_DATES["muse-spark-1.3"] == "2026-09-02" + + def test_muse_wonder_release_date_is_recorded(): """Meta released Muse Wonder on 2026-10-12 (TechCrunch and VentureBeat, both that day), on its Model API and OpenRouter from launch."""