Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions app/src/modelMeta.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ export const MODEL_RELEASE_DATES: Record<string, string> = {
"qwen3.8-max": "2026-08-03",
inkling: "2026-07-15",
"muse-wonder": "2026-10-12",
"muse-spark-1.3": "2026-09-02",
};

export const PROVIDER_LABELS: Record<ProviderKey, string> = {
Expand Down
13 changes: 13 additions & 0 deletions policybench/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,13 @@
# meta/muse-wonder-20261012; openrouter.ai/api/v1/models/meta/
# muse-wonder/endpoints, read 2026-10-12).
"muse-wonder": "openrouter/meta/muse-wonder",
# Meta Muse Spark 1.3 (2026-09-02). Like Muse Wonder it runs through
# OpenRouter, whose one endpoint for the id is Meta itself (provider
# "Meta", serving meta/muse-spark-1.3-20260902; openrouter.ai/api/v1/
# models/meta/muse-spark-1.3/endpoints, read 2026-10-13). The cheaper
# meta/muse-spark-1.3-contributor listing is Meta's contributor mode, a
# different endpoint with its own rate card; the board runs standard mode.
"muse-spark-1.3": "openrouter/meta/muse-spark-1.3",
}

# Configured per-1M-token USD list prices, keyed by the predictions.csv.gz model
Expand Down Expand Up @@ -173,6 +180,12 @@
# the OpenRouter listing, both read 2026-10-12); litellm's price map
# does not yet cover the id.
"muse-wonder": {"input": 4.0, "output": 20.0, "cache_read": 0.20},
# muse-spark-1.3: $1.25 / $4.25 per 1M input/output tokens, $0.15 cached
# input, standard mode, one rate at every prompt length ("No long-context
# premium"; Meta's rate card, dev.meta.ai/docs/pricing-rate-limits, and
# the OpenRouter endpoint listing, both read 2026-10-13); litellm's price
# map is not relied on for the id.
"muse-spark-1.3": {"input": 1.25, "output": 4.25, "cache_read": 0.15},
# gemini-3.6-flash: $1.50 / $7.50 per 1M input/output tokens (Google
# launch pricing, 2026-07-21); litellm's price map does not yet cover it.
"gemini-3.6-flash": {"input": 1.5, "output": 7.5},
Expand Down
35 changes: 35 additions & 0 deletions policybench/model_cards.py
Original file line number Diff line number Diff line change
Expand Up @@ -601,6 +601,41 @@ class ModelCard:
"cache reads."
),
),
"openrouter/meta/muse-spark-1.3": ModelCard(
litellm_id="openrouter/meta/muse-spark-1.3",
answer_contract="json",
thinking_budget=True,
expected_cost_per_scenario_usd=0.02,
notes=(
"Onboarded 2026-10-13; Meta released it on 2026-09-02. OpenRouter "
"lists one endpoint, Meta's own (meta/muse-spark-1.3-20260902), so "
"OpenRouter passes requests to Meta's API. As for Muse Wonder, "
"that API accepts tool_choice 'auto' only, and Meta rejects forced "
"tool choices: a named-function or 'required' tool choice returned "
'a 400 on each of three tries (\'only "auto" is supported for '
"tool_choice'), so the row runs the JSON contract whole-scenario. "
"A request with no reasoning parameter reasons at the provider "
"default: on a one-question probe, 4,311, 7,684 and 8,174 of "
"4,690, 8,073 and 8,586 completion tokens with no tools, and "
"4,844, 5,481 and 6,022 of 5,055, 5,667 and 6,266 with "
"tool_choice auto. No reasoning-suppression sensitivity run is "
"needed: the row already reasons on the JSON contract. Gauntlet: "
"without the reasoning budget the 3-variable JSON probe stopped at "
"the token limit (384 tokens); with it, 3/3 and 16/16 parsed on "
"the JSON contract (8,163 and 4,258 completion tokens, 106s and "
"58s), about $0.020 per scenario at $1.25/$4.25 per 1M with $0.15 "
"cache reads. A four-household smoke on the run path (16, 20, 28 "
"and 32 outputs) parsed 96/96 in one request each. The board run "
"answered all 100 households in one request each (four replayed "
"from the smoke, below), the largest "
"(scenario_008, 56 outputs) included: 1,984/1,984 parsed, no "
"budget escalation, at most 96s and 9,871 completion tokens per "
"household, so the row keeps the thinking class's 300s timeout "
"and 16,384-token starting budget. Its four smoke households "
"replay the smoke's responses from the harness cache (identical "
"requests), as the smoke households of earlier rows do."
),
),
"openrouter/z-ai/glm-5.2": ModelCard(
litellm_id="openrouter/z-ai/glm-5.2",
answer_contract="json",
Expand Down
5 changes: 5 additions & 0 deletions policybench/paper_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -3345,6 +3345,11 @@ def last_engine_upgrade(self) -> EngineUpgrade:
# released alongside it, is limited to Muse Code subscriptions and has
# no API, so the board cannot run it.
"muse-wonder": "2026-10-12",
# research.meta.ai/blog/introducing-muse-spark-1-3 (2026-09-02: "available
# today in Muse Code and in Meta Model API"); datanorth.ai 2026-09-03;
# OpenRouter lists meta/muse-spark-1.3 from 2026-09-02. Proprietary: no
# weights.
"muse-spark-1.3": "2026-09-02",
# openrouter.ai/stealth/ox-alpha ("released on August 20, 2026" as a
# stealth preview; the 2026-09-01 table carried 08-21 from the listing's
# first observed day). After the row's run Z.ai identified the preview as
Expand Down
41 changes: 41 additions & 0 deletions tests/test_eval_no_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -3087,6 +3087,47 @@ def test_local_claude_registration_works_without_the_remote_cost_map():
assert all(out["models"].values()), out["models"]


def test_muse_spark_1_3_routes_through_openrouter_at_metas_standard_rate():
"""Muse Spark 1.3 runs through OpenRouter, whose one endpoint for the id
is Meta's own, at Meta's standard-mode rate: $1.25 / $4.25 per 1M
input/output tokens and $0.15 cached input, at every prompt length. The
contributor-mode listing is a different endpoint and is not the board's."""
assert MODELS["muse-spark-1.3"] == "openrouter/meta/muse-spark-1.3"
assert PRICE_OVERRIDES_PER_1M["muse-spark-1.3"] == {
"input": 1.25,
"output": 4.25,
"cache_read": 0.15,
}


@pytest.mark.parametrize(
("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"),
[
(1_000_000, 0, 0, 1.25),
(0, 1_000_000, 0, 4.25),
(1_000_000, 1_000_000, 0, 5.50),
(1_000_000, 0, 1_000_000, 0.15),
(1_000_000, 0, 250_000, 0.975),
# The smoke's largest household: 2,080 prompt and 6,966 completion
# tokens, which Meta billed at $0.0322.
(2_080, 6_966, 0, 0.0322055),
],
)
def test_muse_spark_1_3_override_reconstructs_token_costs(
prompt_tokens, completion_tokens, cached_prompt_tokens, expected
):
reconstructed = _reconstruct_token_cost(
model_name="muse-spark-1.3",
model_id="openrouter/meta/muse-spark-1.3",
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
cached_prompt_tokens=cached_prompt_tokens,
)

assert reconstructed.usd == pytest.approx(expected)
assert reconstructed.is_estimated is False


def test_muse_wonder_routes_through_openrouter_at_metas_list_price():
"""Muse Wonder runs through OpenRouter, whose one endpoint for the id is
Meta's own, at Meta's list price: $4 / $20 per 1M input/output tokens
Expand Down
1 change: 1 addition & 0 deletions tests/test_model_cards.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
"openrouter/z-ai/glm-5.2": ("json", 3, 300, 16_384),
"openrouter/minimax/minimax-m3": ("tool", None, 300, 16_384),
"openrouter/qwen/qwen3.7-max": ("json", 3, 600, 16_384),
"openrouter/meta/muse-spark-1.3": ("json", None, 300, 16_384),
"gemini/gemini-3.8-flash": ("tool", None, 300, 16_384),
"gemini/gemini-3.5-flash-lite": ("tool", None, 300, 16_384),
"gpt-6-astra": ("tool", None, 300, 16_384),
Expand Down
7 changes: 7 additions & 0 deletions tests/test_paper_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,13 @@ def test_claude_haiku_5_5_release_date_is_recorded():
assert MODEL_RELEASE_DATES["claude-haiku-5.5"] == "2026-10-07"


def test_muse_spark_1_3_release_date_is_recorded():
"""Meta's post "Introducing Muse Spark 1.3" is dated 2026-09-02 and says
the model is available that day in Muse Code and the Meta Model API;
OpenRouter lists meta/muse-spark-1.3 from the same day."""
assert MODEL_RELEASE_DATES["muse-spark-1.3"] == "2026-09-02"


def test_muse_wonder_release_date_is_recorded():
"""Meta released Muse Wonder on 2026-10-12 (TechCrunch and VentureBeat,
both that day), on its Model API and OpenRouter from launch."""
Expand Down
Loading