diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts index d0f84fdc..6f037d5c 100644 --- a/app/src/modelMeta.ts +++ b/app/src/modelMeta.ts @@ -142,6 +142,7 @@ export const MODEL_RELEASE_DATES: Record = { "gemini-3.8-flash": "2026-09-02", "gemini-3.7-flash": "2026-08-13", "gemini-3.5-flash-lite": "2026-07-21", + "gemini-4-argon": "2026-10-13", "gemini-3.1-flash-lite-preview": "2026-03-03", "gpt-5.4-mini": "2026-03-17", "gpt-5.4-nano": "2026-03-17", diff --git a/policybench/config.py b/policybench/config.py index b829b466..837397de 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -72,6 +72,13 @@ "gemini-3.7-flash": "gemini/gemini-3.7-flash", "gemini-3.8-flash": "gemini/gemini-3.8-flash", "gemini-3.5-flash-lite": "gemini/gemini-3.5-flash-lite", + # Gemini 4 Argon. Google announced it on 2026-09-30 for Fairwind Program + # cyber partners only and opened it to the public on 2026-10-13, when the + # Gemini API changelog lists gemini-4-argon as generally available + # (ai.google.dev/gemini-api/docs/changelog, read 2026-10-13). It runs on + # the first-party Gemini API like the other Gemini rows. The id is the + # model page's one stable version; Google lists no dated snapshot. + "gemini-4-argon": "gemini/gemini-4-argon", # Cloaked OpenRouter preview listed 2026-08-20 (free window); Z.ai later # identified it as GLM-5.3-Flash. On the board as a labeled preview row. "ox-alpha": "openrouter/stealth/ox-alpha", @@ -183,6 +190,17 @@ "gemini-3.8-flash": {"input": 0.75, "output": 3.75}, # gemini-3.5-flash-lite: $0.30 / $2.50 per 1M (Google list, 2026-09-03). "gemini-3.5-flash-lite": {"input": 0.30, "output": 2.50}, + # gemini-4-argon: $2 / $10 per 1M input/output tokens (output includes + # thinking tokens) and $0.10 cached input, the standard-tier rates Google + # lists "through November 30, 2026"; from 2026-12-01 they double to + # $4 / $20 and $0.20 (ai.google.dev/gemini-api/docs/pricing, last updated + # and read 2026-10-13). The row is priced at the rate in force on its run + # day, as gemini-3.7-flash and gemini-3.8-flash are at their dated + # introductory rates. The page lists one rate at every prompt length and + # implicit caching only. litellm's remote price map carries the same + # three rates (read 2026-10-13); the override keeps the row priced if the + # map is unavailable or changes on 2026-12-01. + "gemini-4-argon": {"input": 2.0, "output": 10.0, "cache_read": 0.10}, # glm-5.3: $1.40 / $4.40 per 1M (OpenRouter list, 2026-09-03). "glm-5.3": {"input": 1.40, "output": 4.40}, # deepseek dated snapshots via OpenRouter (list prices 2026-09-03). diff --git a/policybench/model_cards.py b/policybench/model_cards.py index e07bdd7f..dfa6365d 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -448,6 +448,82 @@ class ModelCard: "whole-scenario (622 completion tokens)." ), ), + # Every field below comes from the 2026-10-13 onboarding record in + # ~/reviews/policybench-gemini-4-argon/{probes,onboard,smoke,board-run} + # (the reasoning probes, the output-limit probe, the gauntlet report, the + # two-scenario smoke, and the forced and tool_choice auto 100-household + # runs' comparison): + # - answer_contract "tool": the Gemini API accepts a forced function call + # on this model (litellm sends mode ANY with allowed_function_names), + # and the gauntlet passed on it, so the row takes the board's forced + # answer tool, as Gemini 3.6, 3.7 and 3.8 Flash do. + # - explanation_chunk_size None: 16/16 on the canonical whole-scenario + # request (chunking is closed to new models anyway). + # - thinking_budget True: the model reasons by default (Google's guide + # gives a default thinking_level of medium), so it is in the + # thinking-default class and takes that class's 300s timeout. The + # 16,384-token starting budget is the Gemini family's. + # - request_timeout_seconds None: the class default; the slowest of 87 + # harness-path probe calls took 41s, and the slowest board household + # took 53s in the first board run and 47s in the second. + # - provider_max_completion_tokens 32_768. Google's pages disagree: the + # Gemini 4 Argon guide for generateContent says "One non-streaming + # request returns up to 32,768 tokens" (64K per stream), and the + # Models API's outputTokenLimit is 32,768 (models-get-gemini-4-argon + # .json in the onboarding record, read 2026-10-13); the model page + # lists 65,536 as "default synchronous". litellm makes one + # non-streaming REST request, so the card follows the guide; 32,768 is + # the safe value either way. + # - expected_cost_per_scenario_usd 0.032: the gauntlet's full-scenario + # probe at $2/$10 per 1M. + "gemini/gemini-4-argon": ModelCard( + litellm_id="gemini/gemini-4-argon", + answer_contract="tool", + thinking_budget=True, + provider_max_completion_tokens=32_768, + expected_cost_per_scenario_usd=0.032, + notes=( + "Onboarded 2026-10-13, the day Google opened the model to the " + "public (the Gemini API changelog lists gemini-4-argon as " + "generally available that day; the 2026-09-30 launch was limited " + "to Fairwind Program cyber partners). The Gemini API accepts a " + "forced function call on this model, so the row takes the " + "board's forced answer tool: the gauntlet, run with no card, " + "passed 3/3 (442 completion tokens, 5s) and 16/16 whole-scenario " + "(2,619 tokens, 24s) on the tool contract, about $0.032 per " + "scenario at $2/$10 per 1M. The harness sends no thinkingConfig, " + "and Google's guide gives a default thinking_level of medium. " + "The model decides call by call whether to think. A forced tool " + "does not switch its reasoning off, and no statistically " + "significant reduction in how often it reasons was detected. Over " + "the 100 board households a first forced-tool board run returned " + "reasoning tokens on 80 and a tool_choice auto run on 82 (Fisher " + "exact p = 0.86; 70 households reasoned in both runs, 10 only " + "when forced, 12 only under auto, 8 in neither), so the row has no " + "tool_choice auto sensitivity run; the second board run, below, " + "reasoned on 85. Earlier probes agree: on the " + "board's own request for 13 households (29 calls a shape) the " + "forced tool returned reasoning tokens on 20 calls, auto on 25 and " + "the JSON contract on 22 (forced against auto, p = 0.21), and all " + "87 calls parsed every output; on a one-question probe (10 calls a " + "shape) it returned thought tokens on 6 forced calls, 7 auto, 5 " + "with mode ANY and no named function, and none with no tools or " + "with a JSON response type. The first board run answered all 100 " + "households in one request each, 1,984/1,984 parsed, with no " + "budget escalation, for $2.77 (a median of 20s and about 2,030 " + "completion tokens per household, at most 6,374 against the " + "16,384-token starting budget). It ran with a 65,536-token " + "escalation ceiling, the model page's figure, but Google's " + "generateContent guide and the Models API give 32,768 for one " + "non-streaming request, which is what the harness sends, so the " + "board row is a second run on this card: all 100 households in one " + "request each, 1,984/1,984 parsed, no budget escalation, $2.79, " + "reasoning tokens on 85 households (a median of 22s and about " + "1,990 completion tokens per household, at most 47s and 4,872). " + "Requests asking for up to 131,072 tokens returned 200, which " + "shows only that the parameter is accepted." + ), + ), "openrouter/z-ai/glm-5.3": ModelCard( litellm_id="openrouter/z-ai/glm-5.3", answer_contract="tool", diff --git a/policybench/paper_results.py b/policybench/paper_results.py index f2e43339..0ca51aac 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -3265,6 +3265,15 @@ def last_engine_upgrade(self) -> EngineUpgrade: "gemini-3.8-flash": "2026-09-02", # blog.google/products/gemini/gemini-3-7-flash (2026-08-13) "gemini-3.7-flash": "2026-08-13", + # ai.google.dev/gemini-api/docs/changelog ("October 13, 2026": Gemini 4 + # Argon, gemini-4-argon, generally available) and blog.google/ + # innovation-and-ai/models-and-research/gemini-models/ + # introducing-gemini-4-argon (2026-10-13: the paid tier of the Gemini + # API, and Google AI Pro and Ultra subscribers in the Gemini app), both + # read 2026-10-13. Google announced the model on 2026-09-30 for Fairwind + # Program cyber partners only (techcrunch.com 2026-09-30); that day is + # excluded under the public-availability rule. + "gemini-4-argon": "2026-10-13", # blog.google gemini-3-1-flash-lite; siliconangle.com 2026-03-03 "gemini-3.1-flash-lite-preview": "2026-03-03", # openai.com/index/introducing-gpt-5-4-mini-and-nano; 9to5mac 2026-03-17 diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index 1d16347e..b5e20c29 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -7,6 +7,8 @@ import litellm import pytest +from hypothesis import given +from hypothesis import strategies as st from policybench.config import ( COUNTRY_PROGRAMS, @@ -3124,6 +3126,115 @@ def test_muse_wonder_override_reconstructs_token_costs( assert reconstructed.is_estimated is False +def test_gemini_4_argon_runs_first_party_at_googles_dated_rate(): + """Gemini 4 Argon runs on the first-party Gemini API, like the other + Gemini rows, at the rate Google's pricing page lists through 2026-11-30: + $2 / $10 per 1M input/output tokens and $0.10 cached input, at every + prompt length (the page doubles all three from 2026-12-01).""" + assert MODELS["gemini-4-argon"] == "gemini/gemini-4-argon" + assert PRICE_OVERRIDES_PER_1M["gemini-4-argon"] == { + "input": 2.0, + "output": 10.0, + "cache_read": 0.10, + } + + +@pytest.mark.parametrize( + ("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"), + [ + (1_000_000, 0, 0, 2.0), + (0, 1_000_000, 0, 10.0), + (1_000_000, 1_000_000, 0, 12.0), + (1_000_000, 0, 1_000_000, 0.10), + (1_000_000, 0, 250_000, 1.525), + # The gauntlet's whole-scenario probe used 2,619 completion tokens + # (reasoning included); the same request's prompt is 2,762 tokens in + # the harness probe. That is the card's $0.032. + (2_762, 2_619, 0, 0.031714), + ], +) +def test_gemini_4_argon_override_reconstructs_token_costs( + prompt_tokens, completion_tokens, cached_prompt_tokens, expected +): + reconstructed = _reconstruct_token_cost( + model_name="gemini-4-argon", + model_id="gemini/gemini-4-argon", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ) + + assert reconstructed.usd == pytest.approx(expected) + assert reconstructed.is_estimated is False + + +@given( + prompt_tokens=st.integers(min_value=0, max_value=1_048_576), + completion_tokens=st.integers(min_value=0, max_value=131_072), + cached_share=st.floats(min_value=0.0, max_value=1.0), +) +def test_gemini_4_argon_cost_is_linear_in_tokens_and_bounded_by_the_cache_rates( + prompt_tokens, completion_tokens, cached_share +): + """Invariants of the override's reconstruction: the cost is the sum of + uncached input at $2, cached input at $0.10 and output at $10 per 1M; it + is never negative; and it lies between the all-cached and the uncached + cost of the same call.""" + cached = int(prompt_tokens * cached_share) + + def cost(cached_prompt_tokens): + return _reconstruct_token_cost( + model_name="gemini-4-argon", + model_id="gemini/gemini-4-argon", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ).usd + + expected = ( + (prompt_tokens - cached) * 2.0 + cached * 0.10 + completion_tokens * 10.0 + ) / 1_000_000 + assert cost(cached) == pytest.approx(expected, abs=1e-12) + assert cost(cached) >= 0 + assert cost(prompt_tokens) - 1e-12 <= cost(cached) <= cost(0) + 1e-12 + + +def test_gemini_4_argon_request_forces_the_answer_tool_and_sets_no_reasoning( + mini_scenario, +): + """The Gemini API accepts a forced function call on Gemini 4 Argon, so + its card selects the board's forced answer tool (the older Gemini rows + answer as JSON by the family default). The request carries no reasoning + parameter: the model reasons at Google's default. It takes the + thinking-default class's 300s timeout and the Gemini family's + 16,384-token starting budget.""" + from policybench.eval_no_tools import ( + ANSWER_FUNCTION_NAME, + _answer_contract_for_model, + _chat_completion_request_kwargs, + ) + + model_id = "gemini/gemini-4-argon" + assert _answer_contract_for_model(model_id, env={}) == "tool" + assert _thinking_configuration(model_id) == {"mode": "provider_default"} + assert _request_timeout_seconds(model_id) == 300 + + _, kwargs = _chat_completion_request_kwargs(mini_scenario, ["income_tax"], model_id) + + assert kwargs["tool_choice"] == { + "type": "function", + "function": {"name": ANSWER_FUNCTION_NAME}, + } + assert [tool["function"]["name"] for tool in kwargs["tools"]] == [ + ANSWER_FUNCTION_NAME + ] + assert kwargs["max_completion_tokens"] == 16_384 + assert kwargs["timeout"] == 300 + assert "response_format" not in kwargs + for reasoning_key in ("reasoning", "reasoning_effort", "thinking"): + assert reasoning_key not in kwargs + + def test_claude_haiku_5_5_reasons_by_default_but_not_under_the_forced_tool(): """Haiku 5.5 thinks when the request omits the thinking parameter, so it is a thinking-default Claude: provider-default thinking, the thinking diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index 179a7343..3503d076 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -69,6 +69,7 @@ "openrouter/deepseek/deepseek-v4-flash-0731": ("tool", None, 2400, 98_304), "openrouter/z-ai/glm-5.3": ("tool", None, 2400, 16_384), "openrouter/meta/muse-wonder": ("json", None, 300, 16_384), + "gemini/gemini-4-argon": ("tool", None, 300, 16_384), } @@ -151,6 +152,7 @@ def test_provider_max_clamps_initial_completion_budget(monkeypatch): ("gemini/gemini-3.5-flash", 65_535), ("gemini/gemini-3-flash-preview", 65_535), ("gemini/gemini-3.6-flash", 65_536), + ("gemini/gemini-4-argon", 32_768), ], ) def test_roster_model_provider_maximums_limit_escalation(model_id, expected): diff --git a/tests/test_paper_results.py b/tests/test_paper_results.py index 46dac785..aa298c4d 100644 --- a/tests/test_paper_results.py +++ b/tests/test_paper_results.py @@ -81,6 +81,15 @@ def test_muse_wonder_release_date_is_recorded(): assert MODEL_RELEASE_DATES["muse-wonder"] == "2026-10-12" +def test_gemini_4_argon_release_date_is_its_public_day(): + """Google announced Gemini 4 Argon on 2026-09-30 for Fairwind Program + cyber partners only (TechCrunch, that day) and opened it to the public on + 2026-10-13 (the Gemini API changelog's general-availability entry and + Google's launch post, both that day). The partner-only day is excluded + under the public-availability rule, as GPT-6 Astra's is.""" + assert MODEL_RELEASE_DATES["gemini-4-argon"] == "2026-10-13" + + def test_app_release_dates_mirror_the_paper_registry(): """app/src/modelMeta.ts copies MODEL_RELEASE_DATES (its comment says to update both together); every runnable model has a date, and the two