From d8dd84527446c3e2bb82ba989763f36b28a75a26 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 13:18:34 -0400 Subject: [PATCH 1/5] Add Gemini 4 Argon's model card: the forced answer tool, which Google accepts Google opened Gemini 4 Argon to the public on 2026-10-13 (the Gemini API changelog lists gemini-4-argon as generally available that day; the 2026-09-30 launch was limited to Fairwind Program cyber partners). It runs on the first-party Gemini API as gemini/gemini-4-argon, like the other Gemini rows. Price: $2 / $10 per 1M input/output tokens and $0.10 cached input, the rates Google's pricing page lists through 2026-11-30 (they double on 2026-12-01; page last updated and read 2026-10-13). Card: the Gemini API accepts a forced function call on this model, and the gauntlet, run with no card, passed on the tool contract (3/3 in 5s with 442 completion tokens, 16/16 whole-scenario in 24s with 2,619), so the row takes the board's forced answer tool, as Gemini 3.6, 3.7 and 3.8 Flash do. Reasoning: the harness sends no thinkingConfig, and the model decides call by call whether to think. On the board's own request for 13 frozen households (29 calls a shape) it returned reasoning tokens on 20 forced calls, 25 with tool_choice auto and 22 on the JSON contract; all 87 parsed every output. Forced against auto is within sampling error (Fisher exact p = 0.21), so the 100-household runs decide whether a tool_choice auto sensitivity run is published (the rule is fixed in the onboarding record). A two-scenario smoke on the run path completed 2/2 with no missing prediction or explanation for $0.05. Co-Authored-By: Claude Opus 5.5 --- app/src/modelMeta.ts | 1 + policybench/config.py | 18 ++++++ policybench/model_cards.py | 55 +++++++++++++++++ policybench/paper_results.py | 9 +++ tests/test_eval_no_tools.py | 111 +++++++++++++++++++++++++++++++++++ tests/test_model_cards.py | 2 + tests/test_paper_results.py | 9 +++ 7 files changed, 205 insertions(+) diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts index d0f84fdc..6f037d5c 100644 --- a/app/src/modelMeta.ts +++ b/app/src/modelMeta.ts @@ -142,6 +142,7 @@ export const MODEL_RELEASE_DATES: Record = { "gemini-3.8-flash": "2026-09-02", "gemini-3.7-flash": "2026-08-13", "gemini-3.5-flash-lite": "2026-07-21", + "gemini-4-argon": "2026-10-13", "gemini-3.1-flash-lite-preview": "2026-03-03", "gpt-5.4-mini": "2026-03-17", "gpt-5.4-nano": "2026-03-17", diff --git a/policybench/config.py b/policybench/config.py index b829b466..837397de 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -72,6 +72,13 @@ "gemini-3.7-flash": "gemini/gemini-3.7-flash", "gemini-3.8-flash": "gemini/gemini-3.8-flash", "gemini-3.5-flash-lite": "gemini/gemini-3.5-flash-lite", + # Gemini 4 Argon. Google announced it on 2026-09-30 for Fairwind Program + # cyber partners only and opened it to the public on 2026-10-13, when the + # Gemini API changelog lists gemini-4-argon as generally available + # (ai.google.dev/gemini-api/docs/changelog, read 2026-10-13). It runs on + # the first-party Gemini API like the other Gemini rows. The id is the + # model page's one stable version; Google lists no dated snapshot. + "gemini-4-argon": "gemini/gemini-4-argon", # Cloaked OpenRouter preview listed 2026-08-20 (free window); Z.ai later # identified it as GLM-5.3-Flash. On the board as a labeled preview row. "ox-alpha": "openrouter/stealth/ox-alpha", @@ -183,6 +190,17 @@ "gemini-3.8-flash": {"input": 0.75, "output": 3.75}, # gemini-3.5-flash-lite: $0.30 / $2.50 per 1M (Google list, 2026-09-03). "gemini-3.5-flash-lite": {"input": 0.30, "output": 2.50}, + # gemini-4-argon: $2 / $10 per 1M input/output tokens (output includes + # thinking tokens) and $0.10 cached input, the standard-tier rates Google + # lists "through November 30, 2026"; from 2026-12-01 they double to + # $4 / $20 and $0.20 (ai.google.dev/gemini-api/docs/pricing, last updated + # and read 2026-10-13). The row is priced at the rate in force on its run + # day, as gemini-3.7-flash and gemini-3.8-flash are at their dated + # introductory rates. The page lists one rate at every prompt length and + # implicit caching only. litellm's remote price map carries the same + # three rates (read 2026-10-13); the override keeps the row priced if the + # map is unavailable or changes on 2026-12-01. + "gemini-4-argon": {"input": 2.0, "output": 10.0, "cache_read": 0.10}, # glm-5.3: $1.40 / $4.40 per 1M (OpenRouter list, 2026-09-03). "glm-5.3": {"input": 1.40, "output": 4.40}, # deepseek dated snapshots via OpenRouter (list prices 2026-09-03). diff --git a/policybench/model_cards.py b/policybench/model_cards.py index e07bdd7f..27071934 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -448,6 +448,61 @@ class ModelCard: "whole-scenario (622 completion tokens)." ), ), + # Every field below comes from the 2026-10-13 onboarding record in + # ~/reviews/policybench-gemini-4-argon/{probes,onboard,smoke} (the + # reasoning probes, the output-limit probe, the gauntlet report and the + # two-scenario smoke): + # - answer_contract "tool": the Gemini API accepts a forced function call + # on this model (litellm sends mode ANY with allowed_function_names), + # and the gauntlet passed on it, so the row takes the board's forced + # answer tool, as Gemini 3.6, 3.7 and 3.8 Flash do. + # - explanation_chunk_size None: 16/16 on the canonical whole-scenario + # request (chunking is closed to new models anyway). + # - thinking_budget True: the model reasons by default (Google's guide + # gives a default thinking_level of medium), so it is in the + # thinking-default class and takes that class's 300s timeout. The + # 16,384-token starting budget is the Gemini family's. + # - request_timeout_seconds None: the class default; the slowest of 87 + # harness-path probe calls took 41s. + # - provider_max_completion_tokens 65_536: the model page's synchronous + # maximum output. + # - expected_cost_per_scenario_usd 0.032: the gauntlet's full-scenario + # probe at $2/$10 per 1M. + "gemini/gemini-4-argon": ModelCard( + litellm_id="gemini/gemini-4-argon", + answer_contract="tool", + thinking_budget=True, + provider_max_completion_tokens=65_536, + expected_cost_per_scenario_usd=0.032, + notes=( + "Onboarded 2026-10-13, the day Google opened the model to the " + "public (the Gemini API changelog lists gemini-4-argon as " + "generally available that day; the 2026-09-30 launch was limited " + "to Fairwind Program cyber partners). The Gemini API accepts a " + "forced function call on this model, so the row takes the " + "board's forced answer tool: the gauntlet, run with no card, " + "passed 3/3 (442 completion tokens, 5s) and 16/16 whole-scenario " + "(2,619 tokens, 24s) on the tool contract, about $0.032 per " + "scenario at $2/$10 per 1M. The harness sends no thinkingConfig, " + "and Google's guide gives a default thinking_level of medium. " + "The model decides call by call whether to think, in every " + "request shape. On the board's own request for 13 frozen " + "households (29 calls a shape), the forced tool returned " + "reasoning tokens on 20 calls (480 to 2,737 tokens), " + "tool_choice auto on 25 (490 to 2,985) and the JSON contract on " + "22 (161 to 3,014); all 87 calls parsed every output. Forced " + "against auto, that difference is within sampling error (Fisher " + "exact p = 0.21). On a one-question probe (10 calls a shape) it " + "returned thought tokens on 6 forced calls, 7 auto, 5 with " + "mode ANY and no named function, and none with no tools or with " + "a JSON response type. So a forced tool does not switch this " + "model's reasoning off, as it does Claude Opus 5's. The Models " + "API lists an output limit of 32,768 tokens and the model page a " + "synchronous maximum of 65,536; requests asking for up to 131,072 " + "returned 200. No probe call used more than 4,296 completion " + "tokens against the 16,384-token starting budget." + ), + ), "openrouter/z-ai/glm-5.3": ModelCard( litellm_id="openrouter/z-ai/glm-5.3", answer_contract="tool", diff --git a/policybench/paper_results.py b/policybench/paper_results.py index f2e43339..0ca51aac 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -3265,6 +3265,15 @@ def last_engine_upgrade(self) -> EngineUpgrade: "gemini-3.8-flash": "2026-09-02", # blog.google/products/gemini/gemini-3-7-flash (2026-08-13) "gemini-3.7-flash": "2026-08-13", + # ai.google.dev/gemini-api/docs/changelog ("October 13, 2026": Gemini 4 + # Argon, gemini-4-argon, generally available) and blog.google/ + # innovation-and-ai/models-and-research/gemini-models/ + # introducing-gemini-4-argon (2026-10-13: the paid tier of the Gemini + # API, and Google AI Pro and Ultra subscribers in the Gemini app), both + # read 2026-10-13. Google announced the model on 2026-09-30 for Fairwind + # Program cyber partners only (techcrunch.com 2026-09-30); that day is + # excluded under the public-availability rule. + "gemini-4-argon": "2026-10-13", # blog.google gemini-3-1-flash-lite; siliconangle.com 2026-03-03 "gemini-3.1-flash-lite-preview": "2026-03-03", # openai.com/index/introducing-gpt-5-4-mini-and-nano; 9to5mac 2026-03-17 diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index 1d16347e..b5e20c29 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -7,6 +7,8 @@ import litellm import pytest +from hypothesis import given +from hypothesis import strategies as st from policybench.config import ( COUNTRY_PROGRAMS, @@ -3124,6 +3126,115 @@ def test_muse_wonder_override_reconstructs_token_costs( assert reconstructed.is_estimated is False +def test_gemini_4_argon_runs_first_party_at_googles_dated_rate(): + """Gemini 4 Argon runs on the first-party Gemini API, like the other + Gemini rows, at the rate Google's pricing page lists through 2026-11-30: + $2 / $10 per 1M input/output tokens and $0.10 cached input, at every + prompt length (the page doubles all three from 2026-12-01).""" + assert MODELS["gemini-4-argon"] == "gemini/gemini-4-argon" + assert PRICE_OVERRIDES_PER_1M["gemini-4-argon"] == { + "input": 2.0, + "output": 10.0, + "cache_read": 0.10, + } + + +@pytest.mark.parametrize( + ("prompt_tokens", "completion_tokens", "cached_prompt_tokens", "expected"), + [ + (1_000_000, 0, 0, 2.0), + (0, 1_000_000, 0, 10.0), + (1_000_000, 1_000_000, 0, 12.0), + (1_000_000, 0, 1_000_000, 0.10), + (1_000_000, 0, 250_000, 1.525), + # The gauntlet's whole-scenario probe used 2,619 completion tokens + # (reasoning included); the same request's prompt is 2,762 tokens in + # the harness probe. That is the card's $0.032. + (2_762, 2_619, 0, 0.031714), + ], +) +def test_gemini_4_argon_override_reconstructs_token_costs( + prompt_tokens, completion_tokens, cached_prompt_tokens, expected +): + reconstructed = _reconstruct_token_cost( + model_name="gemini-4-argon", + model_id="gemini/gemini-4-argon", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ) + + assert reconstructed.usd == pytest.approx(expected) + assert reconstructed.is_estimated is False + + +@given( + prompt_tokens=st.integers(min_value=0, max_value=1_048_576), + completion_tokens=st.integers(min_value=0, max_value=131_072), + cached_share=st.floats(min_value=0.0, max_value=1.0), +) +def test_gemini_4_argon_cost_is_linear_in_tokens_and_bounded_by_the_cache_rates( + prompt_tokens, completion_tokens, cached_share +): + """Invariants of the override's reconstruction: the cost is the sum of + uncached input at $2, cached input at $0.10 and output at $10 per 1M; it + is never negative; and it lies between the all-cached and the uncached + cost of the same call.""" + cached = int(prompt_tokens * cached_share) + + def cost(cached_prompt_tokens): + return _reconstruct_token_cost( + model_name="gemini-4-argon", + model_id="gemini/gemini-4-argon", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ).usd + + expected = ( + (prompt_tokens - cached) * 2.0 + cached * 0.10 + completion_tokens * 10.0 + ) / 1_000_000 + assert cost(cached) == pytest.approx(expected, abs=1e-12) + assert cost(cached) >= 0 + assert cost(prompt_tokens) - 1e-12 <= cost(cached) <= cost(0) + 1e-12 + + +def test_gemini_4_argon_request_forces_the_answer_tool_and_sets_no_reasoning( + mini_scenario, +): + """The Gemini API accepts a forced function call on Gemini 4 Argon, so + its card selects the board's forced answer tool (the older Gemini rows + answer as JSON by the family default). The request carries no reasoning + parameter: the model reasons at Google's default. It takes the + thinking-default class's 300s timeout and the Gemini family's + 16,384-token starting budget.""" + from policybench.eval_no_tools import ( + ANSWER_FUNCTION_NAME, + _answer_contract_for_model, + _chat_completion_request_kwargs, + ) + + model_id = "gemini/gemini-4-argon" + assert _answer_contract_for_model(model_id, env={}) == "tool" + assert _thinking_configuration(model_id) == {"mode": "provider_default"} + assert _request_timeout_seconds(model_id) == 300 + + _, kwargs = _chat_completion_request_kwargs(mini_scenario, ["income_tax"], model_id) + + assert kwargs["tool_choice"] == { + "type": "function", + "function": {"name": ANSWER_FUNCTION_NAME}, + } + assert [tool["function"]["name"] for tool in kwargs["tools"]] == [ + ANSWER_FUNCTION_NAME + ] + assert kwargs["max_completion_tokens"] == 16_384 + assert kwargs["timeout"] == 300 + assert "response_format" not in kwargs + for reasoning_key in ("reasoning", "reasoning_effort", "thinking"): + assert reasoning_key not in kwargs + + def test_claude_haiku_5_5_reasons_by_default_but_not_under_the_forced_tool(): """Haiku 5.5 thinks when the request omits the thinking parameter, so it is a thinking-default Claude: provider-default thinking, the thinking diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index 179a7343..037ac30e 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -69,6 +69,7 @@ "openrouter/deepseek/deepseek-v4-flash-0731": ("tool", None, 2400, 98_304), "openrouter/z-ai/glm-5.3": ("tool", None, 2400, 16_384), "openrouter/meta/muse-wonder": ("json", None, 300, 16_384), + "gemini/gemini-4-argon": ("tool", None, 300, 16_384), } @@ -151,6 +152,7 @@ def test_provider_max_clamps_initial_completion_budget(monkeypatch): ("gemini/gemini-3.5-flash", 65_535), ("gemini/gemini-3-flash-preview", 65_535), ("gemini/gemini-3.6-flash", 65_536), + ("gemini/gemini-4-argon", 65_536), ], ) def test_roster_model_provider_maximums_limit_escalation(model_id, expected): diff --git a/tests/test_paper_results.py b/tests/test_paper_results.py index 46dac785..aa298c4d 100644 --- a/tests/test_paper_results.py +++ b/tests/test_paper_results.py @@ -81,6 +81,15 @@ def test_muse_wonder_release_date_is_recorded(): assert MODEL_RELEASE_DATES["muse-wonder"] == "2026-10-12" +def test_gemini_4_argon_release_date_is_its_public_day(): + """Google announced Gemini 4 Argon on 2026-09-30 for Fairwind Program + cyber partners only (TechCrunch, that day) and opened it to the public on + 2026-10-13 (the Gemini API changelog's general-availability entry and + Google's launch post, both that day). The partner-only day is excluded + under the public-availability rule, as GPT-6 Astra's is.""" + assert MODEL_RELEASE_DATES["gemini-4-argon"] == "2026-10-13" + + def test_app_release_dates_mirror_the_paper_registry(): """app/src/modelMeta.ts copies MODEL_RELEASE_DATES (its comment says to update both together); every runnable model has a date, and the two From 5910f79774a3bf1a785a6b76d74283d9279edc51 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 13:36:12 -0400 Subject: [PATCH 2/5] Record that a forced tool does not suppress Gemini 4 Argon's reasoning The 100-household runs decide the question the card left open. The forced-tool board run returned reasoning tokens on 80 households and a tool_choice auto run on 82 (Fisher exact p = 0.86; 70 reasoned in both, 10 only when forced, 12 only under auto, 8 in neither). Under the rule written before the runs, the row gets no tool_choice auto sensitivity run; the auto run stays in the onboarding record as evidence. The notes also record the board run: 100/100 households in one request each, 1,984/1,984 answers parsed, no budget escalation, $2.77. Co-Authored-By: Claude Opus 5.5 --- policybench/model_cards.py | 44 ++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index 27071934..399ecf2d 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -449,9 +449,10 @@ class ModelCard: ), ), # Every field below comes from the 2026-10-13 onboarding record in - # ~/reviews/policybench-gemini-4-argon/{probes,onboard,smoke} (the - # reasoning probes, the output-limit probe, the gauntlet report and the - # two-scenario smoke): + # ~/reviews/policybench-gemini-4-argon/{probes,onboard,smoke,board-run} + # (the reasoning probes, the output-limit probe, the gauntlet report, the + # two-scenario smoke, and the forced and tool_choice auto 100-household + # runs' comparison): # - answer_contract "tool": the Gemini API accepts a forced function call # on this model (litellm sends mode ANY with allowed_function_names), # and the gauntlet passed on it, so the row takes the board's forced @@ -463,7 +464,8 @@ class ModelCard: # thinking-default class and takes that class's 300s timeout. The # 16,384-token starting budget is the Gemini family's. # - request_timeout_seconds None: the class default; the slowest of 87 - # harness-path probe calls took 41s. + # harness-path probe calls took 41s and the board run's slowest + # household 53s. # - provider_max_completion_tokens 65_536: the model page's synchronous # maximum output. # - expected_cost_per_scenario_usd 0.032: the gauntlet's full-scenario @@ -485,22 +487,26 @@ class ModelCard: "(2,619 tokens, 24s) on the tool contract, about $0.032 per " "scenario at $2/$10 per 1M. The harness sends no thinkingConfig, " "and Google's guide gives a default thinking_level of medium. " - "The model decides call by call whether to think, in every " - "request shape. On the board's own request for 13 frozen " - "households (29 calls a shape), the forced tool returned " - "reasoning tokens on 20 calls (480 to 2,737 tokens), " - "tool_choice auto on 25 (490 to 2,985) and the JSON contract on " - "22 (161 to 3,014); all 87 calls parsed every output. Forced " - "against auto, that difference is within sampling error (Fisher " - "exact p = 0.21). On a one-question probe (10 calls a shape) it " - "returned thought tokens on 6 forced calls, 7 auto, 5 with " - "mode ANY and no named function, and none with no tools or with " - "a JSON response type. So a forced tool does not switch this " - "model's reasoning off, as it does Claude Opus 5's. The Models " - "API lists an output limit of 32,768 tokens and the model page a " + "The model decides call by call whether to think, and a forced " + "tool does not suppress it. Over the 100 board households the " + "forced-tool board run returned reasoning tokens on 80 and a " + "tool_choice auto run on 82 (Fisher exact p = 0.86; 70 households " + "reasoned in both runs, 10 only when forced, 12 only under auto, " + "8 in neither), so the row has no tool_choice auto sensitivity " + "run. Earlier probes agree: on the board's own request for 13 " + "households (29 calls a shape) the forced tool returned reasoning " + "tokens on 20 calls, auto on 25 and the JSON contract on 22 " + "(forced against auto, p = 0.21), and all 87 calls parsed every " + "output; on a one-question probe (10 calls a shape) it returned " + "thought tokens on 6 forced calls, 7 auto, 5 with mode ANY and no " + "named function, and none with no tools or with a JSON response " + "type. The board run answered all 100 households in one request " + "each, 1,984/1,984 parsed, with no budget escalation, for $2.77 " + "(a median of 20s and about 2,030 completion tokens per household, " + "at most 6,374 against the 16,384-token starting budget). The Models API " + "lists an output limit of 32,768 tokens and the model page a " "synchronous maximum of 65,536; requests asking for up to 131,072 " - "returned 200. No probe call used more than 4,296 completion " - "tokens against the 16,384-token starting budget." + "returned 200." ), ), "openrouter/z-ai/glm-5.3": ModelCard( From e4b4d054d19528d37566600e69bdc9f9f701d590 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 14:27:55 -0400 Subject: [PATCH 3/5] Cap Gemini 4 Argon's escalation ceiling at the 32,768 tokens one request returns (card review P2, P3) Google's Gemini 4 Argon guide: "One non-streaming request returns up to 32,768 tokens" (read 2026-10-13); the Models API's outputTokenLimit agrees. The model page's 65,536 is the streaming and SDK figure, reached by continuing across requests, and litellm sends one non-streaming REST request. So provider_max_completion_tokens is 32,768, not 65,536. The output-limit probe's 200s at up to 131,072 showed only that the parameter is accepted. The first board run recorded the 65,536 ceiling in its treatment fingerprint, which the freeze checks against the registry, so the board row is a second run on this card. The first run never escalated (largest household 6,374 completion tokens), and it stays in the record, with the tool_choice auto run, as the reasoning comparison's evidence. The reasoning sentence now says what the comparison shows: a forced tool does not switch reasoning off, and no statistically significant reduction in how often the model reasons was detected. Co-Authored-By: Claude Opus 5.5 --- policybench/model_cards.py | 54 ++++++++++++++++++++++---------------- tests/test_model_cards.py | 2 +- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index 399ecf2d..958841cb 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -466,15 +466,19 @@ class ModelCard: # - request_timeout_seconds None: the class default; the slowest of 87 # harness-path probe calls took 41s and the board run's slowest # household 53s. - # - provider_max_completion_tokens 65_536: the model page's synchronous - # maximum output. + # - provider_max_completion_tokens 32_768: the most one non-streaming + # generateContent request returns ("One non-streaming request returns + # up to 32,768 tokens", Google's Gemini 4 Argon guide, read 2026-10-13; + # the Models API's outputTokenLimit says the same). The model page's + # 65,536 is the streaming and SDK figure: the SDKs continue across + # requests, but litellm makes one REST request, which stops at 32,768. # - expected_cost_per_scenario_usd 0.032: the gauntlet's full-scenario # probe at $2/$10 per 1M. "gemini/gemini-4-argon": ModelCard( litellm_id="gemini/gemini-4-argon", answer_contract="tool", thinking_budget=True, - provider_max_completion_tokens=65_536, + provider_max_completion_tokens=32_768, expected_cost_per_scenario_usd=0.032, notes=( "Onboarded 2026-10-13, the day Google opened the model to the " @@ -487,26 +491,30 @@ class ModelCard: "(2,619 tokens, 24s) on the tool contract, about $0.032 per " "scenario at $2/$10 per 1M. The harness sends no thinkingConfig, " "and Google's guide gives a default thinking_level of medium. " - "The model decides call by call whether to think, and a forced " - "tool does not suppress it. Over the 100 board households the " - "forced-tool board run returned reasoning tokens on 80 and a " - "tool_choice auto run on 82 (Fisher exact p = 0.86; 70 households " - "reasoned in both runs, 10 only when forced, 12 only under auto, " - "8 in neither), so the row has no tool_choice auto sensitivity " - "run. Earlier probes agree: on the board's own request for 13 " - "households (29 calls a shape) the forced tool returned reasoning " - "tokens on 20 calls, auto on 25 and the JSON contract on 22 " - "(forced against auto, p = 0.21), and all 87 calls parsed every " - "output; on a one-question probe (10 calls a shape) it returned " - "thought tokens on 6 forced calls, 7 auto, 5 with mode ANY and no " - "named function, and none with no tools or with a JSON response " - "type. The board run answered all 100 households in one request " - "each, 1,984/1,984 parsed, with no budget escalation, for $2.77 " - "(a median of 20s and about 2,030 completion tokens per household, " - "at most 6,374 against the 16,384-token starting budget). The Models API " - "lists an output limit of 32,768 tokens and the model page a " - "synchronous maximum of 65,536; requests asking for up to 131,072 " - "returned 200." + "The model decides call by call whether to think. A forced tool " + "does not switch its reasoning off, and no statistically " + "significant reduction in how often it reasons was detected. Over " + "the 100 board households a first forced-tool board run returned " + "reasoning tokens on 80 and a tool_choice auto run on 82 (Fisher " + "exact p = 0.86; 70 households reasoned in both runs, 10 only " + "when forced, 12 only under auto, 8 in neither), so the row has no " + "tool_choice auto sensitivity run. Earlier probes agree: on the " + "board's own request for 13 households (29 calls a shape) the " + "forced tool returned reasoning tokens on 20 calls, auto on 25 and " + "the JSON contract on 22 (forced against auto, p = 0.21), and all " + "87 calls parsed every output; on a one-question probe (10 calls a " + "shape) it returned thought tokens on 6 forced calls, 7 auto, 5 " + "with mode ANY and no named function, and none with no tools or " + "with a JSON response type. The first board run answered all 100 " + "households in one request each, 1,984/1,984 parsed, with no " + "budget escalation, for $2.77 (a median of 20s and about 2,030 " + "completion tokens per household, at most 6,374 against the " + "16,384-token starting budget). It ran with a 65,536-token " + "escalation ceiling, the model page's figure, which one " + "non-streaming request cannot reach (32,768, above), so the board " + "row is a second run on this card. The Models API and the request " + "limit agree on 32,768; requests asking for up to 131,072 returned " + "200, which shows only that the parameter is accepted." ), ), "openrouter/z-ai/glm-5.3": ModelCard( diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py index 037ac30e..3503d076 100644 --- a/tests/test_model_cards.py +++ b/tests/test_model_cards.py @@ -152,7 +152,7 @@ def test_provider_max_clamps_initial_completion_budget(monkeypatch): ("gemini/gemini-3.5-flash", 65_535), ("gemini/gemini-3-flash-preview", 65_535), ("gemini/gemini-3.6-flash", 65_536), - ("gemini/gemini-4-argon", 65_536), + ("gemini/gemini-4-argon", 32_768), ], ) def test_roster_model_provider_maximums_limit_escalation(model_id, expected): From d1e7685beb2018b2a1eef170c2904cbef852928a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 14:39:24 -0400 Subject: [PATCH 4/5] Record Gemini 4 Argon's board run on the corrected card The second board run, on the 32,768-token ceiling (e4b4d054): all 100 households in one request each, 1,984/1,984 answers parsed, no budget escalation, $2.79, reasoning tokens on 85 households. None of its responses replays the first run's (no shared provider response id). Co-Authored-By: Claude Opus 5.5 --- policybench/model_cards.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index 958841cb..6226c083 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -512,7 +512,11 @@ class ModelCard: "16,384-token starting budget). It ran with a 65,536-token " "escalation ceiling, the model page's figure, which one " "non-streaming request cannot reach (32,768, above), so the board " - "row is a second run on this card. The Models API and the request " + "row is a second run on this card: all 100 households in one " + "request each, 1,984/1,984 parsed, no budget escalation, $2.79, " + "reasoning tokens on 85 households (a median of 22s and about " + "1,990 completion tokens per household, at most 47s and 4,872). " + "The Models API and the request " "limit agree on 32,768; requests asking for up to 131,072 returned " "200, which shows only that the parameter is accepted." ), From 0109ea2f260cd8da90dc6557e9c1e72643c8d0c3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 15:20:05 -0400 Subject: [PATCH 5/5] Say that Google's pages disagree on the output limit, and cite the saved record (card review r2) P2: the model page lists 65,536 as "default synchronous"; Google's generateContent guide gives 32,768 for one non-streaming request and the Models API's outputTokenLimit is 32,768, now saved in the onboarding record (probes/models-get-gemini-4-argon.json). The card follows the guide because litellm makes one non-streaming REST request; 32,768 is safe either way. P3: the timeout comment gives both board runs' slowest households (53s, 47s), and the notes add that the second board run reasoned on 85 households. Co-Authored-By: Claude Opus 5.5 --- policybench/model_cards.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/policybench/model_cards.py b/policybench/model_cards.py index 6226c083..dfa6365d 100644 --- a/policybench/model_cards.py +++ b/policybench/model_cards.py @@ -464,14 +464,16 @@ class ModelCard: # thinking-default class and takes that class's 300s timeout. The # 16,384-token starting budget is the Gemini family's. # - request_timeout_seconds None: the class default; the slowest of 87 - # harness-path probe calls took 41s and the board run's slowest - # household 53s. - # - provider_max_completion_tokens 32_768: the most one non-streaming - # generateContent request returns ("One non-streaming request returns - # up to 32,768 tokens", Google's Gemini 4 Argon guide, read 2026-10-13; - # the Models API's outputTokenLimit says the same). The model page's - # 65,536 is the streaming and SDK figure: the SDKs continue across - # requests, but litellm makes one REST request, which stops at 32,768. + # harness-path probe calls took 41s, and the slowest board household + # took 53s in the first board run and 47s in the second. + # - provider_max_completion_tokens 32_768. Google's pages disagree: the + # Gemini 4 Argon guide for generateContent says "One non-streaming + # request returns up to 32,768 tokens" (64K per stream), and the + # Models API's outputTokenLimit is 32,768 (models-get-gemini-4-argon + # .json in the onboarding record, read 2026-10-13); the model page + # lists 65,536 as "default synchronous". litellm makes one + # non-streaming REST request, so the card follows the guide; 32,768 is + # the safe value either way. # - expected_cost_per_scenario_usd 0.032: the gauntlet's full-scenario # probe at $2/$10 per 1M. "gemini/gemini-4-argon": ModelCard( @@ -498,7 +500,8 @@ class ModelCard: "reasoning tokens on 80 and a tool_choice auto run on 82 (Fisher " "exact p = 0.86; 70 households reasoned in both runs, 10 only " "when forced, 12 only under auto, 8 in neither), so the row has no " - "tool_choice auto sensitivity run. Earlier probes agree: on the " + "tool_choice auto sensitivity run; the second board run, below, " + "reasoned on 85. Earlier probes agree: on the " "board's own request for 13 households (29 calls a shape) the " "forced tool returned reasoning tokens on 20 calls, auto on 25 and " "the JSON contract on 22 (forced against auto, p = 0.21), and all " @@ -510,15 +513,15 @@ class ModelCard: "budget escalation, for $2.77 (a median of 20s and about 2,030 " "completion tokens per household, at most 6,374 against the " "16,384-token starting budget). It ran with a 65,536-token " - "escalation ceiling, the model page's figure, which one " - "non-streaming request cannot reach (32,768, above), so the board " - "row is a second run on this card: all 100 households in one " + "escalation ceiling, the model page's figure, but Google's " + "generateContent guide and the Models API give 32,768 for one " + "non-streaming request, which is what the harness sends, so the " + "board row is a second run on this card: all 100 households in one " "request each, 1,984/1,984 parsed, no budget escalation, $2.79, " "reasoning tokens on 85 households (a median of 22s and about " "1,990 completion tokens per household, at most 47s and 4,872). " - "The Models API and the request " - "limit agree on 32,768; requests asking for up to 131,072 returned " - "200, which shows only that the parameter is accepted." + "Requests asking for up to 131,072 tokens returned 200, which " + "shows only that the parameter is accepted." ), ), "openrouter/z-ai/glm-5.3": ModelCard(