From 268cf64f321098058bd2d1cff07940ef9e750223 Mon Sep 17 00:00:00 2001 From: Nick Merrill Date: Mon, 19 Oct 2026 14:37:29 -0700 Subject: [PATCH] The prompt goes to pi on stdin, never as an argument Linux hands one argument at most 128 KiB (MAX_ARG_STRLEN), and AIRO's joint prompt passed that on 2026-10-19 when every incident rung became a deaths question and a damages question. pi reads piped stdin as its first prompt, verbatim but for surrounding whitespace, so harness.run now writes the prompt to pi's stdin and pi_command carries no positional message. Every caller -- AIRO's instrument, the ForecastBench driver -- is uncapped by it. harness.PROMPT_ON_STDIN lets a caller check before sending a prompt that would not fit an argument. Tests: the fake pi records what arrived on stdin; the real pi (0.99.2) receives a prompt past the limit and the scripted model sees it, verbatim but for pi's trim, as the first user message. Co-Authored-By: Claude Fable 5.1 --- README.md | 4 +++- fri_forecaster/harness.py | 19 +++++++++++++++---- tests/test_forecaster.py | 9 +++++++-- tests/test_pi_tools.py | 16 ++++++++++++++++ 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4b171e7..78d2f58 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,9 @@ Value on 2026-10-09: 671.2 (SPY's market close on 2026-10-09.) Give one forecast for EACH of these resolution dates (the probability the question resolves Yes when evaluated on that date): 2026-10-18, 2026-11-10 ``` -`system.txt` begins *"You are a careful, calibrated forecaster…"*. `spec.json` says which +The harness hands pi the prompt on stdin, never as an argument (Linux allows one at most +128 KiB, and AIRO's joint prompt is past that); pi reads it as its first message, trimmed +of surrounding whitespace. `system.txt` begins *"You are a careful, calibrated forecaster…"*. `spec.json` says which tools pi gets, how much research comes first, and what an answer looks like: ```json diff --git a/fri_forecaster/harness.py b/fri_forecaster/harness.py index 010550b..a330d52 100644 --- a/fri_forecaster/harness.py +++ b/fri_forecaster/harness.py @@ -86,9 +86,19 @@ def pi_version(pi=DEFAULT_PI): return None -def pi_command(model, system, prompt, *, thinking="high", pi=DEFAULT_PI): +# The prompt goes to pi on STDIN, never as an argument. Linux hands one +# argument at most 128 KiB (MAX_ARG_STRLEN), and AIRO's joint prompt passed +# that on 2026-10-19; pi reads piped stdin as its first prompt, verbatim but +# for surrounding whitespace (it trims), so what the model reads and prompt.txt +# differ by at most a trailing newline. Callers can check this flag before +# sending a prompt that would not fit an argument. (The same cap is why a +# debrief validator's value reaches tool_cli on stdin: pi_tools.ts.) +PROMPT_ON_STDIN = True + + +def pi_command(model, system, *, thinking="high", pi=DEFAULT_PI): return pi_argv(pi) + ["-p", "--mode", "json", *ISOLATION, "-e", EXT, - "--model", model, "--thinking", thinking, "--system-prompt", system, prompt] + "--model", model, "--thinking", thinking, "--system-prompt", system] def new_attempt_dir(workdir): @@ -176,13 +186,14 @@ def run(*, model, system, prompt, workdir, cell_schema, key_fields, allowed_keys tool_parameters=tool_parameters, **loop) open(os.path.join(workdir, "prompt.txt"), "w").write(prompt) open(os.path.join(workdir, "system.txt"), "w").write(system) - cmd = pi_command(model, system, prompt, thinking=thinking, pi=pi) + cmd = pi_command(model, system, thinking=thinking, pi=pi) full_env = dict(os.environ, **(env or {}), FORECASTER_OUT=workdir, FORECASTER_PY=sys.executable) t0 = datetime.now(timezone.utc) events_path = os.path.join(workdir, "events.jsonl") with open(events_path, "w") as ev, open(os.path.join(workdir, "stderr.txt"), "w") as err: try: - rc = subprocess.run(cmd, stdout=ev, stderr=err, env=full_env, cwd=cwd, timeout=timeout).returncode + rc = subprocess.run(cmd, input=prompt.encode(), stdout=ev, stderr=err, env=full_env, cwd=cwd, + timeout=timeout).returncode except subprocess.TimeoutExpired: rc = "timeout" secs = (datetime.now(timezone.utc) - t0).total_seconds() diff --git a/tests/test_forecaster.py b/tests/test_forecaster.py index 88a39d6..112ee3f 100644 --- a/tests/test_forecaster.py +++ b/tests/test_forecaster.py @@ -15,6 +15,7 @@ #!{py} import json, os, sys out = os.environ["FORECASTER_OUT"] + open(os.path.join(out, "stdin.txt"), "w").write(sys.stdin.read()) # the prompt, as pi would read it spec = json.load(open(os.path.join(out, "spec.json"))) keys = spec["allowed_keys"] field = spec["key_fields"][0] @@ -54,6 +55,7 @@ def test_run_merges_pieces_and_final(self): cell_schema=SCHEMA, key_fields=["k"], allowed_keys=["a", "b", "c"], tool_descriptions=DESC, pi=fake_pi(d)) self.assertEqual(res["rc"], 0) + self.assertEqual(open(os.path.join(res["workdir"], "stdin.txt")).read(), "P") # the prompt, on stdin self.assertEqual(res["cells"]["a"]["probability"], 0.1) # from a piece self.assertEqual(res["cells"]["c"]["probability"], 0.3) # from the final self.assertNotIn("bogus", res["cells"]) # pieces filtered to allowed keys @@ -98,12 +100,15 @@ def test_spec_offers_only_requested_tools(self): self.assertEqual(spec["research_tools"], []) def test_command_loads_extension_without_builtins(self): - cmd = harness.pi_command("m/x", "S", "P", thinking="max", pi="pi") + cmd = harness.pi_command("m/x", "S", thinking="max", pi="pi") for flag in ("--no-builtin-tools", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-context-files"): self.assertIn(flag, cmd) - self.assertEqual(harness.pi_command("m/x", "S", "P", pi="/x/cli.js")[:2], ["node", "/x/cli.js"]) + self.assertEqual(harness.pi_command("m/x", "S", pi="/x/cli.js")[:2], ["node", "/x/cli.js"]) self.assertEqual(cmd[cmd.index("-e") + 1], harness.EXT) self.assertTrue(os.path.exists(harness.EXT)) + # No positional message: the prompt goes on stdin (harness.PROMPT_ON_STDIN). + self.assertEqual(cmd[-2:], ["--system-prompt", "S"]) + self.assertTrue(harness.PROMPT_ON_STDIN) QSET = {"forecast_due_date": "2026-10-11", "question_set": "2026-10-11-llm.json", "questions": [ diff --git a/tests/test_pi_tools.py b/tests/test_pi_tools.py index f83c4a9..edf10c3 100644 --- a/tests/test_pi_tools.py +++ b/tests/test_pi_tools.py @@ -190,6 +190,22 @@ def test_no_research_offers_only_submit_forecast(self): offered = {t["function"]["name"] for t in model.requests[0]["tools"]} self.assertEqual(offered, {"submit_forecast"}) # no research tools, and none of pi's built-ins self.assertEqual({k: c["probability"] for k, c in res["cells"].items()}, {"a": 0.4, "b": 0.6}) + self.assertEqual(model.last_user(1), "P") # the prompt, from stdin, as the first user message + + def test_a_prompt_past_the_argv_limit_reaches_the_model(self): + """The prompt goes on stdin: Linux hands one argument at most 128 KiB, + and AIRO's joint prompt is past that (2026-10-19).""" + prompt = "Forecast.\n" + "".join(f"--- question {i} ---\nWill it happen?\n" for i in range(6000)) + self.assertGreater(len(prompt.encode()), 131072) + res, model = self.run_pi([ + ("submit_forecast", {"forecasts": [{"k": "a", "probability": 0.4}, {"k": "b", "probability": 0.6}], + "rationale": "r"}), + "done", + ], prompt=prompt, research_tools=(), min_research=0, cells_tool=False, + tool_descriptions={"submit_forecast": "f"}) + self.assertEqual(res["rc"], 0, open(os.path.join(res["workdir"], "stderr.txt")).read()) + self.assertEqual(model.last_user(1), prompt.strip()) # verbatim but for pi's trim + self.assertEqual(open(os.path.join(res["workdir"], "prompt.txt")).read(), prompt) SEARCH_WITH_CAUSE = {"type": "object", "required": ["query", "cause"],