diff --git a/packages/fri-forecaster/fri_forecaster/harness.py b/packages/fri-forecaster/fri_forecaster/harness.py index 3afc341..ede9a38 100644 --- a/packages/fri-forecaster/fri_forecaster/harness.py +++ b/packages/fri-forecaster/fri_forecaster/harness.py @@ -102,19 +102,22 @@ def pi_version(pi=DEFAULT_PI): return None -# The prompt goes to pi on STDIN, never as an argument. Linux hands one -# argument at most 128 KiB (MAX_ARG_STRLEN), and AIRO's joint prompt passed -# that on 2026-10-19; pi reads piped stdin as its first prompt, verbatim but -# for surrounding whitespace (it trims), so what the model reads and prompt.txt -# differ by at most a trailing newline. Callers can check this flag before -# sending a prompt that would not fit an argument. (The same cap is why a -# debrief validator's value reaches tool_cli on stdin: pi_tools.ts.) +# The prompt goes to pi on STDIN and the system prompt as a file, never as +# arguments. Linux hands one argument at most 128 KiB (MAX_ARG_STRLEN): AIRO's +# joint prompt passed that on 2026-10-19, and so can a long system prompt +# shared across calls for the provider to cache. pi reads piped stdin as its +# first prompt, verbatim but for surrounding whitespace (it trims), so what the +# model reads and prompt.txt differ by at most a trailing newline; given a path +# that exists, `--system-prompt` reads the file (system.txt) as is. Callers can +# check this flag before sending a prompt that would not fit an argument. (The +# same cap is why a debrief validator's value reaches tool_cli on stdin: +# pi_tools.ts.) PROMPT_ON_STDIN = True -def pi_command(model, system, *, thinking="high", pi=DEFAULT_PI): +def pi_command(model, system_file, *, thinking="high", pi=DEFAULT_PI): return pi_argv(pi) + ["-p", "--mode", "json", *ISOLATION, "-e", EXT, - "--model", model, "--thinking", thinking, "--system-prompt", system] + "--model", model, "--thinking", thinking, "--system-prompt", system_file] def new_attempt_dir(workdir): @@ -245,8 +248,9 @@ def _call(*, model, system, prompt, workdir, cell_schema, key_fields, allowed_ke research_tools=research_tools, final_extra=final_extra, cells_tool=cells_tool, tool_parameters=tool_parameters, **loop) open(os.path.join(workdir, "prompt.txt"), "w").write(prompt) - open(os.path.join(workdir, "system.txt"), "w").write(system) - cmd = pi_command(model, system, thinking=thinking, pi=pi) + system_file = os.path.abspath(os.path.join(workdir, "system.txt")) # pi may run in another cwd + open(system_file, "w").write(system) + cmd = pi_command(model, system_file, thinking=thinking, pi=pi) full_env = dict(env, FORECASTER_OUT=workdir, FORECASTER_PY=sys.executable) t0 = datetime.now(timezone.utc) events_path = os.path.join(workdir, "events.jsonl") diff --git a/packages/fri-forecaster/tests/test_forecaster.py b/packages/fri-forecaster/tests/test_forecaster.py index b4d7bcd..6f2bac8 100644 --- a/packages/fri-forecaster/tests/test_forecaster.py +++ b/packages/fri-forecaster/tests/test_forecaster.py @@ -108,14 +108,15 @@ def test_spec_offers_only_requested_tools(self): self.assertEqual(spec["research_tools"], []) def test_command_loads_extension_without_builtins(self): - cmd = harness.pi_command("m/x", "S", thinking="max", pi="pi") + cmd = harness.pi_command("m/x", "/w/system.txt", thinking="max", pi="pi") for flag in ("--no-builtin-tools", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-context-files"): self.assertIn(flag, cmd) self.assertEqual(harness.pi_command("m/x", "S", pi="/x/cli.js")[:2], ["node", "/x/cli.js"]) self.assertEqual(cmd[cmd.index("-e") + 1], harness.EXT) self.assertTrue(os.path.exists(harness.EXT)) - # No positional message: the prompt goes on stdin (harness.PROMPT_ON_STDIN). - self.assertEqual(cmd[-2:], ["--system-prompt", "S"]) + # No prompt text in argv: the prompt goes on stdin and the system + # prompt in a file (harness.PROMPT_ON_STDIN). + self.assertEqual(cmd[-2:], ["--system-prompt", "/w/system.txt"]) self.assertTrue(harness.PROMPT_ON_STDIN) diff --git a/packages/fri-forecaster/tests/test_pi_tools.py b/packages/fri-forecaster/tests/test_pi_tools.py index 303dc56..db4c160 100644 --- a/packages/fri-forecaster/tests/test_pi_tools.py +++ b/packages/fri-forecaster/tests/test_pi_tools.py @@ -86,18 +86,20 @@ def tools(self, n): """The tools declared in request n (1-based).""" return {t["function"]["name"] for t in self.requests[n - 1].get("tools") or []} - def last_user(self, n): - """The text of the last user message in request n (1-based).""" - msg = [m for m in self.requests[n - 1]["messages"] if m.get("role") == "user"][-1] + def text(self, n, *roles): + """The text of the last message with one of these roles in request n (1-based).""" + msg = [m for m in self.requests[n - 1]["messages"] if m.get("role") in roles][-1] content = msg["content"] return content if isinstance(content, str) else "".join(p.get("text", "") for p in content) + def last_user(self, n): + """The text of the last user message in request n (1-based).""" + return self.text(n, "user") + def tool_result(self, n): """The tool result the model saw at the start of request n (1-based): the answer to the call it made in turn n-1.""" - msg = [m for m in self.requests[n - 1]["messages"] if m.get("role") == "tool"][-1] - content = msg["content"] - return content if isinstance(content, str) else "".join(p.get("text", "") for p in content) + return self.text(n, "tool") def close(self): self.server.shutdown() @@ -208,6 +210,21 @@ def test_a_prompt_past_the_argv_limit_reaches_the_model(self): self.assertEqual(model.last_user(1), prompt.strip()) # verbatim but for pi's trim self.assertEqual(open(os.path.join(res["workdir"], "prompt.txt")).read(), prompt) + def test_a_system_prompt_past_the_argv_limit_reaches_the_model(self): + """The system prompt goes in a file: a survey shared across calls, for + the provider to cache, is past 128 KiB (2026-10-20).""" + system = "The survey.\n" + "".join(f"=== G{i:04d} ===\nWhat will it be?\n" for i in range(6000)) + self.assertGreater(len(system.encode()), 131072) + res, model = self.run_pi([ + ("submit_forecast", {"forecasts": [{"k": "a", "probability": 0.4}, {"k": "b", "probability": 0.6}], + "rationale": "r"}), + "done", + ], system=system, research_tools=(), min_research=0, cells_tool=False, + tool_descriptions={"submit_forecast": "f"}) + self.assertEqual(res["rc"], 0, open(os.path.join(res["workdir"], "stderr.txt")).read()) + self.assertTrue(model.text(1, "system", "developer").startswith(system)) # pi appends its section + self.assertEqual(model.last_user(1), "P") + SEARCH_WITH_CAUSE = {"type": "object", "required": ["query", "cause"], "properties": {"query": {"type": "string"}, "recent_days": {"type": "integer"},