Skip to content
This repository was archived by the owner on Oct 19, 2026. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,9 @@ Value on 2026-10-09: 671.2 (SPY's market close on 2026-10-09.)
Give one forecast for EACH of these resolution dates (the probability the question resolves Yes when evaluated on that date): 2026-10-18, 2026-11-10
```

`system.txt` begins *"You are a careful, calibrated forecaster…"*. `spec.json` says which
The harness hands pi the prompt on stdin, never as an argument (Linux allows one at most
128 KiB, and AIRO's joint prompt is past that); pi reads it as its first message, trimmed
of surrounding whitespace. `system.txt` begins *"You are a careful, calibrated forecaster…"*. `spec.json` says which
tools pi gets, how much research comes first, and what an answer looks like:

```json
Expand Down
19 changes: 15 additions & 4 deletions fri_forecaster/harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,19 @@ def pi_version(pi=DEFAULT_PI):
return None


def pi_command(model, system, prompt, *, thinking="high", pi=DEFAULT_PI):
# The prompt goes to pi on STDIN, never as an argument. Linux hands one
# argument at most 128 KiB (MAX_ARG_STRLEN), and AIRO's joint prompt passed
# that on 2026-10-19; pi reads piped stdin as its first prompt, verbatim but
# for surrounding whitespace (it trims), so what the model reads and prompt.txt
# differ by at most a trailing newline. Callers can check this flag before
# sending a prompt that would not fit an argument. (The same cap is why a
# debrief validator's value reaches tool_cli on stdin: pi_tools.ts.)
PROMPT_ON_STDIN = True


def pi_command(model, system, *, thinking="high", pi=DEFAULT_PI):
return pi_argv(pi) + ["-p", "--mode", "json", *ISOLATION, "-e", EXT,
"--model", model, "--thinking", thinking, "--system-prompt", system, prompt]
"--model", model, "--thinking", thinking, "--system-prompt", system]


def new_attempt_dir(workdir):
Expand Down Expand Up @@ -176,13 +186,14 @@ def run(*, model, system, prompt, workdir, cell_schema, key_fields, allowed_keys
tool_parameters=tool_parameters, **loop)
open(os.path.join(workdir, "prompt.txt"), "w").write(prompt)
open(os.path.join(workdir, "system.txt"), "w").write(system)
cmd = pi_command(model, system, prompt, thinking=thinking, pi=pi)
cmd = pi_command(model, system, thinking=thinking, pi=pi)
full_env = dict(os.environ, **(env or {}), FORECASTER_OUT=workdir, FORECASTER_PY=sys.executable)
t0 = datetime.now(timezone.utc)
events_path = os.path.join(workdir, "events.jsonl")
with open(events_path, "w") as ev, open(os.path.join(workdir, "stderr.txt"), "w") as err:
try:
rc = subprocess.run(cmd, stdout=ev, stderr=err, env=full_env, cwd=cwd, timeout=timeout).returncode
rc = subprocess.run(cmd, input=prompt.encode(), stdout=ev, stderr=err, env=full_env, cwd=cwd,
timeout=timeout).returncode
except subprocess.TimeoutExpired:
rc = "timeout"
secs = (datetime.now(timezone.utc) - t0).total_seconds()
Expand Down
9 changes: 7 additions & 2 deletions tests/test_forecaster.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#!{py}
import json, os, sys
out = os.environ["FORECASTER_OUT"]
open(os.path.join(out, "stdin.txt"), "w").write(sys.stdin.read()) # the prompt, as pi would read it
spec = json.load(open(os.path.join(out, "spec.json")))
keys = spec["allowed_keys"]
field = spec["key_fields"][0]
Expand Down Expand Up @@ -54,6 +55,7 @@ def test_run_merges_pieces_and_final(self):
cell_schema=SCHEMA, key_fields=["k"], allowed_keys=["a", "b", "c"],
tool_descriptions=DESC, pi=fake_pi(d))
self.assertEqual(res["rc"], 0)
self.assertEqual(open(os.path.join(res["workdir"], "stdin.txt")).read(), "P") # the prompt, on stdin
self.assertEqual(res["cells"]["a"]["probability"], 0.1) # from a piece
self.assertEqual(res["cells"]["c"]["probability"], 0.3) # from the final
self.assertNotIn("bogus", res["cells"]) # pieces filtered to allowed keys
Expand Down Expand Up @@ -98,12 +100,15 @@ def test_spec_offers_only_requested_tools(self):
self.assertEqual(spec["research_tools"], [])

def test_command_loads_extension_without_builtins(self):
cmd = harness.pi_command("m/x", "S", "P", thinking="max", pi="pi")
cmd = harness.pi_command("m/x", "S", thinking="max", pi="pi")
for flag in ("--no-builtin-tools", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-context-files"):
self.assertIn(flag, cmd)
self.assertEqual(harness.pi_command("m/x", "S", "P", pi="/x/cli.js")[:2], ["node", "/x/cli.js"])
self.assertEqual(harness.pi_command("m/x", "S", pi="/x/cli.js")[:2], ["node", "/x/cli.js"])
self.assertEqual(cmd[cmd.index("-e") + 1], harness.EXT)
self.assertTrue(os.path.exists(harness.EXT))
# No positional message: the prompt goes on stdin (harness.PROMPT_ON_STDIN).
self.assertEqual(cmd[-2:], ["--system-prompt", "S"])
self.assertTrue(harness.PROMPT_ON_STDIN)


QSET = {"forecast_due_date": "2026-10-11", "question_set": "2026-10-11-llm.json", "questions": [
Expand Down
16 changes: 16 additions & 0 deletions tests/test_pi_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,22 @@ def test_no_research_offers_only_submit_forecast(self):
offered = {t["function"]["name"] for t in model.requests[0]["tools"]}
self.assertEqual(offered, {"submit_forecast"}) # no research tools, and none of pi's built-ins
self.assertEqual({k: c["probability"] for k, c in res["cells"].items()}, {"a": 0.4, "b": 0.6})
self.assertEqual(model.last_user(1), "P") # the prompt, from stdin, as the first user message

def test_a_prompt_past_the_argv_limit_reaches_the_model(self):
"""The prompt goes on stdin: Linux hands one argument at most 128 KiB,
and AIRO's joint prompt is past that (2026-10-19)."""
prompt = "Forecast.\n" + "".join(f"--- question {i} ---\nWill it happen?\n" for i in range(6000))
self.assertGreater(len(prompt.encode()), 131072)
res, model = self.run_pi([
("submit_forecast", {"forecasts": [{"k": "a", "probability": 0.4}, {"k": "b", "probability": 0.6}],
"rationale": "r"}),
"done",
], prompt=prompt, research_tools=(), min_research=0, cells_tool=False,
tool_descriptions={"submit_forecast": "f"})
self.assertEqual(res["rc"], 0, open(os.path.join(res["workdir"], "stderr.txt")).read())
self.assertEqual(model.last_user(1), prompt.strip()) # verbatim but for pi's trim
self.assertEqual(open(os.path.join(res["workdir"], "prompt.txt")).read(), prompt)


SEARCH_WITH_CAUSE = {"type": "object", "required": ["query", "cause"],
Expand Down
Loading