Loop options for AIRO's instrument: floor, coverage, turn limit, debrief

README.md

@@ -48,6 +48,19 @@ with the tools it is given, and the answers land in the same folder, which is th
 ```
 
 Every attempt gets a fresh `attempt-N` folder, so a retry never reads an older answer.
+
+The loop has options, all off by default (the ForecastBench driver's loop); AIRO's
+runner turns them on to match its own instrument. `harness.run(..., **options)`:
+
+| option | what it does |
+|---|---|
+| `floor_counts="returned"` | failed research calls count toward the research floor too |
+| `coverage=[keys]` | submission waits for a recent-window `web_search` tagged with each key |
+| `withhold_final=True` | the submit tools are not offered at all until the floor and coverage are met |
+| `warn_missing_keys=False` | the final answer is accepted as sent, with no warning about missing cells |
+| `max_turns=N` | from turn N only `submit_forecast` is offered and asked for, for up to two more turns; a reply in prose gets a nudge |
+| `debrief={...}` | after the answer, one more turn for a structured debrief, validated, with retries, saved to `debrief.json` |
+| `tool_parameters={...}` | replace a tool's parameter schema, e.g. to require a `cause` on `web_search` |
 pi runs isolated: no built-in tools, no discovered extensions or skills, no
 AGENTS.md/CLAUDE.md, so the inbox is the whole prompt.
 

fri_forecaster/harness.py

@@ -43,16 +43,28 @@ def key_of(cell, key_fields):
     return "|".join(str(cell.get(k)) for k in key_fields)
 
 
+# The loop's optional settings (pi_tools.ts's header says what each does).
+# Unset, a call runs the default loop: the ForecastBench driver's.
+LOOP_OPTIONS = ("floor_counts", "coverage", "withhold_final", "warn_missing_keys", "max_turns", "debrief")
+
+
 def write_spec(workdir, *, cell_schema, key_fields, allowed_keys, tool_descriptions,
-               min_research=0, research_tools=RESEARCH_TOOLS, final_extra=None, cells_tool=True):
+               min_research=0, research_tools=RESEARCH_TOOLS, final_extra=None, cells_tool=True,
+               tool_parameters=None, **loop):
     """spec.json for pi_tools.ts. tool_descriptions: {tool name: description}
     for every tool offered (research tools, submit_forecast, and submit_cells
-    unless cells_tool is False)."""
-    tools = {name: {"description": tool_descriptions[name]}
+    unless cells_tool is False); tool_parameters: {tool name: JSON schema} to
+    replace a tool's built-in parameters; `loop`: LOOP_OPTIONS."""
+    unknown = set(loop) - set(LOOP_OPTIONS)
+    if unknown:
+        raise TypeError(f"unknown loop options {sorted(unknown)}; have {LOOP_OPTIONS}")
+    tools = {name: {"description": tool_descriptions[name],
+                    **({"parameters": tool_parameters[name]} if name in (tool_parameters or {}) else {})}
              for name in list(research_tools) + (["submit_cells"] if cells_tool else []) + ["submit_forecast"]}
     spec = {"tool_cli": TOOL_CLI, "research_tools": list(research_tools), "min_research": min_research,
             "tools": tools, "cell_schema": cell_schema, "key_fields": list(key_fields),
-            "allowed_keys": list(allowed_keys), "final_extra": final_extra or {}}
+            "allowed_keys": list(allowed_keys), "final_extra": final_extra or {},
+            **{k: v for k, v in loop.items() if v is not None}}
     os.makedirs(workdir, exist_ok=True)
     with open(os.path.join(workdir, "spec.json"), "w") as f:
         json.dump(spec, f, indent=1)
@@ -153,14 +165,15 @@ def collect(workdir, key_fields, allowed_keys):
 
 def run(*, model, system, prompt, workdir, cell_schema, key_fields, allowed_keys, tool_descriptions,
         min_research=0, research_tools=RESEARCH_TOOLS, final_extra=None, cells_tool=True,
-        thinking="high", pi=DEFAULT_PI, cwd=None, env=None, timeout=None):
+        thinking="high", pi=DEFAULT_PI, cwd=None, env=None, timeout=None, tool_parameters=None, **loop):
     """One pi call, in a fresh workdir/attempt-N (returned as "workdir").
     Writes spec.json, prompt.txt, system.txt, events.jsonl, stderr.txt there
-    (and pi_tools.ts writes pieces.jsonl, final.json)."""
+    (and pi_tools.ts writes pieces.jsonl, final.json, debrief.json)."""
     workdir = new_attempt_dir(workdir)
     write_spec(workdir, cell_schema=cell_schema, key_fields=key_fields, allowed_keys=allowed_keys,
                tool_descriptions=tool_descriptions, min_research=min_research,
-               research_tools=research_tools, final_extra=final_extra, cells_tool=cells_tool)
+               research_tools=research_tools, final_extra=final_extra, cells_tool=cells_tool,
+               tool_parameters=tool_parameters, **loop)
     open(os.path.join(workdir, "prompt.txt"), "w").write(prompt)
     open(os.path.join(workdir, "system.txt"), "w").write(system)
     cmd = pi_command(model, system, prompt, thinking=thinking, pi=pi)
@@ -177,5 +190,12 @@ def run(*, model, system, prompt, workdir, cell_schema, key_fields, allowed_keys
     usage["harness"] = "pi"
     usage["seconds"] = round(secs)
     cells, answer, delivery = collect(workdir, key_fields, allowed_keys)
+    final = read_json(os.path.join(workdir, "final.json")) or {}
     return {"rc": rc, "seconds": secs, "evidence": evidence, "usage": usage, "cells": cells,
-            "answer": answer, "delivery": delivery, "trace": events_path, "workdir": workdir}
+            "answer": answer, "delivery": delivery, "forced": bool(final.get("forced")),
+            "debrief": read_json(os.path.join(workdir, "debrief.json")),
+            "trace": events_path, "workdir": workdir}
+
+
+def read_json(path):
+    return json.load(open(path)) if os.path.exists(path) else None