elsehow · Oct 14, 2026 · Oct 14, 2026
diff --git a/README.md b/README.md
@@ -89,12 +89,17 @@ pytest
 ## Elicit new forecasts
 
 ```bash
-pip install -e '.[acquire]'                      # litellm, and nothing else
+pip install -e '.[acquire,pi]'                   # litellm, and agent-box (the pi loop)
 python code/run_unified.py --joint --dry-run     # print the prompt, call nothing
 python code/run_unified.py --joint --repeats 1 --workers 2 \
     --conditions data/combined_conditions.json --out results/runs/<stamp>.jsonl
 ```
 
+Since combined-v9 the calls go through pi (redlines/pi_loop.py): node 22 and
+pi 0.99.2, whose `cli.js` `FORECASTER_PI` names (agent-box's README, "Requirements").
+`--harness litellm --experiment <name>` runs the same instrument through
+redlines.llm's own loop, which is the cron's shadow.
+
 Keys go in `~/.config/redlines/env` (mode 600), one `KEY=value` per line:
 `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `GOOGLE_API_KEY`,
 `XAI_API_KEY`, `TAVILY_API_KEY`. A repo-local `.env` is read as a fallback;
@@ -196,7 +201,7 @@ redlines/                   the pipeline package (stdlib only; litellm only insi
   hydrate.py, pages.py      window.__NAME__ blob injection; manifest-driven page assembly
   export.py                 forecasts.csv, rationales.csv, the download bundle, the ForecastBench export
   llm.py, tools.py          ACQUISITION ONLY: key loading + the agentic tool-use loop; the Tavily tools
-  pi_loop.py                the same loop through FRI's agent-box (pi), for --harness pi shadow runs
+  pi_loop.py                call_tools through FRI's agent-box (pi): the published loop since combined-v9
   views/                    one pure build() per panel: graph1..graph4, observational, causal,
                             databank, timeline, conditional, capability, axes, method
   paper/                    the paper's figures (graph1, graph3, graph4, incident_ladders,
@@ -212,7 +217,7 @@ code/                       acquisition scripts, generators and thin shims (see
   graph3_comparability.py   the Graph 3 methodology decomposition (needs FB_DATASETS_ROOT)
   observational/            Graph 5: the observational bench, vendored (spec.md, runners, scoring)
   causal/                   Graph 6: the Starsim causal bench's generating code, frozen at causal-v4-locked
-  compare_harness.py        a --harness pi shadow run against the cron's, cell by cell
+  compare_harness.py        a --harness shadow run (the other loop) against the cron's, cell by cell
 data/
   auto-arc/                 the Auto-ARC question workbook (.xlsx) and definitions (.md) by date
   autoarc_ladder.json, autoarc_crosscutting.json   the generated question sets (never hand-edited)

diff --git a/code/compare_harness.py b/code/compare_harness.py
@@ -1,19 +1,21 @@
 #!/usr/bin/env python3
-"""The same instrument through two loops, cell by cell: the cron's litellm run
-against a `--harness pi` shadow run of the same day (redlines/pi_loop.py).
+"""The same instrument through two loops, cell by cell: the cron's published
+run against a `--harness` shadow run of the same day through the other loop
+(pi since combined-v9, litellm before; redlines/pi_loop.py says what differs).
 
     python code/compare_harness.py \\
         --base results/conditional_runs_combined/2026-10-21T1200Z.jsonl \\
-        --pi results/experiments/pi-shadow-2026-10-21T1405Z-instrument.jsonl \\
+        --other results/experiments/litellm-shadow-2026-10-21T1405Z-instrument.jsonl \\
         --prev results/conditional_runs_combined/2026-10-16T1200Z.jsonl
 
 A cell is (model, question, condition, horizon). Per model: how many cells
 both runs answered, the mean and median |difference| in probability, the
-share that moved more than 0.1, and the mean signed difference (pi minus
-litellm); then each call's research, turns, cost and debrief. One draw per
+share that moved more than 0.1, and the mean signed difference (other minus
+published); then each call's research, turns, cost and debrief. One draw per
 loop cannot tell a loop effect from run-to-run spread, so `--prev` (the
-previous cron run) prints the same numbers for litellm against itself: a
-loop difference that sits inside that spread is not a difference yet.
+previous cron run) prints the same numbers for the published loop against
+itself: a loop difference that sits inside that spread is not a difference
+yet. The loops are named from the rows (usage.harness; none = litellm).
 """
 import argparse
 import json
@@ -60,6 +62,14 @@ def compare(a_rows, b_rows):
     return out
 
 
+def loop_name(rows):
+    """The loop a run went through: pi rows carry usage.harness; litellm's none."""
+    for r in calls(rows).values():
+        h = r["usage"].get("harness")
+        return h["name"] if isinstance(h, dict) else "litellm"
+    return "?"
+
+
 def call_line(r):
     u = r.get("usage") or {}
     st = r.get("debrief_status") or {}
@@ -79,21 +89,23 @@ def table(title, result):
 
 def main(argv=None):
     ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
-    ap.add_argument("--base", required=True, help="the cron run's instrument rows (litellm)")
-    ap.add_argument("--pi", required=True, help="the shadow run's instrument rows (--harness pi)")
-    ap.add_argument("--prev", help="the previous cron run's instrument rows: litellm's own spread")
+    ap.add_argument("--base", required=True, help="the cron run's instrument rows (the published loop)")
+    ap.add_argument("--other", required=True, help="the shadow run's instrument rows (--harness, the other loop)")
+    ap.add_argument("--prev", help="the previous cron run's instrument rows: the published loop's own spread")
     args = ap.parse_args(argv)
-    base, pi = load(args.base), load(args.pi)
-    if not compare(base, pi):
+    base, other = load(args.base), load(args.other)
+    b, o = loop_name(base), loop_name(other)
+    if not compare(base, other):
         sys.exit("no cells in common: different instruments, or a run without forecasts")
-    table("pi vs litellm, same day (pi minus litellm):", compare(base, pi))
+    table(f"{o} vs {b}, same day ({o} minus {b}):", compare(base, other))
     if args.prev:
-        table("\nlitellm vs litellm, previous run (this run minus previous):", compare(load(args.prev), base))
+        table(f"\n{b} vs {b}, previous run (this run minus previous):", compare(load(args.prev), base))
     print("\ncalls:")
+    w = max(len(b), len(o))
     for label, r in calls(base).items():
-        print(f"  {label:16} litellm {call_line(r)}")
-        p = calls(pi).get(label)
-        print(f"  {'':16} pi      {call_line(p) if p else 'no call'}")
+        print(f"  {label:16} {b:{w}} {call_line(r)}")
+        p = calls(other).get(label)
+        print(f"  {'':16} {o:{w}} {call_line(p) if p else 'no call'}")
 
 
 if __name__ == "__main__":

diff --git a/code/cron_run.sh b/code/cron_run.sh
@@ -177,6 +177,10 @@ fi
 # call is longer and dearer than it was. 4 models x 1 = 4 calls. The views
 # draw no interval on a single-draw day; the dots and the median are the
 # reading.
+# THE LOOP is the set's own: agent-box's pi since combined-v9 (2026-10-16;
+# redlines/pi_loop.py, three hours a call, REDLINES_PI_TIMEOUT), so the env
+# file must give FORECASTER_PI and node on PATH (see THE SHADOW below). The
+# request timeout here is the litellm loop's, which the shadow now runs.
 export REDLINES_LLM_TIMEOUT="${REDLINES_LLM_TIMEOUT:-1800}"
 # The runner exits non-zero when any panel model fails (the date cannot
 # publish without all of them), which under `set -e` stops this script before
@@ -223,30 +227,31 @@ fi
 FINISHED=1
 notify "AIRO run $STAMP published" "$(grep -E 'ok, [0-9]+ failed' "$LOG" | tail -1 | sed 's/ -> .*//'). Panel: $PANEL" low white_check_mark
 
-# THE PI SHADOW (2026-10-14): the same instrument, same panel, same day,
-# through agent-box's pi loop instead of redlines.llm (code/run_unified.py
-# --harness pi; redlines/pi_loop.py says what differs). An experiment, never
-# the series: rows go to results/experiments/pi-shadow-*, protocol "...+pi".
-# Then code/compare_harness.py prints the two runs cell by cell, beside the
-# previous run as litellm's own spread. Off unless the env file sets
-# REDLINES_PI_SHADOW=1; needs agent-box in PY (`pip install -e '.[pi]'`), and
-# in the env file FORECASTER_PI (pi 0.99.2's cli.js) and a PATH with node on
-# it (cron's own PATH has none). It runs after publication and can only
-# notify: the published run is already done.
-pi_shadow() {
-    local runs="$REPO/results/conditional_runs_combined" base prev shadow
+# THE SHADOW (2026-10-14): the same instrument, same panel, same day, through
+# the OTHER loop (code/run_unified.py --harness; the set names its own: pi
+# since combined-v9, redlines.llm's litellm before). An experiment, never the
+# series: rows go to results/experiments/<harness>-shadow-*, protocol
+# "<protocol>+<harness>". Then code/compare_harness.py prints the two runs
+# cell by cell, beside the previous run as the published loop's own spread.
+# Off unless the env file sets REDLINES_SHADOW_HARNESS (litellm or pi). The
+# pi loop needs agent-box in PY (`pip install -e '.[pi]'`), and in the env
+# file FORECASTER_PI (pi 0.99.2's cli.js) and a PATH with node on it (cron's
+# own PATH has none) -- for the published run as much as for a shadow. It
+# runs after publication and can only notify: the published run is done.
+shadow() {
+    local h="$1" runs="$REPO/results/conditional_runs_combined" base prev other
     base="$(ls "$runs"/*.jsonl | sort | tail -1)"
     prev="$(ls "$runs"/*.jsonl | sort | tail -2 | head -1)"
-    echo "=== pi shadow: the instrument through pi; litellm's is $base ==="
-    "$PY" code/run_unified.py --joint --repeats 1 --workers 2 --harness pi \
-        --experiment pi-shadow --conditions data/combined_conditions.json || return 1
-    shadow="$(ls "$REPO"/results/experiments/pi-shadow-*-instrument.jsonl | sort | tail -1)"
-    "$PY" code/compare_harness.py --base "$base" --pi "$shadow" --prev "$prev" | tee "${shadow%.jsonl}.compare.txt"
+    echo "=== $h shadow: the instrument through $h; the published run is $base ==="
+    "$PY" code/run_unified.py --joint --repeats 1 --workers 2 --harness "$h" \
+        --experiment "$h-shadow" --conditions data/combined_conditions.json || return 1
+    other="$(ls "$REPO"/results/experiments/"$h"-shadow-*-instrument.jsonl | sort | tail -1)"
+    "$PY" code/compare_harness.py --base "$base" --other "$other" --prev "$prev" | tee "${other%.jsonl}.compare.txt"
 }
-if [ "${REDLINES_PI_SHADOW:-0}" = 1 ]; then
-    if pi_shadow; then
-        notify "AIRO pi shadow $STAMP done" "$(sed -n 2,6p "$(ls "$REPO"/results/experiments/pi-shadow-*.compare.txt | sort | tail -1)")" low
+if [ -n "${REDLINES_SHADOW_HARNESS:-}" ]; then
+    if shadow "$REDLINES_SHADOW_HARNESS"; then
+        notify "AIRO $REDLINES_SHADOW_HARNESS shadow $STAMP done" "$(sed -n 2,6p "$(ls "$REPO"/results/experiments/"$REDLINES_SHADOW_HARNESS"-shadow-*.compare.txt | sort | tail -1)")" low
     else
-        notify "AIRO pi shadow $STAMP FAILED" "The published run is unaffected. log: $LOG" default warning
+        notify "AIRO $REDLINES_SHADOW_HARNESS shadow $STAMP FAILED" "The published run is unaffected. log: $LOG" default warning
     fi
 fi
diff --git a/code/make_combined_conditions.py b/code/make_combined_conditions.py
@@ -116,7 +116,10 @@
 OUT = os.path.join(ROOT, "data", "combined_conditions.json")
 
 SLUG = "combined"
-PROTOCOL = "unified-joint-combined-v8"
+PROTOCOL = "unified-joint-combined-v9"
+# The loop the protocol is elicited through (code/run_unified.py set_harness):
+# agent-box's pi since v9 (redlines/conditional.py PROTOCOL_LINEAGE).
+HARNESS = "pi"
 PLACEHOLDER = "{target_date}"
 
 # ── The authored sentences ──────────────────────────────────────────────────
@@ -172,6 +175,7 @@ def build():
         "slug": SLUG,
         "kind": "policy or capability",
         "protocol": PROTOCOL,
+        "harness": HARNESS,
         "source": {
             "panel": "LEAP, for the policy conditions only; no human panel answers the "
                      "capability conditions",

diff --git a/code/run_unified.py b/code/run_unified.py
@@ -424,6 +424,18 @@ def set_protocol(policies):
     return policies.get("protocol", PROTOCOL_JOINT)
 
 
+def set_harness(policies):
+    """The loop a set's protocol is elicited through (HARNESSES); part of the
+    protocol, so a set changes it only with a new tag."""
+    return policies.get("harness", "litellm")
+
+
+def row_protocol(policies, harness):
+    """The protocol a call's rows carry: the set's, or the set's +<harness>
+    for a run through a loop the set does not name (a shadow)."""
+    return set_protocol(policies) + ("" if harness == set_harness(policies) else f"+{harness}")
+
+
 def _definitions(policies):
     c = policies["conditioning"]
     return c.get("definitions") or f"Definitions. Frontier model: {c['frontier_model']}"
@@ -1354,7 +1366,7 @@ def take_cells(forecasts=None, **_):
                     "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(),
                     "prompt": prompt if first else None,
                     "system_prompt": SYSTEM if first else None,
-                    "protocol": set_protocol(policies) + ("" if harness == "litellm" else f"+{harness}"),
+                    "protocol": row_protocol(policies, harness),
                     "condition_set": set_slug(policies),
                     "call_id": call_id, "attempts": attempts,
                     "batch_size": len(qids),
@@ -1604,13 +1616,18 @@ def main():
                     help="tag every row with this experiment name (needs --out)")
     ap.add_argument("--dry-run", action="store_true",
                     help="print each arm's prompt and exit; no model is called")
-    ap.add_argument("--harness", choices=HARNESSES, default="litellm",
-                    help="the loop: litellm (redlines.llm, the series) or pi (redlines.pi_loop, "
-                         "agent-box; --joint --experiment only)")
+    ap.add_argument("--harness", choices=HARNESSES,
+                    help="the loop: the condition set's own (its `harness`; litellm when it names "
+                         "none) unless given. Another loop runs only as a --joint --experiment, "
+                         "its rows tagged <protocol>+<harness>")
     args = ap.parse_args()
-    if args.harness != "litellm" and not (args.joint and args.experiment):
-        sys.exit(f"--harness {args.harness} runs only as a --joint --experiment: "
-                 "its rows never join the series")
+    if args.harness:
+        if not args.joint:
+            sys.exit("--harness is for --joint runs")
+        own = set_harness(load_policies(args.conditions))
+        if args.harness != own and not args.experiment:
+            sys.exit(f"--harness {args.harness}: this set is elicited through {own}; another loop "
+                     "runs only as an --experiment, so its rows never join the series")
 
     # The combined instrument asks models to forecast and condition on frontier
     # ECI. Refuse to run it against a stale frontier history or a condition
@@ -1747,7 +1764,8 @@ def main_joint(args):
     policies = load_policies(args.conditions)
     conds = expand_conditions(args.condition or ["all"], policies)
     published_set = os.path.abspath(args.conditions) in {os.path.abspath(p) for p in PUBLISHED_SETS}
-    slug, proto = set_slug(policies), set_protocol(policies)
+    harness = args.harness or set_harness(policies)
+    slug, proto = set_slug(policies), row_protocol(policies, harness)
     if args.out and os.path.abspath(args.out) == os.path.abspath(RUNLOG):
         sys.exit("refusing: never append to the shared tracked log; the series "
                  "is one dated file per run under results/runs/ (see cron_run.sh)")
@@ -1815,7 +1833,7 @@ def main_joint(args):
           f"({len(models)} models x {len(arms)} repeat(s)), {n} questions x "
           f"{len(horizons)} horizons = {cells} cells x {len(conds) + 1} conditions "
           f"= {cells * (len(conds) + 1)} probabilities each")
-    print(f"    harness: research floor {MIN_RESEARCH} calls, coverage "
+    print(f"    harness: {harness} loop, research floor {MIN_RESEARCH} calls, coverage "
           f"{', '.join(g['key'] for g in groups)}, guard {MAX_ROUNDS} rounds; reasoning "
           + ", ".join(f"{l}={reasoning_for(m) or 'default'}" for l, m in models))
     print(f"    conditions: {UNCONDITIONAL_KEY}, {', '.join(c['id'] for c in conds)}"
@@ -1838,7 +1856,7 @@ def main_joint(args):
             cf.ThreadPoolExecutor(max_workers=args.workers) as ex:
         futs = {ex.submit(run_one_joint, l, m, groups, by_group, horizons, run_id,
                           args.retries, spec, conds, policies, args.experiment, arm, panel,
-                          today, args.harness): (arm, l)
+                          today, harness): (arm, l)
                 for arm in arms for l, m in models}
         for fut in cf.as_completed(futs):
             arm, label = futs[fut]

diff --git a/data/combined_conditions.json b/data/combined_conditions.json
@@ -2,7 +2,8 @@
  "title": "Policy and capability conditions in one instrument -- LEAP Wave 12 policies beside the self-elicited frontier ECI six months from the run date -- as the runner prepends them",
  "slug": "combined",
  "kind": "policy or capability",
- "protocol": "unified-joint-combined-v8",
+ "protocol": "unified-joint-combined-v9",
+ "harness": "pi",
  "source": {
   "panel": "LEAP, for the policy conditions only; no human panel answers the capability conditions",
   "wave": "LEAP Wave 12 policies + frontier ECI 6 months from the run date (the model's own p10/p25/p50/p75/p90)",