diff --git a/README.md b/README.md index 58406a4..1f2b000 100644 --- a/README.md +++ b/README.md @@ -89,12 +89,17 @@ pytest ## Elicit new forecasts ```bash -pip install -e '.[acquire]' # litellm, and nothing else +pip install -e '.[acquire,pi]' # litellm, and agent-box (the pi loop) python code/run_unified.py --joint --dry-run # print the prompt, call nothing python code/run_unified.py --joint --repeats 1 --workers 2 \ --conditions data/combined_conditions.json --out results/runs/.jsonl ``` +Since combined-v9 the calls go through pi (redlines/pi_loop.py): node 22 and +pi 0.99.2, whose `cli.js` `FORECASTER_PI` names (agent-box's README, "Requirements"). +`--harness litellm --experiment ` runs the same instrument through +redlines.llm's own loop, which is the cron's shadow. + Keys go in `~/.config/redlines/env` (mode 600), one `KEY=value` per line: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `GOOGLE_API_KEY`, `XAI_API_KEY`, `TAVILY_API_KEY`. A repo-local `.env` is read as a fallback; @@ -196,7 +201,7 @@ redlines/ the pipeline package (stdlib only; litellm only insi hydrate.py, pages.py window.__NAME__ blob injection; manifest-driven page assembly export.py forecasts.csv, rationales.csv, the download bundle, the ForecastBench export llm.py, tools.py ACQUISITION ONLY: key loading + the agentic tool-use loop; the Tavily tools - pi_loop.py the same loop through FRI's agent-box (pi), for --harness pi shadow runs + pi_loop.py call_tools through FRI's agent-box (pi): the published loop since combined-v9 views/ one pure build() per panel: graph1..graph4, observational, causal, databank, timeline, conditional, capability, axes, method paper/ the paper's figures (graph1, graph3, graph4, incident_ladders, @@ -212,7 +217,7 @@ code/ acquisition scripts, generators and thin shims (see graph3_comparability.py the Graph 3 methodology decomposition (needs FB_DATASETS_ROOT) observational/ Graph 5: the observational bench, vendored (spec.md, runners, scoring) causal/ Graph 6: the Starsim causal bench's generating code, frozen at causal-v4-locked - compare_harness.py a --harness pi shadow run against the cron's, cell by cell + compare_harness.py a --harness shadow run (the other loop) against the cron's, cell by cell data/ auto-arc/ the Auto-ARC question workbook (.xlsx) and definitions (.md) by date autoarc_ladder.json, autoarc_crosscutting.json the generated question sets (never hand-edited) diff --git a/code/compare_harness.py b/code/compare_harness.py index ee48057..240c5b4 100644 --- a/code/compare_harness.py +++ b/code/compare_harness.py @@ -1,19 +1,21 @@ #!/usr/bin/env python3 -"""The same instrument through two loops, cell by cell: the cron's litellm run -against a `--harness pi` shadow run of the same day (redlines/pi_loop.py). +"""The same instrument through two loops, cell by cell: the cron's published +run against a `--harness` shadow run of the same day through the other loop +(pi since combined-v9, litellm before; redlines/pi_loop.py says what differs). python code/compare_harness.py \\ --base results/conditional_runs_combined/2026-10-21T1200Z.jsonl \\ - --pi results/experiments/pi-shadow-2026-10-21T1405Z-instrument.jsonl \\ + --other results/experiments/litellm-shadow-2026-10-21T1405Z-instrument.jsonl \\ --prev results/conditional_runs_combined/2026-10-16T1200Z.jsonl A cell is (model, question, condition, horizon). Per model: how many cells both runs answered, the mean and median |difference| in probability, the -share that moved more than 0.1, and the mean signed difference (pi minus -litellm); then each call's research, turns, cost and debrief. One draw per +share that moved more than 0.1, and the mean signed difference (other minus +published); then each call's research, turns, cost and debrief. One draw per loop cannot tell a loop effect from run-to-run spread, so `--prev` (the -previous cron run) prints the same numbers for litellm against itself: a -loop difference that sits inside that spread is not a difference yet. +previous cron run) prints the same numbers for the published loop against +itself: a loop difference that sits inside that spread is not a difference +yet. The loops are named from the rows (usage.harness; none = litellm). """ import argparse import json @@ -60,6 +62,14 @@ def compare(a_rows, b_rows): return out +def loop_name(rows): + """The loop a run went through: pi rows carry usage.harness; litellm's none.""" + for r in calls(rows).values(): + h = r["usage"].get("harness") + return h["name"] if isinstance(h, dict) else "litellm" + return "?" + + def call_line(r): u = r.get("usage") or {} st = r.get("debrief_status") or {} @@ -79,21 +89,23 @@ def table(title, result): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - ap.add_argument("--base", required=True, help="the cron run's instrument rows (litellm)") - ap.add_argument("--pi", required=True, help="the shadow run's instrument rows (--harness pi)") - ap.add_argument("--prev", help="the previous cron run's instrument rows: litellm's own spread") + ap.add_argument("--base", required=True, help="the cron run's instrument rows (the published loop)") + ap.add_argument("--other", required=True, help="the shadow run's instrument rows (--harness, the other loop)") + ap.add_argument("--prev", help="the previous cron run's instrument rows: the published loop's own spread") args = ap.parse_args(argv) - base, pi = load(args.base), load(args.pi) - if not compare(base, pi): + base, other = load(args.base), load(args.other) + b, o = loop_name(base), loop_name(other) + if not compare(base, other): sys.exit("no cells in common: different instruments, or a run without forecasts") - table("pi vs litellm, same day (pi minus litellm):", compare(base, pi)) + table(f"{o} vs {b}, same day ({o} minus {b}):", compare(base, other)) if args.prev: - table("\nlitellm vs litellm, previous run (this run minus previous):", compare(load(args.prev), base)) + table(f"\n{b} vs {b}, previous run (this run minus previous):", compare(load(args.prev), base)) print("\ncalls:") + w = max(len(b), len(o)) for label, r in calls(base).items(): - print(f" {label:16} litellm {call_line(r)}") - p = calls(pi).get(label) - print(f" {'':16} pi {call_line(p) if p else 'no call'}") + print(f" {label:16} {b:{w}} {call_line(r)}") + p = calls(other).get(label) + print(f" {'':16} {o:{w}} {call_line(p) if p else 'no call'}") if __name__ == "__main__": diff --git a/code/cron_run.sh b/code/cron_run.sh index db43f79..a33c055 100755 --- a/code/cron_run.sh +++ b/code/cron_run.sh @@ -177,6 +177,10 @@ fi # call is longer and dearer than it was. 4 models x 1 = 4 calls. The views # draw no interval on a single-draw day; the dots and the median are the # reading. +# THE LOOP is the set's own: agent-box's pi since combined-v9 (2026-10-16; +# redlines/pi_loop.py, three hours a call, REDLINES_PI_TIMEOUT), so the env +# file must give FORECASTER_PI and node on PATH (see THE SHADOW below). The +# request timeout here is the litellm loop's, which the shadow now runs. export REDLINES_LLM_TIMEOUT="${REDLINES_LLM_TIMEOUT:-1800}" # The runner exits non-zero when any panel model fails (the date cannot # publish without all of them), which under `set -e` stops this script before @@ -223,30 +227,31 @@ fi FINISHED=1 notify "AIRO run $STAMP published" "$(grep -E 'ok, [0-9]+ failed' "$LOG" | tail -1 | sed 's/ -> .*//'). Panel: $PANEL" low white_check_mark -# THE PI SHADOW (2026-10-14): the same instrument, same panel, same day, -# through agent-box's pi loop instead of redlines.llm (code/run_unified.py -# --harness pi; redlines/pi_loop.py says what differs). An experiment, never -# the series: rows go to results/experiments/pi-shadow-*, protocol "...+pi". -# Then code/compare_harness.py prints the two runs cell by cell, beside the -# previous run as litellm's own spread. Off unless the env file sets -# REDLINES_PI_SHADOW=1; needs agent-box in PY (`pip install -e '.[pi]'`), and -# in the env file FORECASTER_PI (pi 0.99.2's cli.js) and a PATH with node on -# it (cron's own PATH has none). It runs after publication and can only -# notify: the published run is already done. -pi_shadow() { - local runs="$REPO/results/conditional_runs_combined" base prev shadow +# THE SHADOW (2026-10-14): the same instrument, same panel, same day, through +# the OTHER loop (code/run_unified.py --harness; the set names its own: pi +# since combined-v9, redlines.llm's litellm before). An experiment, never the +# series: rows go to results/experiments/-shadow-*, protocol +# "+". Then code/compare_harness.py prints the two runs +# cell by cell, beside the previous run as the published loop's own spread. +# Off unless the env file sets REDLINES_SHADOW_HARNESS (litellm or pi). The +# pi loop needs agent-box in PY (`pip install -e '.[pi]'`), and in the env +# file FORECASTER_PI (pi 0.99.2's cli.js) and a PATH with node on it (cron's +# own PATH has none) -- for the published run as much as for a shadow. It +# runs after publication and can only notify: the published run is done. +shadow() { + local h="$1" runs="$REPO/results/conditional_runs_combined" base prev other base="$(ls "$runs"/*.jsonl | sort | tail -1)" prev="$(ls "$runs"/*.jsonl | sort | tail -2 | head -1)" - echo "=== pi shadow: the instrument through pi; litellm's is $base ===" - "$PY" code/run_unified.py --joint --repeats 1 --workers 2 --harness pi \ - --experiment pi-shadow --conditions data/combined_conditions.json || return 1 - shadow="$(ls "$REPO"/results/experiments/pi-shadow-*-instrument.jsonl | sort | tail -1)" - "$PY" code/compare_harness.py --base "$base" --pi "$shadow" --prev "$prev" | tee "${shadow%.jsonl}.compare.txt" + echo "=== $h shadow: the instrument through $h; the published run is $base ===" + "$PY" code/run_unified.py --joint --repeats 1 --workers 2 --harness "$h" \ + --experiment "$h-shadow" --conditions data/combined_conditions.json || return 1 + other="$(ls "$REPO"/results/experiments/"$h"-shadow-*-instrument.jsonl | sort | tail -1)" + "$PY" code/compare_harness.py --base "$base" --other "$other" --prev "$prev" | tee "${other%.jsonl}.compare.txt" } -if [ "${REDLINES_PI_SHADOW:-0}" = 1 ]; then - if pi_shadow; then - notify "AIRO pi shadow $STAMP done" "$(sed -n 2,6p "$(ls "$REPO"/results/experiments/pi-shadow-*.compare.txt | sort | tail -1)")" low +if [ -n "${REDLINES_SHADOW_HARNESS:-}" ]; then + if shadow "$REDLINES_SHADOW_HARNESS"; then + notify "AIRO $REDLINES_SHADOW_HARNESS shadow $STAMP done" "$(sed -n 2,6p "$(ls "$REPO"/results/experiments/"$REDLINES_SHADOW_HARNESS"-shadow-*.compare.txt | sort | tail -1)")" low else - notify "AIRO pi shadow $STAMP FAILED" "The published run is unaffected. log: $LOG" default warning + notify "AIRO $REDLINES_SHADOW_HARNESS shadow $STAMP FAILED" "The published run is unaffected. log: $LOG" default warning fi fi diff --git a/code/make_combined_conditions.py b/code/make_combined_conditions.py index 57ad8e1..9bf15f7 100644 --- a/code/make_combined_conditions.py +++ b/code/make_combined_conditions.py @@ -116,7 +116,10 @@ OUT = os.path.join(ROOT, "data", "combined_conditions.json") SLUG = "combined" -PROTOCOL = "unified-joint-combined-v8" +PROTOCOL = "unified-joint-combined-v9" +# The loop the protocol is elicited through (code/run_unified.py set_harness): +# agent-box's pi since v9 (redlines/conditional.py PROTOCOL_LINEAGE). +HARNESS = "pi" PLACEHOLDER = "{target_date}" # ── The authored sentences ────────────────────────────────────────────────── @@ -172,6 +175,7 @@ def build(): "slug": SLUG, "kind": "policy or capability", "protocol": PROTOCOL, + "harness": HARNESS, "source": { "panel": "LEAP, for the policy conditions only; no human panel answers the " "capability conditions", diff --git a/code/run_unified.py b/code/run_unified.py index b12d581..d769a3f 100644 --- a/code/run_unified.py +++ b/code/run_unified.py @@ -424,6 +424,18 @@ def set_protocol(policies): return policies.get("protocol", PROTOCOL_JOINT) +def set_harness(policies): + """The loop a set's protocol is elicited through (HARNESSES); part of the + protocol, so a set changes it only with a new tag.""" + return policies.get("harness", "litellm") + + +def row_protocol(policies, harness): + """The protocol a call's rows carry: the set's, or the set's + + for a run through a loop the set does not name (a shadow).""" + return set_protocol(policies) + ("" if harness == set_harness(policies) else f"+{harness}") + + def _definitions(policies): c = policies["conditioning"] return c.get("definitions") or f"Definitions. Frontier model: {c['frontier_model']}" @@ -1354,7 +1366,7 @@ def take_cells(forecasts=None, **_): "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), "prompt": prompt if first else None, "system_prompt": SYSTEM if first else None, - "protocol": set_protocol(policies) + ("" if harness == "litellm" else f"+{harness}"), + "protocol": row_protocol(policies, harness), "condition_set": set_slug(policies), "call_id": call_id, "attempts": attempts, "batch_size": len(qids), @@ -1604,13 +1616,18 @@ def main(): help="tag every row with this experiment name (needs --out)") ap.add_argument("--dry-run", action="store_true", help="print each arm's prompt and exit; no model is called") - ap.add_argument("--harness", choices=HARNESSES, default="litellm", - help="the loop: litellm (redlines.llm, the series) or pi (redlines.pi_loop, " - "agent-box; --joint --experiment only)") + ap.add_argument("--harness", choices=HARNESSES, + help="the loop: the condition set's own (its `harness`; litellm when it names " + "none) unless given. Another loop runs only as a --joint --experiment, " + "its rows tagged +") args = ap.parse_args() - if args.harness != "litellm" and not (args.joint and args.experiment): - sys.exit(f"--harness {args.harness} runs only as a --joint --experiment: " - "its rows never join the series") + if args.harness: + if not args.joint: + sys.exit("--harness is for --joint runs") + own = set_harness(load_policies(args.conditions)) + if args.harness != own and not args.experiment: + sys.exit(f"--harness {args.harness}: this set is elicited through {own}; another loop " + "runs only as an --experiment, so its rows never join the series") # The combined instrument asks models to forecast and condition on frontier # ECI. Refuse to run it against a stale frontier history or a condition @@ -1747,7 +1764,8 @@ def main_joint(args): policies = load_policies(args.conditions) conds = expand_conditions(args.condition or ["all"], policies) published_set = os.path.abspath(args.conditions) in {os.path.abspath(p) for p in PUBLISHED_SETS} - slug, proto = set_slug(policies), set_protocol(policies) + harness = args.harness or set_harness(policies) + slug, proto = set_slug(policies), row_protocol(policies, harness) if args.out and os.path.abspath(args.out) == os.path.abspath(RUNLOG): sys.exit("refusing: never append to the shared tracked log; the series " "is one dated file per run under results/runs/ (see cron_run.sh)") @@ -1815,7 +1833,7 @@ def main_joint(args): f"({len(models)} models x {len(arms)} repeat(s)), {n} questions x " f"{len(horizons)} horizons = {cells} cells x {len(conds) + 1} conditions " f"= {cells * (len(conds) + 1)} probabilities each") - print(f" harness: research floor {MIN_RESEARCH} calls, coverage " + print(f" harness: {harness} loop, research floor {MIN_RESEARCH} calls, coverage " f"{', '.join(g['key'] for g in groups)}, guard {MAX_ROUNDS} rounds; reasoning " + ", ".join(f"{l}={reasoning_for(m) or 'default'}" for l, m in models)) print(f" conditions: {UNCONDITIONAL_KEY}, {', '.join(c['id'] for c in conds)}" @@ -1838,7 +1856,7 @@ def main_joint(args): cf.ThreadPoolExecutor(max_workers=args.workers) as ex: futs = {ex.submit(run_one_joint, l, m, groups, by_group, horizons, run_id, args.retries, spec, conds, policies, args.experiment, arm, panel, - today, args.harness): (arm, l) + today, harness): (arm, l) for arm in arms for l, m in models} for fut in cf.as_completed(futs): arm, label = futs[fut] diff --git a/data/combined_conditions.json b/data/combined_conditions.json index cd7212e..8cb1458 100644 --- a/data/combined_conditions.json +++ b/data/combined_conditions.json @@ -2,7 +2,8 @@ "title": "Policy and capability conditions in one instrument -- LEAP Wave 12 policies beside the self-elicited frontier ECI six months from the run date -- as the runner prepends them", "slug": "combined", "kind": "policy or capability", - "protocol": "unified-joint-combined-v8", + "protocol": "unified-joint-combined-v9", + "harness": "pi", "source": { "panel": "LEAP, for the policy conditions only; no human panel answers the capability conditions", "wave": "LEAP Wave 12 policies + frontier ECI 6 months from the run date (the model's own p10/p25/p50/p75/p90)", diff --git a/paper/numbers.tex b/paper/numbers.tex index a1cd731..63c3bac 100644 --- a/paper/numbers.tex +++ b/paper/numbers.tex @@ -5121,7 +5121,7 @@ \rldef{prompt:horizons:list}{6mo, 12mo, 2028, 2030, 2050, 2100} \rldef{prompt:horizons:n}{6} \rldef{prompt:probabilities:n}{3,444} -\rldef{prompt:protocol}{unified-joint-combined-v8} +\rldef{prompt:protocol}{unified-joint-combined-v9} \rldef{prompt:question:catastrophe:ai}{-{}-{}- catastrophe:ai (severity: deaths reaching 10\% of the population) -{}-{}-\newline What is the probability that a global AI-related catastrophe (AI being the proximate cause, per the but-for standard, of attributable deaths reaching at least 10\% of the population alive at the start of the measurement window) will occur within 6 months / within 12 months / by the end of 2028 / by the end of 2030 / by the end of 2050 / by the end of 2100?\newline Horizons for this question: 6mo, 12mo, 2028, 2030, 2050, 2100\par\smallskip\noindent Resolution criteria:\newline * Catastrophe threshold and the measurement window: A catastrophe at the 10\% level means that deaths attributable to the catastrophe (excess mortality, i.e., deaths beyond the baseline that would have occurred anyway) reach at least 10\% of the population alive at the start of the measurement window.\newline * The measurement window is any contiguous period of at most five years during which the attributable deaths accumulate to the threshold. The window must begin on or after December 31, 2025 and end on or before December 31 of the resolution year (2030, 2050, or 2100). A faster event; the five years is a ceiling on the span over which deaths are counted, not a required duration.\newline * For a threshold to resolve positively, the deaths must arise from a single catastrophe or a single causally-linked chain of events, not from summing unrelated causes.\newline * Data and disagreement: For fatality and population data, we will rely on broadly recognized sources (e.g., Our World in Data, the Economist, the United Nations). Where these disagree in ways that would change resolution, we will resolve by the majority opinion of a panel of experts in demographics and fields relevant to the specific catastrophe (e.g., epidemiology for a pandemic). If reasonable people disagree about whether the event occurred, the question resolves via the same expert panel.\par\smallskip\noindent [Carried forward from "General large-scale catastrophe"]\par\smallskip\noindent * The points above also apply to this question.\newline * Artificial intelligence: Defined as the development of machines capable of sophisticated (intelligent) information processing (Dafoe 2018, AI Governance: A Research Agenda, p. 5 fn. 2).\newline * AI as proximate cause (but-for standard): We consider AI a proximate cause of death under a but-for test: the event would not have occurred, or would have been counterfactually extremely unlikely to occur, but for the substantial involvement of AI.\newline \ \ * Examples that count: an AI-powered system directly killing people at the relevant magnitude; AI autonomously triggering a nuclear launch (including deaths from subsequent nuclear winter); AI being substantially responsible for creating a superweapon that later causes deaths at the relevant magnitude; AI acting autonomously in armed conflict and causing deaths at the relevant magnitude.\newline \ \ * Examples that do not count: AI assisting in targeting a weapon that humans could and would have launched anyway (the AI did not counterfactually enable the event).\newline * There is no fixed time limit between the AI's involvement and the deaths. Creation of a superweapon may precede its use by years and still satisfy the but-for standard; the test is counterfactual materiality, rather than timing.\newline * [Scope of "the resolution year", added 2026-08-28 when these questions moved onto the incident ladder's horizon grid: read it as the resolution date of whichever horizon you are answering, which for a rolling horizon is that many months after today's date. The rule is otherwise unchanged -{}- the measurement window still begins on or after December 31, 2025, must end on or before that resolution date, and still spans at most five years.]} \rldef{prompt:question:catastrophe:general}{-{}-{}- catastrophe:general (severity: deaths reaching 10\% of the population) -{}-{}-\newline What is the probability that any global catastrophe (deaths reaching at least 10\% of the population alive at the start of the measurement window) will occur within 6 months / within 12 months / by the end of 2028 / by the end of 2030 / by the end of 2050 / by the end of 2100?\newline Horizons for this question: 6mo, 12mo, 2028, 2030, 2050, 2100\par\smallskip\noindent Resolution criteria:\newline * Catastrophe threshold and the measurement window: A catastrophe at the 10\% level means that deaths attributable to the catastrophe (excess mortality, i.e., deaths beyond the baseline that would have occurred anyway) reach at least 10\% of the population alive at the start of the measurement window.\newline * The measurement window is any contiguous period of at most five years during which the attributable deaths accumulate to the threshold. The window must begin on or after December 31, 2025 and end on or before December 31 of the resolution year (2030, 2050, or 2100). A faster event; the five years is a ceiling on the span over which deaths are counted, not a required duration.\newline * For a threshold to resolve positively, the deaths must arise from a single catastrophe or a single causally-linked chain of events, not from summing unrelated causes.\newline * Data and disagreement: For fatality and population data, we will rely on broadly recognized sources (e.g., Our World in Data, the Economist, the United Nations). Where these disagree in ways that would change resolution, we will resolve by the majority opinion of a panel of experts in demographics and fields relevant to the specific catastrophe (e.g., epidemiology for a pandemic). If reasonable people disagree about whether the event occurred, the question resolves via the same expert panel.\newline * [Scope of "the resolution year", added 2026-08-28 when these questions moved onto the incident ladder's horizon grid: read it as the resolution date of whichever horizon you are answering, which for a rolling horizon is that many months after today's date. The rule is otherwise unchanged -{}- the measurement window still begins on or after December 31, 2025, must end on or before that resolution date, and still spans at most five years.]} \rldef{prompt:question:disempowerment}{-{}-{}- disempowerment (severity: not applicable) -{}-{}-\newline What is the probability that disempowerment of humanity will occur within 6 months / within 12 months / by the end of 2028 / by the end of 2030 / by the end of 2050 / by the end of 2100?\newline Horizons for this question: 6mo, 12mo, 2028, 2030, 2050, 2100\par\smallskip\noindent Resolution criteria:\newline * Disempowerment of humanity is defined as a state where AI systems are the primary determinant of human events globally.\newline * This could occur via an overt takeover, where an AI system takes control of global systems of power (such as national governments, international organizations, multinational corporations) and humans have no role in executing power. It could also occur if the majority of human events are determined by AI decision, even if humans are nominally in control but humans could not behave differently if they wanted to, for example because they do not understand the decisions well enough to make a decision, or they are coerced into certain actions due to threats or other means.} diff --git a/pyproject.toml b/pyproject.toml index 670ee74..2371f86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ acquire = [ # "xhigh" to OpenAI reasoning.effort (redlines/llm.py LITELLM_MIN). "litellm>=1.96", ] -# The pi loop (code/run_unified.py --harness pi, redlines/pi_loop.py): FRI's +# The pi loop (redlines/pi_loop.py; the published loop since combined-v9): FRI's # agent-box, which drives pi 0.99.2 -- FORECASTER_PI names its cli.js. pi = [ "fri-forecaster @ git+https://github.com/forecastingresearch/agent-box", diff --git a/redlines/conditional.py b/redlines/conditional.py index a844ad8..0fde2e0 100644 --- a/redlines/conditional.py +++ b/redlines/conditional.py @@ -83,6 +83,20 @@ def _logit(p): # falls back down the line, so the tab keeps its rows on the day of a bump and # switches the day the first run under the new tag lands. PROTOCOL_LINEAGE = { + # 2026-10-16, v9: the LOOP changes, nothing else. The instrument -- prompt, + # questions, conditions, tools and their schemas, the research floor, the + # coverage rule, the turn limit, the debrief -- is v8's; the calls go + # through FRI's agent-box (pi, redlines/pi_loop.py) instead of + # redlines.llm's litellm loop, as FRI's ForecastBench runs do. The full + # panel through pi on 2026-10-14 (results/experiments/pi-shadow- + # 2026-10-14T2144Z, code/compare_harness.py), against that day's litellm + # run: mean |d| 0.019-0.025 per model over 3,444 cells, 3.9-7.2% moved + # >0.1 -- inside litellm's own spread from the run before (0.022-0.041, + # 4.8-15.3%) for every model. + "unified-joint-combined-v9": ("unified-joint-combined-v8", "unified-joint-combined-v7", + "unified-joint-combined-v6", "unified-joint-combined-v5", + "unified-joint-combined-v4", "unified-joint-combined-v3", + "unified-joint-combined-v2", "unified-joint-combined-v1"), # 2026-10-01, v8: the extinction pair's criteria stop naming their source. # The AI question had said its criteria were XPT's, "kept verbatim so the # forecasts compare with that panel's", and the total-extinction question @@ -145,7 +159,7 @@ def _logit(p): # the same string (tests pin the two together); the views, the run log's # grounding line and the coherence test all read it from here, so a bump is # one edit plus the lineage entry above. -COMBINED_PROTOCOL = "unified-joint-combined-v8" +COMBINED_PROTOCOL = "unified-joint-combined-v9" # The first combined tag elicited by the agentic harness (iterative search + # page reads, 2026-09-02); v1 (2026-08-28) still ran the legacy grounding. AGENTIC_SINCE = "unified-joint-combined-v2" diff --git a/redlines/pi_loop.py b/redlines/pi_loop.py index 8795428..a104569 100644 --- a/redlines/pi_loop.py +++ b/redlines/pi_loop.py @@ -1,16 +1,18 @@ -"""The same call as redlines.llm.call_tools, with pi as the loop. +"""The same call as redlines.llm.call_tools, with pi as the loop: the +published loop since combined-v9 (2026-10-16), whose condition set names it +(`"harness": "pi"`; code/run_unified.py set_harness). -`code/run_unified.py --harness pi` swaps this in for call_tools and changes -nothing else: the prompt, the tools and their schemas, the research floor, the +Under pi, run_unified swaps this in for call_tools and changes nothing else: the prompt, the tools and their schemas, the research floor, the coverage rule, the turn limit, the debrief and its validator, the cell cleaning and the rows are the instrument's own. The loop is FRI's agent-box (github.com/forecastingresearch/agent-box, the `fri_forecaster` package): pi drives the model, and its extension enforces the rules above with the loop options that mirror call_tools (agent-box README, "How it works"). -So a run under `--harness pi` measures the loop and nothing else. Its rows -carry protocol "+pi" and may only be written as an experiment -(run_unified refuses otherwise): a pi row never pools with the series. +So the loop is the only difference between v8 and v9. A run through the loop +a set does not name (`--harness litellm` on v9, the cron's shadow) carries +protocol "+" and may only be written as an experiment +(run_unified refuses otherwise): it never pools with the series. Where the loops differ, by design of pi rather than of the instrument: * pi cannot force a tool choice, so the last turn ASKS for submit_forecast diff --git a/tests/test_conditional.py b/tests/test_conditional.py index 4c485d8..52e9c9d 100644 --- a/tests/test_conditional.py +++ b/tests/test_conditional.py @@ -750,7 +750,7 @@ def test_set_is_current_and_composed_of_the_two_components(self): [c["id"] for c in self.leap["conditions"]] + [c["id"] for c in self.es["conditions"]]) self.assertEqual([g["key"] for g in cb["groups"]], ["policy", "capability"]) self.assertEqual([c["group"] for c in cb["conditions"]], ["policy"] * 8 + ["capability"] * 5) - self.assertEqual(self.ru.set_protocol(cb), "unified-joint-combined-v8") # October 1: the pair unnamed; September 15: the extinction pair (v5 = September 10 counting version) + self.assertEqual(self.ru.set_protocol(cb), "unified-joint-combined-v9") # October 16: the pi loop; October 1: the pair unnamed; September 15: the extinction pair (v5 = September 10 counting version) self.assertEqual(self.ru.set_slug(cb), "combined") self.assertNotIn(self.ru.set_protocol(cb), (self.ru.PROTOCOL_JOINT, self.ru.set_protocol(self.es))) self.assertEqual(cb["elicit"], self.es["elicit"]) @@ -880,7 +880,7 @@ def test_group_view_is_the_component_set_plus_the_assumption(self): self.assertEqual(pol["source"]["panel"], "LEAP") self.assertIn("eci_p50", pol["conditioning"]["assumption"]) self.assertEqual(pol["slug"], "combined") # the log is still the combined one - self.assertEqual(pol["protocol"], "unified-joint-combined-v8") + self.assertEqual(pol["protocol"], "unified-joint-combined-v9") cap = group_view(self.cb, "capability") self.assertEqual([c["field"] for c in cap["conditions"]], ["p10", "p25", "p50", "p75", "p90"]) self.assertEqual(cap["conditioning"]["instruction"], self.es["conditioning"]["instruction"]) @@ -912,7 +912,7 @@ def test_the_tabs_take_the_newest_instrument(self): self.assertEqual(view["questions"], []) self.assertEqual(view["models"], []) self.assertFalse(view["instrumentInfo"]["available"]) - self.assertEqual(view["protocol"], "unified-joint-combined-v8") + self.assertEqual(view["protocol"], "unified-joint-combined-v9") return # Separate provider retries may have distinct run IDs but share the # single explicitly recorded elicitation date and incident windows. diff --git a/tests/test_extinction_addendum.py b/tests/test_extinction_addendum.py index c3e9eaf..abc7829 100644 --- a/tests/test_extinction_addendum.py +++ b/tests/test_extinction_addendum.py @@ -269,13 +269,13 @@ def test_the_published_prompt_names_no_source_above_the_conditions(self): class TestTheTagMoved(unittest.TestCase): - def test_combined_v8_with_its_lineage(self): - # v8 (2026-10-01, the pair unnamed) sits above v7 (2026-09-16, the - # extinction rung) and v6 (the pair); a log holding only v5 and v6 - # rows still reads as v6 until a later run lands. - self.assertEqual(COMBINED_PROTOCOL, "unified-joint-combined-v8") + def test_combined_v9_with_its_lineage(self): + # v9 (2026-10-16, the pi loop) sits above v8 (2026-10-01, the pair + # unnamed), v7 (2026-09-16, the extinction rung) and v6 (the pair); a + # log holding only v5 and v6 rows still reads as v6 until a later run lands. + self.assertEqual(COMBINED_PROTOCOL, "unified-joint-combined-v9") self.assertEqual(protocol_line(COMBINED_PROTOCOL)[1:], - ("unified-joint-combined-v7", "unified-joint-combined-v6", "unified-joint-combined-v5", + ("unified-joint-combined-v8", "unified-joint-combined-v7", "unified-joint-combined-v6", "unified-joint-combined-v5", "unified-joint-combined-v4", "unified-joint-combined-v3", "unified-joint-combined-v2", "unified-joint-combined-v1")) rows = [{"protocol": "unified-joint-combined-v5"}, {"protocol": "unified-joint-combined-v6"}] diff --git a/tests/test_pi_loop.py b/tests/test_pi_loop.py index 07728e7..3663389 100644 --- a/tests/test_pi_loop.py +++ b/tests/test_pi_loop.py @@ -103,11 +103,23 @@ def test_validator_runs_out_of_process(self): class HarnessFlagTest(unittest.TestCase): - def test_pi_rows_only_as_an_experiment(self): - r = subprocess.run([sys.executable, str(REPO / "code" / "run_unified.py"), "--joint", "--harness", "pi"], + def setUp(self): + self.ru = _load_runner() + + def test_the_set_names_its_loop(self): + cb = json.load(open(REPO / "data" / "combined_conditions.json")) + self.assertEqual(self.ru.set_harness(cb), "pi") # since combined-v9 + self.assertEqual(self.ru.row_protocol(cb, "pi"), "unified-joint-combined-v9") + self.assertEqual(self.ru.row_protocol(cb, "litellm"), "unified-joint-combined-v9+litellm") + legacy = {"protocol": "unified-joint-v6"} # a set naming none: litellm + self.assertEqual(self.ru.row_protocol(legacy, "litellm"), "unified-joint-v6") + self.assertEqual(self.ru.row_protocol(legacy, "pi"), "unified-joint-v6+pi") + + def test_another_loop_only_as_an_experiment(self): + r = subprocess.run([sys.executable, str(REPO / "code" / "run_unified.py"), "--joint", "--harness", "litellm"], capture_output=True, text=True, cwd=REPO) self.assertNotEqual(r.returncode, 0) - self.assertIn("only as a --joint --experiment", r.stderr) + self.assertIn("this set is elicited through pi; another loop runs only as an --experiment", r.stderr) if __name__ == "__main__":