diff --git a/README.md b/README.md index 1f2b000..983d980 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,11 @@ One page, `index.html`, assembled from the chunks under `web/demo/`. Its panels: ensemble median. The series starts at the agentic-harness cutover on 2026-09-02 (`redlines/runlog.py::SERIES_START`); earlier elicitations stay in the run log and the download but are not drawn on the same axis. A model that - leaves the panel keeps its points, drawn in gray. + leaves the panel keeps its points, drawn in gray. Since combined-v10 a date + carries one light line per prompt variant (five, whose mean is each model's + published number) and, since 2026-10-19, a dashed sixth for the call with + AI forecasts banned and AIRO's own pages kept out of its search, which is + drawn but never pooled (`data/combined_conditions.json` `unpooled_variants`). - **Graph 1** — the current forecasts by horizon: the cross-cutting questions (general catastrophe, AI catastrophe, human disempowerment) and one expected-loss row per AI-related incident cause, with prior human panels diff --git a/code/cron_run.sh b/code/cron_run.sh index 63d04fa..d32887f 100755 --- a/code/cron_run.sh +++ b/code/cron_run.sh @@ -190,6 +190,15 @@ export REDLINES_LLM_TIMEOUT="${REDLINES_LLM_TIMEOUT:-1800}" # COMBINED-V10 (2026-10-19): the minimal instrument under five prompt variants # (the set declares both), so 4 models x 5 = 20 calls, ten at a time; about # 5x a v9 run's spend, the published number per model the mean of its five. +# THE SIXTH CALL (2026-10-19 evening): the set also declares one unpooled +# variant per model -- the baseline prompt with the AI-forecast ban, and +# agent-box's URL guard against AIRO's own pages passed to that call alone +# (`unpooled_variants`; run_unified set_unpooled) -- 24 calls in all. It is +# a line on the Timeline, never in the published number, and a failed sixth +# call does not fail the run. The guard needs agent-box's `url-guard` in PY +# (FORECASTER_BLOCK_URL_PATTERNS in its tools); an older agent-box ignores +# the variable and the sixth call searches unguarded while its rows say +# otherwise, so update agent-box before the first run under this set. if ! "$PY" code/run_unified.py --joint --repeats 1 --workers 10 \ --conditions data/combined_conditions.json --out "$OUT"; then echo "FATAL: the combined instrument did not complete for every panel model; see above. Skipping the axes instruments and publication." diff --git a/code/make_combined_conditions.py b/code/make_combined_conditions.py index 38a9c09..4564669 100644 --- a/code/make_combined_conditions.py +++ b/code/make_combined_conditions.py @@ -96,6 +96,21 @@ v4 -> v5 on 2026-09-10: the prospective incident-counting definitions (redlines.instrument.CURRENT_INSTRUMENT), same call shape. +v10 + THE SIXTH VARIANT, 2026-10-19 evening: "AI forecasts banned". Beside +the five prompts the set now asks a sixth call per model: the baseline (`none`) +prompt with one added sentence -- forecasts by AI systems carry no information +beyond their inputs; do not consult them -- AND agent-box's URL guard +(FORECASTER_BLOCK_URL_PATTERNS, passed to that call alone) keeping AIRO's own +published forecasts out of its search. It is the `both` arm of the anchor-guard +experiment of 2026-10-16 (elsehow/airo#5), asked every run. It is declared +under `unpooled_variants`: asked with no new flag, drawn as its own line on the +Timeline, and NEVER part of a model's published number -- the mean, the CI, +Graph 1, the data bank and the Policy-levers tab stay the mean of the five +(redlines.runlog.pooled_rows). Its rows carry `variant`, `url_block_patterns` +and `pooled: false`; the five's rows and prompts are unchanged, so the protocol +tag stays v10. A failed sixth call does not fail the run: the day is complete +when the five are (code/validate_launch_run.py). + v5 -> v6 on 2026-09-15: THE EXTINCTION PAIR. Two cross-cutting questions in XPT's wording (human extinction, AI extinction: data/auto-arc/ addendum-extinction-2026-09-15.json, docs/extinction-questions-2026-09-15.md) @@ -128,6 +143,15 @@ # prompts), which found the instrument's guidance moved no headline forecast. INSTRUMENT = "minimal" VARIANTS = ["none", "independent", "recent", "critical", "two-step"] +# The sixth call (2026-10-19 evening): the baseline prompt with the AI-forecast +# ban (code/run_unified.py VARIANTS["ai-banned"], the sentence) and agent-box's +# URL guard against AIRO's own pages, per call. Asked beside the five, never +# pooled into the published number (see the docstring). +UNPOOLED_VARIANTS = [ + {"variant": "ai-banned", + "url_block_patterns": r"forecastingresearch\.org/pdf/airo,forecastingresearch\.org/research/airo," + r"airo\.forecastingresearch\.org"}, +] PLACEHOLDER = "{target_date}" # ── The authored sentences ────────────────────────────────────────────────── @@ -186,6 +210,7 @@ def build(): "harness": HARNESS, "instrument": INSTRUMENT, "variants": VARIANTS, + "unpooled_variants": UNPOOLED_VARIANTS, "source": { "panel": "LEAP, for the policy conditions only; no human panel answers the " "capability conditions", diff --git a/code/run_unified.py b/code/run_unified.py index af7f020..f89ffc9 100644 --- a/code/run_unified.py +++ b/code/run_unified.py @@ -92,7 +92,7 @@ from redlines.llm import call_tools, load_keys, reasoning_for, research_coverage # noqa: E402 from redlines.logs import run_file # noqa: E402 -from redlines.tools import FORECAST_TOOLS # noqa: E402 +from redlines.tools import FORECAST_TOOLS, BLOCK_ENV # noqa: E402 from redlines.registry import DEFAULT_MODEL_SET, MODEL_SETS, panel_provenance # noqa: E402 from redlines.instrument import counting_windows # noqa: E402 from redlines.config import UNBATCHED # noqa: E402 @@ -416,6 +416,18 @@ "improvement or recursive self-improvement. Consider the distribution, not only " "the modal outcome. Then, given possible model capabilities, evaluate the risk " "posed and put these pieces together."), + # THE SIXTH (2026-10-19 evening): the AI-forecast ban, the `both` arm of the + # anchor-guard experiment of 2026-10-16 (results/experiments/anchor-guard- + # 2026-10-16, where it was a system-prompt addendum; the minimal instrument + # has no system prompt, so it goes where a variant's text goes). The set + # pairs it with agent-box's URL guard against AIRO's own pages + # (`unpooled_variants[].url_block_patterns`, set_unpooled), passed to that + # call alone. Asked every run beside the five, drawn on the Timeline, never + # in the published number (redlines.runlog.pooled_rows). + "ai-banned": ( + "Forecasts produced by AI systems, including any automated forecasting " + "dashboard or model ensemble, carry no information beyond their inputs. Do not " + "consult them or use them as evidence."), } DEFAULT_VARIANT = "none" VARIANT_BLOCK = """===== APPROACH ===== @@ -426,10 +438,33 @@ def set_variants(policies): - """The prompt variants a set is asked under (combined-v10: all five).""" + """The prompt variants a set is asked under (combined-v10: all five) -- + the POOLED ones, whose mean is a model's published number.""" return list(policies.get("variants") or [DEFAULT_VARIANT]) +def set_unpooled(policies): + """{variant: spec} for the variants a set asks BESIDE its pooled ones + (`unpooled_variants`; combined-v10 since 2026-10-19 evening: "ai-banned"). + A spec may name `url_block_patterns`, agent-box's URL guard for that + call alone (redlines.tools.BLOCK_ENV; pi only). Such a call's rows carry + the variant, the guard and `pooled: false`, and no view pools them.""" + return {v["variant"]: v for v in policies.get("unpooled_variants") or []} + + +def set_asked_variants(policies): + """Every variant a run of the set asks: the pooled ones, then the unpooled.""" + return set_variants(policies) + [v for v in set_unpooled(policies) if v not in set_variants(policies)] + + +def check_patterns(raw): + """A bad guard pattern fails before the first paid call, not inside it + (the split is redlines.tools.blocked_patterns's: newlines or commas).""" + for p in re.split(r"[\n,]", raw or ""): + if p.strip(): + re.compile(p.strip(), re.I) + + def set_instrument(policies): """"minimal" (combined-v10) or "standard": the guidance the prompt carries.""" return policies.get("instrument", "standard") @@ -1348,6 +1383,20 @@ def run_one_joint(label, model_id, groups, by_group, horizons, run_id, retries, prompt, n, want_cells = build_prompt_joint(groups, by_group, horizons, spec, conds, policies, today) prompt = minimal_joint_prompt(prompt, variant) if minimal else apply_variant(prompt, variant) + # An unpooled variant's URL guard goes to THIS call's tool processes + # through the loop's `env` (redlines/pi_loop.py -> agent-box harness.run), + # never into os.environ, which every worker shares. A guard set in the + # process environment (an --experiment; main() gates it) reaches every + # call through agent-box and redlines.tools alike, and is stamped as #5 + # stamped it. Either way a guarded row says so (`url_block_patterns`). + aside = set_unpooled(policies).get(variant) + guard = (aside or {}).get("url_block_patterns") or os.environ.get(BLOCK_ENV) or None + if aside and aside.get("url_block_patterns") and harness != "pi": + raise RuntimeError(f"variant {variant!r} carries a URL guard, which travels per call " + "through pi only") + extra = {**({"url_block_patterns": guard} if guard else {}), + # Not in the published number: redlines.runlog.pooled_rows. + **({"pooled": False} if aside else {})} block = conditions_block(conds, policies) sha = hashlib.sha256(block.encode()).hexdigest() cond_ids = [c["id"] for c in conds] @@ -1405,7 +1454,9 @@ def take_cells(forecasts=None, **_): min_evidence=0 if minimal else MIN_RESEARCH, usage=usage, partial_tool=partial, coverage=coverage, debrief=debrief, **({"allowed_keys": [f"{q}|{h}" for q, hs in allowed.items() for h in sorted(hs)]} - if harness == "pi" else {})) + if harness == "pi" else {}), + **({"env": {BLOCK_ENV: aside["url_block_patterns"]}} + if aside and aside.get("url_block_patterns") else {})) except Exception as exc: # A provider error or a call that never submitted: worth the same # retry budget as a short grid, and just as visible in the log. @@ -1483,7 +1534,7 @@ def take_cells(forecasts=None, **_): "prompt": prompt if first else None, "system_prompt": (MINIMAL_SYSTEM if minimal else SYSTEM) if first else None, "protocol": row_protocol(policies, harness) - + ("" if variant in set_variants(policies) else f"+{variant}") + + ("" if variant in set_asked_variants(policies) else f"+{variant}") + ("" if minimal == (set_instrument(policies) == "minimal") else ("+minimal" if minimal else "+standard")), "condition_set": set_slug(policies), @@ -1513,6 +1564,7 @@ def take_cells(forecasts=None, **_): # Which model set this call belongs to -- for the ECI # panel, which snapshot chose it (redlines.registry). "panel": panel, + **extra, }) first = False return rows @@ -1737,7 +1789,8 @@ def main(): help="tag every row with this experiment name (needs --out)") ap.add_argument("--variant", action="append", choices=sorted(VARIANTS), help="with --joint: ask under these prompt variants (repeatable) instead of the " - "set's own (combined-v10: all five); others than the set's need --experiment") + "set's own (combined-v10: the five pooled ones and the unpooled " + "ai-banned); others than the set's need --experiment") ap.add_argument("--minimal", action=argparse.BooleanOptionalAction, default=None, help="with --joint: the minimal instrument (default: the set's; combined-v10 is " "minimal); a run against the set's choice needs --experiment") @@ -1748,6 +1801,20 @@ def main(): "none) unless given. Another loop runs only as a --joint --experiment, " "its rows tagged +") args = ap.parse_args() + # A URL guard in the PROCESS environment reaches every call (agent-box + # hands its environment to the tool processes; redlines.tools reads it + # in-process), so it is an experiment's, never the published run's: the + # published run's guard is per call, from the set (set_unpooled). A bad + # pattern fails now rather than on the first paid tool call. + if os.environ.get(BLOCK_ENV): + if not (args.joint and args.experiment) and not args.dry_run: + sys.exit(f"{BLOCK_ENV} is set: a URL-guarded run must be a --joint --experiment, " + "so its rows never join the series and say so. The published set's own " + "guard travels per call (unpooled_variants).") + try: + check_patterns(os.environ[BLOCK_ENV]) + except re.error as e: + sys.exit(f"{BLOCK_ENV}: bad pattern: {e}") if args.harness: if not args.joint: sys.exit("--harness is for --joint runs") @@ -1893,14 +1960,24 @@ def main_joint(args): published_set = os.path.abspath(args.conditions) in {os.path.abspath(p) for p in PUBLISHED_SETS} harness = args.harness or set_harness(policies) slug, proto = set_slug(policies), row_protocol(policies, harness) - variants = args.variant or set_variants(policies) + variants = args.variant or set_asked_variants(policies) + unpooled = set_unpooled(policies) minimal = set_instrument(policies) == "minimal" if args.minimal is None else args.minimal - if (set(variants) - set(set_variants(policies)) or minimal != (set_instrument(policies) == "minimal")) \ + if (set(variants) - set(set_asked_variants(policies)) or minimal != (set_instrument(policies) == "minimal")) \ and not args.experiment and not args.dry_run: sys.exit("refusing: variants or an instrument this set does not name are an experiment; " "tag the run with --experiment") if minimal and harness != "pi": sys.exit("the minimal instrument runs through pi") + guards = {v: unpooled[v]["url_block_patterns"] for v in variants + if unpooled.get(v, {}).get("url_block_patterns")} + if guards and harness != "pi": + sys.exit(f"variant(s) {sorted(guards)} carry a URL guard, which travels per call through pi only") + for v, g in guards.items(): + try: + check_patterns(g) + except re.error as e: + sys.exit(f"variant {v!r}: bad url_block_patterns: {e}") if args.out and os.path.abspath(args.out) == os.path.abspath(RUNLOG): sys.exit("refusing: never append to the shared tracked log; the series " "is one dated file per run under results/runs/ (see cron_run.sh)") @@ -1959,18 +2036,26 @@ def main_joint(args): for v in variants: p = minimal_joint_prompt(prompt, v) if minimal else apply_variant(prompt, v) print(f"##### JOINT [{slug}: {proto}{'' if v == DEFAULT_VARIANT else ' / ' + v}" - f"{' / minimal' if minimal else ''}]: {n} questions, {cells} cells x {len(conds) + 1} " + f"{' / minimal' if minimal else ''}{' / unpooled' if v in unpooled else ''}]: " + f"{n} questions, {cells} cells x {len(conds) + 1} " f"conditions = {cells * (len(conds) + 1)} probabilities, {len(p)} chars #####") print(p) + print(f"##### URL GUARD ({BLOCK_ENV}): " + f"{guards.get(v) or os.environ.get(BLOCK_ENV) or 'off'} #####") return load_keys() if not os.environ.get("TAVILY_API_KEY") and not args.allow_ungrounded: sys.exit("refusing to run: TAVILY_API_KEY is unset (see the unconditional path)") models, panel = select_models(args) - print(f"run {run_id} protocol {proto} [set {slug}]: {len(models) * len(arms)} calls " - f"({len(models)} models x {len(arms)} repeat(s)), {n} questions x " + print(f"run {run_id} protocol {proto} [set {slug}]: {len(models) * len(arms) * len(variants)} calls " + f"({len(models)} models x {len(arms)} repeat(s) x {len(variants)} variant(s)), {n} questions x " f"{len(horizons)} horizons = {cells} cells x {len(conds) + 1} conditions " f"= {cells * (len(conds) + 1)} probabilities each") + if len(variants) > 1 or variants != [DEFAULT_VARIANT]: + print(f" variants: {', '.join(variants)}" + + (f"; unpooled (not in the published number): " + + ", ".join(f"{v}{' + URL guard' if v in guards else ''}" for v in variants if v in unpooled) + if any(v in unpooled for v in variants) else "")) print(f" harness: {harness} loop, research floor {MIN_RESEARCH} calls, coverage " f"{', '.join(g['key'] for g in groups)}, guard {MAX_ROUNDS} rounds; reasoning " + ", ".join(f"{l}={reasoning_for(m) or 'default'}" for l, m in models)) @@ -1988,7 +2073,7 @@ def main_joint(args): cond_out = str(run_file(cond_out, run_id)) os.makedirs(os.path.dirname(os.path.abspath(cond_out)), exist_ok=True) lock = threading.Lock() - done = failed = 0 + done = failed = failed_unpooled = 0 spent = 0.0 with open(out, "a") as f, open(cond_out, "a") as fc, \ cf.ThreadPoolExecutor(max_workers=args.workers) as ex: @@ -2002,8 +2087,14 @@ def main_joint(args): try: rows = fut.result() except Exception as e: - failed += 1 - print(f" FAIL {tag}: {type(e).__name__}: {str(e)[:120]}") + # An unpooled call (the sixth) is not in the published number, + # so its failure does not fail the day (2026-10-19). + if v in unpooled: + failed_unpooled += 1 + else: + failed += 1 + print(f" FAIL {tag}: {type(e).__name__}: {str(e)[:120]}" + + (" (unpooled: the day publishes without it)" if v in unpooled else "")) continue with lock: for r in rows: @@ -2018,7 +2109,8 @@ def main_joint(args): print(f" done {tag} {got}/{len(rows)} question x condition rows answered, {usage_line(rows)}, " f"{ev} evidence, {coverage_line(rows)}, {rows[0]['attempts']} attempt(s), " f"{debrief_note(rows)}") - print(f"{done} ok, {failed} failed, ${spent:.2f} priced spend -> {out} (unconditional) + {cond_out} (instrument)") + print(f"{done} ok, {failed} failed, ${spent:.2f} priced spend -> {out} (unconditional) + {cond_out} (instrument)" + + (f"; {failed_unpooled} unpooled call(s) failed (not in the published number)" if failed_unpooled else "")) if failed: # An incomplete panel cannot be published (code/validate_launch_run.py # needs every model on the date), so the run is a failure, not a diff --git a/code/validate_launch_run.py b/code/validate_launch_run.py index 2f70775..aa839ce 100644 --- a/code/validate_launch_run.py +++ b/code/validate_launch_run.py @@ -47,12 +47,34 @@ def main(): # The debrief (2026-09-15) is asked after the forecasts are accepted # and cannot move them, so a missing or incomplete one is a warning # for the brief's author, never a reason to withhold the page. - 'debrief_warnings': []} + 'debrief_warnings': [], + # The unpooled sixth call (combined-v10 + "ai-banned", 2026-10-19): + # asked beside the five, never in the published number, so a + # missing, short or defective one (check_call) is a warning; + # the day is complete when the five are. Its rows must still + # say what they ran under (check). + 'variant_warnings': []} def check(ok, kind, detail): if not ok: report['errors'].append({'check': kind, 'detail': detail}) + def check_call(ok, kind, detail, row, slug): + """A check on what a CALL returned (its grid, its numbers, its + rationale, its elicit). On an unpooled call (the sixth, `pooled: + false`) a failure is a variant warning, never a reason to withhold + the five; on a pooled call it is an error as before. What a row + must SAY about its run (protocol, prompt, windows, dates, the + unpooled stamp) stays an error on every row: that is the code, not + the model.""" + if ok: + return + if row.get('pooled') is False: + report['variant_warnings'].append({'set': slug, 'variant': row.get('variant') or 'none', + 'label': row['label'], 'problem': kind, 'detail': detail}) + else: + report['errors'].append({'check': kind, 'detail': detail}) + for slug in sets: filename = files[slug] path = args.results / f'conditional_runs_{slug}.jsonl' @@ -72,30 +94,57 @@ def check(ok, kind, detail): prompt_sha = hashlib.sha256(prompt.encode()).hexdigest() # combined-v10: the minimal instrument under each of the set's prompt # variants -- one call per model per variant, each with its own prompt. + # The pooled variants make the grid; an unpooled one (the sixth) is + # optional per model: absent, or complete. variants = ru.set_variants(policies) + unpooled = ru.set_unpooled(policies) + asked = ru.set_asked_variants(policies) minimal = ru.set_instrument(policies) == 'minimal' - prompts = {v: (ru.minimal_joint_prompt(prompt, v) if minimal else ru.apply_variant(prompt, v)) for v in variants} + prompts = {v: (ru.minimal_joint_prompt(prompt, v) if minimal else ru.apply_variant(prompt, v)) for v in asked} shas = {v: hashlib.sha256(p.encode()).hexdigest() for v, p in prompts.items()} conditions = {None} | {c['id'] for c in conds} expected = {(label, v, q, c) for label in labels for v in variants for q in qs for c in conditions} keys = Counter((r['label'], r.get('variant') or 'none', r['question_id'], (r.get('condition') or {}).get('id')) for r in rows) - check(set(keys) == expected and all(v == 1 for v in keys.values()), 'complete_unique_grid', - {'set': slug, 'expected_rows': len(expected), 'actual_rows': len(rows), - 'missing': len(expected-set(keys)), 'extra': len(set(keys)-expected), - 'duplicates': sum(v-1 for v in keys.values())}) + pooled_keys = Counter({k: n for k, n in keys.items() if k[1] not in unpooled}) + check(set(pooled_keys) == expected and all(v == 1 for v in pooled_keys.values()), 'complete_unique_grid', + {'set': slug, 'expected_rows': len(expected), 'actual_rows': sum(pooled_keys.values()), + 'missing': len(expected-set(pooled_keys)), 'extra': len(set(pooled_keys)-expected), + 'duplicates': sum(v-1 for v in pooled_keys.values())}) + unpooled_calls = 0 + for v in unpooled: + for label in labels: + grid = {k: n for k, n in keys.items() if k[0] == label and k[1] == v} + want = {(label, v, q, c) for q in qs for c in conditions} + if not grid: + report['variant_warnings'].append({'set': slug, 'variant': v, 'label': label, + 'problem': 'no call'}) + elif set(grid) != want or any(n != 1 for n in grid.values()): + report['variant_warnings'].append({'set': slug, 'variant': v, 'label': label, + 'problem': 'incomplete grid', 'missing': len(want-set(grid)), + 'extra': len(set(grid)-want), 'duplicates': sum(n-1 for n in grid.values())}) + else: + unpooled_calls += 1 calls, probabilities = defaultdict(list), {} for r in rows: q = qs.get(r['question_id']) if q is None: continue - ident = [slug, r['label'], r.get('variant') or 'none', r['question_id'], (r.get('condition') or {}).get('id')] + v = r.get('variant') or 'none' + ident = [slug, r['label'], v, r['question_id'], (r.get('condition') or {}).get('id')] hs = {h for h in horizons if h in q['horizons']} fs = r.get('forecasts') or [] - check(len(fs) == len(hs) and {f['horizon'] for f in fs} == hs, - 'complete_horizons', ident) + check_call(len(fs) == len(hs) and {f['horizon'] for f in fs} == hs, + 'complete_horizons', ident, r, slug) check(r.get('protocol') == policies['protocol'], 'protocol', ident) - check(r.get('prompt_sha256') == shas.get(r.get('variant') or 'none'), 'prompt_hash', ident) + check(r.get('prompt_sha256') == shas.get(v), 'prompt_hash', ident) + # The sixth's rows say what they ran under; the five's are unchanged. + if v in unpooled: + check(r.get('pooled') is False + and r.get('url_block_patterns') == unpooled[v].get('url_block_patterns'), + 'unpooled_stamp', ident) + else: + check('pooled' not in r and 'url_block_patterns' not in r, 'pooled_row_unchanged', ident) check(r.get('counting_window') == counting_windows(q, horizons, args.date, spec), 'incident_window', ident) check(r.get('resolves_on') == {h: ru.resolves_on(h, args.date, spec) for h in horizons}, @@ -103,7 +152,7 @@ def check(ok, kind, detail): for f in fs: p = f.get('probability') valid = isinstance(p, (int, float)) and not isinstance(p, bool) and math.isfinite(p) and 0 <= p <= 1 - check(valid, 'bounded_probability', ident+[f.get('horizon'), p]) + check_call(valid, 'bounded_probability', ident+[f.get('horizon'), p], r, slug) if valid: probabilities[tuple(ident[1:])+ (f['horizon'],)] = p calls[r['call_id']].append(r) @@ -120,7 +169,7 @@ def check(ok, kind, detail): if not minimal: # the minimal instrument has no research floor check(len(evidence) >= ru.MIN_RESEARCH and all(r.get('grounded') for r in rs), 'grounding_floor', call_id) - check(bool(first.get('rationale')), 'rationale_present', call_id) + check_call(bool(first.get('rationale')), 'rationale_present', call_id, first, slug) status = first.get('debrief_status') or {} if not minimal and not (isinstance(first.get('debrief'), dict) and status.get('complete')): report['debrief_warnings'].append({ @@ -129,8 +178,11 @@ def check(ok, kind, detail): 'problems': status.get('problems') or (['no debrief recorded'] if not status else [])}) for elicit in ru.set_elicits(policies): value = (first.get('elicited') or {}).get(elicit['key']) - check(bool(ru.clean_elicit(value, elicit)), 'elicited_metadata', [call_id, elicit['key']]) - call_report.append({'call_id': call_id, 'label': first['label'], 'rows': len(rs), + check_call(bool(ru.clean_elicit(value, elicit)), 'elicited_metadata', + [call_id, elicit['key']], first, slug) + call_report.append({'call_id': call_id, 'label': first['label'], + 'variant': first.get('variant') or 'none', + 'pooled': bool(first.get('pooled', True)), 'rows': len(rs), 'evidence': len(evidence), 'usage': first.get('usage'), 'submission': first.get('delivery')}) rungs = [r['short'] for r in spec['rungs']] @@ -141,7 +193,7 @@ def ordered(left, right, kind): report['coherence_warnings'].append({'set': slug, 'check': kind, 'left': left, 'right': right, 'values': [a, b], 'excess_pp': (a-b)*100}) - for label, v in ((l, v) for l in labels for v in variants): + for label, v in ((l, v) for l in labels for v in asked): for c in conditions: for q in qs: hs = sorted([h for h in horizons if h in qs[q]['horizons']], @@ -158,9 +210,13 @@ def ordered(left, right, kind): for rung in rungs: ordered((label,v,f'ladder:{cause}:{rung}',c,h), (label,v,f'ladder:ai:{rung}',c,h), 'subtype_within_ai') + # 4 x 6 = 24 calls under combined-v10 + the sixth; the unpooled ones + # count where they answered, and are listed as missing where not. report['instruments'][slug] = {'file_sha256': hashlib.sha256(raw).hexdigest(), 'prompt_sha256': prompt_sha, 'rows': len(rows), 'probabilities': len(probabilities), - 'expected_probabilities': cells * len(conditions) * len(labels) * len(variants), 'calls': call_report} + 'expected_probabilities': cells * len(conditions) * (len(labels) * len(variants) + unpooled_calls), + 'expected_calls': len(labels) * len(asked), 'pooled_calls': len(labels) * len(variants), + 'unpooled_calls': unpooled_calls, 'calls': call_report} report['passed_integrity_checks'] = not report['errors'] report['recorded_cost_usd'] = round(sum((c.get('usage') or {}).get('cost_usd',0) for s in report['instruments'].values() for c in s['calls']),6) @@ -169,6 +225,7 @@ def ordered(left, right, kind): print(json.dumps({'integrity_errors': len(report['errors']), 'coherence_warnings': len(report['coherence_warnings']), 'debrief_warnings': len(report['debrief_warnings']), + 'variant_warnings': len(report['variant_warnings']), 'recorded_cost_usd': report['recorded_cost_usd'], 'report': str(args.out)})) return int(bool(report['errors'])) diff --git a/data/combined_conditions.json b/data/combined_conditions.json index fce63f6..7c08d61 100644 --- a/data/combined_conditions.json +++ b/data/combined_conditions.json @@ -12,6 +12,12 @@ "critical", "two-step" ], + "unpooled_variants": [ + { + "variant": "ai-banned", + "url_block_patterns": "forecastingresearch\\.org/pdf/airo,forecastingresearch\\.org/research/airo,airo\\.forecastingresearch\\.org" + } + ], "source": { "panel": "LEAP, for the policy conditions only; no human panel answers the capability conditions", "wave": "LEAP Wave 12 policies + frontier ECI 6 months from the run date (the model's own p10/p25/p50/p75/p90)", diff --git a/index.html b/index.html index 34450ec..4cfc309 100644 --- a/index.html +++ b/index.html @@ -528,11 +528,22 @@ const ci = (q.ci || {})[h] || []; // Prompt variants (combined-v10): one light line per variant, the panel // median of that prompt's calls; the ink line is the median of the models' - // means over the five. Hover a line for its prompt. - const VLABEL = { none: "Baseline", independent: "Independent", recent: "Recent", critical: "Critical", "two-step": "Two-step" }; - const vtext = Object.fromEntries((TL.variants || []).map(v => [v.id, v.text])); - const variants = Object.entries(q.variants || {}).map(([id, hs]) => ({ id, label: VLABEL[id] || id, vals: hs[h] || [] })); - const hasVar = i => variants.some(v => v.vals[i] != null); + // means over the five. Hover a line for its prompt. An UNPOOLED variant + // (the sixth call since 2026-10-19: the baseline prompt with the + // AI-forecast ban and a URL guard against AIRO's own pages) is dashed, its + // hover shows the sentence and the guard, and it is not in the mean. + const VLABEL = { none: "Baseline", independent: "Independent", recent: "Recent", critical: "Critical", "two-step": "Two-step", "ai-banned": "AI forecasts banned" }; + const vinfo = Object.fromEntries((TL.variants || []).map(v => [v.id, v])); + const variants = Object.entries(q.variants || {}).map(([id, hs]) => { + const info = vinfo[id] || {}; + return { id, label: VLABEL[id] || id, vals: hs[h] || [], text: info.text, guard: info.guard, + pooled: info.pooled !== false }; + }); + const pooledVariants = variants.filter(v => v.pooled); + const hasVar = i => pooledVariants.some(v => v.vals[i] != null); + const vtitle = v => v.label + (v.text ? ": " + v.text : ": no added instruction") + + (v.guard ? " Search guard (URL patterns blocked for this call): " + v.guard : "") + + (v.pooled ? "" : " Not in the published number."); let y, yticks, tickText, tickSub; if (loss) { @@ -646,12 +657,13 @@ rx="2" fill="var(--ink-faint)" opacity="0.22" /> ) : null)} - {/* One light line per prompt variant, under the median. */} + {/* One light line per prompt variant, under the median; an unpooled one dashed. */} {variants.map(v => ( - {v.label + (vtext[v.id] ? ": " + vtext[v.id] : ": no added instruction")} + {vtitle(v)} {segs(v.vals).map((pts, k) => pts.length > 1 ? ( - + ) : null)} {v.vals.map((val, i) => val == null ? null : ( @@ -702,7 +714,7 @@ boxShadow: "var(--shadow)", pointerEvents: "none", minWidth: 210, maxWidth: "calc(100% - 16px)" }}>
{snapshots[hv].date} · by {h} - {hasVar(hv) ? ` · ${variants.filter(v => v.vals[hv] != null).length} prompts per model · point = mean of the prompts` + {hasVar(hv) ? ` · ${pooledVariants.filter(v => v.vals[hv] != null).length} prompts per model · point = mean of the prompts` : snapshots[hv].draws > 1 ? ` · ${snapshots[hv].draws} draws per model · point = mean, range = 95% CI` : ""}{loss ? " · death-equivalents" : ""}
@@ -714,7 +726,7 @@
{variants.filter(v => v.vals[hv] != null).map(v => (
- — {v.label} prompt{fmtV(v.vals[hv])} + {v.pooled ? "— " : "- - "}{v.label}{v.pooled ? " prompt" : " · not in the mean"}{fmtV(v.vals[hv])}
))} {series.filter(s => s.vals[hv] != null).map(s => ( diff --git a/redlines/conditional.py b/redlines/conditional.py index d57412c..ecfc291 100644 --- a/redlines/conditional.py +++ b/redlines/conditional.py @@ -90,6 +90,11 @@ def _logit(p): # variant, the published number per model the mean of its five # (redlines.runlog.pool). The evidence: the prompt-variant experiments of # 2026-10-14..19, where the dropped guidance moved no headline forecast. + # The same evening a SIXTH call per model joined the run -- the baseline + # prompt with the AI-forecast ban and a URL guard against AIRO's own pages + # (code/make_combined_conditions.py, `unpooled_variants`) -- drawn on the + # Timeline and never pooled (redlines.runlog.pooled_rows). The five's rows + # and prompts did not change, so the tag did not move. "unified-joint-combined-v10": ("unified-joint-combined-v9", "unified-joint-combined-v8", "unified-joint-combined-v7", "unified-joint-combined-v6", "unified-joint-combined-v5", "unified-joint-combined-v4", @@ -192,12 +197,16 @@ def newest_protocol(rows, line): return [r for r in rows if r.get("protocol") == proto], proto -def load_conditional(path=CONDITIONAL_LOG, experiment=None, protocol=None): +def load_conditional(path=CONDITIONAL_LOG, experiment=None, protocol=None, pooled_only=True): """Rows of one experiment and/or one protocol. Two protocols write here -- separate calls (unified-batch-v2, one condition per call) and the joint instrument (unified-joint-v1, every condition in one call) -- and their unconditional arms are different elicitations, so a summary is always of - one protocol; pass protocol=None only to list what the log holds.""" + one protocol; pass protocol=None only to list what the log holds. + `pooled_only` drops the rows that are not in the published number (the + sixth call since 2026-10-19 evening, stamped `pooled: false`; + redlines.runlog.pooled_rows), so the tabs' per-model means stay the + mean of the five prompts; False reads everything.""" rows = [] for one in _paths(path): with open(one) as f: @@ -214,6 +223,8 @@ def load_conditional(path=CONDITIONAL_LOG, experiment=None, protocol=None): continue if is_superseded(r): # redlines.superseded: replaced the same day continue + if pooled_only and not r.get("pooled", True): + continue rows.append(r) return rows diff --git a/redlines/pi_loop.py b/redlines/pi_loop.py index 2dca3b6..3028519 100644 --- a/redlines/pi_loop.py +++ b/redlines/pi_loop.py @@ -47,11 +47,15 @@ def pi_model(model): def call_tools(model, prompt, tools, final_tool, *, max_iters=6, system=None, max_tokens=8000, temperature=0.0, complete=None, min_evidence=0, usage=None, partial_tool=None, coverage=None, debrief=None, - allowed_keys=None, run=None): + allowed_keys=None, env=None, run=None): """redlines.llm.call_tools's contract: -> (answer_args, evidence); fills `usage` and `debrief` in place, feeds `partial_tool["fn"]` every piece in order, raises if the model never submits. `allowed_keys` ("qid|horizon") are the cells the call asks for, which pi's tools check pieces against. + `env` is THIS call's environment for pi and its tool processes, over the + process's own (agent-box `harness.run(env=...)`): the URL guard + (redlines.tools.BLOCK_ENV) travels here, per call, never in os.environ, + because the runner's workers run calls side by side (2026-10-19). `max_tokens`, `temperature` and `complete` are the litellm path's and do nothing here; `run` replaces fri_forecaster.harness.run (tests).""" if run is None: @@ -83,7 +87,8 @@ def call_tools(model, prompt, tools, final_tool, *, max_iters=6, system=None, research_tools=tuple(n for n in ("web_search", "read_page") if n in names), cells_tool=bool(partial_tool), thinking=thinking, timeout=int(os.environ.get("REDLINES_PI_TIMEOUT", DEFAULT_TIMEOUT)), - env={"PI_CODING_AGENT_DIR": os.environ.get("PI_CODING_AGENT_DIR") or os.path.join(base, "agent")}, + env={"PI_CODING_AGENT_DIR": os.environ.get("PI_CODING_AGENT_DIR") or os.path.join(base, "agent"), + **(env or {})}, **loop) if usage is not None: u = res["usage"] diff --git a/redlines/runlog.py b/redlines/runlog.py index 495c742..c8c5143 100644 --- a/redlines/runlog.py +++ b/redlines/runlog.py @@ -242,7 +242,8 @@ def load_runlog(runlog=RUNLOG, runs_dir=RUNS_DIR, panel_only=True): """All forecast rows: the historical log plus one file per scheduled run. `panel_only` (the default) hands back panel rows only (panel_rows: what - the views draw); False hands back every row in the log -- the download + the views draw), minus the rows that are not in the published number + (pooled_rows); False hands back every row in the log -- the download bundle wants everything (project lead, 2026-09-08: include everything). Scheduled runs (code/cron_run.sh on the exe.dev box) write their own dated @@ -267,7 +268,24 @@ def load_runlog(runlog=RUNLOG, runs_dir=RUNS_DIR, panel_only=True): continue row["forecasts"] = _clean_forecasts(row.get("forecasts")) rows.append(row) - return panel_rows(rows) if panel_only else rows + return pooled_rows(panel_rows(rows)) if panel_only else rows + + +def pooled_rows(rows): + """Only the rows that enter a model's published number. + + Since 2026-10-19 evening the combined set asks a sixth call per model + beside its five prompts -- the baseline prompt with the AI-forecast ban + and agent-box's URL guard against AIRO's own pages + (data/combined_conditions.json `unpooled_variants`; code/run_unified.py + set_unpooled). Its rows are stamped `pooled: false`: the Timeline draws + them as their own line (redlines.views.timeline.variant_medians), and + nothing else reads them -- not pool(), not latest_pooled(), not the + Policy-levers summary (redlines.conditional.load_conditional). A row + without the stamp is pooled, so every row before that date is unchanged. + The download bundle (load_runlog(panel_only=False)) keeps everything. + """ + return [r for r in rows if r.get("pooled", True)] def panel_rows(rows): diff --git a/redlines/tools.py b/redlines/tools.py index 565cf68..65fc5a8 100644 --- a/redlines/tools.py +++ b/redlines/tools.py @@ -44,6 +44,7 @@ """ import json import os +import re import urllib.error import urllib.request from datetime import date, timedelta @@ -56,6 +57,29 @@ # its envelope; the model pages through a longer document with `offset`. PAGE_CHARS = 7000 +# URL GUARD (2026-10-16, AIRO's anchor-guard experiment). OFF unless the +# environment sets FORECASTER_BLOCK_URL_PATTERNS: regexes separated by newlines +# or commas (so no commas inside a pattern), searched case-insensitively +# against each URL. pi_tools.ts hands its +# environment to every tool process, so one variable reaches both tools. +# read_page refuses a matching URL without calling Tavily, with the ordinary +# failure shape and a neutral reason; web_search drops matching results, and +# results titled as the AIRO paper (BLOCK_TITLES), and blanks Tavily's +# `answer` when it dropped anything, because that summary was written from +# the unfiltered results. ForecastBench shares these tools, so nothing here +# changes when the variable is unset. +BLOCK_ENV = "FORECASTER_BLOCK_URL_PATTERNS" +BLOCK_TITLES = ("Automated Artificial Intelligence Risk Outlook",) + + +def blocked_patterns(): + raw = os.environ.get(BLOCK_ENV) or "" + return [re.compile(p.strip(), re.I) for p in re.split(r"[\n,]", raw) if p.strip()] + + +def is_blocked(url, patterns): + return bool(url) and any(p.search(url) for p in patterns) + def search_body(query, max_results=5, recent_days=None, today=None): """The Tavily request for a search, minus the key. With `recent_days` the @@ -106,9 +130,14 @@ def web_search(query, max_results=5, recent_days=None, cause=None): except (urllib.error.URLError, ValueError, TimeoutError) as e: return {"query": query, **tag, "results": [], "error": f"web search failed: {e}"} out = {"query": query, **tag, "answer": data.get("answer"), "results": []} + guard = blocked_patterns() if recent_days: out["window"] = {"start": spec["start_date"], "end": spec["end_date"]} for r in data.get("results", []): + if guard and (is_blocked(r.get("url"), guard) + or any(t in (r.get("title") or "") for t in BLOCK_TITLES)): + out["answer"] = None + continue item = {"title": r.get("title"), "url": r.get("url"), # Tavily already returns trimmed snippets; cap defensively "content": (r.get("content") or "")[:1500]} @@ -127,6 +156,8 @@ def read_page(url, offset=0): page cannot be fetched, returns a result carrying an `error` string and no text, so the loop keeps going. """ + if is_blocked(url, blocked_patterns()): + return {"url": url, "text": "", "error": "page read failed: page unavailable"} key = os.environ.get("TAVILY_API_KEY") if not key: return {"url": url, "text": "", diff --git a/redlines/views/timeline.py b/redlines/views/timeline.py index cfc5c93..541e1c7 100644 --- a/redlines/views/timeline.py +++ b/redlines/views/timeline.py @@ -60,7 +60,7 @@ from ..rail import grouping as rail_grouping, panel_questions as rail_questions from ..registry import model_colors, published_panel from ..runlog import (answered_horizons, grounding, load_runlog, pool, - rows_by_day, series_rows) + pooled_rows, rows_by_day, series_rows) from ..runlog import instrument_info, complete_panel_rows, current_panel, elicitation_date, SERIES_START from ..conditional import USD_PER_DEATH, apply_or, ladder_rungs, level_or from ..config import REPO_ROOT @@ -69,13 +69,17 @@ from .graph1 import _loss_question, _loss_series # code/run_unified.py VARIANTS, verbatim (the runner is not importable from -# the stdlib-only build); tests/test_timeline_variants.py keeps the two equal. +# the stdlib-only build); tests/test_variants_v10.py keeps the two equal. +# "ai-banned" (2026-10-19 evening) is the sixth, UNPOOLED call: its rows say +# `pooled: false` and carry the URL guard (`url_block_patterns`) the call ran +# under; the line is drawn, the number is not in the mean. VARIANT_TEXT = { "none": None, "independent": "Try to make your own forecasts. Do not take into account external surveys assessing extinction or catastrophic risk, or other probabilities given by prominent figures. The goal is to reach your own independent conclusion.", "recent": "If you search external information, focus on information that arrived within the past year, and weight more heavily sources that are more up to date about recent developments.", "critical": "Evaluate existing information critically. If you take into account forecasts by specific people or surveyed groups, consider their recent and historical forecasting accuracy, as well as the information available to the forecasters when they made their forecasts.", "two-step": "Proceed in two steps. First, make the best predictions you can about model capability progress over the relevant time frame, via human generated improvement or recursive self-improvement. Consider the distribution, not only the modal outcome. Then, given possible model capabilities, evaluate the risk posed and put these pieces together.", + "ai-banned": "Forecasts produced by AI systems, including any automated forecasting dashboard or model ensemble, carry no information beyond their inputs. Do not consult them or use them as evidence.", } import json @@ -144,9 +148,10 @@ def noise_floor(by_day, days): def variant_medians(rows, days): """{(date, question_id): {variant: {horizon: panel median, in %}}} for dates - asked under more than one prompt variant (combined-v10: five), else {}. - The pooled point per model is the mean of its variants (runlog.pool); - these are the per-variant panel medians the Timeline draws beside it.""" + asked under more than one prompt variant (combined-v10: five, and the + unpooled sixth), else {}. The pooled point per model is the mean of its + pooled variants (runlog.pool); these are the per-variant panel medians + the Timeline draws beside it, the unpooled ones included.""" by = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) for r in rows: v = r.get("variant") or "none" @@ -159,6 +164,18 @@ def variant_medians(rows, days): return out +def variant_info(rows): + """{variant: {"pooled": bool, "guard": patterns or None}} as the rows + say: an unpooled variant's rows carry `pooled: false` and the URL guard + its call ran under (the newest row's, should it ever change).""" + info = {} + for r in sorted(rows, key=lambda r: r["elicited_at"]): + v = r.get("variant") or "none" + info[v] = {"pooled": bool(r.get("pooled", True)), + "guard": r.get("url_block_patterns") or None} + return info + + def _prob_row(q, rail, by_day, days, horizons, var_meds=None): """One cross-cutting question: per model, lists aligned to `days` of the pooled point, its draws, their min-max, and the model's 95% CI (logit @@ -324,6 +341,11 @@ def build(): and r.get("question_id") in legacy_probability_ids and elicitation_date(r) >= SERIES_START] current_series_rows = [r for r in current_rows_all if elicitation_date(r) >= SERIES_START] + # The unpooled rows (the sixth call, `pooled: false`) are drawn as their + # own variant line and enter nothing else: not the pooled point, not the + # median, not the CI, not the day's completeness (runlog.pooled_rows). + aside_rows = [r for r in current_series_rows if not r.get("pooled", True)] + current_series_rows = pooled_rows(current_series_rows) rows = current_series_rows + legacy_rows batches = rows_by_day(rows) newest = max(batches, default=None) @@ -384,7 +406,12 @@ def build(): "panel": sorted({(r.get("panel") or {}).get("set") or "frontier5" for r in drows}), }) - var_meds = variant_medians(rows, days) + # The variant lines: the pooled rows' prompts and, on the same days, the + # unpooled sixth's. A day the five did not complete draws no line at all. + drawn = {elicitation_date(r) for r in rows} + aside_rows = [r for r in aside_rows if elicitation_date(r) in drawn] + var_meds = variant_medians(rows + aside_rows, days) + var_info = variant_info(rows + aside_rows) # Cross-cutting questions as themselves; each incident cause as ONE # expected-loss row (since 2026-08-27, matching Graph 1 and the # Policy-levers tab), emitted at its first rung and never again. @@ -450,7 +477,11 @@ def build(): "usdPerDeath": USD_PER_DEATH, "marks": _reference_marks(), # The prompt variants a date may carry (combined-v10), for the lines' - # labels and their hover text. - **({"variants": [{"id": v, "text": t} for v, t in VARIANT_TEXT.items()]} + # labels and their hover text: the instruction, whether the variant is + # in the published number, and the URL guard its calls ran under. + **({"variants": [{"id": v, "text": t, + "pooled": var_info.get(v, {}).get("pooled", True), + "guard": var_info.get(v, {}).get("guard")} + for v, t in VARIANT_TEXT.items()]} if var_meds else {}), } diff --git a/tests/test_pi_loop.py b/tests/test_pi_loop.py index c515e91..2712fb9 100644 --- a/tests/test_pi_loop.py +++ b/tests/test_pi_loop.py @@ -90,6 +90,20 @@ def test_no_answer_raises_for_the_retry(self): with self.assertRaises(RuntimeError): self.call(FakeRun({})) + def test_the_calls_env_reaches_pi_without_touching_the_process(self): + """The URL guard (redlines.tools.BLOCK_ENV) is per call: it goes to + agent-box's harness.run(env=...), beside PI_CODING_AGENT_DIR, and never + into os.environ, which the runner's workers share.""" + from redlines.tools import BLOCK_ENV + run = FakeRun({"forecasts": [], "rationale": "r"}) + self.call(run, env={BLOCK_ENV: "airo\\.example"}) + self.assertEqual(run.kw["env"][BLOCK_ENV], "airo\\.example") + self.assertIn("PI_CODING_AGENT_DIR", run.kw["env"]) + self.assertNotIn(BLOCK_ENV, os.environ) + run = FakeRun({"forecasts": [], "rationale": "r"}) + self.call(run) + self.assertNotIn(BLOCK_ENV, run.kw["env"]) # off unless the call says so + def test_validator_runs_out_of_process(self): try: from fri_forecaster.tool_cli import validate diff --git a/tests/test_sixth_call.py b/tests/test_sixth_call.py new file mode 100644 index 0000000..c984770 --- /dev/null +++ b/tests/test_sixth_call.py @@ -0,0 +1,252 @@ +"""A synthetic combined-v10 launch run WITH the sixth call: 4 panel models x +(5 pooled prompts + the unpooled "ai-banned"), through run_one_joint with a +fake loop, then the launch validator and the views over those rows. + +What this pins (the owner's decisions of 2026-10-19): + * the validator accepts 24 calls, and the day with a sixth call missing; + a missing pooled call still fails it; + * the Timeline draws six variant lines, the sixth flagged unpooled with + its guard, and every published number -- each model's point, the + ensemble median, Graph 1's latest reading, the Policy-levers baseline -- + is the mean of the FIVE prompts, the sixth never in it. +No model or search is called (the loop is a fake).""" +import importlib.util +import json +import statistics +import subprocess +import sys +import tempfile +import unittest +from datetime import date +from pathlib import Path +from unittest.mock import patch + +REPO = Path(__file__).resolve().parent.parent +if str(REPO) not in sys.path: + sys.path.insert(0, str(REPO)) + +from redlines import conditional, runlog # noqa: E402 +from redlines.registry import published_panel, record_snapshot # noqa: E402 +from redlines.views import timeline # noqa: E402 + + +def _runner(): + spec = importlib.util.spec_from_file_location("run_unified", REPO / "code" / "run_unified.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +ru = _runner() +TODAY = date(2026, 10, 19) +RUN_ID = "2026-10-19T1200Z" +# Each model's five prompts differ a little; the sixth sits well apart, so a +# mean that let it in would be visibly wrong. +OFFSET = {"none": 0.0, "independent": 0.001, "recent": 0.002, "critical": 0.003, "two-step": 0.004, + "ai-banned": 0.05} + + +def synthetic_rows(): + """Every row of a day's run: 4 panel models x the six variants, through + run_one_joint with a fake loop. -> rows""" + pol = ru.load_policies(ru.COMBINED) + g, b, h, _, spec = ru.load_batch(ru.LADDER, ru.CROSS, ru.UNBATCHED) + h = ru.set_horizons(pol, h) + cond_ids = [c["id"] for c in ru.resolve_set(pol, TODAY)["conditions"]] + labels = [m["label"] for m, _ in published_panel()] + panel = {"set": "eci_topk", "k": len(labels), "snapshot": record_snapshot()[0].isoformat()} + base = {label: 0.01 * (i + 1) for i, label in enumerate(labels)} + + def fake_for(label, variant): + p = base[label] + OFFSET[variant] + + def fake(model, prompt, tools, final_tool, **kw): + cells = [{"question_id": q["id"], "horizon": hz, "probabilities": {k: p for k in ["unconditional"] + cond_ids}} + for gg in g for q in b[gg["key"]] for hz in h if hz in q["horizons"]] + return {"forecasts": cells, "rationale": "synthetic", + "eci_forecast": {"p10": 168, "p25": 170, "p50": 172, "p75": 175, "p90": 178}}, [] + return fake + + rows = [] + for v in ru.set_asked_variants(pol): + for label in labels: + with patch.object(ru, "loop_for", lambda harness, f=fake_for(label, v): f): + rows += ru.run_one_joint(label, "x/" + label, g, b, h, RUN_ID, 0, spec, pol["conditions"], pol, + panel=panel, today=TODAY, harness="pi", variant=v, minimal=True) + return rows + + +def write_run(results, rows, drop=()): + """Write a day's run under `results` (results/runs/.jsonl and + results/conditional_runs_combined/.jsonl), minus the (label, variant) + calls in `drop`.""" + rows = [r for r in rows if (r["label"], r["variant"]) not in set(drop)] + runs, cond = results / "runs", results / "conditional_runs_combined" + runs.mkdir(parents=True), cond.mkdir(parents=True) + with open(runs / f"{RUN_ID}.jsonl", "w") as f, open(cond / f"{RUN_ID}.jsonl", "w") as fc: + for r in rows: + fc.write(json.dumps(r) + "\n") + if r["condition"] is None: + f.write(json.dumps(r) + "\n") + return rows + + +def validate(results): + out = results / "report.json" + r = subprocess.run([sys.executable, str(REPO / "code" / "validate_launch_run.py"), "--results", str(results), + "--date", TODAY.isoformat(), "--out", str(out)], capture_output=True, text=True, cwd=REPO) + return r.returncode, json.loads(out.read_text()), r.stdout + r.stderr + + +class TestSixthCallLaunch(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.results = Path(cls.tmp.name) / "full" + cls.rows = synthetic_rows() + write_run(cls.results, cls.rows) + cls.labels = [m["label"] for m, _ in published_panel()] + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_the_run_is_24_calls_with_the_sixth_stamped(self): + calls = {r["call_id"] for r in self.rows} + self.assertEqual(len(calls), 24) + sixth = [r for r in self.rows if r["variant"] == "ai-banned"] + five = [r for r in self.rows if r["variant"] != "ai-banned"] + self.assertEqual(len({r["call_id"] for r in sixth}), 4) + self.assertTrue(all(r["pooled"] is False and r["url_block_patterns"] for r in sixth)) + self.assertFalse(any("pooled" in r or "url_block_patterns" in r for r in five)) + self.assertEqual({r["protocol"] for r in self.rows}, {"unified-joint-combined-v10"}) + + def test_the_validator_passes_the_full_day(self): + rc, report, log = validate(self.results) + self.assertEqual(rc, 0, log) + self.assertEqual(report["errors"], []) + self.assertEqual(report["variant_warnings"], []) + inst = report["instruments"]["combined"] + self.assertEqual((inst["expected_calls"], inst["pooled_calls"], inst["unpooled_calls"]), (24, 20, 4)) + self.assertEqual(len(inst["calls"]), 24) + self.assertEqual(inst["probabilities"], inst["expected_probabilities"]) + self.assertEqual(sum(1 for c in inst["calls"] if not c["pooled"]), 4) + + def test_a_missing_sixth_call_is_a_warning_and_a_missing_pooled_call_an_error(self): + results = Path(self.tmp.name) / "no-sixth" + write_run(results, self.rows, drop=[(self.labels[0], "ai-banned")]) + rc, report, log = validate(results) + self.assertEqual(rc, 0, log) + self.assertEqual(report["errors"], []) + self.assertEqual(report["variant_warnings"], + [{"set": "combined", "variant": "ai-banned", "label": self.labels[0], "problem": "no call"}]) + self.assertEqual(report["instruments"]["combined"]["unpooled_calls"], 3) + self.assertEqual(report["instruments"]["combined"]["probabilities"], + report["instruments"]["combined"]["expected_probabilities"]) + results = Path(self.tmp.name) / "no-fifth" + write_run(results, self.rows, drop=[(self.labels[0], "two-step")]) + rc, report, _ = validate(results) + self.assertEqual(rc, 1) + self.assertEqual([e["check"] for e in report["errors"]][:1], ["complete_unique_grid"]) + + def test_a_defective_sixth_call_is_a_warning_and_a_defective_pooled_call_an_error(self): + """What a call returned -- no rationale, an ECI elicit that never + came back ordered, a probability out of bounds, a short grid -- is + a variant warning on the sixth and an error on a pooled call; the + day publishes without a defective sixth (the reviewer of 1f9d0c6 + reproduced the first two blocking it).""" + import copy + label = self.labels[0] + + def mutate(rows, variant, how): + rows = copy.deepcopy(rows) + for r in rows: + if r["label"] == label and r["variant"] == variant: + if how == "rationale": + r["rationale"] = None + elif how == "elicit": # run_one_joint writes rows after retries with the elicit unordered + r["elicited"]["eci_forecast"] = None + elif how == "bounds": + r["forecasts"] = [{**f, "probability": 1.5} for f in r["forecasts"]] + elif how == "short": + r["forecasts"] = r["forecasts"][:1] + return rows + + for how, kind in (("rationale", "rationale_present"), ("elicit", "elicited_metadata"), + ("bounds", "bounded_probability"), ("short", "complete_horizons")): + results = Path(self.tmp.name) / f"sixth-{how}" + write_run(results, mutate(self.rows, "ai-banned", how)) + rc, report, log = validate(results) + self.assertEqual(rc, 0, (how, log)) + self.assertEqual(report["errors"], [], how) + self.assertTrue(report["variant_warnings"], how) + self.assertEqual({w["problem"] for w in report["variant_warnings"]}, {kind}, how) + self.assertEqual({(w["variant"], w["label"]) for w in report["variant_warnings"]}, {("ai-banned", label)}, how) + self.assertEqual(report["instruments"]["combined"]["unpooled_calls"], 4, how) # every row is there + results = Path(self.tmp.name) / f"pooled-{how}" + write_run(results, mutate(self.rows, "critical", how)) + rc, report, _ = validate(results) + self.assertEqual(rc, 1, how) + self.assertIn(kind, {e["check"] for e in report["errors"]}, how) + self.assertEqual(report["variant_warnings"], [], how) + + def _load(self, **kw): + return runlog.load_runlog(runlog=self.results / "none.jsonl", runs_dir=self.results / "runs", **kw) + + def test_the_published_number_is_the_mean_of_the_five(self): + rows = self._load() # the views' default read + self.assertEqual({r["variant"] for r in rows}, set(ru.set_variants(ru.load_policies(ru.COMBINED)))) + latest = runlog.latest_pooled(runlog.current_rows(runlog.complete_panel_rows(runlog.latest_instrument_rows(rows)))) + for label in self.labels: + pooled = latest[("catastrophe:ai", label)] + self.assertEqual(pooled["replicates"], 5) + want = statistics.mean(0.01 * (self.labels.index(label) + 1) + OFFSET[v] for v in ru.set_variants(ru.load_policies(ru.COMBINED))) + got = next(f["probability"] for f in pooled["forecasts"] if f["horizon"] == "2030") + self.assertAlmostEqual(got, want, places=12) + self.assertEqual({r["variant"] for r in self._load(panel_only=False)} - set(ru.VARIANTS), set()) + self.assertIn("ai-banned", {r["variant"] for r in self._load(panel_only=False)}) # the download keeps it + + def test_the_timeline_draws_six_lines_over_a_five_prompt_mean(self): + with patch.object(timeline, "load_runlog", self._load): + blob = timeline.build() + snap = blob["snapshots"][-1] + self.assertEqual((snap["date"], snap["draws"], snap["calls"]), (TODAY.isoformat(), 5, 20)) + info = {v["id"]: v for v in blob["variants"]} + self.assertEqual(set(info), set(ru.VARIANTS)) + self.assertFalse(info["ai-banned"]["pooled"]) + self.assertEqual(info["ai-banned"]["guard"], ru.set_unpooled(ru.load_policies(ru.COMBINED))["ai-banned"]["url_block_patterns"]) + self.assertEqual(info["ai-banned"]["text"], ru.VARIANTS["ai-banned"]) + self.assertTrue(all(v["pooled"] and v["guard"] is None for k, v in info.items() if k != "ai-banned")) + q = next(q for q in blob["questions"] if q["id"] == "catastrophe:ai") + i = len(blob["snapshots"]) - 1 + self.assertEqual(set(q["variants"]), set(ru.VARIANTS)) # six lines + means = {} + for s in q["series"]: + if s["ps"]["2030"][i] is None: + continue + want = 100 * statistics.mean(0.01 * (self.labels.index(s["label"]) + 1) + OFFSET[v] + for v in ru.set_variants(ru.load_policies(ru.COMBINED))) + self.assertAlmostEqual(s["ps"]["2030"][i], want, places=3) + self.assertEqual(len(s["draws"]["2030"][i]), 5) + self.assertIsNotNone(s["ci"]["2030"][i]) # five draws: an interval + means[s["label"]] = s["ps"]["2030"][i] + self.assertEqual(set(means), set(self.labels)) + self.assertAlmostEqual(q["median"]["2030"][i], statistics.median(means.values()), places=3) + self.assertIsNone(q["rerun"]["2030"][i]) # the draws are prompts, not re-asks + # The sixth's line is the panel median of ITS calls, 5pp above the baseline's. + self.assertAlmostEqual(q["variants"]["ai-banned"]["2030"][i] - q["variants"]["none"]["2030"][i], 5.0, places=3) + + def test_the_policy_levers_baseline_is_five_calls(self): + rows = conditional.load_conditional(self.results / "conditional_runs_combined.jsonl") + self.assertNotIn("ai-banned", {r["variant"] for r in rows}) + pol = conditional.group_view(ru.load_policies(ru.COMBINED), "policy") + s = conditional.summarize(conditional.group_rows(rows, "policy"), pol, horizons=("2030",)) + cell = s["questions"]["catastrophe:ai"]["2030"] + for label in self.labels: + self.assertEqual(cell["models"][label]["baseline_n"], 5) + all_rows = conditional.load_conditional(self.results / "conditional_runs_combined.jsonl", pooled_only=False) + self.assertEqual(len({r["call_id"] for r in all_rows}), 24) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tools_guard.py b/tests/test_tools_guard.py new file mode 100644 index 0000000..a4faafd --- /dev/null +++ b/tests/test_tools_guard.py @@ -0,0 +1,77 @@ +"""redlines.tools URL guard (FORECASTER_BLOCK_URL_PATTERNS), with the network +mocked: off unless set; read_page refuses without calling Tavily; web_search +drops matching results and the answer built from them. A port of agent-box's +tests/test_url_guard.py (branch `url-guard`) against the mirror here, which +tests/test_tools_parity.py keeps identical to agent-box's copy.""" +import io +import json +import os +import unittest +from unittest.mock import patch + +from redlines import tools + +PATTERNS = ("forecastingresearch\\.org/pdf/airo\n" + "forecastingresearch\\.org/research/airo,airo\\.forecastingresearch\\.org") +PAPER = "https://forecastingresearch.org/pdf/airo-working-paper.pdf" +SEARCH = {"answer": "AIRO says 0.47%", "results": [ + {"title": "AIRO working paper", "url": PAPER, "content": "x"}, + {"title": "Automated Artificial Intelligence Risk Outlook (mirror)", + "url": "https://example.com/mirror", "content": "x"}, + {"title": "XPT", "url": "https://forecastingresearch.org/xpt", "content": "x"}, + {"title": "Launch post", "url": "https://www.lesswrong.com/posts/abc/airo", "content": "x"}, +]} + + +def reply(data): + return io.BytesIO(json.dumps(data).encode()) + + +class UrlGuardTest(unittest.TestCase): + def env(self, patterns=None): + env = {"TAVILY_API_KEY": "k"} + if patterns: + env[tools.BLOCK_ENV] = patterns + return patch.dict(os.environ, env, clear=True) + + def test_read_page_blocked_without_calling_tavily(self): + with self.env(PATTERNS), patch("urllib.request.urlopen") as net: + for url in (PAPER, "https://forecastingresearch.org/research/airo", + "https://airo.forecastingresearch.org/", + "https://Airo.ForecastingResearch.org/", PAPER.replace("airo", "AIRO")): + self.assertEqual(tools.read_page(url), { + "url": url, "text": "", "error": "page read failed: page unavailable"}) + net.assert_not_called() + + def test_read_page_others_still_fetched(self): + page = {"results": [{"raw_content": "XPT text"}]} + for env in (self.env(PATTERNS), self.env()): + with env, patch("urllib.request.urlopen", return_value=reply(page)) as net: + self.assertEqual(tools.read_page("https://forecastingresearch.org/xpt")["text"], "XPT text") + net.assert_called_once() + with self.env(), patch("urllib.request.urlopen", return_value=reply(page)): + self.assertEqual(tools.read_page(PAPER)["text"], "XPT text") # off when unset + + def test_web_search_drops_matches_and_answer(self): + with self.env(PATTERNS), patch("urllib.request.urlopen", return_value=reply(SEARCH)): + out = tools.web_search("ai risk") + self.assertEqual([r["url"] for r in out["results"]], + ["https://forecastingresearch.org/xpt", "https://www.lesswrong.com/posts/abc/airo"]) + self.assertIsNone(out["answer"]) + + def test_web_search_keeps_answer_when_nothing_dropped(self): + clean = {"answer": "ok", "results": SEARCH["results"][2:]} + with self.env(PATTERNS), patch("urllib.request.urlopen", return_value=reply(clean)): + out = tools.web_search("ai risk") + self.assertEqual(out["answer"], "ok") + self.assertEqual(len(out["results"]), 2) + + def test_web_search_unchanged_when_unset(self): + with self.env(), patch("urllib.request.urlopen", return_value=reply(SEARCH)): + out = tools.web_search("ai risk") + self.assertEqual(out["answer"], "AIRO says 0.47%") + self.assertEqual(len(out["results"]), 4) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tools_parity.py b/tests/test_tools_parity.py index dbfe4da..1ed92e6 100644 --- a/tests/test_tools_parity.py +++ b/tests/test_tools_parity.py @@ -20,7 +20,8 @@ @unittest.skipIf(ff_tools is None, "fri_forecaster not installed") class ToolsParity(unittest.TestCase): def test_same_functions(self): - for name in ("search_body", "web_search", "read_page"): + # The URL guard (2026-10-16, agent-box's `url-guard`) is part of the stack. + for name in ("search_body", "web_search", "read_page", "blocked_patterns", "is_blocked"): self.assertEqual(inspect.getsource(getattr(airo_tools, name)), inspect.getsource(getattr(ff_tools, name)), name) diff --git a/tests/test_variants_v10.py b/tests/test_variants_v10.py index 9826c84..cca520d 100644 --- a/tests/test_variants_v10.py +++ b/tests/test_variants_v10.py @@ -1,13 +1,18 @@ -"""combined-v10: the minimal instrument under five prompt variants. +"""combined-v10: the minimal instrument under five prompt variants, and the +unpooled sixth. The set declares both (data/combined_conditions.json "instrument", "variants"); run_unified asks one call per model per variant; the Timeline draws one line -per variant beside the pooled median.""" +per variant beside the pooled median. Since 2026-10-19 evening the set also +declares one UNPOOLED variant ("unpooled_variants": ai-banned, the baseline +prompt with the AI-forecast ban and a per-call URL guard): asked beside the +five, drawn on the Timeline, never in the published number.""" import importlib.util import sys import unittest from datetime import date from pathlib import Path +from unittest.mock import patch REPO = Path(__file__).resolve().parent.parent if str(REPO) not in sys.path: @@ -30,11 +35,39 @@ def test_v10_declares_the_minimal_instrument_and_five_variants(self): self.assertEqual(ru.set_protocol(cb), "unified-joint-combined-v10") self.assertEqual(ru.set_instrument(cb), "minimal") self.assertEqual(ru.set_variants(cb), ["none", "independent", "recent", "critical", "two-step"]) - self.assertEqual(set(ru.set_variants(cb)), set(ru.VARIANTS)) + self.assertEqual(set(ru.set_asked_variants(cb)), set(ru.VARIANTS)) + + def test_the_sixth_is_declared_unpooled_with_its_guard(self): + cb = ru.load_policies(ru.COMBINED) + self.assertEqual(list(ru.set_unpooled(cb)), ["ai-banned"]) + guard = ru.set_unpooled(cb)["ai-banned"]["url_block_patterns"] + # The `both` arm of the anchor-guard experiment (2026-10-16), verbatim. + self.assertEqual(guard, r"forecastingresearch\.org/pdf/airo,forecastingresearch\.org/research/airo," + r"airo\.forecastingresearch\.org") + self.assertEqual(ru.VARIANTS["ai-banned"], + "Forecasts produced by AI systems, including any automated forecasting dashboard " + "or model ensemble, carry no information beyond their inputs. Do not consult them " + "or use them as evidence.") + ru.check_patterns(guard) # compiles + with self.assertRaises(ru.re.error): + ru.check_patterns("ok,(unclosed") + self.assertEqual(ru.set_asked_variants(cb), ru.set_variants(cb) + ["ai-banned"]) + self.assertNotIn("ai-banned", ru.set_variants(cb)) # not a pooled variant + # The guard blocks AIRO's pages and nothing else, case-insensitively. + from redlines.tools import blocked_patterns, is_blocked + with patch.dict("os.environ", {ru.BLOCK_ENV: guard}): + pats = blocked_patterns() + for url in ("https://forecastingresearch.org/pdf/airo-working-paper.pdf", + "https://forecastingresearch.org/research/airo", "https://AIRO.forecastingresearch.org/"): + self.assertTrue(is_blocked(url, pats), url) + for url in ("https://forecastingresearch.org/xpt", "https://www.lesswrong.com/posts/x/airo"): + self.assertFalse(is_blocked(url, pats), url) def test_a_set_without_them_is_one_standard_call(self): leap = ru.load_policies(ru.POLICIES) self.assertEqual(ru.set_variants(leap), ["none"]) + self.assertEqual(ru.set_unpooled(leap), {}) + self.assertEqual(ru.set_asked_variants(leap), ["none"]) self.assertEqual(ru.set_instrument(leap), "standard") @@ -80,7 +113,7 @@ def test_minimal_tools_drop_cause_and_advice(self): class TestRunOneJoint(unittest.TestCase): """One call per variant through a fake loop: what the loop is given, and the rows.""" - def _call(self, variant, answer_cells=True): + def _call(self, variant, answer_cells=True, harness="pi"): today = date(2026, 10, 19) pol = ru.load_policies(ru.COMBINED) g, b, h, _, spec = ru.load_batch(ru.LADDER, ru.CROSS, ru.UNBATCHED) @@ -101,7 +134,7 @@ def fake(model, prompt, tools, final_tool, **kw): ru.loop_for = lambda harness: fake try: rows = ru.run_one_joint("M", "anthropic/x", g, b, h, "2026-10-19T0000Z", 0, spec, - pol["conditions"], pol, today=today, harness="pi", + pol["conditions"], pol, today=today, harness=harness, variant=variant, minimal=True) finally: ru.loop_for = orig @@ -120,6 +153,39 @@ def test_rows_and_loop_settings(self): self.assertEqual(rows[0]["protocol"], "unified-joint-combined-v10") # a declared variant: the set's tag self.assertEqual(rows[0]["system_prompt"], ru.MINIMAL_SYSTEM) self.assertTrue(all(r["forecasts"] for r in rows)) + # A pooled call: no guard reaches the loop, and the rows are as before. + self.assertNotIn("env", seen) + self.assertFalse(any("url_block_patterns" in r or "pooled" in r for r in rows)) + + def test_the_sixth_runs_guarded_and_its_rows_say_so(self): + cb = ru.load_policies(ru.COMBINED) + guard = ru.set_unpooled(cb)["ai-banned"]["url_block_patterns"] + rows, seen = self._call("ai-banned") + # The sentence where a variant's text goes; the guard to THIS call only. + self.assertIn(ru.VARIANTS["ai-banned"], seen["prompt"]) + self.assertLess(seen["prompt"].index(ru.VARIANTS["ai-banned"]), seen["prompt"].index("Deliver the probabilities")) + self.assertEqual(seen["env"], {ru.BLOCK_ENV: guard}) + self.assertNotIn(ru.BLOCK_ENV, ru.os.environ) + self.assertEqual(seen["system"], ru.MINIMAL_SYSTEM) + # The rows: the variant, the guard, unpooled; the protocol stays v10. + self.assertEqual({r["variant"] for r in rows}, {"ai-banned"}) + self.assertEqual({r["url_block_patterns"] for r in rows}, {guard}) + self.assertEqual({r["pooled"] for r in rows}, {False}) + self.assertEqual({r["protocol"] for r in rows}, {"unified-joint-combined-v10"}) + self.assertTrue(rows[0]["call_id"].endswith(":ai-banned")) + self.assertTrue(all(r["forecasts"] for r in rows)) + + def test_a_guarded_variant_needs_pi(self): + with self.assertRaisesRegex(RuntimeError, "URL guard"): + self._call("ai-banned", harness="litellm") + + def test_a_process_environment_guard_is_stamped_as_an_experiment_would_be(self): + # #5's stamping: a guard in os.environ reaches every call and every row says so. + with patch.dict("os.environ", {ru.BLOCK_ENV: "example\\.org"}): + rows, seen = self._call("none") + self.assertNotIn("env", seen) + self.assertEqual({r["url_block_patterns"] for r in rows}, {"example\\.org"}) + self.assertFalse(any("pooled" in r for r in rows)) def test_baseline_call_id_is_unchanged(self): rows, _ = self._call("none") @@ -146,6 +212,40 @@ def test_variant_medians(self): self.assertEqual(out[("2026-10-19", "catastrophe:ai")]["independent"]["2030"], 3.0) self.assertEqual(variant_medians(rows[:2], ["2026-10-19"]), {}) # one prompt: no lines + def test_variant_info_reads_the_stamp_off_the_rows(self): + from redlines.views.timeline import variant_info + rows = [{"elicited_at": "2026-10-19T12:00:00+00:00", "variant": "none"}, + {"elicited_at": "2026-10-19T12:00:00+00:00", "variant": "ai-banned", + "pooled": False, "url_block_patterns": "x\\.org"}] + self.assertEqual(variant_info(rows), {"none": {"pooled": True, "guard": None}, + "ai-banned": {"pooled": False, "guard": "x\\.org"}}) + + +class TestPooledRows(unittest.TestCase): + """The sixth's rows (`pooled: false`) enter no published number.""" + + def test_load_runlog_and_load_conditional_drop_them_unless_asked(self): + import json + import tempfile + from redlines import conditional, runlog + mk = lambda v, **kw: {"elicited_at": "2026-10-19T12:00:00+00:00", "run_date": "2026-10-19", + "question_id": "q", "label": "A", "variant": v, "forecasts": [], **kw} + rows = [mk("none"), mk("ai-banned", pooled=False, url_block_patterns="x")] + self.assertEqual([r["variant"] for r in runlog.pooled_rows(rows)], ["none"]) + with tempfile.TemporaryDirectory() as d: + runs = Path(d) / "runs" + runs.mkdir() + (runs / "2026-10-19T1200Z.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows)) + got = runlog.load_runlog(runlog=Path(d) / "none.jsonl", runs_dir=runs) + self.assertEqual([r["variant"] for r in got], ["none"]) + got = runlog.load_runlog(runlog=Path(d) / "none.jsonl", runs_dir=runs, panel_only=False) + self.assertEqual([r["variant"] for r in got], ["none", "ai-banned"]) + log = Path(d) / "cond.jsonl" + log.write_text("".join(json.dumps(r) + "\n" for r in rows)) + self.assertEqual([r["variant"] for r in conditional.load_conditional(log)], ["none"]) + self.assertEqual([r["variant"] for r in conditional.load_conditional(log, pooled_only=False)], + ["none", "ai-banned"]) + if __name__ == "__main__": unittest.main() diff --git a/web/demo/19-live-timeline.jsx b/web/demo/19-live-timeline.jsx index eeadeb4..71aa56c 100644 --- a/web/demo/19-live-timeline.jsx +++ b/web/demo/19-live-timeline.jsx @@ -87,11 +87,22 @@ function TimelineChart({ q, snapshots, horizon, scale }) { const ci = (q.ci || {})[h] || []; // Prompt variants (combined-v10): one light line per variant, the panel // median of that prompt's calls; the ink line is the median of the models' - // means over the five. Hover a line for its prompt. - const VLABEL = { none: "Baseline", independent: "Independent", recent: "Recent", critical: "Critical", "two-step": "Two-step" }; - const vtext = Object.fromEntries((TL.variants || []).map(v => [v.id, v.text])); - const variants = Object.entries(q.variants || {}).map(([id, hs]) => ({ id, label: VLABEL[id] || id, vals: hs[h] || [] })); - const hasVar = i => variants.some(v => v.vals[i] != null); + // means over the five. Hover a line for its prompt. An UNPOOLED variant + // (the sixth call since 2026-10-19: the baseline prompt with the + // AI-forecast ban and a URL guard against AIRO's own pages) is dashed, its + // hover shows the sentence and the guard, and it is not in the mean. + const VLABEL = { none: "Baseline", independent: "Independent", recent: "Recent", critical: "Critical", "two-step": "Two-step", "ai-banned": "AI forecasts banned" }; + const vinfo = Object.fromEntries((TL.variants || []).map(v => [v.id, v])); + const variants = Object.entries(q.variants || {}).map(([id, hs]) => { + const info = vinfo[id] || {}; + return { id, label: VLABEL[id] || id, vals: hs[h] || [], text: info.text, guard: info.guard, + pooled: info.pooled !== false }; + }); + const pooledVariants = variants.filter(v => v.pooled); + const hasVar = i => pooledVariants.some(v => v.vals[i] != null); + const vtitle = v => v.label + (v.text ? ": " + v.text : ": no added instruction") + + (v.guard ? " Search guard (URL patterns blocked for this call): " + v.guard : "") + + (v.pooled ? "" : " Not in the published number."); let y, yticks, tickText, tickSub; if (loss) { @@ -205,12 +216,13 @@ function TimelineChart({ q, snapshots, horizon, scale }) { rx="2" fill="var(--ink-faint)" opacity="0.22" /> ) : null)} - {/* One light line per prompt variant, under the median. */} + {/* One light line per prompt variant, under the median; an unpooled one dashed. */} {variants.map(v => ( - {v.label + (vtext[v.id] ? ": " + vtext[v.id] : ": no added instruction")} + {vtitle(v)} {segs(v.vals).map((pts, k) => pts.length > 1 ? ( - + ) : null)} {v.vals.map((val, i) => val == null ? null : ( @@ -261,7 +273,7 @@ function TimelineChart({ q, snapshots, horizon, scale }) { boxShadow: "var(--shadow)", pointerEvents: "none", minWidth: 210, maxWidth: "calc(100% - 16px)" }}>
{snapshots[hv].date} · by {h} - {hasVar(hv) ? ` · ${variants.filter(v => v.vals[hv] != null).length} prompts per model · point = mean of the prompts` + {hasVar(hv) ? ` · ${pooledVariants.filter(v => v.vals[hv] != null).length} prompts per model · point = mean of the prompts` : snapshots[hv].draws > 1 ? ` · ${snapshots[hv].draws} draws per model · point = mean, range = 95% CI` : ""}{loss ? " · death-equivalents" : ""}
@@ -273,7 +285,7 @@ function TimelineChart({ q, snapshots, horizon, scale }) {
{variants.filter(v => v.vals[hv] != null).map(v => (
- — {v.label} prompt{fmtV(v.vals[hv])} + {v.pooled ? "— " : "- - "}{v.label}{v.pooled ? " prompt" : " · not in the mean"}{fmtV(v.vals[hv])}
))} {series.filter(s => s.vals[hv] != null).map(s => (