From 6562eceaafe54581f323e186094cb4dda34e86da Mon Sep 17 00:00:00 2001 From: AIRO Coding Agent Date: Mon, 26 Oct 2026 22:07:56 +0000 Subject: [PATCH] Exclude shadow conditions from all timeline publication paths --- index.html | 41 +++++++++++++++++++++++++++++++---- redlines/views/timeline.py | 22 ++++++++++++++----- tests/test_experiments.py | 24 +++++++++++++++++++- tests/test_tooltips.cjs | 25 +++++++++++++++++++++ web/demo/19-live-timeline.jsx | 41 +++++++++++++++++++++++++++++++---- 5 files changed, 139 insertions(+), 14 deletions(-) diff --git a/index.html b/index.html index 9e3e9d4..93d807d 100644 --- a/index.html +++ b/index.html @@ -518,7 +518,8 @@ const tlFmtFor = q => (q && q.value_kind === "loss" ? tlFmtBig : tlFmt); // Colors belong to prompt conditions, never to models or their array order. -// New condition IDs stay neutral until explicitly assigned a color here. +// This is an explicit publication allowlist, not a fallback color registry. +// Shadow/unknown conditions in retained snapshots must never reach the UI. const TL_CONDITION_START = "2026-10-21"; const TL_CONDITIONS = { none: { label: "Baseline", color: "#0969da" }, @@ -529,14 +530,46 @@ }; const tlVariants = (q, horizon, snapshots) => { const info = Object.fromEntries(((TL && TL.variants) || []).map(v => [v.id, v])); - return Object.entries(q.variants || {}).map(([id, hs]) => ({ - id, ...(TL_CONDITIONS[id] || { label: id, color: "var(--ink-soft)" }), + return Object.entries(q.variants || {}).filter(([id]) => Object.hasOwn(TL_CONDITIONS, id)).map(([id, hs]) => ({ + id, ...TL_CONDITIONS[id], vals: (hs[horizon] || []).map((v, i) => snapshots[i].date >= TL_CONDITION_START ? v : null), models: ((q.variantModels || {})[id] || {})[horizon] || [], text: (info[id] || {}).text, })).filter(v => v.vals.some(val => val != null)); }; +// Old published snapshots may include a shadow in both condition dots and +// pooled model means. Repair the displayed aggregates from public condition +// answers on affected dates only; leave the immutable snapshot/downloads alone. +function tlPublicQuestion(q) { + const hidden = Object.keys(q.variants || {}).filter(id => !Object.hasOwn(TL_CONDITIONS, id)); + if (!hidden.length) return q; + const clean = JSON.parse(JSON.stringify(q)); + for (const h of q.horizons) { + for (let i = 0; i < (q.median[h] || []).length; i++) { + if (!hidden.some(id => q.variants[id][h]?.[i] != null)) continue; + for (const model of clean.series) { + const values = Object.keys(TL_CONDITIONS).map(id => + q.variantModels?.[id]?.[h]?.[i]?.[model.label]).filter(v => v != null); + const mean = values.length ? Math.round(values.reduce((a, b) => a + b, 0) / values.length * 1000) / 1000 : null; + model.ps[h][i] = mean; + model.draws[h][i] = values.length ? values : null; + model.lo[h][i] = values.length ? Math.min(...values) : null; + model.hi[h][i] = values.length ? Math.max(...values) : null; + } + const values = clean.series.map(s => s.ps[h][i]).filter(v => v != null).sort((a, b) => a - b); + const mid = Math.floor(values.length / 2); + clean.median[h][i] = values.length ? Math.round((values.length % 2 ? values[mid] : (values[mid - 1] + values[mid]) / 2) * 1000) / 1000 : null; + clean.rerun[h][i] = null; + } + } + for (const id of hidden) { + delete clean.variants[id]; + delete clean.variantModels?.[id]; + } + return clean; +} + /* ---------------- time-series chart ---------------- */ // Before Oct 21 the Baseline median is blue until hovered or selected; then // model answers appear as ink-gray circles. Since Oct 21 each prompt condition @@ -871,7 +904,7 @@ } function TimelinePanelCurrent({ qid = TL_HEADLINE, title = "Probability of an AI catastrophe", sub = "Forecasts over time" } = {}) { const snapshots = TL.snapshots; - const q = TL.questions.find(x => x.id === qid) || TL.questions[0]; + const q = useMemo(() => tlPublicQuestion(TL.questions.find(x => x.id === qid) || TL.questions[0]), [qid]); // The horizon picker, Graph 2's toggle: opens on 2030 like every other // horizon toggle (Nick, 2026-08-27), and is what the line, the dots, the // numbers below and the table all read. diff --git a/redlines/views/timeline.py b/redlines/views/timeline.py index 93cf9be..b1867ae 100644 --- a/redlines/views/timeline.py +++ b/redlines/views/timeline.py @@ -134,6 +134,18 @@ def noise_floor(by_day, days): return None +def public_timeline_rows(rows): + """Fail closed: only the published experiment's pooled conditions may plot. + + Apply before the current/legacy split: old instruments can contain shadow + calls too. Keep raw rows in downloads, never in displayed aggregates. + """ + allowed = set(load_experiment(PUBLISHED)["pooled"]) + return [r for r in rows + if r.get("pooled", True) and r.get("publish", True) + and (r.get("condition") or r.get("variant") or "none") in allowed] + + def condition_cells(rows): """{(date, question_id): {condition: {horizon: {model: answer, in %}}}} for dates asked under more than one condition (redlines.experiments; @@ -141,8 +153,8 @@ def condition_cells(rows): than once that day gets the mean. An old row's condition is its prompt variant (redlines.logs.read_row).""" by = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(list)))) - for r in rows: - v = r.get("condition") or "none" + for r in public_timeline_rows(rows): + v = r.get("condition") or r.get("variant") or "none" for f in r["forecasts"]: by[(elicitation_date(r), r["question_id"])][v][f["horizon"]][r["label"]].append(100 * f["probability"]) return {key: {v: {h: {m: round(statistics.mean(ps), 3) for m, ps in ms.items()} for h, ms in hs.items()} @@ -158,11 +170,11 @@ def condition_variants(rows): def condition_legend(variants): """The variant dots' labels and hover text: the published experiment's - pooled conditions in its order, then any other the rows carry. `text` is + pooled conditions in its order, never arbitrary IDs from the rows. `text` is the instruction its variant adds. An unpooled condition (the sixth call) is never shown.""" config = load_experiment(PUBLISHED)["conditions"] - ids = [c for c, cfg in config.items() if cfg["pooled"]] + [c for c in variants if c not in config] + ids = [c for c, cfg in config.items() if cfg["pooled"] and cfg["publish"]] return [{"id": c, "text": VARIANTS.get(variants.get(c) or config[c]["variant"])} for c in ids] @@ -302,7 +314,7 @@ def build(): # severity legs) re-versions the whole joint call, so the loss rows below # use only current-version rows; the headline questions, whose wording did # not change, show their earlier readings as explicitly legacy points. - all_rows = load_runlog(panel_only=False) + all_rows = public_timeline_rows(load_runlog(panel_only=False)) current_rows_all = instrument_rows(all_rows) legacy_probability_ids = {q["id"] for q in qs if not q.get("cause")} legacy_rows = [r for r in all_rows diff --git a/tests/test_experiments.py b/tests/test_experiments.py index 7e1f04c..3aff8d5 100644 --- a/tests/test_experiments.py +++ b/tests/test_experiments.py @@ -362,7 +362,29 @@ def test_the_legend_is_the_published_experiments_pooled_conditions_in_its_order( self.assertNotIn("ai-banned", pooled) # the sixth is never shown self.assertEqual(legend[0]["text"], ru.VARIANTS["none"]) old = condition_legend({"retired": "recent"}) - self.assertEqual((old[-1]["id"], old[-1]["text"]), ("retired", ru.VARIANTS["recent"])) + self.assertEqual([e["id"] for e in old], pooled) # unknown IDs fail closed + self.assertEqual([e["id"] for e in condition_legend({"ai-banned": "ai-banned"})], pooled) + + def test_shadow_rows_never_enter_current_or_legacy_timeline_aggregates(self): + from redlines.views.timeline import public_timeline_rows, condition_cells + from redlines.runlog import pool + mk = lambda c, version, **kw: { + "condition": c, "instrument_version": version, "label": "A", + "question_id": "catastrophe:ai", "run_date": "2026-10-21", + "elicited_at": "2026-10-21T12:00:00+00:00", + "forecasts": [{"horizon": "2030", "probability": 0.01}], **kw} + for version in ("legacy", "airo-incidents-prospective-v3"): + public = [mk("none", version), mk("independent", version)] + shadows = [mk("ai-banned", version), # even without pooled stamp + mk("recent-shadow-noairo", version), + mk("future-shadow", version), + mk("critical", version, pooled=False), + mk("recent", version, publish=False)] + kept = public_timeline_rows(public + shadows) + self.assertEqual(kept, public) + self.assertEqual(pool(kept)["forecasts"][0]["probability"], 0.01) + self.assertEqual(set(condition_cells(public + shadows)[("2026-10-21", "catastrophe:ai")]), + {"none", "independent"}) class TestPooledRows(unittest.TestCase): diff --git a/tests/test_tooltips.cjs b/tests/test_tooltips.cjs index 27a455a..a0e30d1 100644 --- a/tests/test_tooltips.cjs +++ b/tests/test_tooltips.cjs @@ -71,3 +71,28 @@ test('all app tooltip callers use the shared implementation', () => { assert.ok(!source.includes('position: "absolute"'), file); } }); + +test('timeline rejects shadow/unknown conditions in retained snapshots and aggregates', () => { + const source = fs.readFileSync(path.join(__dirname, '../web/demo/19-live-timeline.jsx'), 'utf8') + .split('/* ---------------- time-series chart ---------------- */')[0]; + const context = vm.createContext({ window: { __TIMELINE__: { variants: [{ id: 'ai-banned' }] } } }); + vm.runInContext(source, context); + const values = v => ({ '2030': [v] }); + const answers = v => ({ '2030': [{ A: v, B: v + 2 }] }); + const q = { horizons: ['2030'], median: values(99), rerun: values(null), + variants: { none: values(2), independent: values(4), 'ai-banned': values(99), 'future-shadow': values(88) }, + variantModels: { none: answers(1), independent: answers(3), 'ai-banned': answers(99), 'future-shadow': answers(88) }, + series: ['A', 'B'].map(label => ({ label, ps: values(99), draws: values([99]), lo: values(99), hi: values(99) })) }; + context.q = q; + context.snapshots = [{ date: '2026-10-21' }]; + assert.equal(vm.runInContext('tlVariants(q, "2030", snapshots).map(v => v.id).join(",")', context), 'none,independent'); + const clean = vm.runInContext('tlPublicQuestion(q)', context); + assert.equal(clean.median['2030'][0], 3); // median of public-only model means 2,4 + assert.equal(clean.series[0].ps['2030'][0], 2); + assert.equal(clean.series[1].ps['2030'][0], 4); + assert.equal(Object.keys(clean.variants).join(','), 'none,independent'); + assert.equal(Object.keys(clean.variantModels).join(','), 'none,independent'); + assert.equal(q.median['2030'][0], 99); // do not mutate the stored snapshot + context.clean = clean; + assert.equal(vm.runInContext('tlPublicQuestion(clean) === clean', context), true); +}); diff --git a/web/demo/19-live-timeline.jsx b/web/demo/19-live-timeline.jsx index ba26480..586fc73 100644 --- a/web/demo/19-live-timeline.jsx +++ b/web/demo/19-live-timeline.jsx @@ -30,7 +30,8 @@ const tlFmtRatio = r => (r == null || !isFinite(r) ? "—" : "×" + (r >= 10 ? r const tlFmtFor = q => (q && q.value_kind === "loss" ? tlFmtBig : tlFmt); // Colors belong to prompt conditions, never to models or their array order. -// New condition IDs stay neutral until explicitly assigned a color here. +// This is an explicit publication allowlist, not a fallback color registry. +// Shadow/unknown conditions in retained snapshots must never reach the UI. const TL_CONDITION_START = "2026-10-21"; const TL_CONDITIONS = { none: { label: "Baseline", color: "#0969da" }, @@ -41,14 +42,46 @@ const TL_CONDITIONS = { }; const tlVariants = (q, horizon, snapshots) => { const info = Object.fromEntries(((TL && TL.variants) || []).map(v => [v.id, v])); - return Object.entries(q.variants || {}).map(([id, hs]) => ({ - id, ...(TL_CONDITIONS[id] || { label: id, color: "var(--ink-soft)" }), + return Object.entries(q.variants || {}).filter(([id]) => Object.hasOwn(TL_CONDITIONS, id)).map(([id, hs]) => ({ + id, ...TL_CONDITIONS[id], vals: (hs[horizon] || []).map((v, i) => snapshots[i].date >= TL_CONDITION_START ? v : null), models: ((q.variantModels || {})[id] || {})[horizon] || [], text: (info[id] || {}).text, })).filter(v => v.vals.some(val => val != null)); }; +// Old published snapshots may include a shadow in both condition dots and +// pooled model means. Repair the displayed aggregates from public condition +// answers on affected dates only; leave the immutable snapshot/downloads alone. +function tlPublicQuestion(q) { + const hidden = Object.keys(q.variants || {}).filter(id => !Object.hasOwn(TL_CONDITIONS, id)); + if (!hidden.length) return q; + const clean = JSON.parse(JSON.stringify(q)); + for (const h of q.horizons) { + for (let i = 0; i < (q.median[h] || []).length; i++) { + if (!hidden.some(id => q.variants[id][h]?.[i] != null)) continue; + for (const model of clean.series) { + const values = Object.keys(TL_CONDITIONS).map(id => + q.variantModels?.[id]?.[h]?.[i]?.[model.label]).filter(v => v != null); + const mean = values.length ? Math.round(values.reduce((a, b) => a + b, 0) / values.length * 1000) / 1000 : null; + model.ps[h][i] = mean; + model.draws[h][i] = values.length ? values : null; + model.lo[h][i] = values.length ? Math.min(...values) : null; + model.hi[h][i] = values.length ? Math.max(...values) : null; + } + const values = clean.series.map(s => s.ps[h][i]).filter(v => v != null).sort((a, b) => a - b); + const mid = Math.floor(values.length / 2); + clean.median[h][i] = values.length ? Math.round((values.length % 2 ? values[mid] : (values[mid - 1] + values[mid]) / 2) * 1000) / 1000 : null; + clean.rerun[h][i] = null; + } + } + for (const id of hidden) { + delete clean.variants[id]; + delete clean.variantModels?.[id]; + } + return clean; +} + /* ---------------- time-series chart ---------------- */ // Before Oct 21 the Baseline median is blue until hovered or selected; then // model answers appear as ink-gray circles. Since Oct 21 each prompt condition @@ -383,7 +416,7 @@ function TimelinePanel(props = {}) { } function TimelinePanelCurrent({ qid = TL_HEADLINE, title = "Probability of an AI catastrophe", sub = "Forecasts over time" } = {}) { const snapshots = TL.snapshots; - const q = TL.questions.find(x => x.id === qid) || TL.questions[0]; + const q = useMemo(() => tlPublicQuestion(TL.questions.find(x => x.id === qid) || TL.questions[0]), [qid]); // The horizon picker, Graph 2's toggle: opens on 2030 like every other // horizon toggle (Nick, 2026-08-27), and is what the line, the dots, the // numbers below and the table all read.