From c6fb55b3742c685fb4bb6eeeb4d31bc5da09d03f Mon Sep 17 00:00:00 2001 From: Curtis Schaefer Date: Thu, 22 Oct 2026 15:03:02 -0400 Subject: [PATCH 1/5] Clarify prompt variants beside model rationales --- index.html | 29 ++++++-- redlines/views/rationales.py | 60 +++++++++++++++-- results/rationales_data.json | 3 +- tests/test_rationale_prompts.py | 113 ++++++++++++++++++++++++++++++++ web/demo/65-rationales.jsx | 27 ++++++-- 5 files changed, 214 insertions(+), 18 deletions(-) create mode 100644 tests/test_rationale_prompts.py diff --git a/index.html b/index.html index 925520f..a9ef238 100644 --- a/index.html +++ b/index.html @@ -165,7 +165,7 @@ - + @@ -2565,11 +2565,16 @@

{title}

} // The selected model's forecasts on every horizon, above its explanation: the // explanation was recorded once per question, not once per horizon. -function RationaleForecastTable({ forecasts, model }) { +function RationaleForecastTable({ forecasts, model, calls }) { const rows = forecasts.filter(f => (f.values || {})[model] != null); if (!rows.length) return null; + const variants = new Set(calls.map(([, c]) => c.promptVariant).filter(Boolean)); + const distinctPrompts = variants.size === calls.length; return
+ {calls.length > 1 && } {rows.map(f => @@ -2598,11 +2603,16 @@

{title}

{m} )} - {model && } + {model && } {!calls.length &&

No explanation recorded for this forecast.

} - {calls.map(([id, c]) => { + {calls.map(([id, c], index) => { const explanation = (c.debriefs || {})[qid]; - return
+ const prompt = (RATIONALES.prompts || {})[c.promptId]; + return
+

{c.promptLabel || "Unrecorded"} prompt

+

+ {c.promptVariant === "none" ? "No additional instruction." : c.promptInstruction || "Added instruction text is not available for this call."} +

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}{calls.length > 1 ? " Each call contributes to the displayed model mean." : ""}

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

{explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} @@ -2611,9 +2621,16 @@

{title}

Original batch explanation and provenance {explanation &&

{c.rationale}

} -

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Condition: unconditional

+

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

Download original forecasts, explanations and logs
+
View full prompt + {prompt ? <> + {prompt.system && prompt.system.trim() && <>

System prompt

{prompt.system}
} +

User prompt

+ 
{prompt.user}
+ :

Full prompt not recorded for this call.

} +
; })} ; diff --git a/redlines/views/rationales.py b/redlines/views/rationales.py index c3236b5..38ff471 100644 --- a/redlines/views/rationales.py +++ b/redlines/views/rationales.py @@ -1,23 +1,71 @@ """Original explanations attached to the calls used by current charts.""" +import hashlib +import json from collections import defaultdict +from ..experiments import DEFAULT_VARIANT, PUBLISHED, VARIANTS, load as load_experiment from ..runlog import (load_runlog, complete_panel_rows, latest_instrument_rows, current_rows, elicitation_date) +def _call_id(row): + return row.get('call_id') or f"{row['run_id']}:{row['label']}" + + +def _prompt_metadata(row, recorded, prompts): + """Attach the call's identity and archived wording, never a rebuilt prompt. + + The full prompt is recorded only on a call's first question row. Match the + configured instruction against that record before presenting it verbatim: + a later edit to VARIANTS must not rewrite what an earlier call was asked. + Full prompts are shared across models, so store each (system, user) pair once. + """ + variant = row.get('variant') + condition = row.get('condition') + # The log reader maps old prompt variants into `condition`; never treat a + # scenario dict or an unrecognized condition as a known prompt variant. + if not variant and isinstance(condition, str) and condition in VARIANTS: + variant = condition + label = ('Baseline' if variant == DEFAULT_VARIANT else variant.capitalize()) if variant else 'Unrecorded' + instruction = VARIANTS.get(variant) + full_prompt = recorded.get('prompt') if recorded else None + prompt_id = None + if full_prompt: + prompt = {'system': recorded.get('system_prompt'), 'user': full_prompt} + prompt_id = hashlib.sha256(json.dumps(prompt, sort_keys=True).encode()).hexdigest() + prompts[prompt_id] = prompt + return { + 'condition': condition if isinstance(condition, str) else None, + 'promptVariant': variant, 'promptLabel': label, + 'promptInstruction': instruction if instruction and full_prompt and instruction in full_prompt else None, + 'promptId': prompt_id, + 'promptSha256': recorded.get('prompt_sha256') if recorded else None, + } + + def build(): all_rows = load_runlog() rows = current_rows(complete_panel_rows(latest_instrument_rows(all_rows))) debriefs = {r['call_id']: r['debrief'] for r in all_rows if r.get('debrief')} - calls, questions = {}, defaultdict(lambda: defaultdict(list)) + recorded = {_call_id(r): r for r in all_rows if r.get('prompt')} + calls, prompts, questions = {}, {}, defaultdict(lambda: defaultdict(list)) for r in rows: - cid = r.get('call_id') or f"{r['run_id']}:{r['label']}" - calls[cid] = { + cid = _call_id(r) + if cid not in calls: + calls[cid] = _prompt_metadata(r, recorded.get(cid), prompts) + calls[cid].update({ 'model': r['label'], 'modelId': r['model'], 'run': r['run_id'], 'elicitedAt': r['elicited_at'], 'date': elicitation_date(r), 'instrument': r.get('instrument_version'), 'protocol': r.get('protocol'), 'rationale': r.get('rationale') or '', 'sources': r.get('key_sources') or [], 'debriefs': debriefs.get(cid, {}), - } - questions[r['question_id']][r['label']].append(cid) - return {'calls': calls, 'questions': dict(questions)} + }) + ids = questions[r['question_id']][r['label']] + if cid not in ids: + ids.append(cid) + order = {condition: i for i, condition in enumerate(load_experiment(PUBLISHED)['pooled'])} + for models in questions.values(): + for ids in models.values(): + ids.sort(key=lambda cid: (order.get(calls[cid]['condition'], len(order)), + calls[cid]['elicitedAt'], cid)) + return {'calls': calls, 'questions': dict(questions), 'prompts': prompts} diff --git a/results/rationales_data.json b/results/rationales_data.json index b923dba..8571558 100644 --- a/results/rationales_data.json +++ b/results/rationales_data.json @@ -1,4 +1,5 @@ { "calls": {}, - "questions": {} + "questions": {}, + "prompts": {} } diff --git a/tests/test_rationale_prompts.py b/tests/test_rationale_prompts.py new file mode 100644 index 0000000..86e092b --- /dev/null +++ b/tests/test_rationale_prompts.py @@ -0,0 +1,113 @@ +"""Prompt labels must describe the archived call behind each explanation.""" +import copy +import unittest +from contextlib import ExitStack +from unittest.mock import patch + +from redlines.experiments import DEFAULT_VARIANT, PUBLISHED, VARIANTS, load +from redlines.views import rationales + + +def call_rows(condition, variant, model="Example model"): + """Two questions, with the prompt/debrief recorded only on the first row.""" + instruction = VARIANTS.get(variant) + prompt = "Forecast these questions.\n" + (instruction or "") + first = { + "call_id": f"example:{model}:{condition}", "run_id": "example", + "label": model, "model": model, "elicited_at": "2026-10-22T12:00:00Z", + "question_id": "first", "condition": condition, "variant": variant, + "prompt": prompt, "system_prompt": "Recorded system instruction.", + "prompt_sha256": "recorded-user-prompt-hash", + "rationale": "Original batch rationale.", + "key_sources": ["https://example.org/source"], + "debrief": {"second": {"rationale": "Original question explanation."}}, + } + second = {**first, "question_id": "second", "prompt": None, + "system_prompt": None, "debrief": None} + return [first, second] + + +def build_rows(rows, selected=None): + """Isolate metadata from panel eligibility (covered by instrument tests).""" + with ExitStack() as stack: + stack.enter_context(patch.object(rationales, "load_runlog", return_value=rows)) + stack.enter_context(patch.object(rationales, "latest_instrument_rows", + return_value=rows if selected is None else selected)) + for name in ("complete_panel_rows", "current_rows"): + stack.enter_context(patch.object(rationales, name, side_effect=lambda value: value)) + return rationales.build() + + +class TestRationalePrompts(unittest.TestCase): + def test_pooled_prompt_order_and_original_explanations(self): + experiment = load(PUBLISHED) + rows = [] + for condition in reversed(experiment["pooled"]): + rows.extend(call_rows(condition, experiment["conditions"][condition]["variant"])) + original = copy.deepcopy(rows) + data = build_rows(rows + [rows[-1]]) # duplicate question must not repeat a call + ids = data["questions"]["second"]["Example model"] + self.assertEqual([data["calls"][cid]["condition"] for cid in ids], experiment["pooled"]) + self.assertEqual(len(data["calls"]), len(experiment["pooled"])) + for row in rows: + call = data["calls"][row["call_id"]] + self.assertEqual(call["rationale"], row["rationale"]) + self.assertEqual(call["sources"], row["key_sources"]) + self.assertEqual(call["debriefs"]["second"]["rationale"], "Original question explanation.") + self.assertEqual(call["promptInstruction"], VARIANTS[row["variant"]]) + expected = "Baseline" if row["variant"] == DEFAULT_VARIANT else row["variant"].capitalize() + self.assertEqual(call["promptLabel"], expected) + self.assertEqual(rows, original) + + def test_recovers_first_row_prompt_for_later_question(self): + rows = call_rows("independent", "independent") + data = build_rows(rows, selected=[rows[1]]) + call = data["calls"][rows[1]["call_id"]] + self.assertEqual(data["prompts"][call["promptId"]], + {"system": rows[0]["system_prompt"], "user": rows[0]["prompt"]}) + self.assertEqual(call["promptSha256"], rows[0]["prompt_sha256"]) + self.assertEqual(call["debriefs"], rows[0]["debrief"]) + + def test_shares_identical_prompts_but_keeps_different_system_instructions(self): + first = call_rows("independent", "independent", model="First model") + second = call_rows("independent", "independent", model="Second model") + data = build_rows(first + second) + self.assertEqual(len(data["prompts"]), 1) + second[0]["system_prompt"] = "A different system instruction." + self.assertEqual(len(build_rows(first + second)["prompts"]), 2) + + def test_does_not_substitute_new_instruction_for_archived_wording(self): + rows = call_rows("independent", "independent") + with patch.dict(rationales.VARIANTS, {"independent": "A later instruction."}): + data = build_rows(rows) + call = data["calls"][rows[0]["call_id"]] + self.assertIsNone(call["promptInstruction"]) + self.assertEqual(data["prompts"][call["promptId"]]["user"], rows[0]["prompt"]) + + def test_missing_prompt_does_not_invent_full_text(self): + rows = call_rows("independent", "independent")[1:] + data = build_rows(rows) + call = data["calls"][rows[0]["call_id"]] + self.assertEqual(call["promptLabel"], "Independent") + self.assertIsNone(call["promptInstruction"]) + self.assertIsNone(call["promptId"]) + self.assertEqual(data["prompts"], {}) + + def test_missing_variant_uses_known_condition_but_never_guesses_baseline(self): + for condition, expected in (("independent", "Independent"), ("none", "Baseline"), + (None, "Unrecorded"), ({"id": "ai"}, "Unrecorded"), + ("unrecognized", "Unrecorded")): + with self.subTest(condition=condition): + row = call_rows("example", None)[0] + row["condition"] = condition + data = build_rows([row]) + self.assertEqual(data["calls"][row["call_id"]]["promptLabel"], expected) + + def test_preserves_last_row_batch_rationale_behavior(self): + rows = call_rows("none", DEFAULT_VARIANT) + rows[-1]["rationale"] = "Last recorded batch rationale." + rows[-1]["key_sources"] = ["https://example.org/last-source"] + data = build_rows(rows) + call = data["calls"][rows[0]["call_id"]] + self.assertEqual(call["rationale"], rows[-1]["rationale"]) + self.assertEqual(call["sources"], rows[-1]["key_sources"]) diff --git a/web/demo/65-rationales.jsx b/web/demo/65-rationales.jsx index 0852637..ac00437 100644 --- a/web/demo/65-rationales.jsx +++ b/web/demo/65-rationales.jsx @@ -17,11 +17,16 @@ function RationaleSources({ sources }) { } // The selected model's forecasts on every horizon, above its explanation: the // explanation was recorded once per question, not once per horizon. -function RationaleForecastTable({ forecasts, model }) { +function RationaleForecastTable({ forecasts, model, calls }) { const rows = forecasts.filter(f => (f.values || {})[model] != null); if (!rows.length) return null; + const variants = new Set(calls.map(([, c]) => c.promptVariant).filter(Boolean)); + const distinctPrompts = variants.size === calls.length; return
+ Average forecast across {calls.length} {distinctPrompts ? "prompt variants" : "calls"} +
HorizonForecast
{rationaleHorizonLabel(f.key)}
+ {calls.length > 1 && } {rows.map(f => @@ -50,11 +55,16 @@ function ForecastRationales({ qid, context, forecasts = [], date, controls }) { {m} )} - {model && } + {model && } {!calls.length &&

No explanation recorded for this forecast.

} - {calls.map(([id, c]) => { + {calls.map(([id, c], index) => { const explanation = (c.debriefs || {})[qid]; - return
+ const prompt = (RATIONALES.prompts || {})[c.promptId]; + return
+

{c.promptLabel || "Unrecorded"} prompt

+

+ {c.promptVariant === "none" ? "No additional instruction." : c.promptInstruction || "Added instruction text is not available for this call."} +

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}{calls.length > 1 ? " Each call contributes to the displayed model mean." : ""}

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

{explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} @@ -63,9 +73,16 @@ function ForecastRationales({ qid, context, forecasts = [], date, controls }) {
Original batch explanation and provenance {explanation &&

{c.rationale}

} -

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Condition: unconditional

+

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

Download original forecasts, explanations and logs
+
View full prompt + {prompt ? <> + {prompt.system && prompt.system.trim() && <>

System prompt

{prompt.system}
} +

User prompt

+ 
{prompt.user}
+ :

Full prompt not recorded for this call.

} +
; })} ; From d16ff09ec8f0237b95721ecbad3c784cfc384282 Mon Sep 17 00:00:00 2001 From: Curtis Schaefer Date: Thu, 22 Oct 2026 15:43:28 -0400 Subject: [PATCH 2/5] Collapse each prompt's model rationale by default --- index.html | 23 +++++++++++++---------- web/demo/65-rationales.jsx | 23 +++++++++++++---------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/index.html b/index.html index a9ef238..783052d 100644 --- a/index.html +++ b/index.html @@ -2613,16 +2613,19 @@

{title}

{c.promptVariant === "none" ? "No additional instruction." : c.promptInstruction || "Added instruction text is not available for this call."}

-

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}{calls.length > 1 ? " Each call contributes to the displayed model mean." : ""}

-

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

- {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} -
Sources cited by this model - -
-
Original batch explanation and provenance - {explanation &&

{c.rationale}

} -

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

- Download original forecasts, explanations and logs +
+ Model rationale +

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}{calls.length > 1 ? " Each call contributes to the displayed model mean." : ""}

+

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

+ {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} +
Sources cited by this model + +
+
Original batch explanation and provenance + {explanation &&

{c.rationale}

} +

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

+ Download original forecasts, explanations and logs +
View full prompt {prompt ? <> diff --git a/web/demo/65-rationales.jsx b/web/demo/65-rationales.jsx index ac00437..c46d184 100644 --- a/web/demo/65-rationales.jsx +++ b/web/demo/65-rationales.jsx @@ -65,16 +65,19 @@ function ForecastRationales({ qid, context, forecasts = [], date, controls }) {

{c.promptVariant === "none" ? "No additional instruction." : c.promptInstruction || "Added instruction text is not available for this call."}

-

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}{calls.length > 1 ? " Each call contributes to the displayed model mean." : ""}

-

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

- {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} -
Sources cited by this model - -
-
Original batch explanation and provenance - {explanation &&

{c.rationale}

} -

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

- Download original forecasts, explanations and logs +
+ Model rationale +

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}{calls.length > 1 ? " Each call contributes to the displayed model mean." : ""}

+

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

+ {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} +
Sources cited by this model + +
+
Original batch explanation and provenance + {explanation &&

{c.rationale}

} +

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

+ Download original forecasts, explanations and logs +
View full prompt {prompt ? <> From 4442d1b0444a83d973d8a12b75a1f485dfdd692b Mon Sep 17 00:00:00 2001 From: Curtis Schaefer Date: Thu, 22 Oct 2026 18:05:09 -0400 Subject: [PATCH 3/5] Select prompt condition before model rationale --- index.html | 159 ++++++++++++++++++++------------ redlines/views/rationales.py | 11 ++- results/rationales_data.json | 9 +- tests/test_rationale_prompts.py | 22 +++++ web/demo/65-rationales.jsx | 157 +++++++++++++++++++------------ 5 files changed, 236 insertions(+), 122 deletions(-) diff --git a/index.html b/index.html index 783052d..82601b0 100644 --- a/index.html +++ b/index.html @@ -165,7 +165,7 @@ - + @@ -2563,79 +2563,118 @@

{title}

; })}; } -// The selected model's forecasts on every horizon, above its explanation: the -// explanation was recorded once per question, not once per horizon. -function RationaleForecastTable({ forecasts, model, calls }) { - const rows = forecasts.filter(f => (f.values || {})[model] != null); - if (!rows.length) return null; - const variants = new Set(calls.map(([, c]) => c.promptVariant).filter(Boolean)); - const distinctPrompts = variants.size === calls.length; +const rationaleConditionKey = c => c.condition || c.promptVariant || "unrecorded"; +// One condition only. Raw call probabilities are fractions; convert to percent +// once, after averaging any repeat calls that answered this question/horizon. +function rationaleConditionForecasts(qid, horizons, models, calls) { + return horizons.map(key => { + const values = {}, counts = {}; + models.forEach(model => { + const ps = calls.filter(([, c]) => c.model === model) + .map(([, c]) => (((c.forecasts || {})[qid] || {})[key])) + .filter(p => typeof p === "number" && Number.isFinite(p)); + counts[model] = ps.length; + values[model] = ps.length ? 100 * ps.reduce((sum, p) => sum + p, 0) / ps.length : null; + }); + return { key, values, counts }; + }); +} +function RationaleForecastTable({ rows, models, conditionLabel }) { + if (!rows.length || !models.length) return null; + const repeated = rows.some(row => models.some(model => row.counts[model] > 1)); return
+ Average forecast across {calls.length} {distinctPrompts ? "prompt variants" : "calls"} +
HorizonForecast
{rationaleHorizonLabel(f.key)}
- {calls.length > 1 && } - - {rows.map(f => - - + + {models.map(model => )} + {rows.map(row => + + {models.map(model => )} )}
- Average forecast across {calls.length} {distinctPrompts ? "prompt variants" : "calls"} -
HorizonForecast
{rationaleHorizonLabel(f.key)}{rationalePct(f.values[model])}
 + {conditionLabel} prompt: forecasts by model +
Horizon{model}
{rationaleHorizonLabel(row.key)} 1 ? `Mean of ${row.counts[model]} recorded calls` : undefined}> + {rationalePct(row.values[model])} +
+ {repeated &&

Each cell averages the available repeat calls for that model and horizon within this prompt condition.

} +
; +} +function RationaleExplanation({ id, call: c, qid }) { + const explanation = (c.debriefs || {})[qid]; + const prompt = (RATIONALES.prompts || {})[c.promptId]; + return
+
+ Model rationale +

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}

+

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

+ {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} +
Sources cited by this model + +
+
Original batch explanation and provenance + {explanation &&

{c.rationale}

} +

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

+ Download original forecasts, explanations and logs +
+
+
View full prompt + {prompt ? <> + {prompt.system && prompt.system.trim() && <>

System prompt

{prompt.system}
} +

User prompt

+ 
{prompt.user}
+ :

Full prompt not recorded for this call.

} +
; } -// `forecasts`: [{ key: horizon, values: { model label: probability } }], every -// horizon the question was asked on. `date` picks the calls shown (the -// elicitation whose forecasts the chart displays); it is not shown. +// `forecasts` supplies the chart's horizons and model roster, not its pooled +// probabilities. The comparison table uses only archived calls for the chosen +// prompt condition, question, and displayed date. function ForecastRationales({ qid, context, forecasts = [], date, controls }) { - const [selected, setSelected] = useState(""); + const [selectedCondition, setSelectedCondition] = useState(""); + const [selectedModel, setSelectedModel] = useState(""); + const [selectedCall, setSelectedCall] = useState(""); const entries = RATIONALES.questions[qid] || {}; const models = Object.keys(entries).filter(m => forecasts.some(f => (f.values || {})[m] != null)); - const model = models.includes(selected) ? selected : models[0]; - const calls = (entries[model] || []).map(id => [id, RATIONALES.calls[id]]) + const allCalls = models.flatMap(model => (entries[model] || []).map(id => [id, RATIONALES.calls[id]])) .filter(([, c]) => c && (!date || c.date === date)); + const order = RATIONALES.conditionOrder || []; + const rank = condition => order.includes(condition) ? order.indexOf(condition) : order.length; + const conditions = [...new Set(allCalls.map(([, c]) => rationaleConditionKey(c)))].sort((a, b) => rank(a) - rank(b)); + const condition = conditions.includes(selectedCondition) ? selectedCondition : conditions[0]; + const conditionCalls = allCalls.filter(([, c]) => rationaleConditionKey(c) === condition); + const availableModels = models.filter(model => conditionCalls.some(([, c]) => c.model === model)); + const model = availableModels.includes(selectedModel) ? selectedModel : availableModels[0]; + const calls = conditionCalls.filter(([, c]) => c.model === model); + const current = calls.find(([id]) => id === selectedCall) || calls[0]; + const chosen = current && current[1]; + const label = c => c.promptLabel || "Unrecorded"; + const rows = rationaleConditionForecasts(qid, forecasts.map(f => f.key), models, conditionCalls); return
Model rationale for its forecasts {controls}

{context}

-
- {models.map(m => )} -
- {model && } - {!calls.length &&

No explanation recorded for this forecast.

} - {calls.map(([id, c], index) => { - const explanation = (c.debriefs || {})[qid]; - const prompt = (RATIONALES.prompts || {})[c.promptId]; - return
-

{c.promptLabel || "Unrecorded"} prompt

-

- {c.promptVariant === "none" ? "No additional instruction." : c.promptInstruction || "Added instruction text is not available for this call."} -

-
- Model rationale -

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}{calls.length > 1 ? " Each call contributes to the displayed model mean." : ""}

-

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

- {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} -
Sources cited by this model - -
-
Original batch explanation and provenance - {explanation &&

{c.rationale}

} -

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

- Download original forecasts, explanations and logs -
-
-
View full prompt - {prompt ? <> - {prompt.system && prompt.system.trim() && <>

System prompt

{prompt.system}
} -

User prompt

- 
{prompt.user}
- :

Full prompt not recorded for this call.

} -
-
; - })} + {!chosen ?

No explanation recorded for this forecast.

: <> +
+ +
+

{label(chosen)} prompt

+

+ {chosen.promptVariant === "none" ? "No additional instruction." : chosen.promptInstruction || "Added instruction text is not available for this call."} +

+ + {conditions.length > 1 &&

The main charts average across prompt conditions; this table shows the selected condition.

} +
+ + {calls.length > 1 && } +
+ + }
; } // `causes`: [{key, label, rungs}], each cause's rungs from BOTH ladders (the diff --git a/redlines/views/rationales.py b/redlines/views/rationales.py index 38ff471..8a6bd23 100644 --- a/redlines/views/rationales.py +++ b/redlines/views/rationales.py @@ -52,7 +52,7 @@ def build(): for r in rows: cid = _call_id(r) if cid not in calls: - calls[cid] = _prompt_metadata(r, recorded.get(cid), prompts) + calls[cid] = {**_prompt_metadata(r, recorded.get(cid), prompts), 'forecasts': {}} calls[cid].update({ 'model': r['label'], 'modelId': r['model'], 'run': r['run_id'], 'elicitedAt': r['elicited_at'], 'date': elicitation_date(r), @@ -60,6 +60,12 @@ def build(): 'rationale': r.get('rationale') or '', 'sources': r.get('key_sources') or [], 'debriefs': debriefs.get(cid, {}), }) + # Keep the original [0, 1] probabilities by call and question. The + # rationale table can select one condition without borrowing a mean + # across prompts from the main chart, or another question's numbers. + calls[cid]['forecasts'][r['question_id']] = { + f['horizon']: f['probability'] for f in r.get('forecasts', []) + } ids = questions[r['question_id']][r['label']] if cid not in ids: ids.append(cid) @@ -68,4 +74,5 @@ def build(): for ids in models.values(): ids.sort(key=lambda cid: (order.get(calls[cid]['condition'], len(order)), calls[cid]['elicitedAt'], cid)) - return {'calls': calls, 'questions': dict(questions), 'prompts': prompts} + return {'calls': calls, 'questions': dict(questions), 'prompts': prompts, + 'conditionOrder': list(order)} diff --git a/results/rationales_data.json b/results/rationales_data.json index 8571558..2c20329 100644 --- a/results/rationales_data.json +++ b/results/rationales_data.json @@ -1,5 +1,12 @@ { "calls": {}, "questions": {}, - "prompts": {} + "prompts": {}, + "conditionOrder": [ + "none", + "independent", + "recent", + "critical", + "two-step" + ] } diff --git a/tests/test_rationale_prompts.py b/tests/test_rationale_prompts.py index 86e092b..e976c10 100644 --- a/tests/test_rationale_prompts.py +++ b/tests/test_rationale_prompts.py @@ -48,6 +48,7 @@ def test_pooled_prompt_order_and_original_explanations(self): data = build_rows(rows + [rows[-1]]) # duplicate question must not repeat a call ids = data["questions"]["second"]["Example model"] self.assertEqual([data["calls"][cid]["condition"] for cid in ids], experiment["pooled"]) + self.assertEqual(data["conditionOrder"], experiment["pooled"]) self.assertEqual(len(data["calls"]), len(experiment["pooled"])) for row in rows: call = data["calls"][row["call_id"]] @@ -111,3 +112,24 @@ def test_preserves_last_row_batch_rationale_behavior(self): call = data["calls"][rows[0]["call_id"]] self.assertEqual(call["rationale"], rows[-1]["rationale"]) self.assertEqual(call["sources"], rows[-1]["key_sources"]) + + def test_forecasts_remain_separate_by_call_condition_and_question(self): + baseline = call_rows("none", DEFAULT_VARIANT) + baseline[0]["forecasts"] = [{"horizon": "2030", "probability": 0}, + {"horizon": "2100", "probability": 1}] + baseline[1]["forecasts"] = [{"horizon": "2030", "probability": 0.2}] + independent = call_rows("independent", "independent") + independent[0]["forecasts"] = [{"horizon": "2030", "probability": 0.8}] + repeat = copy.deepcopy(baseline[0]) + repeat["call_id"] += ":repeat" + repeat["forecasts"] = [{"horizon": "2030", "probability": 0.6}] + rows = baseline + independent + [repeat] + original = copy.deepcopy(rows) + data = build_rows(rows) + self.assertEqual(data["calls"][baseline[0]["call_id"]]["forecasts"], + {"first": {"2030": 0, "2100": 1}, "second": {"2030": 0.2}}) + self.assertEqual(data["calls"][independent[0]["call_id"]]["forecasts"], + {"first": {"2030": 0.8}, "second": {}}) + self.assertEqual(data["calls"][repeat["call_id"]]["forecasts"], + {"first": {"2030": 0.6}}) + self.assertEqual(rows, original) diff --git a/web/demo/65-rationales.jsx b/web/demo/65-rationales.jsx index c46d184..cab2b64 100644 --- a/web/demo/65-rationales.jsx +++ b/web/demo/65-rationales.jsx @@ -15,79 +15,118 @@ function RationaleSources({ sources }) { ; })}; } -// The selected model's forecasts on every horizon, above its explanation: the -// explanation was recorded once per question, not once per horizon. -function RationaleForecastTable({ forecasts, model, calls }) { - const rows = forecasts.filter(f => (f.values || {})[model] != null); - if (!rows.length) return null; - const variants = new Set(calls.map(([, c]) => c.promptVariant).filter(Boolean)); - const distinctPrompts = variants.size === calls.length; +const rationaleConditionKey = c => c.condition || c.promptVariant || "unrecorded"; +// One condition only. Raw call probabilities are fractions; convert to percent +// once, after averaging any repeat calls that answered this question/horizon. +function rationaleConditionForecasts(qid, horizons, models, calls) { + return horizons.map(key => { + const values = {}, counts = {}; + models.forEach(model => { + const ps = calls.filter(([, c]) => c.model === model) + .map(([, c]) => (((c.forecasts || {})[qid] || {})[key])) + .filter(p => typeof p === "number" && Number.isFinite(p)); + counts[model] = ps.length; + values[model] = ps.length ? 100 * ps.reduce((sum, p) => sum + p, 0) / ps.length : null; + }); + return { key, values, counts }; + }); +} +function RationaleForecastTable({ rows, models, conditionLabel }) { + if (!rows.length || !models.length) return null; + const repeated = rows.some(row => models.some(model => row.counts[model] > 1)); return
- {calls.length > 1 && } - - {rows.map(f => - - + + {models.map(model => )} + {rows.map(row => + + {models.map(model => )} )}
- Average forecast across {calls.length} {distinctPrompts ? "prompt variants" : "calls"} -
HorizonForecast
{rationaleHorizonLabel(f.key)}{rationalePct(f.values[model])}
 + {conditionLabel} prompt: forecasts by model +
Horizon{model}
{rationaleHorizonLabel(row.key)} 1 ? `Mean of ${row.counts[model]} recorded calls` : undefined}> + {rationalePct(row.values[model])} +
+ {repeated &&

Each cell averages the available repeat calls for that model and horizon within this prompt condition.

} +
; +} +function RationaleExplanation({ id, call: c, qid }) { + const explanation = (c.debriefs || {})[qid]; + const prompt = (RATIONALES.prompts || {})[c.promptId]; + return
+
+ Model rationale +

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}

+

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

+ {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} +
Sources cited by this model + +
+
Original batch explanation and provenance + {explanation &&

{c.rationale}

} +

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

+ Download original forecasts, explanations and logs +
+
+
View full prompt + {prompt ? <> + {prompt.system && prompt.system.trim() && <>

System prompt

{prompt.system}
} +

User prompt

+ 
{prompt.user}
+ :

Full prompt not recorded for this call.

} +
; } -// `forecasts`: [{ key: horizon, values: { model label: probability } }], every -// horizon the question was asked on. `date` picks the calls shown (the -// elicitation whose forecasts the chart displays); it is not shown. +// `forecasts` supplies the chart's horizons and model roster, not its pooled +// probabilities. The comparison table uses only archived calls for the chosen +// prompt condition, question, and displayed date. function ForecastRationales({ qid, context, forecasts = [], date, controls }) { - const [selected, setSelected] = useState(""); + const [selectedCondition, setSelectedCondition] = useState(""); + const [selectedModel, setSelectedModel] = useState(""); + const [selectedCall, setSelectedCall] = useState(""); const entries = RATIONALES.questions[qid] || {}; const models = Object.keys(entries).filter(m => forecasts.some(f => (f.values || {})[m] != null)); - const model = models.includes(selected) ? selected : models[0]; - const calls = (entries[model] || []).map(id => [id, RATIONALES.calls[id]]) + const allCalls = models.flatMap(model => (entries[model] || []).map(id => [id, RATIONALES.calls[id]])) .filter(([, c]) => c && (!date || c.date === date)); + const order = RATIONALES.conditionOrder || []; + const rank = condition => order.includes(condition) ? order.indexOf(condition) : order.length; + const conditions = [...new Set(allCalls.map(([, c]) => rationaleConditionKey(c)))].sort((a, b) => rank(a) - rank(b)); + const condition = conditions.includes(selectedCondition) ? selectedCondition : conditions[0]; + const conditionCalls = allCalls.filter(([, c]) => rationaleConditionKey(c) === condition); + const availableModels = models.filter(model => conditionCalls.some(([, c]) => c.model === model)); + const model = availableModels.includes(selectedModel) ? selectedModel : availableModels[0]; + const calls = conditionCalls.filter(([, c]) => c.model === model); + const current = calls.find(([id]) => id === selectedCall) || calls[0]; + const chosen = current && current[1]; + const label = c => c.promptLabel || "Unrecorded"; + const rows = rationaleConditionForecasts(qid, forecasts.map(f => f.key), models, conditionCalls); return
Model rationale for its forecasts {controls}

{context}

-
- {models.map(m => )} -
- {model && } - {!calls.length &&

No explanation recorded for this forecast.

} - {calls.map(([id, c], index) => { - const explanation = (c.debriefs || {})[qid]; - const prompt = (RATIONALES.prompts || {})[c.promptId]; - return
-

{c.promptLabel || "Unrecorded"} prompt

-

- {c.promptVariant === "none" ? "No additional instruction." : c.promptInstruction || "Added instruction text is not available for this call."} -

-
- Model rationale -

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}{calls.length > 1 ? " Each call contributes to the displayed model mean." : ""}

-

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

- {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} -
Sources cited by this model - -
-
Original batch explanation and provenance - {explanation &&

{c.rationale}

} -

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

- Download original forecasts, explanations and logs -
-
-
View full prompt - {prompt ? <> - {prompt.system && prompt.system.trim() && <>

System prompt

{prompt.system}
} -

User prompt

- 
{prompt.user}
- :

Full prompt not recorded for this call.

} -
-
; - })} + {!chosen ?

No explanation recorded for this forecast.

: <> +
+ +
+

{label(chosen)} prompt

+

+ {chosen.promptVariant === "none" ? "No additional instruction." : chosen.promptInstruction || "Added instruction text is not available for this call."} +

+ + {conditions.length > 1 &&

The main charts average across prompt conditions; this table shows the selected condition.

} +
+ + {calls.length > 1 && } +
+ + }
; } // `causes`: [{key, label, rungs}], each cause's rungs from BOTH ladders (the From 3d021b04cd19d11b5f16b4326db6d14a800e784d Mon Sep 17 00:00:00 2001 From: Curtis Schaefer Date: Fri, 23 Oct 2026 11:03:23 -0400 Subject: [PATCH 4/5] Use shared toggles and show model rationale directly --- index.html | 84 +++++++++++++++++++------------------- web/demo/60-atoms.jsx | 7 ++-- web/demo/65-rationales.jsx | 77 +++++++++++++++++----------------- 3 files changed, 84 insertions(+), 84 deletions(-) diff --git a/index.html b/index.html index 82601b0..255e397 100644 --- a/index.html +++ b/index.html @@ -2498,11 +2498,12 @@

{title}

); } -function Toggle({ options, value, onChange }) { +function Toggle({ options, value, onChange, label }) { return ( -
+
{options.map(o => ( - ))} @@ -2579,22 +2580,30 @@

{title}

return { key, values, counts }; }); } -function RationaleForecastTable({ rows, models, conditionLabel }) { - if (!rows.length || !models.length) return null; - const repeated = rows.some(row => models.some(model => row.counts[model] > 1)); +function RationaleToggle({ label, options, value, onChange }) { + return
+ {label} + +
; +} +function RationaleForecastTable({ rows, model, conditionLabel }) { + if (!rows.length || !model) return null; + const repeated = rows.some(row => row.counts[model] > 1); return
- {models.map(model => )} - {rows.map(row => - - {models.map(model => {rows.map(row => )} + + + {rows.map(row => )} - )} +
- {conditionLabel} prompt: forecasts by model + Forecasts for the {conditionLabel} prompt
Horizon{model}
{rationaleHorizonLabel(row.key)}
Model + {rationaleHorizonLabel(row.key).replace(/ \(by .*\)$/, "")} +
{model} 1 ? `Mean of ${row.counts[model]} recorded calls` : undefined}> {rationalePct(row.values[model])}
{repeated &&

Each cell averages the available repeat calls for that model and horizon within this prompt condition.

}
; @@ -2603,19 +2612,17 @@

{title}

const explanation = (c.debriefs || {})[qid]; const prompt = (RATIONALES.prompts || {})[c.promptId]; return
-
- Model rationale -

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}

-

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

- {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} -
Sources cited by this model - -
-
Original batch explanation and provenance - {explanation &&

{c.rationale}

} -

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

- Download original forecasts, explanations and logs -
+

Model rationale

+

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}

+

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

+ {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} +
Sources cited by this model + +
+
Original batch explanation and provenance + {explanation &&

{c.rationale}

} +

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

+ Download original forecasts, explanations and logs
View full prompt {prompt ? <> @@ -2627,7 +2634,7 @@

{title}

; } // `forecasts` supplies the chart's horizons and model roster, not its pooled -// probabilities. The comparison table uses only archived calls for the chosen +// probabilities. The forecast row uses only archived calls for the chosen // prompt condition, question, and displayed date. function ForecastRationales({ qid, context, forecasts = [], date, controls }) { const [selectedCondition, setSelectedCondition] = useState(""); @@ -2648,31 +2655,24 @@

{title}

const current = calls.find(([id]) => id === selectedCall) || calls[0]; const chosen = current && current[1]; const label = c => c.promptLabel || "Unrecorded"; - const rows = rationaleConditionForecasts(qid, forecasts.map(f => f.key), models, conditionCalls); + const rows = rationaleConditionForecasts(qid, forecasts.map(f => f.key), model ? [model] : [], conditionCalls); return
Model rationale for its forecasts {controls}

{context}

{!chosen ?

No explanation recorded for this forecast.

: <> -
- -
+ ({ key, label: label(allCalls.find(([, c]) => rationaleConditionKey(c) === key)[1]) }))} />

{label(chosen)} prompt

{chosen.promptVariant === "none" ? "No additional instruction." : chosen.promptInstruction || "Added instruction text is not available for this call."}

- - {conditions.length > 1 &&

The main charts average across prompt conditions; this table shows the selected condition.

} -
- - {calls.length > 1 && } -
+ ({ key: name, label: name }))} /> + {calls.length > 1 && ({ key: id, label: `Call ${index + 1}` }))} />} + + {conditions.length > 1 &&

The main charts average across prompt conditions; this row shows the selected condition.

} }
; diff --git a/web/demo/60-atoms.jsx b/web/demo/60-atoms.jsx index aa163d0..a887b55 100644 --- a/web/demo/60-atoms.jsx +++ b/web/demo/60-atoms.jsx @@ -93,11 +93,12 @@ function Stat({ label, value, color }) {
); } -function Toggle({ options, value, onChange }) { +function Toggle({ options, value, onChange, label }) { return ( -
+
{options.map(o => ( - ))} diff --git a/web/demo/65-rationales.jsx b/web/demo/65-rationales.jsx index cab2b64..3276aa5 100644 --- a/web/demo/65-rationales.jsx +++ b/web/demo/65-rationales.jsx @@ -31,22 +31,30 @@ function rationaleConditionForecasts(qid, horizons, models, calls) { return { key, values, counts }; }); } -function RationaleForecastTable({ rows, models, conditionLabel }) { - if (!rows.length || !models.length) return null; - const repeated = rows.some(row => models.some(model => row.counts[model] > 1)); +function RationaleToggle({ label, options, value, onChange }) { + return
+ {label} + +
; +} +function RationaleForecastTable({ rows, model, conditionLabel }) { + if (!rows.length || !model) return null; + const repeated = rows.some(row => row.counts[model] > 1); return
- {models.map(model => )} - {rows.map(row => - - {models.map(model => {rows.map(row => )} + + + {rows.map(row => )} - )} +
- {conditionLabel} prompt: forecasts by model + Forecasts for the {conditionLabel} prompt
Horizon{model}
{rationaleHorizonLabel(row.key)}
Model + {rationaleHorizonLabel(row.key).replace(/ \(by .*\)$/, "")} +
{model} 1 ? `Mean of ${row.counts[model]} recorded calls` : undefined}> {rationalePct(row.values[model])}
{repeated &&

Each cell averages the available repeat calls for that model and horizon within this prompt condition.

}
; @@ -55,19 +63,17 @@ function RationaleExplanation({ id, call: c, qid }) { const explanation = (c.debriefs || {})[qid]; const prompt = (RATIONALES.prompts || {})[c.promptId]; return
-
- Model rationale -

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}

-

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

- {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} -
Sources cited by this model - -
-
Original batch explanation and provenance - {explanation &&

{c.rationale}

} -

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

- Download original forecasts, explanations and logs -
+

Model rationale

+

{explanation ? "Model’s explanation for this question, recorded after its forecasts. Covers all horizons." : "Model’s original explanation for its full forecast batch; not specific to this question or horizon."}

+

{(explanation && explanation.rationale) || c.rationale || "No written rationale recorded."}

+ {explanation && explanation.weakest_link &&

Weakest assumption: {explanation.weakest_link}

} +
Sources cited by this model + +
+
Original batch explanation and provenance + {explanation &&

{c.rationale}

} +

Model: {c.modelId}
Elicited: {c.elicitedAt}
Run: {c.run}
Call: {id}
Instrument: {c.instrument}
Protocol: {c.protocol}
Prompt condition: {c.condition || "Not recorded"}
Scenario: unconditional

+ Download original forecasts, explanations and logs
View full prompt {prompt ? <> @@ -79,7 +85,7 @@ function RationaleExplanation({ id, call: c, qid }) {
; } // `forecasts` supplies the chart's horizons and model roster, not its pooled -// probabilities. The comparison table uses only archived calls for the chosen +// probabilities. The forecast row uses only archived calls for the chosen // prompt condition, question, and displayed date. function ForecastRationales({ qid, context, forecasts = [], date, controls }) { const [selectedCondition, setSelectedCondition] = useState(""); @@ -100,31 +106,24 @@ function ForecastRationales({ qid, context, forecasts = [], date, controls }) { const current = calls.find(([id]) => id === selectedCall) || calls[0]; const chosen = current && current[1]; const label = c => c.promptLabel || "Unrecorded"; - const rows = rationaleConditionForecasts(qid, forecasts.map(f => f.key), models, conditionCalls); + const rows = rationaleConditionForecasts(qid, forecasts.map(f => f.key), model ? [model] : [], conditionCalls); return
Model rationale for its forecasts {controls}

{context}

{!chosen ?

No explanation recorded for this forecast.

: <> -
- -
+ ({ key, label: label(allCalls.find(([, c]) => rationaleConditionKey(c) === key)[1]) }))} />

{label(chosen)} prompt

{chosen.promptVariant === "none" ? "No additional instruction." : chosen.promptInstruction || "Added instruction text is not available for this call."}

- - {conditions.length > 1 &&

The main charts average across prompt conditions; this table shows the selected condition.

} -
- - {calls.length > 1 && } -
+ ({ key: name, label: name }))} /> + {calls.length > 1 && ({ key: id, label: `Call ${index + 1}` }))} />} + + {conditions.length > 1 &&

The main charts average across prompt conditions; this row shows the selected condition.

} }
; From 9182ff0c0a710a42a471e490b4d9eaf30b10a8e8 Mon Sep 17 00:00:00 2001 From: Curtis Schaefer Date: Fri, 23 Oct 2026 15:11:09 -0400 Subject: [PATCH 5/5] Simplify selected-model forecast rendering and metadata --- index.html | 27 +++++++++++---------------- redlines/views/rationales.py | 1 - tests/test_rationale_prompts.py | 2 -- web/demo/65-rationales.jsx | 27 +++++++++++---------------- 4 files changed, 22 insertions(+), 35 deletions(-) diff --git a/index.html b/index.html index 255e397..1502832 100644 --- a/index.html +++ b/index.html @@ -2565,19 +2565,14 @@

{title}

})}; } const rationaleConditionKey = c => c.condition || c.promptVariant || "unrecorded"; -// One condition only. Raw call probabilities are fractions; convert to percent -// once, after averaging any repeat calls that answered this question/horizon. -function rationaleConditionForecasts(qid, horizons, models, calls) { +// Calls are already filtered to the selected model, condition, and date. +// Average repeat calls for each question/horizon, then convert to percent. +function rationaleConditionForecasts(qid, horizons, calls) { return horizons.map(key => { - const values = {}, counts = {}; - models.forEach(model => { - const ps = calls.filter(([, c]) => c.model === model) - .map(([, c]) => (((c.forecasts || {})[qid] || {})[key])) - .filter(p => typeof p === "number" && Number.isFinite(p)); - counts[model] = ps.length; - values[model] = ps.length ? 100 * ps.reduce((sum, p) => sum + p, 0) / ps.length : null; - }); - return { key, values, counts }; + const probabilities = calls.map(([, c]) => (((c.forecasts || {})[qid] || {})[key])) + .filter(p => typeof p === "number" && Number.isFinite(p)); + const count = probabilities.length; + return { key, count, value: count ? 100 * probabilities.reduce((sum, p) => sum + p, 0) / count : null }; }); } function RationaleToggle({ label, options, value, onChange }) { @@ -2588,7 +2583,7 @@

{title}

} function RationaleForecastTable({ rows, model, conditionLabel }) { if (!rows.length || !model) return null; - const repeated = rows.some(row => row.counts[model] > 1); + const repeated = rows.some(row => row.count > 1); return
{rows.map(row => )}
@@ -2600,8 +2595,8 @@

{title}

{model} 1 ? `Mean of ${row.counts[model]} recorded calls` : undefined}> - {rationalePct(row.values[model])} + title={row.count > 1 ? `Mean of ${row.count} recorded calls` : undefined}> + {rationalePct(row.value)}
@@ -2655,7 +2650,7 @@

Model rationale

const current = calls.find(([id]) => id === selectedCall) || calls[0]; const chosen = current && current[1]; const label = c => c.promptLabel || "Unrecorded"; - const rows = rationaleConditionForecasts(qid, forecasts.map(f => f.key), model ? [model] : [], conditionCalls); + const rows = rationaleConditionForecasts(qid, forecasts.map(f => f.key), calls); return
Model rationale for its forecasts {controls} diff --git a/redlines/views/rationales.py b/redlines/views/rationales.py index 8a6bd23..a30260b 100644 --- a/redlines/views/rationales.py +++ b/redlines/views/rationales.py @@ -39,7 +39,6 @@ def _prompt_metadata(row, recorded, prompts): 'promptVariant': variant, 'promptLabel': label, 'promptInstruction': instruction if instruction and full_prompt and instruction in full_prompt else None, 'promptId': prompt_id, - 'promptSha256': recorded.get('prompt_sha256') if recorded else None, } diff --git a/tests/test_rationale_prompts.py b/tests/test_rationale_prompts.py index e976c10..916e6e5 100644 --- a/tests/test_rationale_prompts.py +++ b/tests/test_rationale_prompts.py @@ -17,7 +17,6 @@ def call_rows(condition, variant, model="Example model"): "label": model, "model": model, "elicited_at": "2026-10-22T12:00:00Z", "question_id": "first", "condition": condition, "variant": variant, "prompt": prompt, "system_prompt": "Recorded system instruction.", - "prompt_sha256": "recorded-user-prompt-hash", "rationale": "Original batch rationale.", "key_sources": ["https://example.org/source"], "debrief": {"second": {"rationale": "Original question explanation."}}, @@ -66,7 +65,6 @@ def test_recovers_first_row_prompt_for_later_question(self): call = data["calls"][rows[1]["call_id"]] self.assertEqual(data["prompts"][call["promptId"]], {"system": rows[0]["system_prompt"], "user": rows[0]["prompt"]}) - self.assertEqual(call["promptSha256"], rows[0]["prompt_sha256"]) self.assertEqual(call["debriefs"], rows[0]["debrief"]) def test_shares_identical_prompts_but_keeps_different_system_instructions(self): diff --git a/web/demo/65-rationales.jsx b/web/demo/65-rationales.jsx index 3276aa5..1d965ca 100644 --- a/web/demo/65-rationales.jsx +++ b/web/demo/65-rationales.jsx @@ -16,19 +16,14 @@ function RationaleSources({ sources }) { })}; } const rationaleConditionKey = c => c.condition || c.promptVariant || "unrecorded"; -// One condition only. Raw call probabilities are fractions; convert to percent -// once, after averaging any repeat calls that answered this question/horizon. -function rationaleConditionForecasts(qid, horizons, models, calls) { +// Calls are already filtered to the selected model, condition, and date. +// Average repeat calls for each question/horizon, then convert to percent. +function rationaleConditionForecasts(qid, horizons, calls) { return horizons.map(key => { - const values = {}, counts = {}; - models.forEach(model => { - const ps = calls.filter(([, c]) => c.model === model) - .map(([, c]) => (((c.forecasts || {})[qid] || {})[key])) - .filter(p => typeof p === "number" && Number.isFinite(p)); - counts[model] = ps.length; - values[model] = ps.length ? 100 * ps.reduce((sum, p) => sum + p, 0) / ps.length : null; - }); - return { key, values, counts }; + const probabilities = calls.map(([, c]) => (((c.forecasts || {})[qid] || {})[key])) + .filter(p => typeof p === "number" && Number.isFinite(p)); + const count = probabilities.length; + return { key, count, value: count ? 100 * probabilities.reduce((sum, p) => sum + p, 0) / count : null }; }); } function RationaleToggle({ label, options, value, onChange }) { @@ -39,7 +34,7 @@ function RationaleToggle({ label, options, value, onChange }) { } function RationaleForecastTable({ rows, model, conditionLabel }) { if (!rows.length || !model) return null; - const repeated = rows.some(row => row.counts[model] > 1); + const repeated = rows.some(row => row.count > 1); return
{rows.map(row => )}
@@ -51,8 +46,8 @@ function RationaleForecastTable({ rows, model, conditionLabel }) {
{model} 1 ? `Mean of ${row.counts[model]} recorded calls` : undefined}> - {rationalePct(row.values[model])} + title={row.count > 1 ? `Mean of ${row.count} recorded calls` : undefined}> + {rationalePct(row.value)}
@@ -106,7 +101,7 @@ function ForecastRationales({ qid, context, forecasts = [], date, controls }) { const current = calls.find(([id]) => id === selectedCall) || calls[0]; const chosen = current && current[1]; const label = c => c.promptLabel || "Unrecorded"; - const rows = rationaleConditionForecasts(qid, forecasts.map(f => f.key), model ? [model] : [], conditionCalls); + const rows = rationaleConditionForecasts(qid, forecasts.map(f => f.key), calls); return
Model rationale for its forecasts {controls}