fix(report): an execution with no scores is the worst one, not the best

resources/js/components/report/PrecisionPanel.tsx

@@ -27,8 +27,14 @@ const PrecisionPanel = ({ precision, aggregates }: Props) => {
   }
 
   const unstable = unstableAggregates(aggregates);
-  const resolution = precision.resolution;
-  const resolvable = precision.target_resolvable === true;
+
+  // The run-level block is a documented shape of the same payload, so a report
+  // that records its statistics there must not read as "no statistics".
+  const resolution = precision.resolution ?? precision.run?.resolution;
+  const resolvable =
+    typeof precision.target_resolvable === 'boolean'
+      ? precision.target_resolvable
+      : precision.run?.target_resolvable;
 
   return (
     <div className="space-y-4">
@@ -47,11 +53,17 @@ const PrecisionPanel = ({ precision, aggregates }: Props) => {
           <p className="text-2xl font-semibold">
             {typeof precision.target_delta === 'number' ? formatPercent(precision.target_delta) : '—'}
           </p>
-          <p className={`mt-1 text-xs ${resolvable ? 'text-emerald-600' : 'text-amber-600'}`}>
-            {resolvable
-              ? t('text_target_resolvable')
-              : t('text_target_not_resolvable').replace(':n', String(precision.required_repetitions ?? '—'))}
-          </p>
+          {/* Three states, not two: claiming "not detectable" when the run
+              simply did not record it is an assertion nobody made. */}
+          {resolvable === undefined ? (
+            <p className="mt-1 text-xs text-slate-500">{t('text_target_unknown')}</p>
+          ) : (
+            <p className={`mt-1 text-xs ${resolvable ? 'text-emerald-600' : 'text-amber-600'}`}>
+              {resolvable
+                ? t('text_target_resolvable')
+                : t('text_target_not_resolvable').replace(':n', String(precision.required_repetitions ?? '—'))}
+            </p>
+          )}
         </div>
       </div>
 

resources/js/components/report/reportPanels.test.tsx

@@ -108,6 +108,20 @@ describe('reportBlocks', () => {
     expect(worstExecutionOf(samples, 'refund-window')?.repetition).toBe(0);
     expect(worstExecutionOf(samples, 'nothing-here')).toBeNull();
   });
+
+  /**
+   * An execution with no scores is an errored one, and it is the most
+   * interesting thing that can have happened to the row. Ranking it best would
+   * show a successful sibling on a row the reader opened because it failed.
+   */
+  it('ranks an unscored execution as the worst, not the best', () => {
+    const withError: SampleExecution[] = [
+      { id: 'r', repetition: 0, actual_output: 'a good answer', scores: { 'exact-match': { score: 1 } } },
+      { id: 'r', repetition: 1, actual_output: '' },
+    ];
+
+    expect(worstExecutionOf(withError, 'r')?.repetition).toBe(1);
+  });
 });
 
 describe('PrecisionPanel', () => {
@@ -141,6 +155,33 @@ describe('PrecisionPanel', () => {
     expect(screen.queryByText('clean-row')).not.toBeInTheDocument();
   });
 
+  /**
+   * The run block is a documented shape of the same payload; a report that
+   * records its statistics there must not read as "no statistics".
+   */
+  it('falls back to the nested run block', () => {
+    render(
+      <PrecisionPanel
+        precision={{ repetitions: 5, run: { observations: 50, resolution: 0.12, target_resolvable: true } }}
+        aggregates={aggregates}
+      />,
+    );
+
+    expect(screen.getByText('12.0%')).toBeInTheDocument();
+    expect(screen.getByText(/can detect the difference/i)).toBeInTheDocument();
+  });
+
+  /**
+   * "Not detectable" is an assertion. A run that never recorded it made no
+   * such assertion, and saying so is a third state, not the negative one.
+   */
+  it('does not claim a difference is undetectable when the run never said', () => {
+    render(<PrecisionPanel precision={{ repetitions: 1 }} aggregates={aggregates} />);
+
+    expect(screen.getByText(/did not record whether/i)).toBeInTheDocument();
+    expect(screen.queryByText(/repetitions would be needed/i)).not.toBeInTheDocument();
+  });
+
   it('degrades to an explanation rather than to zeroes', () => {
     render(<PrecisionPanel precision={null} aggregates={[]} />);
 

resources/js/i18n/messages.ts

@@ -142,6 +142,7 @@ const catalog: Record<I18nLocale, MessageCatalog> = {
     text_cost_help: 'Declare token rates under eval-harness.costs.models to price a run.',
     text_resolution_help: 'Differences smaller than this are noise, not signal.',
     text_target_resolvable: 'This run can detect the difference you are gating on.',
+    text_target_unknown: 'This run did not record whether that difference is detectable.',
     text_target_not_resolvable: 'Not detectable at this sample size — :n repetitions would be needed.',
     text_unstable_help: 'Rows that disagree with themselves across repetitions: the ones that fail builds nobody broke.',
     text_no_unstable_rows: 'No row disagreed with itself.',
@@ -291,6 +292,7 @@ const catalog: Record<I18nLocale, MessageCatalog> = {
     text_cost_help: 'Dichiara le tariffe token in eval-harness.costs.models per attribuire un costo alla run.',
     text_resolution_help: 'Differenze piu piccole di questa sono rumore, non segnale.',
     text_target_resolvable: 'Questa run rileva la differenza su cui stai gatando.',
+    text_target_unknown: 'Questa run non ha registrato se quella differenza sia rilevabile.',
     text_target_not_resolvable: 'Non rilevabile con questo numero di campioni: servirebbero :n ripetizioni.',
     text_unstable_help: 'Righe che non concordano con se stesse tra le ripetizioni: quelle che fanno fallire build che nessuno ha rotto.',
     text_no_unstable_rows: 'Nessuna riga in disaccordo con se stessa.',

resources/js/utils/reportBlocks.ts

@@ -78,12 +78,17 @@ export const worstExecutionOf = (samples: SampleExecution[], sampleId: string):
     return null;
   }
 
+  // An execution with no scores at all is an *errored* one, and it is the most
+  // interesting thing that can have happened to a row. Ranking it as the best
+  // candidate (positive infinity) hid it behind a sibling execution that
+  // happened to score — showing a successful answer on a row the reader opened
+  // precisely because it failed.
   const meanScore = (sample: SampleExecution): number => {
     const scores = Object.values(sample.scores ?? {})
       .map((entry) => entry?.score)
       .filter((score): score is number => typeof score === 'number');
 
-    return scores.length === 0 ? Number.POSITIVE_INFINITY : scores.reduce((a, b) => a + b, 0) / scores.length;
+    return scores.length === 0 ? Number.NEGATIVE_INFINITY : scores.reduce((a, b) => a + b, 0) / scores.length;
   };
 
   return candidates.reduce((worst, sample) => (meanScore(sample) < meanScore(worst) ? sample : worst));