feat: add paired experimental results analysis

aicp/results_analysis.py

@@ -0,0 +1,84 @@
+"""Descriptive, paired summaries for recorded experimental runs.
+
+No statistical significance or causal claims are inferred.
+"""
+import json
+from pathlib import Path
+from statistics import mean
+
+REQUIRED = {"task", "repetition", "condition", "passed", "duration_seconds", "human_interventions", "review_minutes", "cost_usd"}
+
+
+def analyze_runs(runs):
+    if not isinstance(runs, list) or not runs:
+        raise ValueError("Expected nonempty experimental runs")
+    pairs = {}
+    for run in runs:
+        if not isinstance(run, dict) or set(run) != REQUIRED:
+            raise ValueError("Invalid run fields")
+        if not isinstance(run["task"], str) or not run["task"].strip():
+            raise ValueError("Invalid task")
+        if type(run["repetition"]) is not int or run["repetition"] < 1:
+            raise ValueError("Invalid repetition")
+        if run["condition"] not in ("baseline", "playbook") or type(run["passed"]) is not bool:
+            raise ValueError("Invalid condition or pass status")
+        for key in ("duration_seconds", "human_interventions", "review_minutes", "cost_usd"):
+            value = run[key]
+            if value is not None and (type(value) not in (int, float) or value < 0):
+                raise ValueError("Invalid metric: " + key)
+        key = (run["task"], run["repetition"])
+        pair = pairs.setdefault(key, {})
+        if run["condition"] in pair:
+            raise ValueError("Duplicate task/repetition/condition")
+        pair[run["condition"]] = run
+
+    complete = [pair for pair in pairs.values() if set(pair) == {"baseline", "playbook"}]
+    if not complete:
+        raise ValueError("No complete baseline/playbook pairs")
+    summary = {
+        "schema_version": 1,
+        "complete_pairs": len(complete),
+        "incomplete_pairs": len(pairs) - len(complete),
+        "conditions": {},
+        "paired_pass_rate_difference": mean(
+            int(p["playbook"]["passed"]) - int(p["baseline"]["passed"]) for p in complete
+        ),
+        "limitations": "descriptive-only; paired observations are not proof of causality or statistical significance",
+    }
+    for condition in ("baseline", "playbook"):
+        rows = [p[condition] for p in complete]
+        result = {"runs": len(rows), "passed": sum(r["passed"] for r in rows), "pass_rate": mean(r["passed"] for r in rows)}
+        for key in ("duration_seconds", "human_interventions", "review_minutes", "cost_usd"):
+            values = [r[key] for r in rows if r[key] is not None]
+            result["mean_" + key] = mean(values) if values else None
+            result["observed_" + key] = len(values)
+        summary["conditions"][condition] = result
+    return summary
+
+
+def analyze_file(path):
+    path = Path(path)
+    if path.is_symlink() or not path.is_file():
+        raise ValueError("Results file must be a regular file")
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(data, dict) or set(data) != {"schema_version", "runs"} or data["schema_version"] != 1:
+        raise ValueError("Invalid results file")
+    return analyze_runs(data["runs"])
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Summarize recorded experimental runs")
+    parser.add_argument("results", type=Path)
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+    report = json.dumps(analyze_file(args.results), indent=2) + "\n"
+    if args.output:
+        with args.output.open("x", encoding="utf-8") as handle:
+            handle.write(report)
+    else:
+        print(report, end="")
+
+
+if __name__ == "__main__":
+    main()

evaluations/experiments/RESULTS.md

@@ -0,0 +1,30 @@
+# Experimental results analysis
+
+**No live agent benchmark results have been collected or published by this project yet.** This analyzer reports descriptive statistics from explicitly supplied observations. It does not execute agents, measure elapsed time itself, calculate confidence intervals, or establish causality.
+
+## Input
+
+Prepare a JSON file containing `{"schema_version": 1, "runs": [...]}`. Each run has exactly these fields:
+
+| Field | Type | Meaning |
+|---|---|---|
+| `task` | string | Stable synthetic task identifier |
+| `repetition` | positive integer | Matched run index |
+| `condition` | string | `baseline` or `playbook` |
+| `passed` | boolean | Independent held-out acceptance outcome |
+| `duration_seconds` | nonnegative number or null | Observed elapsed time |
+| `human_interventions` | nonnegative number or null | Observed intervention count |
+| `review_minutes` | nonnegative number or null | Observed review effort |
+| `cost_usd` | nonnegative number or null | Observed monetary cost |
+
+Unknown telemetry must be `null`, **not zero**. Task and repetition identify a pair. The analyzer rejects duplicate observations and summarizes complete pairs only; it counts incomplete pairs separately.
+
+## Run
+
+```sh
+python3 -m aicp.results_analysis /path/to/observations.json --output /tmp/aicp-analysis.json
+```
+
+The output contains pass rates, paired pass-rate difference (playbook minus baseline), mean observed metrics and observation counts. These are descriptive statistics; they are not a statistical significance test or proof that the playbook caused an improvement.
+
+Follow the [experimental protocol](../../docs/EXPERIMENTAL_PROTOCOL.md) for randomized or counterbalanced experiments, independent acceptance checks, and transparent reporting of failures. Publish the exact environment, fixtures, agent/model versions, all raw outcomes, and limitations before drawing conclusions.

tests/test_results_analysis.py

@@ -0,0 +1,50 @@
+"""Descriptive experiment results tests; data is illustrative, not measured."""
+import unittest
+
+from aicp.results_analysis import analyze_runs
+
+
+def run(condition, passed, repetition=1, duration=10.0):
+    return {
+        "task": "bug-fix", "repetition": repetition, "condition": condition,
+        "passed": passed, "duration_seconds": duration,
+        "human_interventions": 0, "review_minutes": None, "cost_usd": None,
+    }
+
+
+class ResultsAnalysisTests(unittest.TestCase):
+    def test_paired_rates_and_missing_telemetry(self):
+        report = analyze_runs([
+            run("baseline", False, 1, 20),
+            run("playbook", True, 1, 10),
+            run("baseline", True, 2, 12),
+            run("playbook", True, 2, 14),
+        ])
+        self.assertEqual(report["complete_pairs"], 2)
+        self.assertEqual(report["paired_pass_rate_difference"], 0.5)
+        self.assertEqual(report["conditions"]["baseline"]["pass_rate"], 0.5)
+        self.assertIsNone(report["conditions"]["playbook"]["mean_cost_usd"])
+        self.assertEqual(report["conditions"]["baseline"]["observed_cost_usd"], 0)
+
+    def test_incomplete_pairs_not_scored(self):
+        report = analyze_runs([run("baseline", False), run("playbook", True), run("baseline", True, 2)])
+        self.assertEqual(report["complete_pairs"], 1)
+        self.assertEqual(report["incomplete_pairs"], 1)
+
+    def test_duplicates_rejected(self):
+        with self.assertRaises(ValueError):
+            analyze_runs([run("baseline", True), run("baseline", False), run("playbook", True)])
+
+    def test_missing_pairs_rejected(self):
+        with self.assertRaises(ValueError):
+            analyze_runs([run("baseline", True)])
+
+    def test_negative_metric_rejected(self):
+        row = run("baseline", True)
+        row["cost_usd"] = -1
+        with self.assertRaises(ValueError):
+            analyze_runs([row, run("playbook", True)])
+
+
+if __name__ == "__main__":
+    unittest.main()