diff --git a/aicp/results_analysis.py b/aicp/results_analysis.py new file mode 100644 index 0000000..9c8adb6 --- /dev/null +++ b/aicp/results_analysis.py @@ -0,0 +1,84 @@ +"""Descriptive, paired summaries for recorded experimental runs. + +No statistical significance or causal claims are inferred. +""" +import json +from pathlib import Path +from statistics import mean + +REQUIRED = {"task", "repetition", "condition", "passed", "duration_seconds", "human_interventions", "review_minutes", "cost_usd"} + + +def analyze_runs(runs): + if not isinstance(runs, list) or not runs: + raise ValueError("Expected nonempty experimental runs") + pairs = {} + for run in runs: + if not isinstance(run, dict) or set(run) != REQUIRED: + raise ValueError("Invalid run fields") + if not isinstance(run["task"], str) or not run["task"].strip(): + raise ValueError("Invalid task") + if type(run["repetition"]) is not int or run["repetition"] < 1: + raise ValueError("Invalid repetition") + if run["condition"] not in ("baseline", "playbook") or type(run["passed"]) is not bool: + raise ValueError("Invalid condition or pass status") + for key in ("duration_seconds", "human_interventions", "review_minutes", "cost_usd"): + value = run[key] + if value is not None and (type(value) not in (int, float) or value < 0): + raise ValueError("Invalid metric: " + key) + key = (run["task"], run["repetition"]) + pair = pairs.setdefault(key, {}) + if run["condition"] in pair: + raise ValueError("Duplicate task/repetition/condition") + pair[run["condition"]] = run + + complete = [pair for pair in pairs.values() if set(pair) == {"baseline", "playbook"}] + if not complete: + raise ValueError("No complete baseline/playbook pairs") + summary = { + "schema_version": 1, + "complete_pairs": len(complete), + "incomplete_pairs": len(pairs) - len(complete), + "conditions": {}, + "paired_pass_rate_difference": mean( + int(p["playbook"]["passed"]) - int(p["baseline"]["passed"]) for p in complete + ), + "limitations": "descriptive-only; paired observations are not proof of causality or statistical significance", + } + for condition in ("baseline", "playbook"): + rows = [p[condition] for p in complete] + result = {"runs": len(rows), "passed": sum(r["passed"] for r in rows), "pass_rate": mean(r["passed"] for r in rows)} + for key in ("duration_seconds", "human_interventions", "review_minutes", "cost_usd"): + values = [r[key] for r in rows if r[key] is not None] + result["mean_" + key] = mean(values) if values else None + result["observed_" + key] = len(values) + summary["conditions"][condition] = result + return summary + + +def analyze_file(path): + path = Path(path) + if path.is_symlink() or not path.is_file(): + raise ValueError("Results file must be a regular file") + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict) or set(data) != {"schema_version", "runs"} or data["schema_version"] != 1: + raise ValueError("Invalid results file") + return analyze_runs(data["runs"]) + + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Summarize recorded experimental runs") + parser.add_argument("results", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + report = json.dumps(analyze_file(args.results), indent=2) + "\n" + if args.output: + with args.output.open("x", encoding="utf-8") as handle: + handle.write(report) + else: + print(report, end="") + + +if __name__ == "__main__": + main() diff --git a/evaluations/experiments/RESULTS.md b/evaluations/experiments/RESULTS.md new file mode 100644 index 0000000..c3674c9 --- /dev/null +++ b/evaluations/experiments/RESULTS.md @@ -0,0 +1,30 @@ +# Experimental results analysis + +**No live agent benchmark results have been collected or published by this project yet.** This analyzer reports descriptive statistics from explicitly supplied observations. It does not execute agents, measure elapsed time itself, calculate confidence intervals, or establish causality. + +## Input + +Prepare a JSON file containing `{"schema_version": 1, "runs": [...]}`. Each run has exactly these fields: + +| Field | Type | Meaning | +|---|---|---| +| `task` | string | Stable synthetic task identifier | +| `repetition` | positive integer | Matched run index | +| `condition` | string | `baseline` or `playbook` | +| `passed` | boolean | Independent held-out acceptance outcome | +| `duration_seconds` | nonnegative number or null | Observed elapsed time | +| `human_interventions` | nonnegative number or null | Observed intervention count | +| `review_minutes` | nonnegative number or null | Observed review effort | +| `cost_usd` | nonnegative number or null | Observed monetary cost | + +Unknown telemetry must be `null`, **not zero**. Task and repetition identify a pair. The analyzer rejects duplicate observations and summarizes complete pairs only; it counts incomplete pairs separately. + +## Run + +```sh +python3 -m aicp.results_analysis /path/to/observations.json --output /tmp/aicp-analysis.json +``` + +The output contains pass rates, paired pass-rate difference (playbook minus baseline), mean observed metrics and observation counts. These are descriptive statistics; they are not a statistical significance test or proof that the playbook caused an improvement. + +Follow the [experimental protocol](../../docs/EXPERIMENTAL_PROTOCOL.md) for randomized or counterbalanced experiments, independent acceptance checks, and transparent reporting of failures. Publish the exact environment, fixtures, agent/model versions, all raw outcomes, and limitations before drawing conclusions. diff --git a/tests/test_results_analysis.py b/tests/test_results_analysis.py new file mode 100644 index 0000000..ed3de16 --- /dev/null +++ b/tests/test_results_analysis.py @@ -0,0 +1,50 @@ +"""Descriptive experiment results tests; data is illustrative, not measured.""" +import unittest + +from aicp.results_analysis import analyze_runs + + +def run(condition, passed, repetition=1, duration=10.0): + return { + "task": "bug-fix", "repetition": repetition, "condition": condition, + "passed": passed, "duration_seconds": duration, + "human_interventions": 0, "review_minutes": None, "cost_usd": None, + } + + +class ResultsAnalysisTests(unittest.TestCase): + def test_paired_rates_and_missing_telemetry(self): + report = analyze_runs([ + run("baseline", False, 1, 20), + run("playbook", True, 1, 10), + run("baseline", True, 2, 12), + run("playbook", True, 2, 14), + ]) + self.assertEqual(report["complete_pairs"], 2) + self.assertEqual(report["paired_pass_rate_difference"], 0.5) + self.assertEqual(report["conditions"]["baseline"]["pass_rate"], 0.5) + self.assertIsNone(report["conditions"]["playbook"]["mean_cost_usd"]) + self.assertEqual(report["conditions"]["baseline"]["observed_cost_usd"], 0) + + def test_incomplete_pairs_not_scored(self): + report = analyze_runs([run("baseline", False), run("playbook", True), run("baseline", True, 2)]) + self.assertEqual(report["complete_pairs"], 1) + self.assertEqual(report["incomplete_pairs"], 1) + + def test_duplicates_rejected(self): + with self.assertRaises(ValueError): + analyze_runs([run("baseline", True), run("baseline", False), run("playbook", True)]) + + def test_missing_pairs_rejected(self): + with self.assertRaises(ValueError): + analyze_runs([run("baseline", True)]) + + def test_negative_metric_rejected(self): + row = run("baseline", True) + row["cost_usd"] = -1 + with self.assertRaises(ValueError): + analyze_runs([row, run("playbook", True)]) + + +if __name__ == "__main__": + unittest.main()