feat: add five reproducible synthetic pilot tasks

aicp/synthetic_fixtures.py

@@ -0,0 +1,61 @@
+"""Deterministic synthetic pilot fixtures; held-out tests stay outside agent workspace."""
+from pathlib import Path
+
+TASKS = {
+    "bug-fix": {
+        "objective": "Fix clamp so values below lower bound return lower and above upper return upper.",
+        "starter": "def clamp(value, low, high):\n    return min(low, max(value, high))\n",
+        "tests": "from app import clamp\nassert clamp(2, 0, 10) == 2\nassert clamp(-3, 0, 10) == 0\nassert clamp(20, 0, 10) == 10\n",
+    },
+    "test-addition": {
+        "objective": "Add tests/test_app.py using unittest for is_even, including negative, zero, and odd inputs. Do not modify app.py.",
+        "starter": "def is_even(number):\n    return number % 2 == 0\n",
+        "tests": "from pathlib import Path\nimport unittest\np = Path('tests/test_app.py')\nassert p.is_file()\nsource = p.read_text()\nassert 'unittest' in source and 'assert' in source\nsuite = unittest.defaultTestLoader.discover('tests')\nresult = unittest.TextTestRunner(verbosity=0).run(suite)\nassert result.wasSuccessful() and result.testsRun >= 3\n",
+    },
+    "refactoring": {
+        "objective": "Refactor total_cost to remove the explicit loop while preserving results and signature.",
+        "starter": "def total_cost(prices):\n    total = 0\n    for price in prices:\n        total += price\n    return total\n",
+        "tests": "from app import total_cost\nimport ast\nfrom pathlib import Path\nassert total_cost([]) == 0\nassert total_cost([2, 3, 5]) == 10\ntree = ast.parse(Path('app.py').read_text())\nfn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'total_cost')\nassert not any(isinstance(n, (ast.For, ast.While)) for n in ast.walk(fn))\n",
+    },
+    "documentation": {
+        "objective": "Add README.md explaining how to call slugify, its input and output, and one executable Python example.",
+        "starter": "def slugify(text):\n    return '-'.join(text.lower().split())\n",
+        "tests": "from pathlib import Path\np = Path('README.md')\nassert p.is_file()\ns = p.read_text().lower()\nassert 'slugify' in s and 'python' in s and 'hello' in s\n",
+    },
+    "feature": {
+        "objective": "Add unique_sorted(values) to app.py; return sorted distinct values without modifying the input.",
+        "starter": "def identity(value):\n    return value\n",
+        "tests": "from app import unique_sorted\nvalues = [3, 1, 3, 2]\nassert unique_sorted(values) == [1, 2, 3]\nassert values == [3, 1, 3, 2]\nassert unique_sorted([]) == []\n",
+    },
+}
+
+
+def materialize_fixture(name, workspace, evaluator):
+    if name not in TASKS:
+        raise ValueError("Unknown synthetic task")
+    workspace = Path(workspace)
+    evaluator = Path(evaluator)
+    if workspace.resolve() == evaluator.resolve() or workspace.exists() or evaluator.exists():
+        raise ValueError("Workspace and evaluator must be distinct, absent destinations")
+    task = TASKS[name]
+    workspace.mkdir(parents=True)
+    evaluator.mkdir(parents=True)
+    (workspace / "app.py").write_text(task["starter"], encoding="utf-8")
+    (workspace / "TASK.md").write_text(task["objective"] + "\n", encoding="utf-8")
+    (evaluator / "acceptance.py").write_text(task["tests"], encoding="utf-8")
+    return {"task": name, "workspace": str(workspace), "evaluator": str(evaluator)}
+
+
+def main():
+    import argparse
+    import json
+    parser = argparse.ArgumentParser(description="Materialize a synthetic benchmark fixture")
+    parser.add_argument("--task", choices=sorted(TASKS), required=True)
+    parser.add_argument("--workspace", type=Path, required=True)
+    parser.add_argument("--evaluator", type=Path, required=True)
+    args = parser.parse_args()
+    print(json.dumps(materialize_fixture(args.task, args.workspace, args.evaluator), indent=2))
+
+
+if __name__ == "__main__":
+    main()

evaluations/experiments/README.md

@@ -0,0 +1,24 @@
+# Synthetic pilot task suite
+
+This is an **experimental personal side project**, not a validated productivity study.
+
+The suite provides five small Python tasks: bug fixing, test addition, refactoring, documentation, and feature development. Each task has a starter workspace and a separate acceptance script.
+
+## Create a fixture
+
+```sh
+python3 -m aicp.synthetic_fixtures --task bug-fix \
+  --workspace /tmp/aicp-pilot/workspace \
+  --evaluator /tmp/aicp-pilot/evaluator
+```
+
+The two output paths must be distinct and must not exist already. Provide only the `workspace` directory to the agent; **never mount the evaluator directory**. After agent execution, run `acceptance.py` from the workspace using a trusted process that has access to the evaluator script. This is a manual pilot workflow, not yet a fully isolated automated harness.
+
+For example, in a disposable synthetic environment:
+
+```sh
+cd /tmp/aicp-pilot/workspace
+python3 ../evaluator/acceptance.py
+```
+
+The tasks are deliberately small and the acceptance tests are minimal. In particular, the test-addition and documentation checks are structural smoke tests, not strong measures of engineering quality. They should be improved and independently reviewed before making productivity claims. The task definitions and evaluator source live in the same public repository for transparency; the evaluation environment must keep them out of the agent's runtime context. Follow the [experimental protocol](../../docs/EXPERIMENTAL_PROTOCOL.md) and report all failures and limitations.

tests/test_synthetic_fixtures.py

@@ -0,0 +1,39 @@
+"""Synthetic task fixture isolation and validity tests."""
+from pathlib import Path
+import tempfile
+import unittest
+
+from aicp.synthetic_fixtures import TASKS, materialize_fixture
+
+
+class SyntheticFixtureTests(unittest.TestCase):
+    def test_five_distinct_tasks_materialize(self):
+        self.assertEqual(set(TASKS), {"bug-fix", "test-addition", "refactoring", "documentation", "feature"})
+        for name in TASKS:
+            with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                workspace, evaluator = root / "workspace", root / "evaluator"
+                materialize_fixture(name, workspace, evaluator)
+                self.assertTrue((workspace / "app.py").is_file())
+                self.assertTrue((workspace / "TASK.md").is_file())
+                self.assertTrue((evaluator / "acceptance.py").is_file())
+                self.assertFalse((workspace / "acceptance.py").exists())
+                compile((workspace / "app.py").read_text(encoding="utf-8"), "app.py", "exec")
+                compile((evaluator / "acceptance.py").read_text(encoding="utf-8"), "acceptance.py", "exec")
+
+    def test_never_overwrites_workspace(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / "workspace").mkdir()
+            with self.assertRaises(ValueError):
+                materialize_fixture("bug-fix", root / "workspace", root / "evaluator")
+
+    def test_unknown_task_rejected(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            with self.assertRaises(ValueError):
+                materialize_fixture("unknown", root / "workspace", root / "evaluator")
+
+
+if __name__ == "__main__":
+    unittest.main()