From 2ef258f14ba77aadb9c55036a7c2de3baea452b4 Mon Sep 17 00:00:00 2001
From: Ioannis Polyzos <git@ipolyzos.com>
Date: Sat, 10 Oct 2026 16:20:31 +0100
Subject: [PATCH 1/4] feat: define five synthetic tasks with separated
 acceptance checks

---
 aicp/synthetic_fixtures.py | 46 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 aicp/synthetic_fixtures.py

diff --git a/aicp/synthetic_fixtures.py b/aicp/synthetic_fixtures.py
new file mode 100644
index 0000000..70ca100
--- /dev/null
+++ b/aicp/synthetic_fixtures.py
@@ -0,0 +1,46 @@
+"""Deterministic synthetic pilot fixtures; held-out tests stay outside agent workspace."""
+from pathlib import Path
+
+TASKS = {
+    "bug-fix": {
+        "objective": "Fix clamp so values below lower bound return lower and above upper return upper.",
+        "starter": "def clamp(value, low, high):\n    return min(low, max(value, high))\n",
+        "tests": "from app import clamp\nassert clamp(2, 0, 10) == 2\nassert clamp(-3, 0, 10) == 0\nassert clamp(20, 0, 10) == 10\n",
+    },
+    "test-addition": {
+        "objective": "Add tests/test_app.py using unittest for is_even, including negative, zero, and odd inputs. Do not modify app.py.",
+        "starter": "def is_even(number):\n    return number % 2 == 0\n",
+        "tests": "from pathlib import Path\nimport unittest\np = Path('tests/test_app.py')\nassert p.is_file()\nsource = p.read_text()\nassert 'unittest' in source and 'assert' in source\nsuite = unittest.defaultTestLoader.discover('tests')\nresult = unittest.TextTestRunner(verbosity=0).run(suite)\nassert result.wasSuccessful() and result.testsRun >= 3\n",
+    },
+    "refactoring": {
+        "objective": "Refactor total_cost to remove the explicit loop while preserving results and signature.",
+        "starter": "def total_cost(prices):\n    total = 0\n    for price in prices:\n        total += price\n    return total\n",
+        "tests": "from app import total_cost\nimport ast\nfrom pathlib import Path\nassert total_cost([]) == 0\nassert total_cost([2, 3, 5]) == 10\ntree = ast.parse(Path('app.py').read_text())\nfn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'total_cost')\nassert not any(isinstance(n, (ast.For, ast.While)) for n in ast.walk(fn))\n",
+    },
+    "documentation": {
+        "objective": "Add README.md explaining how to call slugify, its input and output, and one executable Python example.",
+        "starter": "def slugify(text):\n    return '-'.join(text.lower().split())\n",
+        "tests": "from pathlib import Path\np = Path('README.md')\nassert p.is_file()\ns = p.read_text().lower()\nassert 'slugify' in s and 'python' in s and 'hello' in s\n",
+    },
+    "feature": {
+        "objective": "Add unique_sorted(values) to app.py; return sorted distinct values without modifying the input.",
+        "starter": "def identity(value):\n    return value\n",
+        "tests": "from app import unique_sorted\nvalues = [3, 1, 3, 2]\nassert unique_sorted(values) == [1, 2, 3]\nassert values == [3, 1, 3, 2]\nassert unique_sorted([]) == []\n",
+    },
+}
+
+
+def materialize_fixture(name, workspace, evaluator):
+    if name not in TASKS:
+        raise ValueError("Unknown synthetic task")
+    workspace = Path(workspace)
+    evaluator = Path(evaluator)
+    if workspace.resolve() == evaluator.resolve() or workspace.exists() or evaluator.exists():
+        raise ValueError("Workspace and evaluator must be distinct, absent destinations")
+    task = TASKS[name]
+    workspace.mkdir(parents=True)
+    evaluator.mkdir(parents=True)
+    (workspace / "app.py").write_text(task["starter"], encoding="utf-8")
+    (workspace / "TASK.md").write_text(task["objective"] + "\n", encoding="utf-8")
+    (evaluator / "acceptance.py").write_text(task["tests"], encoding="utf-8")
+    return {"task": name, "workspace": str(workspace), "evaluator": str(evaluator)}

From 1bf9dbaab3131fa2045bed0e71a4fdafc371e82d Mon Sep 17 00:00:00 2001
From: Ioannis Polyzos <git@ipolyzos.com>
Date: Sat, 10 Oct 2026 16:20:38 +0100
Subject: [PATCH 2/4] test: validate synthetic task generation and evaluator
 separation

---
 tests/test_synthetic_fixtures.py | 39 ++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 tests/test_synthetic_fixtures.py

diff --git a/tests/test_synthetic_fixtures.py b/tests/test_synthetic_fixtures.py
new file mode 100644
index 0000000..c8966f5
--- /dev/null
+++ b/tests/test_synthetic_fixtures.py
@@ -0,0 +1,39 @@
+"""Synthetic task fixture isolation and validity tests."""
+from pathlib import Path
+import tempfile
+import unittest
+
+from aicp.synthetic_fixtures import TASKS, materialize_fixture
+
+
+class SyntheticFixtureTests(unittest.TestCase):
+    def test_five_distinct_tasks_materialize(self):
+        self.assertEqual(set(TASKS), {"bug-fix", "test-addition", "refactoring", "documentation", "feature"})
+        for name in TASKS:
+            with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                workspace, evaluator = root / "workspace", root / "evaluator"
+                materialize_fixture(name, workspace, evaluator)
+                self.assertTrue((workspace / "app.py").is_file())
+                self.assertTrue((workspace / "TASK.md").is_file())
+                self.assertTrue((evaluator / "acceptance.py").is_file())
+                self.assertFalse((workspace / "acceptance.py").exists())
+                compile((workspace / "app.py").read_text(encoding="utf-8"), "app.py", "exec")
+                compile((evaluator / "acceptance.py").read_text(encoding="utf-8"), "acceptance.py", "exec")
+
+    def test_never_overwrites_workspace(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / "workspace").mkdir()
+            with self.assertRaises(ValueError):
+                materialize_fixture("bug-fix", root / "workspace", root / "evaluator")
+
+    def test_unknown_task_rejected(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            with self.assertRaises(ValueError):
+                materialize_fixture("unknown", root / "workspace", root / "evaluator")
+
+
+if __name__ == "__main__":
+    unittest.main()

From 1ce5696147906586efdc338cc3ef93f7ec1a6026 Mon Sep 17 00:00:00 2001
From: Ioannis Polyzos <git@ipolyzos.com>
Date: Sat, 10 Oct 2026 16:20:44 +0100
Subject: [PATCH 3/4] feat: expose fixture materialization CLI

---
 aicp/synthetic_fixtures.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/aicp/synthetic_fixtures.py b/aicp/synthetic_fixtures.py
index 70ca100..50d3f4f 100644
--- a/aicp/synthetic_fixtures.py
+++ b/aicp/synthetic_fixtures.py
@@ -44,3 +44,18 @@ def materialize_fixture(name, workspace, evaluator):
     (workspace / "TASK.md").write_text(task["objective"] + "\n", encoding="utf-8")
     (evaluator / "acceptance.py").write_text(task["tests"], encoding="utf-8")
     return {"task": name, "workspace": str(workspace), "evaluator": str(evaluator)}
+
+
+def main():
+    import argparse
+    import json
+    parser = argparse.ArgumentParser(description="Materialize a synthetic benchmark fixture")
+    parser.add_argument("--task", choices=sorted(TASKS), required=True)
+    parser.add_argument("--workspace", type=Path, required=True)
+    parser.add_argument("--evaluator", type=Path, required=True)
+    args = parser.parse_args()
+    print(json.dumps(materialize_fixture(args.task, args.workspace, args.evaluator), indent=2))
+
+
+if __name__ == "__main__":
+    main()

From 12fbd57197c7c4025283da452ef413a0319b35c5 Mon Sep 17 00:00:00 2001
From: Ioannis Polyzos <git@ipolyzos.com>
Date: Sat, 10 Oct 2026 16:20:51 +0100
Subject: [PATCH 4/4] docs: explain synthetic fixtures and held-out evaluator
 workflow

---
 evaluations/experiments/README.md | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 evaluations/experiments/README.md

diff --git a/evaluations/experiments/README.md b/evaluations/experiments/README.md
new file mode 100644
index 0000000..9271b54
--- /dev/null
+++ b/evaluations/experiments/README.md
@@ -0,0 +1,24 @@
+# Synthetic pilot task suite
+
+This is an **experimental personal side project**, not a validated productivity study.
+
+The suite provides five small Python tasks: bug fixing, test addition, refactoring, documentation, and feature development. Each task has a starter workspace and a separate acceptance script.
+
+## Create a fixture
+
+```sh
+python3 -m aicp.synthetic_fixtures --task bug-fix \
+  --workspace /tmp/aicp-pilot/workspace \
+  --evaluator /tmp/aicp-pilot/evaluator
+```
+
+The two output paths must be distinct and must not exist already. Provide only the `workspace` directory to the agent; **never mount the evaluator directory**. After agent execution, run `acceptance.py` from the workspace using a trusted process that has access to the evaluator script. This is a manual pilot workflow, not yet a fully isolated automated harness.
+
+For example, in a disposable synthetic environment:
+
+```sh
+cd /tmp/aicp-pilot/workspace
+python3 ../evaluator/acceptance.py
+```
+
+The tasks are deliberately small and the acceptance tests are minimal. In particular, the test-addition and documentation checks are structural smoke tests, not strong measures of engineering quality. They should be improved and independently reviewed before making productivity claims. The task definitions and evaluator source live in the same public repository for transparency; the evaluation environment must keep them out of the agent's runtime context. Follow the [experimental protocol](../../docs/EXPERIMENTAL_PROTOCOL.md) and report all failures and limitations.