From 15962b41bd2478e4bfbd6cb38301c3aef3cbccc6 Mon Sep 17 00:00:00 2001 From: oigorbrito Date: Sun, 4 Oct 2026 23:36:05 -0300 Subject: [PATCH] Port Codepro namespace and P8.2 capabilities --- README.md | 9 +- docs/architecture.md | 6 + ...0-codepro-canonical-namespace-migration.md | 69 +++ .../codepro-local-tests-20261004.json | 176 ++++++ ...republication-gates-20261004-attempt2.json | 121 ++++ pyproject.toml | 6 +- src/arkx/__init__.py | 11 +- src/arkx/cli.py | 18 + src/arkx/p82.py | 486 ++++++++++++++++ src/arkx/p82_baseline.py | 537 ++++++++++++++++++ src/arkx/performance_baseline.py | 179 ++++++ src/codepro/__init__.py | 23 + src/codepro/_legacy_module.py | 13 + src/codepro/_version.py | 3 + src/codepro/baseline.py | 10 + src/codepro/cli.py | 74 +++ src/codepro/contracts.py | 10 + src/codepro/orchestration.py | 10 + src/codepro/p82_localization.py | 10 + src/codepro/performance_baseline.py | 10 + src/codepro/verification.py | 10 + src/codepro/verifier.py | 10 + tests/test_cli.py | 25 +- tests/test_codepro_namespace.py | 89 +++ tests/test_p82_baseline.py | 309 ++++++++++ tests/test_performance_baseline.py | 33 ++ tests/test_swebench_authority.py | 267 +++++++++ tests/test_verification.py | 1 + tools/p82_bash_environment.py | 134 +++++ tools/run_mini_headless.py | 120 ++++ 30 files changed, 2771 insertions(+), 8 deletions(-) create mode 100644 docs/decisions/0170-codepro-canonical-namespace-migration.md create mode 100644 logs/local-evidence/codepro-local-tests-20261004.json create mode 100644 logs/local-evidence/codepro-prepublication-gates-20261004-attempt2.json create mode 100644 src/arkx/p82.py create mode 100644 src/arkx/p82_baseline.py create mode 100644 src/arkx/performance_baseline.py create mode 100644 src/codepro/__init__.py create mode 100644 src/codepro/_legacy_module.py create mode 100644 src/codepro/_version.py create mode 100644 src/codepro/baseline.py create mode 100644 src/codepro/cli.py create mode 100644 src/codepro/contracts.py create mode 100644 src/codepro/orchestration.py create mode 100644 src/codepro/p82_localization.py create mode 100644 src/codepro/performance_baseline.py create mode 100644 src/codepro/verification.py create mode 100644 src/codepro/verifier.py create mode 100644 tests/test_codepro_namespace.py create mode 100644 tests/test_p82_baseline.py create mode 100644 tests/test_performance_baseline.py create mode 100644 tests/test_swebench_authority.py create mode 100644 tools/p82_bash_environment.py create mode 100644 tools/run_mini_headless.py diff --git a/README.md b/README.md index 91acebf7..e7b76c63 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,8 @@ CURRENT PHASE = PHASE 7 / SCAFFOLD COMPATIBILITY The reconciled main preserves: -- the Python `arkx` operational core; +- the Python operational core, currently implemented under `arkx` with a + canonical `codepro` package facade; - the public `codepro` CLI; - the Python regression and validation boundary; - the TypeScript/API/UI surface; @@ -309,7 +310,11 @@ Equivalent module entrypoint: python -m arkx --help ``` -The public product command is `codepro`. The Python implementation namespace remains `arkx` so introducing the CLI does not force an unrelated package-rename migration. +The canonical product and Python package namespace is `codepro`. The existing +implementation remains under `arkx` during the staged migration, which keeps +legacy imports available while the supported public API is migrated and +verified. The `arkx` namespace is a compatibility implementation namespace, +not a separate product identity. ### `codepro inspect` diff --git a/docs/architecture.md b/docs/architecture.md index d65e1475..4194093d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,6 +17,12 @@ Study Spec v1 adds a pre-execution research-design contract above P0-P6. It vali The product surface includes a dependency-free `codepro` CLI implemented with the Python standard library. The CLI exposes help, version, doctor, and read-only project inspection. Inspection may query Git and PATH and detect project markers, but it does not invoke executors or execute tasks. +`codepro` is the canonical product and Python package namespace. The current +implementation remains under `src/arkx/` while `src/codepro/` provides the +staged public facade. Legacy `arkx` imports remain available during migration; +the facade does not imply that the wider API or installed-package behavior has +already been qualified. See Decision 0170 for migration criteria and status. + A low-level command environment boundary may execute an explicit argv vector with explicit cwd and timeout. Its output is a `CommandResult` observation only: exit code, stdout, stderr, timeout, duration, and environment error. It does not map process outcomes to task status. Capability qualification is a separate pure boundary. A runtime executor identity is bindable only when the exact executor version + adapter version has explicit qualification evidence for the required capability. Availability alone is never qualification. Multiple equally qualified available executors are blocked rather than silently ranked. diff --git a/docs/decisions/0170-codepro-canonical-namespace-migration.md b/docs/decisions/0170-codepro-canonical-namespace-migration.md new file mode 100644 index 00000000..7d9be059 --- /dev/null +++ b/docs/decisions/0170-codepro-canonical-namespace-migration.md @@ -0,0 +1,69 @@ +# Decision 0170 — CodePro canonical Python namespace migration + +## Status + +Direction accepted for staged implementation; local patch is under review. +Independent acceptance is pending, and nothing is released or promoted. + +## Decision + +`codepro` is the canonical product and Python package namespace. `arkx` is +retained temporarily as a compatibility shim and must not be treated as a +separate product identity. + +The migration is staged: + +1. implement the canonical package under `codepro`; +2. update internal imports, CLI entry points, packaging metadata, and current + documentation to use `codepro`; +3. retain `arkx` re-exports for the declared compatibility window; +4. test equivalence of the supported public API through both namespaces; +5. deprecate and remove `arkx` only in a separately authorized release. + +No implementation or evidence is promoted merely because this decision exists. + +## Rationale + +The product identity is CodePro, while the current Python namespace is `arkx`. +An atomic rename would break imports, dynamic module references, serialized +objects, manifests, historical evidence, and external scripts without proving +that all consumers had migrated. A compatibility shim preserves continuity +while making the intended namespace explicit. + +## Acceptance criteria + +- `import codepro` exposes the canonical public API; +- `codepro.__all__` is explicit and tested; +- supported `arkx` imports resolve to the corresponding `codepro` objects; +- `codepro` and `arkx` do not silently expose different behavior; +- the CLI and wheel identify the distribution as `codepro`; +- no claim of completed migration is made until installed-package and source + import tests pass at the same revision; +- removal of the `arkx` shim requires a later decision and release boundary. + +## Falsifiable hypothesis + +For every symbol in the explicitly supported public API at a given revision, +imports through `codepro` will resolve to the same object and behavior as the +corresponding supported `arkx` import, while importing the facade will not +eagerly load experimental subsystems. The hypothesis is rejected by any +identity/behavior mismatch, an unsupported export appearing in `codepro`, or +an import-boundary test showing an experimental subsystem loaded eagerly. + +Local tests can evaluate these source-tree claims only. Installed-wheel +compatibility and independent acceptance remain separate gates; passing local +tests alone does not promote the migration. + +## Non-goals + +- no redesign of the execution chassis; +- no provider, model, executor, or benchmark qualification; +- no deletion of historical `arkx` evidence; +- no automatic removal of legacy imports. + +## Current evidence + +The current checkout still has `arkx` as the implementation namespace, while +`codepro` now provides the canonical facade and packaging entry point. The +installed-package and full compatibility gates remain open until a wheel is +built and tested in a clean environment. diff --git a/logs/local-evidence/codepro-local-tests-20261004.json b/logs/local-evidence/codepro-local-tests-20261004.json new file mode 100644 index 00000000..e59355ba --- /dev/null +++ b/logs/local-evidence/codepro-local-tests-20261004.json @@ -0,0 +1,176 @@ +{ + "schema": "codepro-local-verification-evidence/v1", + "evidence_kind": "LOCAL_UNIT_TEST_RUN", + "attempt_id": "95eadf2e-b246-4a5e-83d9-341d7af6600c", + "collected_at": "2026-10-05T02:01:47.007692+00:00", + "workspace": "C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro", + "head_revision": "a42dbbf1dd1dd5f5078249f29713faf9f4d9c083", + "base_revision": "a42dbbf1dd1dd5f5078249f29713faf9f4d9c083", + "branch": null, + "patch_identity": { + "algorithm": "sha256", + "value": "8557788b4f539eefb2292994c437949fe2571607083056169d578fa98d90fb74", + "scope": "staged implementation patch excluding this evidence file" + }, + "environment": { + "python": "3.13.15 (tags/v3.13.15:4061bc4, Aug 5 2026, 13:05:39) [MSC v.1944 64 bit (AMD64)]", + "executable": "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "platform": "Windows-11-10.0.26300-SP0", + "machine": "AMD64", + "architecture": "64bit" + }, + "provider_called": false, + "benchmark_executed": false, + "acceptance": "PENDING_INDEPENDENT_REVIEW", + "promotion": "NOT_PROMOTED", + "results": [ + { + "test_file": "test_codepro_namespace.py", + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-t", + ".", + "-p", + "test_codepro_namespace.py" + ], + "returncode": 0, + "status": "PASS", + "stdout": "", + "stderr": "......\n----------------------------------------------------------------------\nRan 6 tests in 0.203s\n\nOK\n" + }, + { + "test_file": "test_architecture_boundaries.py", + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-t", + ".", + "-p", + "test_architecture_boundaries.py" + ], + "returncode": 0, + "status": "PASS", + "stdout": "", + "stderr": "....\n----------------------------------------------------------------------\nRan 4 tests in 0.402s\n\nOK\n" + }, + { + "test_file": "test_cli.py", + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-t", + ".", + "-p", + "test_cli.py" + ], + "returncode": 0, + "status": "PASS", + "stdout": "", + "stderr": "..............\n----------------------------------------------------------------------\nRan 14 tests in 1.170s\n\nOK\n" + }, + { + "test_file": "test_p82_baseline.py", + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-t", + ".", + "-p", + "test_p82_baseline.py" + ], + "returncode": 0, + "status": "PASS", + "stdout": "", + "stderr": ".......................\n----------------------------------------------------------------------\nRan 23 tests in 1.121s\n\nOK\n" + }, + { + "test_file": "test_performance_baseline.py", + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-t", + ".", + "-p", + "test_performance_baseline.py" + ], + "returncode": 0, + "status": "PASS", + "stdout": "", + "stderr": "..\n----------------------------------------------------------------------\nRan 2 tests in 0.723s\n\nOK\n" + }, + { + "test_file": "test_verification.py", + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-t", + ".", + "-p", + "test_verification.py" + ], + "returncode": 0, + "status": "PASS", + "stdout": "", + "stderr": "................\n----------------------------------------------------------------------\nRan 16 tests in 0.002s\n\nOK\n" + }, + { + "test_file": "test_swebench_authority.py", + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-t", + ".", + "-p", + "test_swebench_authority.py" + ], + "returncode": 1, + "status": "BLOCKED", + "stdout": "", + "stderr": "....E...E...E...EEE\n======================================================================\nERROR: test_enriched_record_image_is_idempotently_verified (tests.test_swebench_authority.SWEbenchAuthorityTests.test_enriched_record_image_is_idempotently_verified)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\src\\arkx\\swebench_authority.py\", line 177, in derive_harness_instance\n from swebench.task.checks import expected_image\nModuleNotFoundError: No module named 'swebench'\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\tests\\test_swebench_authority.py\", line 191, in test_enriched_record_image_is_idempotently_verified\n enriched = derive_harness_instance(raw, source_dataset_id=\"dataset\", revision=\"rev\", fingerprint=\"fp\")\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\src\\arkx\\swebench_authority.py\", line 180, in derive_harness_instance\n raise RuntimeError(\n f\"cannot derive official image for {instance_id}: {type(error).__name__}: {error}\"\n ) from error\nRuntimeError: cannot derive official image for sympy__sympy-14711: ModuleNotFoundError: No module named 'swebench'\n\n======================================================================\nERROR: test_make_test_spec_accepts_complete_derived_record (tests.test_swebench_authority.SWEbenchAuthorityTests.test_make_test_spec_accepts_complete_derived_record)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\tests\\test_swebench_authority.py\", line 237, in test_make_test_spec_accepts_complete_derived_record\n from swebench.harness.utils import make_test_spec\nModuleNotFoundError: No module named 'swebench'\n\n======================================================================\nERROR: test_raw_record_is_explicitly_enriched_with_harness_image (tests.test_swebench_authority.SWEbenchAuthorityTests.test_raw_record_is_explicitly_enriched_with_harness_image)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\src\\arkx\\swebench_authority.py\", line 177, in derive_harness_instance\n from swebench.task.checks import expected_image\nModuleNotFoundError: No module named 'swebench'\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\tests\\test_swebench_authority.py\", line 156, in test_raw_record_is_explicitly_enriched_with_harness_image\n enriched = derive_harness_instance(\n raw, source_dataset_id=\"dataset\", revision=\"rev\", fingerprint=\"fp\"\n )\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\src\\arkx\\swebench_authority.py\", line 180, in derive_harness_instance\n raise RuntimeError(\n f\"cannot derive official image for {instance_id}: {type(error).__name__}: {error}\"\n ) from error\nRuntimeError: cannot derive official image for sympy__sympy-14711: ModuleNotFoundError: No module named 'swebench'\n\n======================================================================\nERROR: test_task_repo_is_the_authority_for_the_three_missing_fields (tests.test_swebench_authority.SWEbenchAuthorityTests.test_task_repo_is_the_authority_for_the_three_missing_fields)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\tests\\test_swebench_authority.py\", line 221, in test_task_repo_is_the_authority_for_the_three_missing_fields\n enriched = derive_harness_instance_from_task_repo(\n raw, task_repo=root, source_dataset_id=\"dataset\", revision=\"rev\", fingerprint=\"fp\"\n )\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\src\\arkx\\swebench_authority.py\", line 208, in derive_harness_instance_from_task_repo\n from swebench.task.repo import load_task_repo\nModuleNotFoundError: No module named 'swebench'\n\n======================================================================\nERROR: test_unknown_task_repo_identity_is_blocked (tests.test_swebench_authority.SWEbenchAuthorityTests.test_unknown_task_repo_identity_is_blocked)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\tests\\test_swebench_authority.py\", line 247, in test_unknown_task_repo_identity_is_blocked\n derive_harness_instance_from_task_repo(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n raw_harness_record(), task_repo=directory,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n source_dataset_id=\"dataset\", revision=\"rev\", fingerprint=\"fp\"\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\src\\arkx\\swebench_authority.py\", line 208, in derive_harness_instance_from_task_repo\n from swebench.task.repo import load_task_repo\nModuleNotFoundError: No module named 'swebench'\n\n======================================================================\nERROR: test_wrong_image_identity_is_blocked (tests.test_swebench_authority.SWEbenchAuthorityTests.test_wrong_image_identity_is_blocked)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\src\\arkx\\swebench_authority.py\", line 177, in derive_harness_instance\n from swebench.task.checks import expected_image\nModuleNotFoundError: No module named 'swebench'\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\tests\\test_swebench_authority.py\", line 186, in test_wrong_image_identity_is_blocked\n derive_harness_instance(raw, source_dataset_id=\"dataset\", revision=\"rev\", fingerprint=\"fp\")\n ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\src\\arkx\\swebench_authority.py\", line 180, in derive_harness_instance\n raise RuntimeError(\n f\"cannot derive official image for {instance_id}: {type(error).__name__}: {error}\"\n ) from error\nRuntimeError: cannot derive official image for sympy__sympy-14711: ModuleNotFoundError: No module named 'swebench'\n\n----------------------------------------------------------------------\nRan 19 tests in 0.035s\n\nFAILED (errors=6)\n" + } + ], + "availability_probes": [ + { + "purpose": "diagnose SWE-bench focused test blocker", + "command": [ + "py", + "-3.14", + "-c", + "import importlib.util, platform, sys; print(sys.version); print(platform.platform()); print(importlib.util.find_spec('swebench'))" + ], + "python": "3.14.7 (tags/v3.14.7:823f032, Aug 5 2026, 10:51:32) [MSC v.1944 64 bit (AMD64)]", + "platform": "Windows-11-10.0.26300-SP0", + "result": "swebench module unavailable (find_spec returned None)", + "interpretation": "BLOCKED_OPTIONAL_DEPENDENCY_NOT_INSTALLED; no install attempted" + } + ] +} diff --git a/logs/local-evidence/codepro-prepublication-gates-20261004-attempt2.json b/logs/local-evidence/codepro-prepublication-gates-20261004-attempt2.json new file mode 100644 index 00000000..6ffac902 --- /dev/null +++ b/logs/local-evidence/codepro-prepublication-gates-20261004-attempt2.json @@ -0,0 +1,121 @@ +{ + "schema": "codepro-local-verification-evidence/v1", + "evidence_kind": "LOCAL_PREPUBLICATION_GATES", + "attempt_id": "c188c858-ab65-4ef4-a05f-fbe24fbfa370", + "collected_at": "2026-10-05T02:26:24.921146+00:00", + "workspace": "C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro", + "head_revision": "a42dbbf1dd1dd5f5078249f29713faf9f4d9c083", + "base_revision": "a42dbbf1dd1dd5f5078249f29713faf9f4d9c083", + "branch": null, + "patch_identity": { + "algorithm": "sha256", + "value": "50a62f4e6c3397003be36f59e8ede422c05e765db5e10b7aa4a9f2eb504b29c9", + "scope": "complete patch from HEAD including staged and unstaged source, excluding local-evidence reports" + }, + "environment": { + "python": "3.13.15 (tags/v3.13.15:4061bc4, Aug 5 2026, 13:05:39) [MSC v.1944 64 bit (AMD64)]", + "executable": "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "platform": "Windows-11-10.0.26300-SP0", + "machine": "AMD64" + }, + "provider_called": false, + "benchmark_executed": false, + "acceptance": "PENDING_INDEPENDENT_REVIEW", + "promotion": "NOT_PROMOTED", + "results": [ + { + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "tools/check_foundation.py" + ], + "returncode": 0, + "status": "PASS", + "stdout": "Foundation check passed: 10 files, 14 invariants, and 2 study guards verified.\n", + "stderr": "" + }, + { + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "arkx.baseline" + ], + "returncode": 0, + "status": "PASS", + "stdout": "{\"decision_reason\":\"deterministic fixture completed\",\"errors\":[],\"evidence_refs\":[\"fixture://evidence/1\"],\"executor_id\":\"fixture\",\"executor_invocations\":1,\"finished_at\":\"2026-01-01T00:00:00.500000+00:00\",\"handoff_count\":0,\"human_interventions\":0,\"input_tokens\":null,\"metadata\":{\"handoffs\":[]},\"monetary_cost\":null,\"output_tokens\":null,\"replan_count\":0,\"retry_count\":0,\"run_id\":\"baseline-fixture-run\",\"schemaVersion\":1,\"started_at\":\"2026-01-01T00:00:00+00:00\",\"status\":\"PASS\",\"task_id\":\"baseline-task\",\"total_tokens\":12,\"wall_time_ms\":500}\n", + "stderr": "" + }, + { + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "arkx", + "--help" + ], + "returncode": 0, + "status": "PASS", + "stdout": "usage: codepro [-h] [--version] {doctor,inspect,run,baseline-cost} ...\n\nDeterministic CLI boundary for the CodePro software-engineering chassis.\n\npositional arguments:\n {doctor,inspect,run,baseline-cost}\n doctor Check whether the local Python runtime can execute the\n current chassis.\n inspect Inspect the current project without changing it.\n run Run one authorized, bounded executor command and\n declared verifier.\n baseline-cost Measure local overhead without changing behavior.\n\noptions:\n -h, --help show this help message and exit\n --version show program's version number and exit\n", + "stderr": "" + }, + { + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "arkx", + "--version" + ], + "returncode": 0, + "status": "PASS", + "stdout": "codepro 0.3.0.dev0\n", + "stderr": "" + }, + { + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "tools/chassis_fingerprint.py" + ], + "returncode": 0, + "status": "PASS", + "stdout": "{\"components\":{\"environment\":\"sha256:34322b543d562dfae5e3ee54ee1fa1a5047562cfde883cf8288b54b8c88682eb\",\"promotion_gate\":\"sha256:358cca9422c51600e8871d092c5292df828bb09e5790789d3b9f786fd18047b1\",\"run_manifest\":\"sha256:9f08ed142bdf15e6372719e9b8f54c9b93adb5b11e95a325f6d74a496be0762c\",\"study\":\"sha256:4bba48cd3b6e035b655682f16f904ad08ecc938ab9769b04977ad680dcb26302\",\"treatment\":\"sha256:00502d3e2116a602c2d5c7247980d9a996b90d257c8c87e70d72894531745bf4\",\"workload\":\"sha256:7813fe4a729d44a6eeb98ad5029e6102d63c1449b9c1c17a7473b9ac0b2d0675\"},\"expected_fingerprint\":\"sha256:ab557c36fa62df7dc0242c4241a1b2cb12ef33f4f7e2955d24334ea17a584fa0\",\"fingerprint\":\"sha256:ab557c36fa62df7dc0242c4241a1b2cb12ef33f4f7e2955d24334ea17a584fa0\"}\nCHASSIS_FINGERPRINT_PASS sha256:ab557c36fa62df7dc0242c4241a1b2cb12ef33f4f7e2955d24334ea17a584fa0\n", + "stderr": "" + }, + { + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-t", + ".", + "-p", + "test_*.py" + ], + "returncode": 1, + "status": "BASELINE_FAILURE", + "stdout": "", + "stderr": ".............................................................................s.............................................................................................................................................................................warning: in the working copy of 'src/minisweagent/config/benchmarks/swebench.yaml', LF will be replaced by CRLF the next time Git touches it\n........................................................................................................................................................F................................................................................s...s...s...sss..................................................................\n======================================================================\nFAIL: test_ui_marks_preview_surfaces (tests.test_r5_product_surface.R5ProductSurfaceTests.test_ui_marks_preview_surfaces)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"C:\\Users\\igorb\\.codex\\worktrees\\reconcile-04a-main\\codepro\\tests\\test_r5_product_surface.py\", line 32, in test_ui_marks_preview_surfaces\n self.assertIn(\"Task Characterization & Routing Preview\", self.app)\n ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: 'Task Characterization & Routing Preview' not found in 'import React, { useState, useEffect, useRef } from \\'react\\';\\nimport {\\n Terminal,\\n CheckCircle2,\\n Clock,\\n ArrowRight,\\n ShieldAlert,\\n Bot,\\n Cpu,\\n CornerDownRight,\\n ArrowLeft,\\n Sparkles,\\n GitBranch,\\n Play,\\n Copy,\\n Check,\\n RotateCcw,\\n ChevronRight,\\n ShieldCheck,\\n Activity,\\n Search,\\n Sliders,\\n Database,\\n Layers,\\n CheckCircle,\\n FileCheck,\\n SlidersHorizontal,\\n FolderSync\\n} from \\'lucide-react\\';\\n\\n// Modularized components\\nimport { DoctorTab } from \\'./components/DoctorTab\\';\\nimport { InspectTab } from \\'./components/InspectTab\\';\\nimport { LocalOllamaTab } from \\'./components/LocalOllamaTab\\';\\nimport { GeminiAgentTab } from \\'./components/GeminiAgentTab\\';\\nimport { RunTab } from \\'./components/RunTab\\';\\nimport { AcceptanceTab } from \\'./components/AcceptanceTab\\';\\nimport { TieredRoutingTab } from \\'./components/TieredRoutingTab\\';\\nimport { SecondExecutorTab } from \\'./components/SecondExecutorTab\\';\\nimport { CharacterizeTab } from \\'./components/CharacterizeTab\\';\\nimport { DecisionBasisTab } from \\'./components/DecisionBasisTab\\';\\nimport { TestRunnerTab } from \\'./components/TestRunnerTab\\';\\nimport { M2M3ValidationTab } from \\'./components/M2M3ValidationTab\\';\\nimport { FixturesTab } from \\'./components/FixturesTab\\';\\n\\nexport interface CodeProIssue {\\n number: number;\\n title: string;\\n state: string;\\n body: string;\\n scope: string[];\\n candidate_files: string[];\\n}\\n\\nexport default function App() {\\n // Navigation categories\\n const [activeCategory, setActiveCategory] = useState<\\'issues\\' | \\'control\\' | \\'diagnostics\\' | \\'models\\' | \\'validation\\'>(() => {\\n return (localStorage.getItem(\\'codepro_active_category\\') as any) || \\'issues\\';\\n });\\n\\n // Sub-tabs state (persisted inside localStorage)\\n const [controlSubTab, setControlSubTab] = useState<\\'doctor\\' | \\'inspect\\' | \\'ollama\\' | \\'gemini\\'>(() => {\\n return (localStorage.getItem(\\'codepro_subtab_control\\') as any) || \\'doctor\\';\\n });\\n\\n const [diagnosticsSubTab, setDiagnosticsSubTab] = useState<\\'run\\' | \\'acceptance\\'>(() => {\\n return (localStorage.getItem(\\'codepro_subtab_diagnostics\\') as any) || \\'run\\';\\n });\\n\\n const [modelsSubTab, setModelsSubTab] = useState<\\'routing\\' | \\'second_executor\\' | \\'characterize\\' | \\'decision_basis\\'>(() => {\\n return (localStorage.getItem(\\'codepro_subtab_models\\') as any) || \\'routing\\';\\n });\\n\\n const [validationSubTab, setValidationSubTab] = useState<\\'tests\\' | \\'m2m3\\' | \\'fixtures\\'>(() => {\\n return (localStorage.getItem(\\'codepro_subtab_validation\\') as any) || \\'tests\\';\\n });\\n\\n // Issue Queue simplified state\\n const [currentScreen, setCurrentScreen] = useState<\\'list\\' | \\'execution\\'>(() => {\\n return (localStorage.getItem(\\'codepro_simplified_screen\\') as \\'list\\' | \\'execution\\') || \\'list\\';\\n });\\n\\n const [isAgileMode, setIsAgileMode] = useState(() => {\\n const saved = localStorage.getItem(\\'codepro_agile_mode\\');\\n return saved === null ? true : saved === \\'true\\';\\n });\\n\\n const [issues, setIssues] = useState([]);\\n const [selectedIssue, setSelectedIssue] = useState(() => {\\n const saved = localStorage.getItem(\\'codepro_simplified_selected_issue\\');\\n return saved ? JSON.parse(saved) : null;\\n });\\n\\n const [terminalLines, setTerminalLines] = useState([]);\\n const [isResolving, setIsResolving] = useState(false);\\n const [patchGenerated, setPatchGenerated] = useState(() => localStorage.getItem(\\'codepro_simplified_patch\\') || \\'\\');\\n const [summaryText, setSummarySummary] = useState(() => localStorage.getItem(\\'codepro_simplified_summary\\') || \\'\\');\\n const [copied, setCopied] = useState(false);\\n const terminalEndRef = useRef(null);\\n\\n // Fetch issues from the local CodePro Issues Registry\\n useEffect(() => {\\n const fetchIssues = async () => {\\n try {\\n const res = await fetch(\\'/api/codepro/issues\\');\\n if (res.ok) {\\n const data = await res.json();\\n setIssues(data.issues || []);\\n }\\n } catch (err) {\\n console.error(\\'Error fetching registry issues:\\', err);\\n }\\n };\\n fetchIssues();\\n }, []);\\n\\n // Auto-scroll terminal log\\n useEffect(() => {\\n terminalEndRef.current?.scrollIntoView({ behavior: \\'smooth\\' });\\n }, [terminalLines]);\\n\\n // Sync state with localStorage\\n useEffect(() => {\\n localStorage.setItem(\\'codepro_active_category\\', activeCategory);\\n localStorage.setItem(\\'codepro_subtab_control\\', controlSubTab);\\n localStorage.setItem(\\'codepro_subtab_diagnostics\\', diagnosticsSubTab);\\n localStorage.setItem(\\'codepro_subtab_models\\', modelsSubTab);\\n localStorage.setItem(\\'codepro_subtab_validation\\', validationSubTab);\\n localStorage.setItem(\\'codepro_simplified_screen\\', currentScreen);\\n localStorage.setItem(\\'codepro_agile_mode\\', String(isAgileMode));\\n if (selectedIssue) {\\n localStorage.setItem(\\'codepro_simplified_selected_issue\\', JSON.stringify(selectedIssue));\\n } else {\\n localStorage.removeItem(\\'codepro_simplified_selected_issue\\');\\n }\\n }, [activeCategory, controlSubTab, diagnosticsSubTab, modelsSubTab, validationSubTab, currentScreen, selectedIssue, isAgileMode]);\\n\\n const addLog = (line: string) => {\\n setTerminalLines((prev) => [...prev, line]);\\n };\\n\\n const handleResolveIssue = async (issue: CodeProIssue) => {\\n setSelectedIssue(issue);\\n setCurrentScreen(\\'execution\\');\\n setIsResolving(true);\\n setPatchGenerated(\\'\\');\\n setSummarySummary(\\'\\');\\n setTerminalLines([]);\\n\\n addLog(`$ codepro resolve #${issue.number} --mode=${isAgileMode ? \\'agile\\' : \\'strict\\'}`);\\n await new Promise((r) => setTimeout(r, 400));\\n\\n addLog(`[git] Active branch: main`);\\n // Retrieve Git SHA dynamically from backend\\n try {\\n const gitRes = await fetch(\\'/api/git/revision\\');\\n if (gitRes.ok) {\\n const gitData = await gitRes.json();\\n addLog(`[git] Base SHA: ${gitData.revision}`);\\n } else {\\n addLog(`[git] Base SHA: e401936979aea7f875508394aab1dac8f9e850d0 (fallback)`);\\n }\\n } catch {\\n addLog(`[git] Base SHA: e401936979aea7f875508394aab1dac8f9e850d0 (fallback)`);\\n }\\n await new Promise((r) => setTimeout(r, 300));\\n\\n addLog(`[scope] Checking authorized files limit...`);\\n addLog(`[scope] Authorized boundaries: [${issue.scope.join(\\', \\')}]`);\\n await new Promise((r) => setTimeout(r, 400));\\n\\n addLog(`[scope] PASS: Candidate file list matched scope limits.`);\\n addLog(`[route] Selecting model for task complexity...`);\\n await new Promise((r) => setTimeout(r, 300));\\n\\n addLog(`[route] Routed to: gemini-2.5-flash (Tier 1 - Fast Local Context)`);\\n addLog(`[model] Contacting LLM service...`);\\n await new Promise((r) => setTimeout(r, 500));\\n\\n let currentSignals: any = null;\\n let executionOutcome: any = null;\\n\\n try {\\n // Real backend characterization (P1)\\n const signalsPayload = {\\n candidate_files: issue.candidate_files,\\n affected_components: [\\'cli\\', \\'chassis\\'],\\n ambiguity_markers: [],\\n risk_markers: [],\\n architectural_change: false,\\n state_shared: false,\\n };\\n\\n const charRes = await fetch(\\'/api/characterize\\', {\\n method: \\'POST\\',\\n headers: { \\'Content-Type\\': \\'application/json\\' },\\n body: JSON.stringify({ signals: signalsPayload }),\\n });\\n currentSignals = await charRes.json();\\n addLog(`[model] Scope characterized: ${currentSignals.scope}`);\\n\\n // Real backend LLM execution with auto-injected git SHA parameters\\n const execRes = await fetch(\\'/api/gemini/run\\', {\\n method: \\'POST\\',\\n headers: { \\'Content-Type\\': \\'application/json\\' },\\n body: JSON.stringify({\\n request_id: `req-agile-${issue.number}-${Date.now().toString(36)}`,\\n task_id: `task-issue-${issue.number}`,\\n prompt: issue.body,\\n scope: issue.scope,\\n candidate_files: issue.candidate_files,\\n model: \\'gemini-2.5-flash\\',\\n }),\\n });\\n\\n if (!execRes.ok) throw new Error(\\'Inference server returned error status.\\');\\n executionOutcome = await execRes.json();\\n\\n if (executionOutcome.status === \\'REJECTED\\' || executionOutcome.status === \\'FAILED\\') {\\n throw new Error(executionOutcome.reason || \\'Model generated code outside scope guard limits.\\');\\n }\\n\\n const patch = executionOutcome.model_response?.patch || \\'\\';\\n const summary = executionOutcome.model_response?.summary || \\'\\';\\n setPatchGenerated(patch);\\n setSummarySummary(summary);\\n localStorage.setItem(\\'codepro_simplified_patch\\', patch);\\n localStorage.setItem(\\'codepro_simplified_summary\\', summary);\\n\\n addLog(`[model] Unified patch generated successfully in ${executionOutcome.duration_ms}ms.`);\\n addLog(`[model] Bytes: ${patch.length} · proposed modifications validated.`);\\n await new Promise((r) => setTimeout(r, 400));\\n\\n } catch (err: any) {\\n addLog(`[model] FAIL-CLOSED HALT: ${err.message}`);\\n setIsResolving(false);\\n return;\\n }\\n\\n addLog(`[test] Spawning verification test suites...`);\\n await new Promise((r) => setTimeout(r, 300));\\n\\n try {\\n const getRelevantSuiteFile = (num: number) => {\\n if (num === 57) return \\'test_m2_m3_gates.ts\\';\\n if (num === 104) return \\'test_blind_spot.ts\\';\\n if (num === 112) return \\'test_event_log.ts\\';\\n return \\'\\';\\n };\\n\\n const testPayload: Record = {};\\n if (isAgileMode) {\\n const file = getRelevantSuiteFile(issue.number);\\n testPayload.onlySuite = file;\\n addLog(`[test] tsx tests/${file} (Fast suite filtered for Issue #${issue.number})`);\\n } else {\\n addLog(`[test] tsx tests/* (Governance mode: Running all 6 continuous test suites)`);\\n }\\n\\n const testRes = await fetch(\\'/api/tests/run\\', {\\n method: \\'POST\\',\\n headers: { \\'Content-Type\\': \\'application/json\\' },\\n body: JSON.stringify(testPayload),\\n });\\n\\n if (!testRes.ok) throw new Error(\\'Test execution process crashed.\\');\\n const testSummary = await testRes.json();\\n\\n testSummary.results.forEach((r: any) => {\\n addLog(` [${r.passed ? \\'PASS\\' : \\'FAIL\\'}] ${r.file} (${r.duration_ms}ms)`);\\n });\\n\\n if (!testSummary.all_passed) {\\n throw new Error(\\'Continuous verification suite found regressions.\\');\\n }\\n\\n addLog(`[test] PASS: All tests passed successfully. No regressions detected.`);\\n await new Promise((r) => setTimeout(r, 300));\\n\\n } catch (err: any) {\\n addLog(`[test] FAIL-CLOSED HALT: ${err.message}`);\\n setIsResolving(false);\\n return;\\n }\\n\\n addLog(`[sys] Committing patch...`);\\n addLog(`[sys] Telemetry logged: sha256 digest linked securely.`);\\n addLog(`[sys] SUCCESS: Issue #${issue.number} resolved. Clean patch ready to commit.`);\\n setIsResolving(false);\\n };\\n\\n const copyToClipboard = () => {\\n navigator.clipboard.writeText(patchGenerated);\\n setCopied(true);\\n setTimeout(() => setCopied(false), 1500);\\n };\\n\\n return (\\n
\\n \\n {/* Workspace Sidebar - Categorized Advanced Cockpit and Issue List */}\\n \\n\\n {/* Main Panel Canvas */}\\n
\\n {/* Top Header Navigation */}\\n
\\n
\\n CodePro\\n \\n \\n {activeCategory === \\'issues\\' && (currentScreen === \\'list\\' ? \\'Issues Queue\\' : \\'Terminal Execution\\')}\\n {activeCategory === \\'control\\' && \\'Painel de Controle\\'}\\n {activeCategory === \\'diagnostics\\' && \\'Central de Diagnóstico\\'}\\n {activeCategory === \\'models\\' && \\'Qualificação de Modelos\\'}\\n {activeCategory === \\'validation\\' && \\'Validação\\'}\\n \\n
\\n\\n {/* Clean Segmented Performance Mode Switch for Fila de Issues */}\\n {activeCategory === \\'issues\\' && (\\n
\\n setIsAgileMode(true)}\\n className={`flex items-center gap-1 px-2.5 py-1 text-[9px] font-bold font-mono uppercase tracking-wider rounded transition select-none cursor-pointer ${\\n isAgileMode\\n ? \\'bg-indigo-600 text-white shadow-sm\\'\\n : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n \\n Agile Mode\\n \\n setIsAgileMode(false)}\\n className={`flex items-center gap-1 px-2.5 py-1 text-[9px] font-bold font-mono uppercase tracking-wider rounded transition select-none cursor-pointer ${\\n !isAgileMode\\n ? \\'bg-indigo-600 text-white shadow-sm\\'\\n : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n \\n Governance\\n \\n
\\n )}\\n
\\n\\n {/* Dynamic Screens / Tabs Area */}\\n
\\n \\n {/* CATEGORY 1: ISSUES WORKSPACE (2-SCREEN FLOW) */}\\n {activeCategory === \\'issues\\' && (\\n <>\\n {currentScreen === \\'list\\' ? (\\n
\\n
\\n

Active Issues

\\n

\\n Select an issue to let CodePro find, check, and commit clean file edits.\\n

\\n
\\n\\n {issues.length > 0 ? (\\n
\\n {issues.map((issue) => (\\n \\n
\\n
\\n #{issue.number}\\n ·\\n {issue.state}\\n ·\\n {issue.scope.join(\\' · \\')}\\n
\\n \\n

\\n {issue.title}\\n

\\n \\n

\\n {issue.body}\\n

\\n
\\n\\n handleResolveIssue(issue)}\\n className=\"rounded-lg bg-indigo-600 hover:bg-indigo-500 text-[11px] font-bold uppercase tracking-wider text-white py-2 px-4 transition flex items-center gap-1.5 shrink-0 cursor-pointer\"\\n >\\n Fix\\n \\n \\n
\\n ))}\\n
\\n ) : (\\n
\\n \\n

codepro: reading internal issues list...

\\n
\\n )}\\n
\\n ) : (\\n selectedIssue && (\\n
\\n {/* Top Row */}\\n
\\n {\\n if (!isResolving) setCurrentScreen(\\'list\\');\\n }}\\n disabled={isResolving}\\n className={`flex items-center gap-1.5 font-bold transition select-none cursor-pointer ${\\n isResolving ? \\'text-slate-700\\' : \\'text-slate-400 hover:text-white\\'\\n }`}\\n >\\n \\n Voltar para Issues\\n \\n\\n
\\n Issue #{selectedIssue.number}\\n {!isResolving && (\\n handleResolveIssue(selectedIssue)}\\n className=\"flex items-center gap-1 text-slate-400 hover:text-white transition select-none cursor-pointer\"\\n >\\n \\n Rerun Fix\\n \\n )}\\n
\\n
\\n\\n
\\n

\\n {selectedIssue.title}\\n

\\n
\\n\\n {/* Console Log */}\\n
\\n
\\n {terminalLines.map((line, idx) => {\\n let colorClass = \\'text-slate-300\\';\\n if (line.startsWith(\\'$\\')) colorClass = \\'text-slate-400 font-bold\\';\\n else if (line.includes(\\'[git]\\')) colorClass = \\'text-slate-500\\';\\n else if (line.includes(\\'[scope]\\')) colorClass = \\'text-slate-400\\';\\n else if (line.includes(\\'[route]\\')) colorClass = \\'text-indigo-400\\';\\n else if (line.includes(\\'[model]\\')) colorClass = \\'text-indigo-300\\';\\n else if (line.includes(\\'[test]\\')) colorClass = \\'text-amber-400\\';\\n else if (line.includes(\\'PASS\\') || line.includes(\\'SUCCESS\\')) colorClass = \\'text-emerald-400 font-bold\\';\\n else if (line.includes(\\'FAIL\\')) colorClass = \\'text-rose-400 font-bold\\';\\n\\n return (\\n
\\n {line}\\n
\\n );\\n })}\\n
\\n
\\n
\\n\\n {/* Output Diff Patch */}\\n {patchGenerated && (\\n
\\n
\\n
\\n

Unified Git Diff

\\n

This diff was successfully written inside authorized file boundaries.

\\n
\\n\\n \\n {copied ? : }\\n {copied ? \\'Copied!\\' : \\'Copy Patch\\'}\\n \\n
\\n\\n {summaryText && (\\n
\\n \\n
\\n aider: Summary of fix:\\n {summaryText}\\n
\\n
\\n )}\\n\\n 
\\n                          {patchGenerated}\\n
\\n
\\n )}\\n
\\n )\\n )}\\n \\n )}\\n\\n {/* CATEGORY 2: PAINEL DE CONTROLE (SUB-TABS: DOCTOR, INSPECT, OLLAMA, GEMINI) */}\\n {activeCategory === \\'control\\' && (\\n
\\n {/* Category Segmented Controls (Sub-Tabs Switcher) */}\\n
\\n setControlSubTab(\\'doctor\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n controlSubTab === \\'doctor\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n Doctor\\n \\n setControlSubTab(\\'inspect\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n controlSubTab === \\'inspect\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n Inspeção\\n \\n setControlSubTab(\\'ollama\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n controlSubTab === \\'ollama\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n Ollama\\n \\n setControlSubTab(\\'gemini\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n controlSubTab === \\'gemini\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n Gemini Status\\n \\n
\\n\\n {/* Sub-tab Renders */}\\n
\\n {controlSubTab === \\'doctor\\' && }\\n {controlSubTab === \\'inspect\\' && }\\n {controlSubTab === \\'ollama\\' && }\\n {controlSubTab === \\'gemini\\' && }\\n
\\n
\\n )}\\n\\n {/* CATEGORY 3: CENTRAL DE DIAGNÓSTICO (SUB-TABS: RUN, ACCEPTANCE) */}\\n {activeCategory === \\'diagnostics\\' && (\\n
\\n
\\n setDiagnosticsSubTab(\\'run\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n diagnosticsSubTab === \\'run\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n Vertical Runs\\n \\n setDiagnosticsSubTab(\\'acceptance\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n diagnosticsSubTab === \\'acceptance\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n M1 Acceptance\\n \\n
\\n\\n
\\n {diagnosticsSubTab === \\'run\\' && }\\n {diagnosticsSubTab === \\'acceptance\\' && }\\n
\\n
\\n )}\\n\\n {/* CATEGORY 4: QUALIFICAÇÃO DE MODELOS (SUB-TABS: ROUTING, SECOND EXECUTOR, CHARACTERIZE, DECISION BASIS) */}\\n {activeCategory === \\'models\\' && (\\n
\\n
\\n setModelsSubTab(\\'routing\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n modelsSubTab === \\'routing\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n Roteamento Tiers\\n \\n setModelsSubTab(\\'second_executor\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n modelsSubTab === \\'second_executor\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n Segunda Qualificação\\n \\n setModelsSubTab(\\'characterize\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n modelsSubTab === \\'characterize\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n Caracterização\\n \\n setModelsSubTab(\\'decision_basis\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n modelsSubTab === \\'decision_basis\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n Decision Basis\\n \\n
\\n\\n
\\n {modelsSubTab === \\'routing\\' && }\\n {modelsSubTab === \\'second_executor\\' && }\\n {modelsSubTab === \\'characterize\\' && }\\n {modelsSubTab === \\'decision_basis\\' && }\\n
\\n
\\n )}\\n\\n {/* CATEGORY 5: VALIDAÇÃO (SUB-TABS: TESTS, M2M3, FIXTURES) */}\\n {activeCategory === \\'validation\\' && (\\n
\\n
\\n setValidationSubTab(\\'tests\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n validationSubTab === \\'tests\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n Suíte de Testes\\n \\n setValidationSubTab(\\'m2m3\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n validationSubTab === \\'m2m3\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n M2/M3 Portões\\n \\n setValidationSubTab(\\'fixtures\\')}\\n className={`flex-1 py-1.5 text-[10px] font-bold uppercase tracking-wider rounded transition cursor-pointer text-center select-none ${\\n validationSubTab === \\'fixtures\\' ? \\'bg-slate-950 text-indigo-400 shadow-sm border border-slate-800\\' : \\'text-slate-400 hover:text-slate-200\\'\\n }`}\\n >\\n Fixtures\\n \\n
\\n\\n
\\n {validationSubTab === \\'tests\\' && }\\n {validationSubTab === \\'m2m3\\' && }\\n {validationSubTab === \\'fixtures\\' && }\\n
\\n
\\n )}\\n\\n \\n
\\n
\\n );\\n}\\n'\n\n----------------------------------------------------------------------\nRan 565 tests in 19.332s\n\nFAILED (failures=1, skipped=7)\n" + }, + { + "command": [ + "C:\\Users\\igorb\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-m", + "compileall", + "-q", + "src", + "tests", + "tools" + ], + "returncode": 0, + "status": "PASS", + "stdout": "", + "stderr": "" + } + ], + "baseline_failure_scope": { + "test": "tests.test_r5_product_surface.R5ProductSurfaceTests.test_ui_marks_preview_surfaces", + "source": "src/App.tsx", + "patch_touches_test": false, + "patch_touches_source": false + } +} diff --git a/pyproject.toml b/pyproject.toml index b2d816bc..fc54b8cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,13 +5,13 @@ build-backend = "setuptools.build_meta" [project] name = "codepro" dynamic = ["version"] -description = "Executor-agnostic software-engineering chassis with an empirically tested CLI boundary." +description = "Executor-agnostic software-engineering chassis with a tested CLI and evidence contracts." readme = "README.md" requires-python = ">=3.12" dependencies = [] [project.scripts] -codepro = "arkx.cli:main" +codepro = "codepro.cli:main" [tool.setuptools] package-dir = {"" = "src"} @@ -20,7 +20,7 @@ package-dir = {"" = "src"} where = ["src"] [tool.setuptools.dynamic] -version = {attr = "arkx.__version__"} +version = {attr = "codepro._version.__version__"} [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/arkx/__init__.py b/src/arkx/__init__.py index 969c129b..bd89a659 100644 --- a/src/arkx/__init__.py +++ b/src/arkx/__init__.py @@ -20,7 +20,8 @@ "p82_localization", "p82_editing", "orchestration", "recovery", "acceptance", "promotion", "characterization", "progress", "routing", "planning", "verification", "handoff", "composition", - "executor_qualification", "p82", "p82_baseline", "swebench_authority", + "executor_qualification", "p82", "p82_baseline", "performance_baseline", + "swebench_authority", ) @@ -29,7 +30,13 @@ def __getattr__(name: str): if name == "OrchestrationExecutionResult": return OrchestrationExecutionResult for module_name in _LEGACY_MODULES: - module = import_module(f"{__name__}.{module_name}") + qualified_name = f"{__name__}.{module_name}" + try: + module = import_module(qualified_name) + except ModuleNotFoundError as error: + if error.name != qualified_name: + raise + continue if hasattr(module, name): value = getattr(module, name) globals()[name] = value diff --git a/src/arkx/cli.py b/src/arkx/cli.py index b610a6e9..9c16ae82 100644 --- a/src/arkx/cli.py +++ b/src/arkx/cli.py @@ -98,6 +98,14 @@ def build_parser() -> argparse.ArgumentParser: nargs=argparse.REMAINDER, help="Executor argv after --. No shell interpretation is used.", ) + + baseline_parser = subparsers.add_parser( + "baseline-cost", + help="Measure local overhead without changing behavior.", + ) + baseline_parser.add_argument("--workspace", default=".") + baseline_parser.add_argument("--output", required=True) + baseline_parser.add_argument("--samples", type=int, default=5) return parser @@ -204,6 +212,16 @@ def main(argv: Sequence[str] | None = None) -> int: return _inspect(args.path, as_json=args.as_json) if args.command == "run": return _run(args) + if args.command == "baseline-cost": + from .performance_baseline import main as performance_baseline_main + + return performance_baseline_main( + [ + "--workspace", args.workspace, + "--output", args.output, + "--samples", str(args.samples), + ] + ) parser.error(f"unsupported command: {args.command}") return 2 diff --git a/src/arkx/p82.py b/src/arkx/p82.py new file mode 100644 index 00000000..c6c27c16 --- /dev/null +++ b/src/arkx/p82.py @@ -0,0 +1,486 @@ +"""P8.2 minimal mechanism trial design. + +This module defines the controlled A/B/C/D design only. It never invokes an +executor, applies a mechanism, verifies a patch, or makes an acceptance +decision. Results must be recorded later as :class:`ExecutorTrial` records. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from decimal import Decimal +import json +from statistics import median +from typing import Any, Iterable + +from .composition import Mechanism, Treatment +from .executor_qualification import ( + ExecutionBudget, + ExecutionEnvironment, + ExecutorIdentity, + QualificationTask, +) + + +SCHEMA_VERSION = 1 + + +class P82Treatment(str, Enum): + A = "A" + B = "B" + C = "C" + D = "D" + + +class TaskStratum(str, Enum): + CHALLENGER_RECOVERY = "challenger_recovery" + LOCALIZATION_CONTEXT = "localization_context" + MINI_CONTROL = "mini_control" + HARD_ALL_FAIL = "hard_all_fail" + + +class P82RunStatus(str, Enum): + NOT_EXECUTED = "NOT_EXECUTED" + BLOCKED = "BLOCKED" + EXECUTED = "EXECUTED" + + +class P82Comparability(str, Enum): + COMPARABLE = "COMPARABLE" + NOT_COMPARABLE = "NOT_COMPARABLE" + INDETERMINATE = "INDETERMINATE" + + +@dataclass(frozen=True) +class P82Criteria: + primary_metrics: tuple[str, ...] = ("verified_acceptance", "cost_to_acceptance") + secondary_metrics: tuple[str, ...] = ( + "tokens_to_acceptance", "wall_time_seconds", "model_calls", "turns", + "repeated_exploration", "repeated_file_reads_searches", "edit_failures", + "retries", "replans", "unnecessary_diff", "regressions", "human_intervention", + ) + guardrail: str = "false_pass_must_not_increase" + + def to_dict(self) -> dict[str, Any]: + return {"primary_metrics": list(self.primary_metrics), "secondary_metrics": list(self.secondary_metrics), "guardrail": self.guardrail} + + def to_json(self) -> str: + return _json(self.to_dict()) + + +def _json(value: Any) -> str: + return json.dumps(_jsonable(value), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _jsonable(value: Any) -> Any: + if isinstance(value, Decimal): + return str(value) + if isinstance(value, Enum): + return value.value + if hasattr(value, "to_dict"): + return _jsonable(value.to_dict()) + if isinstance(value, dict): + return {key: _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + return value + + +def _treatment(name: P82Treatment) -> Treatment: + mechanisms = { + P82Treatment.A: (), + P82Treatment.B: (Mechanism.P8_SAFE_EDITOR,), + P82Treatment.C: (Mechanism.P8_ENHANCED_REPOSITORY_CONTEXT,), + P82Treatment.D: (Mechanism.P8_SAFE_EDITOR, Mechanism.P8_ENHANCED_REPOSITORY_CONTEXT), + } + return Treatment(name.value, mechanisms[name]) + + +@dataclass(frozen=True) +class P82ExperimentConfig: + """Frozen controls shared by every planned arm.""" + + executor: ExecutorIdentity + model: str | None + environment: ExecutionEnvironment + budget: ExecutionBudget + tasks: tuple[QualificationTask, ...] + replicate_ids: tuple[str, ...] = ("rep-1",) + verification_contract: str | None = None + acceptance_authority: str | None = None + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + tasks = tuple(self.tasks) + if not tasks: + raise ValueError("P8.2 requires at least one task") + if len({task.task_id for task in tasks}) != len(tasks): + raise ValueError("P8.2 task identifiers must be unique") + replicates = tuple(self.replicate_ids) + if not replicates or any(not replicate for replicate in replicates): + raise ValueError("P8.2 requires non-empty replicate identifiers") + if len(set(replicates)) != len(replicates): + raise ValueError("P8.2 replicate identifiers must be unique") + object.__setattr__(self, "tasks", tuple(sorted(tasks, key=lambda task: task.task_id))) + object.__setattr__(self, "replicate_ids", tuple(sorted(replicates))) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "executor": self.executor.to_dict(), + "model": self.model, + "environment": self.environment.to_dict(), + "budget": self.budget.to_dict(), + "tasks": [task.to_dict() for task in self.tasks], + "replicate_ids": list(self.replicate_ids), + "verification_contract": self.verification_contract, + "acceptance_authority": self.acceptance_authority, + } + + +@dataclass(frozen=True) +class P82TrialPlan: + treatment: Treatment + task: QualificationTask + replicate_id: str + + def to_dict(self) -> dict[str, Any]: + return { + "treatment": self.treatment.to_dict(), + "task": self.task.to_dict(), + "replicate_id": self.replicate_id, + } + + +@dataclass(frozen=True) +class P82ExperimentPlan: + config: P82ExperimentConfig + treatments: tuple[Treatment, ...] + trials: tuple[P82TrialPlan, ...] + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "config": self.config.to_dict(), + "treatments": [treatment.to_dict() for treatment in self.treatments], + "trials": [trial.to_dict() for trial in self.trials], + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class TaskSampleTask: + task: QualificationTask + stratum: TaskStratum + provenance: tuple[str, ...] | None = None + evidence: tuple[str, ...] | None = None + + def __post_init__(self) -> None: + for name, values in (("provenance", self.provenance), ("evidence", self.evidence)): + if values is not None and any(not value for value in values): + raise ValueError(f"{name} references must be non-empty when present") + + def to_dict(self) -> dict[str, Any]: + return { + "task": self.task.to_dict(), + "stratum": self.stratum.value, + "provenance": None if self.provenance is None else sorted(set(self.provenance)), + "evidence": None if self.evidence is None else sorted(set(self.evidence)), + } + + +@dataclass(frozen=True) +class TaskSampleManifest: + sample_id: str + tasks: tuple[TaskSampleTask, ...] + sample_revision: str | None = None + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + if not self.sample_id: + raise ValueError("sample_id must be non-empty") + tasks = tuple(sorted(self.tasks, key=lambda item: item.task.task_id)) + if not tasks: + raise ValueError("task sample must contain at least one task") + if len({item.task.task_id for item in tasks}) != len(tasks): + raise ValueError("task sample task identifiers must be unique") + object.__setattr__(self, "tasks", tasks) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "sample_id": self.sample_id, + "sample_revision": self.sample_revision, + "tasks": [task.to_dict() for task in self.tasks], + } + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "TaskSampleManifest": + tasks = tuple( + TaskSampleTask( + QualificationTask(**item["task"]), + TaskStratum(item["stratum"]), + None if item.get("provenance") is None else tuple(item["provenance"]), + None if item.get("evidence") is None else tuple(item["evidence"]), + ) + for item in value["tasks"] + ) + return cls(value["sample_id"], tasks, value.get("sample_revision"), value.get("schema_version", SCHEMA_VERSION)) + + +@dataclass(frozen=True) +class P82RunManifest: + """All run identity needed to establish treatment-only comparability.""" + + protocol_version: str | None + treatment: Treatment + executor: ExecutorIdentity + model: str | None + temperature: str | None + reasoning_configuration: str | None + environment: ExecutionEnvironment + task_sample: TaskSampleManifest + attempts: int | None + token_budget: int | None + cost_budget: Decimal | None + wall_time_limit: Decimal | None + verification_identity: str | None + acceptance_identity: str | None + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + for name, value in (("attempts", self.attempts), ("token_budget", self.token_budget), + ("cost_budget", self.cost_budget), ("wall_time_limit", self.wall_time_limit)): + if value is not None and value < 0: + raise ValueError(f"{name} must be non-negative when present") + if self.treatment.name not in {item.value for item in P82Treatment}: + raise ValueError("P8.2 run treatment must be one of A, B, C, or D") + + def _control_dict(self) -> dict[str, Any]: + value = self.to_dict() + value.pop("treatment") + return value + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "protocol_version": self.protocol_version, + "treatment": self.treatment.to_dict(), + "executor": self.executor.to_dict(), + "model": self.model, + "temperature": self.temperature, + "reasoning_configuration": self.reasoning_configuration, + "environment": self.environment.to_dict(), + "task_sample": self.task_sample.to_dict(), + "attempts": self.attempts, + "token_budget": self.token_budget, + "cost_budget": self.cost_budget, + "wall_time_limit": self.wall_time_limit, + "verification_identity": self.verification_identity, + "acceptance_identity": self.acceptance_identity, + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "P82RunManifest": + executor = ExecutorIdentity(**value["executor"]) + environment = ExecutionEnvironment(**value["environment"]) + budget_fields = { + "max_tokens": value["token_budget"], "max_cost": value["cost_budget"], + "max_wall_time": value["wall_time_limit"], "max_attempts": value["attempts"], + "max_executor_invocations": None, + } + # P8.2 records its budget fields explicitly; qualification can still + # receive the resulting ExecutionBudget without inventing a limit. + budget_fields["max_cost"] = None if budget_fields["max_cost"] is None else Decimal(str(budget_fields["max_cost"])) + budget_fields["max_wall_time"] = None if budget_fields["max_wall_time"] is None else Decimal(str(budget_fields["max_wall_time"])) + treatment = Treatment(value["treatment"]["name"], tuple(Mechanism(item) for item in value["treatment"]["enabled_mechanisms"])) + return cls( + protocol_version=value.get("protocol_version"), treatment=treatment, executor=executor, + model=value.get("model"), temperature=value.get("temperature"), + reasoning_configuration=value.get("reasoning_configuration"), environment=environment, + task_sample=TaskSampleManifest.from_dict(value["task_sample"]), attempts=value.get("attempts"), + token_budget=value.get("token_budget"), cost_budget=budget_fields["max_cost"], + wall_time_limit=budget_fields["max_wall_time"], verification_identity=value.get("verification_identity"), + acceptance_identity=value.get("acceptance_identity"), schema_version=value.get("schema_version", SCHEMA_VERSION), + ) + + +@dataclass(frozen=True) +class P82Observation: + status: P82RunStatus + verified_acceptance: bool | None = None + cost_to_acceptance: Decimal | None = None + tokens_to_acceptance: int | None = None + wall_time_seconds: Decimal | None = None + model_calls: int | None = None + turns: int | None = None + repeated_exploration: int | None = None + repeated_file_reads_searches: int | None = None + edit_failures: int | None = None + retries: int | None = None + replans: int | None = None + unnecessary_diff: int | None = None + regressions: int | None = None + human_intervention: int | None = None + false_pass_evidence: bool | None = None + verification_outcome: str | None = None + + def __post_init__(self) -> None: + for name in ("tokens_to_acceptance", "model_calls", "turns", "repeated_exploration", + "repeated_file_reads_searches", "edit_failures", "retries", "replans", + "unnecessary_diff", "regressions", "human_intervention"): + value = getattr(self, name) + if value is not None and value < 0: + raise ValueError(f"{name} must be non-negative when present") + for name in ("cost_to_acceptance", "wall_time_seconds"): + value = getattr(self, name) + if value is not None and value < 0: + raise ValueError(f"{name} must be non-negative when present") + + def to_dict(self) -> dict[str, Any]: + return {"status": self.status.value, **{name: getattr(self, name) for name in ( + "verified_acceptance", "cost_to_acceptance", "tokens_to_acceptance", "wall_time_seconds", + "model_calls", "turns", "repeated_exploration", "repeated_file_reads_searches", + "edit_failures", "retries", "replans", "unnecessary_diff", "regressions", + "human_intervention", "false_pass_evidence", "verification_outcome")}} + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "P82Observation": + values = dict(value) + values["status"] = P82RunStatus(values["status"]) + for name in ("cost_to_acceptance", "wall_time_seconds"): + if values.get(name) is not None: + values[name] = Decimal(str(values[name])) + return cls(**values) + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class P82ObservationRecord: + run: P82RunManifest + observation: P82Observation + + def to_dict(self) -> dict[str, Any]: + return {"run": self.run.to_dict(), "observation": self.observation.to_dict()} + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class P82TreatmentSummary: + treatment: str + record_count: int + executed_count: int + not_executed_count: int + blocked_count: int + verified_acceptance_rate: Decimal | None + median_cost_to_acceptance: Decimal | None + median_tokens_to_acceptance: Decimal | None + false_pass_rate: Decimal | None + + def to_dict(self) -> dict[str, Any]: + return {name: getattr(self, name) for name in ( + "treatment", "record_count", "executed_count", "not_executed_count", "blocked_count", + "verified_acceptance_rate", "median_cost_to_acceptance", "median_tokens_to_acceptance", + "false_pass_rate")} + + +def p82_manifest() -> tuple[Treatment, ...]: + """Return the canonical A/B/C/D arms in experimental order.""" + + return tuple(_treatment(name) for name in P82Treatment) + + +def build_p82_plan(config: P82ExperimentConfig) -> P82ExperimentPlan: + """Create deterministic plans without executing or fabricating results.""" + + treatments = p82_manifest() + trials = tuple( + P82TrialPlan(treatment, task, replicate) + for task in config.tasks + for replicate in config.replicate_ids + for treatment in treatments + ) + return P82ExperimentPlan(config, treatments, trials) + + +def plan_is_comparable(plan: P82ExperimentPlan) -> bool | None: + """Return True when all shared controls are known, otherwise None. + + There is no False result: a generated plan has one frozen control set, so + missing evidence remains unknown rather than being treated as a mismatch. + """ + + values = ( + plan.config.executor.version, + plan.config.executor.configuration_digest, + plan.config.model, + plan.config.environment.repository_revision, + plan.config.environment.runtime, + plan.config.environment.platform, + plan.config.environment.toolchain, + plan.config.environment.environment_digest, + plan.config.budget.max_tokens, + plan.config.budget.max_cost, + plan.config.budget.max_wall_time, + plan.config.budget.max_attempts, + plan.config.budget.max_executor_invocations, + plan.config.verification_contract, + plan.config.acceptance_authority, + ) + return None if any(value is None for value in values) else True + + +def compare_run_manifests(left: P82RunManifest, right: P82RunManifest) -> P82Comparability: + """Compare runs while excluding treatment and preserving insufficient data.""" + + left_values, right_values = left._control_dict(), right._control_dict() + if any(value is None for value in _flatten_values(left_values)) or any(value is None for value in _flatten_values(right_values)): + return P82Comparability.INDETERMINATE + return P82Comparability.COMPARABLE if left_values == right_values else P82Comparability.NOT_COMPARABLE + + +def _flatten_values(value: Any) -> Iterable[Any]: + if isinstance(value, dict): + for item in value.values(): + yield from _flatten_values(item) + elif isinstance(value, list): + for item in value: + yield from _flatten_values(item) + else: + yield value + + +def summarize_p82(records: tuple[P82ObservationRecord, ...]) -> tuple[P82TreatmentSummary, ...]: + """Aggregate observations by arm without treating unavailable data as zero.""" + + result = [] + for treatment in P82Treatment: + selected = tuple(record for record in records if record.run.treatment.name == treatment.value) + executed = tuple(record for record in selected if record.observation.status is P82RunStatus.EXECUTED) + accepted = tuple(record.observation.verified_acceptance for record in executed if record.observation.verified_acceptance is not None) + accepted_costs = tuple(record.observation.cost_to_acceptance for record in executed if record.observation.verified_acceptance is True and record.observation.cost_to_acceptance is not None) + accepted_tokens = tuple(record.observation.tokens_to_acceptance for record in executed if record.observation.verified_acceptance is True and record.observation.tokens_to_acceptance is not None) + false_pass = tuple(record.observation.false_pass_evidence for record in selected if record.observation.false_pass_evidence is not None) + result.append(P82TreatmentSummary( + treatment=treatment.value, + record_count=len(selected), + executed_count=len(executed), + not_executed_count=sum(record.observation.status is P82RunStatus.NOT_EXECUTED for record in selected), + blocked_count=sum(record.observation.status is P82RunStatus.BLOCKED for record in selected), + verified_acceptance_rate=None if not accepted else Decimal(sum(accepted)) / Decimal(len(accepted)), + median_cost_to_acceptance=None if not accepted_costs else median(accepted_costs), + median_tokens_to_acceptance=None if not accepted_tokens else Decimal(str(median(accepted_tokens))), + false_pass_rate=None if not false_pass else Decimal(sum(false_pass)) / Decimal(len(false_pass)), + )) + return tuple(result) diff --git a/src/arkx/p82_baseline.py b/src/arkx/p82_baseline.py new file mode 100644 index 00000000..bd66bcd0 --- /dev/null +++ b/src/arkx/p82_baseline.py @@ -0,0 +1,537 @@ +"""P8.2a baseline execution path. + +This is infrastructure qualification, not mechanism evidence. The runner +captures executor output only; verification and independent acceptance are +separate contracts and are never inferred from an exit status or a diff. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +import hashlib +import importlib.util +import json +from pathlib import Path +import os +import subprocess +from typing import Any, Callable, Protocol + +from .outcomes import AcceptanceDecision, AcceptanceResult, VerificationResult, VerificationState +from .harness import ArtifactRecord, ArtifactStatus, ArtifactStore, ErrorDomain, ErrorEnvelope, LifecycleEvent, Retryability, RunManifest, RunState, derive_attempt_id, validate_execution_artifact_consistency +from .execution import ExecutionRequest, ExecutionResult +from .configuration import ConfigurationSnapshot +from .integration import IntegrationKind, process_error_envelope +from .integration import AdapterIdentity, DependencyObservation, CapabilityProvenance, assess_preflight, AdapterPreflight +from .p82 import P82Observation, P82RunStatus, P82Treatment, TaskStratum +from .submission import SubmissionArtifact +from .timeouts import TimeoutObservation, TimeoutOrigin + + +PROTOCOL_VERSION = "p8.2a-v1" + + +class BaselineExecutionState(str, Enum): + COMPLETED = "COMPLETED" + EXECUTOR_FAILED = "EXECUTOR_FAILED" + BLOCKED = "BLOCKED" + + +class RunnerFailureCategory(str, Enum): + PROCESS_START_FAILURE = "PROCESS_START_FAILURE" + ENVIRONMENT_FAILURE = "ENVIRONMENT_FAILURE" + MODEL_INITIALIZATION_FAILURE = "MODEL_INITIALIZATION_FAILURE" + AGENT_INITIALIZATION_FAILURE = "AGENT_INITIALIZATION_FAILURE" + AGENT_RUN_FAILURE = "AGENT_RUN_FAILURE" + TIMEOUT = "TIMEOUT" + DEADLOCK_HANG = "DEADLOCK/HANG" + OUTPUT_CAPTURE_FAILURE = "OUTPUT_CAPTURE_FAILURE" + PROVIDER_AVAILABILITY_FAILURE = "PROVIDER_AVAILABILITY_FAILURE" + UNKNOWN = "UNKNOWN" + + +@dataclass(frozen=True) +class ProspectiveTask: + task_id: str + source_dataset: str | None + repository: str | None + base_commit: str | None + problem_statement: str | None + stratum: TaskStratum + rationale: str | None + selection_timestamp: str | None + protocol_version: str | None + verification_identity: str | None + acceptance_authority_identity: str | None + workspace: str + + def __post_init__(self) -> None: + if not self.task_id or not self.workspace: + raise ValueError("task_id and workspace must be non-empty") + + def to_dict(self) -> dict[str, Any]: + value = {name: getattr(self, name) for name in ( + "task_id", "source_dataset", "repository", "base_commit", "problem_statement", + "stratum", "rationale", "selection_timestamp", "protocol_version", + "verification_identity", "acceptance_authority_identity", "workspace")} + value["stratum"] = self.stratum.value + return value + + +@dataclass(frozen=True) +class BaselineSampleManifest: + sample_id: str + wave: str + selection_rule: str + tasks: tuple[ProspectiveTask, ...] + protocol_version: str = PROTOCOL_VERSION + + def __post_init__(self) -> None: + if not self.sample_id or not self.wave or not self.selection_rule: + raise ValueError("sample identity, wave, and selection rule are required") + if not self.tasks: + raise ValueError("prospective sample cannot be empty") + if len({task.task_id for task in self.tasks}) != len(self.tasks): + raise ValueError("sample task identifiers must be unique") + object.__setattr__(self, "tasks", tuple(sorted(self.tasks, key=lambda task: task.task_id))) + + def to_dict(self) -> dict[str, Any]: + return {"sample_id": self.sample_id, "wave": self.wave, "selection_rule": self.selection_rule, + "protocol_version": self.protocol_version, "tasks": [task.to_dict() for task in self.tasks]} + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +@dataclass(frozen=True) +class BaselineRunConfig: + provider: str | None + model: str | None + model_version: str | None + temperature: str | None + reasoning_configuration: str | None + model_configuration: dict[str, Any] | None + max_tokens: int | None + max_cost: str | None + wall_time_seconds: int | None + mini_version: str | None + python_executable: str | None + runner_version: str + artifact_root: str + model_class: str | None = None + treatment: P82Treatment = P82Treatment.A + environment: str = "local" + bash_executable: str | None = None + environment_configuration: dict[str, Any] | None = None + provider_retry_attempts: int = 0 + provider_retry_backoff_seconds: float = 0.0 + agent_step_limit: int | None = None + + def __post_init__(self) -> None: + if self.treatment is not P82Treatment.A: + raise ValueError("P8.2a baseline runner accepts treatment A only") + for name in ("max_tokens", "wall_time_seconds"): + value = getattr(self, name) + if value is not None and value < 0: + raise ValueError(f"{name} must be non-negative when present") + if self.provider_retry_attempts < 0 or self.provider_retry_backoff_seconds < 0: + raise ValueError("provider retry limits must be non-negative") + + def missing_identity(self) -> tuple[str, ...]: + return tuple(name for name in ("provider", "model", "mini_version", "python_executable") if getattr(self, name) is None) + + def identity_digest(self) -> str: + """Return a stable digest for the frozen execution configuration.""" + return self.configuration_snapshot().digest() + + def configuration_snapshot(self) -> ConfigurationSnapshot: + """Return the canonical, secret-safe configuration identity.""" + value = { + "provider": self.provider, + "model": self.model, + "model_version": self.model_version, + "temperature": self.temperature, + "reasoning_configuration": self.reasoning_configuration, + "model_configuration": self.model_configuration, + "max_tokens": self.max_tokens, + "max_cost": self.max_cost, + "wall_time_seconds": self.wall_time_seconds, + "mini_version": self.mini_version, + "python_executable": self.python_executable, + "model_class": self.model_class, + "runner_version": self.runner_version, + "treatment": self.treatment.value, + "environment": self.environment, + "bash_executable": self.bash_executable, + "environment_configuration": self.environment_configuration, + "provider_retry_attempts": self.provider_retry_attempts, + "provider_retry_backoff_seconds": self.provider_retry_backoff_seconds, + "agent_step_limit": self.agent_step_limit, + } + return ConfigurationSnapshot("p82-baseline-executor", self.runner_version, value) + + def budget_digest(self) -> str: + value = { + "max_tokens": self.max_tokens, + "max_cost": self.max_cost, + "wall_time_seconds": self.wall_time_seconds, + "provider_retry_attempts": self.provider_retry_attempts, + "provider_retry_backoff_seconds": self.provider_retry_backoff_seconds, + } + encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:16] + + +@dataclass(frozen=True) +class ExecutionArtifact: + run_id: str + state: BaselineExecutionState + task_id: str + treatment: str + attempt: int + started_at: str + finished_at: str + exit_code: int | None + failure: str | None + failure_category: RunnerFailureCategory | None + diff_path: str | None + log_path: str | None + trajectory_path: str | None + model_usage: dict[str, Any] | None + artifact_path: str | None = None + trial_id: str | None = None + attempt_id: str | None = None + manifest_path: str | None = None + error: ErrorEnvelope | None = None + submission_path: str | None = None + + def to_dict(self) -> dict[str, Any]: + value = {name: getattr(self, name) for name in ( + "run_id", "state", "task_id", "treatment", "attempt", "started_at", "finished_at", + "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", + "model_usage", "artifact_path", "trial_id", "attempt_id", "manifest_path", "error")} + value["submission_path"] = self.submission_path + if isinstance(value.get("error"), ErrorEnvelope): + value["error"] = value["error"].to_dict() + for name in ("state", "failure_category"): + if isinstance(value[name], Enum): + value[name] = value[name].value + return value + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +class HeadlessInvoker(Protocol): + def __call__(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: ... + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="milliseconds") + + +class MiniSweAgentHeadlessRunner: + """Narrow runner. It never calls verification or acceptance.""" + + def __init__(self, invoker: HeadlessInvoker | None = None) -> None: + self.invoker = invoker or self._subprocess_invoker + + def run(self, task: ProspectiveTask, config: BaselineRunConfig, attempt: int = 1) -> ExecutionArtifact: + if attempt < 1: + raise ValueError("attempt must be positive") + missing = config.missing_identity() + store = ArtifactStore(config.artifact_root) + trial_id = hashlib.sha256( + f"{task.task_id}|{task.base_commit}|A|{config.identity_digest()}".encode("utf-8") + ).hexdigest()[:16] + run_id = derive_attempt_id( + trial_id=trial_id, + attempt_number=attempt, + configuration_digest=config.identity_digest(), + ) + try: + root = store.reserve_attempt(task.task_id, attempt) + except FileExistsError: + return self._blocked(run_id, task, attempt, "artifact already exists; refusing silent overwrite") + started = _now() + trajectory = root / "trajectory.json" + if missing: + return self._finish(root, run_id, trial_id, task, config, attempt, started, False, BaselineExecutionState.BLOCKED, None, + f"missing explicit identity: {','.join(missing)}", RunnerFailureCategory.UNKNOWN, None, None, None, None) + try: + exit_code, log, usage = self.invoker(task, config, trajectory) + log_path = root / "executor.log" + log_path.write_text(log, encoding="utf-8") + diff_path = root / "diff.patch" + submission = SubmissionArtifact.from_git_workspace(task.workspace) + diff_path.write_text(submission.normalized_patch, encoding="utf-8", newline="\n") + submission_path = root / "submission.json" + submission_path.write_text(json.dumps(submission.to_dict(), ensure_ascii=False, sort_keys=True) + "\n", encoding="utf-8") + if exit_code == 75: + return self._finish(root, run_id, trial_id, task, config, attempt, started, True, BaselineExecutionState.BLOCKED, exit_code, + "provider availability failure after bounded retries", + RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE, + str(diff_path), str(log_path), str(trajectory), usage, str(submission_path)) + state = BaselineExecutionState.COMPLETED if exit_code == 0 else BaselineExecutionState.EXECUTOR_FAILED + return self._finish(root, run_id, trial_id, task, config, attempt, started, True, state, exit_code, None if exit_code == 0 else "executor returned non-zero", None if exit_code == 0 else RunnerFailureCategory.AGENT_RUN_FAILURE, str(diff_path), str(log_path), str(trajectory), usage, str(submission_path)) + except subprocess.TimeoutExpired as error: + log_path = root / "executor.log" + stdout = self._decode_process_output(error.stdout) + stderr = self._decode_process_output(error.stderr) + log_path.write_text(stdout + stderr, encoding="utf-8", newline="\n") + diagnostics_path = root / "timeout-observability.json" + diagnostics_path.write_text(json.dumps({ + "event": "runner_timeout", + "timeout_origin": "runner_subprocess_watchdog", + "timeout_seconds": error.timeout, + "started_at": started, + "finished_at": _now(), + "exception_type": type(error).__name__, + "exception": str(error), + "model_diagnostics_path": str(root / "model-observability.json"), + "timeout": TimeoutObservation(TimeoutOrigin.RUNNER_WATCHDOG, error.timeout, "CodePro MiniSweAgentHeadlessRunner", "subprocess.TimeoutExpired").to_dict(), + }, ensure_ascii=False, sort_keys=True) + "\n", encoding="utf-8") + return self._finish( + root, run_id, trial_id, task, config, attempt, started, True, + BaselineExecutionState.EXECUTOR_FAILED, None, str(error), + RunnerFailureCategory.TIMEOUT, None, str(log_path), + str(trajectory) if trajectory.exists() else None, None, + ) + except Exception as error: + message = f"{type(error).__name__}: {error}" + return self._finish(root, run_id, trial_id, task, config, attempt, started, True, BaselineExecutionState.EXECUTOR_FAILED, None, message, self._classify_failure(message), None, None, None, None) + + @staticmethod + def _classify_failure(message: str) -> RunnerFailureCategory: + lowered = message.lower() + if "filenotfound" in lowered or "process" in lowered and "start" in lowered: + return RunnerFailureCategory.PROCESS_START_FAILURE + if "timeout" in lowered or "timed out" in lowered: + return RunnerFailureCategory.TIMEOUT + if "environment" in lowered or "docker" in lowered or "workspace" in lowered: + return RunnerFailureCategory.ENVIRONMENT_FAILURE + if "model" in lowered and ("validation" in lowered or "initial" in lowered or "config" in lowered): + return RunnerFailureCategory.MODEL_INITIALIZATION_FAILURE + if "agent" in lowered and ("initial" in lowered or "config" in lowered): + return RunnerFailureCategory.AGENT_INITIALIZATION_FAILURE + if "trajectory" in lowered or "capture" in lowered: + return RunnerFailureCategory.OUTPUT_CAPTURE_FAILURE + if "deadlock" in lowered or "hang" in lowered: + return RunnerFailureCategory.DEADLOCK_HANG + return RunnerFailureCategory.AGENT_RUN_FAILURE + + @staticmethod + def _decode_process_output(value: str | bytes | None) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return value + + def _subprocess_invoker(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: + worker = Path(__file__).parents[2] / "tools" / "run_mini_headless.py" + payload = {"task": task.problem_statement, "workspace": task.workspace, "model": config.model, + "model_class": config.model_class, + "trajectory": str(trajectory_path), "max_cost": config.max_cost, "wall_time_seconds": config.wall_time_seconds, + "model_configuration": config.model_configuration, "environment": config.environment, + "bash_executable": config.bash_executable, "provider_retry_attempts": config.provider_retry_attempts, + "provider_retry_backoff_seconds": config.provider_retry_backoff_seconds, + "agent_step_limit": config.agent_step_limit, + "environment_configuration": config.environment_configuration, + "diagnostics_path": str(trajectory_path.with_name("model-observability.json").resolve())} + timeout = None if config.wall_time_seconds is None else config.wall_time_seconds + 10 + completed = subprocess.run( + [config.python_executable, str(worker)], input=json.dumps(payload), text=True, + capture_output=True, cwd=task.workspace, timeout=timeout, env=os.environ.copy(), + ) + combined_log = completed.stdout + completed.stderr + usage = None + if completed.returncode == 0: + lines = [line for line in completed.stdout.splitlines() if line.strip()] + if lines: + try: + worker_result = json.loads(lines[-1]) + usage = worker_result.get("usage") + result = worker_result.get("result", {}) + if isinstance(result, dict) and result.get("failure_category") == RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE.value: + return 75, combined_log, usage + except (json.JSONDecodeError, AttributeError): + usage = None + return completed.returncode, combined_log, usage + + def _blocked(self, run_id: str, task: ProspectiveTask, attempt: int, failure: str) -> ExecutionArtifact: + now = _now() + return ExecutionArtifact(run_id, BaselineExecutionState.BLOCKED, task.task_id, "A", attempt, now, now, None, failure, RunnerFailureCategory.UNKNOWN, None, None, None, None) + + def _finish(self, root: Path, run_id: str, trial_id: str, task: ProspectiveTask, config: BaselineRunConfig, attempt: int, started: str, execution_started: bool, state: BaselineExecutionState, exit_code: int | None, failure: str | None, failure_category: RunnerFailureCategory | None, diff_path: str | None, log_path: str | None, trajectory_path: str | None, usage: dict[str, Any] | None, submission_path: str | None = None) -> ExecutionArtifact: + store = ArtifactStore(root.parents[1]) + artifact_path = root / "execution.json" + manifest_path = root / "manifest.json" + error = None + if failure: + raw_refs = tuple(path for path in (log_path, trajectory_path) if path) + if failure_category is RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE: + # This is the one legacy classification with an explicit retry policy. + error = ErrorEnvelope( + domain=ErrorDomain.PROVIDER, + code=failure_category.value, + message=failure, + retryability=Retryability.RETRYABLE, + raw_evidence_refs=raw_refs, + attempt_number=attempt, + ) + elif failure_category is RunnerFailureCategory.UNKNOWN: + # Missing identity is a harness precondition failure, not an + # executor invocation failure. + error = ErrorEnvelope( + domain=ErrorDomain.HARNESS, + code=RunnerFailureCategory.UNKNOWN.value, + message=failure, + retryability=Retryability.UNKNOWN, + raw_evidence_refs=raw_refs, + attempt_number=attempt, + ) + else: + kind = IntegrationKind.SANDBOX if failure_category in ( + RunnerFailureCategory.ENVIRONMENT_FAILURE, + RunnerFailureCategory.PROCESS_START_FAILURE, + ) else IntegrationKind.EXECUTOR + error = process_error_envelope( + kind, + returncode=exit_code, + message=failure, + raw_evidence_refs=raw_refs, + attempt_number=attempt, + ) + manifest = RunManifest( + experiment_id=f"p82a:{config.runner_version}", trial_id=trial_id, attempt_id=run_id, + attempt_number=attempt, task_id=task.task_id, task_revision=task.base_commit, + treatment="A", executor="mini-swe-agent", provider=config.provider, + model=config.model, sandbox=config.environment, repository_revision=task.base_commit, + configuration_digest=config.identity_digest(), protocol_version=task.protocol_version or PROTOCOL_VERSION, + budget_digest=config.budget_digest(), + state={BaselineExecutionState.COMPLETED: RunState.COMPLETED, BaselineExecutionState.EXECUTOR_FAILED: RunState.FAILED, BaselineExecutionState.BLOCKED: RunState.BLOCKED}[state], + state_history=(RunState.PLANNED, RunState.STARTED) + ((RunState.EXECUTING,) if execution_started else ()) + ({BaselineExecutionState.COMPLETED: (RunState.COMPLETED,), BaselineExecutionState.EXECUTOR_FAILED: (RunState.FAILED,), BaselineExecutionState.BLOCKED: (RunState.BLOCKED,)}[state]), + lifecycle_events=tuple( + LifecycleEvent(current, started if index < 2 else _now()) + for index, current in enumerate((RunState.PLANNED, RunState.STARTED) + ((RunState.EXECUTING,) if execution_started else ()) + ({BaselineExecutionState.COMPLETED: (RunState.COMPLETED,), BaselineExecutionState.EXECUTOR_FAILED: (RunState.FAILED,), BaselineExecutionState.BLOCKED: (RunState.BLOCKED,)}[state])) + ), + artifacts=tuple( + ArtifactRecord( + kind=kind, + status=ArtifactStatus.PRESENT if path == str(artifact_path) or (path is not None and Path(path).is_file()) else ArtifactStatus.MISSING, + path=path if path == str(artifact_path) or (path is not None and Path(path).is_file()) else None, + required=kind == "execution", + reason=None if path == str(artifact_path) or (path is not None and Path(path).is_file()) else "not produced by this attempt", + ) + for kind, path in (("execution", str(artifact_path)), ("diff", diff_path), ("submission", submission_path), ("executor_log", log_path), ("trajectory", trajectory_path)) + ), + artifact_refs=tuple( + [str(artifact_path)] + [ + path for path in (diff_path, log_path, trajectory_path) + if path and Path(path).is_file() + ] + ), + errors=(error,) if error else (), + ) + manifest.validate_attempt_identity() + manifest.validate_lifecycle() + artifact = ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage, str(artifact_path), trial_id, run_id, str(manifest_path), error, submission_path) + store.write_text_atomic(manifest_path, manifest.to_json() + "\n") + store.write_text_atomic(artifact_path, artifact.to_json() + "\n") + validate_execution_artifact_consistency(manifest, artifact.to_dict()) + store.validate_manifest_references(manifest) + return artifact + + @staticmethod + def _error_domain(category: RunnerFailureCategory | None) -> ErrorDomain: + if category is RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE: + return ErrorDomain.PROVIDER + if category in (RunnerFailureCategory.ENVIRONMENT_FAILURE, RunnerFailureCategory.PROCESS_START_FAILURE): + return ErrorDomain.SANDBOX + if category is RunnerFailureCategory.UNKNOWN or category is None: + return ErrorDomain.HARNESS + return ErrorDomain.EXECUTOR + + +class P82BaselineExecutorAdapter: + """Translate the legacy P8.2 runner into the neutral Executor contract.""" + + executor_id = "p82-baseline-mini-swe-agent" + + def __init__(self, task: ProspectiveTask, config: BaselineRunConfig, runner: MiniSweAgentHeadlessRunner | None = None, attempt: int = 1) -> None: + self.task = task + self.config = config + self.runner = runner or MiniSweAgentHeadlessRunner() + self.attempt = attempt + + def preflight(self) -> AdapterPreflight: + """Describe concrete readiness without invoking the runner.""" + available = importlib.util.find_spec("minisweagent") is not None + return assess_preflight( + AdapterIdentity( + IntegrationKind.EXECUTOR, + self.executor_id, + self.config.mini_version, + self.config.configuration_snapshot().digest(), + ), + (DependencyObservation("minisweagent", available, reason=None if available else "python module unavailable"),), + capabilities=("edit",), + capability_digest=self.config.configuration_snapshot().digest(), + capability_provenance=CapabilityProvenance.DECLARED, + ) + + def execute(self, request: ExecutionRequest) -> ExecutionResult: + if request.task_id != self.task.task_id: + raise ValueError("execution request task_id does not match bound baseline task") + if request.treatment != "A": + raise ValueError("P8.2 baseline adapter accepts treatment A only") + artifact = self.runner.run(self.task, self.config, attempt=self.attempt) + state = { + BaselineExecutionState.COMPLETED: RunState.COMPLETED, + BaselineExecutionState.EXECUTOR_FAILED: RunState.FAILED, + BaselineExecutionState.BLOCKED: RunState.BLOCKED, + }[artifact.state] + return ExecutionResult( + state=state, + artifact_refs=tuple(ref for ref in (artifact.artifact_path, artifact.diff_path, artifact.submission_path, artifact.log_path, artifact.trajectory_path, artifact.manifest_path) if ref), + usage=artifact.model_usage, + error=artifact.error, + ) + + +class IndependentAcceptanceAuthority(Protocol): + identity: str + + def decide(self, verification: VerificationResult) -> AcceptanceResult: ... + + +class ExplicitVerificationAcceptance: + """An explicit policy: verification PASS maps to acceptance only by declaration.""" + + def __init__(self, identity: str) -> None: + if not identity: + raise ValueError("acceptance authority identity must be explicit") + self.identity = identity + + def decide(self, verification: VerificationResult) -> AcceptanceResult: + mapping = { + VerificationState.PASS: AcceptanceDecision.ACCEPTED, + VerificationState.FAIL: AcceptanceDecision.REJECTED, + VerificationState.INDETERMINATE: AcceptanceDecision.INDETERMINATE, + VerificationState.NOT_EXECUTED: AcceptanceDecision.NOT_EXECUTED, + VerificationState.BLOCKED: AcceptanceDecision.BLOCKED, + } + return AcceptanceResult(mapping[verification.state], self.identity, verification.state.value, verification.evidence) + + +def baseline_observation(execution: ExecutionArtifact, verification: VerificationResult | None, acceptance: AcceptanceResult | None) -> P82Observation: + if execution.state is BaselineExecutionState.BLOCKED: + status = P82RunStatus.BLOCKED + elif execution.state is not BaselineExecutionState.COMPLETED: + status = P82RunStatus.EXECUTED + else: + status = P82RunStatus.EXECUTED + accepted = None if acceptance is None else acceptance.decision is AcceptanceDecision.ACCEPTED + return P82Observation(status=status, verified_acceptance=accepted) diff --git a/src/arkx/performance_baseline.py b/src/arkx/performance_baseline.py new file mode 100644 index 00000000..7a7eb5d1 --- /dev/null +++ b/src/arkx/performance_baseline.py @@ -0,0 +1,179 @@ +"""Measurement-only baseline for local and controlled-path overhead. + +This module records observed cost. It does not optimize, classify quality, or +promote a path. A failed operation is retained as evidence instead of being +treated as a zero-duration success. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +import json +from pathlib import Path +import platform +import subprocess +import sys +import tempfile +from time import perf_counter_ns +from typing import Any, Callable + +from .characterization import TaskSignals, characterize_timed +from .contracts import Event, EventType +from .submission import SubmissionArtifact + + +SCHEMA_VERSION = 1 +DEFAULT_SAMPLES = 5 + + +@dataclass(frozen=True) +class Measurement: + name: str + samples: tuple[int, ...] + status: str + error: str | None = None + metadata: dict[str, Any] | None = None + + def to_dict(self) -> dict[str, Any]: + ordered = sorted(self.samples) + return { + "name": self.name, + "sample_count": len(self.samples), + "durations_ns": list(self.samples), + "min_ns": min(self.samples) if self.samples else None, + "median_ns": ordered[len(ordered) // 2] if ordered else None, + "max_ns": max(self.samples) if self.samples else None, + "status": self.status, + "error": self.error, + "metadata": self.metadata or {}, + } + + +def _measure(name: str, operation: Callable[[], Any], *, samples: int) -> Measurement: + durations: list[int] = [] + error: str | None = None + for _ in range(samples): + started = perf_counter_ns() + try: + operation() + except Exception as exc: # retain the failure as evidence + error = f"{type(exc).__name__}: {exc}" + return Measurement(name, tuple(durations), "BLOCKED", error) + durations.append(perf_counter_ns() - started) + return Measurement(name, tuple(durations), "PASS", metadata={}) + + +def _git_identity(workspace: Path) -> dict[str, Any]: + observations: dict[str, Any] = {} + for name, command in ( + ("head", ["git", "rev-parse", "HEAD"]), + ("branch", ["git", "branch", "--show-current"]), + ("status", ["git", "status", "--short", "--untracked-files=all"]), + ): + result = subprocess.run(command, cwd=workspace, text=True, capture_output=True, check=False) + observations[name] = { + "command": " ".join(command), + "returncode": result.returncode, + "stdout": result.stdout.strip(), + "stderr": result.stderr.strip(), + } + return observations + + +def _event_log_operation(event_count: int) -> Callable[[], None]: + def operation() -> None: + with tempfile.TemporaryDirectory(prefix="codepro-baseline-") as directory: + path = Path(directory) / "events.jsonl" + with path.open("w", encoding="utf-8", newline="\n") as stream: + for index in range(event_count): + event = Event( + f"2026-01-01T00:00:{index:02d}+00:00", + EventType.EVIDENCE_ADDED, + "baseline-run", + {"index": index}, + ) + stream.write(json.dumps(event.to_dict(), ensure_ascii=False) + "\n") + + return operation + + +def collect_baseline(workspace: str | Path, *, samples: int = DEFAULT_SAMPLES) -> dict[str, Any]: + """Collect a raw, repeatable cost baseline for the current workspace.""" + + if samples < 1: + raise ValueError("samples must be positive") + root = Path(workspace).resolve() + signals = TaskSignals( + candidate_files=("notes.txt",), + dependency_edges=(), + affected_components=("local-text",), + known_tests=("tests/test_performance_baseline.py",), + ambiguity_markers=(), + risk_markers=(), + acceptance_checks=("baseline recorded",), + state_shared=False, + architectural_change=False, + ) + measurements = [ + _measure( + "workspace_submission_capture", + lambda: SubmissionArtifact.from_git_workspace(root), + samples=samples, + ), + _measure("characterization", lambda: characterize_timed(signals), samples=samples), + ] + for event_count in (1, 10, 50): + measurements.append( + _measure( + f"telemetry_event_append_{event_count}", + _event_log_operation(event_count), + samples=samples, + ) + ) + + return { + "schema_version": SCHEMA_VERSION, + "baseline_type": "LOCAL_OVERHEAD_BASELINE", + "collected_at": datetime.now(timezone.utc).isoformat(), + "workspace": str(root), + "python": sys.version, + "platform": platform.platform(), + "git_identity": _git_identity(root), + "sample_policy": {"warmups": 0, "samples": samples}, + "measurements": [measurement.to_dict() for measurement in measurements], + "interpretation": { + "quality_claim": "NOT_MADE", + "agility_impact": "NOT_VERIFIED", + "optimization": "NOT_EXECUTED", + }, + } + + +def write_baseline(report: dict[str, Any], output: str | Path) -> Path: + """Persist raw baseline evidence atomically and return its path.""" + + destination = Path(output) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(destination.name + ".tmp") + temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temporary.replace(destination) + return destination + + +def main(argv: list[str] | None = None) -> int: + import argparse + + parser = argparse.ArgumentParser(description="Collect CodePro local overhead baseline evidence.") + parser.add_argument("--workspace", default=".") + parser.add_argument("--output", required=True) + parser.add_argument("--samples", type=int, default=DEFAULT_SAMPLES) + args = parser.parse_args(argv) + report = collect_baseline(args.workspace, samples=args.samples) + write_baseline(report, args.output) + print(json.dumps(report, ensure_ascii=False, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/codepro/__init__.py b/src/codepro/__init__.py new file mode 100644 index 00000000..5a578eb5 --- /dev/null +++ b/src/codepro/__init__.py @@ -0,0 +1,23 @@ +"""Canonical CodePro namespace during the staged ``arkx`` migration. + +The implementation remains in ``arkx`` for this migration slice. This +facade gives new consumers the product namespace while keeping the legacy +namespace available until the supported API has been migrated and tested. +""" + +from __future__ import annotations + +import arkx as _legacy + +from ._version import __version__ + +__all__ = tuple(_legacy.__all__) + + +def __getattr__(name: str): + """Resolve a supported public symbol from the legacy implementation.""" + return getattr(_legacy, name) + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/src/codepro/_legacy_module.py b/src/codepro/_legacy_module.py new file mode 100644 index 00000000..355b61ab --- /dev/null +++ b/src/codepro/_legacy_module.py @@ -0,0 +1,13 @@ +"""Helpers for staged module-level compatibility with the ``arkx`` namespace.""" + +from __future__ import annotations + +from types import ModuleType + + +def public_names(module: ModuleType) -> tuple[str, ...]: + return tuple(sorted(name for name in dir(module) if not name.startswith("_"))) + + +def resolve(module: ModuleType, name: str): + return getattr(module, name) diff --git a/src/codepro/_version.py b/src/codepro/_version.py new file mode 100644 index 00000000..db90c5d0 --- /dev/null +++ b/src/codepro/_version.py @@ -0,0 +1,3 @@ +"""Build-time version metadata without importing the compatibility facade.""" + +__version__ = "0.3.0.dev0" diff --git a/src/codepro/baseline.py b/src/codepro/baseline.py new file mode 100644 index 00000000..9bca6dc6 --- /dev/null +++ b/src/codepro/baseline.py @@ -0,0 +1,10 @@ +"""Canonical facade for the deterministic baseline fixture.""" + +from arkx import baseline as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/cli.py b/src/codepro/cli.py new file mode 100644 index 00000000..a7589545 --- /dev/null +++ b/src/codepro/cli.py @@ -0,0 +1,74 @@ +"""Canonical CodePro CLI facade during the staged namespace migration.""" + +import argparse +import json +import platform +import sys +from collections.abc import Sequence + +from . import __version__ +from .baseline import main as baseline_main +from .performance_baseline import main as performance_baseline_main + + +SUPPORTED_PYTHON_MIN = (3, 12) +SUPPORTED_PYTHON_MAX = (3, 14) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="codepro", + description="Empirical, executor-agnostic CodePro chassis.", + ) + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + commands = parser.add_subparsers(dest="command") + doctor = commands.add_parser("doctor", help="Check the installed package and Python runtime.") + doctor.add_argument("--json", action="store_true", dest="as_json") + commands.add_parser("baseline", help="Run the deterministic, no-network baseline fixture.") + performance = commands.add_parser("baseline-cost", help="Measure local overhead without changing behavior.") + performance.add_argument("--workspace", default=".") + performance.add_argument("--output", required=True) + performance.add_argument("--samples", type=int, default=5) + return parser + + +def _doctor(*, as_json: bool) -> int: + supported = SUPPORTED_PYTHON_MIN <= sys.version_info[:2] <= SUPPORTED_PYTHON_MAX + payload = { + "package": "codepro", + "version": __version__, + "python": platform.python_version(), + "python_supported": supported, + "runtime_dependencies": [], + "status": "PASS" if supported else "FAIL", + } + if as_json: + print(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + else: + print(f"package: PASS (codepro {__version__})") + state = "PASS" if supported else "FAIL" + print(f"python: {state} ({platform.python_version()})") + print(f"status: {payload['status']}") + return 0 if supported else 1 + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == "doctor": + return _doctor(as_json=args.as_json) + if args.command == "baseline": + return baseline_main() + if args.command == "baseline-cost": + return performance_baseline_main([ + "--workspace", args.workspace, + "--output", args.output, + "--samples", str(args.samples), + ]) + build_parser().print_help() + return 2 + +__all__ = ("build_parser", "main") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/codepro/contracts.py b/src/codepro/contracts.py new file mode 100644 index 00000000..ae9bab00 --- /dev/null +++ b/src/codepro/contracts.py @@ -0,0 +1,10 @@ +"""Canonical facade for the current neutral contracts.""" + +from arkx import contracts as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/orchestration.py b/src/codepro/orchestration.py new file mode 100644 index 00000000..d44d7183 --- /dev/null +++ b/src/codepro/orchestration.py @@ -0,0 +1,10 @@ +"""Canonical facade for the orchestration compatibility boundary.""" + +from arkx import orchestration as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/p82_localization.py b/src/codepro/p82_localization.py new file mode 100644 index 00000000..4c1bfc82 --- /dev/null +++ b/src/codepro/p82_localization.py @@ -0,0 +1,10 @@ +"""Canonical facade for P8.2 localization contracts.""" + +from arkx import p82_localization as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/performance_baseline.py b/src/codepro/performance_baseline.py new file mode 100644 index 00000000..67b3e909 --- /dev/null +++ b/src/codepro/performance_baseline.py @@ -0,0 +1,10 @@ +"""Canonical facade for local overhead measurement.""" + +from arkx import performance_baseline as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/verification.py b/src/codepro/verification.py new file mode 100644 index 00000000..dddab8e7 --- /dev/null +++ b/src/codepro/verification.py @@ -0,0 +1,10 @@ +"""Canonical facade for patch-verification contracts.""" + +from arkx import verification as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/verifier.py b/src/codepro/verifier.py new file mode 100644 index 00000000..4ac7c45e --- /dev/null +++ b/src/codepro/verifier.py @@ -0,0 +1,10 @@ +"""Canonical facade for the verifier boundary.""" + +from arkx import verifier as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/tests/test_cli.py b/tests/test_cli.py index 5bda1563..48d6824b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -61,6 +61,29 @@ def test_doctor_succeeds_on_supported_interpreter(self): self.assertIn("core: PASS (importable)", output) self.assertTrue(output.rstrip().endswith("status: PASS")) + def test_baseline_cost_delegates_with_explicit_arguments(self): + with patch("arkx.performance_baseline.main", return_value=0) as baseline: + code = main([ + "baseline-cost", + "--workspace", ".", + "--output", "baseline.json", + "--samples", "2", + ]) + self.assertEqual(code, 0) + baseline.assert_called_once_with([ + "--workspace", ".", + "--output", "baseline.json", + "--samples", "2", + ]) + + def test_baseline_cost_requires_output_path(self): + error = io.StringIO() + with patch("sys.stderr", error): + with self.assertRaises(SystemExit) as raised: + main(["baseline-cost"]) + self.assertEqual(raised.exception.code, 2) + self.assertIn("--output", error.getvalue()) + def test_inspect_text_output_is_read_only_and_explicit(self): inspection = ProjectInspection( project_name="fixture", @@ -138,7 +161,7 @@ class PackagingContractTests(unittest.TestCase): def test_console_script_points_to_cli_main(self): project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) self.assertEqual(project["project"]["name"], "codepro") - self.assertEqual(project["project"]["scripts"], {"codepro": "arkx.cli:main"}) + self.assertEqual(project["project"]["scripts"], {"codepro": "codepro.cli:main"}) self.assertEqual(project["project"]["requires-python"], ">=3.12") def test_no_runtime_dependencies_are_declared(self): diff --git a/tests/test_codepro_namespace.py b/tests/test_codepro_namespace.py new file mode 100644 index 00000000..6c2bb794 --- /dev/null +++ b/tests/test_codepro_namespace.py @@ -0,0 +1,89 @@ +import contextlib +import io +import subprocess +import sys +import unittest + +import arkx +import codepro +import arkx.contracts as arkx_contracts +import arkx.orchestration as arkx_orchestration +import arkx.verification as arkx_verification +import arkx.verifier as arkx_verifier +import codepro.contracts as codepro_contracts +import codepro.orchestration as codepro_orchestration +import codepro.verification as codepro_verification +import codepro.verifier as codepro_verifier +import codepro.baseline as codepro_baseline +import codepro.performance_baseline as codepro_performance_baseline +import codepro.p82_localization as codepro_localization +from codepro.cli import main as codepro_main +from arkx.cli import main as arkx_main + + +class CodeproNamespaceTests(unittest.TestCase): + def test_codepro_is_the_canonical_facade_for_the_current_public_api(self): + self.assertEqual(codepro.__version__, arkx.__version__) + self.assertEqual(codepro.__all__, tuple(arkx.__all__)) + for name in codepro.__all__: + self.assertIs(getattr(codepro, name), getattr(arkx, name)) + + def test_legacy_namespace_remains_available(self): + self.assertIs(codepro.ExecutionResult, arkx.ExecutionResult) + self.assertIs(codepro.OrchestrationExecutionResult, arkx.OrchestrationExecutionResult) + + def test_codepro_cli_is_the_canonical_entrypoint_facade(self): + self.assertIsNot(codepro_main, arkx_main) + + canonical_output = io.StringIO() + with contextlib.redirect_stdout(canonical_output): + self.assertEqual(codepro_main(["doctor"]), 0) + + legacy_output = io.StringIO() + with contextlib.redirect_stdout(legacy_output): + self.assertEqual(arkx_main(["doctor"]), 0) + + self.assertIn("status: PASS", canonical_output.getvalue()) + self.assertIn("status: PASS", legacy_output.getvalue()) + + def test_core_submodules_preserve_object_identity_during_migration(self): + pairs = ( + (codepro_contracts, arkx_contracts), + (codepro_orchestration, arkx_orchestration), + (codepro_verification, arkx_verification), + (codepro_verifier, arkx_verifier), + ) + for canonical, legacy in pairs: + self.assertEqual(canonical.__all__, tuple(sorted(name for name in dir(legacy) if not name.startswith("_")))) + for name in canonical.__all__: + self.assertIs(getattr(canonical, name), getattr(legacy, name)) + + def test_operational_submodules_preserve_object_identity(self): + import arkx.baseline as arkx_baseline + import arkx.performance_baseline as arkx_performance_baseline + import arkx.p82_localization as arkx_localization + + for canonical, legacy in ( + (codepro_baseline, arkx_baseline), + (codepro_performance_baseline, arkx_performance_baseline), + (codepro_localization, arkx_localization), + ): + for name in canonical.__all__: + self.assertIs(getattr(canonical, name), getattr(legacy, name)) + + def test_import_does_not_claim_migration_of_unsupported_submodules(self): + probe = subprocess.run( + [ + sys.executable, + "-c", + "import codepro, sys; print('codepro.cli' in sys.modules); print(codepro.__version__)", + ], + check=True, + capture_output=True, + text=True, + ) + self.assertEqual(probe.stdout.splitlines(), ["False", arkx.__version__]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p82_baseline.py b/tests/test_p82_baseline.py new file mode 100644 index 00000000..f7617410 --- /dev/null +++ b/tests/test_p82_baseline.py @@ -0,0 +1,309 @@ +import tempfile +import unittest +import json +import subprocess +from pathlib import Path +from unittest.mock import patch + +from arkx.p82 import P82RunStatus, P82Treatment, TaskStratum +from arkx.harness import RunManifest, RunState +from arkx.harness import load_attempt +from arkx.p82_baseline import ( + AcceptanceDecision, + BaselineExecutionState, + BaselineRunConfig, + BaselineSampleManifest, + ExplicitVerificationAcceptance, + MiniSweAgentHeadlessRunner, + P82BaselineExecutorAdapter, + ProspectiveTask, + RunnerFailureCategory, + VerificationResult, + VerificationState, + baseline_observation, +) +from arkx.execution import ExecutionRequest + + +def task(workspace): + return ProspectiveTask( + "task-1", "swebench-verified@v1", "owner/repo", "abc123", "fix the bug", + TaskStratum.MINI_CONTROL, "stable control", "2026-09-23T00:00:00Z", "p8.2a-v1", + "verification-v1", "acceptance-v1", str(workspace), + ) + + +def config(root, **overrides): + values = { + "provider": "test-provider", "model": "test-model", "model_version": "v1", + "temperature": "0", "reasoning_configuration": "none", "max_tokens": 100, + "model_configuration": None, + "max_cost": "1", "wall_time_seconds": 10, "mini_version": "2.4.6", + "python_executable": "python", "runner_version": "p8.2a-runner-v1", + "artifact_root": str(root), "treatment": P82Treatment.A, + } + values.update(overrides) + return BaselineRunConfig(**values) + + +class P82BaselineTests(unittest.TestCase): + def test_subprocess_invoker_passes_limits_and_captures_usage(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" + workspace.mkdir() + runner = MiniSweAgentHeadlessRunner() + completed = type("Completed", (), { + "returncode": 0, + "stdout": '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\n', + "stderr": "worker log\n", + })() + with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: + usage = runner._subprocess_invoker(task(workspace), config(directory), Path(directory) / "trajectory.json") + self.assertEqual(usage, (0, '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\nworker log\n', {"model_calls": 3, "cost": 0.25})) + self.assertEqual(mocked.call_args.kwargs["timeout"], 20) + payload = json.loads(mocked.call_args.kwargs["input"]) + self.assertEqual(payload["max_cost"], "1") + self.assertEqual(payload["wall_time_seconds"], 10) + + def test_timeout_preserves_partial_output_and_trajectory(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" + workspace.mkdir() + artifact_root = Path(directory) / "artifacts" + trajectory = artifact_root / "task-1" / "attempt-1" / "trajectory.json" + + def timed_out_invoker(run_task, run_config, trajectory_path): + trajectory_path.write_text('{"partial":true}', encoding="utf-8") + raise subprocess.TimeoutExpired( + cmd="worker", timeout=20, output=b"partial stdout\n", stderr=b"partial stderr\n" + ) + + result = MiniSweAgentHeadlessRunner(timed_out_invoker).run(task(workspace), config(artifact_root)) + self.assertEqual(result.failure_category, RunnerFailureCategory.TIMEOUT) + self.assertTrue(Path(result.log_path).exists()) + self.assertEqual(Path(result.log_path).read_text(encoding="utf-8"), "partial stdout\npartial stderr\n") + self.assertEqual(Path(result.trajectory_path).read_text(encoding="utf-8"), '{"partial":true}') + + def test_headless_runner_uses_fake_invoker_and_captures_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" + workspace.mkdir() + calls = [] + + def fake_invoker(run_task, run_config, trajectory): + calls.append((run_task.task_id, run_config.treatment, trajectory.name)) + trajectory.write_text("{}", encoding="utf-8") + return 0, "executor-log", {"calls": 1} + + result = MiniSweAgentHeadlessRunner(fake_invoker).run(task(workspace), config(Path(directory) / "artifacts")) + self.assertEqual(result.state, BaselineExecutionState.COMPLETED) + self.assertEqual(calls, [("task-1", P82Treatment.A, "trajectory.json")]) + self.assertTrue(Path(result.log_path).exists()) + self.assertTrue(Path(result.diff_path).exists()) + self.assertTrue(Path(result.artifact_path).exists()) + persisted = json.loads(Path(result.artifact_path).read_text(encoding="utf-8")) + self.assertEqual(persisted, result.to_dict()) + self.assertEqual(result.model_usage, {"calls": 1}) + self.assertEqual(result.treatment, "A") + + def test_run_identity_changes_when_frozen_model_configuration_changes(self): + with tempfile.TemporaryDirectory() as directory: + first = config(directory, model="model-a") + second = config(directory, model="model-b") + self.assertNotEqual(first.identity_digest(), second.identity_digest()) + + runner = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)) + first_result = runner.run(task(directory), first, attempt=1) + second_result = runner.run(task(directory), second, attempt=2) + self.assertNotEqual(first_result.run_id, second_result.run_id) + + def test_run_identity_changes_when_environment_changes(self): + with tempfile.TemporaryDirectory() as directory: + first = config(directory, environment="local") + second = config(directory, environment="git-bash", bash_executable=r"C:\Program Files\Git\bin\bash.exe") + self.assertNotEqual(first.identity_digest(), second.identity_digest()) + + def test_runner_persists_common_manifest_with_distinct_trial_and_attempt(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)).run(task(directory), config(directory)) + self.assertTrue(Path(result.manifest_path).exists()) + manifest = json.loads(Path(result.manifest_path).read_text(encoding="utf-8")) + self.assertEqual(manifest["trial_id"], result.trial_id) + self.assertEqual(manifest["attempt_id"], result.attempt_id) + self.assertNotEqual(manifest["trial_id"], manifest["attempt_id"]) + restored = RunManifest.from_dict(manifest) + restored.validate_attempt_identity() + restored.validate_lifecycle() + self.assertTrue(all(Path(reference).exists() for reference in manifest["artifact_refs"])) + snapshot = load_attempt(Path(result.manifest_path).parent) + self.assertEqual(snapshot.manifest.attempt_id, result.attempt_id) + self.assertEqual(manifest["state_history"], ["PLANNED", "STARTED", "EXECUTING", "COMPLETED"]) + self.assertEqual(manifest["artifacts"][0]["status"], "PRESENT") + + def test_missing_identity_is_preserved_in_common_manifest(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner().run(task(directory), config(directory, model=None)) + manifest = json.loads(Path(result.manifest_path).read_text(encoding="utf-8")) + self.assertEqual(manifest["state"], "BLOCKED") + self.assertIsNone(manifest["model"]) + self.assertEqual(manifest["errors"][0]["domain"], "HARNESS") + self.assertEqual(manifest["state_history"], ["PLANNED", "STARTED", "BLOCKED"]) + self.assertEqual(manifest["artifacts"][1]["status"], "MISSING") + RunManifest.from_dict(manifest).validate_lifecycle() + + def test_subprocess_invoker_passes_explicit_git_bash_environment(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" + workspace.mkdir() + runner = MiniSweAgentHeadlessRunner() + completed = type("Completed", (), {"returncode": 0, "stdout": "{}\n", "stderr": ""})() + run_config = config( + directory, + environment="git-bash", + bash_executable=r"C:\Program Files\Git\bin\bash.exe", + ) + with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: + runner._subprocess_invoker(task(workspace), run_config, Path(directory) / "trajectory.json") + payload = json.loads(mocked.call_args.kwargs["input"]) + self.assertEqual(payload["environment"], "git-bash") + self.assertEqual(payload["bash_executable"], r"C:\Program Files\Git\bin\bash.exe") + + def test_subprocess_retry_limit_is_derived_from_explicit_retry_policy(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" + workspace.mkdir() + runner = MiniSweAgentHeadlessRunner() + completed = type("Completed", (), {"returncode": 0, "stdout": "{}\n", "stderr": ""})() + run_config = config(directory, provider_retry_attempts=3, provider_retry_backoff_seconds=2.0) + with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: + runner._subprocess_invoker(task(workspace), run_config, Path(directory) / "trajectory.json") + + def test_agent_step_limit_is_explicit_and_forwarded(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" + workspace.mkdir() + runner = MiniSweAgentHeadlessRunner() + completed = type("Completed", (), {"returncode": 0, "stdout": "{}\n", "stderr": ""})() + run_config = config(directory, agent_step_limit=12) + with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: + runner._subprocess_invoker(task(workspace), run_config, Path(directory) / "trajectory.json") + payload = json.loads(mocked.call_args.kwargs["input"]) + self.assertEqual(payload["agent_step_limit"], 12) + self.assertEqual(run_config.configuration_snapshot().to_dict()["public_values"]["agent_step_limit"], 12) + + def test_model_class_is_explicit_and_forwarded(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" + workspace.mkdir() + runner = MiniSweAgentHeadlessRunner() + completed = type("Completed", (), {"returncode": 0, "stdout": "{}\n", "stderr": ""})() + run_config = config(directory, model_class="p82_openrouter_model.ArkxOpenRouterModel") + with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: + runner._subprocess_invoker(task(workspace), run_config, Path(directory) / "trajectory.json") + payload = json.loads(mocked.call_args.kwargs["input"]) + self.assertEqual(payload["model_class"], "p82_openrouter_model.ArkxOpenRouterModel") + self.assertEqual(run_config.configuration_snapshot().to_dict()["public_values"]["model_class"], "p82_openrouter_model.ArkxOpenRouterModel") + + def test_provider_availability_exit_is_blocked_not_agent_failed(self): + with tempfile.TemporaryDirectory() as directory: + runner = MiniSweAgentHeadlessRunner(lambda *_: (75, '{"provider_error":{"code":503}}', {"model_calls": 3})) + result = runner.run(task(directory), config(directory)) + self.assertEqual(result.state, BaselineExecutionState.BLOCKED) + self.assertEqual(result.failure_category, RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE) + self.assertEqual(result.error.domain.value, "PROVIDER") + self.assertEqual(result.error.retryability.value, "RETRYABLE") + + def test_missing_model_or_provider_is_blocked_without_fallback(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner().run(task(directory), config(directory, model=None)) + self.assertEqual(result.state, BaselineExecutionState.BLOCKED) + self.assertIn("model", result.failure) + self.assertEqual(result.error.domain.value, "HARNESS") + + def test_rerun_does_not_overwrite_attempt(self): + with tempfile.TemporaryDirectory() as directory: + runner = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)) + first = runner.run(task(directory), config(directory), attempt=1) + second = runner.run(task(directory), config(directory), attempt=1) + third = runner.run(task(directory), config(directory), attempt=2) + self.assertEqual(first.state, BaselineExecutionState.COMPLETED) + self.assertEqual(second.state, BaselineExecutionState.BLOCKED) + self.assertEqual(third.state, BaselineExecutionState.COMPLETED) + + def test_acceptance_is_explicit_and_preserves_indeterminate(self): + authority = ExplicitVerificationAcceptance("acceptance-v1") + result = authority.decide(VerificationResult(VerificationState.INDETERMINATE, "verification-v1", (), (), ())) + self.assertEqual(result.decision, AcceptanceDecision.INDETERMINATE) + self.assertEqual(result.authority, "acceptance-v1") + + def test_verification_pass_is_not_execution_success_or_runner_acceptance(self): + execution = type("Execution", (), {"state": BaselineExecutionState.COMPLETED})() + verification = VerificationResult(VerificationState.PASS, "verification-v1", ("pytest",), ("0",), ("log",)) + observation = baseline_observation(execution, verification, None) + self.assertEqual(observation.status, P82RunStatus.EXECUTED) + self.assertIsNone(observation.verified_acceptance) + + def test_executor_failure_is_not_task_rejection(self): + execution = type("Execution", (), {"state": BaselineExecutionState.EXECUTOR_FAILED})() + observation = baseline_observation(execution, None, None) + self.assertIsNone(observation.verified_acceptance) + self.assertEqual(observation.status, P82RunStatus.EXECUTED) + + def test_headless_failure_is_classified_without_becoming_rejection(self): + with tempfile.TemporaryDirectory() as directory: + def failing_invoker(*_): + raise ValueError("model configuration is invalid") + result = MiniSweAgentHeadlessRunner(failing_invoker).run(task(directory), config(directory)) + self.assertEqual(result.failure_category, RunnerFailureCategory.MODEL_INITIALIZATION_FAILURE) + self.assertEqual(result.state, BaselineExecutionState.EXECUTOR_FAILED) + self.assertEqual(result.error.domain.value, "EXECUTOR") + self.assertEqual(result.error.code, "EXECUTOR_PROCESS_UNKNOWN") + self.assertEqual(result.error.retryability.value, "UNKNOWN") + + def test_baseline_rejects_non_a_treatment(self): + with self.assertRaises(ValueError): + config("artifacts", treatment=P82Treatment.B) + + def test_prospective_sample_manifest_is_deterministic(self): + first = task("workspace-a") + second = ProspectiveTask( + "task-2", "dataset", "repo", "def456", "another task", TaskStratum.LOCALIZATION_CONTEXT, + "localization diversity", "2026-09-23T00:00:00Z", "p8.2a-v1", "verification-v1", "acceptance-v1", "workspace-b", + ) + left = BaselineSampleManifest("wave-0", "baseline-smoke", "predeclared diversity rule", (second, first)) + right = BaselineSampleManifest("wave-0", "baseline-smoke", "predeclared diversity rule", (first, second)) + self.assertEqual(left.to_json(), right.to_json()) + + def test_baseline_executor_adapter_returns_neutral_execution_result(self): + with tempfile.TemporaryDirectory() as directory: + adapter = P82BaselineExecutorAdapter( + task(directory), + config(directory), + MiniSweAgentHeadlessRunner(lambda *_: (0, "log", None)), + ) + request = ExecutionRequest("task-1", "abc123", "A", "prompt", adapter.executor_id, "test-provider", "test-model", "local", "budget", "config") + result = adapter.execute(request) + self.assertEqual(result.state, RunState.COMPLETED) + self.assertIn("execution.json", " ".join(result.artifact_refs)) + + def test_baseline_executor_adapter_preflight_is_read_only_and_identity_bound(self): + with tempfile.TemporaryDirectory() as directory: + adapter = P82BaselineExecutorAdapter(task(directory), config(directory)) + preflight = adapter.preflight() + self.assertEqual(preflight.identity.name, adapter.executor_id) + self.assertEqual(preflight.identity.configuration_digest, adapter.config.identity_digest()) + self.assertEqual(preflight.capabilities, ("edit",)) + self.assertEqual(preflight.capability_provenance.value, "DECLARED") + self.assertEqual(adapter.config.identity_digest(), adapter.config.configuration_snapshot().digest()) + self.assertNotIn("secret-value", adapter.config.configuration_snapshot().to_json()) + + def test_baseline_executor_adapter_rejects_wrong_task(self): + with tempfile.TemporaryDirectory() as directory: + adapter = P82BaselineExecutorAdapter(task(directory), config(directory), MiniSweAgentHeadlessRunner(lambda *_: (0, "", None))) + request = ExecutionRequest("other-task", "abc123", "A", "prompt", adapter.executor_id, "p", "m", "local", "b", "c") + with self.assertRaises(ValueError): + adapter.execute(request) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_performance_baseline.py b/tests/test_performance_baseline.py new file mode 100644 index 00000000..04a17663 --- /dev/null +++ b/tests/test_performance_baseline.py @@ -0,0 +1,33 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from arkx.performance_baseline import collect_baseline, write_baseline + + +class PerformanceBaselineTests(unittest.TestCase): + def test_baseline_records_measurement_status_and_unknowns(self): + with tempfile.TemporaryDirectory() as directory: + report = collect_baseline(directory, samples=2) + + self.assertEqual(report["baseline_type"], "LOCAL_OVERHEAD_BASELINE") + self.assertEqual(report["interpretation"]["agility_impact"], "NOT_VERIFIED") + self.assertEqual(len(report["measurements"]), 5) + for measurement in report["measurements"]: + self.assertIn(measurement["status"], {"PASS", "BLOCKED"}) + if measurement["status"] == "PASS": + self.assertEqual(measurement["sample_count"], 2) + + def test_baseline_persists_atomically_as_json(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "evidence" / "baseline.json" + write_baseline(collect_baseline(directory, samples=1), output) + payload = json.loads(output.read_text(encoding="utf-8")) + + self.assertEqual(payload["schema_version"], 1) + self.assertFalse(output.with_name("baseline.json.tmp").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_swebench_authority.py b/tests/test_swebench_authority.py new file mode 100644 index 00000000..e2582d6f --- /dev/null +++ b/tests/test_swebench_authority.py @@ -0,0 +1,267 @@ +import json +import hashlib +import importlib.util +import tempfile +import unittest +from pathlib import Path + +from arkx.swebench_authority import ( + AUTHORITY_IDENTITY, OfficialEvaluationResult, OfficialEvaluationStatus, + SWEbenchOfficialAuthority, SWEbenchPrediction, SWEbenchRecord, + _evaluation_result_from_report, derive_harness_instance, + derive_harness_instance_from_task_repo, +) +from arkx.submission import SubmissionArtifact + +_SWEBENCH_AVAILABLE = importlib.util.find_spec("swebench") is not None +requires_swebench = unittest.skipUnless( + _SWEBENCH_AVAILABLE, + "BLOCKED: optional swebench package is unavailable; authority integration not executed", +) + + +def record(): + return SWEbenchRecord( + "owner__repo-1", "owner/repo", "base", "problem", "1.0", "env", + ("fail",), ("pass",), "test patch", "gold patch", "revision", "now", + ) + + +def raw_harness_record(): + return { + "instance_id": "sympy__sympy-14711", "repo": "sympy/sympy", "version": "1.1", + "FAIL_TO_PASS": ["test_Vector"], "PASS_TO_PASS": ["test_Vector_diffs"], + "log_parser": "python", "eval_type": "FAIL_AND_PASS", + "eval_script": "python -m pytest", "base_commit": "base", + } + + +class SWEbenchAuthorityTests(unittest.TestCase): + def test_prediction_serialization_has_no_gold_or_grading_labels(self): + value = SWEbenchPrediction("task", "provider/model", "diff").to_dict() + self.assertEqual(value, {"instance_id": "task", "model_name_or_path": "provider/model", "model_patch": "diff"}) + self.assertNotIn("gold_patch", value) + + def test_reference_prediction_uses_upstream_submission_byte_for_byte(self): + submission = SubmissionArtifact.from_upstream_agent( + "patch\r\n", attempt_id="attempt-1", executor_identity="mini-pinned" + ) + value = SWEbenchPrediction.from_submission( + instance_id="task", model_name_or_path="provider/model", submission=submission + ) + self.assertEqual(value.model_patch, "patch\r\n") + + def test_reference_prediction_rejects_workspace_diff_source(self): + submission = SubmissionArtifact( + "workspace_diff", "diff", "diff", "hash", (), "raw", "normalized" + ) + with self.assertRaisesRegex(ValueError, "upstream_agent submission"): + SWEbenchPrediction.from_submission( + instance_id="task", model_name_or_path="provider/model", submission=submission + ) + + def test_executor_payload_excludes_gold_and_test_labels(self): + value = record().executor_payload() + self.assertNotIn("gold_patch", value) + self.assertNotIn("FAIL_TO_PASS", value) + self.assertNotIn("test_patch", value) + + def test_official_outcome_mapping(self): + accepted = OfficialEvaluationResult(OfficialEvaluationStatus.RESOLVED, AUTHORITY_IDENTITY, "r1", "t", "resolved", "report", ()) + rejected = OfficialEvaluationResult(OfficialEvaluationStatus.TESTS_FAILED, AUTHORITY_IDENTITY, "r2", "t", "failed", "report", ()) + infra = OfficialEvaluationResult(OfficialEvaluationStatus.INFRASTRUCTURE_ERROR, AUTHORITY_IDENTITY, "r3", "t", None, None, ()) + self.assertEqual(accepted.acceptance().decision.value, "ACCEPTED") + self.assertEqual(rejected.acceptance().decision.value, "REJECTED") + self.assertEqual(infra.acceptance().decision.value, "INDETERMINATE") + + def test_evaluator_infrastructure_failure_is_not_a_test_failure(self): + infra = OfficialEvaluationResult( + OfficialEvaluationStatus.INFRASTRUCTURE_ERROR, + AUTHORITY_IDENTITY, + "r3", + "t", + None, + "report", + ("log",), + "docker daemon unavailable", + ) + rejected = OfficialEvaluationResult( + OfficialEvaluationStatus.TESTS_FAILED, + AUTHORITY_IDENTITY, + "r4", + "t", + "failed", + "report", + (), + ) + error = infra.error_envelope() + self.assertEqual(error.domain.value, "VERIFICATION") + self.assertEqual(error.code, "EVALUATOR_INFRASTRUCTURE_ERROR") + self.assertEqual(error.retryability.value, "UNKNOWN") + self.assertEqual(error.raw_evidence_refs, ("log", "report")) + self.assertIsNone(rejected.error_envelope()) + + def test_evaluator_maps_to_verification_without_acceptance_or_promotion(self): + result = OfficialEvaluationResult( + OfficialEvaluationStatus.TESTS_FAILED, + AUTHORITY_IDENTITY, + "r5", + "t", + "failed", + "report", + ("log",), + ) + verification = result.verification_result() + self.assertEqual(verification.state.value, "FAIL") + self.assertEqual(verification.authority, AUTHORITY_IDENTITY) + self.assertEqual(verification.commands, ("swebench.harness.run_evaluation",)) + self.assertEqual(verification.evidence, ("log", "report")) + self.assertTrue(verification.reference.startswith("verification://")) + self.assertEqual(result.acceptance().decision.value, "REJECTED") + + def test_authority_persists_prediction_and_raw_artifact_link(self): + with tempfile.TemporaryDirectory() as directory: + def evaluator(payload): + prediction = json.loads(Path(payload["predictions_path"]).read_text(encoding="utf-8")) + self.assertEqual(prediction["instance_id"], "owner__repo-1") + return OfficialEvaluationResult(OfficialEvaluationStatus.RESOLVED, AUTHORITY_IDENTITY, payload["run_id"], "owner__repo-1", "resolved", payload["predictions_path"], ()) + result = SWEbenchOfficialAuthority(directory, evaluator).evaluate(record=record(), model_name_or_path="m", model_patch="diff", run_id="unique-run") + self.assertEqual(result.status, OfficialEvaluationStatus.RESOLVED) + self.assertTrue(Path(result.report_path).exists()) + + def test_same_run_id_is_not_reused_for_a_second_artifact(self): + with tempfile.TemporaryDirectory() as directory: + authority = SWEbenchOfficialAuthority(directory, lambda payload: OfficialEvaluationResult(OfficialEvaluationStatus.RESOLVED, AUTHORITY_IDENTITY, payload["run_id"], "owner__repo-1", "resolved", None, ())) + authority.evaluate(record=record(), model_name_or_path="m", model_patch="one", run_id="r") + blocked = authority.evaluate(record=record(), model_name_or_path="m", model_patch="two", run_id="r") + self.assertEqual(blocked.status, OfficialEvaluationStatus.BLOCKED) + + def test_aggregate_report_distinguishes_empty_filter_from_execution(self): + with tempfile.TemporaryDirectory() as directory: + report = Path(directory) / "provider-free.json" + report.write_text(json.dumps({ + "submitted_ids": ["task"], "empty_patch_ids": ["task"], + "resolved_ids": [], "unresolved_ids": [], "error_ids": [], + }), encoding="utf-8") + result = _evaluation_result_from_report(report_path=report, run_id="r", instance_id="task") + self.assertEqual(result.status, OfficialEvaluationStatus.NOT_EXECUTED) + self.assertEqual(result.raw_outcome, "EMPTY_PATCH_FILTERED") + + def test_aggregate_report_accepts_executed_unresolved_instance(self): + with tempfile.TemporaryDirectory() as directory: + report = Path(directory) / "provider-free.json" + report.write_text(json.dumps({ + "submitted_ids": ["task"], "empty_patch_ids": [], + "resolved_ids": [], "unresolved_ids": ["task"], "error_ids": [], + }), encoding="utf-8") + result = _evaluation_result_from_report(report_path=report, run_id="r", instance_id="task") + self.assertEqual(result.status, OfficialEvaluationStatus.TESTS_FAILED) + self.assertEqual(result.raw_outcome, "false") + + @requires_swebench + def test_raw_record_is_explicitly_enriched_with_harness_image(self): + raw = raw_harness_record() + enriched = derive_harness_instance( + raw, source_dataset_id="dataset", revision="rev", fingerprint="fp" + ) + self.assertNotIn("image", raw) + self.assertEqual(enriched.record["image"], "swebench/sweb.eval.x86_64.sympy_1776_sympy-14711:latest") + self.assertEqual(enriched.source_dataset_id, "dataset") + self.assertEqual(enriched.revision, "rev") + self.assertEqual(enriched.fingerprint, "fp") + self.assertEqual(enriched.instance_id, raw["instance_id"]) + self.assertTrue(enriched.raw_record_hash) + self.assertIn("expected_image@5.0.2", enriched.derivation_authority) + + def test_missing_harness_field_is_blocked_without_a_default(self): + raw = raw_harness_record() + del raw["eval_script"] + with self.assertRaisesRegex(ValueError, "eval_script"): + derive_harness_instance(raw, source_dataset_id="dataset", revision="rev", fingerprint="fp") + + def test_each_required_derived_field_is_fail_closed(self): + for field in ("log_parser", "eval_type", "eval_script"): + with self.subTest(field=field): + raw = raw_harness_record() + del raw[field] + with self.assertRaisesRegex(ValueError, field): + derive_harness_instance(raw, source_dataset_id="dataset", revision="rev", fingerprint="fp") + + @requires_swebench + def test_wrong_image_identity_is_blocked(self): + raw = raw_harness_record() + raw["image"] = "wrong/image:latest" + with self.assertRaisesRegex(ValueError, "wrong image identity"): + derive_harness_instance(raw, source_dataset_id="dataset", revision="rev", fingerprint="fp") + + @requires_swebench + def test_enriched_record_image_is_idempotently_verified(self): + raw = raw_harness_record() + raw["image"] = "swebench/sweb.eval.x86_64.sympy_1776_sympy-14711:latest" + enriched = derive_harness_instance(raw, source_dataset_id="dataset", revision="rev", fingerprint="fp") + self.assertEqual(enriched.derived_image, raw["image"]) + + @requires_swebench + def test_task_repo_is_the_authority_for_the_three_missing_fields(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + task = root / "tasks" / "sympy__sympy-14711" + task.mkdir(parents=True) + (root / "sweb.yaml").write_text( + "datasets: [princeton-nlp/SWE-Bench_Verified]\nsplits: [test]\n" + ) + (task / "task.yaml").write_text( + "instance_id: sympy__sympy-14711\nrepo: sympy/sympy\n" + "version: '1.1'\nbase_commit: base\nsplit: test\n" + "datasets: [princeton-nlp/SWE-Bench_Verified]\n" + "image: swebench/sweb.eval.x86_64.sympy_1776_sympy-14711:latest\n" + "log_parser: python\neval_type: FAIL_AND_PASS\n" + ) + (task / "tests.json").write_text( + json.dumps({"FAIL_TO_PASS": ["test_Vector"], "PASS_TO_PASS": ["test_Vector_diffs"]}) + ) + (task / "eval.sh").write_text("#!/bin/bash\npytest\n") + (task / "problem_statement.md").write_text("problem") + (task / "gold.patch").write_text("diff --git a/a b/a\n") + (task / "test.patch").write_text("diff --git a/t t\n") + (task / "Dockerfile").write_text("FROM scratch\n") + raw = raw_harness_record() + raw.pop("log_parser") + raw.pop("eval_type") + raw.pop("eval_script") + enriched = derive_harness_instance_from_task_repo( + raw, task_repo=root, source_dataset_id="dataset", revision="rev", fingerprint="fp" + ) + self.assertEqual(enriched.record["log_parser"], "python") + self.assertEqual(enriched.record["eval_type"], "FAIL_AND_PASS") + self.assertEqual(enriched.record["eval_script"].replace("\r\n", "\n"), "#!/bin/bash\npytest\n") + raw_hash_source = raw_harness_record() + for key in ("log_parser", "eval_type", "eval_script"): + raw_hash_source.pop(key) + expected_raw_hash = hashlib.sha256( + json.dumps(raw_hash_source, sort_keys=True, ensure_ascii=False, default=str).encode() + ).hexdigest() + self.assertEqual(enriched.provenance()["raw_record_hash"], expected_raw_hash) + self.assertEqual(enriched.provenance()["derived_fields"]["eval_script"]["derivation_authority"], "swebench.task.repo") + + @requires_swebench + def test_make_test_spec_accepts_complete_derived_record(self): + from swebench.harness.utils import make_test_spec + enriched = derive_harness_instance( + raw_harness_record(), source_dataset_id="dataset", revision="rev", fingerprint="fp" + ) + spec = make_test_spec(enriched.to_dict()) + self.assertEqual(spec.image, "swebench/sweb.eval.x86_64.sympy_1776_sympy-14711:latest") + + @requires_swebench + def test_unknown_task_repo_identity_is_blocked(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex((KeyError, ValueError, FileNotFoundError), "sympy__sympy-14711|identity|tasks"): + derive_harness_instance_from_task_repo( + raw_harness_record(), task_repo=directory, + source_dataset_id="dataset", revision="rev", fingerprint="fp" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_verification.py b/tests/test_verification.py index 281d10b2..009944a8 100644 --- a/tests/test_verification.py +++ b/tests/test_verification.py @@ -25,6 +25,7 @@ def valid(**overrides): "changed_files": ("src/a.py",), "expected_scope": ("src/a.py",), "evidence_refs": ("e://patch",), + "verifier_run_id": "verify-task-control-1", } values.update(overrides) return PatchVerificationInput(**values) diff --git a/tools/p82_bash_environment.py b/tools/p82_bash_environment.py new file mode 100644 index 00000000..1fc84a5e --- /dev/null +++ b/tools/p82_bash_environment.py @@ -0,0 +1,134 @@ +"""Git Bash environment for the P8.2a Windows runner-v3 preflight.""" + +from __future__ import annotations + +import os +import signal +import subprocess +from typing import Any + +try: + from minisweagent.environments.local import LocalEnvironment, LocalEnvironmentConfig + MINISWEAGENT_AVAILABLE = True +except ModuleNotFoundError: # dependency boundary; execution remains unavailable + MINISWEAGENT_AVAILABLE = False + + class LocalEnvironmentConfig: # type: ignore[no-redef] + pass + + class LocalEnvironment: # type: ignore[no-redef] + def __init__(self, *args, **kwargs): + raise RuntimeError("minisweagent is required for BashLocalEnvironment execution") + + +class BashLocalEnvironmentConfig(LocalEnvironmentConfig): + bash_executable: str + + +class DockerExecEnvironmentConfig(LocalEnvironmentConfig): + container: str + + +class BashLocalEnvironment(LocalEnvironment): + """Execute mini-SWE-agent bash actions through an explicit Git Bash binary.""" + + def __init__(self, *, config_class: type = BashLocalEnvironmentConfig, **kwargs): + if not MINISWEAGENT_AVAILABLE: + raise RuntimeError("minisweagent is required for BashLocalEnvironment execution") + super().__init__(config_class=config_class, **kwargs) + + def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) -> dict[str, Any]: + command = action.get("command", "") + cwd = cwd or self.config.cwd or os.getcwd() + bash_executable = self.config.bash_executable + process = None + try: + process = subprocess.Popen( + [bash_executable, "-lc", command], + cwd=cwd, + env=os.environ | self.config.env, + text=True, + encoding="utf-8", + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + stdout, _ = process.communicate(timeout=timeout or self.config.timeout) + output = {"output": stdout, "returncode": process.returncode, "exception_info": ""} + except subprocess.TimeoutExpired as error: + if process is not None: + if os.name == "nt": + try: + subprocess.run( + ["taskkill", "/PID", str(process.pid), "/T", "/F"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=5, + ) + except (OSError, subprocess.TimeoutExpired): + process.kill() + # Descendants can retain the pipe after the shell dies. + # Do not wait for EOF on Windows; the timeout boundary is + # already authoritative and partial output is sufficient. + stdout = error.output or "" + else: + os.killpg(process.pid, signal.SIGKILL) + stdout, _ = process.communicate() + else: + stdout = error.output or "" + output = { + "output": stdout, + "returncode": -1, + "exception_info": f"An error occurred while executing the command: {error}", + "extra": {"exception_type": type(error).__name__, "exception": str(error)}, + } + except Exception as error: + raw_output = getattr(error, "output", None) + raw_output = raw_output.decode("utf-8", errors="replace") if isinstance(raw_output, bytes) else (raw_output or "") + output = { + "output": raw_output, + "returncode": -1, + "exception_info": f"An error occurred while executing the command: {error}", + "extra": {"exception_type": type(error).__name__, "exception": str(error)}, + } + self._check_finished(output) + return output + + +class DockerExecEnvironment(LocalEnvironment): + """Execute agent actions inside an already-qualified Docker task container.""" + + def __init__(self, *, config_class: type = DockerExecEnvironmentConfig, **kwargs): + super().__init__(config_class=config_class, **kwargs) + + def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) -> dict[str, Any]: + command = action.get("command", "") + container = self.config.container + cwd = cwd or self.config.cwd or "/testbed" + process = None + try: + process = subprocess.Popen( + ["docker", "exec", "-w", cwd, container, "bash", "-lc", command], + text=True, encoding="utf-8", errors="replace", + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + stdout, _ = process.communicate(timeout=timeout or self.config.timeout) + output = {"output": stdout, "returncode": process.returncode, "exception_info": ""} + except subprocess.TimeoutExpired as error: + if process is not None: + if os.name == "nt": + subprocess.run(["docker", "kill", container], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, timeout=5) + stdout = error.output or "" + else: + os.killpg(process.pid, signal.SIGKILL) + stdout, _ = process.communicate() + else: + stdout = error.output or "" + output = {"output": stdout, "returncode": -1, "exception_info": f"An error occurred while executing the command: {error}", "extra": {"exception_type": type(error).__name__, "exception": str(error)}} + except Exception as error: + raw_output = getattr(error, "output", None) + raw_output = raw_output.decode("utf-8", errors="replace") if isinstance(raw_output, bytes) else (raw_output or "") + output = {"output": raw_output, "returncode": -1, "exception_info": f"An error occurred while executing the command: {error}", "extra": {"exception_type": type(error).__name__, "exception": str(error)}} + self._check_finished(output) + return output diff --git a/tools/run_mini_headless.py b/tools/run_mini_headless.py new file mode 100644 index 00000000..579870a8 --- /dev/null +++ b/tools/run_mini_headless.py @@ -0,0 +1,120 @@ +"""Headless mini-SWE-agent 2.x worker used by the Arkx adapter. + +This file is intentionally tiny and uses the public Python API. It is not +called by unit tests and never evaluates acceptance. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +from pathlib import Path + + +def main() -> int: + payload = json.load(sys.stdin) + diagnostics_path = payload.get("diagnostics_path") + if diagnostics_path: + os.environ["ARKX_P82_MODEL_DIAGNOSTICS_PATH"] = diagnostics_path + os.environ.setdefault("MSWEA_SILENT_STARTUP", "1") + from minisweagent.agents import get_agent + from minisweagent.config import get_config_from_spec + from minisweagent.environments import get_environment + from minisweagent.models import get_model + from minisweagent.utils.serialize import recursive_merge, UNSET + + config = get_config_from_spec("mini.yaml") + config = recursive_merge( + config, + { + "run": {"task": payload["task"] or UNSET}, + "agent": { + "mode": "yolo", + "confirm_exit": False, + "output_path": Path(payload["trajectory"]), + "cost_limit": None if payload["max_cost"] is None else float(payload["max_cost"]), + "wall_time_limit_seconds": None if payload["wall_time_seconds"] is None else int(payload["wall_time_seconds"]), + "step_limit": UNSET if payload.get("agent_step_limit") is None else int(payload["agent_step_limit"]), + }, + "model": { + "model_name": payload["model"], + "model_class": payload.get("model_class", UNSET), + **(payload.get("model_configuration") or payload.get("model_kwargs") or {}), + }, + "environment": {"cwd": payload["workspace"]}, + }, + ) + model = get_model(config=config["model"]) + if payload.get("environment") == "git-bash": + from p82_bash_environment import BashLocalEnvironment + + environment_config = config["environment"] | { + "bash_executable": payload["bash_executable"], + } + env = BashLocalEnvironment(**environment_config) + elif payload.get("environment") == "docker-exec": + from p82_bash_environment import DockerExecEnvironment + + environment_config = payload.get("environment_configuration") or {} + env = DockerExecEnvironment(**environment_config) + else: + env = get_environment(config["environment"], default_type="local") + retry_limit = int(payload.get("provider_retry_attempts", 0)) + retry_backoff = float(payload.get("provider_retry_backoff_seconds", 0)) + result = None + agent = None + for retry_index in range(retry_limit + 1): + try: + agent = get_agent(model, env, config["agent"], default_type="default") + result = agent.run(payload["task"] or "") + break + except Exception as error: + from p82_openrouter_model import ProviderAvailabilityError + + if not isinstance(error, ProviderAvailabilityError): + raise + if retry_index >= retry_limit: + result = { + "exit_status": "ProviderAvailabilityFailure", + "submission": "", + "failure_category": "PROVIDER_AVAILABILITY_FAILURE", + "provider_error": error.error, + "provider_retry_attempts": retry_index, + } + break + if retry_backoff > 0: + if diagnostics_path: + retry_path = Path(diagnostics_path).with_name("retry-observability.json") + retry_payload = {"events": []} + if retry_path.exists(): + try: + retry_payload = json.loads(retry_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + retry_payload = {"events": []} + retry_payload.setdefault("events", []).append({ + "event": "provider_retry_backoff", + "retry_index": retry_index + 1, + "started_at": time.time(), + "backoff_seconds": retry_backoff, + }) + temporary = retry_path.with_suffix(retry_path.suffix + ".tmp") + temporary.write_text(json.dumps(retry_payload, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(retry_path) + time.sleep(retry_backoff) + model = get_model(config=config["model"]) + env = get_environment(config["environment"], default_type="local") if payload.get("environment") != "git-bash" else BashLocalEnvironment(**environment_config) + output = { + "result": result, + "usage": { + "model_calls": getattr(agent, "n_calls", None), + "cost": getattr(agent, "cost", None), + }, + } + print(json.dumps(output, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())