diff --git a/.github/workflows/foundation.yml b/.github/workflows/foundation.yml index 4d728e05..79c0e1cb 100644 --- a/.github/workflows/foundation.yml +++ b/.github/workflows/foundation.yml @@ -17,4 +17,38 @@ jobs: python-version: "3.x" - name: Check foundation contract run: python tools/check_foundation.py + - name: Run baseline fixture + run: python -m arkx.baseline + env: + PYTHONPATH: src + - name: Run unit tests + run: python -m unittest discover -s tests -t . -v + env: + PYTHONPATH: src + - name: Compile Python sources + run: python -m compileall -q src tests tools + packaged-cli: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Build and install wheel + run: | + python -m pip install --upgrade pip + python -m pip install build twine + python -m build + twine check dist/* + python -m venv .venv-release-check + . .venv-release-check/bin/activate + python -m pip install --no-index --find-links dist codepro + - name: Verify packaged public boundary + run: | + . .venv-release-check/bin/activate + test "$(codepro --version)" = "codepro 0.3.0.dev0" + codepro doctor --json + codepro baseline + python -m pip check + python -m unittest discover -s tests -t . diff --git a/.gitignore b/.gitignore index 1bef7ae6..07f50c0d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__/ .mypy_cache/ .coverage htmlcov/ +*.egg-info/ # IDE and operating-system files .idea/ @@ -18,3 +19,6 @@ Thumbs.db experiments/**/runs/ experiments/**/artifacts/ +# Local preflight workspaces and temporary virtual environment +.p82-local-preflight-*-workspace/ +.venv-mini/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..5837a8af --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,120 @@ +# Codepro agent operating rules + +The product identity is Codepro. `arkx` is the current Python namespace and +must not be treated as a separate product identity. + +## Scope + +Treat this repository as an evidence-oriented software-engineering project. +The presence of a module, adapter, test, or document does not prove that the +corresponding capability is qualified for adoption. + +## Required implementation standard + +Every material capability change must have: + +1. an explicit contract or decision record; +2. a falsifiable hypothesis and empirical acceptance criteria; +3. executable tests for deterministic behavior and failure boundaries; +4. recorded raw evidence, including environment and identity where relevant; +5. an independent acceptance decision before promotion. + +Local unit-test success is local evidence only. It must not be reported as +upstream, scientific, benchmark, or provider evidence. + +## Harness rules + +- Preserve provider, model, version, configuration, budget, runtime, and + treatment identity in every execution record. +- Distinguish identity levels: `experiment_id` identifies the protocol, + `trial_id` identifies a frozen comparison cell, and `attempt_id` identifies + one execution attempt. A verifier `run_id` must be unique for every + control, patch, and retry that can produce a distinct result; it must never + be reused merely because the task is the same. +- Use stable identity for equivalent frozen protocol inputs, but include the + control role, patch/diff identity, and attempt semantics wherever a provider + or verifier can cache results. +- Execute serially when the experiment requires controlled comparison. +- Never silently overwrite an attempt directory or select a fallback executor. +- Persist execution artifacts atomically and keep execution, verification, + acceptance, and promotion as separate states. +- A qualification record must identify the dataset revision or fingerprint, + task/base revision, environment image and digest when applicable, + OS/architecture, executor/provider/model/configuration, verifier identity, + budget, treatment, control role, and whether the provider was called. +- A provider-free verifier qualification requires both controls: an empty or + no-op patch must fail as expected and an official gold/oracle patch must + resolve as expected. A missing control is `NOT_EXECUTED` or `BLOCKED`, never + a successful qualification. +- Represent missing identity, missing evidence, blocked infrastructure, and + indeterminate outcomes explicitly; none may become success by inference. +- A failure must retain enough raw output and environment facts to support + diagnosis and reproducibility. + +## Routing and promotion + +Route the request before selecting an executor. Simple, localized, and +repository-wide paths are treatment regimes, not implicit executor cascades. +Fallbacks, executor switches, scope expansion, acceptance, and promotion must +be explicit and evidence-backed. + +Do not promote a provider, model, executor, or treatment until the declared +authority has accepted the required evidence. A Gold or infrastructure +qualification validates its stated authority and does not by itself prove +mechanism effectiveness or general model quality. + +## Agent execution in local repositories + +The execution agent is a task worker, not an implicit release manager. + +- Start each task by recording the workspace, branch/worktree, base revision, + and pre-existing working-tree changes. +- Treat one task, one workspace, and one attempt as the default unit. Retries + receive new attempt identity and preserve the previous artifacts. +- Preserve unrelated local changes. Never use broad `git add -A`, reset, clean, + checkout, stash, or file deletion to make a workspace look clean unless the + user explicitly authorizes that exact operation. +- The default deliverable is a verified patch plus evidence. Do not create a + commit, branch, tag, merge, or pull request unless the request explicitly + authorizes it. +- When a commit is authorized, make at most the task-scoped commit after the + declared checks pass. Do not create checkpoint or speculative commits. +- When a pull request is authorized, first locate the task's existing branch + or open PR and update it when appropriate. Do not create duplicate PRs for + retries, blocked attempts, or the same task. A blocked, failed, or + indeterminate attempt is not PR-ready. +- Never infer completion from a clean diff, a created commit, or an open PR. + Completion requires the declared verification and acceptance evidence. + +The complete workspace/patch/commit/PR protocol is in +[`docs/agent-execution-workflow.md`](docs/agent-execution-workflow.md). + +## Change discipline + +- Preserve unrelated working-tree changes. +- Prefer the smallest change that satisfies the contract. +- Add or update tests with behavioral changes. +- Update the relevant architecture, protocol, or decision documentation when + the boundary changes. +- Run the focused tests first, then the complete test suite when practical. +- Do not claim merge readiness or a green local baseline while required + evidence or acceptance records are missing, or while the declared local + suite is not green. Historical passing runs must retain their revision, + command, environment, and limitations. + +## Documentation status + +- Normative rules live in this file and in the current experimental protocol. +- Decision records explain why a boundary exists; they do not silently replace + the current protocol. +- Logs and manifests are evidence, not policy. A historical log must not be + presented as the current state without its revision and collection date. +- When a protocol boundary changes, update the protocol, the relevant + architecture page, and one decision record together. Avoid adding a new + document when an existing normative contract can be updated. + +## Current baseline + +The recorded external authority result is `resolved: true` for the official +SWE-bench Docker Gold path on the frozen tasks under `logs/evaluation/`. This +does not qualify a real provider/model or the A/B/C/D mechanism trial. diff --git a/README.md b/README.md index 9ed2c815..29b9650f 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,33 @@ -# Arkx +# Codepro -Arkx is a greenfield project for empirical, incremental development of reliable capability. +Codepro is a repository for incremental implementation and empirical +evaluation of software capability. + +The current Python import namespace is `arkx` for compatibility; the product, +distribution, CLI, and release identity are `codepro`. > **MINIMUM SUFFICIENT ARCHITECTURE** > **FOR MAXIMUM RELIABLE CAPABILITY** -This repository is the canonical root for the project. It intentionally contains only the foundation in this bootstrap: the project contract, experimental protocol, structural placeholders, and a minimal CI check. +This repository is the canonical root for the project. It contains runtime +contracts, an execution harness, the project contract, the experimental +protocol, and structural checks. -## Scope of this bootstrap +## Current scope Included: - explicit project invariants; -- a small, evidence-oriented experimental protocol; -- stable top-level boundaries for future work; -- a deterministic structural check and minimal CI. +- an evidence-oriented experimental protocol; +- stable top-level boundaries; +- a deterministic structural check, executable contracts, and a Treatment A + baseline harness. -Deferred: +Still deferred or unqualified: -- routing and planning; - multi-agent orchestration; -- OpenHands, ReX, mini-SWE-agent, and SWE-agent integrations; +- OpenHands, ReX, and SWE-agent integrations; +- real provider/model adoption and the A/B/C/D mechanism trial; - product dependencies or automatic migration from `smag-rex`. ## Development posture @@ -28,6 +35,41 @@ Deferred: Every capability must be introduced through a falsifiable hypothesis, an explicit implementation, an executed experiment, and recorded evidence. Local success is not scientific or upstream evidence. No fallback, executor switch, or scope expansion may be silent. See [the project contract](docs/project-contract.md) and [the experimental protocol](docs/experimental-protocol.md). +Agent workspace, patch, commit, retry and pull-request behavior is governed by +[the agent execution workflow](docs/agent-execution-workflow.md). + +## P0 baseline and telemetry + +P0 provides executor-agnostic execution records, explicit event metrics, nullable resource measurements, and fail-closed status resolution. It does not implement performance optimization, routing, planning, or any executor. See [the P0 metrics contract](docs/baseline-metrics.md). + +Install the development distribution in a clean environment and run the +deterministic fixture and tests: + +```text +python -m pip install . +python -m arkx.baseline +python -m unittest discover -s tests -t . -v +``` + +The distribution is currently `codepro==0.3.0.dev0`; it is a packaged +pre-release chassis. The public commands are limited to health and the +deterministic baseline: + +```text +codepro --version +codepro doctor --json +codepro baseline +``` + +There is intentionally no task-submission or executor command yet. + +P1 adds deterministic task characterization from explicit signals only. See [the P1 contract](docs/task-characterization.md). It does not select executors or route work. + +P2 adds deterministic progress/stagnation assessment from explicit snapshots only. See [the P2 contract](docs/progress-stagnation.md). It does not control execution or authorize retry/replan. + +P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke concrete executors. + +Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. P8.2a adds a Treatment A baseline runner and official SWE-bench acceptance adapter; neither owns general orchestration or promotion. ## Checks @@ -36,4 +78,3 @@ The bootstrap check is dependency-free: ```text python tools/check_foundation.py ``` - diff --git a/docs/agent-execution-workflow.md b/docs/agent-execution-workflow.md new file mode 100644 index 00000000..87fb1762 --- /dev/null +++ b/docs/agent-execution-workflow.md @@ -0,0 +1,134 @@ +# Agent execution workflow for local repositories + +## Purpose + +This document governs how the Codepro execution agent works on a local +repository. It addresses workspace state, patches, commits, retries and pull +requests. It does not grant authority to merge, release, promote or erase +unrelated user work. + +The policy follows the useful boundary exposed by current agent harnesses: +the task instruction defines the requested outcome, the environment is the +agent's work surface, the verifier decides the task result, and an oracle or +reference solution is a control. A patch, commit or pull request is an output +artifact, not proof that the task is correct. + +## Default execution unit + +The default unit is: + +```text +one task + one workspace/worktree + one attempt identity +``` + +Before making changes, the agent records: + +- workspace path and branch/worktree identity; +- base revision and current `HEAD`; +- pre-existing modified and untracked paths; +- task scope and requested output; +- attempt identity and execution configuration. + +The agent must not silently adopt unrelated changes as part of the task. If +the requested change overlaps a pre-existing modification, the attempt is +`BLOCKED` until the overlap is explicitly resolved or the user authorizes the +scope. + +## Patch-first delivery + +The default delivery is a patch and an evidence record: + +```text +task -> inspect -> edit -> focused checks -> full checks when practical + -> diff/status/evidence -> verification -> acceptance +``` + +The agent must report at least the changed paths, diff summary, commands run, +observed results, remaining blockers, and the attempt/artifact references. + +The agent must not treat any of the following as completion by itself: + +- a clean working tree; +- a generated diff; +- a successful local test unrelated to the declared verifier; +- a created commit; +- an open pull request. + +## Commit policy + +Commits are optional packaging artifacts, not an automatic execution step. +Create one only when the user explicitly requests a commit or the surrounding +workflow has already granted that authority. + +When authorized: + +1. keep the commit limited to the task scope; +2. inspect the staged file list and diff before committing; +3. run the declared focused checks before the commit; +4. use one task-scoped commit rather than checkpoint commits; +5. record the commit SHA and the evidence revision together. + +Never use `git add -A` as a substitute for selecting task files. Never commit +secrets, generated evidence directories, unrelated user changes or an +unverified workaround. + +## Retry and continuation policy + +A retry is a new attempt, not a new task and not a new pull request. It must: + +- receive a new `attempt_id` and verifier `run_id` when its result can differ; +- preserve the previous raw output and failure classification; +- state what changed in the retry configuration or environment; +- avoid overwriting the previous attempt directory; +- reuse the existing task branch/worktree unless isolation was explicitly + requested. + +If a previous attempt is blocked by infrastructure, the next attempt must not +be reported as a task failure or silently switch executor/provider. + +## Pull-request policy + +Opening or updating a pull request is an explicit external action. It requires +the user to request it or an already-authorized workflow to include it. + +Before opening one, the agent must search for an existing branch or PR for the +same task. The default is to update the existing task PR, not create another. +Retries, new verifier runs and additional evidence do not justify duplicate +PRs. + +A PR is not ready when the attempt is `BLOCKED`, `FAILED`, `UNKNOWN` or +`NOT_EXECUTED`, unless the PR is explicitly for documenting that failure. A +ready PR record includes: + +- task and repository identity; +- base and head revisions; +- changed-file and diff summary; +- focused/full check results and environment limitations; +- verification and independent acceptance state; +- relevant attempt, evidence and commit references. + +Merge and promotion remain separate decisions. The execution agent must not +merge its own PR or infer promotion from review availability. + +## State crosswalk + +```text +EXECUTED + patch != VERIFIED +VERIFIED != ACCEPTED +ACCEPTED != COMMITTED +COMMITTED != PR_READY +PR_READY != MERGED +MERGED != PROMOTED +``` + +Each transition needs the evidence and authority declared by the task. Missing +evidence remains missing; it is never filled by the existence of a later Git +artifact. + +## External precedent and scope + +Harbor's task format separates instruction, environment, tests and optional +oracle solution. SWE-agent documents local patch application separately from +opening a PR, with PR creation as an explicit action. Codepro adopts those +boundaries while adding stricter preservation of user work, attempt lineage, +verification and independent acceptance. diff --git a/docs/architecture-audit-20260923.md b/docs/architecture-audit-20260923.md new file mode 100644 index 00000000..b41f0380 --- /dev/null +++ b/docs/architecture-audit-20260923.md @@ -0,0 +1,182 @@ +# Arkx — architectural audit, 2026-09-23 + +## Scope and evidence boundary + +This is a read-oriented architectural audit of the current working tree on +branch `codex/auditoriaarquitetural`, with source revision `0eff3dca` at audit +collection. The checkout contains +the committed architecture, documentation, tests and evidence plus two nested +experimental workspaces with local modifications. Those workspaces were +inspected and preserved; unrelated changes were not reset or deleted. + +Evidence inspected included the P0–P8 history, decision records 0012–0083, +contracts, tests, runners, adapters, telemetry, experimental manifests, +evaluation logs, and the current package API. The complete deterministic local +suite passed: 367 tests. This is local evidence only. It is not provider, +model, Docker, SWE-bench, benchmark, or scientific qualification evidence. + +The current external authority boundary remains explicit: the official +SWE-bench Docker Gold path is recorded as `resolved: true` for frozen tasks, +but the real provider/model and the A/B/C/D mechanism trial remain +unqualified. Local OpenRouter/minisweagent and Git Bash paths are blocked; +Docker access is blocked by infrastructure permissions; Ollama identity and +configuration are not frozen sufficiently for qualification. + +## Module audit + +The entries below use the requested dimensions in compact form: +responsibility/API/invariants; dependencies and coupling; error/recovery; +idempotency and persistence; observability/tests; risks/debt/maturity. + +| Module | Responsibility, contract and invariants | Dependencies, errors, recovery, persistence and tests | Coupling, risks and maturity | +|---|---|---|---| +| request | Authorizes a request with task, scope, authority, environment and budget. Public request/governance contracts reject missing authority/scope/budget and preserve explicit blocked state. | Upstream of characterization/routing/selection. Error is blocked governance; no executor recovery. Deterministic serialization and request tests exist. | Provider/model-independent, but policy semantics are still embedded in request types. `FUNCTIONAL_BUT_WEAK`. | +| selection | Selects a treatment and executor binding from explicit registries and qualification status. | Depends on routing/treatment definitions and executor registry; blocks rather than cascades. Selection tests cover advertised capabilities and invalid bindings. | Executor identity/capability fields still enter selection; no full capability discovery or dynamic registry persistence. `FUNCTIONAL_BUT_WEAK`. | +| localization | P8.2 evidence-budgeted candidate file/symbol/context localization. | Depends on repository input and explicit evidence budget; overflow is deterministic and non-retryable. Artifacts serialize evidence budget; focused tests exist. | Treatment-specific P8.2 implementation, not yet a generalized localization protocol. `PARTIAL`. | +| editing | P8.2 edit artifact contract and patch validation. | Depends on localization and repository paths; invalid/missing patch evidence blocks. Persistence is artifact-based; tests cover deterministic patch boundaries. | Still treatment/P8.2 shaped and not a universal edit operation protocol. `PARTIAL`. | +| orchestration | Sequentially composes governance, characterization, routing, selection, preflight, execution, progress, verification, acceptance and terminal telemetry. | Depends on all gates and optional event log. It separates verification/acceptance and never selects fallback executors. Budgeted plans require `run_with_budget`; persistence failures are explicit. Tests cover order, gates, terminal event and capability preflight. | Central seam is still a large composition function and replay-driven resume remains outside the live pipeline. `FUNCTIONAL_BUT_WEAK`. | +| acceptance | Independent authority converts verification into explicit acceptance result. | Depends on verification evidence and authority identity; mismatch/blocked/indeterminate states are preserved. Causal verification reference is included. Tests cover decisions and boundary failures. | Authority contract is explicit, but policy composition and durable authority evidence remain thin. `FUNCTIONAL_BUT_WEAK`. | +| verification | Neutral verification result separates pass/fail/blocked/indeterminate/not-executed from execution. | Depends on execution artifacts/evaluator boundary; failures map to explicit outcome/error envelopes. Tests cover outcome mapping and causal reference. | Verifier protocol exists; evaluator abstraction and multiple concrete verifiers are not generalized. `FUNCTIONAL_BUT_WEAK`. | +| recovery | Bounded recovery/replan budget and lineage contracts. | Depends on prior attempt identity and explicit retry/recovery policy; no implicit executor switch. Lineage persists in artifact store and is replay-auditable. Tests cover exhausted budget and identity drift. | Planner is not yet integrated into the main orchestration lifecycle as a first-class state transition. `PARTIAL`. | +| promotion | Gate that requires independent acceptance plus matching verification and causal references. | Depends on verification/acceptance and manifest state; rejects mismatch and lacks automatic promotion. Promotion event requires prior acceptance in event log. Tests cover rejection and event persistence. | Good boundary, but promotion is not yet a fully independent persisted workflow with operator decision records. `FUNCTIONAL_BUT_WEAK`. | +| baseline / execution | Neutral `Executor`, `ExecutionRequest`, `ExecutionPlan`, `ExecutionResult`, budgets, retry policy, capability profiles and fake implementations. | Downstream adapters execute; upstream orchestration plans. Contract lifecycle and budget ledger are immutable/idempotent. Budget enforcement is an explicit runner capability with deterministic rejection when absent. | External concrete execution remains blocked/unqualified and legacy callers without the capability remain blocked. `FUNCTIONAL_BUT_WEAK`. | +| provider adapters | Provider protocol and normalized provider error envelope exist. | Provider failures preserve raw context and retryability; concrete OpenRouter path is import-safe but execution-strict. Adapter preflight records identity/dependencies. | Provider identity, model and configuration still live in P8.2 adapter surfaces; no independently qualified provider registry. `EXPERIMENT_COUPLED`. | +| executor adapters | Executor protocol, preflight, capability profile, and P8.2 baseline adapter exist. | Adapter errors map to neutral domains; minisweagent absence and Git Bash access failure remain explicit blocked/unknown. | Concrete adapter is still a qualification/treatment adapter rather than a stable production plugin boundary. `EXPERIMENT_COUPLED`. | +| telemetry | Event contracts, lifecycle counters, replay state, structured error codes and terminal facts. | Depends on event log and artifact references; deterministic reduction exists; malformed/missing facts remain incomplete/ inconsistent. Tests cover replay, telemetry consistency and terminal immutability. | No shared trace correlation across all external process/provider logs; telemetry is mostly event-level. `FUNCTIONAL_BUT_WEAK`. | +| error taxonomy | Neutral `ErrorDomain`, `ErrorEnvelope`, retryability and adapter normalization. | Upstream adapters and downstream retry/recovery consume it; unknown/blocked are preserved. Tests cover provider/process/exception/preflight mappings. | Codes are still partly stringly typed and executor-specific codes can leak in `raw` fields. `FUNCTIONAL_BUT_WEAK`. | +| task state | `RunState`, transition validation, orchestration status and explicit terminal lifecycle. | State transitions are deterministic and terminal states immutable. Manifest and event log both persist state-related facts. Tests cover illegal transitions and replay mismatch. | Two state vocabularies (`RunState` and `OrchestrationStatus`) are bridged manually; no single lifecycle state machine authority. `FUNCTIONAL_BUT_WEAK`. | +| artifact/state persistence | Atomic manifest/artifact store, attempt/retry lineage, event log, audit and replay. | Depends on stable references and identity digests; incomplete chains are not accepted. Tests cover atomicity, audit, comparison and replay. | Cross-file persistence is deliberately audit-based, not transactionally atomic; real-runtime crash-resume effectiveness remains unqualified. `FUNCTIONAL_BUT_WEAK`. | +| experimental protocol | Frozen task/treatment identity, manifests, comparability and qualification gates. | Depends on provider/model/config/budget/runtime identity and raw logs. Blocks missing identity and separates local evidence from external authority. Tests cover audited experiments and comparability. | Experimental protocol is strong conceptually but still P8.2-shaped and lacks a universal experiment service. `FUNCTIONAL_BUT_WEAK`. | +| configuration | Configuration digests and typed secret-safe snapshots exist on plans, adapters and experiments. | Configuration is passed through request/plan/adapter identities; orchestration retains a supplied snapshot and rejects digest mismatch. | Propagation remains optional for legacy callers and there is no universal resolution record for every adapter. `FUNCTIONAL_BUT_WEAK`. | +| CLI / entrypoints | Tools and scripts expose baseline/preflight/probe flows. | Entrypoints preserve strict failure and write logs/manifests where implemented. Tests cover selected tools, not every invocation path. | CLI surface is experimental and not a stable public command contract. `PARTIAL`. | +| package/public API | `arkx.__init__` exports neutral contracts, lifecycle, routing, integration and audit APIs. | Import boundaries are tested and optional dependencies are lazy/strict at execution. | Public API is broad and currently exposes both legacy P8.2 and neutral seams without deprecation/version policy. `FUNCTIONAL_BUT_WEAK`. | + +## 1. Architecture Gap Matrix + +| módulo | maturidade | principal fraqueza | impacto | dependências | ação necessária | +|---|---|---|---|---|---| +| request | FUNCTIONAL_BUT_WEAK | policies embedded in request | inconsistent policy evolution | governance, routing | extract versioned policy inputs | +| selection | FUNCTIONAL_BUT_WEAK | shallow capability/qualification input | wrong treatment binding | registry, routing | capability-aware selection contract | +| localization/editing | PARTIAL | P8.2-specific contracts | poor composition | treatment, repository | generalize evidence and edit interfaces | +| orchestration | FUNCTIONAL_BUT_WEAK | monolithic composition seam | hard isolated evolution; live resume remains external | all gates | split plan/run/gate/persist services when empirical consumers require it | +| acceptance/verification | FUNCTIONAL_BUT_WEAK | evaluator/authority implementations thin | false confidence or blocked decisions | artifacts, authority | evaluator/verifier adapters and independent records | +| recovery | PARTIAL | not a first-class lifecycle stage | incomplete resume/replan | attempt store, retry | integrate recovery planner and state transitions | +| promotion | FUNCTIONAL_BUT_WEAK | event exists but workflow is thin | premature or non-reproducible promotion | acceptance, verification | durable promotion gate record | +| execution/baseline | FUNCTIONAL_BUT_WEAK | external adapters remain unqualified | portability and operational uncertainty | executor runners | qualify concrete combinations; keep non-capable legacy runners blocked | +| provider/executor adapters | EXPERIMENT_COUPLED | concrete paths tied to P8.2/runtime | portability and qualification risk | minisweagent, OpenRouter, Git Bash | stable plugin adapters plus capability qualification | +| telemetry/errors | FUNCTIONAL_BUT_WEAK | string codes and partial trace correlation | diagnosis/replay friction | event log, artifacts | canonical error/trace schema | +| task state/persistence | FUNCTIONAL_BUT_WEAK | dual vocabularies and audit-based cross-file persistence | divergent replay/resume under real crashes | manifests, event log | retain explicit crosswalk; add stronger coordination only for concurrent/distributed consumers | +| protocol/config/CLI/API | PARTIAL / FUNCTIONAL_BUT_WEAK | no central versioned config/CLI policy | reproducibility and compatibility debt | all modules | versioned schemas and compatibility policy | + +## 2. Missing Architecture + +Remaining or conditionally required architecture, not all immediately required: + +1. A concrete adapter qualification registry that persists capability + provenance and qualification authority independently from selection. +2. A coordinated resume/replay protocol that defines what may be resumed after + a crash and what must remain blocked or indeterminate. +3. A stable evaluator/verifier adapter boundary with raw result retention and + multiple independently exercised concrete verifiers. +4. A single lifecycle state machine is optional at current scope: the explicit + `RunState`/`OrchestrationStatus` crosswalk is the current tested decision. + +Already present and therefore not recommended for speculative duplication: +neutral Executor/Provider/Sandbox protocols, capability registry, routing +decision, execution plan, artifact store, event log/replay, budget ledger, +retry policy, recovery lineage, promotion gate, and experiment identity digest. +An event bus would be overengineering now; the append-only event log is enough +until multiple independent consumers or distributed execution require fan-out. + +## 3. Coupling Map + +```text +request -> routing -> selection -> orchestration -> ExecutorRunner + | | + | +--> P8.2 baseline/minisweagent + | +--> provider/model config + | +--> Docker/Git Bash/Ollama runtime + | + +--> verification/evaluator + +--> acceptance authority + +--> event log/artifact store + +P8.2 treatment ----> localization/editing ----> SWE-bench authority +provider/model ----> preflight + execution logs +Docker ------------> baseline qualification and external authority +SWE-bench ---------> authority result, not neutral execution contract +``` + +The leaks are concentrated in concrete adapter and qualification layers, while +the orchestration seam now blocks a runner without an explicit budget- +enforcement capability. SWE-bench is correctly outside the neutral execution contract; +the remaining risk is that P8.2 assumptions enter through selection and runner +interfaces rather than through an explicit treatment adapter. + +## 4. Critical Path + +1. Qualify at least one real executor/provider/sandbox combination with the + already-established identity, preflight, budget, and replay gates. +2. Split orchestration into plan, preflight, execute, verify, accept and + persist components with one lifecycle state authority. +3. Generalize evaluator/verifier and recovery seams, including crash-resume + rules. +4. Freeze a versioned configuration snapshot and trace identity across all + adapter logs. +5. Do not use a provider result to infer architecture quality; acceptance and + promotion remain independent authorities. + +## 5. Parallelizable Work + +Can proceed without waiting for any external executor result: + +- lifecycle state-machine consolidation; +- evaluator/verifier adapter contract; +- recovery planner integration tests using fake executor/provider/sandbox; +- configuration snapshot schema and canonical digest tests; +- crash/replay/resume property tests; +- adapter capability registry and qualification-record tests; +- CLI contract tests and package API deprecation/version checks; +- update of the experimental protocol to require these identities. + +## 6. Experiment-dependent + +Must wait for infrastructure or external evidence: + +- qualification of OpenRouter/minisweagent execution; +- Docker/SWE-bench authority runs; +- provider/model mechanism comparison; +- real sandbox performance, timeout and cost distributions; +- empirical retry/recovery effectiveness; +- promotion of any executor, model, provider or treatment. + +## Latest local verification + +The lineage-event-chain audit block was verified with the focused replay and +restored-chain tests, followed by the complete deterministic suite: 365 tests +passed. This is local evidence only; it does not qualify a real provider, +model, executor, sandbox, or SWE-bench treatment. + +The budget-enforced legacy-runner block was then verified with the complete +deterministic suite: 366 tests passed. A budgeted plan now blocks a runner that +cannot declare `run_with_budget`; the neutral contract bridge is the first +implementation of that boundary. This closes the previously identified silent +legacy budget-bypass gap locally, but does not qualify any external runner. + +The orchestration-configuration-snapshot block was then verified with 367 +passing deterministic tests. A supplied typed snapshot now survives into the +execution plan and is checked against executor identity; legacy callers without +one remain explicitly weaker rather than being upgraded by inference. + +## Conclusion + +Arkx is no longer merely a collection of executor-specific experiments: the +neutral contract, capability, lifecycle, audit and replay foundations are +materially present. It is not yet a complete executor-agnostic architecture. +The decisive remaining issues are concrete adapter qualification and empirical +validation of retry/recovery under real runtimes. Persistence coordination is +deliberately audit-based at the current single-process scope; a transaction +coordinator or event bus is deferred until concurrent or distributed consumers +create evidence for that need. Budget enforcement and typed configuration +identity now have explicit local gates, while legacy callers without the new +evidence remain blocked or weaker rather than being upgraded by inference. diff --git a/docs/architecture-strengthening-plan.md b/docs/architecture-strengthening-plan.md new file mode 100644 index 00000000..8d376b2d --- /dev/null +++ b/docs/architecture-strengthening-plan.md @@ -0,0 +1,470 @@ +# Plano de fortalecimento arquitetural do Arkx + +## Objetivo + +Fortalecer o núcleo do Arkx para que ele seja executor-agnostic, reproduzível +e evolua independentemente de mini-SWE-agent, Nemotron, NOOA, SWE-agent ou +qualquer outro executor. + +O plano não tem como objetivo escolher executor. O objetivo é criar boundaries +que permitam conectar, medir, comparar, aceitar, promover ou remover executores +sem alterar request, routing, verification, acceptance e promotion. + +## Princípios de decisão + +Toda mudança deve satisfazer simultaneamente: + +```text +CONTRATO EXPLÍCITO +AND +TESTE EXECUTÁVEL +AND +EVIDÊNCIA REPRODUZÍVEL +AND +ACEITAÇÃO INDEPENDENTE +``` + +As seguintes distinções são obrigatórias: + +```text +IMPLEMENTAÇÃO != EXECUÇÃO +EXECUÇÃO != VERIFICATION +VERIFICATION != ACCEPTANCE +ACCEPTANCE != PROMOTION +LOCAL PASS != EXPERIMENTAL EVIDENCE +BLOCKED != PASS +UNKNOWN != PASS +``` + +## Estado inicial conhecido + +- O núcleo P0–P7 possui contratos determinísticos e testes unitários fortes. +- P8.2/P8.2a adicionam execução real, mas ainda acoplam o baseline a + mini-SWE-agent, provider, subprocess, Git Bash e SWE-bench. +- Os outcomes de verification/acceptance já foram extraídos para um módulo + neutro; `p82_baseline` mantém reexports temporários. +- A autoridade SWE-bench Docker possui evidência `resolved: true` para tarefas + Gold congeladas. +- O Wave 0 Treatment A está bloqueado por indisponibilidade do Docker daemon. +- O ambiente atual não possui `pytest` nem `minisweagent` instalados; testes + core requerem `PYTHONPATH=src`. + +Esses fatos são baseline da auditoria, não justificativa para adoção de +executor. + +## Confrontação com a decisão 0012 + +A decisão [0012](decisions/0012-evidence-routed-treatment-architecture.md) +continua sendo a referência estratégica: rotear por tratamento/capability, +executar depois da seleção, verificar antes de aceitar e exigir evidência antes +de promover. + +Ela não substitui os contratos técnicos deste plano. Em particular: + +1. `ROUTE BY REQUIRED CAPABILITY` precisa de um capability contract versionado, + não somente de nomes de tratamento ou strings no registry. +2. `SUFFICIENT EXECUTOR` precisa ser uma decisão sobre um executor capaz de + executar o contrato, não apenas sobre metadata qualificada. +3. A separação entre P8.2, P8.3 e P8.4 é uma ordem experimental; não autoriza + o núcleo a importar os componentes concretos dessas fases. +4. A afirmação retrospectiva de complementaridade entre executores só pode ser + usada como evidência normativa depois que seus dados brutos e identidade + experimental forem localizados e reproduzíveis no repositório ou em uma + autoridade referenciada. + +Assim, a decisão 0012 permanece válida como hipótese arquitetural e princípio +de desenho. Os números retrospectivos que não possuem rastreabilidade local +devem permanecer como `UNVERIFIED` para fins de promoção. + +## Ordem de execução + +### Fase 0 — Controle do trabalho e baseline + +Responsabilidade: tornar o estado de desenvolvimento e teste reproduzível. + +Entregáveis: + +- comando documentado para instalar/importar o pacote; +- comando de teste determinístico; +- relatório de baseline com branch, HEAD, ambiente e dependências; +- separação entre arquivos de trabalho, fixtures e artifacts de execução; +- teste mínimo de import público. + +Hipótese falsificável: + +> Um ambiente documentado consegue importar o pacote e executar todos os +> testes determinísticos sem depender de configuração manual implícita. + +Critérios de aceitação: + +- ambiente limpo reproduz o mesmo resultado; +- falhas de dependência aparecem como bloqueio explícito; +- nenhuma execução seleciona fallback silencioso; +- o conjunto de testes não grava artifacts de experimento no repositório. + +### Fase 1 — Núcleo de outcomes e erros + +Responsabilidade: remover dependências de P8.2 das fronteiras centrais. + +Entregáveis: + +- outcomes neutros de verification e acceptance; +- error envelope comum; +- códigos de erro com domínio, origem, severidade e retryability; +- compatibilidade temporária com imports legados. + +Hipótese: + +> Acceptance, promotion e orchestration podem operar sem importar qualquer +> runner ou adapter experimental. + +Critérios: + +- análise estática confirma ausência de imports proibidos; +- testes preservam serialização e estados existentes; +- erros de provider, executor, sandbox, verification e policy mantêm origem; +- `BLOCKED`, `UNKNOWN` e `NOT_EXECUTED` não são convertidos em sucesso. + +Progresso verificável: outcomes neutros foram extraídos para `arkx.outcomes` +e `arkx.harness` agora define `RunManifest`, `ErrorEnvelope` e identidade +determinística de attempt. Os testes focados passam; a integração do adapter +OpenRouter permanece bloqueada pela dependência externa ausente. A fase +continua aberta até a migração controlada de um runner e o teste de replay. + +### Fase 2 — Contratos de execução + +Responsabilidade: definir interfaces neutras, sem implementar outro executor. + +Entregáveis: + +- `Executor` protocol; +- `Provider` protocol; +- `Sandbox` protocol; +- `ExecutionRequest`; +- `ExecutionResult`; +- `ExecutionArtifact` genérico; +- capability descriptors versionados. + +Hipótese: + +> Um fake executor determinístico pode percorrer o fluxo completo sem que o +> núcleo conheça mini-SWE-agent, OpenRouter, Docker ou SWE-bench. + +Critérios: + +- fake executor cobre completed, failed, blocked, timeout e not executed; +- núcleo não importa SDK, subprocess ou Docker; +- capabilities declaradas não são tratadas como qualification empírica; +- provider e sandbox podem ser substituídos por fakes. + +Bloco 2 concluído com evidência local: `Executor`, `Provider`, `Sandbox`, +requests/results e fakes determinísticos foram adicionados. A integração do +runner P8.2 real ainda não foi feita e nenhum executor/provider foi promovido. + +Bloco 3 concluído com evidência local: `ContractRun` percorre lifecycle fake +completo e mantém verification/acceptance como attachments independentes. + +Bloco 5 concluído com evidência local: orchestration consome o protocolo +`Executor` através de uma ponte explícita, sem mover verification, acceptance +ou promotion para o executor. + +Bloco 6 concluído com evidência local: a entrypoint de orchestration aceita o +protocolo neutro diretamente e preserva equivalência comportamental com a +ponte legada. + +Bloco 8 concluído com evidência local: `arkx.ExecutionResult` agora aponta +para o contrato neutro e o resultado legado possui nome público distinto. + +Bloco 9 concluído com evidência local: um teste de arquitetura impede novos +imports do resultado legado fora da camada de compatibilidade. + +Bloco 10 concluído com evidência local: `ExecutionPlan` separa intenção +congelada de execução de `ExecutionResult` e produz requests sem efeitos +externos. + +### Fase 3 — Lifecycle, plan e budgets + +Responsabilidade: transformar estados dispersos em fluxo persistível e +verificável. + +Entregáveis: + +- task lifecycle state machine; +- execution plan; +- budget manager; +- retry policy; +- recovery planner; +- transições válidas e inválidas documentadas. + +Hipótese: + +> Um fluxo interrompido pode ser retomado ou bloqueado de forma determinística +> sem repetir etapas já confirmadas e sem ultrapassar budgets. + +Critérios: + +- cada transição possui teste positivo e negativo; +- retryable e non-retryable são distinguíveis; +- budgets são consumidos uma única vez por attempt; +- recovery não troca executor nem expande escopo silenciosamente; +- replay do mesmo input produz o mesmo plano e identidade. + +### Fase 4 — Artifact store, trace e replay + +Responsabilidade: tornar a execução auditável e reproduzível. + +Entregáveis: + +- `ArtifactStore` local; +- manifests e checksums; +- lineage request → plan → execution → verification → acceptance; +- event log persistente; +- trace/replay contract; +- writes atômicos e attempts imutáveis. + +Hipótese: + +> Toda decisão material pode ser reconstruída a partir de artifacts persistidos +> sem depender de memória do processo ou logs informais. + +Critérios: + +- artifact parcial não é apresentado como completo; +- dois runs equivalentes compartilham identidade estável; +- artifacts divergentes produzem identidade divergente; +- replay preserva decisões e estados, mas não finge reexecução; +- falhas mantêm raw output suficiente para diagnóstico. + +### Fase 5 — Adaptação do baseline experimental + +Responsabilidade: adaptar mini-SWE-agent ao novo boundary, sem promovê-lo. + +Entregáveis: + +- adapter do runner atual para `Executor`; +- adapter OpenRouter para `Provider`; +- Git Bash/local/Docker atrás de `Sandbox`; +- SWE-bench atrás de `Evaluator`/`AcceptanceAuthority`; +- preservação dos artifacts atuais. + +Hipótese: + +> O baseline atual pode ser conectado aos contratos neutros sem alterar seus +> resultados observados nem fazer o núcleo depender dele. + +Critérios: + +- os artifacts atuais continuam legíveis; +- provider failure, executor failure e sandbox failure permanecem distintos; +- não há fallback implícito; +- testes com fake adapters passam sem dependências externas; +- testes de integração externa são marcados como bloqueados quando faltarem + Docker, provider, modelo ou pacote. + +### Fase 6 — Verificação empírica dos tratamentos + +Responsabilidade: somente depois das fases anteriores, executar experimentos. + +Entregáveis: + +- protocolo congelado; +- task sample congelado; +- identidade completa de executor/provider/model/sandbox/config; +- replicates serializados; +- raw evidence; +- acceptance independente; +- decisão de promotion separada. + +Hipóteses possíveis: + +- B1 melhora tarefas localizadas sob o mesmo executor; +- C1 melhora localização sem aumentar desnecessariamente o escopo; +- D1 apresenta interação entre B1 e C1; +- outro executor possui capability gap comprovado contra o baseline. + +Nenhuma dessas hipóteses deve ser assumida como verdadeira antes da execução. + +Critérios: + +- controles divergentes são `INCOMPARABLE`; +- identidade ausente é `UNKNOWN` ou `BLOCKED`; +- falha de infraestrutura não é rejeição da tarefa; +- resultado experimental não promove automaticamente componente algum. + +## Gate de engenharia de software + +Antes de aceitar cada fase, revisar: + +- responsabilidade única; +- API pública mínima; +- dependências acíclicas e direcionais; +- invariantes explícitos; +- tratamento de erro completo; +- idempotência e reentrância; +- persistência segura; +- compatibilidade de schema; +- testes determinísticos; +- ausência de acoplamento experimental indevido. + +## Gate de reprodutibilidade empírica + +Antes de aceitar qualquer evidência experimental, revisar: + +- hipótese falsificável; +- inputs e task sample congelados; +- executor/provider/model/sandbox identificados; +- versão e configuração registradas; +- budget registrado; +- ambiente e identidade registrados; +- run identity estável; +- execução serial quando exigida; +- raw artifacts preservados; +- verification independente; +- acceptance explícita; +- promoção separada; +- blocked/unknown/not-executed preservados. + +## Regras de parada + +O trabalho deve parar e registrar `BLOCKED` quando: + +- o contrato necessário ainda for implícito; +- uma dependência externa impedir reproducibilidade; +- a identidade do executor/provider/modelo estiver ausente; +- um artifact obrigatório não puder ser preservado; +- uma comparação tiver controles divergentes; +- a única evidência disponível for um teste local; +- a próxima ação exigisse escolher executor sem hipótese ou critério. + +## Ordem imediata + +1. Finalizar a Fase 0 com baseline executável de ambiente e imports. +2. Completar a Fase 1 com error envelope comum. +3. Implementar os contratos da Fase 2 usando apenas fakes. +4. Só então iniciar lifecycle, persistence e adaptação do baseline. + +O plano será atualizado apenas com evidência observada, nunca por inferência de +que uma implementação existente já provou a capacidade correspondente. + +## Continuidade registrada — blocos 10 e 11 + +O bloco 10 introduziu o `ExecutionPlan` como contrato serializável e separado +do resultado da execução. O bloco 11 integrou esse plano à orquestração antes +da invocação do executor. A evidência local atual é de 101 testes focados +passando, com determinismo de serialização, validação de identidade e +compatibilidade entre executor neutro e ponte legada. + +Essa evidência não promove executor, provider, modelo, sandbox ou tratamento. +As integrações externas continuam bloqueadas quando faltam suas dependências e +identidades observáveis. + +O bloco seguinte adiciona um `CapabilityProfile` e um `CapabilityRegistry` +neutros. A evidência local de 104 testes confirma serialização estável, +ordenação determinística e rejeição de identidades duplicadas. A implementação +não infere capacidades a partir de nomes e não substitui qualification externa. + +O bloco seguinte adiciona `ExecutionBudgetSpec` como declaração comum de limites +de execução e identidade por digest. A evidência local de 105 testes cobre +validação e determinismo. A semântica de consumo ainda não foi unificada entre +request, routing, recovery e qualification; essa unificação exige decisão e +testes próprios. + +O bloco seguinte integra essa lineage ao `ArtifactStore`: a reserva do próximo +attempt é append-only e grava o vínculo anterior/seguinte atomicamente. A +evidência local de 111 testes cobre persistência, duplicidade e drift de +identidade; execução e verificação continuam fora da operação. + +O bloco seguinte integra leitura de lineage ao `AttemptStore`. A evidência local +de 112 testes confirma round-trip e rejeição de drift, preservando artifacts +históricos sem inferir retry a partir apenas do número do attempt. + +O bloco seguinte adiciona `RetryPolicy` como autorização determinística e +separada de recovery. A evidência local de 107 testes cobre retryability, +domínio permitido e limites de política/budget. A política não executa retries, +não troca executor e não faz fallback implícito. + +O bloco seguinte adiciona lineage determinístico para retries. A evidência local +de 109 testes confirma novo `attempt_id`, preservação do attempt anterior e +ausência de execução automática. A criação de diretório e persistência continuam +separadas para manter reentrância e evitar sobrescrita silenciosa. + +O bloco seguinte adiciona `BudgetLedger` imutável, com consumo idempotente por +attempt e rejeição não mutante quando um limite é excedido. A evidência local de +114 testes confirma a propriedade; a integração semântica com os demais budgets +ainda requer decisão própria. + +O bloco seguinte liga a especificação de budget ao `ExecutionPlan`, validando o +digest e preservando a identidade durante a orquestração. A evidência local de +115 testes confirma a fronteira; consumo de ledger e enforcement em executor +real continuam separados. + +O bloco seguinte persiste o ledger no `RunManifest` com vínculo explícito ao +attempt e ao digest. A evidência local de 116 testes confirma rejeição de drift e +imutabilidade do snapshot; consumo automático pelo executor ainda não é +assumido. + +O bloco seguinte persiste atomicamente o manifest validado no artifact store. +A evidência local de 119 testes confirma round-trip e validação prévia de +identidade/lifecycle; verification, acceptance e promotion permanecem estados +separados. + +O bloco seguinte integra o ledger ao lifecycle contratual: consumo aceito libera +uma invocação e consumo rejeitado bloqueia antes do executor. A evidência local +de 118 testes confirma a ordem e a ausência de side effect no caminho bloqueado. + +O bloco seguinte adiciona `EventLog` persistente sobre o contrato `Event`, com +ordenação temporal e identidade de run verificadas. A evidência local de 121 +testes cobre round-trip e fronteiras de erro; o log não substitui manifest, +verification ou promotion. + +O bloco seguinte adiciona replay determinístico dos eventos persistidos. A +evidência local de 123 testes cobre identidade e fatos declarados, mantendo +reexecução e acceptance fora do reducer. + +O bloco seguinte adiciona validação cruzada entre replay e manifest. A evidência +local de 124 testes confirma detecção de divergência sem escolher uma fonte de +verdade ou reparar estado automaticamente. + +O bloco seguinte mantém verification e acceptance como outcomes independentes +no replay. A evidência local de 125 testes confirma comparação com evidências +externas e rejeição de conflito, sem inferência a partir da execução. + +O bloco seguinte reconstrói a cadeia completa de referências persistidas. A +evidência local de 127 testes cobre ordem causal e ausência explícita, mantendo +reexecução e decisões fora do reducer. + +O bloco seguinte classifica referências explícitas contra um inventário de +artifacts. A evidência local de 129 testes separa presença, ausência e +indeterminação sem dereferenciar URIs ou inventar evidência física. + +O bloco seguinte conecta replay ao inventário físico validado do attempt. A +evidência local de 130 testes confirma presença e ausência sem scanning externo, +seleção de substitutos ou promoção implícita. + +O bloco seguinte adiciona classificação explícita de integridade da cadeia. A +evidência local de 131 testes distingue completude, ausência e inconsistência +antes de qualquer decisão de acceptance/promotion. + +O bloco seguinte materializa a decisão de recovery em plano persistível com +lineage do attempt. A evidência local de 137 testes confirma determinismo e +separação da execução concreta. + +O bloco seguinte integra recovery à cadeia de replay como estágio opcional entre +execution e verification. A evidência local de 138 testes confirma causalidade, +compatibilidade e ausência de execução automática. + +O bloco seguinte valida a identidade canônica do plano de recovery no replay. A +evidência local de 138 testes confirma rejeição de referências divergentes sem +introduzir execução ou handoff implícito. + +O bloco seguinte fecha a identidade entre `ExecutionPlan` e attempt de execução. +A evidência local de 139 testes confirma referências canônicas e rejeição de +drift antes de verification/acceptance. + +O bloco seguinte fecha identidade determinística de verification e acceptance, +mantendo namespaces separados. A evidência local de 139 testes confirma +rejeição de drift antes de promotion. + +O bloco seguinte fecha a cadeia `verification → acceptance → promotion` com +identidades verificáveis. A evidência local de 139 testes confirma lineage sem +alterar o policy engine ou executar promotion. diff --git a/docs/architecture.md b/docs/architecture.md index 3a54efe0..d3ad8643 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,23 +1,100 @@ -# Architecture boundary +# Codepro architecture boundary -The bootstrap architecture is intentionally small: +Codepro separates contracts, execution, verification, acceptance, and +promotion. `arkx` remains the current Python namespace and source package path +for compatibility. +The executable contracts cover the request-to-acceptance path. Runtime +capability is qualified per provider, model, executor, and treatment; the +presence of a contract does not imply adoption. ```text -Arkx -├── docs/ contracts, protocol, and decisions -├── experiments/ experiment records and fixtures; no runtime yet -├── src/ reserved product boundary; no implementation yet -├── tests/ reserved verification boundary; no test suite yet -└── tools/ dependency-free repository checks +Codepro +├── docs/ contracts, protocol, decisions, and evidence boundaries +├── experiments/ experiment records and synthetic fixtures; not orchestration +├── src/arkx/ executable contracts and bounded policy/runtime surfaces +├── tests/ executable contract and behavior verification +└── tools/ repository checks and narrow execution adapters ``` +The current executable boundary is layered as follows: + +- P0: versioned contracts, evidence status resolution, telemetry collection, + and a deterministic baseline fixture; +- P1: explicit-signal task characterization; +- P2/P3: progress assessment, bounded routing, and evidence-based escalation; +- P4/P5/P6: repository planning/state, patch verification, and handoff/context + accounting; +- P7/P8.1/P8.2: composition records, executor qualification contracts, and + controlled treatment plans; +- P8.2a: a narrow Treatment A baseline runner and the official SWE-bench + acceptance adapter; +- current orchestration surfaces: request governance, treatment/executor + selection, sequential execution, recovery, independent acceptance, and + promotion gates. + +The canonical flow is: + +```text +request + -> governance / authority + -> task characterization + -> treatment route + -> sufficient executor + -> progress / recovery + -> verification + -> independent acceptance + -> evidence for possible promotion +``` + +Routing selects the minimum sufficient treatment before selecting an executor. +Simple, localized, and repository-wide paths are capabilities/regimes, not +automatic executor cascades. A fallback, executor switch, scope expansion, +acceptance, or promotion must remain explicit and evidence-backed. + +The official SWE-bench Docker authority has been qualified on the frozen Gold +tasks. That validates the acceptance path and infrastructure, not a real +provider/model, Treatment A execution, or A/B/C/D mechanism evidence. + ## Boundary rules -- `src/` is reserved for product code and is empty in this bootstrap. +- `src/arkx/` contains product contracts and bounded policy/runtime code; it + must not silently grow into an unqualified general-purpose orchestrator. - `experiments/` contains evidence-oriented records, not an implicit runtime. -- `tools/` may validate repository contracts but must not become orchestration. -- `tests/` is reserved for executable verification as capabilities are added. -- Integrations and executors require an explicit decision record and experiment evidence. +- `tools/` may validate contracts or host a narrow adapter, but must not become + an untracked orchestration layer. +- `tests/` verifies contracts and behavior; passing tests are local evidence, + not scientific or upstream evidence. +- Integrations, providers, models, and executors require explicit identity, + qualification evidence, and an acceptance authority. +- Missing provider/model identity, missing evidence, and blocked execution are + preserved as blocked or unknown; they never become success. +- The working tree may contain an implementation under qualification. Merge + readiness requires the corresponding decision record, tests, and evidence to + be committed together. + +The boundary is explicit: creating a module, fixture, or directory does not by +itself prove that the corresponding capability exists or is suitable for +promotion. + +## Qualification evidence boundary + +Qualification is indexed by one machine-readable experiment manifest and its +referenced raw artifacts. The manifest must bind, at the declared level of +authority, the Codepro/runner revision, dataset revision or fingerprint, +task/base revision, treatment and control role, executor/provider/model and +configuration, environment image/digest and OS/architecture, verifier +identity, budget, identity levels (`experiment_id`, `trial_id`, `attempt_id`, +and verifier `run_id`), `provider_called`, and the independent acceptance +decision. -An empty boundary is intentional. Creating a directory does not imply that the corresponding capability exists. +The manifest is not a replacement for raw evidence. It is the index that makes +the evidence discoverable and comparable. A verifier qualification is invalid +unless the same declared verifier path demonstrates both an expected failure +for a no-op/empty patch and an expected resolution for an official gold/oracle +patch. Neither control authorizes promotion of a provider, model, executor, +treatment, or product capability. +Historical architecture audits and decision records remain valuable, but their +claims are scoped to the revision and collection date recorded in the file. +They must not be read as the current repository state without a current +manifest or acceptance record. diff --git a/docs/baseline-metrics.md b/docs/baseline-metrics.md new file mode 100644 index 00000000..f7a351e7 --- /dev/null +++ b/docs/baseline-metrics.md @@ -0,0 +1,35 @@ +# P0 baseline metrics + +P0 establishes measurement semantics; it does not establish a performance baseline. + +| Metric | Meaning | Unknown representation | +| --- | --- | --- | +| `acceptance_status` | Explicit final execution status, including `UNVERIFIED` when evidence is insufficient | `NOT_EXECUTED` before a task finish event | +| `tokens` | Observed input, output, and total token usage from events | JSON `null` when not observed | +| `cost` | Observed monetary cost from events or executor metadata | JSON `null` when not observed | +| `wall_time` | Elapsed time between first `TASK_STARTED` and last `TASK_FINISHED` | JSON `null` when either boundary is absent | +| `retries` | Count of explicit `RETRY` events | `0` means no such events were recorded | +| `replans` | Count of explicit `REPLAN` events | `0` means no such events were recorded | +| `executor_invocations` | Count of `EXECUTOR_STARTED` events | `0` means no such events were recorded | +| `handoffs` | Count of explicit `HANDOFF` events | `0` means no such events were recorded | +| `human_interventions` | Count of explicit `HUMAN_INTERVENTION` events | `0` means no such events were recorded | + +Counts are event-derived. Resource measurements are nullable: unknown tokens are not zero tokens, and unknown cost is not zero cost. + +## Boundary + +`TASK_FINISHED` with a requested `PASS` but no `EVIDENCE_ADDED` reference resolves to `UNVERIFIED`. P0 does not verify evidence quality and does not turn an executor claim into acceptance. + +```text +P0 DOES NOT CLAIM PERFORMANCE +P0 ONLY MAKES PERFORMANCE MEASURABLE +``` + +The deterministic fixture can be run with: + +```text +PYTHONPATH=src python -m arkx.baseline +``` + +No real run artifacts are committed by the test suite. + diff --git a/docs/benchmark-and-test-confrontation.md b/docs/benchmark-and-test-confrontation.md new file mode 100644 index 00000000..625ca0ab --- /dev/null +++ b/docs/benchmark-and-test-confrontation.md @@ -0,0 +1,229 @@ +# Confrontação do plano com testes e benchmarks + +## Pergunta + +Os testes locais e os benchmarks já registrados sustentam a direção de +fortalecer um núcleo executor-agnostic, separar execution/verification/ +acceptance/promotion e preservar bloqueios como estados explícitos? + +## Evidência de testes locais + +Com `PYTHONPATH=src`: + +| Evidência | Resultado | O que sustenta | O que não sustenta | +|---|---:|---|---| +| Testes P0–P7 e P8 determinísticos | 180 aprovados | contratos, serialização, invariantes e estados fail-closed | capacidade de executor ou provider | +| Testes de outcomes/orchestration/baseline/authority/promotion/harness | 75 aprovados | separação de outcomes, identidade de attempt, envelope de erro, manifest, replay, persistência atômica, lifecycle, presença de artefatos, consistência, leitura consolidada, indexação, comparabilidade, relatório de pares, preservação de métricas desconhecidas, autoridades independentes, gate de promotion, vínculo ao snapshot, binding, persistência da decisão e registro consolidado do experimento | execução real end-to-end | +| Teste OpenRouter adapter | bloqueado | dependência externa é explicitamente necessária | qualidade do adapter em ambiente ausente | +| Import público `arkx` | aprovado | pacote é importável com `PYTHONPATH=src` | instalação reproduzível de pacote | +| `pytest` | bloqueado | baseline de ambiente incompleto | resultado negativo do código | + +Os testes sustentam a direção de contratos pequenos, estados explícitos e +decisões puras. Eles não sustentam uma afirmação de capability operacional. + +## Evidência SWE-bench/Docker registrada + +Os artifacts contêm execuções distintas para as mesmas tarefas: + +| Grupo | Resultado observado | Interpretação válida | +|---|---|---| +| Docker runs 01–02 | `unresolved`, razão `missing_module` | falha de execução/ambiente da avaliação | +| Docker runs 03–04 | `resolved=1` | autoridade Docker conseguiu resolver as tarefas congeladas | +| Docker LF runs 01–02 | `resolved=1` | variante de ambiente também resolveu as tarefas | +| Qualification runs 03–04 | erro sem resultado de tarefa | qualification bloqueada/incompleta | +| Wave 0 Treatment A | `BLOCKED`, Docker daemon indisponível | nenhum model run foi executado | + +Esses resultados não devem ser agregados como uma taxa única. Eles possuem +identidades de run, ambientes e estados diferentes. A variação entre +`missing_module`, `resolved` e erro confirma a necessidade de preservar: + +- identidade completa do ambiente; +- executor authority separada da task result; +- falha de infraestrutura distinta de rejeição da tarefa; +- attempts não sobrescritos; +- replay e comparação somente entre controles compatíveis. + +## Confrontação das hipóteses do plano + +| Hipótese arquitetural | Evidência atual | Veredito | +|---|---|---| +| Outcomes devem ser neutros em relação ao executor | 75 testes passam após a extração e o contrato do harness | sustentada localmente | +| Acceptance não deve depender do runner | import dependency existia e foi removida dos módulos centrais | sustentada estruturalmente | +| Capability declarada não equivale a qualification | contratos P8.1 e testes preservam `UNKNOWN`/`BLOCKED` | sustentada localmente | +| Infra failure não deve virar task rejection | logs mostram Docker/provider failures separados | sustentada empiricamente | +| Fallback silencioso é perigoso | runs distintos possuem resultados e blockers diferentes | sustentada empiricamente | +| Um executor protocol neutro é necessário | runner atual importa mini/provider/sandbox indiretamente | sustentada por inspeção estrutural | +| Provider e sandbox precisam de boundaries próprias | OpenRouter, Git Bash e Docker têm falhas independentes | sustentada empiricamente | +| Arquitetura atual já é executor-agnostic | `p82_baseline` e SWE-bench ainda estão na API pública | falsificada pelo estado atual | + +### Evidência incremental mais recente + +Os blocos posteriores adicionaram contratos neutros de executor/provider/sandbox, +lifecycle, adapter do baseline, ponte de orquestração e `ExecutionPlan`. A +suíte focada atual relevante passou com 101 testes, incluindo a verificação de +que o plano é construído deterministicamente antes da invocação do executor e +que os caminhos neutro e legado permanecem compatíveis. + +Isso fortalece a direção arquitetural, mas não altera a conclusão empírica: +trata-se de evidência local de contrato. A qualificação de provider, modelo, +executor, Docker, SWE-bench ou mecanismo continua dependente de execução externa +com identidade e artefatos preservados. +| O plano já pode ser validado com benchmark de capability | Wave 0 está bloqueado e resultados Gold são autoridade de avaliação | não demonstrada | +| Um novo executor deve ser escolhido agora | não há capability gap experimental comparável | rejeitada por insuficiência de evidência | + +## Decisão + +A direção do plano está correta como direção de engenharia e governança +experimental. A evidência não autoriza ainda: + +- promover executor; +- escolher provider/modelo; +- afirmar eficácia de B1/C1/D1; +- usar SWE-bench Gold como benchmark de qualidade do Arkx; +- afirmar que o pipeline runtime está reproduzível end-to-end. + +## Consequência para o plano + +As Fases 0–4 devem continuar antes de qualquer novo experimento de mecanismo +ou comparação de executor. A Fase 5 deve adaptar o baseline somente depois que +`Executor`, `Provider`, `Sandbox`, lifecycle e artifact store tiverem contratos +testados com fakes. + +## Critério para reabrir a questão de executor + +A escolha de executor só pode ser reavaliada quando houver, para pelo menos uma +comparação controlada: + +1. task sample congelado; +2. executor, provider, modelo, sandbox, versão e configuração identificados; +3. mesma verification e acceptance authority; +4. budgets equivalentes; +5. replicates suficientes para a hipótese declarada; +6. raw artifacts preservados; +7. estados de infraestrutura separados dos resultados de tarefa; +8. acceptance independente; +9. decisão explícita de promotion. + +O bloco de capability registry elevou a suíte focada relevante para 104 testes +e adicionou uma forma determinística de comparar declarações de capacidade. +Isso reduz contratos implícitos, mas não é evidência de que a capacidade +declarada funciona em produção: essa distinção permanece explícita. + +O bloco de budget adicionou digest estável para limites de execução e elevou a +suíte focada para 105 testes. Isso melhora a reprodutibilidade da identidade do +plano, mas não prova enforcement de budget nem comportamento de retry. + +O bloco de retry transformou a decisão de nova tentativa em contrato explícito e +elevou a suíte focada para 107 testes. Isso reduz tratamento de erro implícito, +mas ainda não mede eficácia de retries em provider ou executor real. + +O bloco de lineage adicionou identidade determinística para o próximo attempt e +elevou a suíte focada para 109 testes. Isso melhora replay e diagnóstico, mas +não demonstra eficácia de retry nem comportamento de provider real. + +O bloco de artifact store elevou a suíte focada para 111 testes e passou a +persistir a cadeia de retry sem sobrescrita. Isso melhora replay e diagnóstico, +mas ainda não mede a eficácia do retry em infraestrutura real. + +O bloco de replay elevou a suíte focada para 112 testes e passou a validar +lineage durante a leitura. Isso melhora auditoria e reproducibilidade, mas não +transforma artifacts históricos incompletos em evidência de retry. + +O bloco de budget ledger elevou a suíte focada para 114 testes e demonstrou +consumo idempotente por attempt. Isso é evidência de contrato local, não prova +de enforcement distribuído ou de eficácia experimental. + +O bloco de plan binding elevou a suíte focada para 115 testes e tornou o budget +parte explícita da identidade do plano. Isso melhora comparabilidade e replay, +mas ainda não prova enforcement durante execução real. + +O bloco de persistência do ledger elevou a suíte focada para 116 testes e +registrou o uso no manifest com identidade verificável. Isso melhora auditoria, +mas não prova enforcement end-to-end em executor real. + +O bloco de lifecycle budget-aware elevou a suíte focada para 118 testes e +demonstrou gating local da invocação. Isso não substitui validação de enforcement +em executor/provider reais. + +O bloco de persistência atômica elevou a suíte focada para 119 testes e tornou o +manifest replayable. Isso não transforma a escrita em evidência de correção ou +aceitação da tarefa. + +O bloco de event log elevou a suíte focada para 121 testes e tornou o lifecycle +replayable em JSONL. Isso fortalece diagnóstico e reproducibilidade, mas não +converte telemetria em evidência de correção ou acceptance. + +O bloco de replay elevou a suíte focada para 123 testes e demonstrou +reconstrução determinística de fatos do lifecycle. Isso melhora reproducibilidade +sem fingir que replay é uma nova execução ou aceitação. + +O bloco de consistência elevou a suíte focada para 124 testes e passou a detectar +divergência entre eventos e manifest. Isso fortalece auditoria, mas não produz +aceitação nem corrige artifacts inconsistentes. + +O bloco de outcomes independentes elevou a suíte focada para 125 testes e +preservou a separação entre execução, verification e acceptance. Isso fortalece +o protocolo, mas não aceita nenhum patch por replay. + +O bloco de replay completo elevou a suíte focada para 127 testes e explicitou a +cadeia entre os estágios. Isso melhora auditoria, mas não transforma referências +em evidência válida de promotion. + +O bloco de classificação de referências elevou a suíte focada para 129 testes e +impediu que refs registradas fossem tratadas automaticamente como artifacts +válidos. Isso melhora reprodutibilidade e diagnóstico. + +O bloco de inventário do attempt elevou a suíte focada para 130 testes e passou a +classificar refs contra artifacts validados. Isso melhora auditabilidade, mas +não cria artifacts ausentes nem promove resultados. + +O bloco de integridade elevou a suíte focada para 131 testes e tornou explícita +a diferença entre cadeia completa, incompleta e inconsistente. Isso não é uma +decisão de acceptance ou promotion. + +O bloco de recovery plan elevou a suíte focada para 137 testes e vinculou a ação +ao attempt de origem. Isso melhora replay e auditoria, mas não executa recovery +nem demonstra eficácia de escalonamento. + +O bloco de recovery replay elevou a suíte focada para 138 testes e preservou +compatibilidade com runs sem recovery. Isso melhora auditabilidade sem afirmar +que o plano de recovery foi executado. + +O bloco de identidade de recovery manteve a suíte focada em 138 testes e +eliminou referências ambíguas de plano. Isso fortalece replay, mas não mede +eficácia de recovery real. + +O bloco de identidade de execução elevou a suíte focada para 139 testes e +fortaleceu o vínculo plan/attempt. Isso melhora replay, mas não aceita nenhum +resultado por referência. + +O bloco de outcome references manteve a suíte focada em 139 testes e deu +identidade reprodutível aos resultados de verification e acceptance. Isso não +constitui gate de promotion. + +O bloco de cadeia final manteve a suíte focada em 139 testes e validou o vínculo +entre outcomes e promotion. Isso não promove nenhum candidato nem substitui +evidência experimental. + +A validação global alcançou 291 testes e permaneceu bloqueada apenas na +importação do adapter OpenRouter por dependência `minisweagent` ausente. Isso +confirma que a suíte focada local continua útil, mas a suíte completa ainda não +é evidência de integração externa qualificada. + +O controle dependency-isolated passou com 290 testes. A diferença entre 290 +testes verdes e a suíte completa bloqueada por um import foi preservada como +estado de infraestrutura, não convertida em sucesso de integração. + +O Bloco A confirmou que a direção executor-agnostic está correta no núcleo, mas +os adapters externos ainda vazam `minisweagent`, Git Bash, Docker e SWE-bench +para a camada experimental. A matriz de readiness preserva esses blockers e +impede promoção por evidência histórica ou por testes locais. + +O Bloco B fortaleceu a qualification boundary com identidade, dependências, +capabilities e evidência determinísticas. Os 293 testes isolados passaram, mas +isso não qualifica os adapters externos concretos. + +O preflight concreto confirmou que presença de executável não equivale a +runtime utilizável: Git Bash foi classificado como `BLOCKED` por acesso negado e +Docker como `BLOCKED` por daemon inacessível. Essa distinção evita falsos +positivos de readiness. diff --git a/docs/benchmark-derived-mvp-requirements.md b/docs/benchmark-derived-mvp-requirements.md new file mode 100644 index 00000000..10ab4ef9 --- /dev/null +++ b/docs/benchmark-derived-mvp-requirements.md @@ -0,0 +1,33 @@ +# Requisitos do MVP derivados de benchmarks + +## Evidência externa usada + +- [SWE-bench Verified](https://openai.com/index/introducing-swe-bench-verified/) + fornece um repositório/revisão, um issue e verificadores separados para + testes `FAIL_TO_PASS` e `PASS_TO_PASS`. +- [Terminal-Bench](https://www.tbench.ai/news/announcement) usa tarefas com + ambiente dedicado, solução verificada por humanos e scripts de teste; a + versão atual também explicita recursos e versões do benchmark. + +Esses benchmarks medem resolução de tarefas por agentes. Não qualificam o +chassi isoladamente nem autorizam adotar um executor. + +## Tradução para o chassi + +| Observação do benchmark | Requisito do MVP | Estado | +|---|---|---| +| revisão/base precisa ser identificável | request exige `environment.revision` | implementado | +| tarefa precisa de workspace definido | governance exige `environment.workspace` | implementado | +| solução precisa de verificação declarada | `acceptance_checks` e autoridade devem permanecer explícitos | contrato existente | +| verificador precisa ser reproduzível | `TestResult` preserva `command`, `exit_code` e `duration_ms` quando observados | implementado | +| tarefa precisa de um teste executável no ambiente | `run_verification_command` executa argv declarado, sem shell, com timeout e stdout/stderr | implementado | +| resultado precisa separar correção de execução | verification/acceptance continuam estados distintos | contrato existente | +| ambiente e recursos afetam o resultado | environment, configuração e budget precisam ser persistidos | parcial | +| falhas de infraestrutura não são rejeições da tarefa | erro de ambiente permanece `BLOCKED`/explícito | contrato existente | +| comparação precisa de controles compatíveis | identidade de task, revisão, executor, ambiente e budget | contrato existente | + +## Limite da conclusão + +Os requisitos acima tornam a execução mensurável e reproduzível. Não provam que +um executor resolve tarefas. Essa prova só começa no M1, com um executor real, +um task set congelado e verificadores executados de verdade. diff --git a/docs/decisions/0001-greenfield-bootstrap.md b/docs/decisions/0001-greenfield-bootstrap.md index 5b52943e..24f6673f 100644 --- a/docs/decisions/0001-greenfield-bootstrap.md +++ b/docs/decisions/0001-greenfield-bootstrap.md @@ -18,7 +18,17 @@ Start with documentation, empty structural boundaries, a dependency-free foundat - Future changes must introduce their own evidence and decision records. - No donor repository is a dependency of the product. +## P0 follow-up + +P0 baseline telemetry is implemented, executed, and tested on a branch derived from this bootstrap: + +```text +P0_TELEMETRY = IMPLEMENTED / EXECUTED / TESTED +P0_BASELINE_TELEMETRY = EXECUTED_WITH_SCOPE +``` + +It is not marked `ACCEPTED` or `PROMOTED` by this record. + ## Revisit when The first concrete capability has a reproducible hypothesis and a need for executable product code, experiment fixtures, or a runtime dependency. - diff --git a/docs/decisions/0002-deterministic-task-characterization.md b/docs/decisions/0002-deterministic-task-characterization.md new file mode 100644 index 00000000..d364a4d0 --- /dev/null +++ b/docs/decisions/0002-deterministic-task-characterization.md @@ -0,0 +1,24 @@ +# 0002 — Deterministic task characterization + +- Status: executed with synthetic fixtures +- Date: 2026-09-22 + +## Context + +P0 made execution measurement observable. The next safe increment is a deterministic structural hypothesis before execution, without giving the system authority to route or select an executor. + +## Decision + +Use explicit `TaskSignals`, closed enums, centralized conservative thresholds, structured reason codes, and fail-closed output. Preserve the input and derived evidence in a deterministic JSON record. Keep characterization independent from P0 execution status. + +## Consequences + +- Same semantic signals serialize identically after normalization. +- Missing or contradictory signals require qualification. +- Synthetic fixtures demonstrate contract stability only; they do not establish accuracy. +- No executor, router, planner, retry policy, replan policy, or model dependency is introduced. + +## Revisit when + +Real task evidence justifies validating or revising thresholds, adding richer signals, or designing P2 progress/stagnation measurement. + diff --git a/docs/decisions/0003-progress-stagnation-detection.md b/docs/decisions/0003-progress-stagnation-detection.md new file mode 100644 index 00000000..77bd2cde --- /dev/null +++ b/docs/decisions/0003-progress-stagnation-detection.md @@ -0,0 +1,24 @@ +# 0003 — Deterministic progress / stagnation detection + +- Status: executed with synthetic fixtures +- Date: 2026-09-22 + +## Context + +P0 measures execution and P1 characterizes task structure. P2 must make progress observable before any future policy receives authority to change execution. + +## Decision + +Compare normalized, explicit snapshots. Treat new useful evidence and reduced distances as progress. Treat repeated failure signatures without new evidence, or repeated actions with unchanged acceptance distance, as no progress. Return `UNKNOWN` for missing required measurements or contradictory distances. + +## Consequences + +- Snapshot set ordering does not affect assessment serialization. +- The detector is evidence-only and cannot produce `PASS`. +- Retry, replan, routing, executor selection, and escalation remain external concerns. +- No OpenHands dependency or copied donor framework is introduced; this is a local, smaller adaptation of the generic idea that repeated unchanged failure can indicate stagnation. + +## Revisit when + +Empirical runs provide evidence for validating thresholds, adding event integration, or designing a future policy outside P2. + diff --git a/docs/decisions/0004-rule-based-routing-escalation.md b/docs/decisions/0004-rule-based-routing-escalation.md new file mode 100644 index 00000000..4a7ac2a8 --- /dev/null +++ b/docs/decisions/0004-rule-based-routing-escalation.md @@ -0,0 +1,25 @@ +# 0004 — Rule-based routing and evidence-based escalation + +- Status: executed with synthetic fixtures +- Date: 2026-09-22 + +## Context + +P1 provides a structural task hypothesis and P2 observes progress. A limited policy can now select an initial path and propose bounded escalation without selecting an executor or taking execution authority. + +## Decision + +Implement closed contracts for routing and escalation. Route directly from P1 scope, require qualification for unknown input, use a monotonic three-path ladder, require P2/evidence justification for escalation, and enforce explicit attempt/path-escalation budgets. + +## Consequences + +- Decisions are deterministic and serializable. +- `ESCALATE` is a proposal, not an executor invocation. +- The policy cannot produce `PASS`, change acceptance, retry, replan, or switch executors. +- Budget exhaustion and invalid transitions fail closed. +- No concrete executor or donor dependency is introduced. + +## Revisit when + +Empirical execution evidence justifies a separate policy review for retry/replan, executor mapping, or richer verification. Those are not P3 changes. + diff --git a/docs/decisions/0005-repository-planning-state.md b/docs/decisions/0005-repository-planning-state.md new file mode 100644 index 00000000..5dc3b0d2 --- /dev/null +++ b/docs/decisions/0005-repository-planning-state.md @@ -0,0 +1,7 @@ +# 0005 — Explicit repository planning state + +- Status: executed with synthetic fixtures +- Date: 2026-09-22 + +P4 uses explicit repository state, stable plan-step identities, dependency validation, and historical replan references. It has no execution authority and no autonomous replan loop. + diff --git a/docs/decisions/0006-patch-verification-boundary.md b/docs/decisions/0006-patch-verification-boundary.md new file mode 100644 index 00000000..63641f0b --- /dev/null +++ b/docs/decisions/0006-patch-verification-boundary.md @@ -0,0 +1,7 @@ +# 0006 — Patch verification boundary + +- Status: executed with synthetic fixtures +- Date: 2026-09-22 + +P5 verifies patch gates independently from final acceptance. Missing evidence fails closed without converting verification into `PASS`; P5 does not mutate P0 status. + diff --git a/docs/decisions/0007-handoff-context-accounting.md b/docs/decisions/0007-handoff-context-accounting.md new file mode 100644 index 00000000..1912244c --- /dev/null +++ b/docs/decisions/0007-handoff-context-accounting.md @@ -0,0 +1,7 @@ +# 0007 — Handoff context accounting + +- Status: executed with synthetic fixtures +- Date: 2026-09-22 + +P6 records explicit handoff context metrics, derives only observable executor transitions, preserves unknowns as `null`, and reports budget observations without controlling execution. + diff --git a/docs/decisions/0008-experimental-composition-harness.md b/docs/decisions/0008-experimental-composition-harness.md new file mode 100644 index 00000000..1bf2466d --- /dev/null +++ b/docs/decisions/0008-experimental-composition-harness.md @@ -0,0 +1,8 @@ +# 0008 — Experimental composition harness + +- Status: experimental contract +- Date: 2026-09-22 + +P7 introduces only declarative treatment and observation contracts. The harness keeps the five controlled dimensions explicit, canonicalizes mechanism order, rejects duplicate treatment names and semantic compositions, and distinguishes `COMPARABLE`, `INCOMPARABLE`, and `UNKNOWN`. + +P7 does not invoke P1–P6 or any executor. A treatment can record that a mechanism is enabled without granting that mechanism execution authority. The initial treatment matrix is A–F; adaptive composition G is deferred until an adaptive mechanism is qualified. diff --git a/docs/decisions/0009-executor-qualification.md b/docs/decisions/0009-executor-qualification.md new file mode 100644 index 00000000..00c18507 --- /dev/null +++ b/docs/decisions/0009-executor-qualification.md @@ -0,0 +1,8 @@ +# 0009 — Executor qualification boundary + +- Status: experimental contract +- Date: 2026-09-22 + +P8.1 introduces executor identity, controlled environment/task/budget contracts, external observations, explicit comparison axes, and deterministic summaries. It contains no concrete adapter, SDK, subprocess, network, or LLM integration. + +Declared capability is metadata, not qualification evidence. P8 separates execution outcome from external acceptance and preserves unknown measurements as unknown. It does not choose a winner or promote an executor; those decisions remain future P9 work. diff --git a/docs/decisions/0010-p82-minimal-mechanism-trial.md b/docs/decisions/0010-p82-minimal-mechanism-trial.md new file mode 100644 index 00000000..08ec48ba --- /dev/null +++ b/docs/decisions/0010-p82-minimal-mechanism-trial.md @@ -0,0 +1,14 @@ +# 0010 — P8.2 minimal mechanism trial boundary + +- Status: experimental contract +- Date: 2026-09-22 + +P8.2 adds only a declarative A/B/C/D trial plan over the P8.1 qualification +contracts. The executor and all comparability controls are fixed in one config; +only treatment varies across generated trial plans. The harness does not run +LLMs, switch executors, apply mechanisms, infer acceptance, or promote a +mechanism. + +Safe editor and enhanced repository context are donors under trial, not Arkx +dependencies. The P8.2 plan preserves missing values as unknown and leaves +independent acceptance authority outside the harness. diff --git a/docs/decisions/0011-p82a-baseline-execution-path.md b/docs/decisions/0011-p82a-baseline-execution-path.md new file mode 100644 index 00000000..2d9c5149 --- /dev/null +++ b/docs/decisions/0011-p82a-baseline-execution-path.md @@ -0,0 +1,34 @@ +# 0011 — P8.2a baseline execution path + +- Status: experimental infrastructure +- Date: 2026-09-23 + +P8.2a adds a narrow headless adapter around mini-SWE-agent 2.x's public +Python API. A worker process owns the Python environment containing +mini-SWE-agent; Arkx captures logs, trajectory, diff, timestamps, exit state, +and usage metadata. The adapter never invokes verification or acceptance. + +Acceptance is represented by an explicit authority contract and keeps +`ACCEPTED`, `REJECTED`, `INDETERMINATE`, `NOT_EXECUTED`, and `BLOCKED` +distinct. Runs are append-only by task and attempt, so reruns cannot silently +replace prior artifacts. + +This is infrastructure qualification only. No Wave 0 task sample or real +model run is recorded until the public task artifacts and acceptance authority +are available. The frozen Wave 0 manifest uses the official SWE-bench Verified +test split at revision `78f471bf655a3137b2e8a75af1501690ec009ec3`, and records +`UNKNOWN` strata where no prior auditable Arkx classification exists. B/C/D +remain out of scope. + +The official authority adapter serializes only `instance_id`, +`model_name_or_path`, and `model_patch` for inference. It invokes +`swebench.harness.run_evaluation` with a unique run id and maps infrastructure +errors to `INDETERMINATE`/`BLOCKED`, never to task rejection. Gold-patch +qualification is a separate gate. The official package is now available in +the isolated `.venv-swebench` environment (`swebench 5.0.2`, upstream commit +`02e7a74ffd0b707aab73d203fe87bdc7c76afc8e`) and its CLI help passes. The +Modal qualification reached the official harness but is blocked by missing +`~/.modal.toml` credentials. The local Docker Linux engine is also +unavailable. The deterministic headless probe now passes with an explicit +`DeterministicModelConfig.outputs` fixture recorded in +`experiments/p82-deterministic-headless-probe.json`. diff --git a/docs/decisions/0012-evidence-routed-treatment-architecture.md b/docs/decisions/0012-evidence-routed-treatment-architecture.md new file mode 100644 index 00000000..8776dd20 --- /dev/null +++ b/docs/decisions/0012-evidence-routed-treatment-architecture.md @@ -0,0 +1,809 @@ +# Arkx — Arquitetura Canônica Orientada por Tratamentos e Evidência + +## Status + +**Proposta canônica para as fases P8.2 → P8.4** + +Esta decisão consolida a arquitetura que emergiu da evidência retrospectiva e do desenho experimental atual do Arkx. + +Ela substitui uma interpretação simplificada em que o sistema seria apenas um router entre múltiplos executores. + +O princípio arquitetural permanece: + +> **minimum sufficient architecture for maximum reliable capability** + +Os invariantes continuam válidos: + +* `NO_COMPONENT_HAS_TENURE` +* `SCIENTIFIC_SIGNAL != LOCAL_PASS` +* `HYPOTHESIS != IMPLEMENTATION` +* `IMPLEMENTATION != EXECUTED` +* `EXECUTED != ACCEPTED` +* `ACCEPTED != PROMOTED` +* `DONOR != DEPENDENCY` +* `MECHANISM_PASS != EXECUTOR_ADOPTED` +* `BLOCKED != PASS` +* `NOT_EXECUTED != PASS` +* `NO_SILENT_FALLBACK` +* `NO_SILENT_EXECUTOR_SWITCH` +* `NO_SILENT_SCOPE_EXPANSION` + +E as regras operacionais permanecem: + +* `ROUTE BEFORE CASCADING` +* `DECOMPOSE ONLY WHEN JUSTIFIED` +* `ESCALATE ON EVIDENCE` +* `VERIFY BEFORE ACCEPTING` + +--- + +## Decisão + +A arquitetura do Arkx deve rotear primariamente para **regimes de tratamento**, e não diretamente para executores. + +O executor é uma implementação de capacidade dentro de um tratamento. + +A arquitetura canônica é: + +```text + REQUEST + │ + ▼ + GOVERNANCE / AUTHORITY + │ + ▼ + TASK CHARACTERIZATION + │ + ▼ + ROUTE + │ + ┌─────────────┼─────────────┐ + │ │ │ + ▼ ▼ ▼ + SIMPLE LOCALIZED REPOSITORY- + PATH TREATMENT WIDE PATH + │ │ │ + │ localizer / planning + + │ verifier state + │ │ │ + └─────────────┴──────┬──────┘ + │ + ▼ + EXECUTOR + │ + ▼ + PROGRESS? + / \ + yes no + │ │ + │ recover / + │ replan / + │ escalate + │ │ + └──────┬──────┘ + │ + ▼ + VERIFY + │ + ▼ + INDEPENDENT ACCEPTANCE +``` + +Essa estrutura deve ser considerada a referência arquitetural para as próximas fases do projeto. + +--- + +## 1. Request + +`REQUEST` representa a intenção externa submetida ao Arkx. + +Ela ainda não constitui autorização para executar qualquer transformação. + +A existência de uma request não implica: + +```text +REQUEST != AUTHORITY +``` + +--- + +## 2. Governance / Authority + +Governance ocorre antes de characterization, routing e execução. + +Sua responsabilidade é definir os limites sob os quais a request pode ser processada. + +Pode incluir: + +* escopo autorizado; +* permissões; +* budgets; +* limites de tempo; +* restrições de ambiente; +* políticas de modificação; +* critérios de verification; +* definição da acceptance authority; +* invariantes aplicáveis. + +O executor não deve definir retrospectivamente o que constitui sucesso. + +```text +EXECUTOR OUTPUT != ACCEPTANCE AUTHORITY +``` + +--- + +## 3. Task Characterization + +`TASK CHARACTERIZATION` descreve a tarefa antes da escolha do regime operacional. + +Seu propósito não é simplesmente selecionar um executor. + +Ela deve produzir evidência útil para determinar **qual tratamento é necessário**. + +Características possíveis incluem: + +* escopo provável da alteração; +* concentração/localização do problema; +* necessidade de exploração; +* necessidade de contexto de repositório; +* dependências entre arquivos/componentes; +* necessidade de planejamento; +* dificuldade de verification; +* histórico relevante; +* sinais conhecidos de falha ou sucesso de mecanismos. + +A characterization deve ser separada da decisão de routing. + +--- + +## 4. Route + +O primeiro problema de routing é: + +```text +WHAT TREATMENT DOES THIS TASK REQUIRE? +``` + +e não: + +```text +WHICH AGENT SHOULD RUN? +``` + +O router deve selecionar o regime mínimo plausivelmente suficiente. + +Isso preserva o princípio: + +```text +minimum sufficient architecture +``` + +e evita pagar por coordenação, planejamento, contexto ou chassis adicionais em tarefas que não os exigem. + +--- + +# Regimes de tratamento + +## 5. Simple Path + +O `SIMPLE PATH` é o caminho mínimo. + +Destina-se a tarefas nas quais não exista evidência suficiente para justificar mecanismos adicionais. + +Exemplo conceitual: + +```text +TASK + ↓ +minimal context + ↓ +executor + ↓ +verify + ↓ +accept +``` + +O simple path deve minimizar: + +* coordenação; +* contexto adicional; +* planejamento explícito; +* múltiplos agentes; +* ferramentas especializadas; +* overhead de execução. + +O executor baseline esperado atualmente é o `mini-SWE-agent`, sujeito à qualificação experimental. + +Baseline não implica tenure. + +--- + +## 6. Localized Treatment + +O `LOCALIZED TREATMENT` é usado quando existe evidência de que a principal dificuldade está em localizar, editar ou verificar uma região relativamente restrita do problema. + +Pode incorporar mecanismos como: + +* melhor localization; +* contextual retrieval dirigido; +* structural/repository search; +* safe editing; +* verification localizada; +* dependency context estritamente necessário. + +Conceitualmente: + +```text +TASK + ↓ +localize + ↓ +targeted context + ↓ +executor + ↓ +targeted verification +``` + +O objetivo não é introduzir um novo chassis quando um mecanismo pequeno consegue comprar a capacidade necessária. + +Essa é precisamente uma das perguntas de P8.2: + +```text +CAN A SMALL MECHANISM RECOVER CAPABILITY +WITHOUT IMPORTING A LARGER EXECUTOR? +``` + +--- + +## 7. Repository-Wide Path + +O `REPOSITORY-WIDE PATH` é reservado para tarefas em que a evidência indique necessidade de capacidade transversal. + +Pode incluir: + +* planejamento explícito; +* manutenção de estado; +* dependency reasoning; +* architecture context; +* exploração de múltiplas regiões; +* decomposição; +* coordenação de subproblemas; +* recovery/replanning mais sofisticado. + +Esse caminho pode justificar um executor/chassis especializado caso os experimentos demonstrem capacidade confiável que o baseline enriquecido não consegue reproduzir de maneira mais simples. + +Portanto: + +```text +REPOSITORY_WIDE != AUTOMATIC_MULTI_AGENT +``` + +e: + +```text +COMPLEX TASK != AUTOMATIC_SPECIALIZED_EXECUTOR +``` + +É necessário demonstrar capability gap. + +--- + +# Executor como implementação, não como eixo primário + +## 8. Executor Placement + +O executor ocorre **depois da seleção do tratamento**. + +Isso é uma decisão arquitetural importante. + +O Arkx não deve ser modelado primariamente como: + +```text +ROUTER + ├── mini + ├── Moatless + ├── OpenHands + └── Lingxi +``` + +A representação preferida é: + +```text +ROUTER + │ + ├── SIMPLE PATH + │ └── executor apropriado + │ + ├── LOCALIZED TREATMENT + │ ├── mechanisms + │ └── executor apropriado + │ + └── REPOSITORY-WIDE PATH + ├── planning/state + └── executor apropriado +``` + +Executores são candidatos a implementar capacidades dentro desses regimes. + +Eles não constituem, por si só, a taxonomia arquitetural. + +--- + +# Progress, recovery e escalation + +## 9. Progress / Stagnation + +Depois da execução começar, o Arkx deve observar progresso. + +A questão não é apenas se o executor terminou. + +Devem existir sinais capazes de distinguir, quando possível: + +* progresso; +* exploração repetitiva; +* ausência de avanço; +* regressão; +* edit failure; +* verification failure; +* necessidade de replan; +* necessidade de escalation. + +Conceitualmente: + +```text +EXECUTOR + │ + ▼ +PROGRESS? + / \ +yes no + │ │ + │ recover + │ replan + │ escalate + │ │ + └────────┘ +``` + +Escalation não deve equivaler a cascade automática. + +```text +EXECUTOR_A FAILED + != +RUN EXECUTOR_B AUTOMATICALLY +``` + +O segundo executor somente deve ser introduzido quando exista evidência de que outro tratamento ou capability é justificável. + +Isso preserva: + +```text +ROUTE BEFORE CASCADING +ESCALATE ON EVIDENCE +``` + +--- + +# Verification e acceptance + +## 10. Verification + +Verification avalia propriedades técnicas observáveis da solução. + +Pode incluir: + +* patch validity; +* testes; +* regressões; +* estrutura do diff; +* artifacts; +* reprodução do problema; +* checks específicos. + +Verification não possui autoridade automática para promoção ou acceptance. + +```text +VERIFICATION_PASS != ACCEPTED +``` + +--- + +## 11. Independent Acceptance + +Acceptance deve permanecer uma autoridade separada do executor. + +O executor não pode declarar sua própria solução aceita. + +A cadeia conceitual permanece: + +```text +IMPLEMENTED + ↓ +EXECUTED + ↓ +VERIFIED + ↓ +ACCEPTED + ↓ +potential evidence for future promotion +``` + +Nenhuma transição é implícita. + +Acceptance pode produzir estados como: + +```text +ACCEPTED +REJECTED +INDETERMINATE +BLOCKED +NOT_EXECUTED +``` + +Estados desconhecidos ou não executados não devem ser reduzidos a booleanos. + +--- + +# Relação com P8.2 + +## 12. O significado de A/B/C/D + +Os tratamentos experimentais de P8.2 não são quatro arquiteturas candidatas. + +Eles são intervenções destinadas a isolar efeitos de mecanismos. + +Atualmente: + +```text +A = mini baseline + +B = mini + safe editor + +C = mini + enhanced repository context + +D = mini + safe editor + enhanced repository context +``` + +O objetivo é separar: + +```text +MODEL_EFFECT +SCAFFOLD_EFFECT +MECHANISM_EFFECT +MODEL × SCAFFOLD INTERACTION +``` + +e, especificamente em P8.2, produzir evidência sobre `MECHANISM_EFFECT`. + +Os resultados devem ajudar a responder quais mecanismos pertencem ao: + +```text +SIMPLE PATH +``` + +quais justificam: + +```text +LOCALIZED TREATMENT +``` + +e quais capacidades permanecem ausentes e podem exigir: + +```text +REPOSITORY-WIDE PATH +``` + +--- + +# Relação com P8.3 + +## 13. Targeted Executor Qualification + +P8.3 não deve ser tratado como um torneio geral de agentes. + +Seu propósito é investigar capabilities que continuem não satisfeitas após P8.2. + +Os candidatos registrados atualmente incluem: + +* mini enriquecido; +* Moatless; +* OpenHands; +* Lingxi; +* SWE-agent como controle quando informativo; +* candidatos adicionais evidence-qualified. + +A pergunta para cada challenger deve ser: + +```text +WHAT UNIQUE RELIABLE CAPABILITY DOES THIS EXECUTOR BUY? +``` + +e não: + +```text +WHICH EXECUTOR HAS THE HIGHEST GLOBAL SCORE? +``` + +Um chassis adicional somente se justifica se adquirir capacidade útil que não possa ser absorvida mais economicamente como mecanismo. + +--- + +# Relação com P8.4 + +## 14. Routed Composition + +P8.4 deverá validar a composição seletiva. + +O objetivo não é: + +```text +mini + ↓ fail +SWE-agent + ↓ fail +OpenHands + ↓ fail +... +``` + +Esse desenho violaria `ROUTE BEFORE CASCADING`. + +A composição desejada é aproximadamente: + +```text +REQUEST + ↓ +GOVERNANCE + ↓ +CHARACTERIZE + ↓ +ROUTE + │ + ├── SIMPLE + │ + ├── LOCALIZED + │ + └── REPOSITORY-WIDE + │ + ↓ +SELECT SUFFICIENT EXECUTOR/CAPABILITY + ↓ +EXECUTE + ↓ +PROGRESS / STAGNATION + ↓ +VERIFY + ↓ +INDEPENDENT ACCEPTANCE +``` + +Um segundo executor só deve aparecer mediante evidência. + +Essa direção já estava estabelecida no protocolo de composição seletiva do projeto. + +--- + +# Interpretação da evidência existente + +## 15. Complementaridade não implica arquitetura multiagente + +A evidência retrospectiva mostrou discordância relevante entre executores. + +No conjunto comparável registrado: + +* 262 tasks foram resolvidas por todos; +* 93 falharam para todos; +* 145 apresentaram discordância; +* a união oracle atingiu 407/500, ou 81,4%. + +Isso demonstra que `EXECUTOR CHOICE CONTAINS SIGNAL`, mas não demonstra que oracle routing seja realizável. + +Portanto, a consequência arquitetural não é: + +```text +RUN MORE AGENTS +``` + +A consequência é: + +```text +IDENTIFY THE SIGNAL + ↓ +CHARACTERIZE THE TASK + ↓ +ROUTE TO THE MINIMUM SUFFICIENT CAPABILITY +``` + +--- + +# Consequências arquiteturais + +## 16. Mecanismos podem substituir chassis + +Se P8.2 demonstrar que mecanismos pequenos recuperam capacidade que parecia exclusiva de chassis maiores: + +```text +ABSORB THE MECHANISM +``` + +em vez de: + +```text +ADOPT THE CHASSIS +``` + +Isso reduz: + +* dependências; +* manutenção; +* complexidade de routing; +* custo operacional; +* superfície de falha; +* dificuldade de observabilidade. + +--- + +## 17. Specialized executors permanecem permitidos + +A arquitetura não proíbe múltiplos executores. + +Ela exige evidência para cada um. + +Um executor especializado permanece justificável quando: + +```text +SPECIALIZED_EXECUTOR + BUYS +UNIQUE + RELIABLE + USEFUL CAPABILITY +``` + +que não seja economicamente reproduzida pelo baseline enriquecido. + +--- + +## 18. Arquitetura pode encolher com a evidência + +O objetivo de P8 não é necessariamente adicionar componentes. + +Um resultado válido de P8 pode ser remover candidatos. + +A arquitetura final pode acabar contendo: + +```text +1 baseline executor ++ +small proven mechanisms ++ +1 specialized executor +``` + +ou até menos, caso isso seja suficiente. + +Também pode justificar mais componentes se os experimentos demonstrarem necessidade. + +Não existe quantidade pré-determinada de agentes ou executores. + +--- + +# Progressão arquitetural + +A progressão correta das fases é: + +```text +P8.1 +QUALIFICATION CONTRACTS + │ + ▼ +P8.2 +ISOLATE MECHANISM EFFECT + │ + ▼ +DISCOVER REQUIRED PATH CAPABILITIES + │ + ▼ +P8.3 +TARGETED EXECUTOR QUALIFICATION + │ + ▼ +P8.4 +SELECTIVE ROUTED COMPOSITION + │ + ▼ +MINIMUM SUFFICIENT ARKX ARCHITECTURE +``` + +P8.2 deve acontecer antes de importar novos chassis justamente para determinar se mecanismos comprovados podem ser absorvidos de forma mais barata. Essa é a direção científica registrada do projeto: + +```text +ABSORB PROVEN MECHANISMS WHEN CHEAPER + +KEEP SPECIALIZED EXECUTORS ONLY WHEN THEY BUY +UNIQUE RELIABLE CAPABILITY + +DO NOT LIMIT THE SEARCH TO WESTERN MODELS +OR THE THREE OBVIOUS EXECUTORS +``` + +--- + +# Arquitetura canônica resumida + +```text +REQUEST + │ + ▼ +GOVERNANCE / AUTHORITY + │ + ▼ +TASK CHARACTERIZATION + │ + ▼ +ROUTE BY REQUIRED CAPABILITY + │ + ├──────────────┬────────────────────┐ + │ │ │ + ▼ ▼ ▼ +SIMPLE LOCALIZED REPOSITORY-WIDE +PATH TREATMENT PATH + │ │ │ + │ mechanisms planning / state / + │ as justified deeper capability + │ │ │ + └──────────────┴─────────┬──────────┘ + │ + ▼ + SUFFICIENT EXECUTOR + │ + ▼ + PROGRESS / STAGNATION + │ + ┌───────────┴───────────┐ + │ │ + progress insufficient + │ │ + │ recover / replan / + │ escalate + │ │ + └───────────┬───────────┘ + │ + ▼ + VERIFY + │ + ▼ + INDEPENDENT ACCEPTANCE + │ + ▼ + EVIDENCE / DONE +``` + +## Regra final + +O Arkx não deve perguntar primeiro: + +```text +WHICH AGENT? +``` + +Deve perguntar: + +```text +WHAT CAPABILITY DOES THIS TASK REQUIRE? +``` + +Depois: + +```text +WHAT IS THE MINIMUM TREATMENT THAT PROVIDES IT? +``` + +E somente então: + +```text +WHICH EXECUTOR CAN DELIVER THAT TREATMENT +RELIABLY AND ECONOMICALLY? +``` + +Essa é a interpretação arquitetural que melhor preserva, no estado atual do projeto, a evidência acumulada e o princípio de **minimum sufficient architecture for maximum reliable capability**. diff --git a/docs/decisions/0013-core-outcome-contract-boundary.md b/docs/decisions/0013-core-outcome-contract-boundary.md new file mode 100644 index 00000000..ae332b6c --- /dev/null +++ b/docs/decisions/0013-core-outcome-contract-boundary.md @@ -0,0 +1,39 @@ +# 0013 — Core verification and acceptance outcome boundary + +## Status + +Accepted for implementation as a compatibility-preserving refactoring. + +## Problem + +`acceptance`, `promotion`, and `orchestration` currently import +`VerificationResult`, `AcceptanceResult`, and their state enums from +`p82_baseline`. This makes executor-neutral product boundaries depend on the +Treatment A experimental runner. + +## Decision + +Move the shared verification and acceptance outcome types to an executor-neutral +core module. The baseline module may re-export them temporarily for backwards +compatibility, but core modules must no longer import from `p82_baseline`. + +## Falsifiable hypothesis + +If shared outcome types are moved to a neutral module without changing their +serialized shape or semantics, all existing deterministic contract tests will +continue to pass and core modules will no longer have a baseline-runner import +dependency. + +## Acceptance criteria + +1. Existing outcome constructors and `to_dict()` output remain compatible. +2. `acceptance`, `promotion`, and `orchestration` import only the neutral core + outcome module. +3. Existing tests pass with the repository source path configured. +4. The baseline runner remains compatible through explicit re-exports. +5. No provider, model, executor, or sandbox behavior changes. + +## Removal condition + +Remove the compatibility re-exports only after downstream imports have been +migrated and a public API compatibility review has accepted the change. diff --git a/docs/decisions/0014-reproducible-harness-identity-and-failure-contract.md b/docs/decisions/0014-reproducible-harness-identity-and-failure-contract.md new file mode 100644 index 00000000..9e3d6dbc --- /dev/null +++ b/docs/decisions/0014-reproducible-harness-identity-and-failure-contract.md @@ -0,0 +1,56 @@ +# Decision 0014 — Reproducible harness identity and failure contract + +## Status + +Accepted as a compatibility-preserving contract slice. + +## Evidence + +Current P8.2 artifacts contain multiple report shapes and failure vocabularies. +Some attempts preserve `model_usage` or observed retries as `null`, and the +same task-level run identity appears in different rerun artifact roots. The +documents require immutable attempts and complete identity, but no common +executor-neutral manifest enforced those requirements. + +## Decision + +Introduce a small executor-neutral contract with four identity levels: + +```text +experiment_id -> trial_id -> attempt_id -> artifact references +``` + +`RunManifest` records the frozen identity and terminal state of one attempt. +`ErrorEnvelope` records error domain, stable code, retryability and raw +evidence references. The contract does not execute, retry, verify, accept or +promote anything. + +`attempt_id` is deterministic for `(trial_id, attempt_number, +configuration_digest)` and does not use timestamps or fallback selection. + +The P8.2 runner now emits the common error envelope in both `manifest.json` +and the compatibility `execution.json`. The legacy failure category remains +available during migration, but provider, sandbox, harness and executor +domains are mapped explicitly instead of being inferred by consumers. + +## Falsifiable hypothesis + +If harness artifacts use separate trial/attempt identities and a common error +envelope, then equivalent frozen inputs will serialize identically, distinct +attempts will not collide, and infrastructure/provider failures can be +classified without being represented as task acceptance outcomes. + +## Acceptance criteria + +1. deterministic contract tests pass; +2. missing identity and invalid attempt numbers are rejected; +3. raw evidence references and retryability survive serialization; +4. existing P0–P8 contracts remain compatible; +5. the P8.2 baseline runner may emit the common manifest while retaining the + legacy `execution.json` for compatibility; no executor is promoted by this + slice. + +## Non-goals + +This decision does not introduce a persistence backend, event bus, replay +engine, executor implementation, provider selection, or sandbox abstraction. diff --git a/docs/decisions/0015-promotion-requires-independent-verification.md b/docs/decisions/0015-promotion-requires-independent-verification.md new file mode 100644 index 00000000..a11513ae --- /dev/null +++ b/docs/decisions/0015-promotion-requires-independent-verification.md @@ -0,0 +1,34 @@ +# Decision 0015 — Promotion requires independent verification + +## Status + +Accepted. + +## Evidence + +The previous promotion contract required an accepted `AcceptanceResult` and +promotion evidence references, but did not require a separate verification +record. That allowed a caller to present acceptance without proving that an +independent verifier had evaluated the produced artifact. + +## Decision + +The default promotion policy requires independent `VerificationEvidence` with +state `PASS` and at least one raw evidence reference. Verification remains +separate from executor completion and acceptance authority. + +## Falsifiable hypothesis + +If promotion requires explicit verification evidence, then executor completion +or acceptance alone cannot produce `PROMOTED`. + +## Acceptance criteria + +- missing verification is `BLOCKED`; +- verification without evidence is `BLOCKED`; +- verification `INDETERMINATE` is never promoted; +- accepted candidates with valid verification and evidence preserve existing + promotion behavior; +- snapshot promotion rejects non-completed executions and configuration + identity mismatches; +- focused promotion and harness tests pass. diff --git a/docs/decisions/0016-consolidated-experiment-record.md b/docs/decisions/0016-consolidated-experiment-record.md new file mode 100644 index 00000000..d0bbea33 --- /dev/null +++ b/docs/decisions/0016-consolidated-experiment-record.md @@ -0,0 +1,30 @@ +# Decision 0016 — Consolidated experiment record + +## Status + +Accepted as the first complete harness audit block. + +## Decision + +Persist one immutable `ExperimentRecord` that links protocol version, frozen +input identity, trials, attempts, verification references, acceptance +references and promotion decision reference. Loading the record must validate +that referenced attempts are present and independently validated. + +The record is an index and audit boundary; it does not aggregate metrics, +select executors, infer acceptance or promote a component. + +## Falsifiable hypothesis + +If experiment records are loaded only after cross-validating their attempts and +evidence references, then an experiment with missing attempts or orphaned +promotion evidence cannot appear complete by metadata alone. + +## Acceptance criteria + +- record persistence is atomic and immutable; +- all recorded attempts are found and valid; +- verification and acceptance references are preserved; +- promotion references are bound to a recorded attempt; +- malformed or incomplete records are rejected; +- focused contract and promotion tests pass. diff --git a/docs/decisions/0017-executor-provider-sandbox-contracts.md b/docs/decisions/0017-executor-provider-sandbox-contracts.md new file mode 100644 index 00000000..955a824f --- /dev/null +++ b/docs/decisions/0017-executor-provider-sandbox-contracts.md @@ -0,0 +1,29 @@ +# Decision 0017 — Executor, provider and sandbox contracts + +## Status + +Accepted as the second architecture strengthening block. + +## Decision + +Introduce small protocols for `Executor`, `Provider` and `Sandbox`, plus +serializable request/result boundaries. The executor returns execution state +and artifacts only; verification, acceptance, promotion and routing remain +outside the protocol. + +Provider failures use the common error envelope. Missing provider/model +identity remains explicit and does not select a fallback. + +## Falsifiable hypothesis + +If the core flow depends only on these protocols, deterministic fakes can +exercise completed, failed and blocked paths without importing mini-SWE-agent, +OpenRouter, Docker or SWE-bench. + +## Acceptance criteria + +- fake executor, provider and sandbox tests pass; +- failed/blocked execution requires a structured error; +- provider and sandbox can be replaced independently; +- missing provider/model identity remains `None`; +- no real executor/provider is selected or promoted by this block. diff --git a/docs/decisions/0018-contract-execution-lifecycle.md b/docs/decisions/0018-contract-execution-lifecycle.md new file mode 100644 index 00000000..ef705aaf --- /dev/null +++ b/docs/decisions/0018-contract-execution-lifecycle.md @@ -0,0 +1,25 @@ +# Decision 0018 — Contract execution lifecycle + +## Status + +Accepted as the third architecture strengthening block. + +## Decision + +Represent one contract execution as a `ContractRun` containing the request, +execution result, lifecycle history and optional verification/acceptance +evidence. The executor cannot create verification or acceptance implicitly. + +## Falsifiable hypothesis + +If a fake executor can traverse completed and blocked flows while verification +and acceptance remain optional independent attachments, then orchestration can +be tested without a concrete model or provider. + +## Acceptance criteria + +- completed fake execution preserves lifecycle history; +- completed execution alone is not accepted; +- independent verification plus acceptance can be attached explicitly; +- blocked execution requires structured error and is not accepted; +- no real executor/provider is invoked or promoted. diff --git a/docs/decisions/0019-p82-baseline-executor-adapter.md b/docs/decisions/0019-p82-baseline-executor-adapter.md new file mode 100644 index 00000000..ce02baff --- /dev/null +++ b/docs/decisions/0019-p82-baseline-executor-adapter.md @@ -0,0 +1,29 @@ +# Decision 0019 — P8.2 baseline executor adapter + +## Status + +Accepted as the fourth architecture strengthening block. + +## Decision + +Expose the existing P8.2 baseline runner through a thin +`P82BaselineExecutorAdapter`. The adapter translates the legacy +`ExecutionArtifact` into the neutral `ExecutionResult` contract and rejects a +request for a task different from the bound task. + +The legacy runner remains available for compatibility. This adapter does not +qualify mini-SWE-agent, OpenRouter, Docker or SWE-bench. + +## Falsifiable hypothesis + +If the adapter translates one fixed baseline path without changing its input +identity or terminal state, then the neutral execution contract can be tested +against the real integration boundary before selecting another executor. + +## Acceptance criteria + +- completed baseline maps to neutral `COMPLETED`; +- blocked and failed artifacts preserve their error envelope; +- wrong task identity is rejected; +- existing runner tests remain passing; +- no real provider or executor qualification is claimed. diff --git a/docs/decisions/0020-contract-executor-orchestration-bridge.md b/docs/decisions/0020-contract-executor-orchestration-bridge.md new file mode 100644 index 00000000..2ff65940 --- /dev/null +++ b/docs/decisions/0020-contract-executor-orchestration-bridge.md @@ -0,0 +1,26 @@ +# Decision 0020 — Contract executor bridge for orchestration + +## Status + +Accepted as the fifth architecture strengthening block. + +## Decision + +Add a thin `ContractExecutorRunner` bridge so the existing orchestration gate +can consume the neutral `Executor` protocol while retaining its legacy result +shape during migration. The bridge maps execution state and error only; it +does not run verification, acceptance, promotion or fallback. + +## Falsifiable hypothesis + +If orchestration can use a neutral executor through one explicit bridge, then +the existing governance, routing, progress, verification and acceptance gates +remain behaviorally stable while the concrete runner becomes replaceable. + +## Acceptance criteria + +- fake neutral executor reaches existing acceptance flow; +- blocked neutral execution remains blocked; +- task, treatment, sandbox and configuration identity are forwarded; +- no concrete provider or executor is imported by orchestration; +- existing orchestration tests remain passing. diff --git a/docs/decisions/0021-direct-neutral-executor-orchestration.md b/docs/decisions/0021-direct-neutral-executor-orchestration.md new file mode 100644 index 00000000..7990e2e1 --- /dev/null +++ b/docs/decisions/0021-direct-neutral-executor-orchestration.md @@ -0,0 +1,25 @@ +# Decision 0021 — Direct neutral executor orchestration + +## Status + +Accepted as the sixth architecture strengthening block. + +## Decision + +The routed orchestration entrypoint accepts the neutral `Executor` protocol +directly. The legacy `ExecutorRunner` result shape remains a compatibility +boundary, but new callers do not need to know it. + +## Falsifiable hypothesis + +If direct neutral-executor orchestration produces the same result as the +explicit compatibility bridge, then the orchestration gates are independent of +the legacy runner interface. + +## Acceptance criteria + +- direct neutral executor reaches existing acceptance flow; +- direct and bridged paths are behaviorally equivalent; +- blocked and failed states remain fail-closed; +- no fallback or executor switch is introduced; +- existing orchestration tests remain passing. diff --git a/docs/decisions/0022-legacy-execution-result-boundary.md b/docs/decisions/0022-legacy-execution-result-boundary.md new file mode 100644 index 00000000..ceacf32c --- /dev/null +++ b/docs/decisions/0022-legacy-execution-result-boundary.md @@ -0,0 +1,25 @@ +# Decision 0022 — Legacy execution result boundary + +## Status + +Accepted as the seventh architecture strengthening block. + +## Decision + +Centralize conversion from the neutral `arkx.execution.ExecutionResult` to the +legacy orchestration result shape in `legacy_execution_from_contract`. The +legacy shape remains a compatibility boundary and is not a second execution +contract. + +## Falsifiable hypothesis + +If all neutral-to-legacy conversion is deterministic and centralized, then +orchestration compatibility can be maintained while new executors depend only +on the neutral contract. + +## Acceptance criteria + +- conversion preserves terminal state, artifact references and error message; +- direct and bridged orchestration remain equivalent; +- no second provider/executor selection logic is introduced; +- legacy result remains confined to the compatibility boundary. diff --git a/docs/decisions/0023-public-execution-result-boundary.md b/docs/decisions/0023-public-execution-result-boundary.md new file mode 100644 index 00000000..76e62011 --- /dev/null +++ b/docs/decisions/0023-public-execution-result-boundary.md @@ -0,0 +1,25 @@ +# Decision 0023 — Public execution result boundary + +## Status + +Accepted as the eighth architecture strengthening block. + +## Evidence + +The package initializer imported both the neutral execution contract and the +legacy orchestration result under the same public name. Import order caused +the legacy type to overwrite `arkx.ExecutionResult`, making the public API +executor-dependent despite the new neutral contract. + +## Decision + +`arkx.ExecutionResult` is the neutral contract. The legacy orchestration shape +is exposed as `arkx.OrchestrationExecutionResult` during migration. Internal +legacy imports remain valid in their module boundary. + +## Acceptance criteria + +- public `arkx.ExecutionResult` is the neutral type; +- public legacy result has an explicit distinct name; +- no import-order collision remains; +- existing focused tests pass. diff --git a/docs/decisions/0024-legacy-consumer-boundary-test.md b/docs/decisions/0024-legacy-consumer-boundary-test.md new file mode 100644 index 00000000..1b566330 --- /dev/null +++ b/docs/decisions/0024-legacy-consumer-boundary-test.md @@ -0,0 +1,23 @@ +# Decision 0024 — Legacy consumer boundary test + +## Status + +Accepted as the ninth architecture strengthening block. + +## Decision + +The legacy orchestration result is confined to `orchestration.py` and the +explicit package compatibility alias. A static architecture test prevents new +core modules from importing that result directly. + +## Falsifiable hypothesis + +If legacy consumers are checked continuously, future neutral modules cannot +silently reintroduce the executor-dependent result boundary. + +## Acceptance criteria + +- no forbidden legacy imports exist in core modules; +- public neutral and legacy result names remain distinct; +- the boundary test runs with the contract suite; +- existing orchestration compatibility remains intact. diff --git a/docs/decisions/0025-execution-plan-boundary.md b/docs/decisions/0025-execution-plan-boundary.md new file mode 100644 index 00000000..21b6d2ce --- /dev/null +++ b/docs/decisions/0025-execution-plan-boundary.md @@ -0,0 +1,25 @@ +# Decision 0025 — Explicit execution plan boundary + +## Status + +Accepted as the tenth architecture strengthening block. + +## Decision + +Introduce `ExecutionPlan` as the frozen intent for one execution path. It +records task, treatment, executor/provider/model identities, sandbox, budget, +configuration and ordered steps. It can produce an `ExecutionRequest`, but it +does not contain execution result, verification, acceptance or promotion. + +## Falsifiable hypothesis + +If plan and result are separate contracts, routing and orchestration can be +tested for deterministic planning without invoking an executor. + +## Acceptance criteria + +- plan serialization is deterministic; +- plan identity fields are validated; +- plan-to-request conversion preserves declared identities; +- result state is not present in the plan; +- no executor/provider is invoked by plan construction. diff --git a/docs/decisions/0026-orchestration-execution-plan.md b/docs/decisions/0026-orchestration-execution-plan.md new file mode 100644 index 00000000..cf7532cc --- /dev/null +++ b/docs/decisions/0026-orchestration-execution-plan.md @@ -0,0 +1,25 @@ +# Decision 0026 — Orchestration produces an execution plan + +## Status + +Accepted as the eleventh architecture strengthening block. + +## Decision + +After governance, characterization, routing and selection gates pass, +orchestration produces an immutable `ExecutionPlan` before invoking the +executor. The plan is included in the orchestration result and does not +contain execution, verification, acceptance or promotion outcomes. + +## Falsifiable hypothesis + +If equivalent authorized inputs produce the same plan before execution, then +execution intent can be audited and compared independently of executor output. + +## Acceptance criteria + +- plan is produced only after required gates pass; +- equivalent inputs produce deterministic plan identity; +- terminal orchestration results retain the plan; +- executor output cannot mutate the plan; +- existing orchestration behavior remains passing. diff --git a/docs/decisions/0027-capability-profile-registry.md b/docs/decisions/0027-capability-profile-registry.md new file mode 100644 index 00000000..b61d39e5 --- /dev/null +++ b/docs/decisions/0027-capability-profile-registry.md @@ -0,0 +1,33 @@ +# ADR 0027 — Evidence-bearing capability profile registry + +## Decision + +Arkx gains a small neutral contract for runtime capabilities. A +`CapabilityProfile` records component type and identity, optional version and +configuration digest, declared capabilities, and evidence references. +`CapabilityRegistry` provides deterministic ordering and identity-safe lookup. + +The registry is descriptive only in this phase. It does not discover runtime +capabilities, select an executor, perform fallback, or qualify a provider. + +## Rationale + +Executor selection already had string capabilities, while provider and sandbox +contracts exposed identity without a comparable evidence-bearing capability +boundary. This made capability claims difficult to compare or replay. The +contract makes identity and evidence explicit without coupling routing to a +specific provider or model. + +## Falsifiable hypothesis and acceptance + +For equivalent profile inputs, serialization and registry ordering are stable; +duplicate identities are rejected; lookup does not invoke external systems. +Acceptance requires deterministic unit tests. This is local contract evidence, +not qualification evidence. + +## Non-goals + +- dynamic discovery; +- capability inference from a model name; +- executor/provider fallback; +- promotion or benchmark claims. diff --git a/docs/decisions/0028-execution-budget-spec.md b/docs/decisions/0028-execution-budget-spec.md new file mode 100644 index 00000000..719993e3 --- /dev/null +++ b/docs/decisions/0028-execution-budget-spec.md @@ -0,0 +1,23 @@ +# ADR 0028 — Executor-neutral execution budget specification + +## Decision + +Arkx adds `ExecutionBudgetSpec` as the neutral declaration of attempt, token, +wall-time, cost, and executor-invocation limits. The declaration validates +non-negative values and produces a stable digest for plan and manifest identity. + +It does not consume budget, retry, escalate, or override request governance. +Existing routing, recovery, qualification, and request budgets remain separate +until their semantics are reconciled by a later evidence-backed change. + +## Rationale + +The repository currently has several budget types with different scopes. A +shared execution-level shape is useful for reproducibility, but silently +merging all budget semantics would create an unsafe architectural assumption. +This block adds only the common declaration and identity boundary. + +## Acceptance + +Equivalent values must produce the same digest; negative values must be rejected; +no executor or provider is invoked by the contract. diff --git a/docs/decisions/0029-explicit-retry-policy.md b/docs/decisions/0029-explicit-retry-policy.md new file mode 100644 index 00000000..3ec1da88 --- /dev/null +++ b/docs/decisions/0029-explicit-retry-policy.md @@ -0,0 +1,18 @@ +# ADR 0029 — Explicit retry authorization policy + +## Decision + +Arkx adds `RetryPolicy` and `RetryDecision` at the neutral execution boundary. +The policy authorizes a retry only when the error is explicitly retryable, its +domain is allowed, the retry limit remains available, and the execution budget +allows the next attempt. + +The policy is pure: it does not invoke an executor, sleep, mutate artifacts, +replace an executor, or perform recovery. Recovery remains responsible for +choosing a broader action such as replan, escalation, handoff, or block. + +## Acceptance + +Tests must distinguish retryable, non-retryable, unknown, disallowed-domain, +policy-exhausted, and budget-exhausted cases. No automatic retry is inferred +from a local test result. diff --git a/docs/decisions/0030-retry-attempt-lineage.md b/docs/decisions/0030-retry-attempt-lineage.md new file mode 100644 index 00000000..94447c7c --- /dev/null +++ b/docs/decisions/0030-retry-attempt-lineage.md @@ -0,0 +1,16 @@ +# ADR 0030 — Deterministic retry attempt lineage + +## Decision + +An authorized retry produces a `RetryAttemptPlan` with a new deterministic +attempt identity derived from the same trial, the next attempt number, and the +configuration digest. The previous attempt identity is retained and validated. + +The plan is only lineage metadata. It does not reserve a directory, overwrite +artifacts, invoke an executor, or claim that the retry succeeded. + +## Acceptance + +Equivalent retry inputs produce the same next attempt identity and serialized +plan. Unauthorized retry produces no plan. Inconsistent previous identity is +rejected. Artifact reservation remains an explicit later operation. diff --git a/docs/decisions/0031-artifact-store-retry-lineage.md b/docs/decisions/0031-artifact-store-retry-lineage.md new file mode 100644 index 00000000..cf91e1b5 --- /dev/null +++ b/docs/decisions/0031-artifact-store-retry-lineage.md @@ -0,0 +1,17 @@ +# ADR 0031 — Artifact-store reservation for retry lineage + +## Decision + +`ArtifactStore` exposes an explicit `reserve_retry_attempt` operation. It +validates the previous attempt identity, creates the next append-only attempt +directory, and atomically records `retry-lineage.json` with both identities and +the configuration digest. + +The operation refuses identity drift and existing attempt directories. It does +not execute, verify, accept, promote, or recover a task. + +## Acceptance + +The store must preserve the previous attempt, persist the new lineage, reject +duplicate reservation, reject identity drift, and never silently overwrite an +attempt directory. diff --git a/docs/decisions/0032-attempt-store-lineage-replay.md b/docs/decisions/0032-attempt-store-lineage-replay.md new file mode 100644 index 00000000..4f2bfa5c --- /dev/null +++ b/docs/decisions/0032-attempt-store-lineage-replay.md @@ -0,0 +1,16 @@ +# ADR 0032 — Attempt-store lineage validation and replay visibility + +## Decision + +`AttemptSnapshot` exposes optional `RetryLineage` when +`retry-lineage.json` is present. `load_attempt` validates the lineage against +the manifest, trial, attempt number, and configuration digest. + +Historical attempts without the new artifact remain loadable with +`retry_lineage=None`. This preserves compatibility while making missing lineage +explicit; the reader never infers a retry chain from the attempt number alone. + +## Acceptance + +Valid lineage round-trips and is returned by the reader. Drift is rejected. +Legacy attempts remain readable but expose missing lineage explicitly. diff --git a/docs/decisions/0033-idempotent-budget-ledger.md b/docs/decisions/0033-idempotent-budget-ledger.md new file mode 100644 index 00000000..b7c4283b --- /dev/null +++ b/docs/decisions/0033-idempotent-budget-ledger.md @@ -0,0 +1,16 @@ +# ADR 0033 — Idempotent execution budget ledger + +## Decision + +Arkx adds an immutable `BudgetLedger` over `ExecutionBudgetSpec`. Consumption +is keyed by `attempt_id`, so repeating the same consumption is an explicit +`ALREADY_CONSUMED` result and does not double-count usage. A rejected +consumption returns the unchanged ledger with a concrete limit reason. + +The ledger does not retry, execute, persist itself, or reconcile the separate +request/routing/recovery/qualification budgets yet. + +## Acceptance + +Equivalent ledger inputs are deterministic; an attempt is consumed at most +once; exceeded limits do not mutate the ledger; and no executor is invoked. diff --git a/docs/decisions/0034-plan-budget-binding.md b/docs/decisions/0034-plan-budget-binding.md new file mode 100644 index 00000000..aad3b18f --- /dev/null +++ b/docs/decisions/0034-plan-budget-binding.md @@ -0,0 +1,16 @@ +# ADR 0034 — Binding between execution plan and budget specification + +## Decision + +`ExecutionPlan` may carry an `ExecutionBudgetSpec`. When present, its +`budget_digest` is derived or validated against that specification. The routed +orchestration path materializes the request budget into this neutral spec before +execution. + +Legacy plans containing only a digest remain valid. The plan still does not +consume budget or invoke an executor as a side effect. + +## Acceptance + +The plan must serialize the budget, derive a missing digest, reject a mismatched +digest, and preserve the binding through routed orchestration. diff --git a/docs/decisions/0035-persisted-budget-ledger-binding.md b/docs/decisions/0035-persisted-budget-ledger-binding.md new file mode 100644 index 00000000..5cb74c09 --- /dev/null +++ b/docs/decisions/0035-persisted-budget-ledger-binding.md @@ -0,0 +1,15 @@ +# ADR 0035 — Persisted budget ledger binding + +## Decision + +`RunManifest` may persist a serialized budget ledger bound to its own +`attempt_id` and `budget_digest`. `with_budget_ledger` returns a new manifest +snapshot and rejects a ledger from another attempt or configuration. + +This is a persistence boundary only. It does not make the executor consume the +ledger automatically and does not merge the ledger with other budget scopes. + +## Acceptance + +The bound manifest must round-trip its ledger, reject attempt mismatch and +digest mismatch, and remain immutable from the caller's perspective. diff --git a/docs/decisions/0036-budget-aware-execution-lifecycle.md b/docs/decisions/0036-budget-aware-execution-lifecycle.md new file mode 100644 index 00000000..1dd997a3 --- /dev/null +++ b/docs/decisions/0036-budget-aware-execution-lifecycle.md @@ -0,0 +1,17 @@ +# ADR 0036 — Budget-aware execution lifecycle + +## Decision + +`ContractRun.execute_with_budget` consumes the immutable budget ledger before +invoking an executor. Accepted consumption proceeds to execution and carries +the updated ledger. Rejected consumption returns a structured `BLOCKED` result +and invokes no executor. + +This boundary does not persist the manifest itself, retry automatically, or +merge unrelated budget scopes. + +## Acceptance + +Fake execution must prove both orderings: accepted budget leads to one executor +invocation and updated ledger; exhausted budget leads to structured blocked +state and zero executor invocations. diff --git a/docs/decisions/0037-atomic-manifest-persistence.md b/docs/decisions/0037-atomic-manifest-persistence.md new file mode 100644 index 00000000..f12e5071 --- /dev/null +++ b/docs/decisions/0037-atomic-manifest-persistence.md @@ -0,0 +1,16 @@ +# ADR 0037 — Atomic persistence of the execution manifest + +## Decision + +`ArtifactStore.write_manifest` validates attempt identity and lifecycle, then +atomically writes the current `RunManifest` snapshot. This is the persistence +boundary for the budget-aware execution result and its ledger. + +The operation does not verify artifacts, accept a patch, promote a treatment, +or infer success from a successful write. + +## Acceptance + +A valid bound manifest must round-trip from disk. Invalid identity or lifecycle +must be rejected before persistence, and the write must use the existing atomic +artifact-store primitive. diff --git a/docs/decisions/0038-persistent-event-log.md b/docs/decisions/0038-persistent-event-log.md new file mode 100644 index 00000000..39847cb1 --- /dev/null +++ b/docs/decisions/0038-persistent-event-log.md @@ -0,0 +1,16 @@ +# ADR 0038 — Persistent lifecycle event log + +## Decision + +Arkx adds `EventLog` over the existing neutral `Event` contract. Events are +persisted as deterministic JSONL, read back in order, and validated for matching +`run_id` and monotonic timestamps. Writes replace the complete serialized log +atomically; callers use the log serially as an append-only record. + +The log records telemetry only. It does not infer acceptance, choose an +executor, or replace the manifest, verification, or promotion states. + +## Acceptance + +Events must round-trip, wrong run identities must be rejected, timestamp +regression must be rejected, and no external executor/provider is invoked. diff --git a/docs/decisions/0039-deterministic-event-replay.md b/docs/decisions/0039-deterministic-event-replay.md new file mode 100644 index 00000000..7a83512b --- /dev/null +++ b/docs/decisions/0039-deterministic-event-replay.md @@ -0,0 +1,15 @@ +# ADR 0039 — Deterministic event replay + +## Decision + +Arkx adds `replay_events`, a pure reducer over persisted `Event` values. It +reconstructs declared facts such as event count, executor invocations, retries, +evidence references, and the status explicitly declared by `TASK_FINISHED`. + +Replay does not invoke an executor, rerun verification, infer acceptance, or +replace the manifest. Missing facts remain missing. + +## Acceptance + +Equivalent event sequences produce the same `ReplayState`; wrong run identity +and timestamp regression are rejected; replay performs no external calls. diff --git a/docs/decisions/0040-replay-manifest-consistency.md b/docs/decisions/0040-replay-manifest-consistency.md new file mode 100644 index 00000000..ce429f24 --- /dev/null +++ b/docs/decisions/0040-replay-manifest-consistency.md @@ -0,0 +1,16 @@ +# ADR 0040 — Replay and manifest consistency validation + +## Decision + +Arkx adds `validate_replay_against_manifest`. It validates that the replay +belongs to the manifest attempt and, when a terminal status was explicitly +recorded, that it matches the manifest state. + +The validator reports divergence; it does not choose a source of truth, repair +state, infer missing events, or execute any component. + +## Acceptance + +Matching replay and manifest pass. Wrong run identity or conflicting declared +status is rejected. Missing declared status remains indeterminate rather than +being inferred. diff --git a/docs/decisions/0041-replay-independent-outcomes.md b/docs/decisions/0041-replay-independent-outcomes.md new file mode 100644 index 00000000..1f5c610c --- /dev/null +++ b/docs/decisions/0041-replay-independent-outcomes.md @@ -0,0 +1,15 @@ +# ADR 0041 — Independent verification and acceptance in replay + +## Decision + +`ReplayState` records `verification_state` and `acceptance_decision` only when +explicitly present in event data. `validate_replay_outcomes` compares those +values with independent evidence when supplied. + +Execution completion does not imply verification, and verification does not +imply acceptance. Missing outcome events remain missing. + +## Acceptance + +Matching independent evidence passes; conflicting verification or acceptance is +rejected; absent outcome data is not inferred from execution state. diff --git a/docs/decisions/0042-full-replay-chain.md b/docs/decisions/0042-full-replay-chain.md new file mode 100644 index 00000000..96410b0a --- /dev/null +++ b/docs/decisions/0042-full-replay-chain.md @@ -0,0 +1,16 @@ +# ADR 0042 — Full explicit replay chain + +## Decision + +Arkx adds `ReplayChain` and `replay_chain` for explicit references across +request, plan, execution, verification, acceptance, and promotion. The reducer +validates run identity and causal stage order, but leaves absent stages as +`None`. + +Replay is an audit operation. It does not execute, verify, accept, promote, or +invent references for missing artifacts. + +## Acceptance + +An ordered explicit chain round-trips into the corresponding references; an +out-of-order stage is rejected; and incomplete chains remain incomplete. diff --git a/docs/decisions/0043-replay-reference-classification.md b/docs/decisions/0043-replay-reference-classification.md new file mode 100644 index 00000000..30637d5f --- /dev/null +++ b/docs/decisions/0043-replay-reference-classification.md @@ -0,0 +1,16 @@ +# ADR 0043 — Replay reference classification + +## Decision + +Arkx adds `classify_chain_references` and `ReferenceReport` to classify explicit +chain references against an available artifact inventory. References are +`PRESENT` when found, `MISSING` when an inventory exists and does not contain +them, and `INDETERMINATE` when no inventory is supplied. + +The classifier does not dereference arbitrary URIs, download artifacts, or +infer that a recorded reference is physically valid. + +## Acceptance + +Known present and missing references are distinguished; absent inventory yields +indeterminate; no status is converted to success by inference. diff --git a/docs/decisions/0044-attempt-inventory-replay.md b/docs/decisions/0044-attempt-inventory-replay.md new file mode 100644 index 00000000..a4352472 --- /dev/null +++ b/docs/decisions/0044-attempt-inventory-replay.md @@ -0,0 +1,14 @@ +# ADR 0044 — Replay classification against the validated attempt inventory + +## Decision + +`classify_snapshot_chain_references` classifies replay chain references only +against paths and manifest references already validated by an +`AttemptSnapshot`. It does not scan arbitrary directories, dereference external +URIs, or select a replacement artifact. + +## Acceptance + +References present in the snapshot inventory are `PRESENT`; absent references +are `MISSING`; and the previous indeterminate behavior remains available when no +inventory is supplied. diff --git a/docs/decisions/0045-replay-chain-integrity.md b/docs/decisions/0045-replay-chain-integrity.md new file mode 100644 index 00000000..8b1b8cf6 --- /dev/null +++ b/docs/decisions/0045-replay-chain-integrity.md @@ -0,0 +1,16 @@ +# ADR 0045 — Explicit replay-chain integrity states + +## Decision + +Arkx adds `ChainIntegrity` with `COMPLETE`, `INCOMPLETE`, and `INCONSISTENT`. +The assessment uses only the explicit chain and its reference report: missing +stages or indeterminate inventory are incomplete; missing references in a known +inventory are inconsistent; all stages present and verified are complete. + +This classification is a precondition signal only. It does not accept, reject, +promote, repair, or reexecute a task. + +## Acceptance + +The three states must be distinguishable with deterministic tests and no +external calls. diff --git a/docs/decisions/0046-recovery-plan-lineage.md b/docs/decisions/0046-recovery-plan-lineage.md new file mode 100644 index 00000000..896955ab --- /dev/null +++ b/docs/decisions/0046-recovery-plan-lineage.md @@ -0,0 +1,15 @@ +# ADR 0046 — Persistible recovery-plan lineage + +## Decision + +Arkx adds `RecoveryPlan`, derived deterministically from a `RecoveryDecision` +and a non-empty source `attempt_id`. The plan records action, reasons, current +path, and target path without executing the action. + +Recovery planning remains separate from executor selection, retry execution, +handoff, verification, acceptance, and promotion. + +## Acceptance + +Equivalent inputs produce the same plan identity and serialization; the source +attempt is retained; and plan construction performs no external calls. diff --git a/docs/decisions/0047-recovery-replay-lineage.md b/docs/decisions/0047-recovery-replay-lineage.md new file mode 100644 index 00000000..0ed86bfe --- /dev/null +++ b/docs/decisions/0047-recovery-replay-lineage.md @@ -0,0 +1,16 @@ +# ADR 0047 — Recovery lineage in the replay chain + +## Decision + +`ReplayChain` accepts an optional `recovery_ref`. When present, replay enforces +the causal order `execution → recovery → verification`; when absent, ordinary +runs remain valid. The field is appended to the public dataclass signature to +preserve positional compatibility for existing consumers. + +Recovery remains a recorded plan, not an executed action. + +## Acceptance + +Recovery references replay in order, out-of-order stages fail, and runs without +recovery remain compatible and do not become incomplete solely because the field +is absent. diff --git a/docs/decisions/0048-recovery-reference-identity.md b/docs/decisions/0048-recovery-reference-identity.md new file mode 100644 index 00000000..9e039b2f --- /dev/null +++ b/docs/decisions/0048-recovery-reference-identity.md @@ -0,0 +1,14 @@ +# ADR 0048 — Canonical recovery-plan reference identity + +## Decision + +`RecoveryPlan.reference` returns the canonical `recovery://` reference. +`validate_recovery_reference` rejects replay or artifact references that do not +match the exact plan identity. + +The validator does not execute, persist, hand off, or promote recovery. + +## Acceptance + +The canonical reference passes validation; a different plan reference fails; +equivalent plans retain the same reference. diff --git a/docs/decisions/0049-execution-reference-identity.md b/docs/decisions/0049-execution-reference-identity.md new file mode 100644 index 00000000..ec2bcc8a --- /dev/null +++ b/docs/decisions/0049-execution-reference-identity.md @@ -0,0 +1,16 @@ +# ADR 0049 — Canonical execution-chain references + +## Decision + +`ExecutionPlan.reference` returns `plan://`, while +`execution_reference(attempt_id)` returns `execution://`. +`validate_execution_chain` requires both references to match the plan and exact +attempt identity. + +This validates lineage only. It does not infer execution success from an +artifact reference or perform verification, acceptance, or promotion. + +## Acceptance + +Matching plan/attempt references pass; plan or attempt drift fails; no external +component is invoked. diff --git a/docs/decisions/0050-outcome-reference-identity.md b/docs/decisions/0050-outcome-reference-identity.md new file mode 100644 index 00000000..7b369fad --- /dev/null +++ b/docs/decisions/0050-outcome-reference-identity.md @@ -0,0 +1,15 @@ +# ADR 0050 — Canonical verification and acceptance references + +## Decision + +`VerificationResult.reference` and `AcceptanceResult.reference` are stable +content-derived references with distinct namespaces. `validate_outcome_references` +requires each supplied reference to match its exact outcome. + +Verification and acceptance remain independent; a matching reference does not +itself authorize promotion. + +## Acceptance + +Equivalent outcomes produce equal references, namespaces remain distinct, and +reference drift is rejected without external calls. diff --git a/docs/decisions/0051-final-outcome-promotion-chain.md b/docs/decisions/0051-final-outcome-promotion-chain.md new file mode 100644 index 00000000..5e5b8a91 --- /dev/null +++ b/docs/decisions/0051-final-outcome-promotion-chain.md @@ -0,0 +1,16 @@ +# ADR 0051 — Final outcome-to-promotion chain validation + +## Decision + +`PromotionDecision.reference` is a deterministic `promotion://` identity. +`validate_final_chain` requires promotion to match the exact attempt and +decision, and requires an attached acceptance outcome to match its reference. +Verification and acceptance references remain independently supplied. + +This validator checks lineage only; it does not promote a candidate or replace +the existing promotion policy. + +## Acceptance + +Matching final references pass; attempt, promotion, or acceptance drift fails; +no promotion side effect occurs during validation. diff --git a/docs/decisions/0052-adapter-qualification-preflight.md b/docs/decisions/0052-adapter-qualification-preflight.md new file mode 100644 index 00000000..bb28804d --- /dev/null +++ b/docs/decisions/0052-adapter-qualification-preflight.md @@ -0,0 +1,24 @@ +# ADR 0052 — Common adapter qualification preflight + +## Decision + +Arkx adds a neutral preflight contract for executor, provider, sandbox, and +evaluator adapters. `AdapterIdentity` records kind, name, version, and +configuration digest. `DependencyObservation` records availability without +inference. `AdapterPreflight` records status, capabilities, dependencies, +evidence, and reason. + +`assess_preflight` produces `READY`, `BLOCKED`, `UNKNOWN`, or `FAILED` without +loading or invoking a concrete adapter and without selecting a fallback. + +`preflight_error` maps a non-ready preflight to the neutral `ErrorEnvelope`, +preserving the integration boundary, preflight reference, and +`Retryability.UNKNOWN`. It does not infer retryability from a missing +dependency or from an adapter's failure. + +## Acceptance + +Ready, blocked, unknown, and failed boundaries remain distinguishable; +equivalent observations serialize deterministically; and missing dependencies do +not become success. Error mapping preserves the boundary and raw evidence +reference without selecting a fallback. diff --git a/docs/decisions/0053-adapter-preflight-error-envelope.md b/docs/decisions/0053-adapter-preflight-error-envelope.md new file mode 100644 index 00000000..b25af77a --- /dev/null +++ b/docs/decisions/0053-adapter-preflight-error-envelope.md @@ -0,0 +1,34 @@ +# ADR 0053 — Adapter preflight errors retain neutral boundaries + +## Decision + +When an adapter preflight is not ready, Arkx converts it to the common +`ErrorEnvelope` through `preflight_error`. The mapping retains the integration +boundary (`EXECUTOR`, `PROVIDER`, `SANDBOX`, or `VERIFICATION`), uses a stable +`PREFLIGHT_` code, and attaches the deterministic preflight reference +as raw evidence. + +Retryability is `UNKNOWN` unless a later, explicit policy has enough evidence +to classify it. Missing identity, missing dependency, permission denial, and +runtime failure therefore cannot silently become retryable, successful, or a +fallback selection. + +## Falsifiable hypothesis + +For equivalent preflight observations, the resulting error envelope preserves +the same boundary, code, retryability, and raw evidence reference. A test can +falsify this by observing boundary drift, inferred retryability, or missing +preflight evidence. + +## Acceptance evidence + +The dependency-isolated suite ran 294 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`; the OpenRouter adapter test remains excluded because the +`minisweagent` dependency is unavailable. Concrete adapter states remain +recorded as `BLOCKED` or `UNKNOWN` in the read-only preflight evidence and are +not qualified by this local result. + +## Status + +Accepted for neutral contract development. No external adapter or provider is +promoted. diff --git a/docs/decisions/0054-import-safe-concrete-adapter-boundaries.md b/docs/decisions/0054-import-safe-concrete-adapter-boundaries.md new file mode 100644 index 00000000..2652b264 --- /dev/null +++ b/docs/decisions/0054-import-safe-concrete-adapter-boundaries.md @@ -0,0 +1,31 @@ +# ADR 0054 — Concrete adapter modules are import-safe but execution-strict + +## Decision + +Concrete adapter modules may be imported when their external dependency is +absent, so contract and diagnostic tests can run. They must not provide a +substitute implementation: construction or execution raises an explicit +boundary error until the required dependency is available. + +This separates module discovery and preflight from runtime qualification. It +does not make an adapter `READY`, and it does not authorize fallback selection. + +## Falsifiable hypothesis + +The complete test discovery can import adapter modules without the external +runtime, while constructing an unavailable adapter still fails explicitly. A +test falsifies this if import fails globally or if construction silently uses a +fallback. + +## Acceptance evidence + +The complete local suite ran 298 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. The read-only concrete preflight still reports OpenRouter and +Git Bash as `BLOCKED`, Docker/SWE-bench as `BLOCKED`, and Ollama as `UNKNOWN`. +These results are integration diagnostics, not provider or benchmark +qualification. + +## Status + +Accepted for integration-boundary development. No concrete adapter is +promoted. diff --git a/docs/decisions/0055-neutral-adapter-error-normalization.md b/docs/decisions/0055-neutral-adapter-error-normalization.md new file mode 100644 index 00000000..7a451813 --- /dev/null +++ b/docs/decisions/0055-neutral-adapter-error-normalization.md @@ -0,0 +1,32 @@ +# ADR 0055 — Normalize adapter failures at the integration boundary + +## Decision + +Arkx exposes neutral error normalization for three concrete failure shapes: + +- provider payloads containing an `error` object; +- process failures from executor, sandbox, or evaluator boundaries; +- adapter exceptions. + +The result is always an `ErrorEnvelope` with an explicit domain, stable code, +raw evidence references, and `Retryability.UNKNOWN` unless a separate policy +has justified a stronger classification. Provider SDKs and concrete runtime +types are not imported by the core normalization functions. + +## Falsifiable hypothesis + +Equivalent provider payloads and process failures produce deterministic domain +and code values while preserving raw references. A test falsifies this if a +provider payload reaches the success parser, a sandbox error becomes a provider +error, or retryability is inferred without policy evidence. + +## Acceptance evidence + +The complete local suite ran 301 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. The OpenRouter adapter test verifies that its provider error +can cross into `ErrorEnvelope`; concrete provider/runtime qualification remains +outside this local result. + +## Status + +Accepted for neutral integration-boundary development. No adapter is promoted. diff --git a/docs/decisions/0056-legacy-runner-neutral-error-boundary.md b/docs/decisions/0056-legacy-runner-neutral-error-boundary.md new file mode 100644 index 00000000..a17cd752 --- /dev/null +++ b/docs/decisions/0056-legacy-runner-neutral-error-boundary.md @@ -0,0 +1,39 @@ +# ADR 0056 — Legacy runner emits neutral error boundaries + +## Decision + +The P8.2 baseline runner retains its legacy `RunnerFailureCategory` in the +execution artifact, but its public manifest error is produced through the +neutral integration boundary where appropriate: + +- missing identity remains a `HARNESS` precondition failure; +- environment and process-start failures are `SANDBOX` process failures; +- model, agent, timeout, capture, and run failures are `EXECUTOR` process + failures; +- provider availability keeps its explicit bounded-retry policy and remains a + `PROVIDER` error. + +This preserves historical artifact meaning while preventing upper layers from +depending on runner-specific categories for domain routing. + +## Falsifiable hypothesis + +Equivalent legacy runner outcomes produce the same neutral domain and retain +raw artifact references, while missing identity does not become an executor +failure and provider retryability is not weakened. Tests falsify this if the +domain changes across equivalent outcomes or if legacy artifacts lose their +category. + +## Acceptance evidence + +The complete local suite ran 301 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Existing baseline tests for missing identity, provider +availability, execution failure, and artifact persistence passed. + +This local result does not qualify the mini-SWE-agent provider, sandbox, or +SWE-bench evaluator. + +## Status + +Accepted for compatibility-preserving boundary migration. No external adapter +is promoted. diff --git a/docs/decisions/0057-evaluator-failure-outcome-separation.md b/docs/decisions/0057-evaluator-failure-outcome-separation.md new file mode 100644 index 00000000..b374c5bb --- /dev/null +++ b/docs/decisions/0057-evaluator-failure-outcome-separation.md @@ -0,0 +1,34 @@ +# ADR 0057 — Evaluator failures remain separate from verification outcomes + +## Decision + +`OfficialEvaluationResult` exposes `error_envelope()` only for evaluator +states that do not represent a completed test verdict. Infrastructure errors, +ambiguous reports, blocked runs, and non-execution map to the neutral +`VERIFICATION` domain with an `EVALUATOR_` code and preserved report/log +references. + +`RESOLVED` and `TESTS_FAILED` do not produce an error envelope. The former is a +completed positive evaluator outcome; the latter is a completed negative test +outcome. Both remain available to the separate acceptance mapping. + +## Falsifiable hypothesis + +An evaluator infrastructure failure cannot be mistaken for a test failure, and +a completed test failure cannot be retried merely because it is negative. Tests +falsify this if the two statuses produce the same error representation or if +raw evaluator evidence is lost. + +## Acceptance evidence + +The complete local suite ran 302 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests verify the distinction between infrastructure error and +`TESTS_FAILED`, including domain, stable code, retryability, and report/log +references. + +Docker/SWE-bench execution remains externally blocked and is not qualified by +this local contract evidence. + +## Status + +Accepted for evaluator/verification boundary development. No promotion occurs. diff --git a/docs/decisions/0058-evaluator-to-verification-contract.md b/docs/decisions/0058-evaluator-to-verification-contract.md new file mode 100644 index 00000000..bd30211a --- /dev/null +++ b/docs/decisions/0058-evaluator-to-verification-contract.md @@ -0,0 +1,35 @@ +# ADR 0058 — Evaluator results become verification outcomes before acceptance + +## Decision + +`OfficialEvaluationResult.verification_result()` converts the official +evaluator observation to the executor-neutral `VerificationResult` contract. +The mapping is explicit: + +- `RESOLVED` → `PASS`; +- `TESTS_FAILED` → `FAIL`; +- infrastructure error or ambiguity → `INDETERMINATE`; +- not executed → `NOT_EXECUTED`; +- blocked → `BLOCKED`. + +Authority, command identity, outcome text, and report/log evidence are carried +into the verification result. Acceptance remains a later policy decision and +promotion remains outside this method. + +## Falsifiable hypothesis + +Equivalent evaluator results produce deterministic verification references and +cannot produce a promotion decision. Tests falsify this if evaluator status is +lost, evidence is reordered nondeterministically, or acceptance/promotion is +implicitly performed during conversion. + +## Acceptance evidence + +The complete local suite ran 303 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. The evaluator tests verify status mapping, authority, +command identity, evidence, and the absence of implicit promotion. + +## Status + +Accepted for evaluator-to-verification integration. No provider, evaluator, or +promotion is qualified by local tests. diff --git a/docs/decisions/0059-causal-verification-acceptance-promotion-chain.md b/docs/decisions/0059-causal-verification-acceptance-promotion-chain.md new file mode 100644 index 00000000..bb254e52 --- /dev/null +++ b/docs/decisions/0059-causal-verification-acceptance-promotion-chain.md @@ -0,0 +1,35 @@ +# ADR 0059 — Promotion requires a causal verification-to-acceptance chain + +## Decision + +Independent acceptance now records the deterministic `verification://` +reference alongside its evidence. The outcome promotion entrypoint requires +that same verification reference to be present in the acceptance evidence +before evaluating promotion policy. + +Promotion therefore consumes a completed verification outcome, an acceptance +decision from the declared authority, and explicit evidence. Missing or +unlinked verification blocks promotion; it cannot be repaired by adding an +unrelated experiment or artifact reference. + +The legacy `VerificationEvidence` path remains available for compatibility, but +new outcome-based callers should use `decide_outcome_promotion`. + +## Falsifiable hypothesis + +Acceptance without the exact verification reference is rejected by the +promotion gate, while a matching accepted outcome can be promoted only when +all existing evidence and authority policy requirements also pass. Tests +falsify this if an unlinked acceptance is promoted or if a linked non-pass +verification is promoted. + +## Acceptance evidence + +The complete local suite ran 304 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover verification reference propagation, mismatch +blocking, matching promotion, and preservation of the legacy promotion path. + +## Status + +Accepted for causal outcome-chain development. No external candidate is +promoted by this local evidence. diff --git a/docs/decisions/0060-causal-stage-event-persistence.md b/docs/decisions/0060-causal-stage-event-persistence.md new file mode 100644 index 00000000..ba9cd579 --- /dev/null +++ b/docs/decisions/0060-causal-stage-event-persistence.md @@ -0,0 +1,30 @@ +# ADR 0060 — Persist causal lifecycle stages as replayable events + +## Decision + +`EventLog.append_stage()` persists request, plan, execution, recovery, +verification, acceptance, and promotion references as causal stage events. +Identical repeated references are idempotent. A different reference for an +already recorded stage is rejected, and `replay_chain()` rejects stage +reference drift or out-of-order events. + +The event log stores facts only; it does not execute an adapter, evaluate a +patch, accept a task, or promote a candidate. + +## Falsifiable hypothesis + +Equivalent event histories replay to the same chain, duplicate stage writes do +not create new facts, and reference mutation cannot be silently accepted. +Tests falsify this if replay fills missing stages, accepts stage drift, or +reexecutes any external component. + +## Acceptance evidence + +The complete local suite ran 306 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover causal append, idempotent duplicate handling, +out-of-order rejection, and reference drift detection. + +## Status + +Accepted for persistent lifecycle telemetry and replay. No external execution +or promotion is performed by the log. diff --git a/docs/decisions/0061-replay-object-chain-consistency.md b/docs/decisions/0061-replay-object-chain-consistency.md new file mode 100644 index 00000000..1c3c39b4 --- /dev/null +++ b/docs/decisions/0061-replay-object-chain-consistency.md @@ -0,0 +1,32 @@ +# ADR 0061 — Replay references must match outcome objects + +## Decision + +`validate_outcome_chain()` cross-checks a replayed `ReplayChain` against the +actual `VerificationResult`, `AcceptanceResult`, and `PromotionDecision` +objects. It requires: + +- replay verification reference equals the verification result reference; +- replay acceptance reference equals the acceptance result reference; +- replay promotion reference equals the promotion decision reference; +- acceptance carries the exact verification reference; +- promotion carries the exact acceptance reference. + +Reference presence alone is insufficient for chain integrity. + +## Falsifiable hypothesis + +A chain with complete but mismatched references is rejected, while a chain +whose references identify the supplied objects is accepted without executing +any external component. Tests falsify this if object drift passes validation. + +## Acceptance evidence + +The complete local suite ran 308 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover a valid causal chain and verification-reference +drift. + +## Status + +Accepted for replay and audit integrity. No external execution or promotion is +performed by this validator. diff --git a/docs/decisions/0062-restored-chain-audit.md b/docs/decisions/0062-restored-chain-audit.md new file mode 100644 index 00000000..4bdcb914 --- /dev/null +++ b/docs/decisions/0062-restored-chain-audit.md @@ -0,0 +1,30 @@ +# ADR 0062 — Restored attempts require manifest, event, and outcome consistency + +## Decision + +`audit_restored_chain()` loads the event log for the restored attempt, replays +its facts, validates replayed terminal status against the manifest, reconstructs +the causal chain, classifies references against the snapshot and supplied +outcomes, and validates outcome-object identity when all outcomes are present. + +The audit does not rerun execution or evaluation. An incomplete chain remains +`INCOMPLETE`; missing references are not inferred from nearby artifacts. +Manifest/replay divergence is rejected as an integrity error. + +## Falsifiable hypothesis + +A restored attempt with matching facts is auditable without execution, while a +manifest/replay status mismatch or outcome-reference drift is rejected. Tests +falsify this if audit executes external components or reports an incomplete +chain as complete. + +## Acceptance evidence + +The complete local suite ran 310 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover restored replay, incomplete-chain reporting, and +manifest status divergence. + +## Status + +Accepted for artifact restoration and audit. No external execution or +promotion is performed by restoration. diff --git a/docs/decisions/0063-audited-attempt-loading.md b/docs/decisions/0063-audited-attempt-loading.md new file mode 100644 index 00000000..2aae237f --- /dev/null +++ b/docs/decisions/0063-audited-attempt-loading.md @@ -0,0 +1,29 @@ +# ADR 0063 — Audited loading is an explicit artifact-store boundary + +## Decision + +`AttemptStore.load_audited()` loads the validated attempt snapshot and invokes +the read-only restored-chain audit before returning an audit result. The +existing `load_attempt()` and `list_attempts()` paths remain available for +compatibility, but consumers that require comparison, acceptance, or +promotion integrity can opt into audited loading explicitly. + +The store does not select a winner, rerun execution, or infer missing event +stages. Audit results remain `INCOMPLETE` when the persisted chain is partial. + +## Falsifiable hypothesis + +Audited loading always passes the restored snapshot through chain validation, +while legacy loading behavior remains unchanged. Tests falsify this if the +audited path bypasses validation or if loading causes external execution. + +## Acceptance evidence + +The complete local suite ran 311 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests verify that `AttemptStore.load_audited()` delegates to +the restored-chain audit and preserves the loaded snapshot boundary. + +## Status + +Accepted for integrity-sensitive artifact consumers. No external execution or +promotion is performed during loading. diff --git a/docs/decisions/0064-audited-comparison-gate.md b/docs/decisions/0064-audited-comparison-gate.md new file mode 100644 index 00000000..6b944c56 --- /dev/null +++ b/docs/decisions/0064-audited-comparison-gate.md @@ -0,0 +1,30 @@ +# ADR 0064 — Decision-grade comparison requires complete audited chains + +## Decision + +`AttemptStore.compare_audited()` loads and audits both attempts before applying +manifest comparability. If either restored chain is not `COMPLETE`, the method +fails explicitly instead of returning a comparison that could support +acceptance or promotion. + +The existing `compare_attempts()` and `compare_attempt_collection()` functions +remain available for lower-level identity diagnostics. They must not be +interpreted as proof that verification, acceptance, or promotion evidence is +complete. + +## Falsifiable hypothesis + +An attempt pair with an incomplete or inconsistent persisted chain cannot pass +the decision-grade comparison gate, while legacy identity comparison remains +available. Tests falsify this if an incomplete chain is returned as eligible +for decision use. + +## Acceptance evidence + +The complete local suite ran 312 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests verify that incomplete audited chains are rejected. + +## Status + +Accepted for integrity-sensitive comparison. No external candidate is +accepted or promoted by comparison alone. diff --git a/docs/decisions/0065-audited-experiment-reproducibility-gate.md b/docs/decisions/0065-audited-experiment-reproducibility-gate.md new file mode 100644 index 00000000..340fcbfa --- /dev/null +++ b/docs/decisions/0065-audited-experiment-reproducibility-gate.md @@ -0,0 +1,30 @@ +# ADR 0065 — Reproducible experiments require audited attempt chains + +## Decision + +`validate_experiment_record_audited()` extends experiment validation for +decision-grade use. It requires one audit per recorded attempt, matching replay +run identity, `COMPLETE` chain integrity, and presence of every declared +verification, acceptance, and promotion reference in the audited chains. + +The existing `validate_experiment_record()` remains available for compatibility +with historical records and basic reference validation. It must not be +interpreted as proof of reproducibility without the audited variant. + +## Falsifiable hypothesis + +An experiment with an incomplete attempt chain or a declared reference absent +from the replayed chain is rejected as reproducible, while a complete matching +experiment passes without rerunning any external component. Tests falsify this +if incomplete evidence is accepted. + +## Acceptance evidence + +The complete local suite ran 314 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover complete audited experiment acceptance and +incomplete-chain rejection. + +## Status + +Accepted for reproducibility claims. No benchmark, provider, or model is +qualified by this local validator. diff --git a/docs/decisions/0066-experiment-execution-identity-digest.md b/docs/decisions/0066-experiment-execution-identity-digest.md new file mode 100644 index 00000000..ed633328 --- /dev/null +++ b/docs/decisions/0066-experiment-execution-identity-digest.md @@ -0,0 +1,30 @@ +# ADR 0066 — Reproducible experiment identity binds execution variables + +## Decision + +Arkx derives an `identity_digest` for an experiment from the ordered attempt +inventory and execution-relevant manifest fields: task and revision, treatment, +executor, provider, model, sandbox, repository revision, configuration digest, +budget digest, and protocol version. + +The audited experiment validator requires this digest and rejects experiments +whose recorded identity differs from the loaded attempts. Historical records +may remain readable through the compatibility validator without making a +reproducibility claim. + +## Falsifiable hypothesis + +Changing any execution-relevant identity field changes the experiment digest, +while reordering attempts does not. An audited experiment with a missing or +drifted digest is rejected. + +## Acceptance evidence + +The complete local suite ran 314 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Existing experiment serialization and audited-validation +tests passed with the new optional identity field. + +## Status + +Accepted for reproducible experiment identity. No model, provider, executor, or +benchmark is qualified by digest computation alone. diff --git a/docs/decisions/0067-versioned-replay-compatibility.md b/docs/decisions/0067-versioned-replay-compatibility.md new file mode 100644 index 00000000..34700929 --- /dev/null +++ b/docs/decisions/0067-versioned-replay-compatibility.md @@ -0,0 +1,30 @@ +# ADR 0067 — Strict reproducibility requires version-compatible replay events + +## Decision + +Restored-chain audits observe protocol and event schema versions from persisted +events. Historical logs without version metadata remain readable when no +expected version is requested. A strict audit with an expected protocol +requires every observed version to be present, homogeneous, and equal to the +expected `protocol_version` and supported event schema version. + +Audited experiment validation additionally requires the audit protocol to match +the experiment protocol and requires explicit event schema metadata. + +## Falsifiable hypothesis + +Protocol or schema drift in a replayed chain is rejected for reproducibility, +while legacy logs remain readable for historical inspection. Tests falsify this +if mixed versions pass strict audit or if a strict experiment accepts an +unversioned chain. + +## Acceptance evidence + +The complete local suite ran 315 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover protocol drift rejection, schema presence for +audited experiments, and compatibility of unversioned historical replay. + +## Status + +Accepted for versioned replay compatibility. No benchmark or provider +qualification is inferred from schema agreement. diff --git a/docs/decisions/0068-frozen-execution-identity-replay.md b/docs/decisions/0068-frozen-execution-identity-replay.md new file mode 100644 index 00000000..9aa683e3 --- /dev/null +++ b/docs/decisions/0068-frozen-execution-identity-replay.md @@ -0,0 +1,31 @@ +# ADR 0068 — Strict replay freezes configuration, budget, and treatment identity + +## Decision + +Strict restored-chain audit observes `configuration_digest`, `budget_digest`, +and `treatment` from persisted lifecycle events. Values must be homogeneous and, +when expected identity is supplied, equal to the expected values. Audited +experiment validation additionally compares them with the attempt manifest. + +This prevents a retry or restoration from reusing an attempt identity with a +different configuration, budget, or treatment. Capability identity is not yet +required because Arkx does not have a canonical persisted capability digest in +the manifest; introducing one without that contract would create false +confidence. + +## Falsifiable hypothesis + +Configuration drift in a replayed event log is rejected, while an unchanged +identity passes. Tests falsify this if drift is silently accepted or if an +unversioned capability field is treated as qualified evidence. + +## Acceptance evidence + +The complete local suite ran 316 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover configuration drift rejection and manifest/event +identity comparison. + +## Status + +Accepted for frozen execution identity. Capability registry persistence remains +an explicit future contract, not an inferred field. diff --git a/docs/decisions/0069-persisted-capability-identity.md b/docs/decisions/0069-persisted-capability-identity.md new file mode 100644 index 00000000..6d235c2a --- /dev/null +++ b/docs/decisions/0069-persisted-capability-identity.md @@ -0,0 +1,31 @@ +# ADR 0069 — Persist capability identity without inferring capability quality + +## Decision + +`CapabilityRegistry` now exposes a deterministic digest and a +`capability://` reference derived from its canonical JSON representation. Strict +replay audit can observe and validate `capability_digest`; audited experiments +must record the same digest in their identity and event chain. + +This contract establishes identity and consistency only. It does not assert +that a declared capability works, that an adapter is ready, or that a provider +is qualified. Those claims still require preflight and independent empirical +evidence. + +## Falsifiable hypothesis + +Equivalent registries produce the same digest, registry changes produce a +different digest, and an audited experiment with capability drift is rejected. +Tests falsify this if capability identity changes are invisible or if digest +presence is treated as functional qualification. + +## Acceptance evidence + +The complete local suite ran 317 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover deterministic registry identity and its +experiment/replay consistency boundary. + +## Status + +Accepted for capability identity persistence. Capability effectiveness remains +unqualified until empirical preflight/evaluation evidence exists. diff --git a/docs/decisions/0070-capability-preflight-provenance.md b/docs/decisions/0070-capability-preflight-provenance.md new file mode 100644 index 00000000..d39bccf4 --- /dev/null +++ b/docs/decisions/0070-capability-preflight-provenance.md @@ -0,0 +1,31 @@ +# ADR 0070 — Capability preflight records provenance without conflating qualification + +## Decision + +`AdapterPreflight` carries an optional capability digest and explicit +`CapabilityProvenance`: `DECLARED`, `OBSERVED`, or `QUALIFIED`. + +- `DECLARED` identifies what a registry or adapter claims; +- `OBSERVED` identifies what a runtime preflight observed; +- `QUALIFIED` is permitted only when evidence references are present. + +Preflight status and capability provenance are independent dimensions. A +`READY` preflight does not imply qualified capability, and a capability digest +does not prove effectiveness. + +## Falsifiable hypothesis + +Equivalent capability identity serializes deterministically, provenance survives +preflight serialization, and `QUALIFIED` without evidence is rejected. Tests +falsify this if declared capability becomes qualified by inference. + +## Acceptance evidence + +The complete local suite ran 318 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover capability digest/provenance transport and +evidence requirements for qualification claims. + +## Status + +Accepted for capability/preflight boundary development. No adapter capability +is qualified by this contract alone. diff --git a/docs/decisions/0071-capability-provenance-routing-policy.md b/docs/decisions/0071-capability-provenance-routing-policy.md new file mode 100644 index 00000000..d8c2f2c9 --- /dev/null +++ b/docs/decisions/0071-capability-provenance-routing-policy.md @@ -0,0 +1,33 @@ +# ADR 0071 — Routing policy must respect capability provenance + +## Decision + +`assess_capability_policy()` authorizes a capability only when: + +- the adapter preflight is `READY`; +- a capability identity digest is present; +- all required capabilities are present; +- the observed provenance meets the policy minimum. + +The provenance order is `DECLARED < OBSERVED < QUALIFIED`. A failed or blocked +preflight returns a policy denial, not an executor switch or fallback route. +This function authorizes capability use only; it does not choose or execute an +adapter. + +## Falsifiable hypothesis + +A declared-only capability cannot satisfy an observed/qualified policy, and a +blocked preflight cannot authorize routing. An observed capability satisfies an +observed minimum when identity and required names match. Tests falsify this if +policy silently upgrades provenance or selects a fallback. + +## Acceptance evidence + +The complete local suite ran 321 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover provenance insufficiency, blocked preflight, and +successful observed-capability authorization. + +## Status + +Accepted for routing policy. No executor/provider is selected or qualified by +this policy function alone. diff --git a/docs/decisions/0072-execution-plan-capability-gate.md b/docs/decisions/0072-execution-plan-capability-gate.md new file mode 100644 index 00000000..45f28139 --- /dev/null +++ b/docs/decisions/0072-execution-plan-capability-gate.md @@ -0,0 +1,30 @@ +# ADR 0072 — Execution plans carry and validate capability requirements + +## Decision + +`ExecutionPlan` records required capability names and a minimum capability +provenance. `ExecutionPlan.validate_capabilities()` applies the neutral routing +policy against an adapter preflight before execution. + +A plan may be structurally valid while still being unauthorized to execute if +its preflight is blocked, its capability identity is absent, a required +capability is missing, or its provenance is below policy. Plan construction does +not select a fallback or invoke an executor. + +## Falsifiable hypothesis + +A declared-only capability cannot satisfy a plan requiring observed capability, +while the same plan can be authorized after an observed preflight with matching +identity and capability. Tests falsify this if plan validation silently upgrades +provenance or executes an adapter. + +## Acceptance evidence + +The complete local suite ran 322 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover capability requirements carried by a plan and +denial before execution when provenance is insufficient. + +## Status + +Accepted for execution-plan gating. No executor or provider is selected by plan +validation alone. diff --git a/docs/decisions/0073-orchestration-capability-preflight-gate.md b/docs/decisions/0073-orchestration-capability-preflight-gate.md new file mode 100644 index 00000000..81dfa6c7 --- /dev/null +++ b/docs/decisions/0073-orchestration-capability-preflight-gate.md @@ -0,0 +1,32 @@ +# ADR 0073 — Orchestration must validate plan capabilities before execution + +## Decision + +`build_execution_plan()` carries the selected treatment's required +capabilities. `run_routed_pipeline()` accepts an optional concrete preflight and +checks the resulting plan before invoking `executor.run`. + +If a plan requires capabilities and no preflight is supplied, orchestration +returns `CAPABILITY_PREFLIGHT_REQUIRED`. If the preflight does not satisfy the +capability policy, orchestration returns `CAPABILITY_POLICY_BLOCKED`. Neither +case selects a fallback executor. + +Legacy plans with no capability requirements remain compatible; they do not +claim capability qualification merely because execution was invoked. + +## Falsifiable hypothesis + +A capability-requiring plan cannot invoke an executor without a qualifying +preflight, while a plan with no capability requirement preserves legacy +behavior. Tests falsify this if `executor.run` is called before the gate. + +## Acceptance evidence + +The complete local suite ran 323 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests verify required capabilities are carried into the plan +and missing preflight blocks orchestration before executor invocation. + +## Status + +Accepted for orchestration gating. No executor/provider is selected or +qualified by this gate alone. diff --git a/docs/decisions/0074-concrete-adapter-preflight-plan-binding.md b/docs/decisions/0074-concrete-adapter-preflight-plan-binding.md new file mode 100644 index 00000000..a9310d9f --- /dev/null +++ b/docs/decisions/0074-concrete-adapter-preflight-plan-binding.md @@ -0,0 +1,32 @@ +# ADR 0074 — Concrete adapter preflight must bind to the execution plan + +## Decision + +`P82BaselineExecutorAdapter.preflight()` returns an executor-scoped +`AdapterPreflight` without invoking the runner. It records executor identity, +version, configuration digest, dependency availability, declared capability, +and capability provenance. + +`ExecutionPlan.preflight_matches_identity()` requires the preflight to describe +the same executor kind, executor id, and configuration digest as the plan. +Orchestration rejects mismatched preflight before invoking the executor. + +## Falsifiable hypothesis + +A preflight from another executor or configuration cannot authorize a plan, and +preflight collection does not execute the runner. Tests falsify this if identity +drift passes or if preflight has execution side effects. + +## Acceptance evidence + +The complete local suite ran 325 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover P8.2 preflight identity, declared capability +provenance, and plan rejection of another executor's preflight. + +The concrete dependency may still be unavailable; that state remains an +explicit preflight result and is not converted to a fallback. + +## Status + +Accepted for concrete adapter binding. No adapter is promoted by preflight +identity alone. diff --git a/docs/decisions/0075-persisted-preflight-orchestration-event.md b/docs/decisions/0075-persisted-preflight-orchestration-event.md new file mode 100644 index 00000000..54bf902d --- /dev/null +++ b/docs/decisions/0075-persisted-preflight-orchestration-event.md @@ -0,0 +1,30 @@ +# ADR 0075 — Orchestration persists preflight before execution + +## Decision + +The causal replay chain includes an optional `preflight` stage between plan and +execution. When a plan requires capabilities, orchestration must receive an +attempt-bound `EventLog` and persist the preflight reference before invoking +the executor. + +If the preflight is absent, mismatched, denied by policy, or cannot be +persisted because no attempt-bound event log exists, orchestration blocks. It +does not fabricate a run identity or select a fallback. + +## Falsifiable hypothesis + +A capability-requiring orchestration path cannot invoke an executor before a +preflight event is persisted, and replay reconstructs that stage in causal +order. Tests falsify this if execution occurs without the event or if preflight +is replayed after execution. + +## Acceptance evidence + +The complete local suite ran 326 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover missing event-log persistence and preflight stage +handling. + +## Status + +Accepted for preflight persistence. No external executor or provider is +qualified by event persistence alone. diff --git a/docs/decisions/0076-orchestration-persisted-lifecycle-chain.md b/docs/decisions/0076-orchestration-persisted-lifecycle-chain.md new file mode 100644 index 00000000..87d3f621 --- /dev/null +++ b/docs/decisions/0076-orchestration-persisted-lifecycle-chain.md @@ -0,0 +1,35 @@ +# ADR 0076 — Orchestration persists the execution-to-acceptance lifecycle + +## Decision + +When supplied an attempt-bound `EventLog`, orchestration persists lifecycle +references in causal order: + +`plan → preflight? → execution → verification → acceptance`. + +Execution, verification, and acceptance references are derived from their +neutral contracts. If a required lifecycle write fails, orchestration returns +an explicit blocked result rather than reporting a later acceptance without +persisted evidence. Promotion remains a separate stage and is not performed by +orchestration. + +Legacy callers without an event log remain compatible for non-capability plans, +but they do not produce a replay-complete lifecycle claim. + +## Falsifiable hypothesis + +An event-backed accepted pipeline replays all persisted stages in order, while +missing persistence prevents an acceptance claim. Tests falsify this if stages +are reordered, references drift, or acceptance is returned after a failed +write. + +## Acceptance evidence + +The complete local suite ran 327 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests verify persisted plan, execution, verification, and +acceptance references and causal replay. + +## Status + +Accepted for persisted orchestration lifecycle. Promotion remains an +independent gate. diff --git a/docs/decisions/0077-promotion-event-persistence-gate.md b/docs/decisions/0077-promotion-event-persistence-gate.md new file mode 100644 index 00000000..5a8e90f1 --- /dev/null +++ b/docs/decisions/0077-promotion-event-persistence-gate.md @@ -0,0 +1,28 @@ +# ADR 0077 — Promotion persistence requires the preceding causal chain + +## Decision + +`append_promotion_decision()` persists a `promotion` stage only when the same +`EventLog` already contains the supplied verification reference and the +acceptance reference embedded in the `PromotionDecision`. + +Promotion without persisted acceptance, acceptance from another chain, or +verification reference drift is rejected. The function records the promotion +decision reference and status but does not execute or mutate the candidate. + +## Falsifiable hypothesis + +A valid verification/acceptance/promotion chain replays all three references, +while promotion attempted before acceptance or with another acceptance fails. +Tests falsify this if a partial chain receives a promotion event. + +## Acceptance evidence + +The complete local suite ran 329 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover successful persistence and rejection of missing +or mismatched acceptance. + +## Status + +Accepted for promotion event persistence. This does not itself approve or +promote any external component. diff --git a/docs/decisions/0078-replay-operational-telemetry.md b/docs/decisions/0078-replay-operational-telemetry.md new file mode 100644 index 00000000..669a2c1e --- /dev/null +++ b/docs/decisions/0078-replay-operational-telemetry.md @@ -0,0 +1,29 @@ +# ADR 0078 — Replay reconstructs operational telemetry without inferring success + +## Decision + +Persisted lifecycle events may carry operational facts for execution outcome, +preflight status, promotion status, and structured error codes. `ReplayState` +reconstructs these fields deterministically in addition to stage references, +verification state, and acceptance decision. + +These fields are diagnostic facts only. Replay does not infer acceptance, +promotion, retryability, or provider quality from their presence. + +## Falsifiable hypothesis + +Equivalent event histories reconstruct equivalent operational telemetry, and a +legacy error representation remains readable without crashing replay. Tests +falsify this if telemetry changes the lifecycle decision implicitly or if +structured error data is lost. + +## Acceptance evidence + +The complete local suite ran 330 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover reconstruction of execution outcome and error +codes and compatibility with legacy textual errors. + +## Status + +Accepted for replay observability. No lifecycle outcome is promoted by +telemetry alone. diff --git a/docs/decisions/0079-replay-telemetry-result-consistency.md b/docs/decisions/0079-replay-telemetry-result-consistency.md new file mode 100644 index 00000000..75946157 --- /dev/null +++ b/docs/decisions/0079-replay-telemetry-result-consistency.md @@ -0,0 +1,28 @@ +# ADR 0079 — Replay telemetry must match loaded result objects + +## Decision + +`validate_replay_telemetry()` compares persisted replay facts with any loaded +execution, verification, acceptance, and promotion objects. It rejects +execution outcome, structured error code, verification state, acceptance +decision, or promotion status drift. + +The validator is observational: it does not rerun an executor, verifier, +acceptance authority, or promotion policy. Missing telemetry remains a separate +incompleteness concern and is not filled by inference. + +## Falsifiable hypothesis + +Matching persisted facts and result objects pass; any recorded mismatch fails. +Tests falsify this if stale telemetry is accepted or if validation invokes an +external component. + +## Acceptance evidence + +The complete local suite ran 331 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover verification-state drift against a loaded result. + +## Status + +Accepted for replay integrity. No lifecycle state is promoted by telemetry +validation alone. diff --git a/docs/decisions/0080-terminal-lifecycle-error-events.md b/docs/decisions/0080-terminal-lifecycle-error-events.md new file mode 100644 index 00000000..63503d96 --- /dev/null +++ b/docs/decisions/0080-terminal-lifecycle-error-events.md @@ -0,0 +1,26 @@ +# ADR 0080 — Terminal lifecycle errors are explicit and replayable + +## Decision + +`EventLog.append_terminal()` persists a single terminal `TASK_FINISHED` fact +with status and optional error code. Repeating the same terminal status is +idempotent; changing a terminal status is rejected. Replay reconstructs the +declared status and error code without creating verification or acceptance +outcomes. + +## Falsifiable hypothesis + +Blocked, failed, and indeterminate terminal paths can be replayed with their +error code, while terminal status mutation is rejected. Tests falsify this if a +terminal status changes silently or if replay infers later lifecycle stages. + +## Acceptance evidence + +The complete local suite ran 333 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. Tests cover idempotent terminal persistence, status mutation +rejection, and replay reconstruction. + +## Status + +Accepted for terminal lifecycle telemetry. Terminal events do not imply +verification, acceptance, or promotion. diff --git a/docs/decisions/0081-orchestration-terminal-event-integration.md b/docs/decisions/0081-orchestration-terminal-event-integration.md new file mode 100644 index 00000000..b5a28528 --- /dev/null +++ b/docs/decisions/0081-orchestration-terminal-event-integration.md @@ -0,0 +1,30 @@ +# ADR 0081 — Orchestration records terminal status after persisted lifecycle + +## Decision + +When an attempt-bound event log is supplied, orchestration appends one terminal +`TASK_FINISHED` event after execution/verification/acceptance outcomes, carrying +the orchestration status and primary reason code. Terminal append is idempotent +and does not create missing verification or acceptance stages. + +Pre-execution governance failures without a legitimate `run_id` do not receive +a fabricated terminal event. Persistence failure downgrades the returned +result to an explicit blocked state. + +## Falsifiable hypothesis + +An event-backed pipeline replays its terminal status, while a pre-execution +failure without attempt identity does not fabricate lifecycle facts. Tests +falsify this if terminal status is absent after a run or if an unmaterialized +request receives an attempt event. + +## Acceptance evidence + +The complete local suite ran 333 tests successfully on Python 3.11.9 with +`PYTHONPATH=src`. The persisted orchestration lifecycle test verifies terminal +status replay. + +## Status + +Accepted for terminal orchestration telemetry. Terminal status remains separate +from promotion. diff --git a/docs/decisions/0082-lifecycle-persistence-error-taxonomy.md b/docs/decisions/0082-lifecycle-persistence-error-taxonomy.md new file mode 100644 index 00000000..96bb2133 --- /dev/null +++ b/docs/decisions/0082-lifecycle-persistence-error-taxonomy.md @@ -0,0 +1,42 @@ +# ADR 0082: Neutral lifecycle persistence error taxonomy + +## Status + +Accepted for local architectural evidence; not an external qualification. + +## Context + +The orchestration lifecycle persists plan, preflight, execution, verification, +acceptance, and terminal events. A missing event log is a capability-specific +precondition when a plan requires persisted capability evidence. Other +persistence failures are generic lifecycle failures and must not be reported +as capability failures. + +## Decision + +Use `CAPABILITY_PREFLIGHT_PERSISTENCE_REQUIRED` only when a capability-bearing +plan cannot persist its required preflight evidence. Use +`LIFECYCLE_PERSISTENCE_FAILED` for generic stage persistence failures and for a +failure to append the terminal lifecycle event. No fallback executor or +implicit success is permitted. + +## Falsifiable hypothesis and acceptance criteria + +H1: persistence failures are classified by the violated lifecycle invariant, +not by the last feature added. The focused regression test must distinguish the +generic and capability-specific reasons, and terminal persistence failure must +remain explicit. + +## Evidence + +Raw evidence is recorded in +`logs/architecture/lifecycle-persistence-error-taxonomy-block-20260923.json`. +The local deterministic suite passed with 334 tests. This is local evidence +only; no provider, model, Docker, or SWE-bench qualification was performed. + +## Consequences + +Operators can distinguish a missing capability-evidence precondition from a +general lifecycle persistence outage. Callers still need to decide whether a +blocked result is retryable according to policy; the taxonomy does not grant +automatic retry or promotion. diff --git a/docs/decisions/0083-terminal-fact-integrity.md b/docs/decisions/0083-terminal-fact-integrity.md new file mode 100644 index 00000000..554fe8b1 --- /dev/null +++ b/docs/decisions/0083-terminal-fact-integrity.md @@ -0,0 +1,25 @@ +# ADR 0083: Terminal facts are immutable and explicitly validated + +## Status + +Accepted for local architectural evidence; not an external qualification. + +## Decision + +`EventLog.append_terminal()` treats status and error code as one immutable +terminal identity. Repeating an identical terminal fact is idempotent; changing +either field is rejected. `validate_terminal_replay()` compares a replayed +terminal fact with an explicitly supplied expected status and optional error +code. A missing terminal event is an error, not an inferred success. + +## Falsifiable hypothesis and acceptance criteria + +H1: replay cannot silently change the terminal explanation while preserving the +same status. Tests must reject changed error codes, reject missing terminal +facts, reject wrong error codes, and accept an exact terminal match. + +## Evidence + +The focused tests and the complete local suite provide deterministic evidence. +This block does not qualify an executor, provider, model, Docker, or +SWE-bench authority. diff --git a/docs/decisions/0084-orchestration-budget-enforcement.md b/docs/decisions/0084-orchestration-budget-enforcement.md new file mode 100644 index 00000000..576b5355 --- /dev/null +++ b/docs/decisions/0084-orchestration-budget-enforcement.md @@ -0,0 +1,27 @@ +# ADR 0084: Enforce invocation budget at the orchestration seam + +## Status + +Accepted for local architectural evidence; partial enforcement only. + +## Decision + +When an execution plan contains a budget, orchestration consumes one immutable +ledger entry before invoking any runner and records the resulting ledger in the +execution event data. A budget that rejects the invocation blocks before the +runner is called. The plan reference is the deterministic local consumption +identity for this single pipeline invocation. + +This decision enforces attempts/invocations at the neutral seam. Token, wall +time and cost consumption still require adapter-reported measurements and are +not inferred from this ledger entry. + +## Falsifiable hypothesis and acceptance criteria + +H1: a zero invocation budget never reaches a runner. The focused test must +prove that the runner is not called and that the result is explicitly blocked. + +## Evidence + +The complete local suite is the acceptance evidence for this block. External +provider/model/Docker/SWE-bench qualification remains unchanged. diff --git a/docs/decisions/0085-measured-budget-usage.md b/docs/decisions/0085-measured-budget-usage.md new file mode 100644 index 00000000..e33584fa --- /dev/null +++ b/docs/decisions/0085-measured-budget-usage.md @@ -0,0 +1,29 @@ +# ADR 0085: Budget usage must be measured at the execution boundary + +## Status + +Accepted for local architectural evidence; external qualification unchanged. + +## Decision + +`BudgetLedger.record_usage()` records measured token, wall-time and cost usage +for an attempt already admitted by the ledger. Orchestration requires the +measurements corresponding to declared finite limits. If a runner does not +provide a required measurement, the result is explicitly blocked with +`BUDGET_USAGE_UNAVAILABLE`; usage is never inferred as zero. If measured usage +exceeds a limit, the result is explicitly blocked with +`EXECUTION_BUDGET_EXHAUSTED` after retaining the execution evidence. + +## Falsifiable hypothesis and acceptance criteria + +H1: a runner cannot appear budget-compliant merely because it returned an +execution result. Tests must accept recorded measured usage, reject usage for +an unconsumed attempt, and block orchestration when required measurement is +missing. + +## Consequences + +Concrete adapters must expose canonical `total_tokens`, `wall_time_ms`, and +`cost` fields when the plan declares those limits. Zero is a valid measured +value; missing is not equivalent to zero. Adapter-specific raw usage may still +be retained alongside the canonical fields. diff --git a/docs/decisions/0086-explicit-lifecycle-state-crosswalk.md b/docs/decisions/0086-explicit-lifecycle-state-crosswalk.md new file mode 100644 index 00000000..3450b67f --- /dev/null +++ b/docs/decisions/0086-explicit-lifecycle-state-crosswalk.md @@ -0,0 +1,25 @@ +# ADR 0086: Keep lifecycle vocabularies separate but crosswalked explicitly + +## Status + +Accepted for local architectural evidence. + +## Decision + +`RunState` remains the execution lifecycle vocabulary. `ExecutionOutcome` is +the neutral adapter boundary, and `OrchestrationStatus` is the composed result +vocabulary that also includes verification and acceptance. Their meanings are +not collapsed. Total pure crosswalk functions are public and tested so mapping +drift cannot be hidden in ad-hoc dictionaries inside the pipeline. + +## Falsifiable hypothesis and acceptance criteria + +H1: every execution outcome and acceptance decision maps deterministically to +exactly one orchestration status. The crosswalk test must cover every enum +member. + +## Rationale + +Creating a third universal state machine now would duplicate semantics. The +explicit crosswalk is the smallest evidence-backed correction while recovery +and crash-resume semantics are still being designed. diff --git a/docs/decisions/0087-causal-recovery-persistence.md b/docs/decisions/0087-causal-recovery-persistence.md new file mode 100644 index 00000000..f5e299ed --- /dev/null +++ b/docs/decisions/0087-causal-recovery-persistence.md @@ -0,0 +1,23 @@ +# ADR 0087: Recovery persistence requires a recorded source execution + +## Status + +Accepted for local architectural evidence; recovery execution remains explicit. + +## Decision + +`append_recovery_plan()` persists a recovery stage only when the plan's source +attempt identity exactly matches an execution reference already present in the +same event log. The recovery action is retained as data. Persisting recovery +does not execute a replan, switch executors, or imply promotion. + +## Falsifiable hypothesis and acceptance criteria + +H1: a recovery artifact cannot float free of the attempt it claims to recover. +The focused test must accept a matching execution and reject a mismatched +execution reference. + +## Consequences + +Recovery is now replay-auditable at the persistence boundary. Full recovery +resume and replan execution remain future work and must use this causal record. diff --git a/docs/decisions/0088-canonical-configuration-snapshot.md b/docs/decisions/0088-canonical-configuration-snapshot.md new file mode 100644 index 00000000..7dfc9b1d --- /dev/null +++ b/docs/decisions/0088-canonical-configuration-snapshot.md @@ -0,0 +1,24 @@ +# ADR 0088: Canonical secret-safe configuration snapshots + +## Status + +Accepted for local architectural evidence. + +## Decision + +`ConfigurationSnapshot` is the neutral representation for reproducible +configuration identity. It records component, version, public values and +digests of secret values, never secret material. Its canonical JSON and digest +are deterministic and can populate existing `configuration_digest` fields. + +## Falsifiable hypothesis and acceptance criteria + +H1: equivalent configuration mappings produce the same identity, while public +configuration or secret digests changing produce different identities. Tests +must cover canonical ordering, secret non-serialization and invalid identity. + +## Scope boundary + +This contract does not resolve configuration from environment variables or +secret stores. Adapters remain responsible for producing the snapshot and +recording the external resolver identity. diff --git a/docs/decisions/0089-plan-configuration-snapshot-binding.md b/docs/decisions/0089-plan-configuration-snapshot-binding.md new file mode 100644 index 00000000..164a250c --- /dev/null +++ b/docs/decisions/0089-plan-configuration-snapshot-binding.md @@ -0,0 +1,24 @@ +# ADR 0089: Bind canonical configuration snapshots to execution plans + +## Status + +Accepted for local architectural evidence. + +## Decision + +`ExecutionPlan.configuration_snapshot` is optional for compatibility, but when +present its canonical digest is the plan's `configuration_digest`. A plan may +derive the digest from the snapshot or reject a mismatching supplied digest. +The serialized plan retains the snapshot metadata and public values, never +secret material. + +## Falsifiable hypothesis and acceptance criteria + +H1: a plan cannot carry a configuration digest inconsistent with its canonical +snapshot. Tests must derive a missing digest and reject an inconsistent one. + +## Consequences + +Adapters can progressively migrate from opaque strings to canonical snapshots. +Plans without snapshots remain valid legacy contracts but are weaker for +reproducibility and should be treated as such by qualification policy. diff --git a/docs/decisions/0090-p82-adapter-configuration-snapshot.md b/docs/decisions/0090-p82-adapter-configuration-snapshot.md new file mode 100644 index 00000000..c47ca9fe --- /dev/null +++ b/docs/decisions/0090-p82-adapter-configuration-snapshot.md @@ -0,0 +1,24 @@ +# ADR 0090: P8.2 baseline adapter adopts canonical configuration snapshots + +## Status + +Accepted for local architectural evidence; P8.2 remains experiment-coupled. + +## Decision + +`BaselineRunConfig.configuration_snapshot()` is the canonical source for the +existing P8.2 `identity_digest()`. The preflight identity and capability digest +use the same snapshot digest. The snapshot contains the existing frozen public +configuration fields and no secret material. + +## Falsifiable hypothesis and acceptance criteria + +H1: P8.2 preflight and configuration identity cannot drift because they derive +from one canonical snapshot. Tests must prove digest equality and secret-safe +serialization. + +## Consequences + +This is an adoption step, not a qualification result. It does not make +minisweagent available, qualify a provider/model, or generalize the P8.2 +treatment into a universal executor. diff --git a/docs/decisions/0091-versioned-provider-identity.md b/docs/decisions/0091-versioned-provider-identity.md new file mode 100644 index 00000000..5ca8247b --- /dev/null +++ b/docs/decisions/0091-versioned-provider-identity.md @@ -0,0 +1,23 @@ +# ADR 0091: Providers require versioned configuration-bound identity + +## Status + +Accepted for local architectural evidence. + +## Decision + +The neutral `Provider` protocol exposes `provider_identity` as an +`AdapterIdentity` with `IntegrationKind.PROVIDER`. `validate_provider_identity` +rejects anonymous providers and providers lacking version or configuration +digest. The provider identity is separate from the model fields in +`ProviderRequest`. + +## Falsifiable hypothesis and acceptance criteria + +H1: a provider cannot enter the neutral contract without reproducible identity. +Tests must accept the versioned fake provider and reject an anonymous provider. + +## Boundary + +This does not qualify provider behavior, model quality, pricing, latency, or +availability. Those remain preflight and experimental evidence concerns. diff --git a/docs/decisions/0092-versioned-sandbox-identity.md b/docs/decisions/0092-versioned-sandbox-identity.md new file mode 100644 index 00000000..95fdd18f --- /dev/null +++ b/docs/decisions/0092-versioned-sandbox-identity.md @@ -0,0 +1,24 @@ +# ADR 0092: Sandboxes require versioned configuration-bound identity + +## Status + +Accepted for local architectural evidence. + +## Decision + +The neutral `Sandbox` protocol exposes `sandbox_identity` as an +`AdapterIdentity` with `IntegrationKind.SANDBOX`. The identity must include +version and configuration digest. `validate_sandbox_identity` rejects +anonymous or incomplete sandbox implementations. + +## Falsifiable hypothesis and acceptance criteria + +H1: a sandbox cannot enter the neutral contract without reproducible runtime +identity. Tests must accept the versioned fake sandbox and reject an anonymous +implementation. + +## Boundary + +Identity validation does not prove isolation, Docker correctness, filesystem +safety, or runtime availability. Those remain preflight and qualification +responsibilities. diff --git a/docs/decisions/0093-plan-integration-identity-binding.md b/docs/decisions/0093-plan-integration-identity-binding.md new file mode 100644 index 00000000..ee4ca6ab --- /dev/null +++ b/docs/decisions/0093-plan-integration-identity-binding.md @@ -0,0 +1,23 @@ +# ADR 0093: Bind provider and sandbox identities to the execution plan + +## Status + +Accepted for local architectural evidence. + +## Decision + +`ExecutionPlan` may carry versioned `provider_identity` and `sandbox_identity` +objects. When present, each identity must have the correct integration kind, +version, configuration digest, and name matching the corresponding plan id. +Legacy plans without these identities remain readable but are weaker for +reproducibility and qualification. + +## Falsifiable hypothesis and acceptance criteria + +H1: a plan cannot bind a provider or sandbox identity from another component. +Tests must accept matching identities and reject an id mismatch. + +## Consequences + +The plan now provides one place to compare executor, provider and sandbox +identity. It does not select providers, start sandboxes, or qualify behavior. diff --git a/docs/decisions/0094-orchestration-integration-identity-binding.md b/docs/decisions/0094-orchestration-integration-identity-binding.md new file mode 100644 index 00000000..60e96ca8 --- /dev/null +++ b/docs/decisions/0094-orchestration-integration-identity-binding.md @@ -0,0 +1,24 @@ +# ADR 0094: Orchestration accepts explicit provider and sandbox bindings + +## Status + +Accepted for local architectural evidence. + +## Decision + +`build_execution_plan()` and `run_routed_pipeline()` accept optional provider +and sandbox identities. They are passed into the plan and validated there. +Omitting them preserves legacy compatibility but does not invent identities or +select defaults. + +## Falsifiable hypothesis and acceptance criteria + +H1: identities supplied to orchestration are present on the exact plan that is +executed. The focused test must compare both identity objects after pipeline +construction. + +## Consequences + +Concrete callers can bind qualified integrations before execution. Provider and +sandbox selection remains outside orchestration, and missing identities remain +observable as a qualification/reproducibility weakness. diff --git a/docs/decisions/0095-multi-integration-preflight-replay.md b/docs/decisions/0095-multi-integration-preflight-replay.md new file mode 100644 index 00000000..b47702f9 --- /dev/null +++ b/docs/decisions/0095-multi-integration-preflight-replay.md @@ -0,0 +1,24 @@ +# ADR 0095: Persist separate preflight references for each bound integration + +## Status + +Accepted for local architectural evidence. + +## Decision + +Replay chains distinguish executor `preflight_ref`, `provider_preflight_ref`, +and `sandbox_preflight_ref`. Orchestration requires a matching READY preflight +for every provider or sandbox identity explicitly bound to the plan and +persists each reference before execution. Missing or mismatched preflights +block without choosing a default integration. + +## Falsifiable hypothesis and acceptance criteria + +H1: a multi-integration plan can be replayed without losing one preflight to +another. End-to-end tests must require the preflights and recover both distinct +references from the event log. + +## Consequences + +The event log has a small schema extension rather than overloading one generic +preflight slot. Existing executor-only chains remain compatible. diff --git a/docs/decisions/0096-safe-recovery-resume-boundaries.md b/docs/decisions/0096-safe-recovery-resume-boundaries.md new file mode 100644 index 00000000..cefe84a9 --- /dev/null +++ b/docs/decisions/0096-safe-recovery-resume-boundaries.md @@ -0,0 +1,26 @@ +# ADR 0096: Resume only from persisted causal boundaries + +## Status + +Accepted for local architectural evidence. + +## Decision + +`decide_resume()` is a pure replay decision. A terminal event means no resume +is allowed. An execution reference without verification resumes verification; +verification without acceptance resumes acceptance; acceptance without +promotion resumes promotion. A plan without execution requires explicit +restart, because missing execution evidence cannot prove that the executor did +not run. A recovery reference without a new execution requires explicit replan. + +## Falsifiable hypothesis and acceptance criteria + +H1: crash recovery never reexecutes a stage whose evidence is missing by +assumption. Tests must cover restart-required, verification-resume, +acceptance-resume, and terminal decisions. + +## Consequences + +The protocol is safe but conservative. A future resume service may execute the +returned action only after checking the same identities, budgets and policies; +this decision does not authorize an executor switch or fallback. diff --git a/docs/decisions/0097-audited-resume-decision.md b/docs/decisions/0097-audited-resume-decision.md new file mode 100644 index 00000000..0504fa5c --- /dev/null +++ b/docs/decisions/0097-audited-resume-decision.md @@ -0,0 +1,23 @@ +# ADR 0097: Restored attempt audits expose the safe resume decision + +## Status + +Accepted for local architectural evidence. + +## Decision + +`RestoredChainAudit` includes the `ResumeDecision` derived from its replayed +chain and terminal state. Loading an attempt therefore exposes both evidence +integrity and the safe continuation boundary. The audit does not execute the +decision or mutate artifacts. + +## Falsifiable hypothesis and acceptance criteria + +H1: an audited attempt cannot be consumed for resume without an explicit +decision. The restored-chain test must expose `RESTART_REQUIRED` when execution +evidence is absent. + +## Consequences + +Resume services can consume one audited result instead of reimplementing replay +logic. They must still create a new attempt for restart/replan actions. diff --git a/docs/decisions/0098-recovery-attempt-identity.md b/docs/decisions/0098-recovery-attempt-identity.md new file mode 100644 index 00000000..9d497b2a --- /dev/null +++ b/docs/decisions/0098-recovery-attempt-identity.md @@ -0,0 +1,23 @@ +# ADR 0098: Restart and replan create a new attempt identity + +## Status + +Accepted for local architectural evidence. + +## Decision + +`RecoveryAttemptPlan` derives the next attempt id from trial id, consecutive +attempt number and configuration digest, while retaining the source attempt +and recovery reference. It is declarative only: it does not reserve a +directory, invoke an executor, switch provider, or select a treatment. + +## Falsifiable hypothesis and acceptance criteria + +H1: recovery cannot silently reuse the source attempt. Tests must derive a +different next identity and reject a source id that does not match lineage. + +## Consequences + +Artifact reservation and execution of the new attempt remain separate gates, +which preserves idempotency and prevents a replay operation from becoming a +hidden retry. diff --git a/docs/decisions/0099-atomic-recovery-attempt-reservation.md b/docs/decisions/0099-atomic-recovery-attempt-reservation.md new file mode 100644 index 00000000..c8ad17b6 --- /dev/null +++ b/docs/decisions/0099-atomic-recovery-attempt-reservation.md @@ -0,0 +1,22 @@ +# ADR 0099: Recovery attempt reservation is atomic and lineage-bound + +## Status + +Accepted for local architectural evidence. + +## Decision + +`ArtifactStore.reserve_recovery_attempt()` validates the source attempt id, +derives the next attempt id, reserves a new directory, and atomically writes +`recovery-lineage.json`. Existing directories are never overwritten. Manifest +creation and execution remain separate operations. + +## Falsifiable hypothesis and acceptance criteria + +H1: a recovery attempt cannot overwrite an existing attempt or lose its source +lineage. Tests must inspect the lineage and reject a second reservation. + +## Consequences + +The artifact boundary is safe for a future resume service. It still does not +authorize execution, promotion, or automatic executor switching. diff --git a/docs/decisions/0100-recovery-lineage-audited-loading.md b/docs/decisions/0100-recovery-lineage-audited-loading.md new file mode 100644 index 00000000..46df0a7a --- /dev/null +++ b/docs/decisions/0100-recovery-lineage-audited-loading.md @@ -0,0 +1,24 @@ +# ADR 0100: Recovery lineage is part of audited attempt loading + +## Status + +Accepted for local architectural evidence. + +## Decision + +`load_attempt()` reads and validates `recovery-lineage.json` when present. +Recovery lineage must match the manifest's trial, attempt, attempt number and +configuration digest. An attempt cannot contain both retry and recovery +lineage, because that would make the causal origin ambiguous. + +## Falsifiable hypothesis and acceptance criteria + +H1: recovery lineage written by the artifact store cannot be silently ignored +or drift from the loaded manifest. Tests must round-trip valid lineage and +reject an invalid source identity. + +## Consequences + +Audit and comparison consumers can distinguish ordinary retry from recovery. +The manifest still needs explicit policy fields if future workflows permit +multiple recovery types. diff --git a/docs/decisions/0101-recovery-manifest-binding.md b/docs/decisions/0101-recovery-manifest-binding.md new file mode 100644 index 00000000..c9726bde --- /dev/null +++ b/docs/decisions/0101-recovery-manifest-binding.md @@ -0,0 +1,23 @@ +# ADR 0101: Recovery manifests must match recovery lineage + +## Status + +Accepted for local architectural evidence. + +## Decision + +When an artifact root contains `recovery-lineage.json`, +`ArtifactStore.write_manifest()` validates the lineage before writing the +manifest. Trial, attempt id, attempt number and configuration digest must all +match. A mismatch is rejected before the manifest is persisted. + +## Falsifiable hypothesis and acceptance criteria + +H1: a recovery directory cannot be made audit-ready by writing an unrelated +manifest. The test must reject a mismatched attempt manifest. + +## Consequences + +Recovery reservation, manifest persistence and execution remain distinct +states, but the manifest cannot sever the causal identity established at +reservation. diff --git a/docs/decisions/0102-recovery-budget-gate.md b/docs/decisions/0102-recovery-budget-gate.md new file mode 100644 index 00000000..f551184e --- /dev/null +++ b/docs/decisions/0102-recovery-budget-gate.md @@ -0,0 +1,24 @@ +# ADR 0102: Recovery reservation consumes the persisted attempt budget + +## Status + +Accepted for local architectural evidence. + +## Decision + +`ArtifactStore.reserve_recovery_attempt()` optionally accepts the persisted +budget ledger. When present, the source attempt must be in the ledger and the +next attempt must remain within `max_attempts`. Budget exhaustion is rejected +before directory creation. A missing ledger remains a weaker legacy path and +must be qualified by the caller. + +## Falsifiable hypothesis and acceptance criteria + +H1: recovery cannot bypass a persisted attempts budget. The test must reject a +source with `max_attempts` already consumed and prove no directory was created. + +## Consequences + +The method validates budget state but does not mutate the ledger. The caller +must persist the updated ledger atomically as part of the new attempt's +manifest/lifecycle. diff --git a/docs/decisions/0103-retry-budget-gate.md b/docs/decisions/0103-retry-budget-gate.md new file mode 100644 index 00000000..6d5ee6f8 --- /dev/null +++ b/docs/decisions/0103-retry-budget-gate.md @@ -0,0 +1,18 @@ +# ADR 0103: Retry reservation uses the same persisted attempt budget + +## Status + +Accepted for local architectural evidence. + +## Decision + +`ArtifactStore.reserve_retry_attempt()` applies the same optional persisted +ledger gate as recovery reservation. The previous attempt must be consumed and +the next attempt must remain within `max_attempts`; otherwise no directory or +lineage is written. + +## Falsifiable hypothesis and acceptance criteria + +H1: retry and recovery cannot disagree about attempt-budget enforcement. The +test must reject retry reservation at an exhausted budget and prove no target +directory exists. diff --git a/docs/decisions/0104-persisted-recovery-budget-ledger.md b/docs/decisions/0104-persisted-recovery-budget-ledger.md new file mode 100644 index 00000000..b6101ced --- /dev/null +++ b/docs/decisions/0104-persisted-recovery-budget-ledger.md @@ -0,0 +1,23 @@ +# ADR 0104: Persist the updated budget ledger with a new attempt + +## Status + +Accepted for local architectural evidence. + +## Decision + +When retry or recovery reservation receives a persisted budget ledger, it +writes `budget-ledger.json` containing the new attempt id before returning the +new artifact root. `ArtifactStore.write_manifest()` requires the manifest's +budget ledger to match that persisted ledger exactly. + +## Falsifiable hypothesis and acceptance criteria + +H1: accepted attempt reservation cannot leave budget consumption implicit. The +test must observe the new attempt in `budget-ledger.json`; a manifest with a +different ledger must be rejected. + +## Consequences + +The ledger is copied, not mutated in memory. Callers still need to write the +matching manifest and event log; reservation alone is not execution success. diff --git a/docs/decisions/0105-budget-ledger-replay-binding.md b/docs/decisions/0105-budget-ledger-replay-binding.md new file mode 100644 index 00000000..40c7643e --- /dev/null +++ b/docs/decisions/0105-budget-ledger-replay-binding.md @@ -0,0 +1,22 @@ +# ADR 0105: Budget ledger is part of execution replay + +## Status + +Accepted for local architectural evidence. + +## Decision + +Replay captures the `budget_ledger` carried by execution evidence. Repeated +ledger facts must be identical; drift is rejected. When an audited manifest +declares a persisted budget ledger, `audit_restored_chain()` requires the +replayed ledger to match it exactly. + +## Falsifiable hypothesis and acceptance criteria + +H1: budget consumption cannot differ between artifact manifest and event-log +replay. Tests must restore a ledger and reject a changed ledger fact. + +## Consequences + +Budget state is now part of the reproducibility chain. Legacy attempts without +ledger evidence remain weaker and are not upgraded by inference. diff --git a/docs/decisions/0106-lineage-event-chain-audit.md b/docs/decisions/0106-lineage-event-chain-audit.md new file mode 100644 index 00000000..b17b599f --- /dev/null +++ b/docs/decisions/0106-lineage-event-chain-audit.md @@ -0,0 +1,28 @@ +# ADR 0106: Retry and recovery lineage must match the replayed event chain + +## Status + +Accepted for local architectural evidence. + +## Decision + +An audited retry or recovery attempt is valid only when its persisted lineage +agrees with the replayed event chain. The execution reference must identify the +manifest attempt, a recovery lineage must identify the persisted recovery +reference and source attempt, and the replayed budget ledger must equal the +manifest ledger when one is declared. + +## Falsifiable hypothesis and acceptance criteria + +H1: an attempt with inconsistent execution, recovery, source, or budget facts +cannot pass lineage audit. + +- matching execution/recovery/source/ledger facts: PASS; +- missing or mismatched recovery reference: PASS rejection; +- local regression suite: 365 tests PASS; +- external qualification: not performed. + +## Consequences + +Recovery evidence is now causally bound to the attempt being audited. Legacy +attempts without these facts remain weaker and are not upgraded by inference. diff --git a/docs/decisions/0107-budget-enforced-legacy-runner-boundary.md b/docs/decisions/0107-budget-enforced-legacy-runner-boundary.md new file mode 100644 index 00000000..e440ca5e --- /dev/null +++ b/docs/decisions/0107-budget-enforced-legacy-runner-boundary.md @@ -0,0 +1,33 @@ +# ADR 0107: Budgeted orchestration requires an explicit runner capability + +## Status + +Accepted for local architectural evidence. + +## Decision + +When an execution plan carries a budget, the orchestration seam requires the +runner to expose `run_with_budget(request, selection, attempt_id, +budget_ledger)`. A legacy runner that exposes only `run()` is blocked before +execution with `BUDGET_ENFORCEMENT_REQUIRED`. The neutral +`ContractExecutorRunner` implements the capability only after the attempt has +been reserved in the immutable ledger. + +## Falsifiable hypothesis and acceptance criteria + +H1: a budgeted request cannot reach a runner that has not declared the budget +enforcement capability. + +- legacy runner without the capability is blocked before its `run()` method: + PASS; +- contract bridge requires the reserved attempt identity: PASS; +- missing post-execution usage remains blocked as unavailable: PASS; +- deterministic local suite: 366 tests PASS; +- external qualification: not performed. + +## Consequences + +The old runner seam is no longer silently trusted for budgeted execution. +Existing adapters must implement the capability or remain explicitly blocked; +this is a compatibility break by design because silent budget bypass is not an +acceptable executor-agnostic behavior. diff --git a/docs/decisions/0108-orchestration-configuration-snapshot-propagation.md b/docs/decisions/0108-orchestration-configuration-snapshot-propagation.md new file mode 100644 index 00000000..a8b6a381 --- /dev/null +++ b/docs/decisions/0108-orchestration-configuration-snapshot-propagation.md @@ -0,0 +1,31 @@ +# ADR 0108: Orchestration persists the typed configuration snapshot + +## Status + +Accepted for local architectural evidence. + +## Decision + +`build_execution_plan()` and `run_routed_pipeline()` accept an optional +`ConfigurationSnapshot`. When supplied, its canonical digest becomes the plan +configuration identity, the snapshot is retained in the `ExecutionPlan`, and a +conflict with the selected executor identity is rejected. The plan identity +includes the snapshot content, preventing two configurations with the same +task and treatment from silently collapsing into one plan. + +## Falsifiable hypothesis and acceptance criteria + +H1: a typed configuration snapshot supplied to orchestration survives into the +execution plan and cannot disagree with the executor identity. + +- snapshot retained and digest propagated: PASS; +- identity mismatch rejected: PASS by `ExecutionPlan`/orchestration contract; +- deterministic local suite: 367 tests PASS; +- external qualification: not performed. + +## Consequences + +Configuration identity is no longer only an unstructured digest at the +orchestration boundary. Callers that have a canonical snapshot can preserve it +through planning and replay; legacy callers remain supported but have weaker +configuration evidence until they provide one. diff --git a/docs/decisions/0109-persistence-coordination-scope.md b/docs/decisions/0109-persistence-coordination-scope.md new file mode 100644 index 00000000..8ee12988 --- /dev/null +++ b/docs/decisions/0109-persistence-coordination-scope.md @@ -0,0 +1,38 @@ +# ADR 0109: Persistence coordination remains audit-based at current scope + +## Status + +Accepted for local architectural evidence. + +## Decision + +Arkx keeps atomic artifact writes plus an append-only event log as the current +single-process persistence boundary. It does not introduce a distributed event +bus, a multi-file transaction coordinator, or a database-backed lifecycle +store yet. Restored-chain audit and replay must detect missing, incomplete, or +inconsistent records and preserve them as non-success states. + +## Evidence and falsifiable criteria + +H1: a partial or divergent persisted lifecycle cannot be treated as a complete +accepted attempt without a new transaction subsystem. + +- atomic manifest/artifact writes are exercised: PASS; +- missing and indeterminate chain references are classified: PASS; +- replay/manifest and outcome drift are rejected: PASS; +- full deterministic suite: 367 tests PASS; +- multi-writer/distributed execution: NOT IN SCOPE and not qualified. + +## Rationale + +The current harness is local and serial. A transaction coordinator or event bus +would add failure modes and operational surface without an empirical consumer +requiring fan-out or concurrent writers. The decision can be revisited when +parallel workers, remote artifact stores, or distributed lifecycle consumers +are introduced. + +## Consequences + +Persistence is crash-detecting rather than crash-atomic across independent +files. A restored attempt with incomplete evidence is blocked or indeterminate; +it is never repaired or promoted by inference. diff --git a/docs/decisions/0110-project-identity-and-namespace.md b/docs/decisions/0110-project-identity-and-namespace.md new file mode 100644 index 00000000..e42d2987 --- /dev/null +++ b/docs/decisions/0110-project-identity-and-namespace.md @@ -0,0 +1,31 @@ +# 0110 — Codepro project identity and `arkx` namespace + +## Status + +Accepted. Product and distribution identity is `Codepro`; `arkx` remains the +technical Python namespace until a separately evaluated migration exists. + +## Decision + +Current and new user-facing documentation, release metadata, experiment +reports, and evidence indexes must identify the product as **Codepro**. The +installed distribution and CLI already use `codepro`. + +The import namespace `arkx`, paths under `src/arkx`, and historical evidence +references are preserved for compatibility and traceability. They are not a +second product identity. A namespace migration would be a separate breaking +change requiring import-compatibility tests, artifact migration, and a new +acceptance decision. + +The physical workspace path `D:\\projetos\\arkx` is environment state, not +release metadata, and is not renamed by this decision. + +## Acceptance criteria + +- current README and release-facing documents use Codepro as product name; +- package metadata and CLI remain consistently `codepro`; +- technical references to `arkx` state that it is the Python namespace where + ambiguity is possible; +- historical logs and audit records are not rewritten; +- no release or qualification status changes as a result of this naming work. + diff --git a/docs/decisions/0111-cost-tier-escalation-policy.md b/docs/decisions/0111-cost-tier-escalation-policy.md new file mode 100644 index 00000000..91e5794e --- /dev/null +++ b/docs/decisions/0111-cost-tier-escalation-policy.md @@ -0,0 +1,76 @@ +# 0111 — Cost-tier escalation policy + +## Status + +Accepted as an experimental policy. Not implemented as automatic routing and +not eligible for promotion until executor qualification and non-routed tier +measurements exist. + +## Decision + +Codepro will model cost as an economic treatment variable, not as a proxy for +quality, intelligence, or preference. Candidate options are classified into +cost tiers: + +- `FREE_SHORT` — free option hypothesized to be suitable for short/local tasks; +- `FREE_GENERAL` — second free option hypothesized to cover broader tasks; +- `FREE_LARGE_CONTEXT` — free option with more context/state capacity; +- `SUBSCRIPTION` — already-paid subscription capacity; +- `PAYG` — usage-priced API, last economic tier. + +The tier label does not assert task suitability. Suitability is measured from +the paired observations. + +## Experimental sequence + +1. enable a second independent executor; +2. qualify executors under decision 0035; +3. run the same frozen task sample across all eligible candidates without + routing or cascade escalation; +4. measure the economic/functional matrix; +5. define the fixed lowest-cost eligible policy as `Economic Baseline v1`; +6. test explicit escalation against that baseline; +7. only then test task/context-based routing; +8. compare every treatment against the fixed baseline and report cost-quality + trade-offs. + +No candidate is silently retried on another tier during steps 3–5. + +## Falsifiable hypotheses + +- H1: at least one lower-cost tier resolves a non-trivial subset of the frozen + task set under the shared verifier; +- H2: a simple explicit escalation rule can improve verified resolution per + unit cost over the lowest-cost baseline without exceeding declared latency + and retry budgets; +- H3: a more complex router must outperform the simple escalation baseline on + predeclared held-out tasks, not merely consume more compute. + +## Required observation matrix + +Each candidate/task/replicate cell records: + +`task → executor → provider → model → tier → marginal cost → context limit → +resolved → patch validity → latency → tokens/context used → calls → retries → +termination reason → blocked/failure class`. + +`executor`, `provider`, `model`, `marginal_cost`, `context_limit`, and +`termination_reason` are separate fields. A provider quota, context limit, or +early termination must not be attributed to executor quality. + +The task revision, verifier identity, acceptance definition, environment, +budget, instrumentation, serial order, and replicate must be frozen and +identical within a comparison block. Missing cost or context usage remains +`UNKNOWN`; it is never inferred as zero. + +## Promotion criteria + +Economic Baseline v1 requires complete comparable cells and raw evidence for +the declared sample. Escalation requires a predeclared signal of insufficiency +and must preserve the original attempt and its evidence. Routing promotion +requires a held-out comparison against the economic baseline with improved +verified resolution per unit cost and no unacceptable regression in latency, +blocked rate, patch validity, or reproducibility. + +Until those criteria are met, this decision is a protocol and hypothesis, not +evidence that any tier or routing policy is effective. diff --git a/docs/decisions/0112-package-boundary-and-distribution.md b/docs/decisions/0112-package-boundary-and-distribution.md new file mode 100644 index 00000000..170f5fe8 --- /dev/null +++ b/docs/decisions/0112-package-boundary-and-distribution.md @@ -0,0 +1,37 @@ +# Decision 0112 — Package boundary and distribution identity + +## Status + +Proposed for G1 validation; not a product-release acceptance decision. + +## Context + +The current audit line contains a usable `arkx` Python package but no build +metadata or clean-install contract. The previous G3 CLI implementation belongs +to a different architecture and cannot be promoted by cherry-pick. A package +boundary is therefore required before a public entrypoint or executor can be +evaluated. + +## Decision + +- The import namespace remains `arkx`. +- The distribution identity is `codepro`, matching the current product name. +- The first packaged version is `0.3.0.dev0`, explicitly non-release. +- The package has no runtime dependencies and requires Python `>=3.12`. +- This decision adds packaging only; it does not add a CLI, provider, model, or + executor. + +## Falsifiable hypothesis and acceptance criteria + +H1: A clean Python 3.13 environment can build and install the distribution +without `PYTHONPATH`, import `arkx`, run the existing foundation check and pass +the complete test suite. + +Acceptance requires a wheel and sdist, SHA-256 hashes, the exact Python and +package versions, raw command output, and an independent acceptance decision. +Any build, import, or test failure is `NO-GO` for G1. + +## Removal condition + +Remove or revise this boundary if the public product identity, supported Python +range, or packaging backend changes; do not silently rename the distribution. diff --git a/docs/decisions/0113-minimal-public-chassis-cli.md b/docs/decisions/0113-minimal-public-chassis-cli.md new file mode 100644 index 00000000..a5a5f75d --- /dev/null +++ b/docs/decisions/0113-minimal-public-chassis-cli.md @@ -0,0 +1,39 @@ +# Decision 0113 — Minimal public chassis CLI + +## Status + +Proposed for G3 pre-executor validation. + +## Context + +The packaged audit line had no public command. The previous fake-executor CLI +was based on a different module graph and could not be cherry-picked. The +current line already has a deterministic no-network baseline fixture, so the +first public boundary should expose only package health and that fixture. + +## Decision + +The `codepro` entrypoint exposes only: + +- `codepro --version`; +- `codepro doctor [--json]`; +- `codepro baseline`. + +It does not submit tasks, choose executors, invoke providers, perform fallback, +or claim product execution readiness. + +## Falsifiable hypothesis and acceptance criteria + +H1: After clean installation from wheel and sdist, the three commands are +available without `PYTHONPATH`; `doctor` reports the supported Python range and +`baseline` emits the existing deterministic PASS record. + +Acceptance requires CLI tests, clean wheel and sdist invocations, exact +version identity, and independent acceptance. Unsupported Python must return a +nonzero status. Any executor invocation or hidden fallback is a failure. + +## Removal condition + +Replace this boundary only when a wider user-facing task contract is defined +and independently accepted; do not expand it implicitly to accommodate an +executor. diff --git a/docs/decisions/0114-mvp-vertical-contract.md b/docs/decisions/0114-mvp-vertical-contract.md new file mode 100644 index 00000000..bc51ad81 --- /dev/null +++ b/docs/decisions/0114-mvp-vertical-contract.md @@ -0,0 +1,60 @@ +# Decision 0114 — MVP vertical operational contract + +## Status + +M0 contract proposal; implementation and executor selection are pending. + +## Goal + +Define the smallest real user journey that can become a private operational +MVP. The journey uses one authorized repository, one task class, one explicitly +selected executor and one acceptance authority. + +## Required input + +The request must identify: + +- task and immutable repository revision; +- authorized workspace and requested scope; +- task instruction; +- acceptance command or declared acceptance authority; +- executor identity and configuration; +- budget and wall-time limit; +- evidence destination and retention policy. + +## Required output + +One immutable execution record must contain: + +- request, plan and executor identities; +- provider/model/sandbox identities when used; +- command/result or structured environment error; +- artifacts and raw output references; +- verification state and evidence; +- acceptance decision and authority; +- terminal state: completed, failed, timeout or blocked. + +No missing identity, missing evidence or blocked infrastructure may become +success. No executor switch, scope expansion or retry may be implicit. + +## Current gap + +The chassi already provides useful neutral contracts for request, budget, +execution result, verification and acceptance. It does not yet expose one +vertical path binding workspace, scope and acceptance commands to a real +execution. `FakeExecutor` remains a contract-test fixture only and cannot +satisfy the MVP. + +## Falsifiable acceptance criteria + +M0 is complete only when a local end-to-end run with one real executor can: + +1. reject an unauthorized scope before execution; +2. execute the declared task in the declared workspace; +3. enforce the declared budget and wall time; +4. persist raw execution and environment evidence atomically; +5. verify the declared acceptance command; +6. return explicit success, failure, timeout and environment-error states; +7. reproduce the same input identity without overwriting the prior attempt. + +Until these criteria pass, the MVP remains `NO-GO`. diff --git a/docs/decisions/0115-neutral-verifier-command-boundary.md b/docs/decisions/0115-neutral-verifier-command-boundary.md new file mode 100644 index 00000000..aae687f5 --- /dev/null +++ b/docs/decisions/0115-neutral-verifier-command-boundary.md @@ -0,0 +1,32 @@ +# Decision 0115 — Neutral verifier command boundary + +## Status + +Implemented as a local contract; not yet the MVP end-to-end path. + +## Decision + +The chassi provides one small verifier boundary that runs a declared argv in an +existing workspace with `shell=False`, a positive timeout, captured stdout and +stderr, exit code, duration and explicit environment errors. A +`CommandVerifier` adapts that observation to the orchestration seam and +`VerificationEvidenceStore` persists the raw observation under the stable +`run_id`/test identity. Equivalent content is idempotent; divergent content +for the same identity is a collision and blocks verification. + +It does not select an executor, interpret provider output, accept a task, or +persist a release decision. The verifier owns raw command evidence; the caller +still owns lifecycle/event-log persistence, independent acceptance and release +promotion. + +## Falsifiable criteria + +- exit code zero becomes `PASSED`; +- nonzero exit becomes `FAILED`; +- timeout, missing executable and invocation errors become `UNKNOWN` with an + explicit error kind; +- invalid command or workspace is rejected before invocation; +- shell interpretation is never enabled. +- raw stdout/stderr and result metadata are persisted atomically; +- a second observation with divergent content for the same run/test identity + becomes `BLOCKED`, never a silent overwrite. diff --git a/docs/decisions/0116-executor-qualification-v1-paired-protocol.md b/docs/decisions/0116-executor-qualification-v1-paired-protocol.md new file mode 100644 index 00000000..39c1bf79 --- /dev/null +++ b/docs/decisions/0116-executor-qualification-v1-paired-protocol.md @@ -0,0 +1,62 @@ +# 0116 — Executor Qualification v1: paired protocol + +## Status + +Contract implemented. Real qualification is `NOT_EXECUTED` until two concrete +executors produce observations under this protocol. + +## Decision + +The first executor comparison is a paired, executor-only experiment. Routing, +fallback, retry policy changes, composition and multi-agent behavior are out of +scope. Every executor receives the same task/revision/acceptance definition, +model, environment, budget, treatment, verifier identity and instrumentation +identity. + +The harness validates comparability before analysis. It does not rank, select, +accept or promote an executor. + +## Falsifiable hypothesis + +Two independently implemented executors can pass through the same Codepro +execution, verification and evidence pipeline without executor-specific +branches, while preserving comparable observations for every task/replicate +cell. + +## Acceptance criteria + +- at least two distinct executor identities are present; +- every `(task, revision, acceptance definition, replicate)` cell has one trial + for each executor; +- no control dimension differs inside a paired cell; +- verifier and instrumentation identities are present and equal by experiment; +- raw evidence and terminal outcomes exist for every trial; +- `BLOCKED`, `UNKNOWN`, incomplete cells and control mismatches are retained as + non-passing states; +- the report contains resolution, patch validity, cost, wall time, tokens, + invocations, retries, failures and blocked rates where observed; +- no routing or fallback is invoked. + +Passing these criteria makes the observations comparable. It does not prove +that an executor is better, generally reliable, or ready for promotion. + +## Required run record + +The experiment must freeze: + +1. task-set manifest and repository revisions; +2. executor identities and configuration digests; +3. model/provider identities, if applicable; +4. environment and sandbox identity; +5. budget and invocation limits; +6. treatment identity; +7. verifier and instrumentation identities; +8. serial execution order and replicate identifiers; +9. raw execution, verification and acceptance evidence; +10. independent acceptance decision. + +## Explicit non-goals + +This decision does not authorize a real provider, executor, external CLI, +benchmark publication, routing treatment or release promotion. Those require a +later decision backed by executed evidence. diff --git a/docs/decisions/0146-qualification-identity-and-controls.md b/docs/decisions/0146-qualification-identity-and-controls.md new file mode 100644 index 00000000..ed0da14f --- /dev/null +++ b/docs/decisions/0146-qualification-identity-and-controls.md @@ -0,0 +1,54 @@ +# Decision 0146 — Qualification identity and verifier controls + +Date: 2026-09-24 +Status: accepted as documentation policy +Classification: `PROTOCOL_STRENGTHENING` + +## Decision + +Codepro adopts four explicit identity levels for empirical work: + +```text +experiment_id = protocol and scope +trial_id = frozen comparison cell +attempt_id = one execution attempt +verifier_run_id = one verifier cache namespace for one control/patch attempt +``` + +Stable identity applies to equivalent frozen protocol inputs. A verifier run +ID must still be unique across distinct controls, patches and retries because +benchmark harnesses may cache by run identity rather than by diff content. + +Every qualification manifest must index dataset/task, environment, executor, +provider/model, configuration, verifier, budget, treatment, control role, +provider-called status, raw artifacts, and independent acceptance. Missing +identity or raw evidence remains an explicit non-success state. + +Provider-free verifier qualification requires a paired control: + +```text +no-op/empty patch -> expected failure +gold/oracle patch -> expected resolution +``` + +The controls qualify only the declared verifier and environment authority. +They do not qualify an agent, provider, model, treatment, or product release. + +## Rationale and acceptance criteria + +The current repository already separates execution, verification, acceptance +and promotion, but identity requirements were distributed across manifests, +ADRs and logs. The update makes the comparison unit and verifier cache boundary +explicit without adding a runtime service or changing executor behavior. + +This documentation decision is accepted when: + +- `AGENTS.md`, the experimental protocol and architecture page state the same + identity levels and control rule; +- future qualification records can locate raw evidence from one manifest; +- no-op and gold controls are represented as required states, not inferred + from a candidate result; +- local tests and benchmark results remain clearly separated from external + authority and promotion. + +No provider, executor, model, or treatment is promoted by this decision. diff --git a/docs/decisions/0147-agent-local-workspace-and-pr-policy.md b/docs/decisions/0147-agent-local-workspace-and-pr-policy.md new file mode 100644 index 00000000..9ce41842 --- /dev/null +++ b/docs/decisions/0147-agent-local-workspace-and-pr-policy.md @@ -0,0 +1,36 @@ +# Decision 0147 — Agent local workspace and PR policy + +Date: 2026-09-24 +Status: accepted as execution policy +Classification: `AGENT_WORKFLOW_BOUNDARY` + +## Decision + +The Codepro execution agent defaults to patch-first work in one task-scoped +workspace and one attempt. Commits, branches, pull requests, merges and +promotion are separate authorized transitions, not automatic consequences of +editing files or passing a local test. + +Retries receive new attempt/verifier identities and preserve prior artifacts. +An existing branch or PR for the same task is updated when authorized; the +agent does not create duplicate PRs for retries or additional evidence. + +The agent records initial workspace state, preserves unrelated changes, and +reports verification and acceptance separately from Git artifacts. + +## Rationale + +Agent harnesses commonly separate task instruction, environment, verifier and +oracle/reference solution. SWE-agent also exposes local patch application and +PR opening as distinct options. The policy makes that boundary explicit for +Codepro's local repository workflow and prevents PR accumulation from being +mistaken for task progress or acceptance. + +## Acceptance criteria + +- The workflow is referenced from `AGENTS.md`. +- Default execution produces a patch/evidence record without requiring a + commit or PR. +- Retry, commit and PR states remain distinct from verification and acceptance. +- Duplicate PR creation is prohibited unless explicitly authorized. +- No rule authorizes merge, promotion, destructive cleanup or scope expansion. diff --git a/docs/decisions/0148-pr30-decomposition-and-conflict-boundary.md b/docs/decisions/0148-pr30-decomposition-and-conflict-boundary.md new file mode 100644 index 00000000..c82fcf5f --- /dev/null +++ b/docs/decisions/0148-pr30-decomposition-and-conflict-boundary.md @@ -0,0 +1,149 @@ +# Decision 0148 — PR30 decomposition and conflict boundary + +Date: 2026-09-24; compatibility-gate update: 2026-09-25 + +## Decision + +PR30 must not be merged as a single unit. Its current branch replaces several +foundational `main` contracts and cannot be reconciled safely by choosing one +side of the merge or by concatenating both APIs. + +The integration path is decomposed into independently testable tranches. A +tranche may advance only when its imports, tests, and public contract are +compatible with the current `main` baseline. + +## Evidence + +- A merge simulation from `86b3822499de186a779e9a3887285d92f60fafea` produced + 31 conflicts. +- Conflicts included the package boundary, CLI, contracts, acceptance, + planning, promotion, recovery, routing, telemetry, verification, tests, + documentation, and CI workflow. +- The PR30 side replaces or removes current `main` modules including + `command`, `environment`, `governance`, `project`, `qualification`, + `provenance`, `spine`, `study`, `treatment`, `validity`, and `workload`. +- Keeping `main`'s foundational modules allowed 109 focused tests to pass. +- The remaining focused write test was blocked by Windows temporary-directory + permissions, not by a product assertion. +- The combined suite also exposed PR30-only consumers importing contracts that + no longer exist in `main` (`AcceptancePolicy`, `PromotionDecision`, + `RecoveryPlan`, and replay/event-log dependencies). +- Unrestricted pytest discovery collected historical `logs/` text artifacts; + this is a harness/discovery issue, not product evidence. + +## Integration tranches + +1. **Foundation compatibility gate** — keep `main`'s CLI, command/environment, + governance, project, qualification, promotion, recovery, and verification + contracts authoritative. No PR30 implementation enters this tranche without + an explicit adapter or contract migration. +2. **Passive evidence modules** — evaluate PR30's evidence/provenance and + read-only records that do not import replaced contracts. +3. **Execution/harness modules** — evaluate `execution`, `harness`, + `integration`, and `orchestration` together only after their identities, + lifecycle, and verifier contracts are mapped to `main`. +4. **P82/benchmark experiments** — keep experimental modules and raw evidence + outside the product core until the preceding tranche has an accepted + adapter and provider-free tests. +5. **Historical documentation/logs** — preserve as evidence, but do not let + them participate in test discovery or become implicit product APIs. + +## Non-decisions + +This record does not delete PR30 files, choose a new executor, promote any +experiment, or authorize a commit/push. The isolated merge was aborted after +the evidence was collected; the main checkout was not changed by the merge. + +## Next gate + +The next block is the **Foundation compatibility gate**: enumerate the exact +PR30 consumers of each replaced `main` contract and select the first passive +module that can be tested without importing an incompatible contract. If no +such module exists, classify the tranche as blocked rather than creating a +compatibility facade without tests. + +## Foundation compatibility result + +The symbol audit divides the shared modules into two groups: + +### Additive candidates + +- `contracts.py` — same core public types; PR30 adds behavior that must be + checked against `main` validation. +- `telemetry.py` — public surface is equivalent; compare serialization and + timestamp behavior. +- `handoff.py`, `planning.py`, `routing.py`, and `verification.py` — mostly + additive or validation differences; eligible for an isolated adapter/merge + test, one module at a time. + +### Semantic conflicts + +- `acceptance.py` — independent acceptance authority in `main` versus the + outcome-policy API in PR30. +- `promotion.py` — frozen promotion gates in `main` versus persisted outcome + promotion decisions in PR30. +- `recovery.py` — request/controller lifecycle in `main` versus recovery-plan + and attempt-lineage lifecycle in PR30. +- `cli.py` — read-only inspection/doctor CLI in `main` versus baseline fixture + CLI in PR30. + +The next executable tranche is therefore `contracts.py` only. It must preserve +the `main` validation contract and demonstrate any PR30 additions with focused +tests before `telemetry.py` or the other additive candidates are considered. + +## Tranche result + +`contracts.py` is **not accepted as an integration candidate** in its current +PR30 form: its diff removes `main`'s constructor, timestamp, numeric, JSON, and +terminal-record validation. A passing happy-path test is insufficient evidence +for that regression. + +The first currently viable tranche is instead the **pure contract tranche**: +`composition.py`, `configuration.py`, `outcomes.py`, `p82_editing.py`, +`p82_localization.py`, and `executor_qualification.py`. These modules have no +provider, filesystem, CLI, or runtime side effects; their only internal edge +is `executor_qualification -> composition`. Their focused suite passed 43/43 +on Python 3.13 in the PR30 checkout. This is local evidence only; integration +against a clean `main` checkout remains the next required check. + +The clean-`main` integration check copied only those six modules, their two +referenced fixtures, and their focused tests into a detached worktree at +`origin/main`. The result was 41/41 passed on Python 3.13. The separate +`test_core_outcomes.py` was not counted because it imports `p82_baseline`, which +belongs to a later execution tranche. + +The isolated telemetry gate rejected the PR30 implementation: the canonical +`main` telemetry suite ran 16 tests successfully but failed 4 validation tests +for blank evidence, duplicate task boundaries, events after completion, and +regressive timestamps. The `main` telemetry implementation remains authoritative; +the PR30 version is not an additive candidate in its current form. + +The isolated handoff gate also rejected the PR30 implementation: 8 canonical +tests passed and 6 failed on identity/measurement validation, schema version, +duplicate identifiers, and repeated transition accounting. The `main` handoff +contract remains authoritative. + +The planning gate could not collect its canonical suite because the PR30 +version removes `build_replan_result`, which is part of the current `main` +contract. This is classified as an API incompatibility before behavioral +testing, not as a passing or failing planning result. + +The routing gate was also not isolable: importing the PR30 implementation +requires `arkx.integration`, which belongs to the later PR30 execution/harness +architecture and is absent from the `main` baseline. It is therefore deferred +with that architecture tranche rather than pulled in as a hidden dependency. + +## Execution/harness dependency gate + +The following cluster is not file-isolable because its internal edges cross +the semantic conflicts above: + +`configuration -> execution -> harness/integration -> orchestration`, with +`event_log`, `verifier`, `request`, `selection`, `routing`, `promotion`, and +`recovery` as additional contract edges. `p82_baseline` adds further +dependencies on submission and timeout contracts. + +This cluster is classified `DEFERRED_ADAPTER_REQUIRED`. Bringing it into +`main` requires one explicit compatibility design for lifecycle, acceptance, +promotion, and recovery identities; copying the files together would merely +hide the conflict and would not constitute qualification. diff --git a/docs/decisions/0149-execution-harness-adapter-boundary.md b/docs/decisions/0149-execution-harness-adapter-boundary.md new file mode 100644 index 00000000..ceb7986c --- /dev/null +++ b/docs/decisions/0149-execution-harness-adapter-boundary.md @@ -0,0 +1,52 @@ +# Decision 0149 — Execution/harness adapter boundary + +Date: 2026-09-25 + +## Result + +The PR30 execution/harness cluster cannot be integrated into `main` through a +mechanical adapter at the current boundary. + +`main`'s governed spine accepts a bound executor invoker that returns a +`CommandResult`: an observation of one authorized command, including process +status, stdout/stderr, timeout, and environment error. PR30's executor contract +returns an `ExecutionResult`: a lifecycle-classified run with `RunState`, +artifact references, usage, and a harness/provider/executor error envelope. + +Those results have different authority and semantics. A conversion from +`ExecutionResult` to `CommandResult` would have to invent an argv, cwd, +process outcome, or command observation. A conversion in the other direction +would have to invent task completion, artifacts, usage, or executor state. +Neither is a valid adapter without a declared execution authority. + +## Classification + +`ADAPTER_CONTRACT_UNDEFINED` + +This is an architectural decision boundary, not a runtime or test failure. +No provider was called and no product behavior was changed. + +## Evidence + +- `main` boundary: `src/arkx/spine.py::BoundExecutorInvoker.invoke` returns + `CommandResult`. +- `main` command contract: `src/arkx/command.py::CommandResult` requires + argv, cwd, exit code, stdout/stderr, timeout state, duration, and environment + error semantics. +- PR30 boundary: `src/arkx/execution.py::Executor.execute` returns + `ExecutionResult`. +- PR30 result contract: `ExecutionResult` carries `RunState`, artifact refs, + usage, and an `ErrorEnvelope`, but no command/process observation. + +## Allowed next decisions + +1. Make `main`'s governed spine the authority and port PR30 execution to a + concrete `BoundExecutorInvoker` that produces real `CommandResult` values; + or +2. Make PR30's execution contract the authority and add an explicit governed + adapter layer that records command observations before producing + `ExecutionResult`. + +Until one authority is selected, the execution/harness cluster remains +`DEFERRED_ADAPTER_REQUIRED`. No routing, provider, benchmark, or promotion +work should proceed through this unresolved boundary. diff --git a/docs/decisions/0150-pr30-execution-cluster-disposition.md b/docs/decisions/0150-pr30-execution-cluster-disposition.md new file mode 100644 index 00000000..1b7cd90c --- /dev/null +++ b/docs/decisions/0150-pr30-execution-cluster-disposition.md @@ -0,0 +1,39 @@ +# Decision 0150 — PR30 execution cluster disposition + +Date: 2026-09-25 + +## Decision + +For this integration, the current `main` governed execution spine remains the +authority. The PR30 execution/harness cluster is **not integrated** because no +lossless adapter exists between its `ExecutionResult` lifecycle contract and +`main`'s `CommandResult` observation contract. + +This is a disposition of the stale PR30 cluster, not a claim that either +contract is universally superior. The choice follows the repository's current +baseline and avoids replacing an accepted authority with an unqualified, +semantically different runtime. + +## Scope retained + +The pure, provider-free contract tranche remains eligible for a separate +change: `composition`, `configuration`, `outcomes`, `p82_editing`, +`p82_localization`, and `executor_qualification`, with their referenced +fixtures. It passed 41/41 focused tests in a clean `origin/main` worktree. + +## Scope excluded + +Do not integrate the PR30 versions of `contracts`, `telemetry`, `handoff`, +`planning`, `routing`, `acceptance`, `promotion`, `recovery`, `execution`, +`harness`, `integration`, `orchestration`, `event_log`, `verifier`, or +`p82_baseline` through this merge effort. Several regress validation or depend +on the unresolved execution cluster. + +## Evidence limitation + +The native `main` provider-free suite ran in a clean worktree with 71 passed, +1 skipped, and 7 Windows temporary-directory permission failures. The latter +are environment evidence and do not change the architectural disposition. + +No provider was called, no product code was changed, and no promotion or +commit/push was authorized by this decision. diff --git a/docs/decisions/0151-pr30-non-code-disposition.md b/docs/decisions/0151-pr30-non-code-disposition.md new file mode 100644 index 00000000..1bbc00e0 --- /dev/null +++ b/docs/decisions/0151-pr30-non-code-disposition.md @@ -0,0 +1,64 @@ +# Decision 0151 — PR30 non-code disposition + +Date: 2026-09-25 + +## Inventory + +The PR30 delta contains 127 documentation files, 228 log files, and 12 +experiment fixtures relative to `origin/main`. The logs include raw runs, +workspaces, and at least one nested repository cache with its own `.git` +objects. These are not interchangeable with product source files. + +## Classification + +- Decision records, protocol pages, and release/qualification plans: + `HISTORICAL_NORMATIVE_CANDIDATE`. Preserve until each is mapped to the + current authority; duplicate or superseded records must not silently become + current policy. +- JSON experiment fixtures: + `EXPERIMENT_INPUT_CANDIDATE`. Keep only when a current test or protocol + references them; otherwise classify as historical. +- Raw JSON/log/trajectory/report artifacts: + `HISTORICAL_EVIDENCE`. Preserve provenance and keep them outside test + discovery. +- Nested workspaces, repository caches, and copied source trees under `logs/`: + `ARCHIVE_ONLY`. They are evidence payloads, not runtime inputs or package + data; their size and nested repositories require an explicit archival/removal + decision before any deletion. + +## Decision + +No non-code artifact is deleted in this block. The only safe immediate action +is classification and discovery isolation. The previous deletion of the +explicitly identified mini-adaptation helper files remains separate and is not +expanded to historical evidence. + +## Next gate + +The fixture/reference audit is complete. Historical logs remain unchanged; +their retention or archival requires an exact path set and owner. + +## Fixture audit result + +All 12 added JSON fixtures parsed successfully. Nine are referenced directly +by tests or by another current protocol artifact. Three have no active +reference detected outside themselves: `cost-tier-comparison-protocol.json`, +`p82-block3-t1-functional-v2.json`, and `p82-wave0-comparison-protocol.json`. +They are classified `HISTORICAL_UNREFERENCED_CANDIDATE`, not deleted. + +## Decision-record audit + +All decision Markdown files have no broken local references under the checked +`docs/decisions/` and `logs/architecture/` paths. The six duplicate IDs were +normalized in an isolated worktree and then applied here: + +- 0031 package boundary → 0112 +- 0032 minimal CLI → 0113 +- 0033 MVP vertical contract → 0114 +- 0034 neutral verifier boundary → 0115 +- 0035 executor qualification protocol → 0116 +- 0037 second-executor enablement → 0152 + +The earlier lineage/persistence records retain their IDs. After the change, +duplicate numeric IDs are zero and old filename references are zero. No +document content was deleted. diff --git a/docs/decisions/0152-second-executor-enablement.md b/docs/decisions/0152-second-executor-enablement.md new file mode 100644 index 00000000..9263f35c --- /dev/null +++ b/docs/decisions/0152-second-executor-enablement.md @@ -0,0 +1,100 @@ +# 0152 — Second Executor Enablement + +## Status + +`BLOCKED`: a qualification paired run cannot start until a second concrete +executor is available and independently identified. + +## Decision + +The next work item is infrastructure enablement, not routing, composition, +fallback, ranking, or another qualification abstraction. The project must make +one additional task-solving executor runnable through the existing neutral +contract and verifier. + +Changing the Ollama model, provider, or runtime does not satisfy this decision +when the task is still executed by the same `mini-swe-agent` implementation. +Those are treatment/provider variables and must remain labeled as such. + +## Falsifiable hypothesis + +An independently implemented executor can execute the same frozen task, +revision, acceptance definition, budget, verifier, and instrumentation as the +existing executor, without executor-specific branches or hidden fallback. + +## Required enablement evidence + +The second executor is not enabled until all of the following are recorded: + +1. stable executor identity, implementation/version, configuration digest, and + runtime identity; +2. headless invocation with a bounded timeout and explicit exit status; +3. extracted patch/artefact and raw stdout/stderr; +4. observable budget, calls/tokens where supported, and wall time; +5. the same verifier and acceptance boundary used by executor one; +6. an isolated smoke task that proves the complete execution → verification → + acceptance path; +7. no fallback, executor switch, or executor-specific treatment branch; +8. independent acceptance of the preflight evidence. + +## Current preflight + +The latest read-only collection found: + +- `mini-swe-agent`: installed and usable as the existing P82 execution path; +- Ollama `0.34.3`: available as a provider/runtime only; not a second + executor; +- Git Bash: `BLOCKED` by executable/access failure; +- Docker `29.7.2`: CLI present, daemon access `BLOCKED` by permission denied; +- OpenRouter boundary: `UNKNOWN` because provider/configuration identity is not + frozen for the current run. + +Additional installed candidates were checked on 2026-09-24. Claude Code was +blocked by missing login, OpenCode was blocked while spawning its server, and +Aider was blocked by local application-control policy. Codex was not counted +because it is the controlling agent, not an independent comparator. Raw output +is preserved in +`logs/architecture/second-executor-preflight-20260924.json`; the attempted +qualification is recorded as `BLOCKED` in +`logs/qualification/executor-qualification-v1-20260924.json`. + +The dedicated OpenCode audit found version `2.0.15`, a documented headless +`run` entrypoint, model syntax `provider/model#variant`, and JSON output via +`--format json`. However, the installed CLI rejects `--standalone` when used +with `run`, while the default and isolated invocations fail before execution +with `EPERM`. Detailed observations are preserved in +`logs/architecture/opencode-headless-audit-20260924.json`. + +OpenHands CLI was also audited as an independent candidate. Version `1.21.0` +exposes `--headless --json`, explicit local workspace/persistence overrides, +and `LLM_BASE_URL`/`LLM_MODEL` overrides. It initialized against local Ollama +`qwen2.5-coder:3b`, but failed before the first task cycle while creating its +Jinja cache under the protected user profile. It remains a promising candidate +but is `BLOCKED_RUNTIME`, not qualified. Raw observations are in +`logs/architecture/openhands-headless-audit-20260924.json`. + +The installed source was checked for a cache override. Persistence and +workspace directories are configurable, but the Jinja and several SDK caches +are constructed directly from `Path.home()/.openhands`; no supported CLI cache +directory setting was found. A process-wide profile override or package +monkeypatch is intentionally not part of this qualification. The remaining +blocker therefore requires a writable execution profile supplied by the +environment, not a protocol change. + +Antigravity/Google CLI was checked locally on 2026-09-24. No matching process, +PATH command, desktop shortcut, or Start Menu entry was discovered, and the +Windows UI automation surface was unavailable in this session. A logged-in +browser or desktop state, without an observable headless invocation and +artifact boundary, is not executor evidence. The candidate remains +`NOT_DISCOVERED`; raw checks are in +`logs/architecture/antigravity-preflight-20260924.json`. + +These observations are infrastructure evidence. They do not qualify a second +executor and do not authorize a paired benchmark. + +## Exit criteria + +When the evidence above exists, run `Executor Qualification v1` unchanged: +same frozen task set, revisions, verifier, instrumentation, budget, +replicates, and serial order. Only after that run produces complete comparable +cells may `Baseline v1` begin. Routing remains out of scope until then. diff --git a/docs/decisions/0153-historical-log-retention-audit.md b/docs/decisions/0153-historical-log-retention-audit.md new file mode 100644 index 00000000..bac087c4 --- /dev/null +++ b/docs/decisions/0153-historical-log-retention-audit.md @@ -0,0 +1,29 @@ +# Decision 0153 — Historical log retention audit + +Date: 2026-09-25 + +## Evidence + +The current `logs/` tree contains 29,101 files and approximately 569 MB. Of +that payload, copied workspaces account for approximately 453 MB and nested +repository `.git` objects account for approximately 80 MB. The largest roots +are the P82 treatment runs and mini runtime replay workspaces. + +## Classification + +These payloads are `ARCHIVE_ONLY` evidence. They must not be imported as +package data, discovered as tests, or treated as current product behavior. +Their raw contents are preserved for provenance and are not normalized in this +block. + +Only 228 log paths are tracked by Git, totaling approximately 639 KB. The +large workspace and nested-cache payload is local-only and is not part of the +remote PR surface. This removes the need for a Git-history cleanup in this +block, but does not authorize deleting local evidence. + +## Decision + +No deletion, compression, relocation, or rewriting is performed. Any future +retention change must name an exact root, preserve a manifest/hash, and be +reviewed as a separate destructive/archival block. The current safe action is +test-discovery isolation and explicit evidence classification. diff --git a/docs/decisions/0154-test-discovery-boundary.md b/docs/decisions/0154-test-discovery-boundary.md new file mode 100644 index 00000000..ae2429a9 --- /dev/null +++ b/docs/decisions/0154-test-discovery-boundary.md @@ -0,0 +1,20 @@ +# Decision 0154 — Test discovery boundary + +Date: 2026-09-25 + +## Result + +The current pytest configuration collects only the product test tree. A clean +collection produced exactly 400 test items, all under `tests/`; no historical +`logs/` file, copied workspace, nested repository, or experiment JSON was +collected as a test. + +The complete provider-free suite then passed 400/400 on Python 3.13 in 3.00s, +with 3 warnings and no failures. The run used `-p no:cacheprovider` to avoid +creating host cache artifacts. This is local suite evidence only; it is not +provider, benchmark, or executor qualification evidence. + +## Decision + +The discovery boundary is accepted for the current checkout. Historical log +retention remains a separate concern; no logs or workspaces were deleted. diff --git a/docs/decisions/0170-codepro-canonical-namespace-migration.md b/docs/decisions/0170-codepro-canonical-namespace-migration.md new file mode 100644 index 00000000..0ddedb42 --- /dev/null +++ b/docs/decisions/0170-codepro-canonical-namespace-migration.md @@ -0,0 +1,56 @@ +# Decision 0170 — CodePro canonical Python namespace migration + +## Status + +Accepted for staged implementation; the namespace facade and packaging slice +are implemented locally but not released. + +## Decision + +`codepro` is the canonical product and Python package namespace. `arkx` is +retained temporarily as a compatibility shim and must not be treated as a +separate product identity. + +The migration is staged: + +1. implement the canonical package under `codepro`; +2. update internal imports, CLI entry points, packaging metadata, and current + documentation to use `codepro`; +3. retain `arkx` re-exports for the declared compatibility window; +4. test equivalence of the supported public API through both namespaces; +5. deprecate and remove `arkx` only in a separately authorized release. + +No implementation or evidence is promoted merely because this decision exists. + +## Rationale + +The product identity is CodePro, while the current Python namespace is `arkx`. +An atomic rename would break imports, dynamic module references, serialized +objects, manifests, historical evidence, and external scripts without proving +that all consumers had migrated. A compatibility shim preserves continuity +while making the intended namespace explicit. + +## Acceptance criteria + +- `import codepro` exposes the canonical public API; +- `codepro.__all__` is explicit and tested; +- supported `arkx` imports resolve to the corresponding `codepro` objects; +- `codepro` and `arkx` do not silently expose different behavior; +- the CLI and wheel identify the distribution as `codepro`; +- no claim of completed migration is made until installed-package and source + import tests pass at the same revision; +- removal of the `arkx` shim requires a later decision and release boundary. + +## Non-goals + +- no redesign of the execution chassis; +- no provider, model, executor, or benchmark qualification; +- no deletion of historical `arkx` evidence; +- no automatic removal of legacy imports. + +## Current evidence + +The current checkout still has `arkx` as the implementation namespace, while +`codepro` now provides the canonical facade and packaging entry point. The +installed-package and full compatibility gates remain open until a wheel is +built and tested in a clean environment. diff --git a/docs/executor-qualification.md b/docs/executor-qualification.md new file mode 100644 index 00000000..8f86d99c --- /dev/null +++ b/docs/executor-qualification.md @@ -0,0 +1,18 @@ +# P8.1 executor qualification contracts + +P8.1 defines an experimental boundary for describing executors and recording external observations. It does not integrate, execute, select, rank, accept, promote, or invoke an executor. + +`ExecutorIdentity` and advertised capabilities are metadata. They are not evidence of reliable capability. `ExecutorObservation` preserves acceptance, patch verification, costs, time, retries, replans, handoffs, regressions, and other measurements exactly as supplied; unknown values remain `None` and explicit zero remains zero. + +Comparisons use an explicit axis. `EXECUTOR` requires the same task revision, acceptance definition, model, environment, budget, and P7 treatment while varying executor. `TREATMENT` requires the same task, executor, model, environment, and budget while varying P7 treatment. Divergent controls are `INCOMPARABLE`; missing control evidence is `UNKNOWN`. + +`COMPLETED` with `accepted=False` is a valid observed outcome. Infrastructure, executor, budget, and blocked outcomes are not silently converted into rejection. P8 records external acceptance; it never infers acceptance from patch verification or execution completion. + +Summaries aggregate dimensions without producing a winner or global ranking. Promotion and removal decisions remain outside P8. + +Decision `0035` adds the first executable paired-protocol gate. It requires at +least two executor identities, complete task/replicate cells, shared verifier +and instrumentation identities, and matching control dimensions before a +comparison can be called `QUALIFIABLE`. This is a comparability gate only; it +does not rank or promote an executor. The real protocol remains +`NOT_EXECUTED` until concrete executors produce raw observations. diff --git a/docs/experimental-composition-harness.md b/docs/experimental-composition-harness.md new file mode 100644 index 00000000..29ff52f0 --- /dev/null +++ b/docs/experimental-composition-harness.md @@ -0,0 +1,9 @@ +# P7 experimental composition harness + +P7 defines removable treatment descriptions and records externally supplied observations. It does not execute P1–P6, select an executor, retry, replan, route, accept, promote, or produce `PASS`. + +Each treatment contains a canonical immutable tuple of enabled mechanisms. Mechanisms are serialized in the fixed order P1 through P6, regardless of input order. The initial fixture contains only A–F; adaptive composition G is deliberately absent until an adaptive mechanism exists to qualify. + +Trials are comparable only when task, executor, model, environment, and budget are all known and equal. Divergence is `INCOMPARABLE`; missing identity evidence is `UNKNOWN`. Treatment composition is recorded separately and is not executed by P7. + +Summaries count supplied measurement observations and preserve absence. They never impute zero or false, and they do not represent acceptance or promotion. diff --git a/docs/experimental-protocol.md b/docs/experimental-protocol.md index ee17ed8c..b982a087 100644 --- a/docs/experimental-protocol.md +++ b/docs/experimental-protocol.md @@ -1,6 +1,7 @@ # Experimental protocol -Arkx develops through small, empirical increments. The protocol below is the minimum record required before a capability can be considered for promotion. +Codepro develops through empirical increments. The protocol below defines the +minimum record required before a capability can be considered for promotion. ## Required record @@ -11,13 +12,73 @@ Each experiment records: - `scope`: exact systems, inputs, and exclusions; - `implementation`: what changed, if anything; - `executor`: the exact executor and version/configuration; +- `identity`: resolved dataset revision or fingerprint, task IDs and base + revisions, environment image/digest and OS/architecture when applicable, + verifier identity, provider/model identity, and Codepro/runner revision; +- `attempts`: explicit `experiment_id`, `trial_id`, `attempt_id`, and verifier + `run_id` values. Equivalent frozen trials may share a stable trial identity, + but distinct patches, controls, or retries must not reuse a verifier run ID; - `procedure`: reproducible steps; - `expected_signal`: measurable success and failure criteria; +- `controls`: declared negative/no-op and positive/gold controls, with their + expected states and raw evidence paths; - `result`: `PASS`, `FAIL`, `BLOCKED`, or `NOT_EXECUTED`; - `evidence`: links or paths to raw evidence and environment facts; +- `provider_called`: explicit boolean or `UNKNOWN` when it cannot be + established; - `acceptance`: explicit reviewer decision and rationale; - `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. +The experiment record is the canonical machine-readable index for these +facts. It must reference raw logs, diffs, trajectories, verifier output and +environment captures; it does not replace them. Missing identity or missing +raw evidence makes the relevant claim `BLOCKED`, `UNKNOWN`, or +`NOT_EXECUTED`, according to the declared protocol. + +## Identity and verifier controls + +Use four identity levels: + +```text +experiment_id = protocol and scope +trial_id = frozen comparison cell +attempt_id = one execution attempt +verifier_run_id = one verifier cache namespace for one control/patch attempt +``` + +For provider-free verifier qualification, execute the controls serially with +the same task, base revision, environment and verifier: + +```text +no-op/empty patch -> expected failure +gold/oracle patch -> expected resolution +candidate patch -> observed result, if a candidate exists +``` + +The gold control qualifies only the stated verifier/environment authority. It +does not qualify a provider, model, agent, treatment, or general task-solving +capability. + +P7 composition trials use their own comparability namespace: `COMPARABLE`, `INCOMPARABLE`, and `UNKNOWN`. P7 records externally supplied measurements and does not convert them into acceptance results. + +P8.1 adds executor qualification records with explicit `EXECUTOR` and `TREATMENT` comparison axes. P8.1 records externally supplied execution outcomes and acceptance observations; it does not execute an executor, infer acceptance, rank executors, or promote components. + +P8.2 defines the minimal mechanism trial as a declarative A/B/C/D plan over a +fixed executor: baseline, safe editor, enhanced repository context, and their +combination. Its harness freezes task, replicate, executor, model, +environment, budget, verification, and independent acceptance authority. It +does not invoke an executor or mechanism and does not manufacture acceptance; +results remain P8.1 observations. Missing controls and measurements remain +`UNKNOWN`, and the only intended treatment difference is the declared arm. + +P8.2a adds the Treatment A baseline runner. Each attempt is serial, refuses to +overwrite an existing attempt directory, derives a stable run identity from +the task and frozen execution configuration, and writes an `execution.json` +artifact atomically alongside the log, diff, and trajectory. The artifact +records execution only; verification, independent acceptance, and promotion +remain separate records. A missing provider, model, mini version, or Python +executable blocks the attempt instead of selecting a fallback. + ## State separation The following are distinct claims and must not be inferred from one another: @@ -44,14 +105,29 @@ id: EXP-YYYY-MM-DD-name hypothesis: "..." scope: "..." implementation: "..." +identity: + codepro_revision: "..." + dataset_revision_or_fingerprint: "..." + task_ids: [] + base_revisions: [] + environment_image_digest: "..." + verifier: "..." +attempts: + experiment_id: "..." + trial_id: "..." + attempt_id: "..." + verifier_run_id: "..." executor: "..." procedure: "..." expected_signal: "..." +controls: + negative_noop: {expected: "FAIL", evidence: []} + positive_gold: {expected: "RESOLVED", evidence: []} result: NOT_EXECUTED evidence: local: [] upstream: [] +provider_called: UNKNOWN acceptance: "PENDING" promotion: NOT_PROMOTED ``` - diff --git a/docs/handoff-context-accounting.md b/docs/handoff-context-accounting.md new file mode 100644 index 00000000..01364c18 --- /dev/null +++ b/docs/handoff-context-accounting.md @@ -0,0 +1,10 @@ +# P6 handoff / context accounting + +P6 records explicit handoffs and summarizes observable context cost and loss. Unknown byte counts, duplicate counts, and lost-information declarations remain `null`, not zero. + +Executor transitions are derived only when both source and target are known. Duplicate metrics are explicit structural inputs; no semantic similarity model is used. `lost_information=[]` means explicitly declared no loss, while `null` means unknown. + +`WITHIN_BUDGET`, `BUDGET_EXCEEDED`, and `UNKNOWN` are observations. P6 does not block execution, invoke a chain, or implement orchestration. + +The fixture is `HANDOFF_ACCOUNTING_FIXTURE`, not a performance benchmark. + diff --git a/docs/integration-readiness.md b/docs/integration-readiness.md new file mode 100644 index 00000000..a25ef04d --- /dev/null +++ b/docs/integration-readiness.md @@ -0,0 +1,71 @@ +# Codepro — external integration readiness + +Data da leitura: 2026-09-23 +Branch: `codex/auditoriaarquitetural` +Python: `3.11.9` + +## Regra de interpretação + +`PASS` nesta matriz significa que o boundary ou preflight local foi observado. +Não significa qualificação de executor, provider, modelo ou benchmark. Logs +históricos são raw evidence e permanecem vinculados às identidades que +registraram; não são reutilizados como prova de uma configuração diferente. + +## Readiness matrix + +| Boundary | Estado atual | Evidência | Principal blocker | Decisão | +|---|---|---|---|---| +| `Executor` protocol | READY_FOR_FAKE_CONTRACT | 139 focused tests; fake executor lifecycle | integration identity absent | usable for contract work | +| `Provider` protocol | READY_FOR_FAKE_CONTRACT | provider request/response tests | no provider adapter in `src/arkx` | usable for contract work | +| `Sandbox` protocol | READY_FOR_FAKE_CONTRACT | fake sandbox tests | concrete sandbox remains external | usable for contract work | +| P8 baseline adapter | FUNCTIONAL_BUT_UNQUALIFIED | baseline adapter and artifact tests | real runner dependency/runtime | adapter may be tested, not promoted | +| OpenRouter adapter | BLOCKED | source in `tools/p82_openrouter_model.py`; historical raw logs | `minisweagent` import unavailable | no current qualification | +| Git Bash sandbox | BLOCKED | `bash.EXE` detected | Windows alias returned `E_ACCESSDENIED`; concrete adapter imports `minisweagent` | no sandbox qualification | +| Docker evaluator | BLOCKED | Docker CLI `29.7.2` present | daemon denied access to `dockerDesktopLinuxEngine` | no SWE-bench qualification | +| Ollama runtime | AVAILABLE_RUNTIME_ONLY | Ollama `0.34.3` detected; prior local logs exist | model/config/adapter identity not frozen in current run | not provider-qualified | +| SWE-bench authority | RECORDED_EXTERNAL_AUTHORITY | existing Gold records under `logs/evaluation/` | authority result does not qualify mechanism/provider | authority only | +| full test suite | PASS_LOCAL | 298 tests pass, including adapter import boundaries | concrete runtimes remain unavailable or unidentified | no external qualification claim | + +## Boundary leakage + +- `src/arkx` exposes neutral `Executor`, `Provider`, and `Sandbox` protocols. +- `tools/p82_openrouter_model.py` directly subclasses + `minisweagent.models.openrouter_model.OpenRouterModel`. +- `tools/p82_bash_environment.py` directly subclasses + `minisweagent.environments.local.LocalEnvironment`. +- P8 runner and OpenRouter/Bash adapters therefore remain integration modules, + not neutral core implementations. +- Docker and SWE-bench appear in the experimental authority/evaluation path, + not as generic execution contracts. + +## Acceptance decision + +The neutral contract layer is ready for isolated fake-based development. The +external integration layer is not ready for promotion or comparative claims. +The next implementation should add adapter-level preflight records and +capability identity, not a fallback executor. + +## Preflight contract + +The core contract is now implemented in `src/arkx/integration.py`: + +- `AdapterIdentity` binds kind, name, version, and configuration; +- `DependencyObservation` preserves available, unavailable, and unknown; +- `AdapterPreflight` records capabilities, evidence, and reason; +- `assess_preflight` emits `READY`, `BLOCKED`, `UNKNOWN`, or `FAILED` without + loading a concrete adapter. + +The dependency-isolated suite passed with 298 tests. This is contract evidence; +the concrete OpenRouter, Git Bash, Docker, and SWE-bench boundaries remain in +their previously recorded states. + +The concrete read-only collector is `tools/collect_adapter_preflight.py`. +Its current raw result is recorded in +`logs/architecture/concrete-adapter-preflight-20260923.json`. It distinguishes +executable presence from successful runtime access; Git Bash is therefore +`BLOCKED`, not ready. + +The adapter modules are now import-safe but execution-strict: absence of +`minisweagent` no longer prevents test discovery, while construction still +raises an explicit runtime boundary error. This changes local testability, not +external readiness. diff --git a/docs/local-model-qualification.md b/docs/local-model-qualification.md new file mode 100644 index 00000000..031c41f8 --- /dev/null +++ b/docs/local-model-qualification.md @@ -0,0 +1,101 @@ +# Local model qualification — exploratory wave + +## Status + +This document records an exploratory local-model test wave performed on +2026-09-22. It is evidence about a local runtime, not an executor adoption +decision, a product dependency, or scientific proof of capability. + +No Arkx source code, fixtures, contracts, or experiment results were changed +by this wave. The tests were run outside the repository's experiment harness. + +## Controlled intent + +The same small coding task was used across models: + +> Write a Python function that raises `ValueError("value is required")` for +> `None`, returns the input otherwise, and show the correct exception test. + +The initial smoke test asked for `OK`. A second functional prompt constrained +the output to the function and `pytest.raises(ValueError, +match="value is required")`. Models were started one at a time through the +direct Ollama executable; the previously loaded model was stopped before +switching when possible. + +This was a capability probe only. It did not control temperature, context +window, quantization, thread count, or hardware telemetry sufficiently for a +comparative benchmark. Therefore latency is indicative, not comparable +evidence. + +## Environment + +| Dimension | Recorded value | +| --- | --- | +| Runtime | Ollama 0.34.3 | +| Runtime executable | `C:\Users\igorb\AppData\Local\Programs\Ollama\ollama.exe` | +| Intended hardware envelope | 16 GB RAM, as reported for this evaluation | +| Test date | 2026-09-22 | +| Repository effect | none; existing local Arkx changes were preserved | + +## Results + +| Model | Local artifact | Observation | Qualification state | +| --- | --- | --- | --- | +| Gemma 3 4B | `gemma3:4b`, 3.3 GB | Passed `OK`; generated the requested function and a correct `pytest.raises` assertion. Functional run took about 25 s. | Candidate for deeper qualification | +| Granite 3.3 8B | `granite3.3:8b`, 4.9 GB | Passed the `OK` smoke test in about 23 s. Functional correctness was not fully captured in this wave. | Smoke-qualified only | +| Phi-4-mini | `phi4-mini:latest`, 2.5 GB | Executed the task in about 14 s; response capture was not reliable enough for a correctness verdict. | Inconclusive | +| Nemotron Nano 4B | `nemotron-3-nano:4b`, 2.8 GB | Returned `OK`, but exposed a `Thinking` block despite the bounded-output request. | Rejected for clean executor output until controlled | +| Qwen2.5-Coder 3B | `qwen2.5-coder:3b`, 1.9 GB | Responded, but an earlier coding trial produced an invalid exception test using value equality instead of an exception assertion. | Candidate with review gate | +| Qwen2.5-Coder 7B | `qwen2.5-coder:7b`, 4.7 GB | Passed an isolated `OK` smoke test; broader capture was inconsistent during model switching. | Smoke-qualified only | +| Qwen2.5-Coder 14B | `qwen2.5-coder:14b`, 9.0 GB | Process executed, but the final response was not auditable in the captured terminal output. | Inconclusive | +| Qwen3.6 35B | `qwen3.6:35b`, 22 GB | Process was attempted on the 16 GB envelope; no auditable response was captured. | Not qualified for this envelope | +| DeepSeek-R1 7B | `deepseek-r1:7b`, 4.7 GB | Repeated isolated attempts produced no captured response within the test window. | Not qualified | +| GLM4 9B | `glm4:9b`, 5.5 GB | Installed; smoke execution completed through the interactive runtime, but output capture was inconclusive. | Inconclusive | +| StarCoder2 7B | `starcoder2:7b`, 4.0 GB | Installed; smoke execution completed through the interactive runtime, but output capture was inconclusive. | Inconclusive | + +## Interpretation + +The strongest directly observed result was Gemma 3 4B for bounded coding output. +Granite 3.3 8B and Qwen2.5-Coder 7B remain plausible candidates, but require +the same functional task through a machine-readable adapter before adoption. +Phi-4-mini is attractive for lightweight work because of its size and observed +latency, but it has not yet passed a correctness gate. + +Nemotron's exposed reasoning is a concrete executor-interface concern: a model +may be useful internally while still violating the output contract. It must +not be silently repaired or accepted as clean output. + +The 35B artifact is larger than the reported 16 GB RAM envelope and produced +no auditable result. It is not evidence that the model is unusable in every +environment; it is simply unqualified here. + +These observations support a portfolio hypothesis rather than a single-model +choice: + +- a small model can handle bounded auxiliary tasks; +- a code-oriented model may handle implementation tasks; +- a separate review model may be useful for verification; +- routing among them must remain explicit and observable. + +No routing, retry, fallback, executor switch, acceptance, promotion, or +automatic selection was introduced by this test wave. + +## Limitations and next experiment + +The Ollama interactive terminal emitted control sequences and, for some +models, did not provide a reliably machine-readable response to the calling +shell. Consequently, `executed` must not be interpreted as `accepted` or +`passed`. + +The next qualification step should use one fixed adapter and JSON response +capture, with: + +1. fixed prompt and generation settings; +2. fixed timeout and maximum output size; +3. syntax and test execution of generated Python; +4. explicit output-contract validation; +5. memory and tokens-per-second measurements; +6. repeated trials under the same model/environment identity. + +Until that exists, the states above remain exploratory observations, not +promotion evidence. diff --git a/docs/mvp-release-scope.md b/docs/mvp-release-scope.md new file mode 100644 index 00000000..a8013d88 --- /dev/null +++ b/docs/mvp-release-scope.md @@ -0,0 +1,69 @@ +# Escopo de MVP operacional + +## Decisão + +O produto está em MVP. O objetivo não é publicar a arquitetura acumulada nem +preservar uma solução provisória como se fosse produto. O objetivo é provar uma +única jornada operacional vertical, privada e reproduzível. + +## Regra contra gambiarra + +A única adaptação aceitável é a restrição financeira de infraestrutura: usar +recursos locais, gratuitos ou self-hosted em vez de serviços pagos. Essa +adaptação não pode alterar o contrato, esconder falhas, produzir falso sucesso +ou substituir evidência por opinião. Limitações de hardware, tempo ou custo +devem ser registradas na identidade da execução. + +Nenhum atalho técnico, fallback silencioso, fixture apresentada como execução +real ou camada incompleta será promovido como parte do MVP. + +## O que o MVP precisa fazer + +1. receber uma tarefa real em um repositório autorizado; +2. aplicar uma execução por um único executor explicitamente escolhido; +3. respeitar escopo, orçamento e tempo; +4. executar a verificação declarada; +5. persistir resultado, falha e identidade da execução; +6. retornar estados distintos para sucesso, falha, timeout e ambiente + indisponível; +7. repetir a mesma tarefa com evidência suficiente para comparar os resultados. + +## O que não entra no MVP + +- múltiplos executores; +- fallback automático; +- roteamento sofisticado; +- promessa de qualidade geral de modelo; +- SWE-bench como requisito de primeira versão; +- publicação pública; +- compatibilidade com toda a arquitetura histórica. + +## Classificação do que já existe + +O pacote, `doctor`, `baseline`, testes e checks de artefato são ferramentas de +fundação e harness de desenvolvimento. Eles são necessários para construir o +MVP, mas não constituem por si só uma release operacional. + +## Gates do MVP + +| Gate | Critério | Estado inicial | +|---|---|---| +| M0 | contrato de uma tarefa, executor, ambiente e aceitação | `PENDING` | +| M1 | uma execução real ponta a ponta | `NO-GO` | +| M2 | timeout, falha e ambiente indisponível sem falso sucesso | `PENDING` | +| M3 | repetição controlada com identidade e evidência | `PENDING` | +| M4 | instalação privada e uso por um usuário autorizado | `PENDING` | +| M5 | aceitação independente e release candidate | `PENDING` | + +Nenhum executor adicional, benchmark amplo ou camada arquitetural nova deve ser +adicionada antes de M1–M3 passarem. A implementação deve preferir remover +complexidade do caminho do MVP a criar novos pontos de extensão. + +O contrato detalhado de M0 está em +[0114-mvp-vertical-contract.md](decisions/0114-mvp-vertical-contract.md). + +O inventário atual contém um candidato experimental (`p82-baseline-mini-swe-agent`), +mas ele está `FUNCTIONAL_BUT_UNQUALIFIED`: depende de runtime externo e a +superfície de sandbox Git Bash está bloqueada. Ele só pode entrar no MVP depois +de preflight, identidade congelada, execução local reproduzível e aceitação; +não é um executor adotado por existir no código. diff --git a/docs/p82-minimal-mechanism-trial.md b/docs/p82-minimal-mechanism-trial.md new file mode 100644 index 00000000..c1ade7fa --- /dev/null +++ b/docs/p82-minimal-mechanism-trial.md @@ -0,0 +1,65 @@ +# P8.2 — Minimal mechanism trial + +P8.2 defines a controlled A/B/C/D design with a fixed executor. The harness +creates deterministic trial plans; it does not invoke an LLM or executor, +apply a mechanism, verify a patch, accept a result, or select a winner. + +| Arm | Treatment | +| --- | --- | +| A | mini baseline | +| B | A + safe editor | +| C | A + enhanced repository context | +| D | A + safe editor + enhanced repository context | + +All tasks, replicates, executor identity, exact model, environment, budget, +verification contract, and acceptance authority are frozen in +`P82ExperimentConfig`. Missing control evidence remains `UNKNOWN`; it is never +converted to a default or zero. A result must later be recorded through the +P8.1 `ExecutorTrial` contract with `Treatment` set to the corresponding arm. + +The primary outcomes are independently verified acceptance and cost to +acceptance. False-pass rate is a guardrail. P8.2 produces no promotion or +executor-adoption decision. + +The run manifest also records protocol version, model/temperature/reasoning +configuration, task sample and stratum, attempts, token/cost/wall-time limits, +verification identity, and independent acceptance identity. Task provenance is +evidence metadata only; it grants no authority to a task or treatment. + +Run comparison is tri-state: `COMPARABLE` means controls are known and equal, +`NOT_COMPARABLE` means a confounder differs, and `INDETERMINATE` means a +required control is unknown. Summaries retain `NOT_EXECUTED` and `BLOCKED` +counts, aggregate only observed metrics, and never produce a winner or a +promotion decision. + +## P8.2a baseline execution path + +P8.2a records infrastructure qualification, not mechanism evidence. The +baseline runner accepts treatment A only, invokes mini-SWE-agent through its public +Python API in a headless worker, and records execution artifacts. It does not +run verification or decide acceptance. Verification and independent +acceptance are explicit, nullable records; missing provider/model identity is +`BLOCKED`, with no fallback. + +The Windows runner-v3 qualification adds an explicit Git Bash environment +(`C:\\Program Files\\Git\\bin\\bash.exe`) without changing the model, +prompt, parser, or treatment. The Arkx OpenRouter adapter recognizes provider +error envelopes before action parsing. A retryable `503`/`provider_overloaded` +response is recorded as `BLOCKED` with +`PROVIDER_AVAILABILITY_FAILURE`; it is not converted into a parser or task +failure. The bounded retry policy is an execution-availability policy only. + +Artifacts are append-only under `task_id/attempt-N`. Reusing an existing +attempt is rejected rather than silently overwritten. The prospective Wave 0 +sample is frozen in `experiments/p82-wave0-task-sample.json` from the official +`SWE-bench/SWE-bench_Verified` test split; its two strata are explicitly +`UNKNOWN` because no auditable prior Arkx stratification was available. + +The official acceptance adapter is in `arkx.swebench_authority`. It creates a +SWE-bench prediction without gold patch or grading labels and invokes the +official harness with a unique `run_id`. The frozen Docker Gold authority +returned `resolved: true` for the two Wave 0 tasks recorded under +`logs/evaluation/`; this qualifies the acceptance path and infrastructure only. +Real A runs still +require one exact provider/model/configuration to be frozen. B/C/D and any +mechanism comparison remain out of scope. diff --git a/docs/p82-task-stratification-and-evidence.md b/docs/p82-task-stratification-and-evidence.md new file mode 100644 index 00000000..15399aa8 --- /dev/null +++ b/docs/p82-task-stratification-and-evidence.md @@ -0,0 +1,101 @@ +# P8.2 — Task stratification and evidence confrontation + +## Decision + +Candidate quality is not a scalar property of a model or harness. P8.2 must +compare a `task stratum × model × harness` cell. The current Wave 0 remains +frozen and is not reclassified retrospectively as mechanism evidence. + +The task strata are: + +| Stratum | Operational description | Main suspected bottleneck | +| --- | --- | --- | +| T0 | deterministic fixture, one target file, explicit failing test | startup and action compatibility | +| T1 | localized bug, one function or file, reproducible test | local reasoning and editing | +| T2 | several plausible locations or hidden control flow | repository localization and diagnosis | +| T3 | multiple modules, implicit behavior, incomplete test signal | planning and verification | +| T4 | cross-cutting API, dependency, or architectural change | long-horizon coordination | +| T5 | long-horizon, multimodal, multilingual, or non-Python task | generalization and runtime robustness | + +Classification is metadata, not a result. A task must not be assigned to a +higher stratum merely because an agent failed on it. Unknown task metadata is +recorded as `UNCLASSIFIED`, never inferred from outcome. + +## Task profile + +Every future task selection should record, before execution: + +```text +repository_scope = one_file | few_files | multi_module | cross_cutting +localization_ambiguity = explicit | bounded | competing | unknown +behavior_signal = direct_test | reproducible | implicit | absent +execution_horizon = fixture | short | iterative | long +language_runtime = single | multi_language | unknown +``` + +The stratum is derived from this profile and frozen with the task manifest. +The derivation is not an implementation of C1: it supplies no candidate file, +symbol, or context to an agent. + +## Empirical confrontation + +The literature supports testing several mechanisms, but does not qualify them +for Arkx: + +| Mechanism | External evidence | Evidence class | Arkx interpretation | +| --- | --- | --- | --- | +| minimal scaffold | official SWE-bench leaderboard uses a common mini-SWE-agent/Bash-only setting | official benchmark | control remains necessary | +| workflow without autonomous loop | Agentless reports strong SWE-bench Lite performance at low cost | primary paper, not Arkx | compare against A on T1/T2 | +| repository graph | RepoGraph reports gains when integrated with Agentless and SWE-agent | project/self-reported until independently reproduced | localization hypothesis for T2 | +| AST/repository tools | AutoCodeRover reports Verified results with specialized analysis | project/self-reported | test Python-specific bias on T2/T3 | +| tree search and refinement | SWE-Search reports relative improvement from MCTS/refinement | peer-reviewed paper | search hypothesis for T3/T4 | +| trained verifier | SWE-Gym reports gains from agent/verifier training and best-of-n selection | peer-reviewed paper | verifier hypothesis for T3/T4 | +| distribution shift | SWEE/SWA-Bench reports lower success outside classic SWE-bench distribution | peer-reviewed paper | include T4/T5 before generalizing | + +These claims are external evidence only. They do not open B1/C1, establish an +Arkx improvement, or justify comparing scores obtained with different task +strata. + +## Candidate allocation + +The exploration funnel is deliberately broad: + +```text +E0: 20–40 model × harness smoke cells across T0 fixtures +E1: 10–15 cells on T1/T2 deterministic repository fixtures +E2: 6–8 cells on stratified SWE-bench tasks +E3: 2–3 cells on T3/T4/T5 long-horizon or shifted tasks +``` + +No candidate advances because of a smoke result alone. Advancement requires +the same provider, model identity, generation configuration, executor, +verification authority, and acceptance authority within the comparison cell. + +## Current Wave 0 annotation + +The two frozen tasks have only provisional metadata descriptions: + +```text +sympy__sympy-14711 -> provisional T1 +sympy__sympy-24443 -> provisional T2 +``` + +This annotation is not an empirical result and does not change the frozen +Wave 0 protocol. The actual Wave 0 remains `BLOCKED` by provider availability +and timeout evidence. + +## Acceptance criteria for future stratified evidence + +For a stratum-level claim, Arkx requires: + +1. a pre-execution task manifest with frozen profiles; +2. at least one control cell using A; +3. the same authority and executor across compared cells; +4. raw trajectories, usage, environment, and identity artifacts; +5. explicit separation of `BLOCKED`, executor failure, task rejection, and + accepted resolution; +6. an independent acceptance decision before any promotion. + +Unit tests validate only the schema and deterministic derivation. Published +results validate hypotheses and candidate selection, not Arkx effectiveness. + diff --git a/docs/patch-verification.md b/docs/patch-verification.md new file mode 100644 index 00000000..4fee5836 --- /dev/null +++ b/docs/patch-verification.md @@ -0,0 +1,14 @@ +# P5 patch verification boundary + +P5 verifies explicit patch gates and remains separate from final acceptance: + +```text +PATCH_VERIFIED != PASS +PATCH_VERIFICATION != SMAG_FINAL_ACCEPTANCE +``` + +The verifier records `reproduces_issue_before_patch` and `reproduces_issue_after_patch`; `True` means the issue reproduces and `False` means it does not. A reparative reproduction gate therefore requires `before=True` and `after=False`. Missing observations are `UNKNOWN` or `BLOCKED` according to the gate; they are never fabricated as passes. + +`VERIFIED`, `REJECTED`, `BLOCKED`, and `UNKNOWN` are P5 statuses. P5 never updates `ExecutionRecord.status` and has no semantic diff analyzer. + +The fixture is `PATCH_VERIFICATION_FIXTURE`, not a production acceptance benchmark. diff --git a/docs/progress-stagnation.md b/docs/progress-stagnation.md new file mode 100644 index 00000000..297b8368 --- /dev/null +++ b/docs/progress-stagnation.md @@ -0,0 +1,49 @@ +# P2 progress / stagnation detection + +P2 compares explicit `ProgressSnapshot` values and emits evidence-only assessments: + +```text +PROGRESS_PROVEN +PROGRESS_NOT_PROVEN +NO_PROGRESS +UNKNOWN +``` + +P2 observes progress. P2 does not control execution, authorize retries, authorize replans, select or switch executors, route work, escalate automatically, produce `PASS`, or change acceptance. + +## Semantics + +- New relevant files, new passing tests, newly explained failures, reduced acceptance distance, or reduced diff distance are observable progress evidence. +- The same failure signature with no new evidence can establish `NO_PROGRESS`. +- Repeated actions with unchanged acceptance distance can establish `NO_PROGRESS`. +- A new failure with new useful evidence is not treated as stagnation. +- Missing required measurements and contradictory distance values are `UNKNOWN`. +- `confidence` is ordinal (`HIGH`, `MEDIUM`, `LOW`), not a calibrated probability. + +The required measurement fields are centralized in `ProgressConfig`. Defaults require all snapshot fields, so missing telemetry fails closed. + +## Policy boundary + +P2 exposes no retry or replan policy. Future policy may consume `ProgressAssessment` and independently derive a justification; this module performs no action. + +```text +PROGRESS_PROVEN != PASS +NO_PROGRESS != FAILED +UNKNOWN != PASS +``` + +The manifest at `experiments/progress-detection-fixture.json` is a `PROGRESS_DETECTION_FIXTURE`, not a real benchmark. + +## Telemetry + +Assessments expose `progress_status`, `progress_confidence`, `progress_signal_count`, `repeated_action_count`, `failure_signature_count`, and `assessment_duration_ms`. No tokens or cost are fabricated. + +## State + +```text +P2_PROGRESS_DETECTION = IMPLEMENTED / EXECUTED / TESTED_WITH_SYNTHETIC_FIXTURES +STAGNATION_POLICY_PROMOTED = NOT_CLAIMED +RETRY_POLICY_PROVEN = NOT_CLAIMED +PRODUCTION_READY = NOT_CLAIMED +``` + diff --git a/docs/project-contract.md b/docs/project-contract.md index f2f734a2..82054a43 100644 --- a/docs/project-contract.md +++ b/docs/project-contract.md @@ -40,7 +40,9 @@ NO_SILENT_SCOPE_EXPANSION 5. Any fallback, executor change, or scope change is explicit in the record and review. 6. New architecture requires a decision record describing the observed need, alternatives, evidence, and rollback/removal condition. -## Non-goals of this bootstrap - -This repository does not yet define routing, planning, multi-agent execution, or integrations with OpenHands, ReX, mini-SWE-agent, or SWE-agent. +## Current non-goals +The repository now defines bounded routing, planning/state, verification, +recovery, acceptance, promotion gates, and a mini-SWE-agent Treatment A +baseline path. It does not define general multi-agent execution, automatic +executor adoption, or integrations with OpenHands, ReX, and SWE-agent. diff --git a/docs/release-blocker-plan.md b/docs/release-blocker-plan.md new file mode 100644 index 00000000..f3b10171 --- /dev/null +++ b/docs/release-blocker-plan.md @@ -0,0 +1,88 @@ +# Plano passo a passo de desbloqueio de release + +## Regra de decisão + +Cada bloqueio recebe uma classificação explícita: + +- `PASS`: evidência executada atende ao critério declarado; +- `PENDING`: ainda falta trabalho ou decisão, mas não há falha conhecida; +- `BLOCKED_INFRA`: o teste não iniciou por infraestrutura externa; +- `NOT_APPLICABLE`: não pertence ao alvo de release atual, com justificativa; +- `NO-GO`: falha funcional ou evidência insuficiente para promoção. + +`NOT_APPLICABLE` não pode ser usado para esconder uma capacidade anunciada. +Uma pendência só pode ser ignorada quando o escopo do release não promete a +capacidade correspondente. + +## Alvos distintos + +### Fundação obrigatória — Chassi pré-executor + +O chassi não é um produto alternativo nem um release final. É a fundação +obrigatória do produto operacional: distribuição instalável, contratos, +governança, telemetria, persistência, recuperação, verificação e fronteiras +executor-agnostic. Ele precisa estar validado antes de receber um executor. + +Uma versão do chassi pode ser usada internamente como artefato técnico, mas não +será apresentada como o release do CodePro. + +### Alvo B — Release operacional privado (alvo atual) + +Além do Alvo A, promete submissão de tarefa, execução real, verificação, +persistência de evidência e recuperação. Exige executor qualificado. + +O estado atual pode avançar no Alvo A, mas ainda não atende ao Alvo B. + +Por decisão de escopo, o Alvo B é o objetivo: o primeiro release operacional +será privado, sem promessa de publicação aberta. O chassi é um gate obrigatório +desse alvo. Privado altera a distribuição e o licenciamento público, mas não +reduz os requisitos de funcionamento, evidência, segurança ou aceitação. + +O escopo operacional foi reduzido ao MVP descrito em +[mvp-release-scope.md](mvp-release-scope.md): uma jornada vertical, um executor +e uma classe de tarefa. A arquitetura histórica, o baseline e o CLI mínimo não +serão confundidos com o produto operacional. + +## Matriz de bloqueios + +| Ordem | Gate | Evidência atual | Classificação | Próxima ação | Alvo | +|---|---|---|---|---|---| +| 0 | Escopo e identidade | alvo alterado para release operacional privado | `PASS` | definir contrato operacional e executor autorizado | B | +| 1 | Empacotamento do chassi | wheel/sdist construídos, instalados em venvs limpos, hashes registrados | `PASS_LOCAL` | aceitar independentemente | B | +| 2 | CLI mínimo do chassi | `version`, `doctor`, `baseline`; 5 testes focados; `run` rejeitado | `PASS_LOCAL` | confirmação externa no CI | B | +| 3 | Núcleo determinístico do chassi | 372 testes e archive limpo passam | `PASS_LOCAL` | executar CI quando Billing estiver resolvido | B | +| 4 | Artefatos de publicação do chassi | `twine check` e `pip check` passam localmente | `PASS_LOCAL` | repetir externamente e arquivar artefatos | B | +| 5 | CI externo | jobs criados, mas nenhum passo iniciou por falha de Billing/spending limit | `BLOCKED_INFRA` | corrigir Billing ou configurar runner próprio | A/B | +| 6 | Política privada e notas de release | não há política de distribuição privada nem `CHANGELOG`/release notes identificados | `PENDING_HUMAN_DECISION` | definir acesso, retenção de evidências e notas da versão | B | +| 7 | Aceitação independente | ainda não existe decisão `ACCEPT` para a revisão candidata | `PENDING` | autoridade independente revisar evidências | A/B | +| 8 | Executor/CLI externo | execução real é requisito; candidato P8 baseline está funcional, mas não qualificado e depende de runtime/sandbox bloqueados | `NO-GO` | escolher um executor autorizado e qualificá-lo | B | +| 9 | Benchmark operacional | SWE-bench/provider benchmark requer executor e tarefa real | `PENDING` | executar após o fluxo operacional local passar | B | +| 10 | Promoção | branch de auditoria diverge de `main`; sem tag/release candidate | `PENDING` | congelar revisão, revisar diff, aceitar e só então taggear | A/B | + +## Sequência sem atalhos + +1. Definir o contrato do release operacional privado: entrada, tarefa, saída, + limites, evidências e política de falha. +2. Escolher explicitamente o executor autorizado e sua superfície de integração. +3. Implementar o fluxo operacional real sobre o chassi atual. +4. Executar primeiro toda a validação local: build, `twine check`, instalação, + CLI, `pip check`, suíte, testes negativos e execução real controlada. +5. Corrigir Billing/spending limit ou registrar runner próprio seguro. +6. Reexecutar o workflow externo completo: build, instalação, CLI, + `pip check` e suíte. +7. Arquivar logs, hashes, identidade do ambiente e resultado bruto do CI. +8. Obter decisão independente `ACCEPT` ou manter `NO-GO`. +9. Executar benchmark externo compatível, sem alegar mais do que o teste mede. +10. Criar uma revisão candidata sem artefatos locais; revisar diff contra a + base escolhida. +11. Gerar release candidate e repetir instalação em ambiente limpo. +12. Publicar a release privada somente se todos os gates obrigatórios estiverem + `PASS`. + +## Decisões que não serão mascaradas + +- Billing não vira `PASS` por testes locais; permanece `BLOCKED_INFRA`. +- CLI externo não vira `PASS` por existir documentação; permanece pendente. +- benchmark não é executado artificialmente sem executor e tarefa compatíveis. +- ausência de licença/notas exige decisão; não será corrigida por suposição. +- a branch de auditoria não será mesclada em `main` por conveniência. diff --git a/docs/release-readiness-plan.md b/docs/release-readiness-plan.md new file mode 100644 index 00000000..2fcef2e9 --- /dev/null +++ b/docs/release-readiness-plan.md @@ -0,0 +1,341 @@ +# Codepro — plano de prontidão para release + +Identidade do produto: **Codepro**. O namespace técnico atual é `arkx`, por +compatibilidade; isso não altera a identidade da distribuição nem do release. + +Status inicial: **NO-GO** para release de produto operacional. + +Este documento define a sequência de avaliação do produto. Um gate só pode ser +marcado como `PASS` com evidência executada e arquivada. `PASS` de teste local +não será chamado de benchmark externo, qualificação de executor ou evidência +de usuário. + +Para a matriz operacional de bloqueios e a distinção entre release do chassi e +release operacional, consulte [o plano passo a passo](release-blocker-plan.md). + +## Objetivo do release + +O release de produto só será promovido quando um usuário conseguir, em um +ambiente limpo e suportado: + +1. instalar o artefato; +2. iniciar o CLI; +3. submeter uma tarefa; +4. obter planejamento/roteamento explícito; +5. executar o caminho suportado; +6. receber resultado verificável ou falha explícita; +7. reproduzir a execução usando os artefatos registrados. + +O `v0.2.0` existente não recebe retroativamente essa classificação: a evidência +atual o sustenta apenas como release técnico/chassi. + +## Base metodológica + +- **PyPA Packaging Flow**: a publicação deve partir de uma árvore versionada, + declarar metadados, gerar sdist/wheel e verificar a instalação pelo usuário. + Referências: [flow](https://packaging.python.org/en/latest/flow/) e + [building and publishing](https://packaging.python.org/en/latest/guides/section-build-and-publish/). +- **NIST SSDF 1.1**: fornece práticas de desenvolvimento seguro, rastreabilidade + e redução de risco; não é prova de funcionalidade do agente. Referência: + [NIST SP 800-218](https://doi.org/10.6028/NIST.SP.800-218). +- **SWE-bench**: exige dataset, identidade da execução, timeout, artefatos e + relatório; um caso resolvido não representa qualidade geral. Referências: + [evaluation CLI](https://github.com/SWE-bench/SWE-bench/blob/main/docs/reference/cli.md) + e [evaluation guide](https://github.com/SWE-bench/SWE-bench/blob/main/docs/guides/evaluation.md). +- **Engenharia empírica**: cada afirmação deve ter hipótese falsificável, + workload congelado, tratamento identificado, métrica, repetição, falhas + reportadas e artefatos brutos. Não serão aceitos cherry-picking, fallback + silencioso ou inferência de sucesso por ausência de erro. + +## Gates, em ordem + +### G0 — Identidade e reprodutibilidade + +Hipótese: a revisão avaliada é a mesma que será publicada e pode ser +reconstruída sem depender do estado acidental da máquina. + +Critérios: + +- branch/tag/commit, versão, Python, SO e dependências registrados; +- working tree limpo ou desvios explicitamente anexados; +- branch local reconciliada com a revisão de release; +- comando de avaliação documentado e repetível. + +Evidência atual: **FAIL/INDETERMINADO**. A branch local está 24 commits atrás +e 22 à frente de `origin/main`, com artefatos locais não rastreados. + +### G1 — Distribuição e instalação limpa + +Hipótese: um usuário consegue instalar e iniciar o produto sem `PYTHONPATH`, +checkout especial ou dependência implícita. + +Critérios: + +- `pyproject.toml` com metadados e versão; +- sdist e wheel construídos; +- instalação em venv novo; +- `codepro --version`, `doctor` e `inspect` executam; +- arquivos fora do artefato não são necessários. + +Evidência atual: **FAIL na branch auditada**. O comando comum falha ao importar +`arkx`; os testes passam somente com `PYTHONPATH=src`. + +### G2 — Núcleo determinístico + +Hipótese: contratos, estados, persistência, replay e falhas determinísticas +mantêm o comportamento declarado. + +Critérios: + +- suíte completa em ambiente suportado; +- testes de unidade, integração e fronteiras negativas; +- mutation probe sem mutação sobrevivente nas invariantes críticas; +- nenhum teste depende de ordem, diretório ou lixo de execução anterior. + +### G3 — Caminho mínimo de produto sem provedor + +Hipótese: o produto coordena uma tarefa completa com executor falso, sem +acoplar o contrato central a um fornecedor. + +Critérios: + +- entrada → caracterização → roteamento → plano → execução → verificação → + aceitação/promoção; +- sucesso, timeout, cancelamento, orçamento excedido e dependência ausente; +- eventos e artefatos persistidos atomicamente; +- replay independente reproduz o estado final. + +### G4 — Segurança e operação básica + +Hipótese: o caminho mínimo falha de modo seguro e observável. + +Critérios: + +- escopo autorizado e comandos perigosos bloqueados; +- segredos não aparecem em logs; +- dependências auditadas; +- artefatos possuem hash e identidade; +- documentação informa limites e recuperação. + +### G5 — Qualificação de um executor real (adiado) + +Este gate não faz parte do objetivo atual. Ele só será aberto depois que G0–G4 +estiverem aprovados e quando houver uma decisão explícita de integrar um +executor. O fake de G3 existe apenas para provar o contrato do produto. + +Hipótese futura: um executor específico, com versão/configuração fixas, +completa o fluxo real sob um contrato definido. + +Critérios: + +- identidade do executor, provedor, modelo e versão; +- preflight positivo, sem tratar PATH como disponibilidade; +- tarefa pequena reproduzível em ambiente limpo; +- edição real, testes reais e resultado verificável; +- falhas de autenticação, timeout, processo e dependência registradas; +- repetição serial com o mesmo tratamento e artefatos brutos. + +Dois ou três executores só são necessários se o produto fizer essa promessa. +Não serão usados como fallback implícito. + +### G6 — Capacidade por horizonte de tarefa + +Hipótese: o produto sustenta as classes que declara vender. + +Critérios separados para pequena, média e longa duração: workload congelado, +limite de tempo, orçamento, taxa de conclusão, taxa de regressão, falha de +infraestrutura e custo. Uma vitória em SWE-bench Gold não qualifica todas as +classes nem um provedor/modelo em geral. + +### G7 — Release candidate e promoção + +Hipótese: outra pessoa consegue reconstruir, instalar, operar e auditar o +release a partir dos artefatos publicados. + +Critérios: + +- CI verde na revisão exata; +- artefatos, hashes, SBOM/auditoria e logs publicados; +- relatório com configuração, runs brutos, limitações e falhas; +- README reproduz o caminho suportado; +- decisão independente `ACCEPT` antes de tag/publicação. + +## Ordem de execução adotada + +1. congelar a revisão de produto e separar a linha de auditoria da linha de + release; +2. corrigir G0 e G1 antes de qualquer executor; +3. executar G2 com ambiente limpo; +4. executar G3 e G4, incluindo testes negativos; +5. escolher explicitamente um executor para G5; +6. executar G6 apenas para as classes anunciadas; +7. preparar G7; qualquer falha mantém `NO-GO`. + +## Auditoria inicial do plano + +O plano foi auditado contra as fontes acima e contra a árvore local. + +| Propriedade | Resultado | +|---|---| +| Começa pelo uso do usuário e instalação | PASS | +| Separa teste local de benchmark externo | PASS | +| Exige identidade, configuração e artefatos | PASS | +| Trata falha e infraestrutura como estados explícitos | PASS | +| Evita exigir múltiplos executores sem promessa correspondente | PASS | +| Possui critérios falsificáveis por gate | PASS | +| Estado atual do produto | NO-GO | + +### Evidência da primeira rodada + +- `git diff --check`: `PASS`. +- `python tools/check_foundation.py`: `PASS` — 8 arquivos e 14 invariantes. +- `PYTHONPATH=src python -m unittest discover -s tests -t .`: `PASS` — 367 + testes. +- `python -m unittest discover -s tests -t .`: `FAIL` — o pacote não é + importável sem configuração adicional. +- `python tools/mutation_probe.py`: `NOT MEASURED` nesta branch; o script não + existe nela. A ausência do probe não será convertida em aprovação. +- working tree: `BLOCKED_FOR_RELEASE` por artefatos locais preexistentes e pela + divergência entre a branch auditada e `origin/main`. +- auditoria de `origin/main` em `bb1b4547`: o `pyproject.toml` declara + `requires-python >=3.12`, mas a instalação limpa com Python 3.13 ficou + `BLOCKED_INFRA` porque o ambiente não conseguiu inicializar `venv`, acessar + `setuptools` ou criar/limpar diretórios temporários. Isso não é aprovação nem + reprovação funcional; exige repetição em um ambiente limpo com permissões + válidas. +- repetição de G1 em `origin/main@bb1b4547`, Python 3.13.15: wheel e sdist + construídos; wheel instalado em venv separado; `codepro --version`, `doctor` + e `inspect --json`: `PASS`. +- hashes dos artefatos dessa rodada: wheel + `D9890489615E032B0A071F6E3E9D196FBEC4ADE44997DED6EB25D51AB855CFFF`; + sdist `61AC55CA74CA3F11BF221EC8287E028DCCE34B478BAD30370EE92D009B51C665`. +- G2 na mesma instalação: `344` testes, `1` skip, `PASS` com permissões + adequadas. Sem elevação houve `12` erros de ACL em `tempfile`; esse resultado + permanece como limitação do ambiente, não como aprovação silenciosa. +- G3: contratos internos de governança, roteamento, qualificação e recovery + passaram em `63` testes, mas o CLI público só expõe `doctor` e `inspect`. + `codepro run` é rejeitado pelo parser (`invalid choice`). Não existe ainda + caminho público para submeter tarefa, acionar executor, verificar resultado + ou persistir uma execução de usuário. G3 permanece + `BLOCKED_BY_PRODUCT_SCOPE`. +- implementação de G3 foi preparada isoladamente na branch + `codex/g3-cli-execution`, commit `ef1cf6cb`: `codepro run --executor fake` + cobre sucesso, timeout e erro de ambiente; persiste JSON atomicamente; 348 + testes passaram com 1 skip. O wheel dessa implementação teve SHA-256 + `3F837651F00B40BDDB71A546985A2032B8CF53ECBD12175058EF0C589E67C2D6`. + Isso é evidência de contrato público com fixture, não qualificação de + executor real e ainda não é promoção para `main`. +- escopo atual confirmado: G5 está `DEFERRED`; nenhum executor real será + procurado, selecionado ou promovido durante a preparação do chassi. +- decisão de escopo: integração com CLI externo permanece `PENDING`, fora do + acceptance atual do chassi. Isso não bloqueia a avaliação do pacote, do CLI + mínimo e dos contratos executor-agnostic; bloqueia qualquer release que se + anuncie como produto operacional capaz de executar tarefas reais. +- G4 na implementação G3 (`ef1cf6cb`): `110` testes focados, `1` skip, + `PASS`, cobrindo comando sem shell, governança, autorização, qualificação, + escopo, CLI, verificação, promoção e proveniência. O foundation check também + passou com `10` arquivos, `14` invariantes e `2` study guards. +- validação do artefato empacotado na implementação G3 (`ef1cf6cb`): wheel + construído e instalado em venv Python 3.13 sem `PYTHONPATH`; `codepro + 0.2.0`, `doctor` e `inspect --json`: `PASS`. SHA-256 do wheel: + `3578CD89FD09716ABC9A4093D83AB5050AC795924A73E1749D46141D070ACD34`. +- contrato público G3 no wheel instalado: os três estados foram reproduzidos e + persistidos atomicamente (`READY_FOR_ACCEPTANCE`, `COMMAND_TIMEOUT` e + `ENVIRONMENT_ERROR`), com JSONs brutos separados por tentativa. +- regressão do artefato instalado: `348` testes, `1` skip, `PASS`. +- probe empírico de mutações na implementação G3: `8/8` mutações mortas, + `0` sobreviventes (`MUTATION_PROBE_PASS`). Isso aumenta a confiança nas + invariantes, mas não substitui teste de usuário, integração externa ou + qualificação de executor. +- reconciliação G3 com a linha de auditoria (`acd90a27`): `BLOCKED`. O + cherry-pick de `ef1cf6cb` encontrou conflitos modify/delete em + `src/arkx/cli.py` e `tests/test_cli.py`; a linha de auditoria não contém + esses arquivos nem os módulos antigos importados pelo CLI (`command`, + `governance`, `qualification`, `project` e `spine`). A implementação G3 + continua isolada e não foi promovida por incompatibilidade arquitetural. +- G1 na linha de auditoria antes desta mudança: `BLOCKED_BY_PACKAGING`. A revisão `acd90a27` não + possui `pyproject.toml`, `setup.py` ou `setup.cfg`, nem entrypoint público + `codepro`; portanto ainda não existe um artefato instalável para validar + como produto. `tools/check_foundation.py`: `PASS` — 8 arquivos e 14 + invariantes. A suíte da biblioteca: `367` testes, `PASS`, em Python 3.13 + com `TEMP` controlado. A execução sem esse controle teve `49` erros de ACL; + foi classificada como limitação de ambiente e não como aprovação. +- implementação inicial do limite G1 nesta linha: contrato registrado em + `docs/decisions/0112-package-boundary-and-distribution.md`, distribuição + `codepro==0.3.0.dev0`, namespace `arkx`, sem dependências de runtime. Em + Python 3.13, wheel e sdist foram construídos; o wheel foi instalado em venv + limpo sem `PYTHONPATH`, `arkx` importou, o baseline passou, o foundation + check passou e a suíte passou com `367` testes. Hashes: wheel + `B0EC4466576D6CFB52FD354451E7FD6BBB1AAEB4CC38D0134F530EEC00C22F2B`; + sdist `72832FCE71C474DEB4DC68D18D4A20B5A69986CE450718AC44C8980A4A53029D`. + O sdist também foi instalado em um segundo venv Python 3.13 sem + `PYTHONPATH`; import, baseline e foundation check passaram. Isso é evidência + local de distribuição, ainda sem aceitação independente. +- entrypoint público mínimo na linha atual: decisão `0032`, com apenas + `codepro --version`, `codepro doctor [--json]` e `codepro baseline`; nenhum + comando de tarefa ou executor foi adicionado. Testes focados: `4/4`, `PASS`. + No wheel instalado em venv Python 3.13, sem `PYTHONPATH`, os três comandos + passaram e a suíte completa passou com `371` testes, `OK`. Hash do wheel + desta revisão: `0A983984D1B3207ABA689712147EEBFA978A2513619AEE6F36E7C7B1406FC36C`. + O sdist da mesma revisão também foi instalado e executou os três comandos; + hash `3E5C84BC4F601E672E834303BBE7AB2914FCD13658E846EF0FE4D0D27A4F2094`. + Isso satisfaz a evidência local inicial de G3, mas ainda exige aceitação + independente e não qualifica executor. +- fronteira negativa do G3: teste explícito confirma que `codepro run` é + rejeitado (`SystemExit 2`, `invalid choice`); não há submissão silenciosa de + tarefa. O focused suite passou com `5/5` e a suíte completa desta revisão + passou com `372` testes, `OK`. +- automação adicionada em `.github/workflows/foundation.yml`: job + `packaged-cli` constrói o wheel, instala em venv, verifica `--version`, + `doctor --json`, `baseline` e executa a suíte sem `PYTHONPATH`. A alteração + foi validada localmente por comandos equivalentes, mas o job GitHub ainda é + `NOT_EXECUTED` até uma execução CI observável. +- reprodutibilidade do commit rastreado: `git archive` de `7e4b510a` foi + extraído sem arquivos não rastreados; o wheel foi construído e instalado em + venv limpo, `codepro --version`, `doctor --json` e `baseline` passaram, e a + suíte passou com `372` testes, `OK`. Hash do wheel desse archive: + `89E4C6DCA68F2BE494EC4EF2FD82AC859F6773A5DC3548B5F423BC5AF887B091`. +- execução CI externa do commit `2206e3cb`: `BLOCKED_INFRA`. Os jobs + `foundation` e `packaged-cli` foram criados, mas nenhum passo iniciou; o + GitHub registrou falha de pagamentos recentes ou limite de spending da conta. + Isso não é aprovação nem reprovação funcional. A aceitação externa só pode + ser repetida após a correção de Billing & Plans/spending limit. +- verificação local de artefatos da revisão seguinte: `python -m build` gerou + wheel e sdist; `twine check` aprovou ambos; instalação do wheel em venv + separado e `pip check` retornaram `No broken requirements found`. O workflow + foi fortalecido para executar esses mesmos checks, mas essa alteração ainda + aguarda CI externo após o desbloqueio da conta. +- revisão benchmark-driven do governance boundary: SWE-bench e + Terminal-Bench exigem identidade de revisão, ambiente e verificador. O + chassi agora bloqueia request sem `environment.workspace` ou + `environment.revision`; o focused suite de request passou `9/9` e a suíte + completa passou `373` testes. Isso fortalece reprodutibilidade e falha + fechada, mas não qualifica executor nem demonstra resolução de tarefa. +- melhoria benchmark-driven do verificador: `TestResult` agora preserva + `command`, `exit_code` e `duration_ms` quando observados, com validação de + metadados inválidos. O focused suite de verification passou `14/14` e a + suíte completa passou `375` testes. Esses campos tornam o verificador + auditável; ainda precisam ser preenchidos por uma execução real no M1. +- boundary neutro de verificação implementado: `run_verification_command` + executa argv declarado com `shell=False`, workspace existente e timeout; + captura stdout/stderr, exit code e duração, e classifica timeout, executável + ausente e erro de invocação como `UNKNOWN` explícito. Testes focados: `5/5`; + suíte completa anterior: `380` testes, `OK`. Isso fecha a execução de + comando do verificador, mas o vínculo a um executor real ainda pertence ao + M1. +- persistência do verificador implementada em `VerificationEvidenceStore` e + adaptador `CommandVerifier`: stdout/stderr, resultado e erro bruto são + armazenados por identidade `run_id`/teste; conteúdo divergente para a mesma + identidade bloqueia em vez de sobrescrever. A suíte focada do verificador e + da orquestração passou `32/32` em Python 3.11. A suíte completa em Python + 3.13 continua `INDETERMINATE/ENVIRONMENT_ERROR` neste host por ACL do + diretório temporário (56 falhas de permissão, incluindo cleanup); isso não é + contado como PASS e exige repetição em um ambiente gravável. + +## Primeiro trabalho autorizado pelo plano + +O próximo trabalho é fechar M0/M1 do MVP: ligar request autorizado, workspace, +executor escolhido, verificador neutro e persistência de evidência em uma +jornada real. A aceitação independente, CI externo e promoção continuam +obrigatórios antes de qualquer release. Nenhuma alegação de executor ou +release de produto será feita antes desses gates. diff --git a/docs/repository-planning-state.md b/docs/repository-planning-state.md new file mode 100644 index 00000000..ca09e1c4 --- /dev/null +++ b/docs/repository-planning-state.md @@ -0,0 +1,15 @@ +# P4 repository planning / state + +P4 creates deterministic `RepositoryState` and `RepositoryPlan` artifacts from explicit structure. It does not execute plans, change policy, alter acceptance, or promote a result. + +Collections are normalized for deterministic JSON. Plan steps have stable identities and explicit dependencies. Unknown dependencies and cycles are rejected. A step is never marked `COMPLETED` by textual assertion or by plan creation. + +`ReplanRequest` and `ReplanResult` preserve `previous_plan_ref`; a new plan cannot silently replace its predecessor. Replanning is a contract, not an autonomous loop. + +```text +REPOSITORY_WIDE_PATH != PLANNER_EXECUTED +P4_OFF is supported +``` + +The fixture is `REPOSITORY_PLANNING_FIXTURE`, not a performance claim. + diff --git a/docs/reproducibility-baseline.md b/docs/reproducibility-baseline.md new file mode 100644 index 00000000..fda7c656 --- /dev/null +++ b/docs/reproducibility-baseline.md @@ -0,0 +1,258 @@ +# Reproducibility baseline — 2026-09-23 + +## Scope + +This record describes the local deterministic-test baseline before the next +architecture slice. It is environment evidence, not executor qualification. + +## Repository identity + +| Field | Value | +|---|---| +| Branch | `codex/auditoriaarquitetural` | +| HEAD | `aef1df4d33c95c522dfda8f2e21f9bd09eb6ce6b` | +| OS | Windows | +| Python | 3.11.9 | +| Git | 2.55.0.windows.5 | +| Source import mode | `PYTHONPATH=src` | + +The working tree contains pre-existing modified and untracked files. This +baseline does not claim that the tree is clean or that those changes belong to +this phase. + +## Commands + +Core import smoke test: + +```powershell +$env:PYTHONPATH = "src" +python -c "import arkx; print('ok', len(arkx.__all__))" +``` + +Deterministic test command currently available: + +```powershell +$env:PYTHONPATH = "src" +python -m unittest discover -s tests -p 'test_*.py' +``` + +## Observed result + +- Core import: `PASS` (`151` public names exposed at the time of recording). +- Focused core/outcome/orchestration/baseline/authority/promotion/harness tests: `PASS`, 75 tests. + +The second architecture block adds executor/provider/sandbox contract tests; +its focused evidence is recorded in +`logs/architecture/execution-contract-block-20260923.json` with 79 passing +tests. This remains local contract evidence, not executor qualification. + +The third architecture block adds the fake contract lifecycle; its focused +evidence is recorded in +`logs/architecture/contract-lifecycle-block-20260923.json` with 82 passing +tests. No real executor or provider was invoked. + +The fourth architecture block adds the P8.2 baseline adapter; its focused +evidence is recorded in +`logs/architecture/baseline-executor-adapter-block-20260923.json` with 84 +passing tests. This is adapter evidence, not executor qualification. + +The fifth architecture block adds the orchestration bridge; its focused +evidence is recorded in +`logs/architecture/contract-orchestration-bridge-block-20260923.json` with 92 +passing tests. Verification, acceptance and promotion remain outside the +executor. + +The seventh architecture block centralizes the legacy result boundary; its +focused evidence is recorded in +`logs/architecture/legacy-result-boundary-block-20260923.json` with 95 +passing tests. The legacy API remains for compatibility. + +The eighth architecture block corrects the public result collision; its +focused evidence is recorded in +`logs/architecture/public-result-boundary-block-20260923.json` with 96 +passing tests. + +The tenth architecture block adds `ExecutionPlan`; its focused evidence is +recorded in `logs/architecture/execution-plan-block-20260923.json` with 100 +passing tests. Plan construction invoked no executor or provider. + +The ninth architecture block adds a static legacy-consumer boundary test; its +focused evidence is recorded in +`logs/architecture/legacy-consumer-boundary-block-20260923.json` with 98 +passing tests. + +The sixth architecture block adds direct neutral-executor orchestration; its +focused evidence is recorded in +`logs/architecture/direct-neutral-orchestration-block-20260923.json` with 94 +passing tests. Direct and bridged paths were equivalent under fakes. +- Full unittest discovery: `BLOCKED` by missing `minisweagent` in + `test_p82_openrouter_adapter`; the current discovery reached 205 tests + before the import error. +- `pytest`: `BLOCKED`; package is not installed in the selected environment. +- `minisweagent`: `BLOCKED`; import unavailable. +- Docker server: `BLOCKED`; Docker daemon unavailable through the configured + named pipe. + +## Interpretation + +The passing tests are local deterministic evidence only. They do not qualify +an executor, provider, model, Docker environment, SWE-bench path, or treatment. + +The missing dependencies are preserved as infrastructure blockers. No fallback +executor, provider, test runner, or local evaluator was selected. + +## Acceptance state + +`F0_BASELINE = BLOCKED_FOR_EXTERNAL_INTEGRATIONS` + +The focused evidence for the harness contract slice is recorded in +`logs/architecture/harness-contract-slice-20260923.json`. + +The core import and deterministic tests are usable for continuing contract +work. External adapter qualification remains unavailable until the missing +runtime identities and dependencies are explicitly restored. + +The eleventh architecture block integrates `ExecutionPlan` into the routed +orchestration boundary. Its focused evidence is recorded in +`logs/architecture/orchestration-execution-plan-block-20260923.json` with 101 +passing tests. Equivalent authorized inputs produced equal plans, the plan was +available on orchestration results, and neutral/legacy executor paths remained +compatible. This remains local contract evidence; no external executor, +provider, model, Docker environment, or SWE-bench authority was qualified. + +O bloco seguinte adiciona `CapabilityProfile` e `CapabilityRegistry` para +registrar identidade, versão, configuração, capacidades e referências de +evidência de componentes. A suíte focada passou com 104 testes. O registro é +descritivo e determinístico; não faz descoberta dinâmica, fallback ou +qualificação de provider/modelo. + +O bloco de budget adiciona `ExecutionBudgetSpec`, com limites neutros e digest +estável para identidade de plano/manifest. A suíte focada passou com 105 testes. +O bloco não consome budget nem implementa retry, escalonamento ou reconciliação +com os budgets de routing, recovery, qualification e request. + +O bloco de artifact store integrou a reserva explícita ao lineage. A suíte +focada passou com 111 testes; a operação persiste `retry-lineage.json`, rejeita +duplicidade e drift de identidade, e não executa o executor. + +O bloco de replay integrou `RetryLineage` ao `AttemptSnapshot`. A suíte focada +passou com 112 testes; lineage presente é validada contra o manifest, enquanto +artifacts históricos sem lineage continuam legíveis com ausência explícita. + +O bloco de retry adiciona `RetryPolicy` e `RetryDecision` como autorização pura. +A suíte focada passou com 107 testes, cobrindo erro retryable, erro unknown, +domínio não permitido e esgotamento de política/budget. Nenhum retry automático +foi executado; recovery e execução continuam sendo fronteiras separadas. + +O bloco de lineage adiciona `RetryAttemptPlan`, que deriva deterministicamente +o próximo `attempt_id`, preserva o attempt anterior e não reserva nem sobrescreve +artefatos. A suíte focada passou com 109 testes. A reserva efetiva no artifact +store continua uma operação explícita posterior. + +O bloco de budget ledger adiciona consumo imutável por `attempt_id`. A suíte +focada passou com 114 testes; consumo repetido retorna `ALREADY_CONSUMED` sem +duplicar uso e rejeições preservam o ledger. A reconciliação dos budgets de +outras camadas continua pendente. + +O bloco de binding integra `ExecutionBudgetSpec` ao `ExecutionPlan`. A suíte +focada passou com 115 testes; o plano deriva ou valida o digest e a +orquestração preserva o vínculo. O ledger ainda não é consumido por execução +real. + +O bloco seguinte persiste o `BudgetLedger` no `RunManifest`, vinculado ao +`attempt_id` e ao digest do plano. A suíte focada passou com 116 testes; drift de +attempt ou digest é rejeitado e o manifest continua imutável. Enforcement +end-to-end no executor permanece pendente. + +O bloco seguinte adiciona escrita atômica do `RunManifest` validado no +`ArtifactStore`. A suíte focada passou com 119 testes; identidade e lifecycle +são validados antes da persistência e o manifest pode ser recarregado. Escrita +bem-sucedida não implica verification ou acceptance. + +O bloco de lifecycle budget-aware passou com 118 testes. O ledger é consumido +antes da invocação; budget aceito chama o fake executor uma vez, enquanto budget +excedido produz `BLOCKED` estruturado sem chamada. Persistência de manifest e +execução externa continuam separadas. + +O bloco de event log adiciona persistência JSONL determinística para eventos de +lifecycle. A suíte focada passou com 121 testes; `run_id` e monotonicidade +temporal são validados na escrita e leitura. O log permanece telemetria, não +decisão de acceptance. + +O bloco de replay adiciona `ReplayState` e `replay_events` como redução pura do +event log. A suíte focada passou com 123 testes; fatos declarados são +reconstruídos sem executor, verifier ou inferência de acceptance. + +O bloco de consistência compara replay e `RunManifest`. A suíte focada passou +com 124 testes; identidade divergente e status conflitante são rejeitados, sem +reparo automático ou inferência de estado ausente. + +O bloco seguinte preserva verification e acceptance separadamente no replay. A +suíte focada passou com 125 testes; conflitos com evidência independente são +rejeitados e nenhum outcome é derivado de `COMPLETED`. + +O bloco seguinte adiciona `ReplayChain` para referências explícitas de request, +plan, execution, verification, acceptance e promotion. A suíte focada passou +com 127 testes; ordem causal inválida é rejeitada e estágios ausentes não são +preenchidos. + +O bloco de integridade adiciona `ChainIntegrity` com estados `COMPLETE`, +`INCOMPLETE` e `INCONSISTENT`. A suíte focada passou com 131 testes; a +classificação é apenas pré-condição e não executa acceptance ou promotion. + +O bloco seguinte classifica referências do replay como `PRESENT`, `MISSING` ou +`INDETERMINATE`. A suíte focada passou com 129 testes; sem inventário físico, a +referência permanece indeterminada e não é convertida em sucesso. + +O bloco seguinte conecta a classificação ao inventário validado do +`AttemptSnapshot`. A suíte focada passou com 130 testes; artifacts presentes e +refs ausentes são distinguidos sem varrer diretórios arbitrários ou substituir +artifacts. + +O bloco seguinte adiciona `RecoveryPlan` determinístico e vinculado ao attempt +de origem. A suíte focada passou com 137 testes; o plano registra decisão e +target sem executar recovery ou trocar executor. + +O bloco de recovery replay adiciona `recovery_ref` opcional à cadeia. A suíte +focada passou com 138 testes; a ordem causal é validada sem tornar recovery +obrigatório e sem quebrar consumidores posicionais legados. + +O bloco seguinte vincula `recovery_ref` ao `RecoveryPlan.reference` canônico. A +suíte focada permanece com 138 testes; referência divergente é rejeitada sem +executar recovery. + +O bloco de execução adiciona referências canônicas `plan://` e `execution://`. +A suíte focada passou com 139 testes; drift de plano ou attempt é rejeitado sem +inferir verification ou acceptance. + +O bloco seguinte adiciona referências canônicas para `VerificationResult` e +`AcceptanceResult`. A suíte focada permanece com 139 testes; namespaces são +distintos e drift de referência é rejeitado sem promotion. + +O bloco final de lineage adiciona referência canônica à `PromotionDecision` e +validação da cadeia de outcomes. A suíte focada permanece com 139 testes; +attempt, promotion ou acceptance divergentes são rejeitados sem executar +promotion. + +O bloco de validação global executou a descoberta completa e alcançou 291 +testes, mas ficou `BLOCKED` ao importar `test_p82_openrouter_adapter` por +ausência de `minisweagent`. O bloqueio foi preservado como dependência externa; +nenhum fallback foi selecionado e não foi classificado como falha funcional. + +Como controle, a suíte foi executada excluindo somente o módulo bloqueado e +passou com 290 testes. Isso é evidência ampla de regressão local, mas não +qualifica a integração OpenRouter/minisweagent nem o executor externo. + +O Bloco A de readiness consolidou adapters, dependências e preflights em +`docs/integration-readiness.md`. A conclusão é assimétrica: contratos neutros +estão prontos para desenvolvimento com fakes; OpenRouter, Git Bash, Docker e +SWE-bench permanecem bloqueados ou limitados à autoridade já registrada. + +O Bloco B adiciona o contrato comum de preflight para adapters. A suíte isolada +passou com 293 testes; `READY`, `BLOCKED`, `UNKNOWN` e `FAILED` permanecem +distintos e nenhum fallback é selecionado. + +O coletor concreto do Bloco B registrou OpenRouter, Git Bash e Docker como +`BLOCKED`, e Ollama como `UNKNOWN` por identidade/configuração não congelada. A +evidência raw está em +`logs/architecture/concrete-adapter-preflight-20260923.json`. diff --git a/docs/routing-escalation.md b/docs/routing-escalation.md new file mode 100644 index 00000000..72df9a5f --- /dev/null +++ b/docs/routing-escalation.md @@ -0,0 +1,69 @@ +# P3 rule-based routing and evidence-based escalation + +P3 is the first bounded policy layer. It consumes P1 characterization, P2 progress assessment, external evidence sufficiency, and an explicit budget. It produces serializable decisions only. + +```text +characterization -> select initial path -> bounded outcome -> evidence/progress -> continue | stop | escalate +``` + +## Routing + +P1 scope maps directly to a path: + +```text +SIMPLE -> SIMPLE_PATH +LOCALIZED -> LOCALIZED_PATH +REPOSITORY_WIDE -> REPOSITORY_WIDE_PATH +UNKNOWN -> QUALIFICATION_REQUIRED +``` + +P3 validates that P1's `recommended_path` agrees with the scope. It does not infer a better path or choose a concrete executor. + +## Escalation + +The path ladder is monotonic: + +```text +SIMPLE_PATH -> LOCALIZED_PATH -> REPOSITORY_WIDE_PATH +``` + +`NO_PROGRESS` plus an available higher path and escalation budget produces `ESCALATE`. At the top of the ladder it produces `BLOCK`. A failed attempt alone is not an escalation reason. + +`PROGRESS_PROVEN` plus `SUFFICIENT` evidence produces `STOP_SUFFICIENT_EVIDENCE`, never `PASS`. `UNKNOWN` progress or evidence requires qualification. Insufficient evidence may continue while attempt budget remains. + +## Budget + +Defaults are conservative and explicit: + +```text +max_path_escalations = 2 +max_attempts = 3 +``` + +The policy returns a derived budget snapshot. It does not execute an attempt, increment retry policy, or mutate P0 telemetry. + +## Boundaries + +```text +ROUTE BEFORE CASCADING +ESCALATE ON EVIDENCE +NO_SILENT_EXECUTOR_SWITCH +NO_UNBOUNDED_CHAIN +NO_PROGRESS != AUTOMATIC_FAILURE +CHARACTERIZATION != ACCEPTANCE +ROUTING != ACCEPTANCE +ESCALATE != invoke another executor +``` + +P3 is rule-based, does not learn, does not invoke executors, does not own acceptance, and does not implement retry, replan, planner, multi-agent, or executor selection behavior. The fixture at `experiments/routing-escalation-fixture.json` is synthetic policy evidence, not an accuracy benchmark. + +## State + +```text +P3_ROUTING_ESCALATION = IMPLEMENTED / EXECUTED / TESTED_WITH_SYNTHETIC_FIXTURES +EXECUTOR_SELECTION = NOT_IMPLEMENTED +LEARNED_ROUTING = NOT_IMPLEMENTED +ROUTING_PROMOTED = NOT_CLAIMED +PRODUCTION_READY = NOT_CLAIMED +``` + diff --git a/docs/task-characterization.md b/docs/task-characterization.md new file mode 100644 index 00000000..f316acbb --- /dev/null +++ b/docs/task-characterization.md @@ -0,0 +1,50 @@ +# P1 deterministic task characterization + +P1 transforms explicit `TaskSignals` into a versioned `TaskCharacterization`. It is a structural hypothesis for later experiments, not a router or classifier trained for accuracy. + +```text +TaskSignals -> deterministic characterization -> TaskCharacterization +``` + +P1 does not: + +- invoke an executor or a model; +- select or authorize an executor; +- route, escalate, retry, replan, or alter acceptance; +- claim real-world classifier accuracy. + +## Fail-closed semantics + +Missing required signals, contradictory signals, or ambiguity produce: + +```text +scope = UNKNOWN +recommended_path = QUALIFICATION_REQUIRED +confidence = LOW +``` + +Confidence is ordinal (`HIGH`, `MEDIUM`, `LOW`), not a calibrated probability. P1 uses explicit, centralized thresholds in `CharacterizationConfig`; it does not parse language magically. + +## Evidence + +The serialized characterization preserves input signals, derived counts, closed-enum classification, confidence, reason codes, recommended path, and telemetry. It stores no chain-of-thought. + +```text +CHARACTERIZED != ROUTED != EXECUTED != PASS +``` + +The manifest at `experiments/characterization-fixture.json` is a `CHARACTERIZATION_FIXTURE`: synthetic stability evidence, not a real-world benchmark. + +## Telemetry + +P1 exposes `characterization_duration_ms`, `characterization_scope`, `characterization_confidence`, `recommended_path`, and `signal_count`. Duration is measured locally by `characterize_timed`; tokens and cost remain P0-unknown because P1 invokes no model. + +## State + +```text +P1_TASK_CHARACTERIZATION = IMPLEMENTED / EXECUTED / TESTED_WITH_SYNTHETIC_FIXTURES +ROUTING_PROVEN = NOT_CLAIMED +CLASSIFIER_ACCURATE = NOT_CLAIMED +PRODUCTION_READY = NOT_CLAIMED +``` + diff --git a/experiments/README.md b/experiments/README.md index 551f37e0..fe93b83d 100644 --- a/experiments/README.md +++ b/experiments/README.md @@ -2,3 +2,4 @@ This directory is reserved for explicit experiment records and small fixtures. Use [the experimental protocol](../docs/experimental-protocol.md). No experiment has been executed by the bootstrap. +The exploratory local-model qualification wave is documented in [local model qualification](../docs/local-model-qualification.md). It was run outside the Arkx experiment harness and must not be read as executor adoption or acceptance evidence. diff --git a/experiments/characterization-fixture.json b/experiments/characterization-fixture.json new file mode 100644 index 00000000..1ea9baf7 --- /dev/null +++ b/experiments/characterization-fixture.json @@ -0,0 +1,56 @@ +{ + "fixture_type": "CHARACTERIZATION_FIXTURE", + "description": "Synthetic stability fixtures; not a real-world quality benchmark.", + "tasks": [ + { + "id": "simple", + "signals": { + "candidate_files": ["src/arkx/contracts.py"], + "dependency_edges": [], + "affected_components": ["contracts"], + "known_tests": ["tests/test_contracts.py"], + "ambiguity_markers": [], + "risk_markers": [], + "acceptance_checks": ["unit tests pass"] + } + }, + { + "id": "localized", + "signals": { + "candidate_files": ["src/arkx/telemetry.py", "tests/test_telemetry.py"], + "dependency_edges": [["telemetry", "contracts"]], + "affected_components": ["telemetry"], + "known_tests": ["tests/test_telemetry.py", "tests/test_contracts.py"], + "ambiguity_markers": [], + "risk_markers": [], + "acceptance_checks": ["unit tests pass", "round-trip is preserved"] + } + }, + { + "id": "repository-wide", + "signals": { + "candidate_files": ["src/arkx/a.py", "src/arkx/b.py", "src/arkx/c.py", "tests/test_a.py", "tests/test_b.py", "docs/architecture.md"], + "dependency_edges": [["a", "b"], ["b", "c"], ["c", "shared-state"], ["tests", "a"]], + "affected_components": ["runtime", "storage", "api"], + "known_tests": ["tests/test_a.py", "tests/test_b.py", "tests/test_c.py"], + "ambiguity_markers": [], + "risk_markers": [], + "acceptance_checks": ["unit tests pass", "integration tests pass"], + "state_shared": true, + "architectural_change": true + } + }, + { + "id": "ambiguous", + "signals": { + "candidate_files": null, + "dependency_edges": null, + "affected_components": ["unknown"], + "known_tests": null, + "ambiguity_markers": ["acceptance unclear"], + "risk_markers": [], + "acceptance_checks": null + } + } + ] +} diff --git a/experiments/composition-harness-fixture.json b/experiments/composition-harness-fixture.json new file mode 100644 index 00000000..5f023924 --- /dev/null +++ b/experiments/composition-harness-fixture.json @@ -0,0 +1,7 @@ +{ + "fixture_type": "EXPERIMENTAL_COMPOSITION_FIXTURE", + "description": "Synthetic treatment arms for P7; not an execution or acceptance benchmark.", + "controlled_dimensions": ["task", "executor", "model", "environment", "budget"], + "treatments": ["A", "B", "C", "D", "E", "F"], + "excluded_future_treatments": ["G"] +} diff --git a/experiments/cost-tier-comparison-protocol.json b/experiments/cost-tier-comparison-protocol.json new file mode 100644 index 00000000..56f89b78 --- /dev/null +++ b/experiments/cost-tier-comparison-protocol.json @@ -0,0 +1,51 @@ +{ + "protocol": "cost-tier-comparison-v1", + "project": "Codepro", + "status": "BLOCKED_ON_EXECUTOR_QUALIFICATION", + "routing": false, + "cascade": false, + "purpose": "Measure cost tiers without treating cost as quality or selecting a winner before evidence.", + "tiers": [ + "FREE_SHORT", + "FREE_GENERAL", + "FREE_LARGE_CONTEXT", + "SUBSCRIPTION", + "PAYG" + ], + "controlled_dimensions": [ + "task_id", + "task_revision", + "executor_identity", + "provider_identity", + "model_identity", + "tier", + "environment_identity", + "budget_digest", + "verifier_identity", + "instrumentation_identity", + "replicate_id" + ], + "observations": [ + "resolved", + "patch_validity", + "marginal_cost", + "context_limit", + "context_used", + "wall_time", + "tokens", + "calls", + "retries", + "termination_reason", + "blocked_class", + "failure_class" + ], + "sequence": [ + "SECOND_EXECUTOR_ENABLEMENT", + "EXECUTOR_QUALIFICATION_V1", + "NON_ROUTED_TIER_MATRIX", + "ECONOMIC_BASELINE_V1", + "EXPLICIT_ESCALATION_TREATMENT", + "HELD_OUT_ROUTING_TREATMENT" + ], + "promotion_rule": "No routing promotion without held-out improvement over Economic Baseline v1 in verified resolution per unit cost and declared guardrails." +} diff --git a/experiments/executor-qualification-fixture.json b/experiments/executor-qualification-fixture.json new file mode 100644 index 00000000..b8e166e4 --- /dev/null +++ b/experiments/executor-qualification-fixture.json @@ -0,0 +1,7 @@ +{ + "fixture_type": "EXECUTOR_QUALIFICATION_FIXTURE", + "description": "Synthetic external observations for P8.1; not an executor benchmark.", + "comparison_axes": ["EXECUTOR", "TREATMENT"], + "controlled_dimensions": ["task_revision", "acceptance_definition", "model", "environment", "budget", "treatment"], + "observed_outcomes": ["COMPLETED", "EXECUTOR_FAILED", "INFRASTRUCTURE_FAILED", "BUDGET_EXHAUSTED", "BLOCKED", "UNKNOWN"] +} diff --git a/experiments/handoff-accounting-fixture.json b/experiments/handoff-accounting-fixture.json new file mode 100644 index 00000000..4dd1e4e5 --- /dev/null +++ b/experiments/handoff-accounting-fixture.json @@ -0,0 +1,13 @@ +{ + "fixture_type": "HANDOFF_ACCOUNTING_FIXTURE", + "description": "Synthetic handoff accounting scenarios; not a performance benchmark.", + "scenarios": [ + "A_no_handoff", + "B_one_justified_handoff", + "C_multiple_bounded_handoffs", + "D_duplicated_instructions", + "E_discarded_lost_context", + "F_unknown_context_metrics", + "G_budget_exceeded" + ] +} diff --git a/experiments/p82-block-plan.json b/experiments/p82-block-plan.json new file mode 100644 index 00000000..9ad1d142 --- /dev/null +++ b/experiments/p82-block-plan.json @@ -0,0 +1,77 @@ +{ + "schema_version": "p8.2-block-plan-v1", + "status": "DESIGN_ONLY", + "purpose": "Organize qualification and empirical work into independently auditable blocks.", + "global_rules": [ + "A remains the only executable treatment until its real-agent preflight is qualified.", + "B1 and C1 contracts may be tested, but their operational mechanisms remain disabled.", + "D1 is not executable.", + "A blocked infrastructure outcome is not an agent failure.", + "No candidate advances on smoke evidence alone." + ], + "blocks": [ + { + "id": "BLOCK_0", + "name": "contracts-and-matrix", + "scope": "schemas, identities, strata, outcomes, artifact paths", + "input": "repository state and predeclared protocol", + "output": "validated design artifacts", + "execution": "deterministic tests only", + "stop_when": "schema and identity invariants pass" + }, + { + "id": "BLOCK_1", + "name": "compatibility", + "scope": "model query, action parser, Git Bash, Docker, hardened adapter", + "input": "one model-harness candidate", + "output": "qualified, blocked, or incompatible probe record", + "execution": "bounded disposable probes; no SWE-bench", + "stop_when": "one complete response and required runtime checks are observed" + }, + { + "id": "BLOCK_2", + "name": "difficulty-smoke", + "scope": "T0, T1, and T2 deterministic fixtures", + "input": "Block 1 qualified candidate", + "output": "stratified smoke evidence and diagnostics", + "execution": "serial fixture runs with frozen identity", + "stop_when": "predeclared fixture budget is exhausted" + }, + { + "id": "BLOCK_3", + "name": "candidate-qualification", + "scope": "model x harness x stratum cells", + "input": "Block 2 evidence", + "output": "executor-qualified candidate records", + "execution": "bounded cells; no silent fallback", + "stop_when": "candidate set for the next comparison wave is frozen" + }, + { + "id": "BLOCK_4", + "name": "baseline-A", + "scope": "frozen Treatment A and Wave 0", + "input": "qualified provider/model and Docker authority", + "output": "Wave 0 acceptance evidence", + "execution": "only the predeclared tasks and identity", + "stop_when": "all tasks are terminal or the declared blocker is exhausted" + }, + { + "id": "BLOCK_5", + "name": "stratified-exploration", + "scope": "T1 through T5 exploratory cells", + "input": "Block 3 candidate set and accepted baseline evidence", + "output": "task-stratum-specific comparisons", + "execution": "same authority and identity within each comparison", + "stop_when": "the predeclared exploratory allocation is complete" + }, + { + "id": "BLOCK_6", + "name": "mechanism-gate", + "scope": "authorization of operational B1 or C1", + "input": "independent baseline and mechanism evidence", + "output": "gate decision only", + "execution": "review and acceptance; no automatic promotion", + "stop_when": "an independent acceptance decision is recorded" + } + ] +} diff --git a/experiments/p82-block2-smoke-manifest.json b/experiments/p82-block2-smoke-manifest.json new file mode 100644 index 00000000..acec6fb9 --- /dev/null +++ b/experiments/p82-block2-smoke-manifest.json @@ -0,0 +1,37 @@ +{ + "schema_version": "p8.2-block2-smoke-v1", + "status": "PREDECLARED", + "purpose": "Run deterministic local smoke cells by task stratum without model or SWE-bench execution.", + "evidence_boundary": "contract and harness consistency only; not model capability or mechanism evidence", + "cells": [ + { + "cell_id": "T0-headless", + "stratum": "T0", + "fixture": "experiments/p82-deterministic-headless-probe.json", + "test_modules": ["tests.test_p82_baseline"], + "required": ["initialization_status=PASS", "run_status=Submitted", "api_calls=1"] + }, + { + "cell_id": "T1-localized", + "stratum": "T1", + "fixture": "experiments/characterization-fixture.json:localized", + "test_modules": ["tests.test_characterization"], + "required": ["localized fixture is represented", "deterministic characterization"] + }, + { + "cell_id": "T2-localization-contract", + "stratum": "T2", + "fixture": "C1 contract only", + "test_modules": ["tests.test_p82_localization"], + "required": ["budget enforced", "candidates non-authoritative", "deterministic serialization"] + } + ], + "frozen": { + "model_calls": 0, + "provider": null, + "executor": "local deterministic test runner", + "treatment": "contract-smoke-only", + "fallback": false + }, + "promotion_rule": "A smoke PASS does not qualify a model, executor, treatment, B1, C1, or D1." +} diff --git a/experiments/p82-block3-t1-functional-v2.json b/experiments/p82-block3-t1-functional-v2.json new file mode 100644 index 00000000..4db5e724 --- /dev/null +++ b/experiments/p82-block3-t1-functional-v2.json @@ -0,0 +1,10 @@ +{ + "schema_version": "p8.2-block3-t1-functional-v2", + "status": "PREDECLARED", + "changed_from_v1": "output budget only", + "model_set": ["gemma3:4b", "phi4-mini:latest", "granite3.3:8b", "qwen2.5-coder:3b"], + "prompt": "Return only one valid JSON object with keys code and test. code must define validate_value(value): raise ValueError('value is required') when value is None, otherwise return value. test must use pytest.raises to check the None case and assert that validate_value(7) == 7.", + "generation": {"temperature": 0, "num_predict": 128, "stream": false}, + "validation": ["JSON keys exactly code and test", "both fields parse as Python", "function behavior executes", "test contains pytest.raises"], + "interpretation": "This is a new functional smoke cell; it does not revise the v1 result or qualify SWE-bench resolution." +} diff --git a/experiments/p82-candidate-matrix.json b/experiments/p82-candidate-matrix.json new file mode 100644 index 00000000..ffedffee --- /dev/null +++ b/experiments/p82-candidate-matrix.json @@ -0,0 +1,45 @@ +{ + "schema_version": "p8.2-candidate-matrix-v1", + "status": "DESIGN_ONLY", + "evidence_boundary": "Prior observations are qualification metadata, not resolution evidence.", + "frozen_harnesses": [ + {"id": "mini-gitbash-docker", "agent": "mini-SWE-agent-2.4.6", "environment": "Git Bash", "executor": "Docker", "fallback": false}, + {"id": "ollama-machine-readable", "agent": "bounded-json-probe", "environment": "Ollama", "executor": "local", "fallback": false} + ], + "difficulty_targets": { + "T0": "startup and action compatibility", + "T1": "localized one-file reasoning and editing", + "T2": "repository localization and diagnosis", + "T3": "multi-module planning and verification", + "T4": "cross-cutting API or dependency change", + "T5": "long-horizon or distribution-shifted task" + }, + "candidates": [ + {"id": "local/gemma3-4b", "family": "local", "harness": "ollama-machine-readable", "prior_state": "observed_candidate", "initial_targets": ["T0", "T1"]}, + {"id": "local/granite3.3-8b", "family": "local", "harness": "ollama-machine-readable", "prior_state": "smoke_qualified_only", "initial_targets": ["T0", "T1"]}, + {"id": "local/phi4-mini", "family": "local", "harness": "ollama-machine-readable", "prior_state": "inconclusive", "initial_targets": ["T0", "T1"]}, + {"id": "local/qwen2.5-coder-3b", "family": "local", "harness": "ollama-machine-readable", "prior_state": "review_required", "initial_targets": ["T0", "T1"]}, + {"id": "local/qwen2.5-coder-7b", "family": "local", "harness": "ollama-machine-readable", "prior_state": "smoke_qualified_only", "initial_targets": ["T0", "T1", "T2"]}, + {"id": "local/qwen2.5-coder-14b", "family": "local", "harness": "ollama-machine-readable", "prior_state": "inconclusive", "initial_targets": ["T1", "T2"]}, + {"id": "local/glm4-9b", "family": "local", "harness": "ollama-machine-readable", "prior_state": "inconclusive", "initial_targets": ["T0", "T1"]}, + {"id": "local/starcoder2-7b", "family": "local", "harness": "ollama-machine-readable", "prior_state": "inconclusive", "initial_targets": ["T0", "T1"]}, + {"id": "local/nemotron-nano-4b", "family": "local", "harness": "ollama-machine-readable", "prior_state": "output_contract_risk", "initial_targets": ["T0"]}, + {"id": "local/deepseek-r1-7b", "family": "local", "harness": "ollama-machine-readable", "prior_state": "not_qualified", "initial_targets": ["T0"]}, + {"id": "openrouter/nemotron-ultra-550b", "family": "openrouter_free", "harness": "mini-gitbash-docker", "prior_state": "provider_unstable_executor_pending", "initial_targets": ["T0", "T1", "T2"]}, + {"id": "openrouter/laguna-s-2.1", "family": "openrouter_free", "harness": "mini-gitbash-docker", "prior_state": "unqualified", "initial_targets": ["T0", "T1"]}, + {"id": "openrouter/nex-n2.5-mini", "family": "openrouter_free", "harness": "mini-gitbash-docker", "prior_state": "unqualified", "initial_targets": ["T0", "T1"]}, + {"id": "openrouter/nemotron-3.5-lightning", "family": "openrouter_free", "harness": "mini-gitbash-docker", "prior_state": "unqualified", "initial_targets": ["T0", "T1", "T2"]}, + {"id": "openrouter/liquid", "family": "openrouter_free", "harness": "mini-gitbash-docker", "prior_state": "observed_incompatible_or_unstable", "initial_targets": ["T0"]}, + {"id": "openrouter/north", "family": "openrouter_free", "harness": "mini-gitbash-docker", "prior_state": "observed_incompatible_or_unstable", "initial_targets": ["T0"]}, + {"id": "openrouter/gpt-5-mini", "family": "openrouter_paid", "harness": "mini-gitbash-docker", "prior_state": "not_tested_in_current_wave", "initial_targets": ["T0", "T1", "T2"]}, + {"id": "nooa/0.0.10", "family": "harness_candidate", "harness": "nooa", "prior_state": "investigation_only", "initial_targets": ["T0", "T1", "T2", "T3"]} + ], + "advancement_rules": [ + "A smoke response does not qualify task resolution.", + "A candidate must pass the same machine-readable action and runtime contract before task cells.", + "A candidate is qualified per harness and stratum, never globally by model name.", + "Provider failure, runtime failure, parser incompatibility, and task rejection remain distinct.", + "No fallback or silent harness substitution is allowed." + ], + "next_execution": "bounded T0 compatibility cells, then T1/T2 only for candidates that pass the same adapter contract" +} diff --git a/experiments/p82-deterministic-headless-probe.json b/experiments/p82-deterministic-headless-probe.json new file mode 100644 index 00000000..02b40ade --- /dev/null +++ b/experiments/p82-deterministic-headless-probe.json @@ -0,0 +1,43 @@ +{ + "schema": "p8.2a-deterministic-headless-probe-v1", + "status": "PASS", + "runtime": { + "package": "mini-swe-agent", + "version": "2.4.6", + "api": [ + "get_model", + "get_environment", + "get_agent", + "agent.run" + ] + }, + "explicit_model_configuration": { + "model": "deterministic", + "model_class": "deterministic", + "cost_per_call": 0.0, + "outputs": [ + { + "role": "assistant", + "content": "", + "extra": { + "actions": [ + { + "command": "echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT" + } + ], + "cost": 0.0 + } + } + ] + }, + "result": { + "process_status": "EXIT_CODE_0", + "initialization_status": "PASS", + "run_status": "Submitted", + "api_calls": 1, + "instance_cost": 0.0, + "trajectory": "%TEMP%/arkx-p82-deterministic-probe.json", + "failure_category": null + }, + "interpretation": "Headless runner qualification only; not a model capability result and not treatment-A evidence." +} diff --git a/experiments/p82-minimal-mechanism-trial-fixture.json b/experiments/p82-minimal-mechanism-trial-fixture.json new file mode 100644 index 00000000..2f61883d --- /dev/null +++ b/experiments/p82-minimal-mechanism-trial-fixture.json @@ -0,0 +1,15 @@ +{ + "fixture_type": "P82_MINIMAL_MECHANISM_TRIAL", + "description": "Declarative A/B/C/D plan; no executor invocation or result acceptance.", + "executor": "fixed externally supplied mini-SWE-agent identity", + "treatments": ["A", "B", "C", "D"], + "mechanisms": { + "A": [], + "B": ["P8_SAFE_EDITOR"], + "C": ["P8_ENHANCED_REPOSITORY_CONTEXT"], + "D": ["P8_SAFE_EDITOR", "P8_ENHANCED_REPOSITORY_CONTEXT"] + }, + "controlled_dimensions": ["executor", "model", "environment", "budget", "task", "replicate", "verification", "acceptance_authority"], + "primary_metrics": ["verified_acceptance", "cost_to_acceptance"], + "guardrail": "false_pass_must_not_increase" +} diff --git a/experiments/p82-task-stratification-evidence.json b/experiments/p82-task-stratification-evidence.json new file mode 100644 index 00000000..7aafea88 --- /dev/null +++ b/experiments/p82-task-stratification-evidence.json @@ -0,0 +1,18 @@ +{ + "schema_version": "p8.2-task-stratification-v1", + "status": "DESIGN_ONLY", + "strata": ["T0", "T1", "T2", "T3", "T4", "T5"], + "external_evidence": [ + {"mechanism": "minimal_scaffold", "class": "official_benchmark", "source": "https://www.swebench.com/"}, + {"mechanism": "agentless_workflow", "class": "primary_paper", "source": "https://arxiv.org/abs/2407.01489"}, + {"mechanism": "repository_graph", "class": "project_report", "source": "https://github.com/ozyyshr/RepoGraph"}, + {"mechanism": "ast_repository_tools", "class": "project_report", "source": "https://github.com/AutoCodeRoverSG/auto-code-rover"}, + {"mechanism": "tree_search_refinement", "class": "peer_reviewed_paper", "source": "https://proceedings.iclr.cc/paper_files/paper/2025/hash/a1e6783e4d739196cad3336f12d402bf-Abstract-Conference.html"}, + {"mechanism": "trained_verifier", "class": "peer_reviewed_paper", "source": "https://arxiv.org/abs/2412.21139"}, + {"mechanism": "distribution_shift", "class": "peer_reviewed_paper", "source": "https://proceedings.mlr.press/v267/vergopoulos25a.html"} + ], + "wave0": { + "sympy__sympy-14711": "T1_PROVISIONAL", + "sympy__sympy-24443": "T2_PROVISIONAL" + } +} diff --git a/experiments/p82-wave0-comparison-protocol.json b/experiments/p82-wave0-comparison-protocol.json new file mode 100644 index 00000000..15bad8b7 --- /dev/null +++ b/experiments/p82-wave0-comparison-protocol.json @@ -0,0 +1,100 @@ +{ + "schema_version": "p8.2-comparison-v1", + "status": "PREDECLARED_NOT_EXECUTED", + "purpose": "Freeze the comparison matrix and outcome taxonomy before operational B1 or C1 implementation.", + "task_sample": { + "manifest": "experiments/p82-wave0-task-sample.json", + "tasks": [ + "sympy__sympy-14711", + "sympy__sympy-24443" + ], + "repository_revision_source": "task manifest base_commit per instance", + "task_statement_source": "task manifest problem_statement per instance" + }, + "treatments": { + "A": { + "definition": "original mini-SWE-agent", + "status": "CONTROL_IMPLEMENTED" + }, + "B1": { + "definition": "A + structured editing interface only", + "changed_variable": "editing interface", + "status": "CONTRACT_IMPLEMENTED_MECHANISM_NOT_IMPLEMENTED" + }, + "C1": { + "definition": "A + exactly one bounded pre-execution repository-localization intervention", + "changed_variable": "information presented before the agent's first action", + "status": "CONTRACT_IMPLEMENTED_MECHANISM_NOT_IMPLEMENTED" + }, + "D1": { + "definition": "A + B1 + C1", + "status": "NOT_AUTHORIZED_UNTIL_B1_AND_C1_HAVE_INDEPENDENT_EVIDENCE" + } + }, + "frozen_dimensions": { + "provider": "OpenRouter", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "model_version": "nvidia/nemotron-3-ultra-550b-a55b:free", + "generation": { + "temperature": 0, + "reasoning": "disabled", + "provider_allow_fallbacks": false + }, + "runner_version": "p8.2a-runner-v2", + "mini_version": "2.4.6", + "wall_time_seconds": 180, + "max_cost": "0", + "max_tokens": 64, + "verification_authority": "swebench.harness.run_evaluation", + "acceptance_authority": "swebench.harness.run_evaluation", + "replicates": 1 + }, + "outcomes": { + "primary": [ + "official_task_acceptance", + "official_resolution" + ], + "secondary": [ + "executor_completed", + "patch_created", + "verification_result", + "wall_time_seconds", + "model_calls", + "token_usage", + "cost" + ], + "mechanism_diagnostics": { + "B1": [ + "edit_attempts", + "malformed_edit_requests", + "applied_edits" + ], + "C1": [ + "candidate_files", + "candidate_symbols", + "evidence_budget", + "accepted_patch_touched_candidates" + ] + } + }, + "interpretation_rules": { + "BLOCKED": "not FAIL", + "EXECUTOR_FAILED": "not TASK_REJECTED", + "PATCH_CREATED": "not ACCEPTED", + "VERIFICATION_PASS": "not ACCEPTED", + "CONTRACT_TEST_PASS": "not MECHANISM_EVIDENCE" + }, + "diagnostic_classifications": { + "B1": [ + "structured_request_malformed", + "request_valid_edit_wrong", + "edit_correct_task_failed" + ], + "C1": [ + "localization_wrong", + "localization_plausible_agent_failed", + "localization_correct_patch_rejected" + ] + }, + "execution_gate": "Do not implement or activate operational B1/C1 until A completes real-agent preflight with a responsive provider/model." +} diff --git a/experiments/p82-wave0-task-sample.json b/experiments/p82-wave0-task-sample.json new file mode 100644 index 00000000..e0f01e22 --- /dev/null +++ b/experiments/p82-wave0-task-sample.json @@ -0,0 +1,47 @@ +{ + "schema_version": "p8.2a-wave0-v1", + "sample_id": "p82-wave0-official-verified-2026-09-23", + "wave": "wave-0-baseline-smoke", + "dataset": { + "identity": "SWE-bench/SWE-bench_Verified", + "split": "test", + "revision": "78f471bf655a3137b2e8a75af1501690ec009ec3", + "retrieved_at": "2026-09-23", + "source": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified" + }, + "selection": { + "method": "prospective deterministic first two records among the first 500 test records with the shortest public test metadata; no Arkx result consulted", + "protocol_version": "p8.2a-v1", + "frozen_pre_execution": true, + "acceptance_authority": "swebench.harness.run_evaluation", + "strata": "UNKNOWN: no auditable prior Arkx stratification evidence was available" + }, + "tasks": [ + { + "instance_id": "sympy__sympy-14711", + "repo": "sympy/sympy", + "base_commit": "c6753448b5c34f95e250105d76709fe4d349ca1f", + "problem_statement": "vector add 0 error\n```python\nfrom sympy.physics.vector import ReferenceFrame, Vector\nfrom sympy import symbols\nsum([N.x, (0 * N.x)])\n```\ngives a TypeError because adding zero to a vector is not accepted.\n", + "version": "1.1", + "environment_setup_commit": "ec9e3c0436fbff934fa84e22bf07f1b3ef5bfac3", + "FAIL_TO_PASS": ["test_Vector"], + "PASS_TO_PASS": ["test_Vector_diffs", "test_vector_var_in_dcm"], + "test_patch": "diff --git a/sympy/physics/vector/tests/test_vector.py b/sympy/physics/vector/tests/test_vector.py\n--- a/sympy/physics/vector/tests/test_vector.py\n+++ b/sympy/physics/vector/tests/test_vector.py\n@@ -13,6 +13,8 @@ def test_Vector():\n assert A.x != A.y\n assert A.y != A.z\n assert A.z != A.x\n \n+ assert A.x + 0 == A.x\n+\n v1 = x*A.x + y*A.y + z*A.z\n v2 = x**2*A.x + y**2*A.y + z**2*A.z\n v3 = v1 + v2\n", + "stratum": "UNKNOWN", + "rationale": "Compact public record selected by the frozen prospective rule; no unsupported mechanism stratum inferred." + }, + { + "instance_id": "sympy__sympy-24443", + "repo": "sympy/sympy", + "base_commit": "809c53c077485ca48a206cee78340389cb83b7f1", + "problem_statement": "`_check_homomorphism` is broken on PermutationGroups\n```python\nfrom sympy.combinatorics import *\nfrom sympy.combinatorics.homomorphisms import homomorphism\nD3 = DihedralGroup(3)\nT = homomorphism(D3, D3, D3.generators, D3.generators)\n```\nraises ValueError although the images define a homomorphism. The issue is in `_image()` when an inverted generator is handled.\n", + "version": "1.12", + "environment_setup_commit": "c6cb7c5602fa48034ab1bd43c2347a7e8488f12e", + "FAIL_TO_PASS": ["test_homomorphism"], + "PASS_TO_PASS": ["test_isomorphisms"], + "test_patch": "diff --git a/sympy/combinatorics/tests/test_homomorphisms.py b/sympy/combinatorics/tests/test_homomorphisms.py\n--- a/sympy/combinatorics/tests/test_homomorphisms.py\n+++ b/sympy/combinatorics/tests/test_homomorphisms.py\n@@ -57,6 +57,11 @@ def test_homomorphism():\n assert T.codomain == D\n assert T(a*b) == p\n \n+ D3 = DihedralGroup(3)\n+ T = homomorphism(D3, D3, D3.generators, D3.generators)\n+ assert T.is_isomorphism()\n+\n+\n def test_isomorphisms():\n \n F, a, b = free_group(\"a, b\")\n", + "stratum": "UNKNOWN", + "rationale": "Compact public record selected by the frozen prospective rule; no unsupported mechanism stratum inferred." + } + ] +} diff --git a/experiments/patch-verification-fixture.json b/experiments/patch-verification-fixture.json new file mode 100644 index 00000000..85429e0b --- /dev/null +++ b/experiments/patch-verification-fixture.json @@ -0,0 +1,12 @@ +{ + "fixture_type": "PATCH_VERIFICATION_FIXTURE", + "description": "Synthetic verification gate scenarios; not an acceptance benchmark.", + "scenarios": [ + "A_verified_patch", + "B_missing_reproduction", + "C_regression_failure", + "D_post_patch_reproduction_failure", + "E_scope_expansion", + "F_incomplete_evidence" + ] +} diff --git a/experiments/progress-detection-fixture.json b/experiments/progress-detection-fixture.json new file mode 100644 index 00000000..37876b22 --- /dev/null +++ b/experiments/progress-detection-fixture.json @@ -0,0 +1,108 @@ +{ + "fixture_type": "PROGRESS_DETECTION_FIXTURE", + "description": "Synthetic progression and stagnation sequences; not a real-world benchmark.", + "sequences": [ + { + "id": "A_productive_progression", + "expected": "PROGRESS_PROVEN", + "previous": { + "useful_files": ["src/a.py"], + "passing_tests": [], + "explained_failures": [], + "diff_distance": 4, + "acceptance_distance": 3, + "recent_actions": ["inspect a"], + "failure_signatures": ["missing-test"] + }, + "current": { + "useful_files": ["src/a.py", "tests/test_a.py"], + "passing_tests": ["test_a"], + "explained_failures": ["missing-test"], + "diff_distance": 3, + "acceptance_distance": 2, + "recent_actions": ["inspect a", "add test_a"], + "failure_signatures": ["missing-test"] + } + }, + { + "id": "B_repeated_failure_without_new_evidence", + "expected": "NO_PROGRESS", + "previous": { + "useful_files": ["src/a.py"], + "passing_tests": [], + "explained_failures": [], + "diff_distance": 4, + "acceptance_distance": 3, + "recent_actions": ["run test_a"], + "failure_signatures": ["assertion-a"] + }, + "current": { + "useful_files": ["src/a.py"], + "passing_tests": [], + "explained_failures": [], + "diff_distance": 4, + "acceptance_distance": 3, + "recent_actions": ["run test_a"], + "failure_signatures": ["assertion-a"] + } + }, + { + "id": "C_partial_progress", + "expected": "PROGRESS_PROVEN", + "previous": { + "useful_files": ["src/a.py"], + "passing_tests": [], + "explained_failures": [], + "diff_distance": 5, + "acceptance_distance": 4, + "recent_actions": ["inspect a"], + "failure_signatures": ["assertion-a"] + }, + "current": { + "useful_files": ["src/a.py", "src/contracts.py"], + "passing_tests": [], + "explained_failures": ["assertion-a"], + "diff_distance": 5, + "acceptance_distance": 4, + "recent_actions": ["inspect a", "inspect contracts"], + "failure_signatures": ["assertion-a"] + } + }, + { + "id": "D_unknown_missing_telemetry", + "expected": "UNKNOWN", + "previous": null, + "current": { + "useful_files": null, + "passing_tests": null, + "explained_failures": null, + "diff_distance": null, + "acceptance_distance": null, + "recent_actions": null, + "failure_signatures": null + } + }, + { + "id": "E_contradictory_signals", + "expected": "UNKNOWN", + "previous": { + "useful_files": [], + "passing_tests": [], + "explained_failures": [], + "diff_distance": -1, + "acceptance_distance": 1, + "recent_actions": [], + "failure_signatures": [] + }, + "current": { + "useful_files": ["src/a.py"], + "passing_tests": [], + "explained_failures": [], + "diff_distance": 0, + "acceptance_distance": 1, + "recent_actions": [], + "failure_signatures": [] + } + } + ] +} diff --git a/experiments/repository-planning-fixture.json b/experiments/repository-planning-fixture.json new file mode 100644 index 00000000..a5b35b53 --- /dev/null +++ b/experiments/repository-planning-fixture.json @@ -0,0 +1,11 @@ +{ + "fixture_type": "REPOSITORY_PLANNING_FIXTURE", + "description": "Synthetic planning stability scenarios; not a performance benchmark.", + "scenarios": [ + {"id": "A_multi_file_bounded_plan", "components": ["api", "tests"], "files": ["src/api.py", "tests/test_api.py"]}, + {"id": "B_dependency_chain", "components": ["a", "b", "c"], "files": ["src/a.py", "src/b.py", "src/c.py"], "edges": [["inspect-01", "inspect-02"], ["inspect-02", "inspect-03"]]}, + {"id": "C_invalid_dependency", "invalid": true}, + {"id": "D_replan_after_discovered_dependency", "trigger": "DEPENDENCY_DISCOVERED", "previous_plan_ref": "plan://old"}, + {"id": "E_unresolved_question", "components": ["unknown"], "unresolved_questions": ["Which acceptance surface is authoritative?"]} + ] +} diff --git a/experiments/routing-escalation-fixture.json b/experiments/routing-escalation-fixture.json new file mode 100644 index 00000000..7f6083c4 --- /dev/null +++ b/experiments/routing-escalation-fixture.json @@ -0,0 +1,13 @@ +{ + "fixture_type": "ROUTING_POLICY_FIXTURE", + "description": "Synthetic policy stability scenarios; not an accuracy benchmark.", + "scenarios": [ + {"id": "A_simple_success_path", "route_scope": "SIMPLE", "expected_path": "SIMPLE_PATH", "progress": "PROGRESS_PROVEN", "evidence": "SUFFICIENT", "expected_action": "STOP_SUFFICIENT_EVIDENCE"}, + {"id": "B_simple_to_localized", "route_scope": "SIMPLE", "expected_path": "SIMPLE_PATH", "progress": "NO_PROGRESS", "evidence": "INSUFFICIENT", "expected_action": "ESCALATE", "expected_target": "LOCALIZED_PATH"}, + {"id": "C_localized_to_repository_wide", "route_scope": "LOCALIZED", "expected_path": "LOCALIZED_PATH", "progress": "NO_PROGRESS", "evidence": "INSUFFICIENT", "expected_action": "ESCALATE", "expected_target": "REPOSITORY_WIDE_PATH"}, + {"id": "D_repository_wide_stagnation", "route_scope": "REPOSITORY_WIDE", "expected_path": "REPOSITORY_WIDE_PATH", "progress": "NO_PROGRESS", "evidence": "INSUFFICIENT", "expected_action": "BLOCK"}, + {"id": "E_unknown_characterization", "route_scope": "UNKNOWN", "expected_path": "QUALIFICATION_REQUIRED", "progress": "UNKNOWN", "evidence": "UNKNOWN", "expected_route_decision": "REQUIRE_QUALIFICATION"}, + {"id": "F_exhausted_budget", "route_scope": "SIMPLE", "expected_path": "SIMPLE_PATH", "progress": "NO_PROGRESS", "evidence": "INSUFFICIENT", "attempts_used": 3, "path_escalations_used": 2, "expected_action": "BLOCK"}, + {"id": "G_sufficient_evidence_early_stop", "route_scope": "LOCALIZED", "expected_path": "LOCALIZED_PATH", "progress": "PROGRESS_PROVEN", "evidence": "SUFFICIENT", "expected_action": "STOP_SUFFICIENT_EVIDENCE"} + ] +} diff --git a/logs/architecture/adapter-preflight-error-envelope-block-20260923.json b/logs/architecture/adapter-preflight-error-envelope-block-20260923.json new file mode 100644 index 00000000..e972147e --- /dev/null +++ b/logs/architecture/adapter-preflight-error-envelope-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-adapter-preflight-error-envelope-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "isolated unittest discovery excluding tests.test_p82_openrouter_adapter", + "focused_suite": {"tests_run": 294, "result": "PASS"}, + "criteria": { + "boundary_preserved": "PASS", + "stable_preflight_error_code": "PASS", + "unknown_retryability_preserved": "PASS", + "raw_preflight_reference_attached": "PASS", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/adapter-qualification-preflight-block-20260923.json b/logs/architecture/adapter-qualification-preflight-block-20260923.json new file mode 100644 index 00000000..c4e34776 --- /dev/null +++ b/logs/architecture/adapter-qualification-preflight-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-adapter-qualification-preflight-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "focused_suite": {"tests_run": 294, "result": "PASS", "excluded": "tests.test_p82_openrouter_adapter"}, + "criteria": { + "ready_status_distinguished": "PASS", + "blocked_status_distinguished": "PASS", + "unknown_status_distinguished": "PASS", + "stable_serialization": "PASS", + "fallback_selected": "NO", + "error_envelope_boundary_mapping": "PASS", + "unknown_retryability_preserved": "PASS" + }, + "acceptance": "LOCAL_ADAPTER_PREFLIGHT_CONTRACT_PASS_EXTERNAL_ADAPTERS_UNQUALIFIED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/antigravity-preflight-20260924.json b/logs/architecture/antigravity-preflight-20260924.json new file mode 100644 index 00000000..065c1b76 --- /dev/null +++ b/logs/architecture/antigravity-preflight-20260924.json @@ -0,0 +1,28 @@ +{ + "project": "Codepro", + "date": "2026-09-24", + "candidate": "Antigravity/Google CLI", + "checks": { + "processes": { + "commands": "Get-Process filtered by antigravity|google|gemini|gravity", + "observed": [], + "status": "NOT_DISCOVERED" + }, + "path_commands": { + "commands": "Get-Command filtered by gravity|gemini|google", + "observed": [], + "status": "NOT_DISCOVERED" + }, + "start_menu_desktop_entries": { + "commands": "recursive read-only search for Antigravity|Gemini|Google", + "observed": [], + "status": "NOT_DISCOVERED" + }, + "computer_use_surface": { + "observed": "UI automation RPC unavailable: Trusted RPC service is not configured: sky", + "status": "UNAVAILABLE" + } + }, + "qualification_decision": "NOT_DISCOVERED", + "reason": "A logged-in UI session was not exposed as a local headless executable or inspectable automation surface; no executor contract evidence exists." +} diff --git a/logs/architecture/architecture-audit-final-20260923.json b/logs/architecture/architecture-audit-final-20260923.json new file mode 100644 index 00000000..50910634 --- /dev/null +++ b/logs/architecture/architecture-audit-final-20260923.json @@ -0,0 +1,41 @@ +{ + "audit": "architecture-audit-final", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "audit_source_head": "0eff3dca", + "local_tests": {"command": "PYTHONPATH=src python -m unittest discover -s tests -p test_*.py", "passed": 338}, + "diff_check": "PASS", + "final_suite": {"command": "PYTHONPATH=src python -m unittest discover -s tests -p 'test_*.py'", "passed": 348}, + "resume_audit_verified": true, + "recovery_attempt_identity_verified": true, + "recovery_lineage_audited": true, + "recovery_manifest_binding_verified": true, + "retry_recovery_budget_gate_verified": true, + "persisted_budget_ledger_verified": true, + "budget_ledger_replay_verified": true, + "lineage_event_chain_audit_verified": true, + "latest_verified_suite": {"command": "PYTHONPATH=src python -m unittest discover -s tests -p 'test_*.py'", "passed": 367}, + "budget_enforced_legacy_runner_verified": true, + "latest_verified_suite_after_budget_boundary": {"command": "PYTHONPATH=src python -m unittest discover -s tests -p 'test_*.py'", "passed": 366}, + "orchestration_configuration_snapshot_verified": true, + "latest_verified_suite_after_configuration_snapshot": {"command": "PYTHONPATH=src python -m unittest discover -s tests -p 'test_*.py'", "passed": 367}, + "persistence_coordination_scope_verified": true, + "suite_history": [ + {"passed": 359, "label": "prior local suite"}, + {"passed": 361, "label": "budget/recovery verification"}, + {"passed": 364, "label": "replay ledger verification"}, + {"passed": 365, "label": "lineage event-chain verification"}, + {"passed": 366, "label": "budget-enforced legacy runner"}, + {"passed": 367, "label": "orchestration configuration snapshot"} + ], + "external_qualification": "NOT_COMPLETE", + "external_authority": {"swebench_docker_gold": "resolved_true_for_frozen_tasks", "real_provider_model": "UNQUALIFIED", "mechanism_trial": "UNQUALIFIED"}, + "critical_gaps": [ + "legacy runners without explicit budget enforcement remain blocked", + "orchestration remains a large composition seam, with extraction deferred until an isolated consumer requires it", + "execution and orchestration state vocabularies remain intentionally separate and are crosswalked explicitly", + "configuration snapshot propagation is available but optional for legacy callers; real-runtime crash-resume effectiveness remains unqualified", + "concrete provider/executor/sandbox qualification remains blocked" + ], + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/artifact-store-retry-lineage-block-20260923.json b/logs/architecture/artifact-store-retry-lineage-block-20260923.json new file mode 100644 index 00000000..23a0dd79 --- /dev/null +++ b/logs/architecture/artifact-store-retry-lineage-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-artifact-store-retry-lineage-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Retry lineage can be reserved append-only and persisted atomically without executor invocation or artifact overwrite.", + "focused_test": {"result": "PASS", "tests_run": 111}, + "criteria": { + "lineage_persisted": "PASS", + "duplicate_reservation_rejected": "PASS", + "identity_drift_rejected": "PASS", + "executor_invoked": "NO", + "attempt_overwritten": "NO" + }, + "acceptance": "LOCAL_ARTIFACT_LINEAGE_EVIDENCE_PASS_EXECUTION_SEPARATE", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/atomic-manifest-persistence-block-20260923.json b/logs/architecture/atomic-manifest-persistence-block-20260923.json new file mode 100644 index 00000000..32efbb3e --- /dev/null +++ b/logs/architecture/atomic-manifest-persistence-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-atomic-manifest-persistence-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "A budget-bound execution manifest can be validated and persisted atomically for later replay.", + "focused_test": {"result": "PASS", "tests_run": 119}, + "criteria": { + "manifest_round_trip": "PASS", + "identity_and_lifecycle_validated": "PASS", + "atomic_write_primitive_used": "PASS", + "verification_or_acceptance_inferred": "NO" + }, + "acceptance": "LOCAL_ATOMIC_MANIFEST_EVIDENCE_PASS_VERIFICATION_SEPARATE", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/atomic-recovery-attempt-reservation-block-20260923.json b/logs/architecture/atomic-recovery-attempt-reservation-block-20260923.json new file mode 100644 index 00000000..87123f4f --- /dev/null +++ b/logs/architecture/atomic-recovery-attempt-reservation-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "atomic-recovery-attempt-reservation", + "date": "2026-09-23", + "hypothesis": "A recovery attempt cannot overwrite an existing attempt or lose its source lineage.", + "criteria": {"lineage_persisted": "PASS", "identity_is_new": "PASS", "overwrite_rejected": "PASS"}, + "local_tests_verified": 356, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/attempt-inventory-replay-block-20260923.json b/logs/architecture/attempt-inventory-replay-block-20260923.json new file mode 100644 index 00000000..c0a56aac --- /dev/null +++ b/logs/architecture/attempt-inventory-replay-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-attempt-inventory-replay-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Replay chain references can be classified against the validated attempt inventory without scanning or selecting artifacts.", + "focused_test": {"result": "PASS", "tests_run": 130}, + "criteria": { + "validated_artifact_present": "PASS", + "missing_reference_detected": "PASS", + "arbitrary_directory_scan": "NO", + "artifact_substitution": "NO" + }, + "acceptance": "LOCAL_ATTEMPT_INVENTORY_REPLAY_EVIDENCE_PASS", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/attempt-store-lineage-replay-block-20260923.json b/logs/architecture/attempt-store-lineage-replay-block-20260923.json new file mode 100644 index 00000000..9ad41b1d --- /dev/null +++ b/logs/architecture/attempt-store-lineage-replay-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-attempt-store-lineage-replay-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Attempt loading can validate retry lineage while preserving historical artifacts and representing missing lineage explicitly.", + "focused_test": {"result": "PASS", "tests_run": 112}, + "criteria": { + "valid_lineage_round_trip": "PASS", + "lineage_drift_rejected": "PASS", + "legacy_missing_lineage_preserved": "PASS", + "retry_chain_inferred_from_attempt_number": "NO" + }, + "acceptance": "LOCAL_LINEAGE_REPLAY_EVIDENCE_PASS_LEGACY_MISSING_EXPLICIT", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/audited-attempt-loading-block-20260923.json b/logs/architecture/audited-attempt-loading-block-20260923.json new file mode 100644 index 00000000..23b159d5 --- /dev/null +++ b/logs/architecture/audited-attempt-loading-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-audited-attempt-loading-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 311, "result": "PASS"}, + "criteria": { + "audited_load_delegates_to_chain_audit": "PASS", + "legacy_load_compatibility_preserved": "PASS", + "external_execution_during_load": "NO", + "incomplete_chain_inferred_complete": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/audited-comparison-gate-block-20260923.json b/logs/architecture/audited-comparison-gate-block-20260923.json new file mode 100644 index 00000000..a2c69900 --- /dev/null +++ b/logs/architecture/audited-comparison-gate-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-audited-comparison-gate-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 312, "result": "PASS"}, + "criteria": { + "both_attempts_loaded_and_audited": "PASS", + "incomplete_chain_rejected_for_decision_comparison": "PASS", + "legacy_identity_comparison_preserved": "PASS", + "comparison_alone_accepts_or_promotes": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/audited-experiment-reproducibility-gate-block-20260923.json b/logs/architecture/audited-experiment-reproducibility-gate-block-20260923.json new file mode 100644 index 00000000..3fe775d1 --- /dev/null +++ b/logs/architecture/audited-experiment-reproducibility-gate-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-audited-experiment-reproducibility-gate-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 314, "result": "PASS"}, + "criteria": { + "complete_audited_experiment_accepted": "PASS", + "incomplete_attempt_chain_rejected": "PASS", + "declared_reference_presence_checked": "PASS", + "legacy_record_validation_preserved": "PASS", + "external_reexecution_during_validation": "NO", + "promotion_or_qualification_inferred": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/audited-resume-decision-block-20260923.json b/logs/architecture/audited-resume-decision-block-20260923.json new file mode 100644 index 00000000..b65ff45d --- /dev/null +++ b/logs/architecture/audited-resume-decision-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "audited-resume-decision", + "date": "2026-09-23", + "hypothesis": "An audited attempt cannot be consumed for resume without an explicit decision.", + "criteria": {"resume_decision_exposed": "PASS", "restart_required_without_execution": "PASS", "audit_mutated_artifacts": "NO"}, + "local_tests_verified": 355, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/baseline-executor-adapter-block-20260923.json b/logs/architecture/baseline-executor-adapter-block-20260923.json new file mode 100644 index 00000000..171063b9 --- /dev/null +++ b/logs/architecture/baseline-executor-adapter-block-20260923.json @@ -0,0 +1,21 @@ +{ + "schema_version": "arkx-baseline-executor-adapter-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "A thin adapter can expose the P8.2 baseline runner through the neutral Executor contract without changing its execution semantics.", + "focused_test": { + "command": "PYTHONPATH=src python -m unittest tests.test_execution tests.test_execution_lifecycle tests.test_p82_baseline tests.test_harness tests.test_promotion tests.test_core_outcomes tests.test_p0_telemetry tests.test_swebench_authority", + "result": "PASS", + "tests_run": 84 + }, + "criteria": { + "neutral_execution_result": "PASS", + "wrong_task_rejected": "PASS", + "legacy_runner_preserved": "PASS", + "real_provider_invoked": false, + "real_executor_qualified": false + }, + "acceptance": "LOCAL_ADAPTER_CONTRACT_EVIDENCE_PASS_EXTERNAL_INTEGRATION_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/budget-aware-lifecycle-block-20260923.json b/logs/architecture/budget-aware-lifecycle-block-20260923.json new file mode 100644 index 00000000..0b2b82da --- /dev/null +++ b/logs/architecture/budget-aware-lifecycle-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-budget-aware-lifecycle-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Budget consumption can gate executor invocation deterministically within the contract lifecycle.", + "focused_test": {"result": "PASS", "tests_run": 118}, + "criteria": { + "accepted_budget_invokes_executor_once": "PASS", + "accepted_budget_updates_ledger": "PASS", + "exhausted_budget_blocks": "PASS", + "blocked_budget_invokes_executor": "NO" + }, + "acceptance": "LOCAL_BUDGET_AWARE_LIFECYCLE_EVIDENCE_PASS_EXTERNAL_EXECUTOR_PENDING", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/budget-enforced-legacy-runner-block-20260923.json b/logs/architecture/budget-enforced-legacy-runner-block-20260923.json new file mode 100644 index 00000000..751df34a --- /dev/null +++ b/logs/architecture/budget-enforced-legacy-runner-block-20260923.json @@ -0,0 +1,14 @@ +{ + "block": "budget-enforced-legacy-runner", + "date": "2026-09-23", + "hypothesis": "A budgeted request cannot reach a runner that has not declared the budget enforcement capability.", + "criteria": { + "legacy_runner_blocked_before_run": "PASS", + "contract_bridge_requires_reserved_attempt": "PASS", + "missing_usage_remains_blocked": "PASS", + "local_suite": "PASS", + "external_qualification": "NOT_PERFORMED" + }, + "local_tests_verified": 366, + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/budget-ledger-replay-binding-block-20260923.json b/logs/architecture/budget-ledger-replay-binding-block-20260923.json new file mode 100644 index 00000000..c9503b66 --- /dev/null +++ b/logs/architecture/budget-ledger-replay-binding-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "budget-ledger-replay-binding", + "date": "2026-09-23", + "hypothesis": "Budget consumption cannot differ between artifact manifest and event-log replay.", + "criteria": {"ledger_restored": "PASS", "ledger_drift_rejected": "PASS", "legacy_ledger_inferred": "NO"}, + "local_tests_verified": 364, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/canonical-configuration-snapshot-block-20260923.json b/logs/architecture/canonical-configuration-snapshot-block-20260923.json new file mode 100644 index 00000000..def16b37 --- /dev/null +++ b/logs/architecture/canonical-configuration-snapshot-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "canonical-configuration-snapshot", + "date": "2026-09-23", + "hypothesis": "Equivalent configuration mappings produce the same identity while changes produce different identities without serializing secrets.", + "criteria": {"canonical_digest": "PASS", "secret_material_absent": "PASS", "identity_change_detected": "PASS"}, + "local_tests_verified": 344, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/capability-preflight-provenance-block-20260923.json b/logs/architecture/capability-preflight-provenance-block-20260923.json new file mode 100644 index 00000000..4c88df91 --- /dev/null +++ b/logs/architecture/capability-preflight-provenance-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-capability-preflight-provenance-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 318, "result": "PASS"}, + "criteria": { + "capability_digest_transported": "PASS", + "declared_and_observed_distinguished": "PASS", + "qualified_requires_evidence": "PASS", + "ready_implies_qualified": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/capability-profile-registry-block-20260923.json b/logs/architecture/capability-profile-registry-block-20260923.json new file mode 100644 index 00000000..e876cf56 --- /dev/null +++ b/logs/architecture/capability-profile-registry-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-capability-profile-registry-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Capability identity, configuration, declarations, and evidence can be represented deterministically without runtime discovery or executor coupling.", + "focused_test": {"result": "PASS", "tests_run": 104}, + "criteria": { + "stable_profile_serialization": "PASS", + "stable_registry_ordering": "PASS", + "duplicate_identity_rejected": "PASS", + "external_discovery_invoked": "NO" + }, + "acceptance": "LOCAL_CAPABILITY_CONTRACT_EVIDENCE_PASS_EXTERNAL_QUALIFICATION_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/capability-provenance-routing-policy-block-20260923.json b/logs/architecture/capability-provenance-routing-policy-block-20260923.json new file mode 100644 index 00000000..66dc1442 --- /dev/null +++ b/logs/architecture/capability-provenance-routing-policy-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-capability-provenance-routing-policy-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 321, "result": "PASS"}, + "criteria": { + "declared_capability_cannot_satisfy_observed_policy": "PASS", + "blocked_preflight_denies_routing": "PASS", + "observed_capability_can_satisfy_observed_policy": "PASS", + "fallback_or_executor_switch_selected": "NO", + "qualification_inferred_from_policy": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/causal-recovery-persistence-block-20260923.json b/logs/architecture/causal-recovery-persistence-block-20260923.json new file mode 100644 index 00000000..c61d39fe --- /dev/null +++ b/logs/architecture/causal-recovery-persistence-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "causal-recovery-persistence", + "date": "2026-09-23", + "hypothesis": "A recovery artifact cannot float free of the attempt it claims to recover.", + "criteria": {"matching_execution_accepted": "PASS", "mismatched_execution_rejected": "PASS", "implicit_replan_executed": "NO"}, + "local_tests_verified": 344, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/causal-stage-event-persistence-block-20260923.json b/logs/architecture/causal-stage-event-persistence-block-20260923.json new file mode 100644 index 00000000..35408182 --- /dev/null +++ b/logs/architecture/causal-stage-event-persistence-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-causal-stage-event-persistence-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 306, "result": "PASS"}, + "criteria": { + "causal_stage_append": "PASS", + "idempotent_duplicate_stage": "PASS", + "out_of_order_rejected": "PASS", + "stage_reference_drift_rejected": "PASS", + "external_reexecution_during_replay": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/causal-verification-acceptance-promotion-chain-block-20260923.json b/logs/architecture/causal-verification-acceptance-promotion-chain-block-20260923.json new file mode 100644 index 00000000..970b34f6 --- /dev/null +++ b/logs/architecture/causal-verification-acceptance-promotion-chain-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-causal-verification-acceptance-promotion-chain-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 304, "result": "PASS"}, + "criteria": { + "verification_reference_propagated_to_acceptance": "PASS", + "unlinked_acceptance_blocked": "PASS", + "linked_accepted_outcome_can_pass_gate": "PASS", + "non_pass_verification_not_promoted": "PASS", + "legacy_promotion_compatibility_preserved": "PASS", + "implicit_fallback_or_promotion": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/concrete-adapter-preflight-20260923.json b/logs/architecture/concrete-adapter-preflight-20260923.json new file mode 100644 index 00000000..b84b0b54 --- /dev/null +++ b/logs/architecture/concrete-adapter-preflight-20260923.json @@ -0,0 +1,24 @@ +{ + "schema_version": "arkx-concrete-adapter-preflight-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9"}, + "collector": "tools/collect_adapter_preflight.py", + "read_only": true, + "records": [ + {"kind": "PROVIDER", "name": "openrouter", "status": "BLOCKED", "dependency": "minisweagent", "reason": "python module unavailable"}, + {"kind": "SANDBOX", "name": "git-bash", "status": "BLOCKED", "dependency": "bash", "reason": "executable unavailable or access denied"}, + {"kind": "EVALUATOR", "name": "docker-swebench", "status": "BLOCKED", "dependency": "docker-daemon", "version": "Docker version 29.7.2, build a7dcaa6", "reason": "permission denied while trying to connect to dockerDesktopLinuxEngine"}, + {"kind": "PROVIDER", "name": "ollama", "status": "UNKNOWN", "dependency": "ollama", "version": "ollama version is 0.34.3", "reason": "configuration identity not frozen"} + ], + "isolated_suite": {"tests_run": 298, "result": "PASS", "excluded": null}, + "references": { + "openrouter": "preflight://eaacd4c66101892f", + "git-bash": "preflight://21c3ab3a89acff8f", + "docker-swebench": "preflight://7075937eb9c95440", + "ollama": "preflight://aee4cf549055d570" + }, + "fallback_selected": false, + "acceptance": "CONCRETE_PREFLIGHT_RECORDED_EXTERNAL_BOUNDARIES_NOT_QUALIFIED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/concrete-adapter-preflight-plan-binding-block-20260923.json b/logs/architecture/concrete-adapter-preflight-plan-binding-block-20260923.json new file mode 100644 index 00000000..be872cb3 --- /dev/null +++ b/logs/architecture/concrete-adapter-preflight-plan-binding-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-concrete-adapter-preflight-plan-binding-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 325, "result": "PASS"}, + "criteria": { + "concrete_preflight_is_read_only": "PASS", + "executor_identity_bound": "PASS", + "configuration_identity_bound": "PASS", + "mismatched_preflight_rejected": "PASS", + "fallback_selected": "NO", + "external_qualification": "NOT_PERFORMED" + }, + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/contract-lifecycle-block-20260923.json b/logs/architecture/contract-lifecycle-block-20260923.json new file mode 100644 index 00000000..b689fb98 --- /dev/null +++ b/logs/architecture/contract-lifecycle-block-20260923.json @@ -0,0 +1,21 @@ +{ + "schema_version": "arkx-contract-lifecycle-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "A fake executor can traverse completed and blocked flows while verification and acceptance remain independent attachments.", + "focused_test": { + "command": "PYTHONPATH=src python -m unittest tests.test_execution tests.test_execution_lifecycle tests.test_harness tests.test_promotion tests.test_p82_baseline tests.test_core_outcomes tests.test_p0_telemetry tests.test_swebench_authority", + "result": "PASS", + "tests_run": 82 + }, + "criteria": { + "completed_lifecycle": "PASS", + "completion_does_not_imply_acceptance": "PASS", + "independent_verification_and_acceptance": "PASS", + "blocked_requires_error": "PASS", + "real_executor_invoked": false + }, + "acceptance": "LOCAL_CONTRACT_EVIDENCE_PASS_EXTERNAL_INTEGRATION_NOT_ATTEMPTED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/contract-orchestration-bridge-block-20260923.json b/logs/architecture/contract-orchestration-bridge-block-20260923.json new file mode 100644 index 00000000..3f74f5b9 --- /dev/null +++ b/logs/architecture/contract-orchestration-bridge-block-20260923.json @@ -0,0 +1,21 @@ +{ + "schema_version": "arkx-contract-orchestration-bridge-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "The existing orchestration gates remain behaviorally stable when a neutral Executor is consumed through one explicit bridge.", + "focused_test": { + "command": "PYTHONPATH=src python -m unittest tests.test_orchestration tests.test_execution tests.test_execution_lifecycle tests.test_p82_baseline tests.test_harness tests.test_promotion tests.test_core_outcomes tests.test_p0_telemetry tests.test_swebench_authority", + "result": "PASS", + "tests_run": 92 + }, + "criteria": { + "neutral_executor_reaches_acceptance": "PASS", + "blocked_execution_remains_blocked": "PASS", + "selection_identity_forwarded": "PASS", + "real_provider_invoked": false, + "real_executor_qualified": false + }, + "acceptance": "LOCAL_ORCHESTRATION_BRIDGE_EVIDENCE_PASS_EXTERNAL_INTEGRATION_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/dependency-isolated-suite-block-20260923.json b/logs/architecture/dependency-isolated-suite-block-20260923.json new file mode 100644 index 00000000..bd9eab96 --- /dev/null +++ b/logs/architecture/dependency-isolated-suite-block-20260923.json @@ -0,0 +1,14 @@ +{ + "schema_version": "arkx-dependency-isolated-suite-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "command": "PYTHONPATH=src python -c ", + "excluded_module": "tests.test_p82_openrouter_adapter", + "tests_run": 290, + "result": "PASS", + "interpretation": "Broad local regression evidence; external OpenRouter/minisweagent integration remains unqualified.", + "fallback_selected": false, + "acceptance": "LOCAL_DEPENDENCY_ISOLATED_SUITE_PASS_EXTERNAL_ADAPTER_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/deterministic-event-replay-block-20260923.json b/logs/architecture/deterministic-event-replay-block-20260923.json new file mode 100644 index 00000000..3b713f3c --- /dev/null +++ b/logs/architecture/deterministic-event-replay-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-deterministic-event-replay-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Persisted lifecycle events can reconstruct declared facts deterministically without re-execution or acceptance inference.", + "focused_test": {"result": "PASS", "tests_run": 123}, + "criteria": { + "declared_facts_reconstructed": "PASS", + "identity_validated": "PASS", + "no_executor_or_verifier_invoked": "PASS", + "acceptance_inferred": "NO" + }, + "acceptance": "LOCAL_EVENT_REPLAY_EVIDENCE_PASS_REEXECUTION_SEPARATE", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/direct-neutral-orchestration-block-20260923.json b/logs/architecture/direct-neutral-orchestration-block-20260923.json new file mode 100644 index 00000000..d0df3b53 --- /dev/null +++ b/logs/architecture/direct-neutral-orchestration-block-20260923.json @@ -0,0 +1,20 @@ +{ + "schema_version": "arkx-direct-neutral-orchestration-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Direct neutral-executor orchestration produces the same result as the explicit legacy bridge.", + "focused_test": { + "command": "PYTHONPATH=src python -m unittest tests.test_orchestration tests.test_execution tests.test_execution_lifecycle tests.test_p82_baseline tests.test_harness tests.test_promotion tests.test_core_outcomes tests.test_p0_telemetry tests.test_swebench_authority", + "result": "PASS", + "tests_run": 94 + }, + "criteria": { + "direct_neutral_executor": "PASS", + "direct_bridge_equivalence": "PASS", + "blocked_and_failed_fail_closed": "PASS", + "fallback_or_executor_switch": false + }, + "acceptance": "LOCAL_ORCHESTRATION_EQUIVALENCE_EVIDENCE_PASS_EXTERNAL_INTEGRATION_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/evaluator-failure-outcome-separation-block-20260923.json b/logs/architecture/evaluator-failure-outcome-separation-block-20260923.json new file mode 100644 index 00000000..1b5b71d4 --- /dev/null +++ b/logs/architecture/evaluator-failure-outcome-separation-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-evaluator-failure-outcome-separation-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 302, "result": "PASS"}, + "criteria": { + "infrastructure_error_maps_to_verification_domain": "PASS", + "test_failure_is_not_error_envelope": "PASS", + "raw_report_and_log_references_preserved": "PASS", + "unknown_retryability_preserved": "PASS", + "acceptance_mapping_remains_separate": "PASS", + "fallback_selected": "NO" + }, + "docker_swebench_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/evaluator-to-verification-contract-block-20260923.json b/logs/architecture/evaluator-to-verification-contract-block-20260923.json new file mode 100644 index 00000000..337ef0f1 --- /dev/null +++ b/logs/architecture/evaluator-to-verification-contract-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-evaluator-to-verification-contract-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 303, "result": "PASS"}, + "criteria": { + "resolved_maps_to_verification_pass": "PASS", + "tests_failed_maps_to_verification_fail": "PASS", + "infrastructure_and_ambiguous_map_to_indeterminate": "PASS", + "evidence_is_deterministically_ordered": "PASS", + "acceptance_remains_separate": "PASS", + "promotion_implicitly_performed": "NO" + }, + "docker_swebench_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/execution-budget-spec-block-20260923.json b/logs/architecture/execution-budget-spec-block-20260923.json new file mode 100644 index 00000000..07c4b9bf --- /dev/null +++ b/logs/architecture/execution-budget-spec-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-execution-budget-spec-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Execution-level limits can have a stable, executor-neutral identity without conflating routing, recovery, qualification, and request budgets.", + "focused_test": {"result": "PASS", "tests_run": 105}, + "criteria": { + "stable_budget_digest": "PASS", + "negative_limits_rejected": "PASS", + "executor_invoked": "NO", + "budget_consumption_implemented": "NO" + }, + "acceptance": "LOCAL_EXECUTION_BUDGET_CONTRACT_EVIDENCE_PASS_SEMANTIC_RECONCILIATION_PENDING", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/execution-contract-block-20260923.json b/logs/architecture/execution-contract-block-20260923.json new file mode 100644 index 00000000..e69db950 --- /dev/null +++ b/logs/architecture/execution-contract-block-20260923.json @@ -0,0 +1,22 @@ +{ + "schema_version": "arkx-execution-contract-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Executor, provider and sandbox protocols allow deterministic core tests without importing a concrete executor, provider, Docker or SWE-bench.", + "focused_test": { + "command": "PYTHONPATH=src python -m unittest tests.test_execution tests.test_harness tests.test_promotion tests.test_p82_baseline tests.test_core_outcomes tests.test_p0_telemetry tests.test_swebench_authority", + "result": "PASS", + "tests_run": 79 + }, + "criteria": { + "fake_executor": "PASS", + "fake_provider": "PASS", + "fake_sandbox": "PASS", + "structured_error_for_failed_or_blocked": "PASS", + "missing_provider_and_model_preserved": "PASS", + "real_executor_selected": false + }, + "acceptance": "LOCAL_CONTRACT_EVIDENCE_PASS_EXTERNAL_INTEGRATION_NOT_ATTEMPTED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/execution-plan-block-20260923.json b/logs/architecture/execution-plan-block-20260923.json new file mode 100644 index 00000000..44a9c8cb --- /dev/null +++ b/logs/architecture/execution-plan-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-execution-plan-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Plan and result can remain separate contracts while plan construction stays deterministic and side-effect free.", + "focused_test": {"result": "PASS", "tests_run": 101}, + "criteria": { + "deterministic_serialization": "PASS", + "identity_validation": "PASS", + "plan_to_request_preserves_identity": "PASS", + "result_state_absent_from_plan": "PASS", + "executor_invoked": false + }, + "acceptance": "LOCAL_EXECUTION_PLAN_EVIDENCE_PASS_EXTERNAL_INTEGRATION_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/execution-plan-capability-gate-block-20260923.json b/logs/architecture/execution-plan-capability-gate-block-20260923.json new file mode 100644 index 00000000..bee9aefe --- /dev/null +++ b/logs/architecture/execution-plan-capability-gate-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-execution-plan-capability-gate-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 322, "result": "PASS"}, + "criteria": { + "plan_carries_required_capabilities": "PASS", + "minimum_provenance_enforced": "PASS", + "preflight_denial_prevents_execution_authorization": "PASS", + "fallback_selected": "NO", + "executor_invoked_by_plan_validation": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/execution-reference-identity-block-20260923.json b/logs/architecture/execution-reference-identity-block-20260923.json new file mode 100644 index 00000000..376e52aa --- /dev/null +++ b/logs/architecture/execution-reference-identity-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-execution-reference-identity-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Plan and execution references can be bound to their exact identities before verification or acceptance.", + "focused_test": {"result": "PASS", "tests_run": 139}, + "criteria": { + "matching_references_validated": "PASS", + "plan_drift_rejected": "PASS", + "external_execution_invoked": "NO", + "acceptance_inferred": "NO" + }, + "acceptance": "LOCAL_EXECUTION_REFERENCE_EVIDENCE_PASS", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/experiment-execution-identity-digest-block-20260923.json b/logs/architecture/experiment-execution-identity-digest-block-20260923.json new file mode 100644 index 00000000..c6d03e5e --- /dev/null +++ b/logs/architecture/experiment-execution-identity-digest-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-experiment-execution-identity-digest-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 314, "result": "PASS"}, + "criteria": { + "identity_digest_serializes": "PASS", + "attempt_order_does_not_change_digest": "PASS", + "execution_field_drift_changes_identity": "PASS", + "missing_or_wrong_identity_rejected_for_audited_experiment": "PASS", + "legacy_record_readability_preserved": "PASS", + "qualification_inferred_from_digest": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/explicit-lifecycle-state-crosswalk-block-20260923.json b/logs/architecture/explicit-lifecycle-state-crosswalk-block-20260923.json new file mode 100644 index 00000000..b6bafc7a --- /dev/null +++ b/logs/architecture/explicit-lifecycle-state-crosswalk-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "explicit-lifecycle-state-crosswalk", + "date": "2026-09-23", + "hypothesis": "Every execution outcome and acceptance decision maps deterministically to exactly one orchestration status.", + "criteria": {"execution_mapping_total": "PASS", "acceptance_mapping_total": "PASS", "third_state_machine_created": "NO"}, + "local_tests_verified": 342, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/external-integration-readiness-block-20260923.json b/logs/architecture/external-integration-readiness-block-20260923.json new file mode 100644 index 00000000..ad799200 --- /dev/null +++ b/logs/architecture/external-integration-readiness-block-20260923.json @@ -0,0 +1,21 @@ +{ + "schema_version": "arkx-external-integration-readiness-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "head_observed": "aef1df4d33c95c522dfda8f2e21f9bd09eb6ce6b", + "environment": { + "os": "Windows", + "python": "3.11.9", + "minisweagent_import": false, + "docker_cli": "29.7.2", + "docker_daemon": "BLOCKED_PERMISSION_DENIED", + "bash_executable": "DETECTED_BUT_ALIAS_ACCESS_DENIED", + "ollama": "0.34.3" + }, + "focused_suite": {"tests_run": 139, "result": "PASS"}, + "isolated_suite": {"tests_run": 290, "result": "PASS", "excluded": "tests.test_p82_openrouter_adapter"}, + "full_suite": {"tests_reached": 291, "result": "BLOCKED", "reason": "minisweagent import unavailable"}, + "historical_logs": {"observed": true, "reused_as_current_qualification": false}, + "acceptance": "NEUTRAL_CONTRACTS_READY_EXTERNAL_INTEGRATIONS_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/final-outcome-promotion-chain-block-20260923.json b/logs/architecture/final-outcome-promotion-chain-block-20260923.json new file mode 100644 index 00000000..0af0b449 --- /dev/null +++ b/logs/architecture/final-outcome-promotion-chain-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-final-outcome-promotion-chain-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Promotion lineage can be validated against exact attempt and acceptance identities without performing promotion.", + "focused_test": {"result": "PASS", "tests_run": 139}, + "criteria": { + "promotion_reference_identity": "PASS", + "attempt_binding": "PASS", + "acceptance_binding": "PASS", + "promotion_side_effect": "NO" + }, + "acceptance": "LOCAL_FINAL_CHAIN_EVIDENCE_PASS_PROMOTION_POLICY_SEPARATE", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/frozen-execution-identity-replay-block-20260923.json b/logs/architecture/frozen-execution-identity-replay-block-20260923.json new file mode 100644 index 00000000..3baf6ffc --- /dev/null +++ b/logs/architecture/frozen-execution-identity-replay-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-frozen-execution-identity-replay-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 316, "result": "PASS"}, + "criteria": { + "configuration_drift_rejected": "PASS", + "budget_identity_checked": "PASS", + "treatment_identity_checked": "PASS", + "manifest_event_identity_compared": "PASS", + "capability_identity_invented_without_contract": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/full-replay-chain-block-20260923.json b/logs/architecture/full-replay-chain-block-20260923.json new file mode 100644 index 00000000..1e8bbced --- /dev/null +++ b/logs/architecture/full-replay-chain-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-full-replay-chain-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Explicit stage references can reconstruct the full lifecycle chain without re-execution or filling missing stages.", + "focused_test": {"result": "PASS", "tests_run": 127}, + "criteria": { + "ordered_chain_reconstructed": "PASS", + "causal_order_validated": "PASS", + "missing_stage_inferred": "NO", + "executor_or_acceptance_invoked": "NO" + }, + "acceptance": "LOCAL_FULL_REPLAY_CHAIN_EVIDENCE_PASS_NO_REEXECUTION", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/full-suite-validation-block-20260923.json b/logs/architecture/full-suite-validation-block-20260923.json new file mode 100644 index 00000000..0d6978e0 --- /dev/null +++ b/logs/architecture/full-suite-validation-block-20260923.json @@ -0,0 +1,15 @@ +{ + "schema_version": "arkx-full-suite-validation-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "command": "PYTHONPATH=src python -m unittest discover -s tests -p 'test_*.py'", + "tests_reached": 291, + "result": "BLOCKED", + "blocking_test_module": "tests/test_p82_openrouter_adapter.py", + "blocking_error": "ModuleNotFoundError: No module named 'minisweagent'", + "interpretation": "Infrastructure/dependency blocker; not classified as functional failure.", + "fallback_selected": false, + "acceptance": "FULL_SUITE_BLOCKED_BY_MISSING_EXTERNAL_DEPENDENCY", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/google-cloud-cli-preflight-20260924.json b/logs/architecture/google-cloud-cli-preflight-20260924.json new file mode 100644 index 00000000..8b6ebb2e --- /dev/null +++ b/logs/architecture/google-cloud-cli-preflight-20260924.json @@ -0,0 +1,16 @@ +{ + "project": "Codepro", + "date": "2026-09-24", + "candidate": "Google Cloud CLI", + "binary": "C:\\Users\\igorb\\AppData\\Local\\Google\\Cloud SDK\\google-cloud-sdk\\bin\\gcloud.cmd", + "version": "Google Cloud SDK 586.0.0; core 2026.09.20", + "path_status": "NOT_ON_PATH", + "auth_probe": { + "command": "gcloud auth list --format=value(status)", + "observed": "Unable to create private file ... credentials.db: WinError 5 Access denied", + "account_identity_exposed": false, + "status": "BLOCKED_CONFIG_ACCESS" + }, + "executor_qualification": "NOT_APPLICABLE", + "reason": "Google Cloud CLI is a cloud administration/authentication CLI, not a task-solving executor with patch extraction and shared verifier boundary." +} diff --git a/logs/architecture/harness-contract-slice-20260923.json b/logs/architecture/harness-contract-slice-20260923.json new file mode 100644 index 00000000..c5641b7d --- /dev/null +++ b/logs/architecture/harness-contract-slice-20260923.json @@ -0,0 +1,48 @@ +{ + "schema_version": "arkx-harness-contract-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "head_before_slice": "aef1df4d33c95c522dfda8f2e21f9bd09eb6ce6b", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Separate experiment, trial and attempt identity plus a common error envelope will make frozen inputs deterministic and prevent attempt identity collisions.", + "focused_test": { + "command": "PYTHONPATH=src python -m unittest tests.test_harness tests.test_core_outcomes tests.test_p0_telemetry tests.test_p82_baseline tests.test_swebench_authority", + "result": "PASS", + "tests_run": 75 + }, + "full_test": { + "command": "PYTHONPATH=src python -m unittest discover -s tests -p 'test_*.py'", + "result": "BLOCKED", + "tests_run": 205, + "error": "ModuleNotFoundError: No module named 'minisweagent'", + "affected_test": "tests/test_p82_openrouter_adapter.py" + }, + "runner_integration": "P82_BASELINE_EMITS_MANIFEST_AND_LEGACY_EXECUTION_ARTIFACT", + "replay_validation": "PASS", + "artifact_store": "PASS", + "error_envelope_in_execution_artifact": "PASS", + "lifecycle_validation": "PASS", + "transition_validation": "PASS", + "state_history_validation": "PASS", + "lifecycle_timestamp_validation": "PASS", + "artifact_reference_validation": "PASS", + "artifact_presence_classification": "PASS", + "execution_manifest_consistency": "PASS", + "attempt_snapshot_loader": "PASS", + "attempt_store_index": "PASS", + "attempt_comparability": "PASS", + "comparison_report": "PASS", + "unknown_metric_preservation": "PASS", + "execution_verification_acceptance_separation": "PASS", + "independent_verification_acceptance_evidence": "PASS", + "promotion_verification_gate": "PASS", + "snapshot_promotion_gate": "PASS", + "promotion_decision_binding": "PASS", + "promotion_decision_persistence": "PASS", + "experiment_record_linkage": "PASS", + "experiment_record_loader": "PASS", + "experiment_record_immutability": "PASS", + "block_status": "BLOCK_1_ACCEPTED_LOCAL_EVIDENCE_EXTERNAL_INTEGRATION_BLOCKED", + "acceptance": "LOCAL_CONTRACT_AND_RUNNER_EVIDENCE_PASS_EXTERNAL_INTEGRATION_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/idempotent-budget-ledger-block-20260923.json b/logs/architecture/idempotent-budget-ledger-block-20260923.json new file mode 100644 index 00000000..d0292a9b --- /dev/null +++ b/logs/architecture/idempotent-budget-ledger-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-idempotent-budget-ledger-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Execution budget consumption can be idempotent per attempt and preserve an unchanged ledger on rejection.", + "focused_test": {"result": "PASS", "tests_run": 114}, + "criteria": { + "attempt_consumed_once": "PASS", + "limit_rejection_is_non_mutating": "PASS", + "explicit_reason": "PASS", + "executor_invoked": "NO" + }, + "acceptance": "LOCAL_BUDGET_LEDGER_EVIDENCE_PASS_SEMANTIC_RECONCILIATION_PENDING", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/import-safe-adapter-boundaries-block-20260923.json b/logs/architecture/import-safe-adapter-boundaries-block-20260923.json new file mode 100644 index 00000000..0d774677 --- /dev/null +++ b/logs/architecture/import-safe-adapter-boundaries-block-20260923.json @@ -0,0 +1,21 @@ +{ + "schema_version": "arkx-import-safe-adapter-boundaries-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 298, "result": "PASS"}, + "criteria": { + "adapter_modules_import_without_optional_runtime": "PASS", + "unavailable_adapter_construction_is_explicitly_blocked": "PASS", + "fallback_selected": "NO" + }, + "concrete_preflight": { + "openrouter": "BLOCKED", + "git_bash": "BLOCKED", + "docker_swebench": "BLOCKED", + "ollama": "UNKNOWN" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/legacy-consumer-boundary-block-20260923.json b/logs/architecture/legacy-consumer-boundary-block-20260923.json new file mode 100644 index 00000000..cfbc73f2 --- /dev/null +++ b/logs/architecture/legacy-consumer-boundary-block-20260923.json @@ -0,0 +1,11 @@ +{ + "schema_version": "arkx-legacy-consumer-boundary-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "A static boundary test prevents new core modules from importing the legacy orchestration result.", + "focused_test": {"result": "PASS", "tests_run": 98}, + "criteria": {"legacy_imports_confined": "PASS", "public_names_distinct": "PASS"}, + "acceptance": "LOCAL_ARCHITECTURE_BOUNDARY_EVIDENCE_PASS_EXTERNAL_INTEGRATION_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/legacy-result-boundary-block-20260923.json b/logs/architecture/legacy-result-boundary-block-20260923.json new file mode 100644 index 00000000..1492142f --- /dev/null +++ b/logs/architecture/legacy-result-boundary-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-legacy-result-boundary-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Centralized neutral-to-legacy conversion preserves orchestration compatibility while new executors depend only on the neutral contract.", + "focused_test": {"result": "PASS", "tests_run": 95}, + "criteria": { + "conversion_preserves_state_artifacts_and_error": "PASS", + "direct_and_bridged_equivalence": "PASS", + "legacy_boundary_centralized": "PASS", + "legacy_api_removed": false + }, + "acceptance": "LOCAL_COMPATIBILITY_EVIDENCE_PASS_EXTERNAL_INTEGRATION_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/legacy-runner-neutral-error-boundary-block-20260923.json b/logs/architecture/legacy-runner-neutral-error-boundary-block-20260923.json new file mode 100644 index 00000000..42f6beaa --- /dev/null +++ b/logs/architecture/legacy-runner-neutral-error-boundary-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-legacy-runner-neutral-error-boundary-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 301, "result": "PASS"}, + "criteria": { + "missing_identity_remains_harness": "PASS", + "sandbox_process_boundary_preserved": "PASS", + "executor_failure_boundary_preserved": "PASS", + "provider_retry_policy_preserved": "PASS", + "legacy_failure_category_retained": "PASS", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/lifecycle-persistence-error-taxonomy-block-20260923.json b/logs/architecture/lifecycle-persistence-error-taxonomy-block-20260923.json new file mode 100644 index 00000000..ed511d0d --- /dev/null +++ b/logs/architecture/lifecycle-persistence-error-taxonomy-block-20260923.json @@ -0,0 +1,15 @@ +{ + "block": "lifecycle-persistence-error-taxonomy", + "date": "2026-09-23", + "hypothesis": "Persistence failures are classified by the violated lifecycle invariant.", + "criteria": { + "generic_persistence_failure_neutral_code": "PASS", + "capability_specific_reason_not_reused": "PASS", + "terminal_persistence_failure_explicit": "PASS", + "fallback_selected": "NO" + }, + "tests_run": 334, + "test_command": "PYTHONPATH=src python -m unittest discover -s tests -p test_*.py", + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/lineage-event-chain-audit-block-20260923.json b/logs/architecture/lineage-event-chain-audit-block-20260923.json new file mode 100644 index 00000000..c1f1b360 --- /dev/null +++ b/logs/architecture/lineage-event-chain-audit-block-20260923.json @@ -0,0 +1,13 @@ +{ + "block": "lineage-event-chain-audit", + "date": "2026-09-23", + "hypothesis": "An attempt with inconsistent execution, recovery, source, or budget facts cannot pass lineage audit.", + "criteria": { + "matching_execution_recovery_source_ledger": "PASS", + "recovery_reference_drift_rejected": "PASS", + "local_suite": "PASS", + "external_qualification": "NOT_PERFORMED" + }, + "local_tests_verified": 365, + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/measured-budget-usage-block-20260923.json b/logs/architecture/measured-budget-usage-block-20260923.json new file mode 100644 index 00000000..07a1b4d1 --- /dev/null +++ b/logs/architecture/measured-budget-usage-block-20260923.json @@ -0,0 +1,14 @@ +{ + "block": "measured-budget-usage", + "date": "2026-09-23", + "hypothesis": "A runner cannot appear budget-compliant merely because it returned an execution result.", + "criteria": { + "measured_usage_recorded": "PASS", + "unconsumed_attempt_rejected": "PASS", + "missing_measurement_blocks": "PASS", + "usage_inferred_as_zero": "NO" + }, + "local_tests_verified": 342, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/multi-integration-preflight-replay-block-20260923.json b/logs/architecture/multi-integration-preflight-replay-block-20260923.json new file mode 100644 index 00000000..8943c57f --- /dev/null +++ b/logs/architecture/multi-integration-preflight-replay-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "multi-integration-preflight-replay", + "date": "2026-09-23", + "hypothesis": "A multi-integration plan can be replayed without losing one preflight to another.", + "criteria": {"provider_preflight_required": "PASS", "sandbox_preflight_required": "PASS", "both_references_replayed": "PASS", "implicit_integration_selected": "NO"}, + "local_tests_verified": 354, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/neutral-adapter-error-normalization-block-20260923.json b/logs/architecture/neutral-adapter-error-normalization-block-20260923.json new file mode 100644 index 00000000..be6957e7 --- /dev/null +++ b/logs/architecture/neutral-adapter-error-normalization-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-neutral-adapter-error-normalization-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 301, "result": "PASS"}, + "criteria": { + "provider_payload_maps_to_provider_domain": "PASS", + "process_failure_preserves_sandbox_boundary": "PASS", + "evaluator_exception_maps_to_verification_domain": "PASS", + "raw_evidence_preserved": "PASS", + "retryability_inferred_without_policy": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/opencode-headless-audit-20260924.json b/logs/architecture/opencode-headless-audit-20260924.json new file mode 100644 index 00000000..fc9bca6a --- /dev/null +++ b/logs/architecture/opencode-headless-audit-20260924.json @@ -0,0 +1,57 @@ +{ + "project": "Codepro", + "date": "2026-09-24", + "candidate": "OpenCode", + "binary": "C:\\Users\\igorb\\AppData\\Roaming\\npm\\node_modules\\@opencode\\cli\\bin\\opencode.exe", + "version": { + "observed": "opencode v2.0.15", + "source": "successful --version probe before server-dependent subcommand probe" + }, + "default_environment_reproduction": { + "commands": [ + "opencode --version", + "opencode models", + "opencode run --help" + ], + "exit_code": 1, + "stderr_stdout_observation": "PlatformError: FileSystem.open ... C:\\Users\\igorb\\.local\\share\\opencode\\log\\opencode.log; EPERM", + "status": "BLOCKED_SHARED_LOG" + }, + "isolated_environment": { + "method": "XDG_DATA_HOME and XDG_STATE_HOME pointed to disposable workspace state", + "result": "CLI attempted to start background server and failed with EPERM uv_spawn of opencode.exe", + "status": "BLOCKED_RUNTIME" + }, + "observed_help_contract": { + "root_flags": ["--standalone", "--server", "--prompt", "--model", "--log-level", "--print-logs"], + "run_flags": ["--standalone", "--server", "--model", "--format {default,json}", "--agent", "--file", "--auto"], + "model_syntax": "provider/model#variant", + "structured_output": "--format json", + "headless_entrypoint": "opencode run " + }, + "standalone_probe": { + "command": "opencode --standalone run -m ollama/qwen2.5-coder:3b --format json ", + "exit_code": 1, + "observed": "Unrecognized flag: --standalone in command opencode run", + "status": "CLI_FLAG_INCONSISTENCY" + }, + "free_provider": { + "provider": "ollama", + "runtime_version": "ollama version is 0.34.3", + "model_observed": "qwen2.5-coder:3b", + "model_available_locally": true, + "executor_run": "NOT_EXECUTED", + "reason": "OpenCode failed before provider/model invocation" + }, + "required_but_unobserved": [ + "bounded timeout during task execution", + "provider/model identity in an execution record", + "stdout/stderr from a completed task run", + "exit status from a completed task run", + "workspace patch or diff extraction", + "termination reason from a completed task run", + "absence of fallback during a completed task run" + ], + "qualification_decision": "BLOCKED", + "qualification_reason": "OpenCode is installed but not operational as a headless task executor in this environment; no protocol relaxation is justified." +} diff --git a/logs/architecture/openhands-headless-audit-20260924.json b/logs/architecture/openhands-headless-audit-20260924.json new file mode 100644 index 00000000..5c798b2d --- /dev/null +++ b/logs/architecture/openhands-headless-audit-20260924.json @@ -0,0 +1,73 @@ +{ + "project": "Codepro", + "date": "2026-09-24", + "candidate": "OpenHands CLI", + "version": "OpenHands SDK v1.21.0", + "headless_contract": { + "flags_observed": [ + "--headless", + "--json", + "--task", + "--override-with-envs", + "--exit-without-confirmation" + ], + "structured_output": "JSONL stream when --headless --json are used", + "workspace_override": "OPENHANDS_WORK_DIR", + "persistence_overrides": [ + "OPENHANDS_PERSISTENCE_DIR", + "OPENHANDS_CONVERSATIONS_DIR" + ], + "provider_model_override": [ + "LLM_BASE_URL", + "LLM_API_KEY", + "LLM_MODEL" + ] + }, + "probe": { + "executor": "OpenHands CLI", + "provider": "Ollama", + "base_url": "http://127.0.0.1:11434/v1", + "model": "qwen2.5-coder:3b", + "cost_tier": "FREE_SHORT", + "task": "append exactly CODEPRO_OPENHANDS_PROBE_OK to probe.txt and modify no other file", + "workspace": "disposable workspace under the repository; removed after probe", + "observed": [ + "agent initialized with model qwen2.5-coder:3b", + "workspace identity was resolved", + "no model/tool task cycle completed", + "probe.txt remained unchanged" + ] + }, + "blockers": [ + { + "kind": "PERSISTENCE_CACHE", + "detail": "PermissionError creating C:\\Users\\igorb\\.openhands\\cache\\jinja", + "status": "BLOCKED_RUNTIME" + }, + { + "kind": "NETWORK_PREFLIGHT", + "detail": "LiteLLM attempted three remote cost-map fetches and fell back to its local backup; no paid provider was configured", + "status": "NON_FATAL_BUT_UNDESIRED" + }, + { + "kind": "TERMINAL_OUTPUT", + "detail": "Rich reported a non-interactive terminal; JSONL evidence was not produced because execution failed before the task cycle", + "status": "BLOCKED_EVIDENCE" + }, + { + "kind": "CACHE_CONFIGURATION", + "detail": "Installed CLI exposes persistence/workspace overrides, but Jinja and several SDK caches are built directly from Path.home()/.openhands and no CLI cache-dir override was found in the installed source.", + "status": "BLOCKED_CONFIGURATION" + } + ], + "required_but_unobserved": [ + "bounded task timeout", + "completed-task exit code", + "captured stdout/stderr for a completed task", + "extracted patch/diff", + "termination reason from a completed task", + "fallback absence during a completed task" + ], + "qualification_decision": "BLOCKED", + "qualification_reason": "OpenHands is a technically promising independent headless candidate, but its installed CLI cannot complete a local Ollama task without writing to a protected user cache path. A process-wide HOME/profile override or package monkeypatch was not used because it would alter global identity/state semantics." +} diff --git a/logs/architecture/orchestration-budget-enforcement-block-20260923.json b/logs/architecture/orchestration-budget-enforcement-block-20260923.json new file mode 100644 index 00000000..c4c97137 --- /dev/null +++ b/logs/architecture/orchestration-budget-enforcement-block-20260923.json @@ -0,0 +1,14 @@ +{ + "block": "orchestration-budget-enforcement", + "date": "2026-09-23", + "hypothesis": "A zero invocation budget never reaches a runner.", + "criteria": { + "runner_not_called": "PASS", + "explicit_budget_block": "PASS", + "ledger_persisted_when_execution_occurs": "PASS", + "token_cost_inferred_without_measurement": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "local_tests_verified": 339, + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/orchestration-capability-preflight-gate-block-20260923.json b/logs/architecture/orchestration-capability-preflight-gate-block-20260923.json new file mode 100644 index 00000000..c3b69597 --- /dev/null +++ b/logs/architecture/orchestration-capability-preflight-gate-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-orchestration-capability-preflight-gate-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 323, "result": "PASS"}, + "criteria": { + "treatment_capabilities_copied_to_plan": "PASS", + "missing_preflight_blocks_before_executor": "PASS", + "policy_denial_blocks_before_executor": "PASS", + "legacy_empty-capability_path_preserved": "PASS", + "fallback_or_executor_switch_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/orchestration-configuration-snapshot-block-20260923.json b/logs/architecture/orchestration-configuration-snapshot-block-20260923.json new file mode 100644 index 00000000..455bed1b --- /dev/null +++ b/logs/architecture/orchestration-configuration-snapshot-block-20260923.json @@ -0,0 +1,13 @@ +{ + "block": "orchestration-configuration-snapshot", + "date": "2026-09-23", + "hypothesis": "A typed configuration snapshot supplied to orchestration survives into the execution plan and cannot disagree with executor identity.", + "criteria": { + "snapshot_retained_and_digest_propagated": "PASS", + "identity_mismatch_rejected": "PASS", + "local_suite": "PASS", + "external_qualification": "NOT_PERFORMED" + }, + "local_tests_verified": 367, + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/orchestration-execution-plan-block-20260923.json b/logs/architecture/orchestration-execution-plan-block-20260923.json new file mode 100644 index 00000000..e262db6b --- /dev/null +++ b/logs/architecture/orchestration-execution-plan-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-orchestration-execution-plan-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Equivalent authorized inputs produce the same immutable execution plan before executor invocation.", + "focused_test": {"result": "PASS", "tests_run": 101}, + "criteria": { + "plan_built_before_execution": "PASS", + "equivalent_inputs_produce_equal_plans": "PASS", + "neutral_and_legacy_paths_remain_compatible": "PASS", + "plan_serialization_is_deterministic": "PASS" + }, + "acceptance": "LOCAL_ORCHESTRATION_PLAN_EVIDENCE_PASS_EXTERNAL_INTEGRATION_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/orchestration-integration-identity-binding-block-20260923.json b/logs/architecture/orchestration-integration-identity-binding-block-20260923.json new file mode 100644 index 00000000..4f326b69 --- /dev/null +++ b/logs/architecture/orchestration-integration-identity-binding-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "orchestration-integration-identity-binding", + "date": "2026-09-23", + "hypothesis": "Identities supplied to orchestration are present on the exact plan that is executed.", + "criteria": {"provider_identity_bound": "PASS", "sandbox_identity_bound": "PASS", "implicit_default_selected": "NO"}, + "local_tests_verified": 351, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/orchestration-persisted-lifecycle-chain-block-20260923.json b/logs/architecture/orchestration-persisted-lifecycle-chain-block-20260923.json new file mode 100644 index 00000000..df786c1f --- /dev/null +++ b/logs/architecture/orchestration-persisted-lifecycle-chain-block-20260923.json @@ -0,0 +1,19 @@ +{ + "schema_version": "arkx-orchestration-persisted-lifecycle-chain-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 327, "result": "PASS"}, + "criteria": { + "plan_persisted": "PASS", + "execution_persisted": "PASS", + "verification_persisted": "PASS", + "acceptance_persisted": "PASS", + "causal_replay_order_preserved": "PASS", + "promotion_performed_by_orchestration": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/orchestration-terminal-event-integration-block-20260923.json b/logs/architecture/orchestration-terminal-event-integration-block-20260923.json new file mode 100644 index 00000000..3223a838 --- /dev/null +++ b/logs/architecture/orchestration-terminal-event-integration-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-orchestration-terminal-event-integration-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 333, "result": "PASS"}, + "criteria": { + "terminal_status_replayed": "PASS", + "primary_reason_replayed_as_error_code": "PASS", + "pre_execution_identity_fabricated": "NO", + "persistence_failure_downgrades_result": "PASS", + "promotion_inferred_from_terminal_status": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/outcome-reference-identity-block-20260923.json b/logs/architecture/outcome-reference-identity-block-20260923.json new file mode 100644 index 00000000..91f230e2 --- /dev/null +++ b/logs/architecture/outcome-reference-identity-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-outcome-reference-identity-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Verification and acceptance outcomes can have distinct deterministic identities before promotion.", + "focused_test": {"result": "PASS", "tests_run": 139}, + "criteria": { + "stable_verification_reference": "PASS", + "stable_acceptance_reference": "PASS", + "namespaces_distinct": "PASS", + "reference_drift_rejected": "PASS", + "promotion_invoked": "NO" + }, + "acceptance": "LOCAL_OUTCOME_REFERENCE_EVIDENCE_PASS_PROMOTION_SEPARATE", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/p82-configuration-snapshot-adoption-block-20260923.json b/logs/architecture/p82-configuration-snapshot-adoption-block-20260923.json new file mode 100644 index 00000000..64784ea9 --- /dev/null +++ b/logs/architecture/p82-configuration-snapshot-adoption-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "p82-configuration-snapshot-adoption", + "date": "2026-09-23", + "hypothesis": "P8.2 preflight and configuration identity cannot drift because they derive from one canonical snapshot.", + "criteria": {"identity_equals_snapshot_digest": "PASS", "preflight_uses_snapshot_digest": "PASS", "secret_material_serialized": "NO"}, + "local_tests_verified": 348, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/persisted-budget-ledger-binding-block-20260923.json b/logs/architecture/persisted-budget-ledger-binding-block-20260923.json new file mode 100644 index 00000000..185bab3d --- /dev/null +++ b/logs/architecture/persisted-budget-ledger-binding-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-persisted-budget-ledger-binding-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "A budget ledger can be persisted in a manifest while remaining bound to the exact attempt and plan budget identity.", + "focused_test": {"result": "PASS", "tests_run": 116}, + "criteria": { + "ledger_bound_to_attempt": "PASS", + "digest_mismatch_rejected": "PASS", + "manifest_snapshot_immutable": "PASS", + "automatic_executor_consumption": "NO" + }, + "acceptance": "LOCAL_PERSISTED_BUDGET_BINDING_EVIDENCE_PASS_END_TO_END_ENFORCEMENT_PENDING", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/persisted-capability-identity-block-20260923.json b/logs/architecture/persisted-capability-identity-block-20260923.json new file mode 100644 index 00000000..25f75432 --- /dev/null +++ b/logs/architecture/persisted-capability-identity-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-persisted-capability-identity-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 317, "result": "PASS"}, + "criteria": { + "capability_registry_digest_deterministic": "PASS", + "capability_reference_stable": "PASS", + "capability_digest_bound_to_replay": "PASS", + "capability_drift_rejected": "PASS", + "capability_effectiveness_inferred_from_identity": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/persisted-preflight-orchestration-event-block-20260923.json b/logs/architecture/persisted-preflight-orchestration-event-block-20260923.json new file mode 100644 index 00000000..7f050be0 --- /dev/null +++ b/logs/architecture/persisted-preflight-orchestration-event-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-persisted-preflight-orchestration-event-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 326, "result": "PASS"}, + "criteria": { + "preflight_stage_ordered_before_execution": "PASS", + "preflight_event_persistence_required_for_capability_plan": "PASS", + "attempt_identity_not_fabricated": "PASS", + "executor_invoked_before_preflight_event": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/persisted-recovery-budget-ledger-block-20260923.json b/logs/architecture/persisted-recovery-budget-ledger-block-20260923.json new file mode 100644 index 00000000..dec6c7fa --- /dev/null +++ b/logs/architecture/persisted-recovery-budget-ledger-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "persisted-recovery-budget-ledger", + "date": "2026-09-23", + "hypothesis": "Accepted attempt reservation cannot leave budget consumption implicit.", + "criteria": {"updated_ledger_persisted": "PASS", "ledger_is_copied": "PASS", "manifest_ledger_binding": "PASS"}, + "local_tests_verified": 363, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/persistence-coordination-scope-block-20260923.json b/logs/architecture/persistence-coordination-scope-block-20260923.json new file mode 100644 index 00000000..201214ca --- /dev/null +++ b/logs/architecture/persistence-coordination-scope-block-20260923.json @@ -0,0 +1,14 @@ +{ + "block": "persistence-coordination-scope", + "date": "2026-09-23", + "hypothesis": "Partial or divergent persisted lifecycle cannot be treated as complete accepted attempt under current audit-based persistence.", + "criteria": { + "atomic_artifact_writes": "PASS", + "incomplete_chain_classification": "PASS", + "replay_drift_rejection": "PASS", + "multi_writer_qualification": "NOT_IN_SCOPE", + "local_suite": "PASS" + }, + "local_tests_verified": 367, + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/persistent-event-log-block-20260923.json b/logs/architecture/persistent-event-log-block-20260923.json new file mode 100644 index 00000000..cb067ded --- /dev/null +++ b/logs/architecture/persistent-event-log-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-persistent-event-log-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Lifecycle events can be persisted and replayed in order without coupling telemetry to execution or acceptance.", + "focused_test": {"result": "PASS", "tests_run": 121}, + "criteria": { + "event_round_trip": "PASS", + "run_identity_validated": "PASS", + "timestamp_regression_rejected": "PASS", + "executor_invoked": "NO" + }, + "acceptance": "LOCAL_PERSISTENT_EVENT_LOG_EVIDENCE_PASS_DECISION_STATES_SEPARATE", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/plan-budget-binding-block-20260923.json b/logs/architecture/plan-budget-binding-block-20260923.json new file mode 100644 index 00000000..fb3a1cd0 --- /dev/null +++ b/logs/architecture/plan-budget-binding-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-plan-budget-binding-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Execution plans can carry an explicit budget specification and validate its identity without changing execution behavior.", + "focused_test": {"result": "PASS", "tests_run": 115}, + "criteria": { + "budget_serialized_in_plan": "PASS", + "missing_digest_derived": "PASS", + "mismatched_digest_rejected": "PASS", + "orchestration_preserves_binding": "PASS", + "executor_invoked_by_binding": "NO" + }, + "acceptance": "LOCAL_PLAN_BUDGET_BINDING_EVIDENCE_PASS_LEDGER_CONSUMPTION_SEPARATE", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/plan-configuration-snapshot-binding-block-20260923.json b/logs/architecture/plan-configuration-snapshot-binding-block-20260923.json new file mode 100644 index 00000000..6a429f81 --- /dev/null +++ b/logs/architecture/plan-configuration-snapshot-binding-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "plan-configuration-snapshot-binding", + "date": "2026-09-23", + "hypothesis": "A plan cannot carry a configuration digest inconsistent with its canonical snapshot.", + "criteria": {"digest_derived": "PASS", "digest_mismatch_rejected": "PASS", "secret_material_serialized": "NO"}, + "local_tests_verified": 347, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/plan-integration-identity-binding-block-20260923.json b/logs/architecture/plan-integration-identity-binding-block-20260923.json new file mode 100644 index 00000000..cb3f950d --- /dev/null +++ b/logs/architecture/plan-integration-identity-binding-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "plan-integration-identity-binding", + "date": "2026-09-23", + "hypothesis": "A plan cannot bind a provider or sandbox identity from another component.", + "criteria": {"matching_identities_accepted": "PASS", "identity_mismatch_rejected": "PASS", "provider_or_sandbox_selected_by_plan": "NO"}, + "local_tests_verified": 350, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/pr30-decomposition-20260924.json b/logs/architecture/pr30-decomposition-20260924.json new file mode 100644 index 00000000..b3df9399 --- /dev/null +++ b/logs/architecture/pr30-decomposition-20260924.json @@ -0,0 +1,106 @@ +{ + "decision": "0148", + "classification": "PR30_NOT_MERGEABLE_AS_SINGLE_UNIT", + "date": "2026-09-24", + "updated_at": "2026-09-25", + "main_head": "86b3822499de186a779e9a3887285d92f60fafea", + "merge_base": "033b46da93af614fcd65b556eb4a727408a71644", + "conflict_count": 31, + "merge_worktree": "D:/projetos/arkx/.tmp/pr30-merge", + "merge_state": "ABORTED_AFTER_DIAGNOSTIC", + "provider_called": false, + "focused_tests": { + "passed": 109, + "failed": 0, + "blocked_environment_tests": 1, + "environment_blocker": "WINDOWS_TEMP_DIRECTORY_PERMISSION_DENIED" + }, + "full_suite": { + "status": "NOT_COMPARABLE", + "observed_issues": [ + "HISTORICAL_LOG_DISCOVERY_CONTAMINATION", + "PR30_FOUNDATIONAL_CONTRACT_IMPORT_MISMATCH" + ] + }, + "promotion": "NOT_AUTHORIZED", + "foundation_gate": { + "contracts": "REJECTED_PR30_VALIDATION_REGRESSION", + "pure_contract_tranche": { + "modules": [ + "composition.py", + "configuration.py", + "outcomes.py", + "p82_editing.py", + "p82_localization.py", + "executor_qualification.py" + ], + "clean_main_worktree": "D:/projetos/arkx/.tmp/pr30-pure-tranche", + "fixtures": [ + "experiments/composition-harness-fixture.json", + "experiments/executor-qualification-fixture.json" + ], + "python": "3.13", + "focused_tests_passed": 41, + "focused_tests_failed": 0, + "status": "COMPATIBLE_WITH_SCOPE" + }, + "core_outcomes_test": "NOT_EXECUTED_LATER_DEPENDENCY_P82_BASELINE" + }, + "telemetry_gate": { + "tests_passed": 16, + "tests_failed": 4, + "status": "REJECTED_VALIDATION_REGRESSION", + "failed_invariants": [ + "blank_evidence_rejection", + "duplicate_task_boundary_rejection", + "post_terminal_event_rejection", + "monotonic_event_timestamp_rejection" + ] + }, + "handoff_gate": { + "tests_passed": 8, + "tests_failed": 6, + "status": "REJECTED_CONTRACT_REGRESSION", + "failed_invariants": [ + "identity_and_evidence_validation", + "measurement_validation", + "schema_version", + "duplicate_handoff_id_rejection", + "repeated_transition_accounting" + ] + }, + "planning_gate": { + "status": "REJECTED_API_INCOMPATIBILITY", + "tests_passed": 0, + "tests_failed": 0, + "collection_error": "missing build_replan_result" + }, + "routing_gate": { + "status": "DEFERRED_DEPENDENCY_BOUNDARY", + "tests_passed": 0, + "tests_failed": 0, + "collection_error": "missing arkx.integration" + }, + "execution_harness_gate": { + "status": "DEFERRED_ADAPTER_REQUIRED", + "cluster": [ + "configuration", + "execution", + "harness", + "integration", + "orchestration", + "event_log", + "verifier", + "request", + "selection", + "p82_baseline" + ], + "crosses_semantic_conflicts": [ + "acceptance", + "promotion", + "recovery", + "routing" + ] + }, + "next_block": "PR30_EXECUTION_HARNESS_DEPENDENCY_GATE" +} diff --git a/logs/architecture/pr30-execution-harness-adapter-20250925.json b/logs/architecture/pr30-execution-harness-adapter-20250925.json new file mode 100644 index 00000000..ea671c7a --- /dev/null +++ b/logs/architecture/pr30-execution-harness-adapter-20250925.json @@ -0,0 +1,24 @@ +{ + "decision": "0149", + "date": "2026-09-25", + "classification": "ADAPTER_CONTRACT_UNDEFINED", + "authority_disposition": "MAIN_GOVERNED_SPINE", + "execution_cluster_disposition": "NOT_INTEGRATED", + "provider_called": false, + "main_boundary": { + "symbol": "arkx.spine.BoundExecutorInvoker.invoke", + "result": "arkx.command.CommandResult", + "authority": "governed_command_observation" + }, + "pr30_boundary": { + "symbol": "arkx.execution.Executor.execute", + "result": "arkx.execution.ExecutionResult", + "authority": "executor_lifecycle_contract" + }, + "lossy_mapping_detected": [ + "ExecutionResult_to_CommandResult_would_invent_process_observation", + "CommandResult_to_ExecutionResult_would_invent_task_lifecycle_and_artifacts" + ], + "promotion": "NOT_AUTHORIZED", + "next_block": "PR30_NON_CODE_DISPOSITION_AUDIT" +} diff --git a/logs/architecture/pr30-historical-log-retention-20250925.json b/logs/architecture/pr30-historical-log-retention-20250925.json new file mode 100644 index 00000000..c6fd894b --- /dev/null +++ b/logs/architecture/pr30-historical-log-retention-20250925.json @@ -0,0 +1,20 @@ +{ + "decision": "0153", + "date": "2026-09-25", + "root": "logs/", + "files": 29101, + "bytes": 569137454, + "workspace_files": 26488, + "workspace_bytes": 452922069, + "nested_git_files": 127, + "nested_git_bytes": 80155322, + "tracked_log_paths": 228, + "tracked_log_bytes": 639036, + "local_only_bytes": 568498418, + "remote_log_bloat": false, + "classification": "ARCHIVE_ONLY", + "deletion": "NOT_EXECUTED", + "provider_called": false, + "promotion": "NOT_AUTHORIZED", + "next_block": "LOCAL_LOG_RETENTION_SCOPE_DECISION" +} diff --git a/logs/architecture/pr30-non-code-disposition-20250925.json b/logs/architecture/pr30-non-code-disposition-20250925.json new file mode 100644 index 00000000..2287b19c --- /dev/null +++ b/logs/architecture/pr30-non-code-disposition-20250925.json @@ -0,0 +1,46 @@ +{ + "decision": "0151", + "date": "2026-09-25", + "docs_added": 127, + "logs_added": 228, + "experiment_fixtures_added": 12, + "experiment_fixtures_valid_json": 12, + "experiment_fixtures_active_references": 9, + "experiment_fixtures_unreferenced_candidates": [ + "experiments/cost-tier-comparison-protocol.json", + "experiments/p82-block3-t1-functional-v2.json", + "experiments/p82-wave0-comparison-protocol.json" + ], + "fixture_reference_tests": { + "passed": 54, + "failed": 0, + "provider_called": false + }, + "decision_record_audit": { + "files": 153, + "broken_local_references": 0, + "duplicate_numeric_ids": [], + "old_filename_references": 0, + "renumbering_map": { + "0031-package-boundary-and-distribution": "0112-package-boundary-and-distribution", + "0032-minimal-public-chassis-cli": "0113-minimal-public-chassis-cli", + "0033-mvp-vertical-contract": "0114-mvp-vertical-contract", + "0034-neutral-verifier-command-boundary": "0115-neutral-verifier-command-boundary", + "0035-executor-qualification-v1-paired-protocol": "0116-executor-qualification-v1-paired-protocol", + "0037-second-executor-enablement": "0152-second-executor-enablement" + }, + "status": "TRACEABILITY_NORMALIZED" + }, + "nested_repository_cache_observed": true, + "deletion_performed": false, + "test_discovery_isolation_required": true, + "classifications": { + "decision_and_protocol_docs": "HISTORICAL_NORMATIVE_CANDIDATE", + "experiment_fixtures": "EXPERIMENT_INPUT_CANDIDATE", + "raw_logs_and_reports": "HISTORICAL_EVIDENCE", + "nested_workspaces_and_caches": "ARCHIVE_ONLY" + }, + "provider_called": false, + "promotion": "NOT_AUTHORIZED", + "next_block": "EXPERIMENT_FIXTURE_REFERENCE_AUDIT" +} diff --git a/logs/architecture/pr30-test-discovery-boundary-20250925.json b/logs/architecture/pr30-test-discovery-boundary-20250925.json new file mode 100644 index 00000000..f8e71e35 --- /dev/null +++ b/logs/architecture/pr30-test-discovery-boundary-20250925.json @@ -0,0 +1,21 @@ +{ + "decision": "0154", + "date": "2026-09-25", + "collection": { + "tests": 400, + "outside_tests_collected": 0, + "status": "PASS" + }, + "suite": { + "passed": 400, + "failed": 0, + "warnings": 3, + "python": "3.13", + "provider_called": false, + "command": "py -3.13 -m pytest -q -p no:cacheprovider --disable-warnings" + }, + "classification": "TEST_DISCOVERY_BOUNDARY_PASS", + "logs_deleted": false, + "promotion": "NOT_AUTHORIZED", + "next_block": "LOCAL_LOG_RETENTION_SCOPE_DECISION" +} diff --git a/logs/architecture/promotion-event-persistence-gate-block-20260923.json b/logs/architecture/promotion-event-persistence-gate-block-20260923.json new file mode 100644 index 00000000..2603cfe8 --- /dev/null +++ b/logs/architecture/promotion-event-persistence-gate-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-promotion-event-persistence-gate-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 329, "result": "PASS"}, + "criteria": { + "promotion_persisted_after_verification_and_acceptance": "PASS", + "missing_acceptance_rejected": "PASS", + "mismatched_acceptance_rejected": "PASS", + "promotion_reference_replayable": "PASS", + "candidate_mutated_by_persistence": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/public-result-boundary-block-20260923.json b/logs/architecture/public-result-boundary-block-20260923.json new file mode 100644 index 00000000..12841721 --- /dev/null +++ b/logs/architecture/public-result-boundary-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-public-result-boundary-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "The public package API can expose the neutral ExecutionResult without collision with the legacy orchestration result.", + "focused_test": {"result": "PASS", "tests_run": 96}, + "criteria": { + "public_execution_result_is_neutral": "PASS", + "legacy_result_has_explicit_name": "PASS", + "import_order_collision_removed": "PASS", + "legacy_internal_consumers": "DOCUMENTED_COMPATIBILITY_BOUNDARY" + }, + "acceptance": "LOCAL_PUBLIC_API_EVIDENCE_PASS_EXTERNAL_INTEGRATION_BLOCKED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/recovery-attempt-identity-block-20260923.json b/logs/architecture/recovery-attempt-identity-block-20260923.json new file mode 100644 index 00000000..2fc051da --- /dev/null +++ b/logs/architecture/recovery-attempt-identity-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "recovery-attempt-identity", + "date": "2026-09-23", + "hypothesis": "Recovery cannot silently reuse the source attempt.", + "criteria": {"new_identity_derived": "PASS", "source_lineage_checked": "PASS", "executor_invoked": "NO"}, + "local_tests_verified": 356, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/recovery-budget-gate-block-20260923.json b/logs/architecture/recovery-budget-gate-block-20260923.json new file mode 100644 index 00000000..60883a5b --- /dev/null +++ b/logs/architecture/recovery-budget-gate-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "recovery-budget-gate", + "date": "2026-09-23", + "hypothesis": "Recovery cannot bypass a persisted attempts budget.", + "criteria": {"budget_exhaustion_rejected": "PASS", "directory_not_created": "PASS", "ledger_mutated_by_reservation": "NO"}, + "local_tests_verified": 359, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/recovery-lineage-audited-loading-block-20260923.json b/logs/architecture/recovery-lineage-audited-loading-block-20260923.json new file mode 100644 index 00000000..b8692f1d --- /dev/null +++ b/logs/architecture/recovery-lineage-audited-loading-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "recovery-lineage-audited-loading", + "date": "2026-09-23", + "hypothesis": "Recovery lineage written by the artifact store cannot be silently ignored or drift from the loaded manifest.", + "criteria": {"lineage_round_trip": "PASS", "invalid_source_rejected": "PASS", "retry_recovery_ambiguity_rejected": "PASS"}, + "local_tests_verified": 358, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/recovery-manifest-binding-block-20260923.json b/logs/architecture/recovery-manifest-binding-block-20260923.json new file mode 100644 index 00000000..725ebbe9 --- /dev/null +++ b/logs/architecture/recovery-manifest-binding-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "recovery-manifest-binding", + "date": "2026-09-23", + "hypothesis": "A recovery directory cannot be made audit-ready by writing an unrelated manifest.", + "criteria": {"mismatched_manifest_rejected": "PASS", "manifest_written_before_lineage_validation": "NO"}, + "local_tests_verified": 358, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/recovery-plan-lineage-block-20260923.json b/logs/architecture/recovery-plan-lineage-block-20260923.json new file mode 100644 index 00000000..54bf2e05 --- /dev/null +++ b/logs/architecture/recovery-plan-lineage-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-recovery-plan-lineage-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Recovery decisions can become deterministic, attempt-bound plans without executing recovery or changing executor selection.", + "focused_test": {"result": "PASS", "tests_run": 137}, + "criteria": { + "stable_plan_identity": "PASS", + "source_attempt_preserved": "PASS", + "external_execution_invoked": "NO", + "implicit_executor_switch": "NO" + }, + "acceptance": "LOCAL_RECOVERY_PLAN_EVIDENCE_PASS_EXECUTION_SEPARATE", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/recovery-reference-identity-block-20260923.json b/logs/architecture/recovery-reference-identity-block-20260923.json new file mode 100644 index 00000000..7a816035 --- /dev/null +++ b/logs/architecture/recovery-reference-identity-block-20260923.json @@ -0,0 +1,15 @@ +{ + "schema_version": "arkx-recovery-reference-identity-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "A recovery replay reference can be bound to one deterministic recovery plan identity.", + "focused_test": {"result": "PASS", "tests_run": 138}, + "criteria": { + "canonical_reference_validated": "PASS", + "wrong_reference_rejected": "PASS", + "external_recovery_invoked": "NO" + }, + "acceptance": "LOCAL_RECOVERY_REFERENCE_EVIDENCE_PASS_NO_EXECUTION", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/recovery-replay-lineage-block-20260923.json b/logs/architecture/recovery-replay-lineage-block-20260923.json new file mode 100644 index 00000000..2e6e246d --- /dev/null +++ b/logs/architecture/recovery-replay-lineage-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-recovery-replay-lineage-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Recovery plans can be represented in replay causally without making recovery mandatory or breaking legacy positional consumers.", + "focused_test": {"result": "PASS", "tests_run": 138}, + "criteria": { + "recovery_order_validated": "PASS", + "recovery_optional": "PASS", + "legacy_positional_compatibility": "PASS", + "recovery_executed": "NO" + }, + "acceptance": "LOCAL_RECOVERY_REPLAY_EVIDENCE_PASS_COMPATIBILITY_PRESERVED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/replay-chain-integrity-block-20260923.json b/logs/architecture/replay-chain-integrity-block-20260923.json new file mode 100644 index 00000000..dd32f80e --- /dev/null +++ b/logs/architecture/replay-chain-integrity-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-replay-chain-integrity-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Replay chain integrity can distinguish complete, incomplete, and inconsistent state before acceptance or promotion.", + "focused_test": {"result": "PASS", "tests_run": 131}, + "criteria": { + "complete_distinguished": "PASS", + "incomplete_distinguished": "PASS", + "inconsistent_distinguished": "PASS", + "acceptance_or_promotion_invoked": "NO" + }, + "acceptance": "LOCAL_REPLAY_INTEGRITY_EVIDENCE_PASS_PREDECISION_ONLY", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/replay-independent-outcomes-block-20260923.json b/logs/architecture/replay-independent-outcomes-block-20260923.json new file mode 100644 index 00000000..a855c3af --- /dev/null +++ b/logs/architecture/replay-independent-outcomes-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-replay-independent-outcomes-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Replay can preserve verification and acceptance as separate explicit outcomes without deriving one from execution completion.", + "focused_test": {"result": "PASS", "tests_run": 125}, + "criteria": { + "verification_preserved_separately": "PASS", + "acceptance_preserved_separately": "PASS", + "conflict_rejected": "PASS", + "outcome_inferred_from_completion": "NO" + }, + "acceptance": "LOCAL_INDEPENDENT_OUTCOME_REPLAY_EVIDENCE_PASS", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/replay-manifest-consistency-block-20260923.json b/logs/architecture/replay-manifest-consistency-block-20260923.json new file mode 100644 index 00000000..da066924 --- /dev/null +++ b/logs/architecture/replay-manifest-consistency-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-replay-manifest-consistency-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Replayed lifecycle facts can be checked against a persisted manifest without state repair or execution.", + "focused_test": {"result": "PASS", "tests_run": 124}, + "criteria": { + "matching_replay_accepted": "PASS", + "status_divergence_rejected": "PASS", + "missing_status_inferred": "NO", + "executor_invoked": "NO" + }, + "acceptance": "LOCAL_REPLAY_MANIFEST_CONSISTENCY_EVIDENCE_PASS_NO_STATE_REPAIR", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/replay-object-chain-consistency-block-20260923.json b/logs/architecture/replay-object-chain-consistency-block-20260923.json new file mode 100644 index 00000000..1104178a --- /dev/null +++ b/logs/architecture/replay-object-chain-consistency-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-replay-object-chain-consistency-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 308, "result": "PASS"}, + "criteria": { + "matching_outcome_objects_accepted": "PASS", + "verification_reference_drift_rejected": "PASS", + "acceptance_reference_drift_rejected": "PASS", + "promotion_reference_drift_rejected": "PASS", + "external_reexecution_during_validation": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/replay-operational-telemetry-block-20260923.json b/logs/architecture/replay-operational-telemetry-block-20260923.json new file mode 100644 index 00000000..032b5605 --- /dev/null +++ b/logs/architecture/replay-operational-telemetry-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-replay-operational-telemetry-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 330, "result": "PASS"}, + "criteria": { + "execution_outcome_replayed": "PASS", + "error_code_replayed": "PASS", + "preflight_status_replay_supported": "PASS", + "promotion_status_replay_supported": "PASS", + "legacy_error_compatibility_preserved": "PASS", + "success_inferred_from_telemetry": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/replay-reference-classification-block-20260923.json b/logs/architecture/replay-reference-classification-block-20260923.json new file mode 100644 index 00000000..c59ac5ec --- /dev/null +++ b/logs/architecture/replay-reference-classification-block-20260923.json @@ -0,0 +1,16 @@ +{ + "schema_version": "arkx-replay-reference-classification-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Replay references can distinguish present, missing, and indeterminate artifacts without treating recorded URIs as physical evidence.", + "focused_test": {"result": "PASS", "tests_run": 129}, + "criteria": { + "present_classified": "PASS", + "missing_classified": "PASS", + "no_inventory_is_indeterminate": "PASS", + "reference_inferred_as_success": "NO" + }, + "acceptance": "LOCAL_REFERENCE_CLASSIFICATION_EVIDENCE_PASS_PHYSICAL_VALIDATION_EXPLICIT", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/replay-telemetry-result-consistency-block-20260923.json b/logs/architecture/replay-telemetry-result-consistency-block-20260923.json new file mode 100644 index 00000000..a9398aca --- /dev/null +++ b/logs/architecture/replay-telemetry-result-consistency-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-replay-telemetry-result-consistency-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 331, "result": "PASS"}, + "criteria": { + "verification_telemetry_drift_rejected": "PASS", + "execution_telemetry_check_supported": "PASS", + "acceptance_telemetry_check_supported": "PASS", + "promotion_telemetry_check_supported": "PASS", + "external_reexecution_during_validation": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/restored-chain-audit-block-20260923.json b/logs/architecture/restored-chain-audit-block-20260923.json new file mode 100644 index 00000000..d709d2e4 --- /dev/null +++ b/logs/architecture/restored-chain-audit-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-restored-chain-audit-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 310, "result": "PASS"}, + "criteria": { + "restored_event_log_replayed_without_execution": "PASS", + "manifest_replay_status_divergence_rejected": "PASS", + "incomplete_chain_reported_not_promoted": "PASS", + "outcome_object_consistency_checked": "PASS", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/retry-attempt-lineage-block-20260923.json b/logs/architecture/retry-attempt-lineage-block-20260923.json new file mode 100644 index 00000000..4c80f65c --- /dev/null +++ b/logs/architecture/retry-attempt-lineage-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-retry-attempt-lineage-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "An authorized retry can create deterministic, non-overwriting attempt lineage without invoking execution.", + "focused_test": {"result": "PASS", "tests_run": 109}, + "criteria": { + "stable_next_attempt_identity": "PASS", + "previous_attempt_preserved": "PASS", + "unauthorized_retry_has_no_plan": "PASS", + "executor_invoked": "NO", + "artifact_overwritten": "NO" + }, + "acceptance": "LOCAL_RETRY_LINEAGE_EVIDENCE_PASS_ARTIFACT_RESERVATION_SEPARATE", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/retry-budget-gate-block-20260923.json b/logs/architecture/retry-budget-gate-block-20260923.json new file mode 100644 index 00000000..16450488 --- /dev/null +++ b/logs/architecture/retry-budget-gate-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "retry-budget-gate", + "date": "2026-09-23", + "hypothesis": "Retry and recovery cannot disagree about attempt-budget enforcement.", + "criteria": {"retry_budget_exhaustion_rejected": "PASS", "directory_not_created": "PASS", "fallback_selected": "NO"}, + "local_tests_verified": 360, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/retry-policy-block-20260923.json b/logs/architecture/retry-policy-block-20260923.json new file mode 100644 index 00000000..cad965eb --- /dev/null +++ b/logs/architecture/retry-policy-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-retry-policy-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "hypothesis": "Retry authorization can be deterministic and separate from executor invocation and recovery decisions.", + "focused_test": {"result": "PASS", "tests_run": 107}, + "criteria": { + "retryable_error_authorized_within_limits": "PASS", + "unknown_error_rejected": "PASS", + "wrong_domain_rejected": "PASS", + "policy_and_budget_limits_enforced": "PASS", + "automatic_retry_executed": "NO" + }, + "acceptance": "LOCAL_RETRY_POLICY_EVIDENCE_PASS_AUTOMATIC_EXECUTION_NOT_QUALIFIED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/safe-recovery-resume-boundaries-block-20260923.json b/logs/architecture/safe-recovery-resume-boundaries-block-20260923.json new file mode 100644 index 00000000..20f249e0 --- /dev/null +++ b/logs/architecture/safe-recovery-resume-boundaries-block-20260923.json @@ -0,0 +1,9 @@ +{ + "block": "safe-recovery-resume-boundaries", + "date": "2026-09-23", + "hypothesis": "Crash recovery never reexecutes a stage whose evidence is missing by assumption.", + "criteria": {"restart_required_without_execution": "PASS", "verification_resume": "PASS", "acceptance_resume": "PASS", "terminal_not_resumed": "PASS", "implicit_executor_reexecution": "NO"}, + "local_tests_verified": 355, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/second-executor-preflight-20260924.json b/logs/architecture/second-executor-preflight-20260924.json new file mode 100644 index 00000000..4d4c8ed3 --- /dev/null +++ b/logs/architecture/second-executor-preflight-20260924.json @@ -0,0 +1,47 @@ +{ + "project": "Codepro", + "date": "2026-09-24", + "purpose": "Second Executor Enablement preflight", + "classification": "INFRASTRUCTURE_EVIDENCE_ONLY", + "existing_executor": { + "name": "mini-swe-agent", + "entrypoint": "mini-swe-agent.exe", + "status": "AVAILABLE_ENTRYPOINT", + "qualification": "EXISTING_PATH_ONLY" + }, + "candidates": [ + { + "name": "claude-code", + "entrypoint": "claude.exe", + "headless_probe": "claude -p ... --output-format json --tools \"\" --permission-prompts none --max-turns 1", + "observed": "Not logged in · Please run /login", + "status": "BLOCKED_IDENTITY" + }, + { + "name": "opencode", + "version": "2.0.15", + "headless_probe": "opencode --help", + "observed": "help succeeded; models/server startup failed with EPERM while spawning opencode.exe", + "status": "BLOCKED_RUNTIME" + }, + { + "name": "aider", + "entrypoint": "aider.exe", + "headless_probe": "aider --help", + "observed": "Application Control blocked process start", + "status": "BLOCKED_RUNTIME" + }, + { + "name": "codex", + "entrypoint": "codex.exe", + "observed": "Current controlling agent; not accepted as an independent executor for this experiment", + "status": "NOT_COMPARABLE" + } + ], + "qualification_run_v1": { + "status": "BLOCKED", + "reason": "No second independent executor completed headless preflight with patch/artefact extraction, bounded execution, and the shared verifier.", + "next_gate": "Second Executor Enablement", + "downstream": ["Qualification Run v1", "Baseline v1", "routing"] + } +} diff --git a/logs/architecture/terminal-fact-integrity-block-20260923.json b/logs/architecture/terminal-fact-integrity-block-20260923.json new file mode 100644 index 00000000..5b7153b5 --- /dev/null +++ b/logs/architecture/terminal-fact-integrity-block-20260923.json @@ -0,0 +1,14 @@ +{ + "block": "terminal-fact-integrity", + "date": "2026-09-23", + "hypothesis": "Replay cannot silently change the terminal explanation while preserving the same status.", + "criteria": { + "exact_terminal_match": "PASS", + "changed_error_code_rejected": "PASS", + "missing_terminal_rejected": "PASS", + "wrong_terminal_error_rejected": "PASS", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/terminal-lifecycle-error-events-block-20260923.json b/logs/architecture/terminal-lifecycle-error-events-block-20260923.json new file mode 100644 index 00000000..7e1ac61f --- /dev/null +++ b/logs/architecture/terminal-lifecycle-error-events-block-20260923.json @@ -0,0 +1,18 @@ +{ + "schema_version": "arkx-terminal-lifecycle-error-events-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 333, "result": "PASS"}, + "criteria": { + "terminal_status_persisted": "PASS", + "terminal_error_code_replayed": "PASS", + "terminal_append_idempotent": "PASS", + "terminal_status_mutation_rejected": "PASS", + "verification_or_acceptance_inferred": "NO", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/versioned-provider-identity-block-20260923.json b/logs/architecture/versioned-provider-identity-block-20260923.json new file mode 100644 index 00000000..c18988ce --- /dev/null +++ b/logs/architecture/versioned-provider-identity-block-20260923.json @@ -0,0 +1,10 @@ +{ + "block": "versioned-provider-identity", + "date": "2026-09-23", + "hypothesis": "A provider cannot enter the neutral contract without reproducible identity.", + "criteria": {"versioned_provider_accepted": "PASS", "anonymous_provider_rejected": "PASS", "provider_behavior_qualified": "NO"}, + "latest_local_tests_verified": 350, + "local_tests_verified": 349, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/versioned-replay-compatibility-block-20260923.json b/logs/architecture/versioned-replay-compatibility-block-20260923.json new file mode 100644 index 00000000..8aa62d25 --- /dev/null +++ b/logs/architecture/versioned-replay-compatibility-block-20260923.json @@ -0,0 +1,17 @@ +{ + "schema_version": "arkx-versioned-replay-compatibility-evidence-v1", + "date": "2026-09-23", + "branch": "codex/auditoriaarquitetural", + "environment": {"os": "Windows", "python": "3.11.9", "source_import": "PYTHONPATH=src"}, + "test_command": "python -m unittest discover -s tests -p test_*.py", + "full_suite": {"tests_run": 315, "result": "PASS"}, + "criteria": { + "protocol_drift_rejected_in_strict_audit": "PASS", + "schema_drift_rejected_in_strict_audit": "PASS", + "unversioned_historical_replay_readable": "PASS", + "strict_experiment_requires_version_metadata": "PASS", + "fallback_selected": "NO" + }, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/architecture/versioned-sandbox-identity-block-20260923.json b/logs/architecture/versioned-sandbox-identity-block-20260923.json new file mode 100644 index 00000000..7a45da23 --- /dev/null +++ b/logs/architecture/versioned-sandbox-identity-block-20260923.json @@ -0,0 +1,10 @@ +{ + "block": "versioned-sandbox-identity", + "date": "2026-09-23", + "hypothesis": "A sandbox cannot enter the neutral contract without reproducible runtime identity.", + "criteria": {"versioned_sandbox_accepted": "PASS", "anonymous_sandbox_rejected": "PASS", "isolation_qualified": "NO"}, + "latest_local_tests_verified": 350, + "local_tests_verified": 349, + "external_qualification": "NOT_PERFORMED", + "promotion": "NOT_PROMOTED" +} diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/eval.sh b/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/eval.sh new file mode 100644 index 00000000..04953b01 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/eval.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -uxo pipefail +source /opt/miniconda3/bin/activate +conda activate testbed +cd /testbed +git config --global --add safe.directory /testbed +cd /testbed +git status +git show +git -c core.fileMode=false diff c6753448b5c34f95e250105d76709fe4d349ca1f +source /opt/miniconda3/bin/activate +conda activate testbed +python -m pip install -e . +git checkout c6753448b5c34f95e250105d76709fe4d349ca1f sympy/physics/vector/tests/test_vector.py +git apply -v - <<'EOF_114329324912' +diff --git a/sympy/physics/vector/tests/test_vector.py b/sympy/physics/vector/tests/test_vector.py +--- a/sympy/physics/vector/tests/test_vector.py ++++ b/sympy/physics/vector/tests/test_vector.py +@@ -13,6 +13,8 @@ def test_Vector(): + assert A.y != A.z + assert A.z != A.x + ++ assert A.x + 0 == A.x ++ + v1 = x*A.x + y*A.y + z*A.z + v2 = x**2*A.x + y**2*A.y + z**2*A.z + v3 = v1 + v2 + +EOF_114329324912 +: '>>>>> Start Test Output' +PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose sympy/physics/vector/tests/test_vector.py +SWEBENCH_TEST_EXIT_CODE=$? +: '>>>>> End Test Output' +echo ">>>>> Test Exit Code: $SWEBENCH_TEST_EXIT_CODE" +git checkout c6753448b5c34f95e250105d76709fe4d349ca1f sympy/physics/vector/tests/test_vector.py diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/patch.diff b/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/patch.diff new file mode 100644 index 00000000..9119cf6a --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/patch.diff @@ -0,0 +1,12 @@ +diff --git a/sympy/physics/vector/vector.py b/sympy/physics/vector/vector.py +--- a/sympy/physics/vector/vector.py ++++ b/sympy/physics/vector/vector.py +@@ -57,6 +57,8 @@ def __hash__(self): + + def __add__(self, other): + """The add operator for Vector. """ ++ if other == 0: ++ return self + other = _check_vector(other) + return Vector(self.args + other.args) + diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/report.json b/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/report.json new file mode 100644 index 00000000..124348e0 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/report.json @@ -0,0 +1,32 @@ +{ + "sympy__sympy-14711": { + "patch_is_None": false, + "patch_exists": true, + "patch_successfully_applied": true, + "resolved": false, + "infra_failure": false, + "tests_status": { + "FAIL_TO_PASS": { + "success": [], + "failure": [ + "test_Vector" + ] + }, + "PASS_TO_PASS": { + "success": [], + "failure": [ + "test_Vector_diffs", + "test_vector_var_in_dcm" + ] + }, + "FAIL_TO_FAIL": { + "success": [], + "failure": [] + }, + "PASS_TO_FAIL": { + "success": [], + "failure": [] + } + } + } +} \ No newline at end of file diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/run_instance.log b/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/run_instance.log new file mode 100644 index 00000000..49714045 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/run_instance.log @@ -0,0 +1,48 @@ +2026-09-23 04:39:29,177 - INFO - Image not found locally, attempting to pull... +2026-09-23 04:40:04,464 - INFO - Creating container for sympy__sympy-14711... +2026-09-23 04:40:05,119 - INFO - Container for sympy__sympy-14711 created: 1796e10c728022b699dab3609deb3167c662207437991087974d6346ac838cb9 +2026-09-23 04:40:05,622 - INFO - Container for sympy__sympy-14711 started: 1796e10c728022b699dab3609deb3167c662207437991087974d6346ac838cb9 +2026-09-23 04:40:05,622 - INFO - Intermediate patch for sympy__sympy-14711 written to logs\evaluation\arkx-p82-gold-docker-20260923-01\gold\sympy__sympy-14711\patch.diff, now applying to container... +2026-09-23 04:40:05,800 - INFO - Failed to apply patch to container: git apply --verbose +2026-09-23 04:40:06,309 - INFO - Failed to apply patch to container: git apply --verbose --3way +2026-09-23 04:40:06,557 - INFO - Failed to apply patch to container: git apply --verbose --reject +2026-09-23 04:40:06,785 - INFO - >>>>> Applied Patch: +(Stripping trailing CRs from patch; use --binary to disable.) +patching file sympy/physics/vector/vector.py + +2026-09-23 04:40:06,894 - INFO - Git diff before: +diff --git a/sympy/physics/vector/vector.py b/sympy/physics/vector/vector.py +index a048ee57f..884911921 100644 +--- a/sympy/physics/vector/vector.py ++++ b/sympy/physics/vector/vector.py +@@ -57,6 +57,8 @@ def __hash__(self): + + def __add__(self, other): + """The add operator for Vector. """ ++ if other == 0: ++ return self + other = _check_vector(other) + return Vector(self.args + other.args) +2026-09-23 04:40:06,895 - INFO - Eval script for sympy__sympy-14711 written to logs\evaluation\arkx-p82-gold-docker-20260923-01\gold\sympy__sympy-14711\eval.sh; copying to container... +2026-09-23 04:40:08,925 - INFO - Test runtime: 1.97 seconds +2026-09-23 04:40:08,926 - INFO - Test output for sympy__sympy-14711 written to logs\evaluation\arkx-p82-gold-docker-20260923-01\gold\sympy__sympy-14711\test_output.txt +2026-09-23 04:40:09,029 - INFO - Git diff after: +diff --git a/sympy/physics/vector/vector.py b/sympy/physics/vector/vector.py +index a048ee57f..884911921 100644 +--- a/sympy/physics/vector/vector.py ++++ b/sympy/physics/vector/vector.py +@@ -57,6 +57,8 @@ def __hash__(self): + + def __add__(self, other): + """The add operator for Vector. """ ++ if other == 0: ++ return self + other = _check_vector(other) + return Vector(self.args + other.args) +2026-09-23 04:40:09,029 - INFO - Grading answer for sympy__sympy-14711... +2026-09-23 04:40:09,029 - INFO - report: {'sympy__sympy-14711': {'patch_is_None': False, 'patch_exists': True, 'patch_successfully_applied': True, 'resolved': False, 'infra_failure': False, 'tests_status': {'FAIL_TO_PASS': {'success': [], 'failure': ['test_Vector']}, 'PASS_TO_PASS': {'success': [], 'failure': ['test_Vector_diffs', 'test_vector_var_in_dcm']}, 'FAIL_TO_FAIL': {'success': [], 'failure': []}, 'PASS_TO_FAIL': {'success': [], 'failure': []}}}} +Result for sympy__sympy-14711: resolved: False +2026-09-23 04:40:09,030 - INFO - Attempting to stop container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-20260923-01... +2026-09-23 04:40:24,642 - INFO - Container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-20260923-01 stopped successfully. +2026-09-23 04:40:24,643 - INFO - Attempting to remove container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-20260923-01... +2026-09-23 04:40:24,896 - INFO - Container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-20260923-01 removed successfully. diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/test_output.txt b/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/test_output.txt new file mode 100644 index 00000000..451f7036 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-01/gold/sympy__sympy-14711/test_output.txt @@ -0,0 +1,70 @@ +/eval.sh: line 2: set: pipefail : invalid option name ++ source $'/opt/miniconda3/bin/activate\r' +/eval.sh: line 3: /opt/miniconda3/bin/activate : No such file or directory ++ conda activate $'testbed\r' + +CondaError: Run 'conda init' before 'conda activate' + ++ cd $'/testbed\r' +/eval.sh: line 5: cd: $'/testbed\r': No such file or directory ++ git config --global --add safe.directory $'/testbed\r' ++ cd $'/testbed\r' +/eval.sh: line 7: cd: $'/testbed\r': No such file or directory ++ git $'status\r' +git: 'status ' is not a git command. See 'git --help'. + +The most similar command is + status ++ git $'show\r' +git: 'show ' is not a git command. See 'git --help'. + +The most similar command is + show ++ git -c core.fileMode=false diff $'c6753448b5c34f95e250105d76709fe4d349ca1f\r' +fatal: ambiguous argument 'c6753448b5c34f95e250105d76709fe4d349ca1f?': unknown revision or path not in the working tree. +Use '--' to separate paths from revisions, like this: +'git [...] -- [...]' ++ source $'/opt/miniconda3/bin/activate\r' +/eval.sh: line 11: /opt/miniconda3/bin/activate : No such file or directory ++ conda activate $'testbed\r' + +CondaError: Run 'conda init' before 'conda activate' + ++ python -m pip install -e $'.\r' +ERROR: . is not a valid editable requirement. It should either be a path to a local project or a VCS URL (beginning with bzr+http, bzr+https, bzr+ssh, bzr+sftp, bzr+ftp, bzr+lp, bzr+file, git+http, git+https, git+ssh, git+git, git+file, hg+file, hg+http, hg+https, hg+ssh, hg+static-http, svn+ssh, svn+http, svn+https, svn+svn, svn+file). ++ git checkout c6753448b5c34f95e250105d76709fe4d349ca1f $'sympy/physics/vector/tests/test_vector.py\r' +error: pathspec 'sympy/physics/vector/tests/test_vector.py?' did not match any file(s) known to git ++ git apply -v - +Checking patch sympy/physics/vector/tests/test_vector.py... +error: while searching for: + assert A.y != A.z? + assert A.z != A.x? +? + v1 = x*A.x + y*A.y + z*A.z? + v2 = x**2*A.x + y**2*A.y + z**2*A.z? + v3 = v1 + v2? + +error: patch failed: sympy/physics/vector/tests/test_vector.py:13 +error: sympy/physics/vector/tests/test_vector.py: patch does not apply ++ : '>>>>> Start Test Output ' ++ PYTHONWARNINGS=ignore::UserWarning,ignore::SyntaxWarning ++ bin/test -C --verbose $'sympy/physics/vector/tests/test_vector.py\r' +Traceback (most recent call last): + File "/testbed/sympy/__init__.py", line 19, in + import mpmath +ModuleNotFoundError: No module named 'mpmath' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/testbed/bin/test", line 105, in + import sympy + File "/testbed/sympy/__init__.py", line 21, in + raise ImportError("SymPy now depends on mpmath as an external library. " +ImportError: SymPy now depends on mpmath as an external library. See http://docs.sympy.org/latest/install.html#mpmath for more information. ++ SWEBENCH_TEST_EXIT_CODE=$'1\r' ++ : '>>>>> End Test Output ' ++ echo '>>>>> Test Exit Code: 1 ' ++ git checkout c6753448b5c34f95e250105d76709fe4d349ca1f $'sympy/physics/vector/tests/test_vector.py\r' +>>>>> Test Exit Code: 1 +error: pathspec 'sympy/physics/vector/tests/test_vector.py?' did not match any file(s) known to git diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-01/results.json b/logs/evaluation/arkx-p82-gold-docker-20260923-01/results.json new file mode 100644 index 00000000..c0afc77e --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-01/results.json @@ -0,0 +1,37 @@ +{ + "total_instances": 1, + "submitted_instances": 1, + "completed_instances": 1, + "resolved_instances": 0, + "unresolved_instances": 1, + "infra_failure_instances": 0, + "ambiguous_failure_instances": 1, + "empty_patch_instances": 0, + "error_instances": 0, + "completed_ids": [ + "sympy__sympy-14711" + ], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "sympy__sympy-14711" + ], + "resolved_ids": [], + "unresolved_ids": [ + "sympy__sympy-14711" + ], + "infra_failure_ids": [], + "ambiguous_failure_ids": [ + "sympy__sympy-14711" + ], + "failure_reasons": { + "sympy__sympy-14711": "missing_module" + }, + "error_ids": [], + "schema_version": 2, + "unstopped_instances": 0, + "unstopped_containers": [], + "unremoved_images": [ + "swebench/sweb.eval.x86_64.sympy_1776_sympy-14711:latest" + ] +} diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-01/run.json b/logs/evaluation/arkx-p82-gold-docker-20260923-01/run.json new file mode 100644 index 00000000..06da0966 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-01/run.json @@ -0,0 +1,6 @@ +{ + "dataset": "SWE-bench/SWE-bench_Verified", + "split": "test", + "task_repo": null, + "created_at": "2026-09-23T08:50:36+00:00" +} diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/eval.sh b/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/eval.sh new file mode 100644 index 00000000..7ebf973e --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/eval.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -uxo pipefail +source /opt/miniconda3/bin/activate +conda activate testbed +cd /testbed +git config --global --add safe.directory /testbed +cd /testbed +git status +git show +git -c core.fileMode=false diff 809c53c077485ca48a206cee78340389cb83b7f1 +source /opt/miniconda3/bin/activate +conda activate testbed +python -m pip install -e . +git checkout 809c53c077485ca48a206cee78340389cb83b7f1 sympy/combinatorics/tests/test_homomorphisms.py +git apply -v - <<'EOF_114329324912' +diff --git a/sympy/combinatorics/tests/test_homomorphisms.py b/sympy/combinatorics/tests/test_homomorphisms.py +--- a/sympy/combinatorics/tests/test_homomorphisms.py ++++ b/sympy/combinatorics/tests/test_homomorphisms.py +@@ -57,6 +57,11 @@ def test_homomorphism(): + assert T.codomain == D + assert T(a*b) == p + ++ D3 = DihedralGroup(3) ++ T = homomorphism(D3, D3, D3.generators, D3.generators) ++ assert T.is_isomorphism() ++ ++ + def test_isomorphisms(): + + F, a, b = free_group("a, b") + +EOF_114329324912 +: '>>>>> Start Test Output' +PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose sympy/combinatorics/tests/test_homomorphisms.py +SWEBENCH_TEST_EXIT_CODE=$? +: '>>>>> End Test Output' +echo ">>>>> Test Exit Code: $SWEBENCH_TEST_EXIT_CODE" +git checkout 809c53c077485ca48a206cee78340389cb83b7f1 sympy/combinatorics/tests/test_homomorphisms.py diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/patch.diff b/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/patch.diff new file mode 100644 index 00000000..0e325652 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/patch.diff @@ -0,0 +1,68 @@ +diff --git a/sympy/combinatorics/homomorphisms.py b/sympy/combinatorics/homomorphisms.py +--- a/sympy/combinatorics/homomorphisms.py ++++ b/sympy/combinatorics/homomorphisms.py +@@ -308,42 +308,31 @@ def homomorphism(domain, codomain, gens, images=(), check=True): + return GroupHomomorphism(domain, codomain, images) + + def _check_homomorphism(domain, codomain, images): +- if hasattr(domain, 'relators'): +- rels = domain.relators +- else: +- gens = domain.presentation().generators +- rels = domain.presentation().relators ++ """ ++ Check that a given mapping of generators to images defines a homomorphism. ++ ++ Parameters ++ ========== ++ domain : PermutationGroup, FpGroup, FreeGroup ++ codomain : PermutationGroup, FpGroup, FreeGroup ++ images : dict ++ The set of keys must be equal to domain.generators. ++ The values must be elements of the codomain. ++ ++ """ ++ pres = domain if hasattr(domain, 'relators') else domain.presentation() ++ rels = pres.relators ++ gens = pres.generators ++ symbols = [g.ext_rep[0] for g in gens] ++ symbols_to_domain_generators = dict(zip(symbols, domain.generators)) + identity = codomain.identity + + def _image(r): +- if r.is_identity: +- return identity +- else: +- w = identity +- r_arr = r.array_form +- i = 0 +- j = 0 +- # i is the index for r and j is for +- # r_arr. r_arr[j] is the tuple (sym, p) +- # where sym is the generator symbol +- # and p is the power to which it is +- # raised while r[i] is a generator +- # (not just its symbol) or the inverse of +- # a generator - hence the need for +- # both indices +- while i < len(r): +- power = r_arr[j][1] +- if isinstance(domain, PermutationGroup) and r[i] in gens: +- s = domain.generators[gens.index(r[i])] +- else: +- s = r[i] +- if s in images: +- w = w*images[s]**power +- elif s**-1 in images: +- w = w*images[s**-1]**power +- i += abs(power) +- j += 1 +- return w ++ w = identity ++ for symbol, power in r.array_form: ++ g = symbols_to_domain_generators[symbol] ++ w *= images[g]**power ++ return w + + for r in rels: + if isinstance(codomain, FpGroup): diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/report.json b/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/report.json new file mode 100644 index 00000000..d24c3493 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/report.json @@ -0,0 +1,31 @@ +{ + "sympy__sympy-24443": { + "patch_is_None": false, + "patch_exists": true, + "patch_successfully_applied": true, + "resolved": false, + "infra_failure": false, + "tests_status": { + "FAIL_TO_PASS": { + "success": [], + "failure": [ + "test_homomorphism" + ] + }, + "PASS_TO_PASS": { + "success": [], + "failure": [ + "test_isomorphisms" + ] + }, + "FAIL_TO_FAIL": { + "success": [], + "failure": [] + }, + "PASS_TO_FAIL": { + "success": [], + "failure": [] + } + } + } +} \ No newline at end of file diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/run_instance.log b/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/run_instance.log new file mode 100644 index 00000000..9c4095ff --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/run_instance.log @@ -0,0 +1,162 @@ +2026-09-23 04:40:59,847 - INFO - Image not found locally, attempting to pull... +2026-09-23 04:41:15,725 - INFO - Creating container for sympy__sympy-24443... +2026-09-23 04:41:16,094 - INFO - Container for sympy__sympy-24443 created: 16bca0454f11e34a89ab32a49a27c8cc5bd701cab23bd2140a17a5f17fffaf95 +2026-09-23 04:41:16,322 - INFO - Container for sympy__sympy-24443 started: 16bca0454f11e34a89ab32a49a27c8cc5bd701cab23bd2140a17a5f17fffaf95 +2026-09-23 04:41:16,323 - INFO - Intermediate patch for sympy__sympy-24443 written to logs\evaluation\arkx-p82-gold-docker-20260923-02\gold\sympy__sympy-24443\patch.diff, now applying to container... +2026-09-23 04:41:16,447 - INFO - Failed to apply patch to container: git apply --verbose +2026-09-23 04:41:16,958 - INFO - Failed to apply patch to container: git apply --verbose --3way +2026-09-23 04:41:17,259 - INFO - Failed to apply patch to container: git apply --verbose --reject +2026-09-23 04:41:17,378 - INFO - >>>>> Applied Patch: +(Stripping trailing CRs from patch; use --binary to disable.) +patching file sympy/combinatorics/homomorphisms.py + +2026-09-23 04:41:17,436 - INFO - Git diff before: +diff --git a/sympy/combinatorics/homomorphisms.py b/sympy/combinatorics/homomorphisms.py +index bb7bdf32ba..95b43c5013 100644 +--- a/sympy/combinatorics/homomorphisms.py ++++ b/sympy/combinatorics/homomorphisms.py +@@ -308,42 +308,31 @@ def homomorphism(domain, codomain, gens, images=(), check=True): + return GroupHomomorphism(domain, codomain, images) + + def _check_homomorphism(domain, codomain, images): +- if hasattr(domain, 'relators'): +- rels = domain.relators +- else: +- gens = domain.presentation().generators +- rels = domain.presentation().relators ++ """ ++ Check that a given mapping of generators to images defines a homomorphism. ++ ++ Parameters ++ ========== ++ domain : PermutationGroup, FpGroup, FreeGroup ++ codomain : PermutationGroup, FpGroup, FreeGroup ++ images : dict ++ The set of keys must be equal to domain.generators. ++ The values must be elements of the codomain. ++ ++ """ ++ pres = domain if hasattr(domain, 'relators') else domain.presentation() ++ rels = pres.relators ++ gens = pres.generators ++ symbols = [g.ext_rep[0] for g in gens] ++ symbols_to_domain_generators = dict(zip(symbols, domain.generators)) + identity = codomain.identity + + def _image(r): +- if r.is_identity: +- return identity +- else: +- w = identity +- r_arr = r.array_form +- i = 0 +- j = 0 +- # i is the index for r and j is for +- # r_arr. r_arr[j] is the tuple (sym, p) +- # where sym is the generator symbol +- # and p is the power to which it is +- # raised while r[i] is a generator +- # (not just its symbol) or the inverse of +- # a generator - hence the need for +- # both indices +- while i < len(r): +- power = r_arr[j][1] +- if isinstance(domain, PermutationGroup) and r[i] in gens: +- s = domain.generators[gens.index(r[i])] +- else: +- s = r[i] +- if s in images: +- w = w*images[s]**power +- elif s**-1 in images: +- w = w*images[s**-1]**power +- i += abs(power) +- j += 1 +- return w ++ w = identity ++ for symbol, power in r.array_form: ++ g = symbols_to_domain_generators[symbol] ++ w *= images[g]**power ++ return w + + for r in rels: + if isinstance(codomain, FpGroup): +2026-09-23 04:41:17,437 - INFO - Eval script for sympy__sympy-24443 written to logs\evaluation\arkx-p82-gold-docker-20260923-02\gold\sympy__sympy-24443\eval.sh; copying to container... +2026-09-23 04:41:18,783 - INFO - Test runtime: 1.28 seconds +2026-09-23 04:41:18,783 - INFO - Test output for sympy__sympy-24443 written to logs\evaluation\arkx-p82-gold-docker-20260923-02\gold\sympy__sympy-24443\test_output.txt +2026-09-23 04:41:18,834 - INFO - Git diff after: +diff --git a/sympy/combinatorics/homomorphisms.py b/sympy/combinatorics/homomorphisms.py +index bb7bdf32ba..95b43c5013 100644 +--- a/sympy/combinatorics/homomorphisms.py ++++ b/sympy/combinatorics/homomorphisms.py +@@ -308,42 +308,31 @@ def homomorphism(domain, codomain, gens, images=(), check=True): + return GroupHomomorphism(domain, codomain, images) + + def _check_homomorphism(domain, codomain, images): +- if hasattr(domain, 'relators'): +- rels = domain.relators +- else: +- gens = domain.presentation().generators +- rels = domain.presentation().relators ++ """ ++ Check that a given mapping of generators to images defines a homomorphism. ++ ++ Parameters ++ ========== ++ domain : PermutationGroup, FpGroup, FreeGroup ++ codomain : PermutationGroup, FpGroup, FreeGroup ++ images : dict ++ The set of keys must be equal to domain.generators. ++ The values must be elements of the codomain. ++ ++ """ ++ pres = domain if hasattr(domain, 'relators') else domain.presentation() ++ rels = pres.relators ++ gens = pres.generators ++ symbols = [g.ext_rep[0] for g in gens] ++ symbols_to_domain_generators = dict(zip(symbols, domain.generators)) + identity = codomain.identity + + def _image(r): +- if r.is_identity: +- return identity +- else: +- w = identity +- r_arr = r.array_form +- i = 0 +- j = 0 +- # i is the index for r and j is for +- # r_arr. r_arr[j] is the tuple (sym, p) +- # where sym is the generator symbol +- # and p is the power to which it is +- # raised while r[i] is a generator +- # (not just its symbol) or the inverse of +- # a generator - hence the need for +- # both indices +- while i < len(r): +- power = r_arr[j][1] +- if isinstance(domain, PermutationGroup) and r[i] in gens: +- s = domain.generators[gens.index(r[i])] +- else: +- s = r[i] +- if s in images: +- w = w*images[s]**power +- elif s**-1 in images: +- w = w*images[s**-1]**power +- i += abs(power) +- j += 1 +- return w ++ w = identity ++ for symbol, power in r.array_form: ++ g = symbols_to_domain_generators[symbol] ++ w *= images[g]**power ++ return w + + for r in rels: + if isinstance(codomain, FpGroup): +2026-09-23 04:41:18,834 - INFO - Grading answer for sympy__sympy-24443... +2026-09-23 04:41:18,835 - INFO - report: {'sympy__sympy-24443': {'patch_is_None': False, 'patch_exists': True, 'patch_successfully_applied': True, 'resolved': False, 'infra_failure': False, 'tests_status': {'FAIL_TO_PASS': {'success': [], 'failure': ['test_homomorphism']}, 'PASS_TO_PASS': {'success': [], 'failure': ['test_isomorphisms']}, 'FAIL_TO_FAIL': {'success': [], 'failure': []}, 'PASS_TO_FAIL': {'success': [], 'failure': []}}}} +Result for sympy__sympy-24443: resolved: False +2026-09-23 04:41:18,836 - INFO - Attempting to stop container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-20260923-02... +2026-09-23 04:41:34,375 - INFO - Container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-20260923-02 stopped successfully. +2026-09-23 04:41:34,375 - INFO - Attempting to remove container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-20260923-02... +2026-09-23 04:41:34,618 - INFO - Container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-20260923-02 removed successfully. diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/test_output.txt b/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/test_output.txt new file mode 100644 index 00000000..bbcc2556 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-02/gold/sympy__sympy-24443/test_output.txt @@ -0,0 +1,70 @@ +/eval.sh: line 2: set: pipefail : invalid option name ++ source $'/opt/miniconda3/bin/activate\r' +/eval.sh: line 3: /opt/miniconda3/bin/activate : No such file or directory ++ conda activate $'testbed\r' + +CondaError: Run 'conda init' before 'conda activate' + ++ cd $'/testbed\r' +/eval.sh: line 5: cd: $'/testbed\r': No such file or directory ++ git config --global --add safe.directory $'/testbed\r' ++ cd $'/testbed\r' +/eval.sh: line 7: cd: $'/testbed\r': No such file or directory ++ git $'status\r' +git: 'status ' is not a git command. See 'git --help'. + +The most similar command is + status ++ git $'show\r' +git: 'show ' is not a git command. See 'git --help'. + +The most similar command is + show ++ git -c core.fileMode=false diff $'809c53c077485ca48a206cee78340389cb83b7f1\r' +fatal: ambiguous argument '809c53c077485ca48a206cee78340389cb83b7f1?': unknown revision or path not in the working tree. +Use '--' to separate paths from revisions, like this: +'git [...] -- [...]' ++ source $'/opt/miniconda3/bin/activate\r' +/eval.sh: line 11: /opt/miniconda3/bin/activate : No such file or directory ++ conda activate $'testbed\r' + +CondaError: Run 'conda init' before 'conda activate' + ++ python -m pip install -e $'.\r' +ERROR: . is not a valid editable requirement. It should either be a path to a local project or a VCS URL (beginning with bzr+http, bzr+https, bzr+ssh, bzr+sftp, bzr+ftp, bzr+lp, bzr+file, git+http, git+https, git+ssh, git+git, git+file, hg+file, hg+http, hg+https, hg+ssh, hg+static-http, svn+ssh, svn+http, svn+https, svn+svn, svn+file). ++ git checkout 809c53c077485ca48a206cee78340389cb83b7f1 $'sympy/combinatorics/tests/test_homomorphisms.py\r' +error: pathspec 'sympy/combinatorics/tests/test_homomorphisms.py?' did not match any file(s) known to git ++ git apply -v - +Checking patch sympy/combinatorics/tests/test_homomorphisms.py... +error: while searching for: + assert T.codomain == D? + assert T(a*b) == p? +? +def test_isomorphisms():? +? + F, a, b = free_group("a, b")? + +error: patch failed: sympy/combinatorics/tests/test_homomorphisms.py:57 +error: sympy/combinatorics/tests/test_homomorphisms.py: patch does not apply ++ : '>>>>> Start Test Output ' ++ PYTHONWARNINGS=ignore::UserWarning,ignore::SyntaxWarning ++ bin/test -C --verbose $'sympy/combinatorics/tests/test_homomorphisms.py\r' +Traceback (most recent call last): + File "/testbed/sympy/__init__.py", line 22, in + import mpmath +ModuleNotFoundError: No module named 'mpmath' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/testbed/bin/test", line 95, in + import sympy + File "/testbed/sympy/__init__.py", line 24, in + raise ImportError("SymPy now depends on mpmath as an external library. " +ImportError: SymPy now depends on mpmath as an external library. See https://docs.sympy.org/latest/install.html#mpmath for more information. ++ SWEBENCH_TEST_EXIT_CODE=$'1\r' ++ : '>>>>> End Test Output ' ++ echo '>>>>> Test Exit Code: 1 ' ++ git checkout 809c53c077485ca48a206cee78340389cb83b7f1 $'sympy/combinatorics/tests/test_homomorphisms.py\r' +>>>>> Test Exit Code: 1 +error: pathspec 'sympy/combinatorics/tests/test_homomorphisms.py?' did not match any file(s) known to git diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-02/results.json b/logs/evaluation/arkx-p82-gold-docker-20260923-02/results.json new file mode 100644 index 00000000..9642dc46 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-02/results.json @@ -0,0 +1,37 @@ +{ + "total_instances": 1, + "submitted_instances": 1, + "completed_instances": 1, + "resolved_instances": 0, + "unresolved_instances": 1, + "infra_failure_instances": 0, + "ambiguous_failure_instances": 1, + "empty_patch_instances": 0, + "error_instances": 0, + "completed_ids": [ + "sympy__sympy-24443" + ], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "sympy__sympy-24443" + ], + "resolved_ids": [], + "unresolved_ids": [ + "sympy__sympy-24443" + ], + "infra_failure_ids": [], + "ambiguous_failure_ids": [ + "sympy__sympy-24443" + ], + "failure_reasons": { + "sympy__sympy-24443": "missing_module" + }, + "error_ids": [], + "schema_version": 2, + "unstopped_instances": 0, + "unstopped_containers": [], + "unremoved_images": [ + "swebench/sweb.eval.x86_64.sympy_1776_sympy-24443:latest" + ] +} diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-02/run.json b/logs/evaluation/arkx-p82-gold-docker-20260923-02/run.json new file mode 100644 index 00000000..276cfc13 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-02/run.json @@ -0,0 +1,6 @@ +{ + "dataset": "SWE-bench/SWE-bench_Verified", + "split": "test", + "task_repo": null, + "created_at": "2026-09-23T08:51:23+00:00" +} diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/eval.sh b/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/eval.sh new file mode 100644 index 00000000..04953b01 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/eval.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -uxo pipefail +source /opt/miniconda3/bin/activate +conda activate testbed +cd /testbed +git config --global --add safe.directory /testbed +cd /testbed +git status +git show +git -c core.fileMode=false diff c6753448b5c34f95e250105d76709fe4d349ca1f +source /opt/miniconda3/bin/activate +conda activate testbed +python -m pip install -e . +git checkout c6753448b5c34f95e250105d76709fe4d349ca1f sympy/physics/vector/tests/test_vector.py +git apply -v - <<'EOF_114329324912' +diff --git a/sympy/physics/vector/tests/test_vector.py b/sympy/physics/vector/tests/test_vector.py +--- a/sympy/physics/vector/tests/test_vector.py ++++ b/sympy/physics/vector/tests/test_vector.py +@@ -13,6 +13,8 @@ def test_Vector(): + assert A.y != A.z + assert A.z != A.x + ++ assert A.x + 0 == A.x ++ + v1 = x*A.x + y*A.y + z*A.z + v2 = x**2*A.x + y**2*A.y + z**2*A.z + v3 = v1 + v2 + +EOF_114329324912 +: '>>>>> Start Test Output' +PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose sympy/physics/vector/tests/test_vector.py +SWEBENCH_TEST_EXIT_CODE=$? +: '>>>>> End Test Output' +echo ">>>>> Test Exit Code: $SWEBENCH_TEST_EXIT_CODE" +git checkout c6753448b5c34f95e250105d76709fe4d349ca1f sympy/physics/vector/tests/test_vector.py diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/patch.diff b/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/patch.diff new file mode 100644 index 00000000..9119cf6a --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/patch.diff @@ -0,0 +1,12 @@ +diff --git a/sympy/physics/vector/vector.py b/sympy/physics/vector/vector.py +--- a/sympy/physics/vector/vector.py ++++ b/sympy/physics/vector/vector.py +@@ -57,6 +57,8 @@ def __hash__(self): + + def __add__(self, other): + """The add operator for Vector. """ ++ if other == 0: ++ return self + other = _check_vector(other) + return Vector(self.args + other.args) + diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/report.json b/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/report.json new file mode 100644 index 00000000..73b675b1 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/report.json @@ -0,0 +1,32 @@ +{ + "sympy__sympy-14711": { + "patch_is_None": false, + "patch_exists": true, + "patch_successfully_applied": true, + "resolved": true, + "infra_failure": false, + "tests_status": { + "FAIL_TO_PASS": { + "success": [ + "test_Vector" + ], + "failure": [] + }, + "PASS_TO_PASS": { + "success": [ + "test_Vector_diffs", + "test_vector_var_in_dcm" + ], + "failure": [] + }, + "FAIL_TO_FAIL": { + "success": [], + "failure": [] + }, + "PASS_TO_FAIL": { + "success": [], + "failure": [] + } + } + } +} \ No newline at end of file diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/run_instance.log b/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/run_instance.log new file mode 100644 index 00000000..4aae3913 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/run_instance.log @@ -0,0 +1,47 @@ +2026-09-23 05:41:35,743 - INFO - Creating container for sympy__sympy-14711... +2026-09-23 05:41:35,872 - INFO - Container for sympy__sympy-14711 created: eaea60ff36e3b175514a17192083e88f8cb914b6aa29e875ac64594d13e2e419 +2026-09-23 05:41:36,141 - INFO - Container for sympy__sympy-14711 started: eaea60ff36e3b175514a17192083e88f8cb914b6aa29e875ac64594d13e2e419 +2026-09-23 05:41:36,141 - INFO - Intermediate patch for sympy__sympy-14711 written to logs\evaluation\arkx-p82-gold-docker-20260923-03\gold\sympy__sympy-14711\patch.diff, now applying to container... +2026-09-23 05:41:36,275 - INFO - Failed to apply patch to container: git apply --verbose +2026-09-23 05:41:36,777 - INFO - Failed to apply patch to container: git apply --verbose --3way +2026-09-23 05:41:37,087 - INFO - Failed to apply patch to container: git apply --verbose --reject +2026-09-23 05:41:37,213 - INFO - >>>>> Applied Patch: +(Stripping trailing CRs from patch; use --binary to disable.) +patching file sympy/physics/vector/vector.py + +2026-09-23 05:41:37,272 - INFO - Git diff before: +diff --git a/sympy/physics/vector/vector.py b/sympy/physics/vector/vector.py +index a048ee57f..884911921 100644 +--- a/sympy/physics/vector/vector.py ++++ b/sympy/physics/vector/vector.py +@@ -57,6 +57,8 @@ def __hash__(self): + + def __add__(self, other): + """The add operator for Vector. """ ++ if other == 0: ++ return self + other = _check_vector(other) + return Vector(self.args + other.args) +2026-09-23 05:41:37,273 - INFO - Eval script for sympy__sympy-14711 written to logs\evaluation\arkx-p82-gold-docker-20260923-03\gold\sympy__sympy-14711\eval.sh; copying to container... +2026-09-23 05:41:43,106 - INFO - Test runtime: 5.77 seconds +2026-09-23 05:41:43,107 - INFO - Test output for sympy__sympy-14711 written to logs\evaluation\arkx-p82-gold-docker-20260923-03\gold\sympy__sympy-14711\test_output.txt +2026-09-23 05:41:43,161 - INFO - Git diff after: +diff --git a/sympy/physics/vector/vector.py b/sympy/physics/vector/vector.py +index a048ee57f..884911921 100644 +--- a/sympy/physics/vector/vector.py ++++ b/sympy/physics/vector/vector.py +@@ -57,6 +57,8 @@ def __hash__(self): + + def __add__(self, other): + """The add operator for Vector. """ ++ if other == 0: ++ return self + other = _check_vector(other) + return Vector(self.args + other.args) +2026-09-23 05:41:43,161 - INFO - Grading answer for sympy__sympy-14711... +2026-09-23 05:41:43,162 - INFO - report: {'sympy__sympy-14711': {'patch_is_None': False, 'patch_exists': True, 'patch_successfully_applied': True, 'resolved': True, 'infra_failure': False, 'tests_status': {'FAIL_TO_PASS': {'success': ['test_Vector'], 'failure': []}, 'PASS_TO_PASS': {'success': ['test_Vector_diffs', 'test_vector_var_in_dcm'], 'failure': []}, 'FAIL_TO_FAIL': {'success': [], 'failure': []}, 'PASS_TO_FAIL': {'success': [], 'failure': []}}}} +Result for sympy__sympy-14711: resolved: True +2026-09-23 05:41:43,162 - INFO - Attempting to stop container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-20260923-03... +2026-09-23 05:41:58,635 - INFO - Container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-20260923-03 stopped successfully. +2026-09-23 05:41:58,635 - INFO - Attempting to remove container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-20260923-03... +2026-09-23 05:41:58,905 - INFO - Container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-20260923-03 removed successfully. diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/test_output.txt b/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/test_output.txt new file mode 100644 index 00000000..736d5751 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-03/gold/sympy__sympy-14711/test_output.txt @@ -0,0 +1,345 @@ ++ source /opt/miniconda3/bin/activate +++ _CONDA_ROOT=/opt/miniconda3 +++ . /opt/miniconda3/etc/profile.d/conda.sh ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ '[' -z '' ']' ++++ export CONDA_SHLVL=0 ++++ CONDA_SHLVL=0 ++++ '[' -n '' ']' ++++++ dirname /opt/miniconda3/bin/conda +++++ dirname /opt/miniconda3/bin ++++ PATH=/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export PATH ++++ '[' -z '' ']' ++++ PS1= +++ conda activate +++ local cmd=activate +++ case "$cmd" in +++ __conda_activate activate +++ '[' -n '' ']' +++ local ask_conda ++++ PS1= ++++ __conda_exe shell.posix activate ++++ /opt/miniconda3/bin/conda shell.posix activate +++ ask_conda='PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''1'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ eval 'PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''1'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++++ PS1='(base) ' ++++ export PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export CONDA_PREFIX=/opt/miniconda3 ++++ CONDA_PREFIX=/opt/miniconda3 ++++ export CONDA_SHLVL=1 ++++ CONDA_SHLVL=1 ++++ export CONDA_DEFAULT_ENV=base ++++ CONDA_DEFAULT_ENV=base ++++ export 'CONDA_PROMPT_MODIFIER=(base) ' ++++ CONDA_PROMPT_MODIFIER='(base) ' ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ __conda_hashr +++ '[' -n '' ']' +++ '[' -n '' ']' +++ hash -r ++ conda activate testbed ++ local cmd=activate ++ case "$cmd" in ++ __conda_activate activate testbed ++ '[' -n '' ']' ++ local ask_conda +++ PS1='(base) ' +++ __conda_exe shell.posix activate testbed +++ /opt/miniconda3/bin/conda shell.posix activate testbed ++ ask_conda='PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''2'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_1='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++ eval 'PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''2'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_1='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ PS1='(testbed) ' +++ export PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ export CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ export CONDA_SHLVL=2 +++ CONDA_SHLVL=2 +++ export CONDA_DEFAULT_ENV=testbed +++ CONDA_DEFAULT_ENV=testbed +++ export 'CONDA_PROMPT_MODIFIER=(testbed) ' +++ CONDA_PROMPT_MODIFIER='(testbed) ' +++ export CONDA_PREFIX_1=/opt/miniconda3 +++ CONDA_PREFIX_1=/opt/miniconda3 +++ export CONDA_EXE=/opt/miniconda3/bin/conda +++ CONDA_EXE=/opt/miniconda3/bin/conda +++ export _CE_M= +++ _CE_M= +++ export _CE_CONDA= +++ _CE_CONDA= +++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++ __conda_hashr ++ '[' -n '' ']' ++ '[' -n '' ']' ++ hash -r ++ cd /testbed ++ git config --global --add safe.directory /testbed ++ cd /testbed ++ git status +On branch master +Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + modified: sympy/physics/vector/vector.py + +no changes added to commit (use "git add" and/or "git commit -a") ++ git show +commit a0bbae0b2e7de1110db52bdc844d0e15fe1458fb +Author: SWE-bench +Date: Thu Aug 13 03:57:29 2026 +0000 + + SWE-bench ++ git -c core.fileMode=false diff c6753448b5c34f95e250105d76709fe4d349ca1f +diff --git a/sympy/physics/vector/vector.py b/sympy/physics/vector/vector.py +index a048ee57f..884911921 100644 +--- a/sympy/physics/vector/vector.py ++++ b/sympy/physics/vector/vector.py +@@ -57,6 +57,8 @@ def __hash__(self): + + def __add__(self, other): + """The add operator for Vector. """ ++ if other == 0: ++ return self + other = _check_vector(other) + return Vector(self.args + other.args) + ++ source /opt/miniconda3/bin/activate +++ _CONDA_ROOT=/opt/miniconda3 +++ . /opt/miniconda3/etc/profile.d/conda.sh ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ '[' -z x ']' +++ conda activate +++ local cmd=activate +++ case "$cmd" in +++ __conda_activate activate +++ '[' -n '' ']' +++ local ask_conda ++++ PS1='(testbed) ' ++++ __conda_exe shell.posix activate ++++ /opt/miniconda3/bin/conda shell.posix activate +++ ask_conda='PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''3'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_PREFIX_2='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ eval 'PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''3'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_PREFIX_2='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++++ PS1='(base) ' ++++ export PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export CONDA_PREFIX=/opt/miniconda3 ++++ CONDA_PREFIX=/opt/miniconda3 ++++ export CONDA_SHLVL=3 ++++ CONDA_SHLVL=3 ++++ export CONDA_DEFAULT_ENV=base ++++ CONDA_DEFAULT_ENV=base ++++ export 'CONDA_PROMPT_MODIFIER=(base) ' ++++ CONDA_PROMPT_MODIFIER='(base) ' ++++ export CONDA_PREFIX_2=/opt/miniconda3/envs/testbed ++++ CONDA_PREFIX_2=/opt/miniconda3/envs/testbed ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ __conda_hashr +++ '[' -n '' ']' +++ '[' -n '' ']' +++ hash -r ++ conda activate testbed ++ local cmd=activate ++ case "$cmd" in ++ __conda_activate activate testbed ++ '[' -n '' ']' ++ local ask_conda +++ PS1='(base) ' +++ __conda_exe shell.posix activate testbed +++ /opt/miniconda3/bin/conda shell.posix activate testbed ++ ask_conda='PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''4'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_3='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++ eval 'PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''4'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_3='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ PS1='(testbed) ' +++ export PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ export CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ export CONDA_SHLVL=4 +++ CONDA_SHLVL=4 +++ export CONDA_DEFAULT_ENV=testbed +++ CONDA_DEFAULT_ENV=testbed +++ export 'CONDA_PROMPT_MODIFIER=(testbed) ' +++ CONDA_PROMPT_MODIFIER='(testbed) ' +++ export CONDA_PREFIX_3=/opt/miniconda3 +++ CONDA_PREFIX_3=/opt/miniconda3 +++ export CONDA_EXE=/opt/miniconda3/bin/conda +++ CONDA_EXE=/opt/miniconda3/bin/conda +++ export _CE_M= +++ _CE_M= +++ export _CE_CONDA= +++ _CE_CONDA= +++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++ __conda_hashr ++ '[' -n '' ']' ++ '[' -n '' ']' ++ hash -r ++ python -m pip install -e . +Obtaining file:///testbed + Preparing metadata (setup.py): started + Preparing metadata (setup.py): finished with status 'done' +Requirement already satisfied: mpmath>=0.19 in /opt/miniconda3/envs/testbed/lib/python3.9/site-packages (from sympy==1.1.2.dev0) (1.3.0) +Installing collected packages: sympy + Attempting uninstall: sympy + Found existing installation: sympy 1.1.2.dev0 + Uninstalling sympy-1.1.2.dev0: + Successfully uninstalled sympy-1.1.2.dev0 + DEPRECATION: Legacy editable install of sympy==1.1.2.dev0 from file:///testbed (setup.py develop) is deprecated. pip 25.0 will enforce this behaviour change. A possible replacement is to add a pyproject.toml or enable --use-pep517, and use setuptools >= 64. If the resulting installation is not behaving as expected, try using --config-settings editable_mode=compat. Please consult the setuptools documentation for more information. Discussion can be found at https://github.com/pypa/pip/issues/11457 + Running setup.py develop for sympy +Successfully installed sympy +WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning. ++ git checkout c6753448b5c34f95e250105d76709fe4d349ca1f sympy/physics/vector/tests/test_vector.py +Updated 0 paths from 0bd77345d ++ git apply -v - +Checking patch sympy/physics/vector/tests/test_vector.py... +Applied patch sympy/physics/vector/tests/test_vector.py cleanly. ++ : '>>>>> Start Test Output' ++ PYTHONWARNINGS=ignore::UserWarning,ignore::SyntaxWarning ++ bin/test -C --verbose sympy/physics/vector/tests/test_vector.py +/testbed/sympy/core/basic.py:3: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import Mapping, defaultdict +/testbed/sympy/core/expr.py:12: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import defaultdict, Iterable +/testbed/sympy/core/containers.py:271: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + class OrderedSet(collections.MutableSet): +/testbed/sympy/plotting/plot.py:28: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import Callable +/testbed/sympy/core/basic.py:3: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import Mapping, defaultdict +/testbed/sympy/core/expr.py:12: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import defaultdict, Iterable +/testbed/sympy/core/containers.py:271: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + class OrderedSet(collections.MutableSet): +/testbed/sympy/plotting/plot.py:28: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import Callable +============================= test process starts ============================== +executable: /opt/miniconda3/envs/testbed/bin/python (3.9.20-final-0) [CPython] +architecture: 64-bit +cache: no +ground types: python +numpy: None +random seed: 64776495 +hash randomization: on (PYTHONHASHSEED=1206030234) + +sympy/physics/vector/tests/test_vector.py[4] +test_Vector ok +test_Vector_diffs ok +test_vector_var_in_dcm ok +test_vector_simplify ok [OK] + + +================== tests finished: 4 passed, in 2.26 seconds =================== ++ SWEBENCH_TEST_EXIT_CODE=0 ++ : '>>>>> End Test Output' ++ echo '>>>>> Test Exit Code: 0' ++ git checkout c6753448b5c34f95e250105d76709fe4d349ca1f sympy/physics/vector/tests/test_vector.py +>>>>> Test Exit Code: 0 +Updated 1 path from 0bd77345d diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-03/results.json b/logs/evaluation/arkx-p82-gold-docker-20260923-03/results.json new file mode 100644 index 00000000..2da5b411 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-03/results.json @@ -0,0 +1,33 @@ +{ + "total_instances": 1, + "submitted_instances": 1, + "completed_instances": 1, + "resolved_instances": 1, + "unresolved_instances": 0, + "infra_failure_instances": 0, + "ambiguous_failure_instances": 0, + "empty_patch_instances": 0, + "error_instances": 0, + "completed_ids": [ + "sympy__sympy-14711" + ], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "sympy__sympy-14711" + ], + "resolved_ids": [ + "sympy__sympy-14711" + ], + "unresolved_ids": [], + "infra_failure_ids": [], + "ambiguous_failure_ids": [], + "failure_reasons": {}, + "error_ids": [], + "schema_version": 2, + "unstopped_instances": 0, + "unstopped_containers": [], + "unremoved_images": [ + "swebench/sweb.eval.x86_64.sympy_1776_sympy-14711:latest" + ] +} diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-03/run.json b/logs/evaluation/arkx-p82-gold-docker-20260923-03/run.json new file mode 100644 index 00000000..2928bf26 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-03/run.json @@ -0,0 +1,6 @@ +{ + "dataset": "SWE-bench/SWE-bench_Verified", + "split": "test", + "task_repo": null, + "created_at": "2026-09-23T08:41:27+00:00" +} diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/eval.sh b/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/eval.sh new file mode 100644 index 00000000..7ebf973e --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/eval.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -uxo pipefail +source /opt/miniconda3/bin/activate +conda activate testbed +cd /testbed +git config --global --add safe.directory /testbed +cd /testbed +git status +git show +git -c core.fileMode=false diff 809c53c077485ca48a206cee78340389cb83b7f1 +source /opt/miniconda3/bin/activate +conda activate testbed +python -m pip install -e . +git checkout 809c53c077485ca48a206cee78340389cb83b7f1 sympy/combinatorics/tests/test_homomorphisms.py +git apply -v - <<'EOF_114329324912' +diff --git a/sympy/combinatorics/tests/test_homomorphisms.py b/sympy/combinatorics/tests/test_homomorphisms.py +--- a/sympy/combinatorics/tests/test_homomorphisms.py ++++ b/sympy/combinatorics/tests/test_homomorphisms.py +@@ -57,6 +57,11 @@ def test_homomorphism(): + assert T.codomain == D + assert T(a*b) == p + ++ D3 = DihedralGroup(3) ++ T = homomorphism(D3, D3, D3.generators, D3.generators) ++ assert T.is_isomorphism() ++ ++ + def test_isomorphisms(): + + F, a, b = free_group("a, b") + +EOF_114329324912 +: '>>>>> Start Test Output' +PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose sympy/combinatorics/tests/test_homomorphisms.py +SWEBENCH_TEST_EXIT_CODE=$? +: '>>>>> End Test Output' +echo ">>>>> Test Exit Code: $SWEBENCH_TEST_EXIT_CODE" +git checkout 809c53c077485ca48a206cee78340389cb83b7f1 sympy/combinatorics/tests/test_homomorphisms.py diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/patch.diff b/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/patch.diff new file mode 100644 index 00000000..0e325652 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/patch.diff @@ -0,0 +1,68 @@ +diff --git a/sympy/combinatorics/homomorphisms.py b/sympy/combinatorics/homomorphisms.py +--- a/sympy/combinatorics/homomorphisms.py ++++ b/sympy/combinatorics/homomorphisms.py +@@ -308,42 +308,31 @@ def homomorphism(domain, codomain, gens, images=(), check=True): + return GroupHomomorphism(domain, codomain, images) + + def _check_homomorphism(domain, codomain, images): +- if hasattr(domain, 'relators'): +- rels = domain.relators +- else: +- gens = domain.presentation().generators +- rels = domain.presentation().relators ++ """ ++ Check that a given mapping of generators to images defines a homomorphism. ++ ++ Parameters ++ ========== ++ domain : PermutationGroup, FpGroup, FreeGroup ++ codomain : PermutationGroup, FpGroup, FreeGroup ++ images : dict ++ The set of keys must be equal to domain.generators. ++ The values must be elements of the codomain. ++ ++ """ ++ pres = domain if hasattr(domain, 'relators') else domain.presentation() ++ rels = pres.relators ++ gens = pres.generators ++ symbols = [g.ext_rep[0] for g in gens] ++ symbols_to_domain_generators = dict(zip(symbols, domain.generators)) + identity = codomain.identity + + def _image(r): +- if r.is_identity: +- return identity +- else: +- w = identity +- r_arr = r.array_form +- i = 0 +- j = 0 +- # i is the index for r and j is for +- # r_arr. r_arr[j] is the tuple (sym, p) +- # where sym is the generator symbol +- # and p is the power to which it is +- # raised while r[i] is a generator +- # (not just its symbol) or the inverse of +- # a generator - hence the need for +- # both indices +- while i < len(r): +- power = r_arr[j][1] +- if isinstance(domain, PermutationGroup) and r[i] in gens: +- s = domain.generators[gens.index(r[i])] +- else: +- s = r[i] +- if s in images: +- w = w*images[s]**power +- elif s**-1 in images: +- w = w*images[s**-1]**power +- i += abs(power) +- j += 1 +- return w ++ w = identity ++ for symbol, power in r.array_form: ++ g = symbols_to_domain_generators[symbol] ++ w *= images[g]**power ++ return w + + for r in rels: + if isinstance(codomain, FpGroup): diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/report.json b/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/report.json new file mode 100644 index 00000000..211a4f30 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/report.json @@ -0,0 +1,31 @@ +{ + "sympy__sympy-24443": { + "patch_is_None": false, + "patch_exists": true, + "patch_successfully_applied": true, + "resolved": true, + "infra_failure": false, + "tests_status": { + "FAIL_TO_PASS": { + "success": [ + "test_homomorphism" + ], + "failure": [] + }, + "PASS_TO_PASS": { + "success": [ + "test_isomorphisms" + ], + "failure": [] + }, + "FAIL_TO_FAIL": { + "success": [], + "failure": [] + }, + "PASS_TO_FAIL": { + "success": [], + "failure": [] + } + } + } +} \ No newline at end of file diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/run_instance.log b/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/run_instance.log new file mode 100644 index 00000000..855af93f --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/run_instance.log @@ -0,0 +1,161 @@ +2026-09-23 05:42:31,377 - INFO - Creating container for sympy__sympy-24443... +2026-09-23 05:42:31,502 - INFO - Container for sympy__sympy-24443 created: 331d9beacd2eede9ffbe2c72dfd31301fe886e702eb427cbe5832e97470bf2f2 +2026-09-23 05:42:31,697 - INFO - Container for sympy__sympy-24443 started: 331d9beacd2eede9ffbe2c72dfd31301fe886e702eb427cbe5832e97470bf2f2 +2026-09-23 05:42:31,698 - INFO - Intermediate patch for sympy__sympy-24443 written to logs\evaluation\arkx-p82-gold-docker-20260923-04\gold\sympy__sympy-24443\patch.diff, now applying to container... +2026-09-23 05:42:31,812 - INFO - Failed to apply patch to container: git apply --verbose +2026-09-23 05:42:32,345 - INFO - Failed to apply patch to container: git apply --verbose --3way +2026-09-23 05:42:32,602 - INFO - Failed to apply patch to container: git apply --verbose --reject +2026-09-23 05:42:32,850 - INFO - >>>>> Applied Patch: +(Stripping trailing CRs from patch; use --binary to disable.) +patching file sympy/combinatorics/homomorphisms.py + +2026-09-23 05:42:32,971 - INFO - Git diff before: +diff --git a/sympy/combinatorics/homomorphisms.py b/sympy/combinatorics/homomorphisms.py +index bb7bdf32ba..95b43c5013 100644 +--- a/sympy/combinatorics/homomorphisms.py ++++ b/sympy/combinatorics/homomorphisms.py +@@ -308,42 +308,31 @@ def homomorphism(domain, codomain, gens, images=(), check=True): + return GroupHomomorphism(domain, codomain, images) + + def _check_homomorphism(domain, codomain, images): +- if hasattr(domain, 'relators'): +- rels = domain.relators +- else: +- gens = domain.presentation().generators +- rels = domain.presentation().relators ++ """ ++ Check that a given mapping of generators to images defines a homomorphism. ++ ++ Parameters ++ ========== ++ domain : PermutationGroup, FpGroup, FreeGroup ++ codomain : PermutationGroup, FpGroup, FreeGroup ++ images : dict ++ The set of keys must be equal to domain.generators. ++ The values must be elements of the codomain. ++ ++ """ ++ pres = domain if hasattr(domain, 'relators') else domain.presentation() ++ rels = pres.relators ++ gens = pres.generators ++ symbols = [g.ext_rep[0] for g in gens] ++ symbols_to_domain_generators = dict(zip(symbols, domain.generators)) + identity = codomain.identity + + def _image(r): +- if r.is_identity: +- return identity +- else: +- w = identity +- r_arr = r.array_form +- i = 0 +- j = 0 +- # i is the index for r and j is for +- # r_arr. r_arr[j] is the tuple (sym, p) +- # where sym is the generator symbol +- # and p is the power to which it is +- # raised while r[i] is a generator +- # (not just its symbol) or the inverse of +- # a generator - hence the need for +- # both indices +- while i < len(r): +- power = r_arr[j][1] +- if isinstance(domain, PermutationGroup) and r[i] in gens: +- s = domain.generators[gens.index(r[i])] +- else: +- s = r[i] +- if s in images: +- w = w*images[s]**power +- elif s**-1 in images: +- w = w*images[s**-1]**power +- i += abs(power) +- j += 1 +- return w ++ w = identity ++ for symbol, power in r.array_form: ++ g = symbols_to_domain_generators[symbol] ++ w *= images[g]**power ++ return w + + for r in rels: + if isinstance(codomain, FpGroup): +2026-09-23 05:42:32,971 - INFO - Eval script for sympy__sympy-24443 written to logs\evaluation\arkx-p82-gold-docker-20260923-04\gold\sympy__sympy-24443\eval.sh; copying to container... +2026-09-23 05:42:36,816 - INFO - Test runtime: 3.78 seconds +2026-09-23 05:42:36,816 - INFO - Test output for sympy__sympy-24443 written to logs\evaluation\arkx-p82-gold-docker-20260923-04\gold\sympy__sympy-24443\test_output.txt +2026-09-23 05:42:36,871 - INFO - Git diff after: +diff --git a/sympy/combinatorics/homomorphisms.py b/sympy/combinatorics/homomorphisms.py +index bb7bdf32ba..95b43c5013 100644 +--- a/sympy/combinatorics/homomorphisms.py ++++ b/sympy/combinatorics/homomorphisms.py +@@ -308,42 +308,31 @@ def homomorphism(domain, codomain, gens, images=(), check=True): + return GroupHomomorphism(domain, codomain, images) + + def _check_homomorphism(domain, codomain, images): +- if hasattr(domain, 'relators'): +- rels = domain.relators +- else: +- gens = domain.presentation().generators +- rels = domain.presentation().relators ++ """ ++ Check that a given mapping of generators to images defines a homomorphism. ++ ++ Parameters ++ ========== ++ domain : PermutationGroup, FpGroup, FreeGroup ++ codomain : PermutationGroup, FpGroup, FreeGroup ++ images : dict ++ The set of keys must be equal to domain.generators. ++ The values must be elements of the codomain. ++ ++ """ ++ pres = domain if hasattr(domain, 'relators') else domain.presentation() ++ rels = pres.relators ++ gens = pres.generators ++ symbols = [g.ext_rep[0] for g in gens] ++ symbols_to_domain_generators = dict(zip(symbols, domain.generators)) + identity = codomain.identity + + def _image(r): +- if r.is_identity: +- return identity +- else: +- w = identity +- r_arr = r.array_form +- i = 0 +- j = 0 +- # i is the index for r and j is for +- # r_arr. r_arr[j] is the tuple (sym, p) +- # where sym is the generator symbol +- # and p is the power to which it is +- # raised while r[i] is a generator +- # (not just its symbol) or the inverse of +- # a generator - hence the need for +- # both indices +- while i < len(r): +- power = r_arr[j][1] +- if isinstance(domain, PermutationGroup) and r[i] in gens: +- s = domain.generators[gens.index(r[i])] +- else: +- s = r[i] +- if s in images: +- w = w*images[s]**power +- elif s**-1 in images: +- w = w*images[s**-1]**power +- i += abs(power) +- j += 1 +- return w ++ w = identity ++ for symbol, power in r.array_form: ++ g = symbols_to_domain_generators[symbol] ++ w *= images[g]**power ++ return w + + for r in rels: + if isinstance(codomain, FpGroup): +2026-09-23 05:42:36,871 - INFO - Grading answer for sympy__sympy-24443... +2026-09-23 05:42:36,872 - INFO - report: {'sympy__sympy-24443': {'patch_is_None': False, 'patch_exists': True, 'patch_successfully_applied': True, 'resolved': True, 'infra_failure': False, 'tests_status': {'FAIL_TO_PASS': {'success': ['test_homomorphism'], 'failure': []}, 'PASS_TO_PASS': {'success': ['test_isomorphisms'], 'failure': []}, 'FAIL_TO_FAIL': {'success': [], 'failure': []}, 'PASS_TO_FAIL': {'success': [], 'failure': []}}}} +Result for sympy__sympy-24443: resolved: True +2026-09-23 05:42:36,872 - INFO - Attempting to stop container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-20260923-04... +2026-09-23 05:42:52,408 - INFO - Container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-20260923-04 stopped successfully. +2026-09-23 05:42:52,408 - INFO - Attempting to remove container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-20260923-04... +2026-09-23 05:42:52,655 - INFO - Container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-20260923-04 removed successfully. diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/test_output.txt b/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/test_output.txt new file mode 100644 index 00000000..65aa0a17 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-04/gold/sympy__sympy-24443/test_output.txt @@ -0,0 +1,384 @@ ++ source /opt/miniconda3/bin/activate +++ _CONDA_ROOT=/opt/miniconda3 +++ . /opt/miniconda3/etc/profile.d/conda.sh ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ '[' -z '' ']' ++++ export CONDA_SHLVL=0 ++++ CONDA_SHLVL=0 ++++ '[' -n '' ']' ++++++ dirname /opt/miniconda3/bin/conda +++++ dirname /opt/miniconda3/bin ++++ PATH=/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export PATH ++++ '[' -z '' ']' ++++ PS1= +++ conda activate +++ local cmd=activate +++ case "$cmd" in +++ __conda_activate activate +++ '[' -n '' ']' +++ local ask_conda ++++ PS1= ++++ __conda_exe shell.posix activate ++++ /opt/miniconda3/bin/conda shell.posix activate +++ ask_conda='PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''1'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ eval 'PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''1'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++++ PS1='(base) ' ++++ export PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export CONDA_PREFIX=/opt/miniconda3 ++++ CONDA_PREFIX=/opt/miniconda3 ++++ export CONDA_SHLVL=1 ++++ CONDA_SHLVL=1 ++++ export CONDA_DEFAULT_ENV=base ++++ CONDA_DEFAULT_ENV=base ++++ export 'CONDA_PROMPT_MODIFIER=(base) ' ++++ CONDA_PROMPT_MODIFIER='(base) ' ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ __conda_hashr +++ '[' -n '' ']' +++ '[' -n '' ']' +++ hash -r ++ conda activate testbed ++ local cmd=activate ++ case "$cmd" in ++ __conda_activate activate testbed ++ '[' -n '' ']' ++ local ask_conda +++ PS1='(base) ' +++ __conda_exe shell.posix activate testbed +++ /opt/miniconda3/bin/conda shell.posix activate testbed ++ ask_conda='PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''2'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_1='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++ eval 'PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''2'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_1='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ PS1='(testbed) ' +++ export PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ export CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ export CONDA_SHLVL=2 +++ CONDA_SHLVL=2 +++ export CONDA_DEFAULT_ENV=testbed +++ CONDA_DEFAULT_ENV=testbed +++ export 'CONDA_PROMPT_MODIFIER=(testbed) ' +++ CONDA_PROMPT_MODIFIER='(testbed) ' +++ export CONDA_PREFIX_1=/opt/miniconda3 +++ CONDA_PREFIX_1=/opt/miniconda3 +++ export CONDA_EXE=/opt/miniconda3/bin/conda +++ CONDA_EXE=/opt/miniconda3/bin/conda +++ export _CE_M= +++ _CE_M= +++ export _CE_CONDA= +++ _CE_CONDA= +++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++ __conda_hashr ++ '[' -n '' ']' ++ '[' -n '' ']' ++ hash -r ++ cd /testbed ++ git config --global --add safe.directory /testbed ++ cd /testbed ++ git status +On branch master +Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + modified: sympy/combinatorics/homomorphisms.py + +no changes added to commit (use "git add" and/or "git commit -a") ++ git show +commit f40f22bd8193d8fb3ac4001021b21b08e948d46d +Author: SWE-bench +Date: Thu Aug 13 04:03:50 2026 +0000 + + SWE-bench ++ git -c core.fileMode=false diff 809c53c077485ca48a206cee78340389cb83b7f1 +diff --git a/sympy/combinatorics/homomorphisms.py b/sympy/combinatorics/homomorphisms.py +index bb7bdf32ba..95b43c5013 100644 +--- a/sympy/combinatorics/homomorphisms.py ++++ b/sympy/combinatorics/homomorphisms.py +@@ -308,42 +308,31 @@ def homomorphism(domain, codomain, gens, images=(), check=True): + return GroupHomomorphism(domain, codomain, images) + + def _check_homomorphism(domain, codomain, images): +- if hasattr(domain, 'relators'): +- rels = domain.relators +- else: +- gens = domain.presentation().generators +- rels = domain.presentation().relators ++ """ ++ Check that a given mapping of generators to images defines a homomorphism. ++ ++ Parameters ++ ========== ++ domain : PermutationGroup, FpGroup, FreeGroup ++ codomain : PermutationGroup, FpGroup, FreeGroup ++ images : dict ++ The set of keys must be equal to domain.generators. ++ The values must be elements of the codomain. ++ ++ """ ++ pres = domain if hasattr(domain, 'relators') else domain.presentation() ++ rels = pres.relators ++ gens = pres.generators ++ symbols = [g.ext_rep[0] for g in gens] ++ symbols_to_domain_generators = dict(zip(symbols, domain.generators)) + identity = codomain.identity + + def _image(r): +- if r.is_identity: +- return identity +- else: +- w = identity +- r_arr = r.array_form +- i = 0 +- j = 0 +- # i is the index for r and j is for +- # r_arr. r_arr[j] is the tuple (sym, p) +- # where sym is the generator symbol +- # and p is the power to which it is +- # raised while r[i] is a generator +- # (not just its symbol) or the inverse of +- # a generator - hence the need for +- # both indices +- while i < len(r): +- power = r_arr[j][1] +- if isinstance(domain, PermutationGroup) and r[i] in gens: +- s = domain.generators[gens.index(r[i])] +- else: +- s = r[i] +- if s in images: +- w = w*images[s]**power +- elif s**-1 in images: +- w = w*images[s**-1]**power +- i += abs(power) +- j += 1 +- return w ++ w = identity ++ for symbol, power in r.array_form: ++ g = symbols_to_domain_generators[symbol] ++ w *= images[g]**power ++ return w + + for r in rels: + if isinstance(codomain, FpGroup): ++ source /opt/miniconda3/bin/activate +++ _CONDA_ROOT=/opt/miniconda3 +++ . /opt/miniconda3/etc/profile.d/conda.sh ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ '[' -z x ']' +++ conda activate +++ local cmd=activate +++ case "$cmd" in +++ __conda_activate activate +++ '[' -n '' ']' +++ local ask_conda ++++ PS1='(testbed) ' ++++ __conda_exe shell.posix activate ++++ /opt/miniconda3/bin/conda shell.posix activate +++ ask_conda='PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''3'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_PREFIX_2='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ eval 'PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''3'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_PREFIX_2='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++++ PS1='(base) ' ++++ export PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export CONDA_PREFIX=/opt/miniconda3 ++++ CONDA_PREFIX=/opt/miniconda3 ++++ export CONDA_SHLVL=3 ++++ CONDA_SHLVL=3 ++++ export CONDA_DEFAULT_ENV=base ++++ CONDA_DEFAULT_ENV=base ++++ export 'CONDA_PROMPT_MODIFIER=(base) ' ++++ CONDA_PROMPT_MODIFIER='(base) ' ++++ export CONDA_PREFIX_2=/opt/miniconda3/envs/testbed ++++ CONDA_PREFIX_2=/opt/miniconda3/envs/testbed ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ __conda_hashr +++ '[' -n '' ']' +++ '[' -n '' ']' +++ hash -r ++ conda activate testbed ++ local cmd=activate ++ case "$cmd" in ++ __conda_activate activate testbed ++ '[' -n '' ']' ++ local ask_conda +++ PS1='(base) ' +++ __conda_exe shell.posix activate testbed +++ /opt/miniconda3/bin/conda shell.posix activate testbed ++ ask_conda='PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''4'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_3='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++ eval 'PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''4'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_3='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ PS1='(testbed) ' +++ export PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ export CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ export CONDA_SHLVL=4 +++ CONDA_SHLVL=4 +++ export CONDA_DEFAULT_ENV=testbed +++ CONDA_DEFAULT_ENV=testbed +++ export 'CONDA_PROMPT_MODIFIER=(testbed) ' +++ CONDA_PROMPT_MODIFIER='(testbed) ' +++ export CONDA_PREFIX_3=/opt/miniconda3 +++ CONDA_PREFIX_3=/opt/miniconda3 +++ export CONDA_EXE=/opt/miniconda3/bin/conda +++ CONDA_EXE=/opt/miniconda3/bin/conda +++ export _CE_M= +++ _CE_M= +++ export _CE_CONDA= +++ _CE_CONDA= +++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++ __conda_hashr ++ '[' -n '' ']' ++ '[' -n '' ']' ++ hash -r ++ python -m pip install -e . +Obtaining file:///testbed + Preparing metadata (setup.py): started + Preparing metadata (setup.py): finished with status 'done' +Requirement already satisfied: mpmath>=0.19 in /opt/miniconda3/envs/testbed/lib/python3.9/site-packages (from sympy==1.12.dev0) (1.3.0) +Installing collected packages: sympy + Attempting uninstall: sympy + Found existing installation: sympy 1.12.dev0 + Uninstalling sympy-1.12.dev0: + Successfully uninstalled sympy-1.12.dev0 + DEPRECATION: Legacy editable install of sympy==1.12.dev0 from file:///testbed (setup.py develop) is deprecated. pip 25.0 will enforce this behaviour change. A possible replacement is to add a pyproject.toml or enable --use-pep517, and use setuptools >= 64. If the resulting installation is not behaving as expected, try using --config-settings editable_mode=compat. Please consult the setuptools documentation for more information. Discussion can be found at https://github.com/pypa/pip/issues/11457 + Running setup.py develop for sympy +Successfully installed sympy +WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning. ++ git checkout 809c53c077485ca48a206cee78340389cb83b7f1 sympy/combinatorics/tests/test_homomorphisms.py +Updated 0 paths from 8595b5fff2 ++ git apply -v - +Checking patch sympy/combinatorics/tests/test_homomorphisms.py... +Applied patch sympy/combinatorics/tests/test_homomorphisms.py cleanly. ++ : '>>>>> Start Test Output' ++ PYTHONWARNINGS=ignore::UserWarning,ignore::SyntaxWarning ++ bin/test -C --verbose sympy/combinatorics/tests/test_homomorphisms.py +============================= test process starts ============================== +executable: /opt/miniconda3/envs/testbed/bin/python (3.9.20-final-0) [CPython] +architecture: 64-bit +cache: no +ground types: python +numpy: None +random seed: 42898202 +hash randomization: on (PYTHONHASHSEED=1021889497) + +sympy/combinatorics/tests/test_homomorphisms.py[3] +test_homomorphism ok +test_isomorphisms ok +test_check_homomorphism ok [OK] + + +================== tests finished: 3 passed, in 0.52 seconds =================== +>>>>> Test Exit Code: 0 ++ SWEBENCH_TEST_EXIT_CODE=0 ++ : '>>>>> End Test Output' ++ echo '>>>>> Test Exit Code: 0' ++ git checkout 809c53c077485ca48a206cee78340389cb83b7f1 sympy/combinatorics/tests/test_homomorphisms.py +Updated 1 path from 8595b5fff2 diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-04/results.json b/logs/evaluation/arkx-p82-gold-docker-20260923-04/results.json new file mode 100644 index 00000000..978348da --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-04/results.json @@ -0,0 +1,33 @@ +{ + "total_instances": 1, + "submitted_instances": 1, + "completed_instances": 1, + "resolved_instances": 1, + "unresolved_instances": 0, + "infra_failure_instances": 0, + "ambiguous_failure_instances": 0, + "empty_patch_instances": 0, + "error_instances": 0, + "completed_ids": [ + "sympy__sympy-24443" + ], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "sympy__sympy-24443" + ], + "resolved_ids": [ + "sympy__sympy-24443" + ], + "unresolved_ids": [], + "infra_failure_ids": [], + "ambiguous_failure_ids": [], + "failure_reasons": {}, + "error_ids": [], + "schema_version": 2, + "unstopped_instances": 0, + "unstopped_containers": [], + "unremoved_images": [ + "swebench/sweb.eval.x86_64.sympy_1776_sympy-24443:latest" + ] +} diff --git a/logs/evaluation/arkx-p82-gold-docker-20260923-04/run.json b/logs/evaluation/arkx-p82-gold-docker-20260923-04/run.json new file mode 100644 index 00000000..c188e617 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-20260923-04/run.json @@ -0,0 +1,6 @@ +{ + "dataset": "SWE-bench/SWE-bench_Verified", + "split": "test", + "task_repo": null, + "created_at": "2026-09-23T08:42:25+00:00" +} diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/eval.sh b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/eval.sh new file mode 100644 index 00000000..04953b01 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/eval.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -uxo pipefail +source /opt/miniconda3/bin/activate +conda activate testbed +cd /testbed +git config --global --add safe.directory /testbed +cd /testbed +git status +git show +git -c core.fileMode=false diff c6753448b5c34f95e250105d76709fe4d349ca1f +source /opt/miniconda3/bin/activate +conda activate testbed +python -m pip install -e . +git checkout c6753448b5c34f95e250105d76709fe4d349ca1f sympy/physics/vector/tests/test_vector.py +git apply -v - <<'EOF_114329324912' +diff --git a/sympy/physics/vector/tests/test_vector.py b/sympy/physics/vector/tests/test_vector.py +--- a/sympy/physics/vector/tests/test_vector.py ++++ b/sympy/physics/vector/tests/test_vector.py +@@ -13,6 +13,8 @@ def test_Vector(): + assert A.y != A.z + assert A.z != A.x + ++ assert A.x + 0 == A.x ++ + v1 = x*A.x + y*A.y + z*A.z + v2 = x**2*A.x + y**2*A.y + z**2*A.z + v3 = v1 + v2 + +EOF_114329324912 +: '>>>>> Start Test Output' +PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose sympy/physics/vector/tests/test_vector.py +SWEBENCH_TEST_EXIT_CODE=$? +: '>>>>> End Test Output' +echo ">>>>> Test Exit Code: $SWEBENCH_TEST_EXIT_CODE" +git checkout c6753448b5c34f95e250105d76709fe4d349ca1f sympy/physics/vector/tests/test_vector.py diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/patch.diff b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/patch.diff new file mode 100644 index 00000000..9119cf6a --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/patch.diff @@ -0,0 +1,12 @@ +diff --git a/sympy/physics/vector/vector.py b/sympy/physics/vector/vector.py +--- a/sympy/physics/vector/vector.py ++++ b/sympy/physics/vector/vector.py +@@ -57,6 +57,8 @@ def __hash__(self): + + def __add__(self, other): + """The add operator for Vector. """ ++ if other == 0: ++ return self + other = _check_vector(other) + return Vector(self.args + other.args) + diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/report.json b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/report.json new file mode 100644 index 00000000..73b675b1 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/report.json @@ -0,0 +1,32 @@ +{ + "sympy__sympy-14711": { + "patch_is_None": false, + "patch_exists": true, + "patch_successfully_applied": true, + "resolved": true, + "infra_failure": false, + "tests_status": { + "FAIL_TO_PASS": { + "success": [ + "test_Vector" + ], + "failure": [] + }, + "PASS_TO_PASS": { + "success": [ + "test_Vector_diffs", + "test_vector_var_in_dcm" + ], + "failure": [] + }, + "FAIL_TO_FAIL": { + "success": [], + "failure": [] + }, + "PASS_TO_FAIL": { + "success": [], + "failure": [] + } + } + } +} \ No newline at end of file diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/run_instance.log b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/run_instance.log new file mode 100644 index 00000000..cd3bfede --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/run_instance.log @@ -0,0 +1,47 @@ +2026-09-23 06:02:09,447 - INFO - Creating container for sympy__sympy-14711... +2026-09-23 06:02:09,592 - INFO - Container for sympy__sympy-14711 created: 52a5b1c72026e1d330131a0103ce7bcdb384f877c767142bbc7fd6e2953b9129 +2026-09-23 06:02:09,865 - INFO - Container for sympy__sympy-14711 started: 52a5b1c72026e1d330131a0103ce7bcdb384f877c767142bbc7fd6e2953b9129 +2026-09-23 06:02:09,866 - INFO - Intermediate patch for sympy__sympy-14711 written to logs\evaluation\arkx-p82-gold-docker-lf-20260923-01\gold\sympy__sympy-14711\patch.diff, now applying to container... +2026-09-23 06:02:10,034 - INFO - Failed to apply patch to container: git apply --verbose +2026-09-23 06:02:10,521 - INFO - Failed to apply patch to container: git apply --verbose --3way +2026-09-23 06:02:10,827 - INFO - Failed to apply patch to container: git apply --verbose --reject +2026-09-23 06:02:11,125 - INFO - >>>>> Applied Patch: +(Stripping trailing CRs from patch; use --binary to disable.) +patching file sympy/physics/vector/vector.py + +2026-09-23 06:02:11,179 - INFO - Git diff before: +diff --git a/sympy/physics/vector/vector.py b/sympy/physics/vector/vector.py +index a048ee57f..884911921 100644 +--- a/sympy/physics/vector/vector.py ++++ b/sympy/physics/vector/vector.py +@@ -57,6 +57,8 @@ def __hash__(self): + + def __add__(self, other): + """The add operator for Vector. """ ++ if other == 0: ++ return self + other = _check_vector(other) + return Vector(self.args + other.args) +2026-09-23 06:02:11,179 - INFO - Eval script for sympy__sympy-14711 written to logs\evaluation\arkx-p82-gold-docker-lf-20260923-01\gold\sympy__sympy-14711\eval.sh; copying to container... +2026-09-23 06:02:17,050 - INFO - Test runtime: 5.81 seconds +2026-09-23 06:02:17,052 - INFO - Test output for sympy__sympy-14711 written to logs\evaluation\arkx-p82-gold-docker-lf-20260923-01\gold\sympy__sympy-14711\test_output.txt +2026-09-23 06:02:17,104 - INFO - Git diff after: +diff --git a/sympy/physics/vector/vector.py b/sympy/physics/vector/vector.py +index a048ee57f..884911921 100644 +--- a/sympy/physics/vector/vector.py ++++ b/sympy/physics/vector/vector.py +@@ -57,6 +57,8 @@ def __hash__(self): + + def __add__(self, other): + """The add operator for Vector. """ ++ if other == 0: ++ return self + other = _check_vector(other) + return Vector(self.args + other.args) +2026-09-23 06:02:17,104 - INFO - Grading answer for sympy__sympy-14711... +2026-09-23 06:02:17,105 - INFO - report: {'sympy__sympy-14711': {'patch_is_None': False, 'patch_exists': True, 'patch_successfully_applied': True, 'resolved': True, 'infra_failure': False, 'tests_status': {'FAIL_TO_PASS': {'success': ['test_Vector'], 'failure': []}, 'PASS_TO_PASS': {'success': ['test_Vector_diffs', 'test_vector_var_in_dcm'], 'failure': []}, 'FAIL_TO_FAIL': {'success': [], 'failure': []}, 'PASS_TO_FAIL': {'success': [], 'failure': []}}}} +Result for sympy__sympy-14711: resolved: True +2026-09-23 06:02:17,105 - INFO - Attempting to stop container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-lf-20260923-01... +2026-09-23 06:02:32,501 - INFO - Container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-lf-20260923-01 stopped successfully. +2026-09-23 06:02:32,501 - INFO - Attempting to remove container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-lf-20260923-01... +2026-09-23 06:02:32,641 - INFO - Container sweb.eval.sympy__sympy-14711.arkx-p82-gold-docker-lf-20260923-01 removed successfully. diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/test_output.txt b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/test_output.txt new file mode 100644 index 00000000..13fe19e5 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/gold/sympy__sympy-14711/test_output.txt @@ -0,0 +1,345 @@ ++ source /opt/miniconda3/bin/activate +++ _CONDA_ROOT=/opt/miniconda3 +++ . /opt/miniconda3/etc/profile.d/conda.sh ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ '[' -z '' ']' ++++ export CONDA_SHLVL=0 ++++ CONDA_SHLVL=0 ++++ '[' -n '' ']' ++++++ dirname /opt/miniconda3/bin/conda +++++ dirname /opt/miniconda3/bin ++++ PATH=/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export PATH ++++ '[' -z '' ']' ++++ PS1= +++ conda activate +++ local cmd=activate +++ case "$cmd" in +++ __conda_activate activate +++ '[' -n '' ']' +++ local ask_conda ++++ PS1= ++++ __conda_exe shell.posix activate ++++ /opt/miniconda3/bin/conda shell.posix activate +++ ask_conda='PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''1'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ eval 'PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''1'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++++ PS1='(base) ' ++++ export PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export CONDA_PREFIX=/opt/miniconda3 ++++ CONDA_PREFIX=/opt/miniconda3 ++++ export CONDA_SHLVL=1 ++++ CONDA_SHLVL=1 ++++ export CONDA_DEFAULT_ENV=base ++++ CONDA_DEFAULT_ENV=base ++++ export 'CONDA_PROMPT_MODIFIER=(base) ' ++++ CONDA_PROMPT_MODIFIER='(base) ' ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ __conda_hashr +++ '[' -n '' ']' +++ '[' -n '' ']' +++ hash -r ++ conda activate testbed ++ local cmd=activate ++ case "$cmd" in ++ __conda_activate activate testbed ++ '[' -n '' ']' ++ local ask_conda +++ PS1='(base) ' +++ __conda_exe shell.posix activate testbed +++ /opt/miniconda3/bin/conda shell.posix activate testbed ++ ask_conda='PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''2'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_1='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++ eval 'PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''2'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_1='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ PS1='(testbed) ' +++ export PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ export CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ export CONDA_SHLVL=2 +++ CONDA_SHLVL=2 +++ export CONDA_DEFAULT_ENV=testbed +++ CONDA_DEFAULT_ENV=testbed +++ export 'CONDA_PROMPT_MODIFIER=(testbed) ' +++ CONDA_PROMPT_MODIFIER='(testbed) ' +++ export CONDA_PREFIX_1=/opt/miniconda3 +++ CONDA_PREFIX_1=/opt/miniconda3 +++ export CONDA_EXE=/opt/miniconda3/bin/conda +++ CONDA_EXE=/opt/miniconda3/bin/conda +++ export _CE_M= +++ _CE_M= +++ export _CE_CONDA= +++ _CE_CONDA= +++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++ __conda_hashr ++ '[' -n '' ']' ++ '[' -n '' ']' ++ hash -r ++ cd /testbed ++ git config --global --add safe.directory /testbed ++ cd /testbed ++ git status +On branch master +Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + modified: sympy/physics/vector/vector.py + +no changes added to commit (use "git add" and/or "git commit -a") ++ git show +commit a0bbae0b2e7de1110db52bdc844d0e15fe1458fb +Author: SWE-bench +Date: Thu Aug 13 03:57:29 2026 +0000 + + SWE-bench ++ git -c core.fileMode=false diff c6753448b5c34f95e250105d76709fe4d349ca1f +diff --git a/sympy/physics/vector/vector.py b/sympy/physics/vector/vector.py +index a048ee57f..884911921 100644 +--- a/sympy/physics/vector/vector.py ++++ b/sympy/physics/vector/vector.py +@@ -57,6 +57,8 @@ def __hash__(self): + + def __add__(self, other): + """The add operator for Vector. """ ++ if other == 0: ++ return self + other = _check_vector(other) + return Vector(self.args + other.args) + ++ source /opt/miniconda3/bin/activate +++ _CONDA_ROOT=/opt/miniconda3 +++ . /opt/miniconda3/etc/profile.d/conda.sh ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ '[' -z x ']' +++ conda activate +++ local cmd=activate +++ case "$cmd" in +++ __conda_activate activate +++ '[' -n '' ']' +++ local ask_conda ++++ PS1='(testbed) ' ++++ __conda_exe shell.posix activate ++++ /opt/miniconda3/bin/conda shell.posix activate +++ ask_conda='PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''3'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_PREFIX_2='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ eval 'PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''3'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_PREFIX_2='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++++ PS1='(base) ' ++++ export PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export CONDA_PREFIX=/opt/miniconda3 ++++ CONDA_PREFIX=/opt/miniconda3 ++++ export CONDA_SHLVL=3 ++++ CONDA_SHLVL=3 ++++ export CONDA_DEFAULT_ENV=base ++++ CONDA_DEFAULT_ENV=base ++++ export 'CONDA_PROMPT_MODIFIER=(base) ' ++++ CONDA_PROMPT_MODIFIER='(base) ' ++++ export CONDA_PREFIX_2=/opt/miniconda3/envs/testbed ++++ CONDA_PREFIX_2=/opt/miniconda3/envs/testbed ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ __conda_hashr +++ '[' -n '' ']' +++ '[' -n '' ']' +++ hash -r ++ conda activate testbed ++ local cmd=activate ++ case "$cmd" in ++ __conda_activate activate testbed ++ '[' -n '' ']' ++ local ask_conda +++ PS1='(base) ' +++ __conda_exe shell.posix activate testbed +++ /opt/miniconda3/bin/conda shell.posix activate testbed ++ ask_conda='PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''4'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_3='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++ eval 'PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''4'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_3='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ PS1='(testbed) ' +++ export PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ export CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ export CONDA_SHLVL=4 +++ CONDA_SHLVL=4 +++ export CONDA_DEFAULT_ENV=testbed +++ CONDA_DEFAULT_ENV=testbed +++ export 'CONDA_PROMPT_MODIFIER=(testbed) ' +++ CONDA_PROMPT_MODIFIER='(testbed) ' +++ export CONDA_PREFIX_3=/opt/miniconda3 +++ CONDA_PREFIX_3=/opt/miniconda3 +++ export CONDA_EXE=/opt/miniconda3/bin/conda +++ CONDA_EXE=/opt/miniconda3/bin/conda +++ export _CE_M= +++ _CE_M= +++ export _CE_CONDA= +++ _CE_CONDA= +++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++ __conda_hashr ++ '[' -n '' ']' ++ '[' -n '' ']' ++ hash -r ++ python -m pip install -e . +Obtaining file:///testbed + Preparing metadata (setup.py): started + Preparing metadata (setup.py): finished with status 'done' +Requirement already satisfied: mpmath>=0.19 in /opt/miniconda3/envs/testbed/lib/python3.9/site-packages (from sympy==1.1.2.dev0) (1.3.0) +Installing collected packages: sympy + Attempting uninstall: sympy + Found existing installation: sympy 1.1.2.dev0 + Uninstalling sympy-1.1.2.dev0: + Successfully uninstalled sympy-1.1.2.dev0 + DEPRECATION: Legacy editable install of sympy==1.1.2.dev0 from file:///testbed (setup.py develop) is deprecated. pip 25.0 will enforce this behaviour change. A possible replacement is to add a pyproject.toml or enable --use-pep517, and use setuptools >= 64. If the resulting installation is not behaving as expected, try using --config-settings editable_mode=compat. Please consult the setuptools documentation for more information. Discussion can be found at https://github.com/pypa/pip/issues/11457 + Running setup.py develop for sympy +Successfully installed sympy +WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning. ++ git checkout c6753448b5c34f95e250105d76709fe4d349ca1f sympy/physics/vector/tests/test_vector.py +Updated 0 paths from 0bd77345d ++ git apply -v - +Checking patch sympy/physics/vector/tests/test_vector.py... +Applied patch sympy/physics/vector/tests/test_vector.py cleanly. ++ : '>>>>> Start Test Output' ++ PYTHONWARNINGS=ignore::UserWarning,ignore::SyntaxWarning ++ bin/test -C --verbose sympy/physics/vector/tests/test_vector.py +/testbed/sympy/core/basic.py:3: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import Mapping, defaultdict +/testbed/sympy/core/expr.py:12: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import defaultdict, Iterable +/testbed/sympy/core/containers.py:271: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + class OrderedSet(collections.MutableSet): +/testbed/sympy/plotting/plot.py:28: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import Callable +/testbed/sympy/core/basic.py:3: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import Mapping, defaultdict +/testbed/sympy/core/expr.py:12: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import defaultdict, Iterable +/testbed/sympy/core/containers.py:271: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + class OrderedSet(collections.MutableSet): +/testbed/sympy/plotting/plot.py:28: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.10 it will stop working + from collections import Callable +============================= test process starts ============================== +executable: /opt/miniconda3/envs/testbed/bin/python (3.9.20-final-0) [CPython] +architecture: 64-bit +cache: no +ground types: python +numpy: None +random seed: 27028459 +hash randomization: on (PYTHONHASHSEED=731783106) + +sympy/physics/vector/tests/test_vector.py[4] +test_Vector ok +test_Vector_diffs ok +test_vector_var_in_dcm ok +test_vector_simplify ok [OK] + + +================== tests finished: 4 passed, in 2.43 seconds =================== +>>>>> Test Exit Code: 0 ++ SWEBENCH_TEST_EXIT_CODE=0 ++ : '>>>>> End Test Output' ++ echo '>>>>> Test Exit Code: 0' ++ git checkout c6753448b5c34f95e250105d76709fe4d349ca1f sympy/physics/vector/tests/test_vector.py +Updated 1 path from 0bd77345d diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/results.json b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/results.json new file mode 100644 index 00000000..2da5b411 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/results.json @@ -0,0 +1,33 @@ +{ + "total_instances": 1, + "submitted_instances": 1, + "completed_instances": 1, + "resolved_instances": 1, + "unresolved_instances": 0, + "infra_failure_instances": 0, + "ambiguous_failure_instances": 0, + "empty_patch_instances": 0, + "error_instances": 0, + "completed_ids": [ + "sympy__sympy-14711" + ], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "sympy__sympy-14711" + ], + "resolved_ids": [ + "sympy__sympy-14711" + ], + "unresolved_ids": [], + "infra_failure_ids": [], + "ambiguous_failure_ids": [], + "failure_reasons": {}, + "error_ids": [], + "schema_version": 2, + "unstopped_instances": 0, + "unstopped_containers": [], + "unremoved_images": [ + "swebench/sweb.eval.x86_64.sympy_1776_sympy-14711:latest" + ] +} diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/run.json b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/run.json new file mode 100644 index 00000000..61159d1a --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-01/run.json @@ -0,0 +1,6 @@ +{ + "dataset": "SWE-bench/SWE-bench_Verified", + "split": "test", + "task_repo": null, + "created_at": "2026-09-23T09:02:06+00:00" +} diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/eval.sh b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/eval.sh new file mode 100644 index 00000000..7ebf973e --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/eval.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -uxo pipefail +source /opt/miniconda3/bin/activate +conda activate testbed +cd /testbed +git config --global --add safe.directory /testbed +cd /testbed +git status +git show +git -c core.fileMode=false diff 809c53c077485ca48a206cee78340389cb83b7f1 +source /opt/miniconda3/bin/activate +conda activate testbed +python -m pip install -e . +git checkout 809c53c077485ca48a206cee78340389cb83b7f1 sympy/combinatorics/tests/test_homomorphisms.py +git apply -v - <<'EOF_114329324912' +diff --git a/sympy/combinatorics/tests/test_homomorphisms.py b/sympy/combinatorics/tests/test_homomorphisms.py +--- a/sympy/combinatorics/tests/test_homomorphisms.py ++++ b/sympy/combinatorics/tests/test_homomorphisms.py +@@ -57,6 +57,11 @@ def test_homomorphism(): + assert T.codomain == D + assert T(a*b) == p + ++ D3 = DihedralGroup(3) ++ T = homomorphism(D3, D3, D3.generators, D3.generators) ++ assert T.is_isomorphism() ++ ++ + def test_isomorphisms(): + + F, a, b = free_group("a, b") + +EOF_114329324912 +: '>>>>> Start Test Output' +PYTHONWARNINGS='ignore::UserWarning,ignore::SyntaxWarning' bin/test -C --verbose sympy/combinatorics/tests/test_homomorphisms.py +SWEBENCH_TEST_EXIT_CODE=$? +: '>>>>> End Test Output' +echo ">>>>> Test Exit Code: $SWEBENCH_TEST_EXIT_CODE" +git checkout 809c53c077485ca48a206cee78340389cb83b7f1 sympy/combinatorics/tests/test_homomorphisms.py diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/patch.diff b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/patch.diff new file mode 100644 index 00000000..0e325652 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/patch.diff @@ -0,0 +1,68 @@ +diff --git a/sympy/combinatorics/homomorphisms.py b/sympy/combinatorics/homomorphisms.py +--- a/sympy/combinatorics/homomorphisms.py ++++ b/sympy/combinatorics/homomorphisms.py +@@ -308,42 +308,31 @@ def homomorphism(domain, codomain, gens, images=(), check=True): + return GroupHomomorphism(domain, codomain, images) + + def _check_homomorphism(domain, codomain, images): +- if hasattr(domain, 'relators'): +- rels = domain.relators +- else: +- gens = domain.presentation().generators +- rels = domain.presentation().relators ++ """ ++ Check that a given mapping of generators to images defines a homomorphism. ++ ++ Parameters ++ ========== ++ domain : PermutationGroup, FpGroup, FreeGroup ++ codomain : PermutationGroup, FpGroup, FreeGroup ++ images : dict ++ The set of keys must be equal to domain.generators. ++ The values must be elements of the codomain. ++ ++ """ ++ pres = domain if hasattr(domain, 'relators') else domain.presentation() ++ rels = pres.relators ++ gens = pres.generators ++ symbols = [g.ext_rep[0] for g in gens] ++ symbols_to_domain_generators = dict(zip(symbols, domain.generators)) + identity = codomain.identity + + def _image(r): +- if r.is_identity: +- return identity +- else: +- w = identity +- r_arr = r.array_form +- i = 0 +- j = 0 +- # i is the index for r and j is for +- # r_arr. r_arr[j] is the tuple (sym, p) +- # where sym is the generator symbol +- # and p is the power to which it is +- # raised while r[i] is a generator +- # (not just its symbol) or the inverse of +- # a generator - hence the need for +- # both indices +- while i < len(r): +- power = r_arr[j][1] +- if isinstance(domain, PermutationGroup) and r[i] in gens: +- s = domain.generators[gens.index(r[i])] +- else: +- s = r[i] +- if s in images: +- w = w*images[s]**power +- elif s**-1 in images: +- w = w*images[s**-1]**power +- i += abs(power) +- j += 1 +- return w ++ w = identity ++ for symbol, power in r.array_form: ++ g = symbols_to_domain_generators[symbol] ++ w *= images[g]**power ++ return w + + for r in rels: + if isinstance(codomain, FpGroup): diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/report.json b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/report.json new file mode 100644 index 00000000..211a4f30 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/report.json @@ -0,0 +1,31 @@ +{ + "sympy__sympy-24443": { + "patch_is_None": false, + "patch_exists": true, + "patch_successfully_applied": true, + "resolved": true, + "infra_failure": false, + "tests_status": { + "FAIL_TO_PASS": { + "success": [ + "test_homomorphism" + ], + "failure": [] + }, + "PASS_TO_PASS": { + "success": [ + "test_isomorphisms" + ], + "failure": [] + }, + "FAIL_TO_FAIL": { + "success": [], + "failure": [] + }, + "PASS_TO_FAIL": { + "success": [], + "failure": [] + } + } + } +} \ No newline at end of file diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/run_instance.log b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/run_instance.log new file mode 100644 index 00000000..ee9c5d91 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/run_instance.log @@ -0,0 +1,161 @@ +2026-09-23 06:05:07,612 - INFO - Creating container for sympy__sympy-24443... +2026-09-23 06:05:07,747 - INFO - Container for sympy__sympy-24443 created: e0b71a23e7b7fbe22fd4addb64ba8c88495a8bfcc6e3726964245a93a1f9ce2f +2026-09-23 06:05:08,042 - INFO - Container for sympy__sympy-24443 started: e0b71a23e7b7fbe22fd4addb64ba8c88495a8bfcc6e3726964245a93a1f9ce2f +2026-09-23 06:05:08,042 - INFO - Intermediate patch for sympy__sympy-24443 written to logs\evaluation\arkx-p82-gold-docker-lf-20260923-02\gold\sympy__sympy-24443\patch.diff, now applying to container... +2026-09-23 06:05:08,196 - INFO - Failed to apply patch to container: git apply --verbose +2026-09-23 06:05:08,812 - INFO - Failed to apply patch to container: git apply --verbose --3way +2026-09-23 06:05:09,161 - INFO - Failed to apply patch to container: git apply --verbose --reject +2026-09-23 06:05:09,293 - INFO - >>>>> Applied Patch: +(Stripping trailing CRs from patch; use --binary to disable.) +patching file sympy/combinatorics/homomorphisms.py + +2026-09-23 06:05:09,353 - INFO - Git diff before: +diff --git a/sympy/combinatorics/homomorphisms.py b/sympy/combinatorics/homomorphisms.py +index bb7bdf32ba..95b43c5013 100644 +--- a/sympy/combinatorics/homomorphisms.py ++++ b/sympy/combinatorics/homomorphisms.py +@@ -308,42 +308,31 @@ def homomorphism(domain, codomain, gens, images=(), check=True): + return GroupHomomorphism(domain, codomain, images) + + def _check_homomorphism(domain, codomain, images): +- if hasattr(domain, 'relators'): +- rels = domain.relators +- else: +- gens = domain.presentation().generators +- rels = domain.presentation().relators ++ """ ++ Check that a given mapping of generators to images defines a homomorphism. ++ ++ Parameters ++ ========== ++ domain : PermutationGroup, FpGroup, FreeGroup ++ codomain : PermutationGroup, FpGroup, FreeGroup ++ images : dict ++ The set of keys must be equal to domain.generators. ++ The values must be elements of the codomain. ++ ++ """ ++ pres = domain if hasattr(domain, 'relators') else domain.presentation() ++ rels = pres.relators ++ gens = pres.generators ++ symbols = [g.ext_rep[0] for g in gens] ++ symbols_to_domain_generators = dict(zip(symbols, domain.generators)) + identity = codomain.identity + + def _image(r): +- if r.is_identity: +- return identity +- else: +- w = identity +- r_arr = r.array_form +- i = 0 +- j = 0 +- # i is the index for r and j is for +- # r_arr. r_arr[j] is the tuple (sym, p) +- # where sym is the generator symbol +- # and p is the power to which it is +- # raised while r[i] is a generator +- # (not just its symbol) or the inverse of +- # a generator - hence the need for +- # both indices +- while i < len(r): +- power = r_arr[j][1] +- if isinstance(domain, PermutationGroup) and r[i] in gens: +- s = domain.generators[gens.index(r[i])] +- else: +- s = r[i] +- if s in images: +- w = w*images[s]**power +- elif s**-1 in images: +- w = w*images[s**-1]**power +- i += abs(power) +- j += 1 +- return w ++ w = identity ++ for symbol, power in r.array_form: ++ g = symbols_to_domain_generators[symbol] ++ w *= images[g]**power ++ return w + + for r in rels: + if isinstance(codomain, FpGroup): +2026-09-23 06:05:09,354 - INFO - Eval script for sympy__sympy-24443 written to logs\evaluation\arkx-p82-gold-docker-lf-20260923-02\gold\sympy__sympy-24443\eval.sh; copying to container... +2026-09-23 06:05:15,615 - INFO - Test runtime: 6.19 seconds +2026-09-23 06:05:15,616 - INFO - Test output for sympy__sympy-24443 written to logs\evaluation\arkx-p82-gold-docker-lf-20260923-02\gold\sympy__sympy-24443\test_output.txt +2026-09-23 06:05:15,749 - INFO - Git diff after: +diff --git a/sympy/combinatorics/homomorphisms.py b/sympy/combinatorics/homomorphisms.py +index bb7bdf32ba..95b43c5013 100644 +--- a/sympy/combinatorics/homomorphisms.py ++++ b/sympy/combinatorics/homomorphisms.py +@@ -308,42 +308,31 @@ def homomorphism(domain, codomain, gens, images=(), check=True): + return GroupHomomorphism(domain, codomain, images) + + def _check_homomorphism(domain, codomain, images): +- if hasattr(domain, 'relators'): +- rels = domain.relators +- else: +- gens = domain.presentation().generators +- rels = domain.presentation().relators ++ """ ++ Check that a given mapping of generators to images defines a homomorphism. ++ ++ Parameters ++ ========== ++ domain : PermutationGroup, FpGroup, FreeGroup ++ codomain : PermutationGroup, FpGroup, FreeGroup ++ images : dict ++ The set of keys must be equal to domain.generators. ++ The values must be elements of the codomain. ++ ++ """ ++ pres = domain if hasattr(domain, 'relators') else domain.presentation() ++ rels = pres.relators ++ gens = pres.generators ++ symbols = [g.ext_rep[0] for g in gens] ++ symbols_to_domain_generators = dict(zip(symbols, domain.generators)) + identity = codomain.identity + + def _image(r): +- if r.is_identity: +- return identity +- else: +- w = identity +- r_arr = r.array_form +- i = 0 +- j = 0 +- # i is the index for r and j is for +- # r_arr. r_arr[j] is the tuple (sym, p) +- # where sym is the generator symbol +- # and p is the power to which it is +- # raised while r[i] is a generator +- # (not just its symbol) or the inverse of +- # a generator - hence the need for +- # both indices +- while i < len(r): +- power = r_arr[j][1] +- if isinstance(domain, PermutationGroup) and r[i] in gens: +- s = domain.generators[gens.index(r[i])] +- else: +- s = r[i] +- if s in images: +- w = w*images[s]**power +- elif s**-1 in images: +- w = w*images[s**-1]**power +- i += abs(power) +- j += 1 +- return w ++ w = identity ++ for symbol, power in r.array_form: ++ g = symbols_to_domain_generators[symbol] ++ w *= images[g]**power ++ return w + + for r in rels: + if isinstance(codomain, FpGroup): +2026-09-23 06:05:15,749 - INFO - Grading answer for sympy__sympy-24443... +2026-09-23 06:05:15,751 - INFO - report: {'sympy__sympy-24443': {'patch_is_None': False, 'patch_exists': True, 'patch_successfully_applied': True, 'resolved': True, 'infra_failure': False, 'tests_status': {'FAIL_TO_PASS': {'success': ['test_homomorphism'], 'failure': []}, 'PASS_TO_PASS': {'success': ['test_isomorphisms'], 'failure': []}, 'FAIL_TO_FAIL': {'success': [], 'failure': []}, 'PASS_TO_FAIL': {'success': [], 'failure': []}}}} +Result for sympy__sympy-24443: resolved: True +2026-09-23 06:05:15,751 - INFO - Attempting to stop container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-lf-20260923-02... +2026-09-23 06:05:31,335 - INFO - Container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-lf-20260923-02 stopped successfully. +2026-09-23 06:05:31,335 - INFO - Attempting to remove container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-lf-20260923-02... +2026-09-23 06:05:31,519 - INFO - Container sweb.eval.sympy__sympy-24443.arkx-p82-gold-docker-lf-20260923-02 removed successfully. diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/test_output.txt b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/test_output.txt new file mode 100644 index 00000000..45aceabc --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/gold/sympy__sympy-24443/test_output.txt @@ -0,0 +1,384 @@ ++ source /opt/miniconda3/bin/activate +++ _CONDA_ROOT=/opt/miniconda3 +++ . /opt/miniconda3/etc/profile.d/conda.sh ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ '[' -z '' ']' ++++ export CONDA_SHLVL=0 ++++ CONDA_SHLVL=0 ++++ '[' -n '' ']' ++++++ dirname /opt/miniconda3/bin/conda +++++ dirname /opt/miniconda3/bin ++++ PATH=/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export PATH ++++ '[' -z '' ']' ++++ PS1= +++ conda activate +++ local cmd=activate +++ case "$cmd" in +++ __conda_activate activate +++ '[' -n '' ']' +++ local ask_conda ++++ PS1= ++++ __conda_exe shell.posix activate ++++ /opt/miniconda3/bin/conda shell.posix activate +++ ask_conda='PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''1'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ eval 'PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''1'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++++ PS1='(base) ' ++++ export PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export CONDA_PREFIX=/opt/miniconda3 ++++ CONDA_PREFIX=/opt/miniconda3 ++++ export CONDA_SHLVL=1 ++++ CONDA_SHLVL=1 ++++ export CONDA_DEFAULT_ENV=base ++++ CONDA_DEFAULT_ENV=base ++++ export 'CONDA_PROMPT_MODIFIER=(base) ' ++++ CONDA_PROMPT_MODIFIER='(base) ' ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ __conda_hashr +++ '[' -n '' ']' +++ '[' -n '' ']' +++ hash -r ++ conda activate testbed ++ local cmd=activate ++ case "$cmd" in ++ __conda_activate activate testbed ++ '[' -n '' ']' ++ local ask_conda +++ PS1='(base) ' +++ __conda_exe shell.posix activate testbed +++ /opt/miniconda3/bin/conda shell.posix activate testbed ++ ask_conda='PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''2'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_1='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++ eval 'PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''2'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_1='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ PS1='(testbed) ' +++ export PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ export CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ export CONDA_SHLVL=2 +++ CONDA_SHLVL=2 +++ export CONDA_DEFAULT_ENV=testbed +++ CONDA_DEFAULT_ENV=testbed +++ export 'CONDA_PROMPT_MODIFIER=(testbed) ' +++ CONDA_PROMPT_MODIFIER='(testbed) ' +++ export CONDA_PREFIX_1=/opt/miniconda3 +++ CONDA_PREFIX_1=/opt/miniconda3 +++ export CONDA_EXE=/opt/miniconda3/bin/conda +++ CONDA_EXE=/opt/miniconda3/bin/conda +++ export _CE_M= +++ _CE_M= +++ export _CE_CONDA= +++ _CE_CONDA= +++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++ __conda_hashr ++ '[' -n '' ']' ++ '[' -n '' ']' ++ hash -r ++ cd /testbed ++ git config --global --add safe.directory /testbed ++ cd /testbed ++ git status +On branch master +Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + modified: sympy/combinatorics/homomorphisms.py + +no changes added to commit (use "git add" and/or "git commit -a") ++ git show +commit f40f22bd8193d8fb3ac4001021b21b08e948d46d +Author: SWE-bench +Date: Thu Aug 13 04:03:50 2026 +0000 + + SWE-bench ++ git -c core.fileMode=false diff 809c53c077485ca48a206cee78340389cb83b7f1 +diff --git a/sympy/combinatorics/homomorphisms.py b/sympy/combinatorics/homomorphisms.py +index bb7bdf32ba..95b43c5013 100644 +--- a/sympy/combinatorics/homomorphisms.py ++++ b/sympy/combinatorics/homomorphisms.py +@@ -308,42 +308,31 @@ def homomorphism(domain, codomain, gens, images=(), check=True): + return GroupHomomorphism(domain, codomain, images) + + def _check_homomorphism(domain, codomain, images): +- if hasattr(domain, 'relators'): +- rels = domain.relators +- else: +- gens = domain.presentation().generators +- rels = domain.presentation().relators ++ """ ++ Check that a given mapping of generators to images defines a homomorphism. ++ ++ Parameters ++ ========== ++ domain : PermutationGroup, FpGroup, FreeGroup ++ codomain : PermutationGroup, FpGroup, FreeGroup ++ images : dict ++ The set of keys must be equal to domain.generators. ++ The values must be elements of the codomain. ++ ++ """ ++ pres = domain if hasattr(domain, 'relators') else domain.presentation() ++ rels = pres.relators ++ gens = pres.generators ++ symbols = [g.ext_rep[0] for g in gens] ++ symbols_to_domain_generators = dict(zip(symbols, domain.generators)) + identity = codomain.identity + + def _image(r): +- if r.is_identity: +- return identity +- else: +- w = identity +- r_arr = r.array_form +- i = 0 +- j = 0 +- # i is the index for r and j is for +- # r_arr. r_arr[j] is the tuple (sym, p) +- # where sym is the generator symbol +- # and p is the power to which it is +- # raised while r[i] is a generator +- # (not just its symbol) or the inverse of +- # a generator - hence the need for +- # both indices +- while i < len(r): +- power = r_arr[j][1] +- if isinstance(domain, PermutationGroup) and r[i] in gens: +- s = domain.generators[gens.index(r[i])] +- else: +- s = r[i] +- if s in images: +- w = w*images[s]**power +- elif s**-1 in images: +- w = w*images[s**-1]**power +- i += abs(power) +- j += 1 +- return w ++ w = identity ++ for symbol, power in r.array_form: ++ g = symbols_to_domain_generators[symbol] ++ w *= images[g]**power ++ return w + + for r in rels: + if isinstance(codomain, FpGroup): ++ source /opt/miniconda3/bin/activate +++ _CONDA_ROOT=/opt/miniconda3 +++ . /opt/miniconda3/etc/profile.d/conda.sh ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ '[' -z x ']' +++ conda activate +++ local cmd=activate +++ case "$cmd" in +++ __conda_activate activate +++ '[' -n '' ']' +++ local ask_conda ++++ PS1='(testbed) ' ++++ __conda_exe shell.posix activate ++++ /opt/miniconda3/bin/conda shell.posix activate +++ ask_conda='PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''3'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_PREFIX_2='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ eval 'PS1='\''(base) '\'' +export PATH='\''/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3'\'' +export CONDA_SHLVL='\''3'\'' +export CONDA_DEFAULT_ENV='\''base'\'' +export CONDA_PROMPT_MODIFIER='\''(base) '\'' +export CONDA_PREFIX_2='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++++ PS1='(base) ' ++++ export PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ PATH=/opt/miniconda3/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ++++ export CONDA_PREFIX=/opt/miniconda3 ++++ CONDA_PREFIX=/opt/miniconda3 ++++ export CONDA_SHLVL=3 ++++ CONDA_SHLVL=3 ++++ export CONDA_DEFAULT_ENV=base ++++ CONDA_DEFAULT_ENV=base ++++ export 'CONDA_PROMPT_MODIFIER=(base) ' ++++ CONDA_PROMPT_MODIFIER='(base) ' ++++ export CONDA_PREFIX_2=/opt/miniconda3/envs/testbed ++++ CONDA_PREFIX_2=/opt/miniconda3/envs/testbed ++++ export CONDA_EXE=/opt/miniconda3/bin/conda ++++ CONDA_EXE=/opt/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ __conda_hashr +++ '[' -n '' ']' +++ '[' -n '' ']' +++ hash -r ++ conda activate testbed ++ local cmd=activate ++ case "$cmd" in ++ __conda_activate activate testbed ++ '[' -n '' ']' ++ local ask_conda +++ PS1='(base) ' +++ __conda_exe shell.posix activate testbed +++ /opt/miniconda3/bin/conda shell.posix activate testbed ++ ask_conda='PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''4'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_3='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' ++ eval 'PS1='\''(testbed) '\'' +export PATH='\''/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'\'' +export CONDA_PREFIX='\''/opt/miniconda3/envs/testbed'\'' +export CONDA_SHLVL='\''4'\'' +export CONDA_DEFAULT_ENV='\''testbed'\'' +export CONDA_PROMPT_MODIFIER='\''(testbed) '\'' +export CONDA_PREFIX_3='\''/opt/miniconda3'\'' +export CONDA_EXE='\''/opt/miniconda3/bin/conda'\'' +export _CE_M='\'''\'' +export _CE_CONDA='\'''\'' +export CONDA_PYTHON_EXE='\''/opt/miniconda3/bin/python'\''' +++ PS1='(testbed) ' +++ export PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ PATH=/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/condabin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ export CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ CONDA_PREFIX=/opt/miniconda3/envs/testbed +++ export CONDA_SHLVL=4 +++ CONDA_SHLVL=4 +++ export CONDA_DEFAULT_ENV=testbed +++ CONDA_DEFAULT_ENV=testbed +++ export 'CONDA_PROMPT_MODIFIER=(testbed) ' +++ CONDA_PROMPT_MODIFIER='(testbed) ' +++ export CONDA_PREFIX_3=/opt/miniconda3 +++ CONDA_PREFIX_3=/opt/miniconda3 +++ export CONDA_EXE=/opt/miniconda3/bin/conda +++ CONDA_EXE=/opt/miniconda3/bin/conda +++ export _CE_M= +++ _CE_M= +++ export _CE_CONDA= +++ _CE_CONDA= +++ export CONDA_PYTHON_EXE=/opt/miniconda3/bin/python +++ CONDA_PYTHON_EXE=/opt/miniconda3/bin/python ++ __conda_hashr ++ '[' -n '' ']' ++ '[' -n '' ']' ++ hash -r ++ python -m pip install -e . +Obtaining file:///testbed + Preparing metadata (setup.py): started + Preparing metadata (setup.py): finished with status 'done' +Requirement already satisfied: mpmath>=0.19 in /opt/miniconda3/envs/testbed/lib/python3.9/site-packages (from sympy==1.12.dev0) (1.3.0) +Installing collected packages: sympy + Attempting uninstall: sympy + Found existing installation: sympy 1.12.dev0 + Uninstalling sympy-1.12.dev0: + Successfully uninstalled sympy-1.12.dev0 + DEPRECATION: Legacy editable install of sympy==1.12.dev0 from file:///testbed (setup.py develop) is deprecated. pip 25.0 will enforce this behaviour change. A possible replacement is to add a pyproject.toml or enable --use-pep517, and use setuptools >= 64. If the resulting installation is not behaving as expected, try using --config-settings editable_mode=compat. Please consult the setuptools documentation for more information. Discussion can be found at https://github.com/pypa/pip/issues/11457 + Running setup.py develop for sympy +Successfully installed sympy +WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning. ++ git checkout 809c53c077485ca48a206cee78340389cb83b7f1 sympy/combinatorics/tests/test_homomorphisms.py +Updated 0 paths from 8595b5fff2 ++ git apply -v - +Checking patch sympy/combinatorics/tests/test_homomorphisms.py... +Applied patch sympy/combinatorics/tests/test_homomorphisms.py cleanly. ++ : '>>>>> Start Test Output' ++ PYTHONWARNINGS=ignore::UserWarning,ignore::SyntaxWarning ++ bin/test -C --verbose sympy/combinatorics/tests/test_homomorphisms.py +============================= test process starts ============================== +executable: /opt/miniconda3/envs/testbed/bin/python (3.9.20-final-0) [CPython] +architecture: 64-bit +cache: no +ground types: python +numpy: None +random seed: 96709696 +hash randomization: on (PYTHONHASHSEED=2141092767) + +sympy/combinatorics/tests/test_homomorphisms.py[3] +test_homomorphism ok +test_isomorphisms ok +test_check_homomorphism ok [OK] + + +================== tests finished: 3 passed, in 1.03 seconds =================== ++ SWEBENCH_TEST_EXIT_CODE=0 ++ : '>>>>> End Test Output' ++ echo '>>>>> Test Exit Code: 0' ++ git checkout 809c53c077485ca48a206cee78340389cb83b7f1 sympy/combinatorics/tests/test_homomorphisms.py +>>>>> Test Exit Code: 0 +Updated 1 path from 8595b5fff2 diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/results.json b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/results.json new file mode 100644 index 00000000..978348da --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/results.json @@ -0,0 +1,33 @@ +{ + "total_instances": 1, + "submitted_instances": 1, + "completed_instances": 1, + "resolved_instances": 1, + "unresolved_instances": 0, + "infra_failure_instances": 0, + "ambiguous_failure_instances": 0, + "empty_patch_instances": 0, + "error_instances": 0, + "completed_ids": [ + "sympy__sympy-24443" + ], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "sympy__sympy-24443" + ], + "resolved_ids": [ + "sympy__sympy-24443" + ], + "unresolved_ids": [], + "infra_failure_ids": [], + "ambiguous_failure_ids": [], + "failure_reasons": {}, + "error_ids": [], + "schema_version": 2, + "unstopped_instances": 0, + "unstopped_containers": [], + "unremoved_images": [ + "swebench/sweb.eval.x86_64.sympy_1776_sympy-24443:latest" + ] +} diff --git a/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/run.json b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/run.json new file mode 100644 index 00000000..a1e8bfa4 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-docker-lf-20260923-02/run.json @@ -0,0 +1,6 @@ +{ + "dataset": "SWE-bench/SWE-bench_Verified", + "split": "test", + "task_repo": null, + "created_at": "2026-09-23T09:05:05+00:00" +} diff --git a/logs/evaluation/arkx-p82-gold-qualification-20260923-01/run.json b/logs/evaluation/arkx-p82-gold-qualification-20260923-01/run.json new file mode 100644 index 00000000..6968466e --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-qualification-20260923-01/run.json @@ -0,0 +1,6 @@ +{ + "dataset": "SWE-bench/SWE-bench_Verified", + "split": "test", + "task_repo": null, + "created_at": "2026-09-23T02:47:15+00:00" +} diff --git a/logs/evaluation/arkx-p82-gold-qualification-20260923-02/run.json b/logs/evaluation/arkx-p82-gold-qualification-20260923-02/run.json new file mode 100644 index 00000000..a048599c --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-qualification-20260923-02/run.json @@ -0,0 +1,6 @@ +{ + "dataset": "SWE-bench/SWE-bench_Verified", + "split": "test", + "task_repo": null, + "created_at": "2026-09-23T03:11:56+00:00" +} diff --git a/logs/evaluation/arkx-p82-gold-qualification-20260923-03/results.json b/logs/evaluation/arkx-p82-gold-qualification-20260923-03/results.json new file mode 100644 index 00000000..2ccd5b06 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-qualification-20260923-03/results.json @@ -0,0 +1,26 @@ +{ + "total_instances": 1, + "submitted_instances": 1, + "completed_instances": 0, + "resolved_instances": 0, + "unresolved_instances": 0, + "infra_failure_instances": 0, + "ambiguous_failure_instances": 0, + "empty_patch_instances": 0, + "error_instances": 1, + "completed_ids": [], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "sympy__sympy-14711" + ], + "resolved_ids": [], + "unresolved_ids": [], + "infra_failure_ids": [], + "ambiguous_failure_ids": [], + "failure_reasons": {}, + "error_ids": [ + "sympy__sympy-14711" + ], + "schema_version": 2 +} diff --git a/logs/evaluation/arkx-p82-gold-qualification-20260923-03/run.json b/logs/evaluation/arkx-p82-gold-qualification-20260923-03/run.json new file mode 100644 index 00000000..9c0460a9 --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-qualification-20260923-03/run.json @@ -0,0 +1,6 @@ +{ + "dataset": "SWE-bench/SWE-bench_Verified", + "split": "test", + "task_repo": null, + "created_at": "2026-09-23T03:12:32+00:00" +} diff --git a/logs/evaluation/arkx-p82-gold-qualification-20260923-04/results.json b/logs/evaluation/arkx-p82-gold-qualification-20260923-04/results.json new file mode 100644 index 00000000..85e4f93f --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-qualification-20260923-04/results.json @@ -0,0 +1,26 @@ +{ + "total_instances": 1, + "submitted_instances": 1, + "completed_instances": 0, + "resolved_instances": 0, + "unresolved_instances": 0, + "infra_failure_instances": 0, + "ambiguous_failure_instances": 0, + "empty_patch_instances": 0, + "error_instances": 1, + "completed_ids": [], + "incomplete_ids": [], + "empty_patch_ids": [], + "submitted_ids": [ + "sympy__sympy-24443" + ], + "resolved_ids": [], + "unresolved_ids": [], + "infra_failure_ids": [], + "ambiguous_failure_ids": [], + "failure_reasons": {}, + "error_ids": [ + "sympy__sympy-24443" + ], + "schema_version": 2 +} diff --git a/logs/evaluation/arkx-p82-gold-qualification-20260923-04/run.json b/logs/evaluation/arkx-p82-gold-qualification-20260923-04/run.json new file mode 100644 index 00000000..bdab042f --- /dev/null +++ b/logs/evaluation/arkx-p82-gold-qualification-20260923-04/run.json @@ -0,0 +1,6 @@ +{ + "dataset": "SWE-bench/SWE-bench_Verified", + "split": "test", + "task_repo": null, + "created_at": "2026-09-23T03:13:53+00:00" +} diff --git a/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/diff.patch b/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/diff.patch new file mode 100644 index 00000000..5cc4cdda --- /dev/null +++ b/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/diff.patch @@ -0,0 +1,619 @@ +diff --git a/README.md b/README.md +index 13041af..c729c17 100644 +--- a/README.md ++++ b/README.md +@@ -1,26 +1,29 @@ + # Arkx + +-Arkx is a greenfield project for empirical, incremental development of reliable capability. ++Arkx is a repository for incremental implementation and empirical evaluation of software capability. + + > **MINIMUM SUFFICIENT ARCHITECTURE** + > **FOR MAXIMUM RELIABLE CAPABILITY** + +-This repository is the canonical root for the project. It intentionally contains only the foundation in this bootstrap: the project contract, experimental protocol, structural placeholders, and a minimal CI check. ++This repository is the canonical root for the project. It contains runtime ++contracts, an execution harness, the project contract, the experimental ++protocol, and structural checks. + +-## Scope of this bootstrap ++## Current scope + + Included: + + - explicit project invariants; +-- a small, evidence-oriented experimental protocol; +-- stable top-level boundaries for future work; +-- a deterministic structural check and minimal CI. ++- an evidence-oriented experimental protocol; ++- stable top-level boundaries; ++- a deterministic structural check, executable contracts, and a Treatment A ++ baseline harness. + +-Deferred: ++Still deferred or unqualified: + +-- routing and planning; + - multi-agent orchestration; +-- OpenHands, ReX, mini-SWE-agent, and SWE-agent integrations; ++- OpenHands, ReX, and SWE-agent integrations; ++- real provider/model adoption and the A/B/C/D mechanism trial; + - product dependencies or automatic migration from `smag-rex`. + + ## Development posture +@@ -44,9 +47,9 @@ P1 adds deterministic task characterization from explicit signals only. See [the + + P2 adds deterministic progress/stagnation assessment from explicit snapshots only. See [the P2 contract](docs/progress-stagnation.md). It does not control execution or authorize retry/replan. + +-P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke or select concrete executors. ++P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke concrete executors. + +-Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. None executes a plan, owns acceptance, or creates orchestration. ++Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. P8.2a adds a Treatment A baseline runner and official SWE-bench acceptance adapter; neither owns general orchestration or promotion. + + ## Checks + +diff --git a/docs/architecture.md b/docs/architecture.md +index 17bbba6..98d1e6a 100644 +--- a/docs/architecture.md ++++ b/docs/architecture.md +@@ -1,24 +1,75 @@ + # Architecture boundary + +-The bootstrap architecture is intentionally small: ++Arkx separates contracts, execution, verification, acceptance, and promotion. ++The executable contracts cover the request-to-acceptance path. Runtime ++capability is qualified per provider, model, executor, and treatment; the ++presence of a contract does not imply adoption. + + ```text + Arkx +-├── docs/ contracts, protocol, and decisions +-├── experiments/ experiment records and fixtures; no runtime yet +-├── src/ reserved product boundary; no implementation yet +-├── tests/ reserved verification boundary; no test suite yet +-└── tools/ dependency-free repository checks ++├── docs/ contracts, protocol, decisions, and evidence boundaries ++├── experiments/ experiment records and synthetic fixtures; not orchestration ++├── src/arkx/ executable contracts and bounded policy/runtime surfaces ++├── tests/ executable contract and behavior verification ++└── tools/ repository checks and narrow execution adapters + ``` + +-P0 adds a deliberately small executable core under `src/arkx/`: versioned contracts, evidence status resolution, telemetry collection, and a deterministic baseline fixture. P1 adds explicit-signal task characterization, P2 adds snapshot-based progress assessment, P3 adds bounded routing/escalation policy, and Block B adds independent P4/P5/P6 artifacts in the same boundary. No layer has a concrete executor interface or executor orchestration. ++The current executable boundary is layered as follows: ++ ++- P0: versioned contracts, evidence status resolution, telemetry collection, ++ and a deterministic baseline fixture; ++- P1: explicit-signal task characterization; ++- P2/P3: progress assessment, bounded routing, and evidence-based escalation; ++- P4/P5/P6: repository planning/state, patch verification, and handoff/context ++ accounting; ++- P7/P8.1/P8.2: composition records, executor qualification contracts, and ++ controlled treatment plans; ++- P8.2a: a narrow Treatment A baseline runner and the official SWE-bench ++ acceptance adapter; ++- current orchestration surfaces: request governance, treatment/executor ++ selection, sequential execution, recovery, independent acceptance, and ++ promotion gates. ++ ++The canonical flow is: ++ ++```text ++request ++ -> governance / authority ++ -> task characterization ++ -> treatment route ++ -> sufficient executor ++ -> progress / recovery ++ -> verification ++ -> independent acceptance ++ -> evidence for possible promotion ++``` ++ ++Routing selects the minimum sufficient treatment before selecting an executor. ++Simple, localized, and repository-wide paths are capabilities/regimes, not ++automatic executor cascades. A fallback, executor switch, scope expansion, ++acceptance, or promotion must remain explicit and evidence-backed. ++ ++The official SWE-bench Docker authority has been qualified on the frozen Gold ++tasks. That validates the acceptance path and infrastructure, not a real ++provider/model, Treatment A execution, or A/B/C/D mechanism evidence. + + ## Boundary rules + +-- `src/` is reserved for product code and is empty in this bootstrap. ++- `src/arkx/` contains product contracts and bounded policy/runtime code; it ++ must not silently grow into an unqualified general-purpose orchestrator. + - `experiments/` contains evidence-oriented records, not an implicit runtime. +-- `tools/` may validate repository contracts but must not become orchestration. +-- `tests/` contains P0 executable verification and remains the boundary for future tests. +-- Integrations and executors require an explicit decision record and experiment evidence. ++- `tools/` may validate contracts or host a narrow adapter, but must not become ++ an untracked orchestration layer. ++- `tests/` verifies contracts and behavior; passing tests are local evidence, ++ not scientific or upstream evidence. ++- Integrations, providers, models, and executors require explicit identity, ++ qualification evidence, and an acceptance authority. ++- Missing provider/model identity, missing evidence, and blocked execution are ++ preserved as blocked or unknown; they never become success. ++- The working tree may contain an implementation under qualification. Merge ++ readiness requires the corresponding decision record, tests, and evidence to ++ be committed together. + +-An empty boundary is intentional. Creating a directory does not imply that the corresponding capability exists. ++The boundary is explicit: creating a module, fixture, or directory does not by ++itself prove that the corresponding capability exists or is suitable for ++promotion. +diff --git a/docs/experimental-protocol.md b/docs/experimental-protocol.md +index f24bf94..2b0f92d 100644 +--- a/docs/experimental-protocol.md ++++ b/docs/experimental-protocol.md +@@ -1,6 +1,7 @@ + # Experimental protocol + +-Arkx develops through small, empirical increments. The protocol below is the minimum record required before a capability can be considered for promotion. ++Arkx develops through empirical increments. The protocol below defines the ++minimum record required before a capability can be considered for promotion. + + ## Required record + +@@ -14,6 +15,9 @@ Each experiment records: + - `procedure`: reproducible steps; + - `expected_signal`: measurable success and failure criteria; + - `result`: `PASS`, `FAIL`, `BLOCKED`, or `NOT_EXECUTED`; ++- `evidence`: links or paths to raw evidence and environment facts; ++- `acceptance`: explicit reviewer decision and rationale; ++- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. + + P7 composition trials use their own comparability namespace: `COMPARABLE`, `INCOMPARABLE`, and `UNKNOWN`. P7 records externally supplied measurements and does not convert them into acceptance results. + +@@ -26,9 +30,14 @@ environment, budget, verification, and independent acceptance authority. It + does not invoke an executor or mechanism and does not manufacture acceptance; + results remain P8.1 observations. Missing controls and measurements remain + `UNKNOWN`, and the only intended treatment difference is the declared arm. +-- `evidence`: links or paths to raw evidence and environment facts; +-- `acceptance`: explicit reviewer decision and rationale; +-- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. ++ ++P8.2a adds the Treatment A baseline runner. Each attempt is serial, refuses to ++overwrite an existing attempt directory, derives a stable run identity from ++the task and frozen execution configuration, and writes an `execution.json` ++artifact atomically alongside the log, diff, and trajectory. The artifact ++records execution only; verification, independent acceptance, and promotion ++remain separate records. A missing provider, model, mini version, or Python ++executable blocks the attempt instead of selecting a fallback. + + ## State separation + +diff --git a/docs/p82-minimal-mechanism-trial.md b/docs/p82-minimal-mechanism-trial.md +index b05ef93..43225e8 100644 +--- a/docs/p82-minimal-mechanism-trial.md ++++ b/docs/p82-minimal-mechanism-trial.md +@@ -34,8 +34,8 @@ promotion decision. + + ## P8.2a baseline execution path + +-P8.2a is infrastructure qualification, not mechanism evidence. The baseline +-runner accepts treatment A only, invokes mini-SWE-agent through its public ++P8.2a records infrastructure qualification, not mechanism evidence. The ++baseline runner accepts treatment A only, invokes mini-SWE-agent through its public + Python API in a headless worker, and records execution artifacts. It does not + run verification or decide acceptance. Verification and independent + acceptance are explicit, nullable records; missing provider/model identity is +@@ -49,7 +49,9 @@ sample is frozen in `experiments/p82-wave0-task-sample.json` from the official + + The official acceptance adapter is in `arkx.swebench_authority`. It creates a + SWE-bench prediction without gold patch or grading labels and invokes the +-official harness with a unique `run_id`. Gold qualification and real A runs +-remain blocked until `swebench` and Docker are available, the gold harness +-passes, and one exact provider/model/configuration is frozen. B/C/D and any ++official harness with a unique `run_id`. The frozen Docker Gold authority ++returned `resolved: true` for the two Wave 0 tasks recorded under ++`logs/evaluation/`; this qualifies the acceptance path and infrastructure only. ++Real A runs still ++require one exact provider/model/configuration to be frozen. B/C/D and any + mechanism comparison remain out of scope. +diff --git a/docs/project-contract.md b/docs/project-contract.md +index f2f734a..82054a4 100644 +--- a/docs/project-contract.md ++++ b/docs/project-contract.md +@@ -40,7 +40,9 @@ NO_SILENT_SCOPE_EXPANSION + 5. Any fallback, executor change, or scope change is explicit in the record and review. + 6. New architecture requires a decision record describing the observed need, alternatives, evidence, and rollback/removal condition. + +-## Non-goals of this bootstrap +- +-This repository does not yet define routing, planning, multi-agent execution, or integrations with OpenHands, ReX, mini-SWE-agent, or SWE-agent. ++## Current non-goals + ++The repository now defines bounded routing, planning/state, verification, ++recovery, acceptance, promotion gates, and a mini-SWE-agent Treatment A ++baseline path. It does not define general multi-agent execution, automatic ++executor adoption, or integrations with OpenHands, ReX, and SWE-agent. +diff --git a/experiments/README.md b/experiments/README.md +index 551f37e..fe93b83 100644 +--- a/experiments/README.md ++++ b/experiments/README.md +@@ -2,3 +2,4 @@ + + This directory is reserved for explicit experiment records and small fixtures. Use [the experimental protocol](../docs/experimental-protocol.md). No experiment has been executed by the bootstrap. + ++The exploratory local-model qualification wave is documented in [local model qualification](../docs/local-model-qualification.md). It was run outside the Arkx experiment harness and must not be read as executor adoption or acceptance evidence. +diff --git a/src/arkx/__init__.py b/src/arkx/__init__.py +index 39c7d6c..2508056 100644 +--- a/src/arkx/__init__.py ++++ b/src/arkx/__init__.py +@@ -1,6 +1,62 @@ + """Minimal, executor-agnostic Arkx execution contracts and telemetry.""" + + from .contracts import Event, EventType, ExecutionRecord, ExecutionStatus ++from .request import ( ++ ArkxRequest, ++ GovernanceDecision, ++ GovernancePolicy, ++ GovernanceReason, ++ GovernanceStatus, ++ RequestBudget, ++ RequestEnvironment, ++ authorize_request, ++ characterize_authorized_request, ++) ++from .selection import ( ++ ExecutorBinding, ++ ExecutorRegistry, ++ SelectionDecision, ++ SelectionReason, ++ SelectionStatus, ++ TreatmentCatalog, ++ TreatmentDefinition, ++ select_executor, ++ select_treatment, ++) ++from .p82_localization import ( ++ LocalizationArtifact, ++ LocalizationEvidenceBudget, ++ record_localization_artifact, ++) ++from .p82_editing import ( ++ EditOperation, ++ StructuredEditProposal, ++ record_edit_proposal, ++) ++from .orchestration import ( ++ ExecutionOutcome, ++ ExecutionResult, ++ OrchestrationReason, ++ OrchestrationResult, ++ OrchestrationStatus, ++ run_routed_pipeline, ++) ++from .recovery import ( ++ RecoveryAction, ++ RecoveryBudget, ++ RecoveryDecision, ++ RecoveryReason, ++ decide_recovery, ++) ++from .acceptance import AcceptancePolicy, decide_independent_acceptance ++from .promotion import ( ++ PromotionCandidate, ++ PromotionDecision, ++ PromotionPolicy, ++ PromotionReason, ++ PromotionStatus, ++ decide_promotion, ++) + from .characterization import ( + CharacterizationConfig, + Confidence, +@@ -108,6 +164,43 @@ __all__ = [ + "EventType", + "ExecutionRecord", + "ExecutionStatus", ++ "ArkxRequest", ++ "GovernanceDecision", ++ "GovernancePolicy", ++ "GovernanceReason", ++ "GovernanceStatus", ++ "RequestBudget", ++ "RequestEnvironment", ++ "authorize_request", ++ "characterize_authorized_request", ++ "ExecutorBinding", ++ "ExecutorRegistry", ++ "SelectionDecision", ++ "SelectionReason", ++ "SelectionStatus", ++ "TreatmentCatalog", ++ "TreatmentDefinition", ++ "select_executor", ++ "select_treatment", ++ "ExecutionOutcome", ++ "ExecutionResult", ++ "OrchestrationReason", ++ "OrchestrationResult", ++ "OrchestrationStatus", ++ "run_routed_pipeline", ++ "RecoveryAction", ++ "RecoveryBudget", ++ "RecoveryDecision", ++ "RecoveryReason", ++ "decide_recovery", ++ "AcceptancePolicy", ++ "decide_independent_acceptance", ++ "PromotionCandidate", ++ "PromotionDecision", ++ "PromotionPolicy", ++ "PromotionReason", ++ "PromotionStatus", ++ "decide_promotion", + "CharacterizationConfig", + "Confidence", + "RecommendedPath", +diff --git a/src/arkx/p82_baseline.py b/src/arkx/p82_baseline.py +index 9cd9d72..cd192fa 100644 +--- a/src/arkx/p82_baseline.py ++++ b/src/arkx/p82_baseline.py +@@ -138,6 +138,26 @@ class BaselineRunConfig: + def missing_identity(self) -> tuple[str, ...]: + return tuple(name for name in ("provider", "model", "mini_version", "python_executable") if getattr(self, name) is None) + ++ def identity_digest(self) -> str: ++ """Return a stable digest for the frozen execution configuration.""" ++ value = { ++ "provider": self.provider, ++ "model": self.model, ++ "model_version": self.model_version, ++ "temperature": self.temperature, ++ "reasoning_configuration": self.reasoning_configuration, ++ "model_configuration": self.model_configuration, ++ "max_tokens": self.max_tokens, ++ "max_cost": self.max_cost, ++ "wall_time_seconds": self.wall_time_seconds, ++ "mini_version": self.mini_version, ++ "python_executable": self.python_executable, ++ "runner_version": self.runner_version, ++ "treatment": self.treatment.value, ++ } ++ encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:16] ++ + + @dataclass(frozen=True) + class ExecutionArtifact: +@@ -155,16 +175,21 @@ class ExecutionArtifact: + log_path: str | None + trajectory_path: str | None + model_usage: dict[str, Any] | None ++ artifact_path: str | None = None + + def to_dict(self) -> dict[str, Any]: + value = {name: getattr(self, name) for name in ( + "run_id", "state", "task_id", "treatment", "attempt", "started_at", "finished_at", +- "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", "model_usage")} ++ "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", ++ "model_usage", "artifact_path")} + for name in ("state", "failure_category"): + if isinstance(value[name], Enum): + value[name] = value[name].value + return value + ++ def to_json(self) -> str: ++ return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ + + class HeadlessInvoker(Protocol): + def __call__(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: ... +@@ -185,7 +210,9 @@ class MiniSweAgentHeadlessRunner: + raise ValueError("attempt must be positive") + missing = config.missing_identity() + root = Path(config.artifact_root) / task.task_id / f"attempt-{attempt}" +- run_id = hashlib.sha256(f"{task.task_id}|A|{attempt}|{config.runner_version}".encode()).hexdigest()[:16] ++ run_id = hashlib.sha256( ++ f"{task.task_id}|{task.base_commit}|A|{attempt}|{config.identity_digest()}".encode("utf-8") ++ ).hexdigest()[:16] + if root.exists(): + return self._blocked(run_id, task, attempt, "artifact already exists; refusing silent overwrite") + root.mkdir(parents=True) +@@ -204,7 +231,16 @@ class MiniSweAgentHeadlessRunner: + state = BaselineExecutionState.COMPLETED if exit_code == 0 else BaselineExecutionState.EXECUTOR_FAILED + return self._finish(root, run_id, task, attempt, started, state, exit_code, None if exit_code == 0 else "executor returned non-zero", None if exit_code == 0 else RunnerFailureCategory.AGENT_RUN_FAILURE, str(diff_path), str(log_path), str(trajectory), usage) + except subprocess.TimeoutExpired as error: +- return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, str(error), RunnerFailureCategory.TIMEOUT, None, None, None, None) ++ log_path = root / "executor.log" ++ stdout = self._decode_process_output(error.stdout) ++ stderr = self._decode_process_output(error.stderr) ++ log_path.write_text(stdout + stderr, encoding="utf-8", newline="\n") ++ return self._finish( ++ root, run_id, task, attempt, started, ++ BaselineExecutionState.EXECUTOR_FAILED, None, str(error), ++ RunnerFailureCategory.TIMEOUT, None, str(log_path), ++ str(trajectory) if trajectory.exists() else None, None, ++ ) + except Exception as error: + message = f"{type(error).__name__}: {error}" + return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, message, self._classify_failure(message), None, None, None, None) +@@ -228,21 +264,47 @@ class MiniSweAgentHeadlessRunner: + return RunnerFailureCategory.DEADLOCK_HANG + return RunnerFailureCategory.AGENT_RUN_FAILURE + ++ @staticmethod ++ def _decode_process_output(value: str | bytes | None) -> str: ++ if value is None: ++ return "" ++ if isinstance(value, bytes): ++ return value.decode("utf-8", errors="replace") ++ return value ++ + def _subprocess_invoker(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: + worker = Path(__file__).parents[2] / "tools" / "run_mini_headless.py" + payload = {"task": task.problem_statement, "workspace": task.workspace, "model": config.model, + "trajectory": str(trajectory_path), "max_cost": config.max_cost, "wall_time_seconds": config.wall_time_seconds, + "model_configuration": config.model_configuration} +- completed = subprocess.run([config.python_executable, str(worker)], input=json.dumps(payload), text=True, +- capture_output=True, cwd=task.workspace, timeout=config.wall_time_seconds) +- return completed.returncode, completed.stdout + completed.stderr, None ++ timeout = None if config.wall_time_seconds is None else config.wall_time_seconds + 10 ++ completed = subprocess.run( ++ [config.python_executable, str(worker)], input=json.dumps(payload), text=True, ++ capture_output=True, cwd=task.workspace, timeout=timeout, ++ ) ++ combined_log = completed.stdout + completed.stderr ++ usage = None ++ if completed.returncode == 0: ++ lines = [line for line in completed.stdout.splitlines() if line.strip()] ++ if lines: ++ try: ++ worker_result = json.loads(lines[-1]) ++ usage = worker_result.get("usage") ++ except (json.JSONDecodeError, AttributeError): ++ usage = None ++ return completed.returncode, combined_log, usage + + def _blocked(self, run_id: str, task: ProspectiveTask, attempt: int, failure: str) -> ExecutionArtifact: + now = _now() + return ExecutionArtifact(run_id, BaselineExecutionState.BLOCKED, task.task_id, "A", attempt, now, now, None, failure, RunnerFailureCategory.UNKNOWN, None, None, None, None) + + def _finish(self, root: Path, run_id: str, task: ProspectiveTask, attempt: int, started: str, state: BaselineExecutionState, exit_code: int | None, failure: str | None, failure_category: RunnerFailureCategory | None, diff_path: str | None, log_path: str | None, trajectory_path: str | None, usage: dict[str, Any] | None) -> ExecutionArtifact: +- return ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage) ++ artifact_path = root / "execution.json" ++ artifact = ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage, str(artifact_path)) ++ temporary_path = root / "execution.json.tmp" ++ temporary_path.write_text(artifact.to_json() + "\n", encoding="utf-8", newline="\n") ++ temporary_path.replace(artifact_path) ++ return artifact + + + @dataclass(frozen=True) +diff --git a/tests/test_p82_baseline.py b/tests/test_p82_baseline.py +index d7d45ab..8ddf91f 100644 +--- a/tests/test_p82_baseline.py ++++ b/tests/test_p82_baseline.py +@@ -1,6 +1,9 @@ + import tempfile + import unittest ++import json ++import subprocess + from pathlib import Path ++from unittest.mock import patch + + from arkx.p82 import P82RunStatus, P82Treatment, TaskStratum + from arkx.p82_baseline import ( +@@ -40,6 +43,43 @@ def config(root, **overrides): + + + class P82BaselineTests(unittest.TestCase): ++ def test_subprocess_invoker_passes_limits_and_captures_usage(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ runner = MiniSweAgentHeadlessRunner() ++ completed = type("Completed", (), { ++ "returncode": 0, ++ "stdout": '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\n', ++ "stderr": "worker log\n", ++ })() ++ with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: ++ usage = runner._subprocess_invoker(task(workspace), config(directory), Path(directory) / "trajectory.json") ++ self.assertEqual(usage, (0, '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\nworker log\n', {"model_calls": 3, "cost": 0.25})) ++ self.assertEqual(mocked.call_args.kwargs["timeout"], 20) ++ payload = json.loads(mocked.call_args.kwargs["input"]) ++ self.assertEqual(payload["max_cost"], "1") ++ self.assertEqual(payload["wall_time_seconds"], 10) ++ ++ def test_timeout_preserves_partial_output_and_trajectory(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ artifact_root = Path(directory) / "artifacts" ++ trajectory = artifact_root / "task-1" / "attempt-1" / "trajectory.json" ++ ++ def timed_out_invoker(run_task, run_config, trajectory_path): ++ trajectory_path.write_text('{"partial":true}', encoding="utf-8") ++ raise subprocess.TimeoutExpired( ++ cmd="worker", timeout=20, output=b"partial stdout\n", stderr=b"partial stderr\n" ++ ) ++ ++ result = MiniSweAgentHeadlessRunner(timed_out_invoker).run(task(workspace), config(artifact_root)) ++ self.assertEqual(result.failure_category, RunnerFailureCategory.TIMEOUT) ++ self.assertTrue(Path(result.log_path).exists()) ++ self.assertEqual(Path(result.log_path).read_text(encoding="utf-8"), "partial stdout\npartial stderr\n") ++ self.assertEqual(Path(result.trajectory_path).read_text(encoding="utf-8"), '{"partial":true}') ++ + def test_headless_runner_uses_fake_invoker_and_captures_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" +@@ -56,8 +96,23 @@ class P82BaselineTests(unittest.TestCase): + self.assertEqual(calls, [("task-1", P82Treatment.A, "trajectory.json")]) + self.assertTrue(Path(result.log_path).exists()) + self.assertTrue(Path(result.diff_path).exists()) ++ self.assertTrue(Path(result.artifact_path).exists()) ++ persisted = json.loads(Path(result.artifact_path).read_text(encoding="utf-8")) ++ self.assertEqual(persisted, result.to_dict()) ++ self.assertEqual(result.model_usage, {"calls": 1}) + self.assertEqual(result.treatment, "A") + ++ def test_run_identity_changes_when_frozen_model_configuration_changes(self): ++ with tempfile.TemporaryDirectory() as directory: ++ first = config(directory, model="model-a") ++ second = config(directory, model="model-b") ++ self.assertNotEqual(first.identity_digest(), second.identity_digest()) ++ ++ runner = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)) ++ first_result = runner.run(task(directory), first, attempt=1) ++ second_result = runner.run(task(directory), second, attempt=2) ++ self.assertNotEqual(first_result.run_id, second_result.run_id) ++ + def test_missing_model_or_provider_is_blocked_without_fallback(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner().run(task(directory), config(directory, model=None)) +diff --git a/tools/run_mini_headless.py b/tools/run_mini_headless.py +index eedd60b..d501982 100644 +--- a/tools/run_mini_headless.py ++++ b/tools/run_mini_headless.py +@@ -22,17 +22,37 @@ def main() -> int: + from minisweagent.utils.serialize import recursive_merge, UNSET + + config = get_config_from_spec("mini.yaml") +- config = recursive_merge(config, { +- "run": {"task": payload["task"] or UNSET}, +- "agent": {"mode": "yolo", "confirm_exit": False, "output_path": Path(payload["trajectory"])}, +- "model": {"model_name": payload["model"], "model_class": payload.get("model_class", UNSET), **(payload.get("model_configuration") or payload.get("model_kwargs") or {})}, +- "environment": {"cwd": payload["workspace"]}, +- }) ++ config = recursive_merge( ++ config, ++ { ++ "run": {"task": payload["task"] or UNSET}, ++ "agent": { ++ "mode": "yolo", ++ "confirm_exit": False, ++ "output_path": Path(payload["trajectory"]), ++ "cost_limit": None if payload["max_cost"] is None else float(payload["max_cost"]), ++ "wall_time_limit_seconds": None if payload["wall_time_seconds"] is None else int(payload["wall_time_seconds"]), ++ }, ++ "model": { ++ "model_name": payload["model"], ++ "model_class": payload.get("model_class", UNSET), ++ **(payload.get("model_configuration") or payload.get("model_kwargs") or {}), ++ }, ++ "environment": {"cwd": payload["workspace"]}, ++ }, ++ ) + model = get_model(config=config["model"]) + env = get_environment(config["environment"], default_type="local") + agent = get_agent(model, env, config["agent"], default_type="default") + result = agent.run(payload["task"] or "") +- print(json.dumps(result, ensure_ascii=False, sort_keys=True)) ++ output = { ++ "result": result, ++ "usage": { ++ "model_calls": getattr(agent, "n_calls", None), ++ "cost": getattr(agent, "cost", None), ++ }, ++ } ++ print(json.dumps(output, ensure_ascii=False, sort_keys=True)) + return 0 + + diff --git a/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/execution.json b/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/execution.json new file mode 100644 index 00000000..165f3844 --- /dev/null +++ b/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-local-granite-01\\agent-preflight-local-granite-01\\attempt-1\\execution.json","attempt":1,"diff_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-local-granite-01\\agent-preflight-local-granite-01\\attempt-1\\diff.patch","exit_code":0,"failure":null,"failure_category":null,"finished_at":"2026-09-23T10:34:25.415+00:00","log_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-local-granite-01\\agent-preflight-local-granite-01\\attempt-1\\executor.log","model_usage":{"cost":0.0,"model_calls":3},"run_id":"44f6238c23a30a7a","started_at":"2026-09-23T10:33:25.946+00:00","state":"COMPLETED","task_id":"agent-preflight-local-granite-01","trajectory_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-local-granite-01\\agent-preflight-local-granite-01\\attempt-1\\trajectory.json","treatment":"A"} diff --git a/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/executor.log b/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/executor.log new file mode 100644 index 00000000..5938d622 --- /dev/null +++ b/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/executor.log @@ -0,0 +1 @@ +{"result": {"exit_status": "RepeatedFormatError", "submission": ""}, "usage": {"cost": 0.0, "model_calls": 3}} diff --git a/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/trajectory.json b/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/trajectory.json new file mode 100644 index 00000000..7f790e7d --- /dev/null +++ b/logs/p82-agent-preflight-local-granite-01/agent-preflight-local-granite-01/attempt-1/trajectory.json @@ -0,0 +1,173 @@ +{ + "info": { + "model_stats": { + "instance_cost": 0.0, + "api_calls": 3 + }, + "config": { + "agent": { + "system_template": "You are a helpful assistant that can interact with a computer.\n", + "instance_template": "Please solve this issue: {{task}}\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\n{{system}} {{release}} {{version}} {{machine}}\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:\n\n{%- if system == \"Darwin\" -%}\n\nYou are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.\n\n{%- endif -%}\n\n```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```\n", + "step_limit": 0, + "cost_limit": 0.0, + "wall_time_limit_seconds": 180, + "max_consecutive_format_errors": 3, + "output_path": "D:\\projetos\\arkx\\logs\\p82-agent-preflight-local-granite-01\\agent-preflight-local-granite-01\\attempt-1\\trajectory.json" + }, + "agent_type": "minisweagent.agents.default.DefaultAgent", + "model": { + "model_name": "ollama/granite3.3:8b", + "model_kwargs": { + "drop_params": true, + "temperature": 0.0, + "max_tokens": 256, + "api_base": "http://127.0.0.1:11434", + "custom_llm_provider": "ollama" + }, + "litellm_model_registry": null, + "set_cache_control": null, + "cost_tracking": "ignore_errors", + "format_error_template": "{% if finish_reason is defined and (finish_reason == \"length\" or (finish_reason == \"tool_calls\" and not has_tool_calls)) -%}\nYour previous response reached the output token limit (finish_reason={{ finish_reason }}) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.\n{%- else -%}\nTool call error:\n\n\n{{error}}\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.\n{%- endif %}\n", + "observation_template": "{%- if output.output | length < 10000 -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output\": {{ output.output | tojson }}\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- else -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output_head\": {{ output.output[:5000] | tojson }},\n \"output_tail\": {{ output.output[-5000:] | tojson }},\n \"elided_chars\": {{ output.output | length - 10000 }},\n \"warning\": \"Output too long.\"\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- endif -%}\n", + "multimodal_regex": "" + }, + "model_type": "minisweagent.models.litellm_model.LitellmModel", + "environment": { + "cwd": "D:\\projetos\\arkx\\.p82-local-preflight-granite-workspace", + "env": { + "PAGER": "cat", + "MANPAGER": "cat", + "LESS": "-R", + "PIP_PROGRESS_BAR": "off", + "TQDM_DISABLE": "1" + }, + "timeout": 30 + }, + "environment_type": "minisweagent.environments.local.LocalEnvironment" + }, + "mini_version": "2.4.6", + "exit_status": "RepeatedFormatError", + "submission": "" + }, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: Use the bash tool to create target.py containing exactly: def answer():\\n return 'ARKX_PREFLIGHT_OK'\\n. Then finish the task.\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nWindows 10 10.0.26200 AMD64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + }, + { + "role": "user", + "content": "Tool call error:\n\n\nNo tool calls found in the response. Every response MUST include at least one tool call.\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.", + "extra": { + "interrupt_type": "FormatError", + "cost": 0.0, + "response": { + "id": "chatcmpl-c17b29ba-e25d-4b4c-a6e3-d030d83ac3a7", + "created": 1790159632, + "model": "ollama/granite3.3:8b", + "object": "chat.completion", + "system_fingerprint": null, + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "{\"command\": \"echo 'def answer():\\n return \"}", + "role": "assistant", + "tool_calls": null, + "function_call": null + } + } + ], + "usage": { + "completion_tokens": 20, + "prompt_tokens": 960, + "total_tokens": 980, + "completion_tokens_details": null, + "prompt_tokens_details": null + } + } + } + }, + { + "role": "user", + "content": "Tool call error:\n\n\nNo tool calls found in the response. Every response MUST include at least one tool call.\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.", + "extra": { + "interrupt_type": "FormatError", + "cost": 0.0, + "response": { + "id": "chatcmpl-aa0cd981-1dbb-4a70-8038-237f465e3c67", + "created": 1790159650, + "model": "ollama/granite3.3:8b", + "object": "chat.completion", + "system_fingerprint": null, + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "{\"tool\": \"bash\", \"arguments\": {\"command\": \"echo 'def answer():\\n return \\\"ARKX_PREFLIGHT_OK\\\"' > target.py\"}}", + "role": "assistant", + "tool_calls": null, + "function_call": null + } + } + ], + "usage": { + "completion_tokens": 48, + "prompt_tokens": 1094, + "total_tokens": 1142, + "completion_tokens_details": null, + "prompt_tokens_details": null + } + } + } + }, + { + "role": "user", + "content": "Tool call error:\n\n\nNo tool calls found in the response. Every response MUST include at least one tool call.\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.", + "extra": { + "interrupt_type": "FormatError", + "cost": 0.0, + "response": { + "id": "chatcmpl-b9dd6cf4-d20e-4ea7-b2b6-86238f11a149", + "created": 1790159664, + "model": "ollama/granite3.3:8b", + "object": "chat.completion", + "system_fingerprint": null, + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "{\"tool\": \"bash\", \"arguments\": {\"command\": \"echo 'def answer():\\n return \"}}", + "role": "assistant", + "tool_calls": null, + "function_call": null + } + } + ], + "usage": { + "completion_tokens": 35, + "prompt_tokens": 1228, + "total_tokens": 1263, + "completion_tokens_details": null, + "prompt_tokens_details": null + } + } + } + }, + { + "role": "exit", + "content": "RepeatedFormatError", + "extra": { + "exit_status": "RepeatedFormatError", + "submission": "" + } + } + ], + "trajectory_format": "mini-swe-agent-1.1" +} \ No newline at end of file diff --git a/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/diff.patch b/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/diff.patch new file mode 100644 index 00000000..5cc4cdda --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/diff.patch @@ -0,0 +1,619 @@ +diff --git a/README.md b/README.md +index 13041af..c729c17 100644 +--- a/README.md ++++ b/README.md +@@ -1,26 +1,29 @@ + # Arkx + +-Arkx is a greenfield project for empirical, incremental development of reliable capability. ++Arkx is a repository for incremental implementation and empirical evaluation of software capability. + + > **MINIMUM SUFFICIENT ARCHITECTURE** + > **FOR MAXIMUM RELIABLE CAPABILITY** + +-This repository is the canonical root for the project. It intentionally contains only the foundation in this bootstrap: the project contract, experimental protocol, structural placeholders, and a minimal CI check. ++This repository is the canonical root for the project. It contains runtime ++contracts, an execution harness, the project contract, the experimental ++protocol, and structural checks. + +-## Scope of this bootstrap ++## Current scope + + Included: + + - explicit project invariants; +-- a small, evidence-oriented experimental protocol; +-- stable top-level boundaries for future work; +-- a deterministic structural check and minimal CI. ++- an evidence-oriented experimental protocol; ++- stable top-level boundaries; ++- a deterministic structural check, executable contracts, and a Treatment A ++ baseline harness. + +-Deferred: ++Still deferred or unqualified: + +-- routing and planning; + - multi-agent orchestration; +-- OpenHands, ReX, mini-SWE-agent, and SWE-agent integrations; ++- OpenHands, ReX, and SWE-agent integrations; ++- real provider/model adoption and the A/B/C/D mechanism trial; + - product dependencies or automatic migration from `smag-rex`. + + ## Development posture +@@ -44,9 +47,9 @@ P1 adds deterministic task characterization from explicit signals only. See [the + + P2 adds deterministic progress/stagnation assessment from explicit snapshots only. See [the P2 contract](docs/progress-stagnation.md). It does not control execution or authorize retry/replan. + +-P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke or select concrete executors. ++P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke concrete executors. + +-Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. None executes a plan, owns acceptance, or creates orchestration. ++Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. P8.2a adds a Treatment A baseline runner and official SWE-bench acceptance adapter; neither owns general orchestration or promotion. + + ## Checks + +diff --git a/docs/architecture.md b/docs/architecture.md +index 17bbba6..98d1e6a 100644 +--- a/docs/architecture.md ++++ b/docs/architecture.md +@@ -1,24 +1,75 @@ + # Architecture boundary + +-The bootstrap architecture is intentionally small: ++Arkx separates contracts, execution, verification, acceptance, and promotion. ++The executable contracts cover the request-to-acceptance path. Runtime ++capability is qualified per provider, model, executor, and treatment; the ++presence of a contract does not imply adoption. + + ```text + Arkx +-├── docs/ contracts, protocol, and decisions +-├── experiments/ experiment records and fixtures; no runtime yet +-├── src/ reserved product boundary; no implementation yet +-├── tests/ reserved verification boundary; no test suite yet +-└── tools/ dependency-free repository checks ++├── docs/ contracts, protocol, decisions, and evidence boundaries ++├── experiments/ experiment records and synthetic fixtures; not orchestration ++├── src/arkx/ executable contracts and bounded policy/runtime surfaces ++├── tests/ executable contract and behavior verification ++└── tools/ repository checks and narrow execution adapters + ``` + +-P0 adds a deliberately small executable core under `src/arkx/`: versioned contracts, evidence status resolution, telemetry collection, and a deterministic baseline fixture. P1 adds explicit-signal task characterization, P2 adds snapshot-based progress assessment, P3 adds bounded routing/escalation policy, and Block B adds independent P4/P5/P6 artifacts in the same boundary. No layer has a concrete executor interface or executor orchestration. ++The current executable boundary is layered as follows: ++ ++- P0: versioned contracts, evidence status resolution, telemetry collection, ++ and a deterministic baseline fixture; ++- P1: explicit-signal task characterization; ++- P2/P3: progress assessment, bounded routing, and evidence-based escalation; ++- P4/P5/P6: repository planning/state, patch verification, and handoff/context ++ accounting; ++- P7/P8.1/P8.2: composition records, executor qualification contracts, and ++ controlled treatment plans; ++- P8.2a: a narrow Treatment A baseline runner and the official SWE-bench ++ acceptance adapter; ++- current orchestration surfaces: request governance, treatment/executor ++ selection, sequential execution, recovery, independent acceptance, and ++ promotion gates. ++ ++The canonical flow is: ++ ++```text ++request ++ -> governance / authority ++ -> task characterization ++ -> treatment route ++ -> sufficient executor ++ -> progress / recovery ++ -> verification ++ -> independent acceptance ++ -> evidence for possible promotion ++``` ++ ++Routing selects the minimum sufficient treatment before selecting an executor. ++Simple, localized, and repository-wide paths are capabilities/regimes, not ++automatic executor cascades. A fallback, executor switch, scope expansion, ++acceptance, or promotion must remain explicit and evidence-backed. ++ ++The official SWE-bench Docker authority has been qualified on the frozen Gold ++tasks. That validates the acceptance path and infrastructure, not a real ++provider/model, Treatment A execution, or A/B/C/D mechanism evidence. + + ## Boundary rules + +-- `src/` is reserved for product code and is empty in this bootstrap. ++- `src/arkx/` contains product contracts and bounded policy/runtime code; it ++ must not silently grow into an unqualified general-purpose orchestrator. + - `experiments/` contains evidence-oriented records, not an implicit runtime. +-- `tools/` may validate repository contracts but must not become orchestration. +-- `tests/` contains P0 executable verification and remains the boundary for future tests. +-- Integrations and executors require an explicit decision record and experiment evidence. ++- `tools/` may validate contracts or host a narrow adapter, but must not become ++ an untracked orchestration layer. ++- `tests/` verifies contracts and behavior; passing tests are local evidence, ++ not scientific or upstream evidence. ++- Integrations, providers, models, and executors require explicit identity, ++ qualification evidence, and an acceptance authority. ++- Missing provider/model identity, missing evidence, and blocked execution are ++ preserved as blocked or unknown; they never become success. ++- The working tree may contain an implementation under qualification. Merge ++ readiness requires the corresponding decision record, tests, and evidence to ++ be committed together. + +-An empty boundary is intentional. Creating a directory does not imply that the corresponding capability exists. ++The boundary is explicit: creating a module, fixture, or directory does not by ++itself prove that the corresponding capability exists or is suitable for ++promotion. +diff --git a/docs/experimental-protocol.md b/docs/experimental-protocol.md +index f24bf94..2b0f92d 100644 +--- a/docs/experimental-protocol.md ++++ b/docs/experimental-protocol.md +@@ -1,6 +1,7 @@ + # Experimental protocol + +-Arkx develops through small, empirical increments. The protocol below is the minimum record required before a capability can be considered for promotion. ++Arkx develops through empirical increments. The protocol below defines the ++minimum record required before a capability can be considered for promotion. + + ## Required record + +@@ -14,6 +15,9 @@ Each experiment records: + - `procedure`: reproducible steps; + - `expected_signal`: measurable success and failure criteria; + - `result`: `PASS`, `FAIL`, `BLOCKED`, or `NOT_EXECUTED`; ++- `evidence`: links or paths to raw evidence and environment facts; ++- `acceptance`: explicit reviewer decision and rationale; ++- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. + + P7 composition trials use their own comparability namespace: `COMPARABLE`, `INCOMPARABLE`, and `UNKNOWN`. P7 records externally supplied measurements and does not convert them into acceptance results. + +@@ -26,9 +30,14 @@ environment, budget, verification, and independent acceptance authority. It + does not invoke an executor or mechanism and does not manufacture acceptance; + results remain P8.1 observations. Missing controls and measurements remain + `UNKNOWN`, and the only intended treatment difference is the declared arm. +-- `evidence`: links or paths to raw evidence and environment facts; +-- `acceptance`: explicit reviewer decision and rationale; +-- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. ++ ++P8.2a adds the Treatment A baseline runner. Each attempt is serial, refuses to ++overwrite an existing attempt directory, derives a stable run identity from ++the task and frozen execution configuration, and writes an `execution.json` ++artifact atomically alongside the log, diff, and trajectory. The artifact ++records execution only; verification, independent acceptance, and promotion ++remain separate records. A missing provider, model, mini version, or Python ++executable blocks the attempt instead of selecting a fallback. + + ## State separation + +diff --git a/docs/p82-minimal-mechanism-trial.md b/docs/p82-minimal-mechanism-trial.md +index b05ef93..43225e8 100644 +--- a/docs/p82-minimal-mechanism-trial.md ++++ b/docs/p82-minimal-mechanism-trial.md +@@ -34,8 +34,8 @@ promotion decision. + + ## P8.2a baseline execution path + +-P8.2a is infrastructure qualification, not mechanism evidence. The baseline +-runner accepts treatment A only, invokes mini-SWE-agent through its public ++P8.2a records infrastructure qualification, not mechanism evidence. The ++baseline runner accepts treatment A only, invokes mini-SWE-agent through its public + Python API in a headless worker, and records execution artifacts. It does not + run verification or decide acceptance. Verification and independent + acceptance are explicit, nullable records; missing provider/model identity is +@@ -49,7 +49,9 @@ sample is frozen in `experiments/p82-wave0-task-sample.json` from the official + + The official acceptance adapter is in `arkx.swebench_authority`. It creates a + SWE-bench prediction without gold patch or grading labels and invokes the +-official harness with a unique `run_id`. Gold qualification and real A runs +-remain blocked until `swebench` and Docker are available, the gold harness +-passes, and one exact provider/model/configuration is frozen. B/C/D and any ++official harness with a unique `run_id`. The frozen Docker Gold authority ++returned `resolved: true` for the two Wave 0 tasks recorded under ++`logs/evaluation/`; this qualifies the acceptance path and infrastructure only. ++Real A runs still ++require one exact provider/model/configuration to be frozen. B/C/D and any + mechanism comparison remain out of scope. +diff --git a/docs/project-contract.md b/docs/project-contract.md +index f2f734a..82054a4 100644 +--- a/docs/project-contract.md ++++ b/docs/project-contract.md +@@ -40,7 +40,9 @@ NO_SILENT_SCOPE_EXPANSION + 5. Any fallback, executor change, or scope change is explicit in the record and review. + 6. New architecture requires a decision record describing the observed need, alternatives, evidence, and rollback/removal condition. + +-## Non-goals of this bootstrap +- +-This repository does not yet define routing, planning, multi-agent execution, or integrations with OpenHands, ReX, mini-SWE-agent, or SWE-agent. ++## Current non-goals + ++The repository now defines bounded routing, planning/state, verification, ++recovery, acceptance, promotion gates, and a mini-SWE-agent Treatment A ++baseline path. It does not define general multi-agent execution, automatic ++executor adoption, or integrations with OpenHands, ReX, and SWE-agent. +diff --git a/experiments/README.md b/experiments/README.md +index 551f37e..fe93b83 100644 +--- a/experiments/README.md ++++ b/experiments/README.md +@@ -2,3 +2,4 @@ + + This directory is reserved for explicit experiment records and small fixtures. Use [the experimental protocol](../docs/experimental-protocol.md). No experiment has been executed by the bootstrap. + ++The exploratory local-model qualification wave is documented in [local model qualification](../docs/local-model-qualification.md). It was run outside the Arkx experiment harness and must not be read as executor adoption or acceptance evidence. +diff --git a/src/arkx/__init__.py b/src/arkx/__init__.py +index 39c7d6c..2508056 100644 +--- a/src/arkx/__init__.py ++++ b/src/arkx/__init__.py +@@ -1,6 +1,62 @@ + """Minimal, executor-agnostic Arkx execution contracts and telemetry.""" + + from .contracts import Event, EventType, ExecutionRecord, ExecutionStatus ++from .request import ( ++ ArkxRequest, ++ GovernanceDecision, ++ GovernancePolicy, ++ GovernanceReason, ++ GovernanceStatus, ++ RequestBudget, ++ RequestEnvironment, ++ authorize_request, ++ characterize_authorized_request, ++) ++from .selection import ( ++ ExecutorBinding, ++ ExecutorRegistry, ++ SelectionDecision, ++ SelectionReason, ++ SelectionStatus, ++ TreatmentCatalog, ++ TreatmentDefinition, ++ select_executor, ++ select_treatment, ++) ++from .p82_localization import ( ++ LocalizationArtifact, ++ LocalizationEvidenceBudget, ++ record_localization_artifact, ++) ++from .p82_editing import ( ++ EditOperation, ++ StructuredEditProposal, ++ record_edit_proposal, ++) ++from .orchestration import ( ++ ExecutionOutcome, ++ ExecutionResult, ++ OrchestrationReason, ++ OrchestrationResult, ++ OrchestrationStatus, ++ run_routed_pipeline, ++) ++from .recovery import ( ++ RecoveryAction, ++ RecoveryBudget, ++ RecoveryDecision, ++ RecoveryReason, ++ decide_recovery, ++) ++from .acceptance import AcceptancePolicy, decide_independent_acceptance ++from .promotion import ( ++ PromotionCandidate, ++ PromotionDecision, ++ PromotionPolicy, ++ PromotionReason, ++ PromotionStatus, ++ decide_promotion, ++) + from .characterization import ( + CharacterizationConfig, + Confidence, +@@ -108,6 +164,43 @@ __all__ = [ + "EventType", + "ExecutionRecord", + "ExecutionStatus", ++ "ArkxRequest", ++ "GovernanceDecision", ++ "GovernancePolicy", ++ "GovernanceReason", ++ "GovernanceStatus", ++ "RequestBudget", ++ "RequestEnvironment", ++ "authorize_request", ++ "characterize_authorized_request", ++ "ExecutorBinding", ++ "ExecutorRegistry", ++ "SelectionDecision", ++ "SelectionReason", ++ "SelectionStatus", ++ "TreatmentCatalog", ++ "TreatmentDefinition", ++ "select_executor", ++ "select_treatment", ++ "ExecutionOutcome", ++ "ExecutionResult", ++ "OrchestrationReason", ++ "OrchestrationResult", ++ "OrchestrationStatus", ++ "run_routed_pipeline", ++ "RecoveryAction", ++ "RecoveryBudget", ++ "RecoveryDecision", ++ "RecoveryReason", ++ "decide_recovery", ++ "AcceptancePolicy", ++ "decide_independent_acceptance", ++ "PromotionCandidate", ++ "PromotionDecision", ++ "PromotionPolicy", ++ "PromotionReason", ++ "PromotionStatus", ++ "decide_promotion", + "CharacterizationConfig", + "Confidence", + "RecommendedPath", +diff --git a/src/arkx/p82_baseline.py b/src/arkx/p82_baseline.py +index 9cd9d72..cd192fa 100644 +--- a/src/arkx/p82_baseline.py ++++ b/src/arkx/p82_baseline.py +@@ -138,6 +138,26 @@ class BaselineRunConfig: + def missing_identity(self) -> tuple[str, ...]: + return tuple(name for name in ("provider", "model", "mini_version", "python_executable") if getattr(self, name) is None) + ++ def identity_digest(self) -> str: ++ """Return a stable digest for the frozen execution configuration.""" ++ value = { ++ "provider": self.provider, ++ "model": self.model, ++ "model_version": self.model_version, ++ "temperature": self.temperature, ++ "reasoning_configuration": self.reasoning_configuration, ++ "model_configuration": self.model_configuration, ++ "max_tokens": self.max_tokens, ++ "max_cost": self.max_cost, ++ "wall_time_seconds": self.wall_time_seconds, ++ "mini_version": self.mini_version, ++ "python_executable": self.python_executable, ++ "runner_version": self.runner_version, ++ "treatment": self.treatment.value, ++ } ++ encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:16] ++ + + @dataclass(frozen=True) + class ExecutionArtifact: +@@ -155,16 +175,21 @@ class ExecutionArtifact: + log_path: str | None + trajectory_path: str | None + model_usage: dict[str, Any] | None ++ artifact_path: str | None = None + + def to_dict(self) -> dict[str, Any]: + value = {name: getattr(self, name) for name in ( + "run_id", "state", "task_id", "treatment", "attempt", "started_at", "finished_at", +- "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", "model_usage")} ++ "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", ++ "model_usage", "artifact_path")} + for name in ("state", "failure_category"): + if isinstance(value[name], Enum): + value[name] = value[name].value + return value + ++ def to_json(self) -> str: ++ return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ + + class HeadlessInvoker(Protocol): + def __call__(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: ... +@@ -185,7 +210,9 @@ class MiniSweAgentHeadlessRunner: + raise ValueError("attempt must be positive") + missing = config.missing_identity() + root = Path(config.artifact_root) / task.task_id / f"attempt-{attempt}" +- run_id = hashlib.sha256(f"{task.task_id}|A|{attempt}|{config.runner_version}".encode()).hexdigest()[:16] ++ run_id = hashlib.sha256( ++ f"{task.task_id}|{task.base_commit}|A|{attempt}|{config.identity_digest()}".encode("utf-8") ++ ).hexdigest()[:16] + if root.exists(): + return self._blocked(run_id, task, attempt, "artifact already exists; refusing silent overwrite") + root.mkdir(parents=True) +@@ -204,7 +231,16 @@ class MiniSweAgentHeadlessRunner: + state = BaselineExecutionState.COMPLETED if exit_code == 0 else BaselineExecutionState.EXECUTOR_FAILED + return self._finish(root, run_id, task, attempt, started, state, exit_code, None if exit_code == 0 else "executor returned non-zero", None if exit_code == 0 else RunnerFailureCategory.AGENT_RUN_FAILURE, str(diff_path), str(log_path), str(trajectory), usage) + except subprocess.TimeoutExpired as error: +- return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, str(error), RunnerFailureCategory.TIMEOUT, None, None, None, None) ++ log_path = root / "executor.log" ++ stdout = self._decode_process_output(error.stdout) ++ stderr = self._decode_process_output(error.stderr) ++ log_path.write_text(stdout + stderr, encoding="utf-8", newline="\n") ++ return self._finish( ++ root, run_id, task, attempt, started, ++ BaselineExecutionState.EXECUTOR_FAILED, None, str(error), ++ RunnerFailureCategory.TIMEOUT, None, str(log_path), ++ str(trajectory) if trajectory.exists() else None, None, ++ ) + except Exception as error: + message = f"{type(error).__name__}: {error}" + return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, message, self._classify_failure(message), None, None, None, None) +@@ -228,21 +264,47 @@ class MiniSweAgentHeadlessRunner: + return RunnerFailureCategory.DEADLOCK_HANG + return RunnerFailureCategory.AGENT_RUN_FAILURE + ++ @staticmethod ++ def _decode_process_output(value: str | bytes | None) -> str: ++ if value is None: ++ return "" ++ if isinstance(value, bytes): ++ return value.decode("utf-8", errors="replace") ++ return value ++ + def _subprocess_invoker(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: + worker = Path(__file__).parents[2] / "tools" / "run_mini_headless.py" + payload = {"task": task.problem_statement, "workspace": task.workspace, "model": config.model, + "trajectory": str(trajectory_path), "max_cost": config.max_cost, "wall_time_seconds": config.wall_time_seconds, + "model_configuration": config.model_configuration} +- completed = subprocess.run([config.python_executable, str(worker)], input=json.dumps(payload), text=True, +- capture_output=True, cwd=task.workspace, timeout=config.wall_time_seconds) +- return completed.returncode, completed.stdout + completed.stderr, None ++ timeout = None if config.wall_time_seconds is None else config.wall_time_seconds + 10 ++ completed = subprocess.run( ++ [config.python_executable, str(worker)], input=json.dumps(payload), text=True, ++ capture_output=True, cwd=task.workspace, timeout=timeout, ++ ) ++ combined_log = completed.stdout + completed.stderr ++ usage = None ++ if completed.returncode == 0: ++ lines = [line for line in completed.stdout.splitlines() if line.strip()] ++ if lines: ++ try: ++ worker_result = json.loads(lines[-1]) ++ usage = worker_result.get("usage") ++ except (json.JSONDecodeError, AttributeError): ++ usage = None ++ return completed.returncode, combined_log, usage + + def _blocked(self, run_id: str, task: ProspectiveTask, attempt: int, failure: str) -> ExecutionArtifact: + now = _now() + return ExecutionArtifact(run_id, BaselineExecutionState.BLOCKED, task.task_id, "A", attempt, now, now, None, failure, RunnerFailureCategory.UNKNOWN, None, None, None, None) + + def _finish(self, root: Path, run_id: str, task: ProspectiveTask, attempt: int, started: str, state: BaselineExecutionState, exit_code: int | None, failure: str | None, failure_category: RunnerFailureCategory | None, diff_path: str | None, log_path: str | None, trajectory_path: str | None, usage: dict[str, Any] | None) -> ExecutionArtifact: +- return ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage) ++ artifact_path = root / "execution.json" ++ artifact = ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage, str(artifact_path)) ++ temporary_path = root / "execution.json.tmp" ++ temporary_path.write_text(artifact.to_json() + "\n", encoding="utf-8", newline="\n") ++ temporary_path.replace(artifact_path) ++ return artifact + + + @dataclass(frozen=True) +diff --git a/tests/test_p82_baseline.py b/tests/test_p82_baseline.py +index d7d45ab..8ddf91f 100644 +--- a/tests/test_p82_baseline.py ++++ b/tests/test_p82_baseline.py +@@ -1,6 +1,9 @@ + import tempfile + import unittest ++import json ++import subprocess + from pathlib import Path ++from unittest.mock import patch + + from arkx.p82 import P82RunStatus, P82Treatment, TaskStratum + from arkx.p82_baseline import ( +@@ -40,6 +43,43 @@ def config(root, **overrides): + + + class P82BaselineTests(unittest.TestCase): ++ def test_subprocess_invoker_passes_limits_and_captures_usage(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ runner = MiniSweAgentHeadlessRunner() ++ completed = type("Completed", (), { ++ "returncode": 0, ++ "stdout": '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\n', ++ "stderr": "worker log\n", ++ })() ++ with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: ++ usage = runner._subprocess_invoker(task(workspace), config(directory), Path(directory) / "trajectory.json") ++ self.assertEqual(usage, (0, '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\nworker log\n', {"model_calls": 3, "cost": 0.25})) ++ self.assertEqual(mocked.call_args.kwargs["timeout"], 20) ++ payload = json.loads(mocked.call_args.kwargs["input"]) ++ self.assertEqual(payload["max_cost"], "1") ++ self.assertEqual(payload["wall_time_seconds"], 10) ++ ++ def test_timeout_preserves_partial_output_and_trajectory(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ artifact_root = Path(directory) / "artifacts" ++ trajectory = artifact_root / "task-1" / "attempt-1" / "trajectory.json" ++ ++ def timed_out_invoker(run_task, run_config, trajectory_path): ++ trajectory_path.write_text('{"partial":true}', encoding="utf-8") ++ raise subprocess.TimeoutExpired( ++ cmd="worker", timeout=20, output=b"partial stdout\n", stderr=b"partial stderr\n" ++ ) ++ ++ result = MiniSweAgentHeadlessRunner(timed_out_invoker).run(task(workspace), config(artifact_root)) ++ self.assertEqual(result.failure_category, RunnerFailureCategory.TIMEOUT) ++ self.assertTrue(Path(result.log_path).exists()) ++ self.assertEqual(Path(result.log_path).read_text(encoding="utf-8"), "partial stdout\npartial stderr\n") ++ self.assertEqual(Path(result.trajectory_path).read_text(encoding="utf-8"), '{"partial":true}') ++ + def test_headless_runner_uses_fake_invoker_and_captures_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" +@@ -56,8 +96,23 @@ class P82BaselineTests(unittest.TestCase): + self.assertEqual(calls, [("task-1", P82Treatment.A, "trajectory.json")]) + self.assertTrue(Path(result.log_path).exists()) + self.assertTrue(Path(result.diff_path).exists()) ++ self.assertTrue(Path(result.artifact_path).exists()) ++ persisted = json.loads(Path(result.artifact_path).read_text(encoding="utf-8")) ++ self.assertEqual(persisted, result.to_dict()) ++ self.assertEqual(result.model_usage, {"calls": 1}) + self.assertEqual(result.treatment, "A") + ++ def test_run_identity_changes_when_frozen_model_configuration_changes(self): ++ with tempfile.TemporaryDirectory() as directory: ++ first = config(directory, model="model-a") ++ second = config(directory, model="model-b") ++ self.assertNotEqual(first.identity_digest(), second.identity_digest()) ++ ++ runner = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)) ++ first_result = runner.run(task(directory), first, attempt=1) ++ second_result = runner.run(task(directory), second, attempt=2) ++ self.assertNotEqual(first_result.run_id, second_result.run_id) ++ + def test_missing_model_or_provider_is_blocked_without_fallback(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner().run(task(directory), config(directory, model=None)) +diff --git a/tools/run_mini_headless.py b/tools/run_mini_headless.py +index eedd60b..d501982 100644 +--- a/tools/run_mini_headless.py ++++ b/tools/run_mini_headless.py +@@ -22,17 +22,37 @@ def main() -> int: + from minisweagent.utils.serialize import recursive_merge, UNSET + + config = get_config_from_spec("mini.yaml") +- config = recursive_merge(config, { +- "run": {"task": payload["task"] or UNSET}, +- "agent": {"mode": "yolo", "confirm_exit": False, "output_path": Path(payload["trajectory"])}, +- "model": {"model_name": payload["model"], "model_class": payload.get("model_class", UNSET), **(payload.get("model_configuration") or payload.get("model_kwargs") or {})}, +- "environment": {"cwd": payload["workspace"]}, +- }) ++ config = recursive_merge( ++ config, ++ { ++ "run": {"task": payload["task"] or UNSET}, ++ "agent": { ++ "mode": "yolo", ++ "confirm_exit": False, ++ "output_path": Path(payload["trajectory"]), ++ "cost_limit": None if payload["max_cost"] is None else float(payload["max_cost"]), ++ "wall_time_limit_seconds": None if payload["wall_time_seconds"] is None else int(payload["wall_time_seconds"]), ++ }, ++ "model": { ++ "model_name": payload["model"], ++ "model_class": payload.get("model_class", UNSET), ++ **(payload.get("model_configuration") or payload.get("model_kwargs") or {}), ++ }, ++ "environment": {"cwd": payload["workspace"]}, ++ }, ++ ) + model = get_model(config=config["model"]) + env = get_environment(config["environment"], default_type="local") + agent = get_agent(model, env, config["agent"], default_type="default") + result = agent.run(payload["task"] or "") +- print(json.dumps(result, ensure_ascii=False, sort_keys=True)) ++ output = { ++ "result": result, ++ "usage": { ++ "model_calls": getattr(agent, "n_calls", None), ++ "cost": getattr(agent, "cost", None), ++ }, ++ } ++ print(json.dumps(output, ensure_ascii=False, sort_keys=True)) + return 0 + + diff --git a/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/execution.json b/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/execution.json new file mode 100644 index 00000000..bdd8abcf --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"logs\\p82-agent-preflight-openrouter-liquid-01\\agent-preflight-openrouter-liquid-01\\attempt-1\\execution.json","attempt":1,"diff_path":"logs\\p82-agent-preflight-openrouter-liquid-01\\agent-preflight-openrouter-liquid-01\\attempt-1\\diff.patch","exit_code":0,"failure":null,"failure_category":null,"finished_at":"2026-09-23T10:19:29.385+00:00","log_path":"logs\\p82-agent-preflight-openrouter-liquid-01\\agent-preflight-openrouter-liquid-01\\attempt-1\\executor.log","model_usage":{"cost":0.0,"model_calls":3},"run_id":"7cdc8837f21a5664","started_at":"2026-09-23T10:19:19.898+00:00","state":"COMPLETED","task_id":"agent-preflight-openrouter-liquid-01","trajectory_path":"logs\\p82-agent-preflight-openrouter-liquid-01\\agent-preflight-openrouter-liquid-01\\attempt-1\\trajectory.json","treatment":"A"} diff --git a/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/executor.log b/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/executor.log new file mode 100644 index 00000000..e6b13ce9 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/executor.log @@ -0,0 +1,10 @@ + +Provider List: https://docs.litellm.ai/docs/providers + + +Provider List: https://docs.litellm.ai/docs/providers + + +Provider List: https://docs.litellm.ai/docs/providers + +{"result": {"exit_status": "RepeatedFormatError", "submission": ""}, "usage": {"cost": 0.0, "model_calls": 3}} diff --git a/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/trajectory.json b/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/trajectory.json new file mode 100644 index 00000000..9ec7bde1 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-liquid-01/agent-preflight-openrouter-liquid-01/attempt-1/trajectory.json @@ -0,0 +1,279 @@ +{ + "info": { + "model_stats": { + "instance_cost": 0.0, + "api_calls": 3 + }, + "config": { + "agent": { + "system_template": "You are a helpful assistant that can interact with a computer.\n", + "instance_template": "Please solve this issue: {{task}}\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\n{{system}} {{release}} {{version}} {{machine}}\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:\n\n{%- if system == \"Darwin\" -%}\n\nYou are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.\n\n{%- endif -%}\n\n```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```\n", + "step_limit": 0, + "cost_limit": 0.0, + "wall_time_limit_seconds": 180, + "max_consecutive_format_errors": 3, + "output_path": "logs\\p82-agent-preflight-openrouter-liquid-01\\agent-preflight-openrouter-liquid-01\\attempt-1\\trajectory.json" + }, + "agent_type": "minisweagent.agents.default.DefaultAgent", + "model": { + "model_name": "liquid/lfm-2.5-2.6b:free", + "model_kwargs": { + "drop_params": true, + "custom_llm_provider": "openrouter", + "provider": { + "allow_fallbacks": false + }, + "max_tokens": 64 + }, + "litellm_model_registry": null, + "set_cache_control": null, + "cost_tracking": "ignore_errors", + "format_error_template": "{% if finish_reason is defined and (finish_reason == \"length\" or (finish_reason == \"tool_calls\" and not has_tool_calls)) -%}\nYour previous response reached the output token limit (finish_reason={{ finish_reason }}) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.\n{%- else -%}\nTool call error:\n\n\n{{error}}\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.\n{%- endif %}\n", + "observation_template": "{%- if output.output | length < 10000 -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output\": {{ output.output | tojson }}\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- else -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output_head\": {{ output.output[:5000] | tojson }},\n \"output_tail\": {{ output.output[-5000:] | tojson }},\n \"elided_chars\": {{ output.output | length - 10000 }},\n \"warning\": \"Output too long.\"\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- endif -%}\n", + "multimodal_regex": "" + }, + "model_type": "minisweagent.models.litellm_model.LitellmModel", + "environment": { + "cwd": ".", + "env": { + "PAGER": "cat", + "MANPAGER": "cat", + "LESS": "-R", + "PIP_PROGRESS_BAR": "off", + "TQDM_DISABLE": "1" + }, + "timeout": 30 + }, + "environment_type": "minisweagent.environments.local.LocalEnvironment" + }, + "mini_version": "2.4.6", + "exit_status": "RepeatedFormatError", + "submission": "" + }, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: Reply with exactly OK.\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nWindows 10 10.0.26200 AMD64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + }, + { + "role": "user", + "content": "Your previous response reached the output token limit (finish_reason=length) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.", + "extra": { + "interrupt_type": "FormatError", + "cost": 0.0, + "response": { + "id": "gen-1790158766-CGALnC8C2iJOKAosGcG7", + "created": 1790158766, + "model": "liquid/lfm-2.5-2.6b:free", + "object": "chat.completion", + "system_fingerprint": "vllm-0.29.1rc1.dev9+g2671fedfc-dafbf1d2", + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": null, + "function_call": null, + "reasoning_content": "The user wants me to solve an issue related to \"Reply with exactly OK.\" This seems like a meta-instruction about how to respond. But I also need to actually analyze the codebase and fix some issue. Let me start by exploring the repository structure to understand what we're dealing with.\n\nFirst, I'll list", + "provider_specific_fields": { + "refusal": null, + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user wants me to solve an issue related to \"Reply with exactly OK.\" This seems like a meta-instruction about how to respond. But I also need to actually analyze the codebase and fix some issue. Let me start by exploring the repository structure to understand what we're dealing with.\n\nFirst, I'll list", + "format": "unknown", + "index": 0 + } + ], + "reasoning": "The user wants me to solve an issue related to \"Reply with exactly OK.\" This seems like a meta-instruction about how to respond. But I also need to actually analyze the codebase and fix some issue. Let me start by exploring the repository structure to understand what we're dealing with.\n\nFirst, I'll list" + } + }, + "provider_specific_fields": { + "native_finish_reason": "length" + } + } + ], + "usage": { + "completion_tokens": 64, + "prompt_tokens": 781, + "total_tokens": 845, + "completion_tokens_details": { + "audio_tokens": 0, + "reasoning_tokens": 64, + "image_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 0, + "video_tokens": 0, + "cache_creation_tokens": 0 + }, + "cost": 0.0, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0, + "upstream_inference_completions_cost": 0 + } + }, + "provider": "Liquid", + "service_tier": null + } + } + }, + { + "role": "user", + "content": "Your previous response reached the output token limit (finish_reason=length) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.", + "extra": { + "interrupt_type": "FormatError", + "cost": 0.0, + "response": { + "id": "gen-1790158767-VAV9vVXjplCquaNANccY", + "created": 1790158767, + "model": "liquid/lfm-2.5-2.6b:free", + "object": "chat.completion", + "system_fingerprint": "vllm-0.29.1rc1.dev9+g2671fedfc-dafbf1d2", + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": null, + "function_call": null, + "reasoning_content": "The user wants me to solve an issue related to \"Reply with exactly OK.\" This seems like a simple request to confirm something. But I need to first understand the context - there's no actual issue described yet. The system message says I should analyze the codebase and find relevant files, but I don't have any", + "provider_specific_fields": { + "refusal": null, + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user wants me to solve an issue related to \"Reply with exactly OK.\" This seems like a simple request to confirm something. But I need to first understand the context - there's no actual issue described yet. The system message says I should analyze the codebase and find relevant files, but I don't have any", + "format": "unknown", + "index": 0 + } + ], + "reasoning": "The user wants me to solve an issue related to \"Reply with exactly OK.\" This seems like a simple request to confirm something. But I need to first understand the context - there's no actual issue described yet. The system message says I should analyze the codebase and find relevant files, but I don't have any" + } + }, + "provider_specific_fields": { + "native_finish_reason": "length" + } + } + ], + "usage": { + "completion_tokens": 64, + "prompt_tokens": 837, + "total_tokens": 901, + "completion_tokens_details": { + "audio_tokens": 0, + "reasoning_tokens": 64, + "image_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 768, + "video_tokens": 0, + "cache_creation_tokens": 0 + }, + "cost": 0.0, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0, + "upstream_inference_completions_cost": 0 + } + }, + "provider": "Liquid", + "service_tier": null + } + } + }, + { + "role": "user", + "content": "Your previous response reached the output token limit (finish_reason=length) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.", + "extra": { + "interrupt_type": "FormatError", + "cost": 0.0, + "response": { + "id": "gen-1790158768-WFN3W5tpvktbxQfPjLpz", + "created": 1790158768, + "model": "liquid/lfm-2.5-2.6b:free", + "object": "chat.completion", + "system_fingerprint": "vllm-0.29.1rc1.dev9+g2671fedfc-dafbf1d2", + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": null, + "function_call": null, + "reasoning_content": "The user wants me to solve an issue but I need to first understand what the issue is. They mentioned \"Reply with exactly OK\" which seems like a test instruction. But looking at the context, there's no actual issue described yet - just a general request to solve something. \n\nHowever, I notice the system prompt", + "provider_specific_fields": { + "refusal": null, + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user wants me to solve an issue but I need to first understand what the issue is. They mentioned \"Reply with exactly OK\" which seems like a test instruction. But looking at the context, there's no actual issue described yet - just a general request to solve something. \n\nHowever, I notice the system prompt", + "format": "unknown", + "index": 0 + } + ], + "reasoning": "The user wants me to solve an issue but I need to first understand what the issue is. They mentioned \"Reply with exactly OK\" which seems like a test instruction. But looking at the context, there's no actual issue described yet - just a general request to solve something. \n\nHowever, I notice the system prompt" + } + }, + "provider_specific_fields": { + "native_finish_reason": "length" + } + } + ], + "usage": { + "completion_tokens": 64, + "prompt_tokens": 893, + "total_tokens": 957, + "completion_tokens_details": { + "audio_tokens": 0, + "reasoning_tokens": 64, + "image_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 832, + "video_tokens": 0, + "cache_creation_tokens": 0 + }, + "cost": 0.0, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0, + "upstream_inference_completions_cost": 0 + } + }, + "provider": "Liquid", + "service_tier": null + } + } + }, + { + "role": "exit", + "content": "RepeatedFormatError", + "extra": { + "exit_status": "RepeatedFormatError", + "submission": "" + } + } + ], + "trajectory_format": "mini-swe-agent-1.1" +} \ No newline at end of file diff --git a/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/diff.patch b/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/diff.patch new file mode 100644 index 00000000..5cc4cdda --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/diff.patch @@ -0,0 +1,619 @@ +diff --git a/README.md b/README.md +index 13041af..c729c17 100644 +--- a/README.md ++++ b/README.md +@@ -1,26 +1,29 @@ + # Arkx + +-Arkx is a greenfield project for empirical, incremental development of reliable capability. ++Arkx is a repository for incremental implementation and empirical evaluation of software capability. + + > **MINIMUM SUFFICIENT ARCHITECTURE** + > **FOR MAXIMUM RELIABLE CAPABILITY** + +-This repository is the canonical root for the project. It intentionally contains only the foundation in this bootstrap: the project contract, experimental protocol, structural placeholders, and a minimal CI check. ++This repository is the canonical root for the project. It contains runtime ++contracts, an execution harness, the project contract, the experimental ++protocol, and structural checks. + +-## Scope of this bootstrap ++## Current scope + + Included: + + - explicit project invariants; +-- a small, evidence-oriented experimental protocol; +-- stable top-level boundaries for future work; +-- a deterministic structural check and minimal CI. ++- an evidence-oriented experimental protocol; ++- stable top-level boundaries; ++- a deterministic structural check, executable contracts, and a Treatment A ++ baseline harness. + +-Deferred: ++Still deferred or unqualified: + +-- routing and planning; + - multi-agent orchestration; +-- OpenHands, ReX, mini-SWE-agent, and SWE-agent integrations; ++- OpenHands, ReX, and SWE-agent integrations; ++- real provider/model adoption and the A/B/C/D mechanism trial; + - product dependencies or automatic migration from `smag-rex`. + + ## Development posture +@@ -44,9 +47,9 @@ P1 adds deterministic task characterization from explicit signals only. See [the + + P2 adds deterministic progress/stagnation assessment from explicit snapshots only. See [the P2 contract](docs/progress-stagnation.md). It does not control execution or authorize retry/replan. + +-P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke or select concrete executors. ++P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke concrete executors. + +-Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. None executes a plan, owns acceptance, or creates orchestration. ++Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. P8.2a adds a Treatment A baseline runner and official SWE-bench acceptance adapter; neither owns general orchestration or promotion. + + ## Checks + +diff --git a/docs/architecture.md b/docs/architecture.md +index 17bbba6..98d1e6a 100644 +--- a/docs/architecture.md ++++ b/docs/architecture.md +@@ -1,24 +1,75 @@ + # Architecture boundary + +-The bootstrap architecture is intentionally small: ++Arkx separates contracts, execution, verification, acceptance, and promotion. ++The executable contracts cover the request-to-acceptance path. Runtime ++capability is qualified per provider, model, executor, and treatment; the ++presence of a contract does not imply adoption. + + ```text + Arkx +-├── docs/ contracts, protocol, and decisions +-├── experiments/ experiment records and fixtures; no runtime yet +-├── src/ reserved product boundary; no implementation yet +-├── tests/ reserved verification boundary; no test suite yet +-└── tools/ dependency-free repository checks ++├── docs/ contracts, protocol, decisions, and evidence boundaries ++├── experiments/ experiment records and synthetic fixtures; not orchestration ++├── src/arkx/ executable contracts and bounded policy/runtime surfaces ++├── tests/ executable contract and behavior verification ++└── tools/ repository checks and narrow execution adapters + ``` + +-P0 adds a deliberately small executable core under `src/arkx/`: versioned contracts, evidence status resolution, telemetry collection, and a deterministic baseline fixture. P1 adds explicit-signal task characterization, P2 adds snapshot-based progress assessment, P3 adds bounded routing/escalation policy, and Block B adds independent P4/P5/P6 artifacts in the same boundary. No layer has a concrete executor interface or executor orchestration. ++The current executable boundary is layered as follows: ++ ++- P0: versioned contracts, evidence status resolution, telemetry collection, ++ and a deterministic baseline fixture; ++- P1: explicit-signal task characterization; ++- P2/P3: progress assessment, bounded routing, and evidence-based escalation; ++- P4/P5/P6: repository planning/state, patch verification, and handoff/context ++ accounting; ++- P7/P8.1/P8.2: composition records, executor qualification contracts, and ++ controlled treatment plans; ++- P8.2a: a narrow Treatment A baseline runner and the official SWE-bench ++ acceptance adapter; ++- current orchestration surfaces: request governance, treatment/executor ++ selection, sequential execution, recovery, independent acceptance, and ++ promotion gates. ++ ++The canonical flow is: ++ ++```text ++request ++ -> governance / authority ++ -> task characterization ++ -> treatment route ++ -> sufficient executor ++ -> progress / recovery ++ -> verification ++ -> independent acceptance ++ -> evidence for possible promotion ++``` ++ ++Routing selects the minimum sufficient treatment before selecting an executor. ++Simple, localized, and repository-wide paths are capabilities/regimes, not ++automatic executor cascades. A fallback, executor switch, scope expansion, ++acceptance, or promotion must remain explicit and evidence-backed. ++ ++The official SWE-bench Docker authority has been qualified on the frozen Gold ++tasks. That validates the acceptance path and infrastructure, not a real ++provider/model, Treatment A execution, or A/B/C/D mechanism evidence. + + ## Boundary rules + +-- `src/` is reserved for product code and is empty in this bootstrap. ++- `src/arkx/` contains product contracts and bounded policy/runtime code; it ++ must not silently grow into an unqualified general-purpose orchestrator. + - `experiments/` contains evidence-oriented records, not an implicit runtime. +-- `tools/` may validate repository contracts but must not become orchestration. +-- `tests/` contains P0 executable verification and remains the boundary for future tests. +-- Integrations and executors require an explicit decision record and experiment evidence. ++- `tools/` may validate contracts or host a narrow adapter, but must not become ++ an untracked orchestration layer. ++- `tests/` verifies contracts and behavior; passing tests are local evidence, ++ not scientific or upstream evidence. ++- Integrations, providers, models, and executors require explicit identity, ++ qualification evidence, and an acceptance authority. ++- Missing provider/model identity, missing evidence, and blocked execution are ++ preserved as blocked or unknown; they never become success. ++- The working tree may contain an implementation under qualification. Merge ++ readiness requires the corresponding decision record, tests, and evidence to ++ be committed together. + +-An empty boundary is intentional. Creating a directory does not imply that the corresponding capability exists. ++The boundary is explicit: creating a module, fixture, or directory does not by ++itself prove that the corresponding capability exists or is suitable for ++promotion. +diff --git a/docs/experimental-protocol.md b/docs/experimental-protocol.md +index f24bf94..2b0f92d 100644 +--- a/docs/experimental-protocol.md ++++ b/docs/experimental-protocol.md +@@ -1,6 +1,7 @@ + # Experimental protocol + +-Arkx develops through small, empirical increments. The protocol below is the minimum record required before a capability can be considered for promotion. ++Arkx develops through empirical increments. The protocol below defines the ++minimum record required before a capability can be considered for promotion. + + ## Required record + +@@ -14,6 +15,9 @@ Each experiment records: + - `procedure`: reproducible steps; + - `expected_signal`: measurable success and failure criteria; + - `result`: `PASS`, `FAIL`, `BLOCKED`, or `NOT_EXECUTED`; ++- `evidence`: links or paths to raw evidence and environment facts; ++- `acceptance`: explicit reviewer decision and rationale; ++- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. + + P7 composition trials use their own comparability namespace: `COMPARABLE`, `INCOMPARABLE`, and `UNKNOWN`. P7 records externally supplied measurements and does not convert them into acceptance results. + +@@ -26,9 +30,14 @@ environment, budget, verification, and independent acceptance authority. It + does not invoke an executor or mechanism and does not manufacture acceptance; + results remain P8.1 observations. Missing controls and measurements remain + `UNKNOWN`, and the only intended treatment difference is the declared arm. +-- `evidence`: links or paths to raw evidence and environment facts; +-- `acceptance`: explicit reviewer decision and rationale; +-- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. ++ ++P8.2a adds the Treatment A baseline runner. Each attempt is serial, refuses to ++overwrite an existing attempt directory, derives a stable run identity from ++the task and frozen execution configuration, and writes an `execution.json` ++artifact atomically alongside the log, diff, and trajectory. The artifact ++records execution only; verification, independent acceptance, and promotion ++remain separate records. A missing provider, model, mini version, or Python ++executable blocks the attempt instead of selecting a fallback. + + ## State separation + +diff --git a/docs/p82-minimal-mechanism-trial.md b/docs/p82-minimal-mechanism-trial.md +index b05ef93..43225e8 100644 +--- a/docs/p82-minimal-mechanism-trial.md ++++ b/docs/p82-minimal-mechanism-trial.md +@@ -34,8 +34,8 @@ promotion decision. + + ## P8.2a baseline execution path + +-P8.2a is infrastructure qualification, not mechanism evidence. The baseline +-runner accepts treatment A only, invokes mini-SWE-agent through its public ++P8.2a records infrastructure qualification, not mechanism evidence. The ++baseline runner accepts treatment A only, invokes mini-SWE-agent through its public + Python API in a headless worker, and records execution artifacts. It does not + run verification or decide acceptance. Verification and independent + acceptance are explicit, nullable records; missing provider/model identity is +@@ -49,7 +49,9 @@ sample is frozen in `experiments/p82-wave0-task-sample.json` from the official + + The official acceptance adapter is in `arkx.swebench_authority`. It creates a + SWE-bench prediction without gold patch or grading labels and invokes the +-official harness with a unique `run_id`. Gold qualification and real A runs +-remain blocked until `swebench` and Docker are available, the gold harness +-passes, and one exact provider/model/configuration is frozen. B/C/D and any ++official harness with a unique `run_id`. The frozen Docker Gold authority ++returned `resolved: true` for the two Wave 0 tasks recorded under ++`logs/evaluation/`; this qualifies the acceptance path and infrastructure only. ++Real A runs still ++require one exact provider/model/configuration to be frozen. B/C/D and any + mechanism comparison remain out of scope. +diff --git a/docs/project-contract.md b/docs/project-contract.md +index f2f734a..82054a4 100644 +--- a/docs/project-contract.md ++++ b/docs/project-contract.md +@@ -40,7 +40,9 @@ NO_SILENT_SCOPE_EXPANSION + 5. Any fallback, executor change, or scope change is explicit in the record and review. + 6. New architecture requires a decision record describing the observed need, alternatives, evidence, and rollback/removal condition. + +-## Non-goals of this bootstrap +- +-This repository does not yet define routing, planning, multi-agent execution, or integrations with OpenHands, ReX, mini-SWE-agent, or SWE-agent. ++## Current non-goals + ++The repository now defines bounded routing, planning/state, verification, ++recovery, acceptance, promotion gates, and a mini-SWE-agent Treatment A ++baseline path. It does not define general multi-agent execution, automatic ++executor adoption, or integrations with OpenHands, ReX, and SWE-agent. +diff --git a/experiments/README.md b/experiments/README.md +index 551f37e..fe93b83 100644 +--- a/experiments/README.md ++++ b/experiments/README.md +@@ -2,3 +2,4 @@ + + This directory is reserved for explicit experiment records and small fixtures. Use [the experimental protocol](../docs/experimental-protocol.md). No experiment has been executed by the bootstrap. + ++The exploratory local-model qualification wave is documented in [local model qualification](../docs/local-model-qualification.md). It was run outside the Arkx experiment harness and must not be read as executor adoption or acceptance evidence. +diff --git a/src/arkx/__init__.py b/src/arkx/__init__.py +index 39c7d6c..2508056 100644 +--- a/src/arkx/__init__.py ++++ b/src/arkx/__init__.py +@@ -1,6 +1,62 @@ + """Minimal, executor-agnostic Arkx execution contracts and telemetry.""" + + from .contracts import Event, EventType, ExecutionRecord, ExecutionStatus ++from .request import ( ++ ArkxRequest, ++ GovernanceDecision, ++ GovernancePolicy, ++ GovernanceReason, ++ GovernanceStatus, ++ RequestBudget, ++ RequestEnvironment, ++ authorize_request, ++ characterize_authorized_request, ++) ++from .selection import ( ++ ExecutorBinding, ++ ExecutorRegistry, ++ SelectionDecision, ++ SelectionReason, ++ SelectionStatus, ++ TreatmentCatalog, ++ TreatmentDefinition, ++ select_executor, ++ select_treatment, ++) ++from .p82_localization import ( ++ LocalizationArtifact, ++ LocalizationEvidenceBudget, ++ record_localization_artifact, ++) ++from .p82_editing import ( ++ EditOperation, ++ StructuredEditProposal, ++ record_edit_proposal, ++) ++from .orchestration import ( ++ ExecutionOutcome, ++ ExecutionResult, ++ OrchestrationReason, ++ OrchestrationResult, ++ OrchestrationStatus, ++ run_routed_pipeline, ++) ++from .recovery import ( ++ RecoveryAction, ++ RecoveryBudget, ++ RecoveryDecision, ++ RecoveryReason, ++ decide_recovery, ++) ++from .acceptance import AcceptancePolicy, decide_independent_acceptance ++from .promotion import ( ++ PromotionCandidate, ++ PromotionDecision, ++ PromotionPolicy, ++ PromotionReason, ++ PromotionStatus, ++ decide_promotion, ++) + from .characterization import ( + CharacterizationConfig, + Confidence, +@@ -108,6 +164,43 @@ __all__ = [ + "EventType", + "ExecutionRecord", + "ExecutionStatus", ++ "ArkxRequest", ++ "GovernanceDecision", ++ "GovernancePolicy", ++ "GovernanceReason", ++ "GovernanceStatus", ++ "RequestBudget", ++ "RequestEnvironment", ++ "authorize_request", ++ "characterize_authorized_request", ++ "ExecutorBinding", ++ "ExecutorRegistry", ++ "SelectionDecision", ++ "SelectionReason", ++ "SelectionStatus", ++ "TreatmentCatalog", ++ "TreatmentDefinition", ++ "select_executor", ++ "select_treatment", ++ "ExecutionOutcome", ++ "ExecutionResult", ++ "OrchestrationReason", ++ "OrchestrationResult", ++ "OrchestrationStatus", ++ "run_routed_pipeline", ++ "RecoveryAction", ++ "RecoveryBudget", ++ "RecoveryDecision", ++ "RecoveryReason", ++ "decide_recovery", ++ "AcceptancePolicy", ++ "decide_independent_acceptance", ++ "PromotionCandidate", ++ "PromotionDecision", ++ "PromotionPolicy", ++ "PromotionReason", ++ "PromotionStatus", ++ "decide_promotion", + "CharacterizationConfig", + "Confidence", + "RecommendedPath", +diff --git a/src/arkx/p82_baseline.py b/src/arkx/p82_baseline.py +index 9cd9d72..cd192fa 100644 +--- a/src/arkx/p82_baseline.py ++++ b/src/arkx/p82_baseline.py +@@ -138,6 +138,26 @@ class BaselineRunConfig: + def missing_identity(self) -> tuple[str, ...]: + return tuple(name for name in ("provider", "model", "mini_version", "python_executable") if getattr(self, name) is None) + ++ def identity_digest(self) -> str: ++ """Return a stable digest for the frozen execution configuration.""" ++ value = { ++ "provider": self.provider, ++ "model": self.model, ++ "model_version": self.model_version, ++ "temperature": self.temperature, ++ "reasoning_configuration": self.reasoning_configuration, ++ "model_configuration": self.model_configuration, ++ "max_tokens": self.max_tokens, ++ "max_cost": self.max_cost, ++ "wall_time_seconds": self.wall_time_seconds, ++ "mini_version": self.mini_version, ++ "python_executable": self.python_executable, ++ "runner_version": self.runner_version, ++ "treatment": self.treatment.value, ++ } ++ encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:16] ++ + + @dataclass(frozen=True) + class ExecutionArtifact: +@@ -155,16 +175,21 @@ class ExecutionArtifact: + log_path: str | None + trajectory_path: str | None + model_usage: dict[str, Any] | None ++ artifact_path: str | None = None + + def to_dict(self) -> dict[str, Any]: + value = {name: getattr(self, name) for name in ( + "run_id", "state", "task_id", "treatment", "attempt", "started_at", "finished_at", +- "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", "model_usage")} ++ "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", ++ "model_usage", "artifact_path")} + for name in ("state", "failure_category"): + if isinstance(value[name], Enum): + value[name] = value[name].value + return value + ++ def to_json(self) -> str: ++ return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ + + class HeadlessInvoker(Protocol): + def __call__(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: ... +@@ -185,7 +210,9 @@ class MiniSweAgentHeadlessRunner: + raise ValueError("attempt must be positive") + missing = config.missing_identity() + root = Path(config.artifact_root) / task.task_id / f"attempt-{attempt}" +- run_id = hashlib.sha256(f"{task.task_id}|A|{attempt}|{config.runner_version}".encode()).hexdigest()[:16] ++ run_id = hashlib.sha256( ++ f"{task.task_id}|{task.base_commit}|A|{attempt}|{config.identity_digest()}".encode("utf-8") ++ ).hexdigest()[:16] + if root.exists(): + return self._blocked(run_id, task, attempt, "artifact already exists; refusing silent overwrite") + root.mkdir(parents=True) +@@ -204,7 +231,16 @@ class MiniSweAgentHeadlessRunner: + state = BaselineExecutionState.COMPLETED if exit_code == 0 else BaselineExecutionState.EXECUTOR_FAILED + return self._finish(root, run_id, task, attempt, started, state, exit_code, None if exit_code == 0 else "executor returned non-zero", None if exit_code == 0 else RunnerFailureCategory.AGENT_RUN_FAILURE, str(diff_path), str(log_path), str(trajectory), usage) + except subprocess.TimeoutExpired as error: +- return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, str(error), RunnerFailureCategory.TIMEOUT, None, None, None, None) ++ log_path = root / "executor.log" ++ stdout = self._decode_process_output(error.stdout) ++ stderr = self._decode_process_output(error.stderr) ++ log_path.write_text(stdout + stderr, encoding="utf-8", newline="\n") ++ return self._finish( ++ root, run_id, task, attempt, started, ++ BaselineExecutionState.EXECUTOR_FAILED, None, str(error), ++ RunnerFailureCategory.TIMEOUT, None, str(log_path), ++ str(trajectory) if trajectory.exists() else None, None, ++ ) + except Exception as error: + message = f"{type(error).__name__}: {error}" + return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, message, self._classify_failure(message), None, None, None, None) +@@ -228,21 +264,47 @@ class MiniSweAgentHeadlessRunner: + return RunnerFailureCategory.DEADLOCK_HANG + return RunnerFailureCategory.AGENT_RUN_FAILURE + ++ @staticmethod ++ def _decode_process_output(value: str | bytes | None) -> str: ++ if value is None: ++ return "" ++ if isinstance(value, bytes): ++ return value.decode("utf-8", errors="replace") ++ return value ++ + def _subprocess_invoker(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: + worker = Path(__file__).parents[2] / "tools" / "run_mini_headless.py" + payload = {"task": task.problem_statement, "workspace": task.workspace, "model": config.model, + "trajectory": str(trajectory_path), "max_cost": config.max_cost, "wall_time_seconds": config.wall_time_seconds, + "model_configuration": config.model_configuration} +- completed = subprocess.run([config.python_executable, str(worker)], input=json.dumps(payload), text=True, +- capture_output=True, cwd=task.workspace, timeout=config.wall_time_seconds) +- return completed.returncode, completed.stdout + completed.stderr, None ++ timeout = None if config.wall_time_seconds is None else config.wall_time_seconds + 10 ++ completed = subprocess.run( ++ [config.python_executable, str(worker)], input=json.dumps(payload), text=True, ++ capture_output=True, cwd=task.workspace, timeout=timeout, ++ ) ++ combined_log = completed.stdout + completed.stderr ++ usage = None ++ if completed.returncode == 0: ++ lines = [line for line in completed.stdout.splitlines() if line.strip()] ++ if lines: ++ try: ++ worker_result = json.loads(lines[-1]) ++ usage = worker_result.get("usage") ++ except (json.JSONDecodeError, AttributeError): ++ usage = None ++ return completed.returncode, combined_log, usage + + def _blocked(self, run_id: str, task: ProspectiveTask, attempt: int, failure: str) -> ExecutionArtifact: + now = _now() + return ExecutionArtifact(run_id, BaselineExecutionState.BLOCKED, task.task_id, "A", attempt, now, now, None, failure, RunnerFailureCategory.UNKNOWN, None, None, None, None) + + def _finish(self, root: Path, run_id: str, task: ProspectiveTask, attempt: int, started: str, state: BaselineExecutionState, exit_code: int | None, failure: str | None, failure_category: RunnerFailureCategory | None, diff_path: str | None, log_path: str | None, trajectory_path: str | None, usage: dict[str, Any] | None) -> ExecutionArtifact: +- return ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage) ++ artifact_path = root / "execution.json" ++ artifact = ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage, str(artifact_path)) ++ temporary_path = root / "execution.json.tmp" ++ temporary_path.write_text(artifact.to_json() + "\n", encoding="utf-8", newline="\n") ++ temporary_path.replace(artifact_path) ++ return artifact + + + @dataclass(frozen=True) +diff --git a/tests/test_p82_baseline.py b/tests/test_p82_baseline.py +index d7d45ab..8ddf91f 100644 +--- a/tests/test_p82_baseline.py ++++ b/tests/test_p82_baseline.py +@@ -1,6 +1,9 @@ + import tempfile + import unittest ++import json ++import subprocess + from pathlib import Path ++from unittest.mock import patch + + from arkx.p82 import P82RunStatus, P82Treatment, TaskStratum + from arkx.p82_baseline import ( +@@ -40,6 +43,43 @@ def config(root, **overrides): + + + class P82BaselineTests(unittest.TestCase): ++ def test_subprocess_invoker_passes_limits_and_captures_usage(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ runner = MiniSweAgentHeadlessRunner() ++ completed = type("Completed", (), { ++ "returncode": 0, ++ "stdout": '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\n', ++ "stderr": "worker log\n", ++ })() ++ with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: ++ usage = runner._subprocess_invoker(task(workspace), config(directory), Path(directory) / "trajectory.json") ++ self.assertEqual(usage, (0, '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\nworker log\n', {"model_calls": 3, "cost": 0.25})) ++ self.assertEqual(mocked.call_args.kwargs["timeout"], 20) ++ payload = json.loads(mocked.call_args.kwargs["input"]) ++ self.assertEqual(payload["max_cost"], "1") ++ self.assertEqual(payload["wall_time_seconds"], 10) ++ ++ def test_timeout_preserves_partial_output_and_trajectory(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ artifact_root = Path(directory) / "artifacts" ++ trajectory = artifact_root / "task-1" / "attempt-1" / "trajectory.json" ++ ++ def timed_out_invoker(run_task, run_config, trajectory_path): ++ trajectory_path.write_text('{"partial":true}', encoding="utf-8") ++ raise subprocess.TimeoutExpired( ++ cmd="worker", timeout=20, output=b"partial stdout\n", stderr=b"partial stderr\n" ++ ) ++ ++ result = MiniSweAgentHeadlessRunner(timed_out_invoker).run(task(workspace), config(artifact_root)) ++ self.assertEqual(result.failure_category, RunnerFailureCategory.TIMEOUT) ++ self.assertTrue(Path(result.log_path).exists()) ++ self.assertEqual(Path(result.log_path).read_text(encoding="utf-8"), "partial stdout\npartial stderr\n") ++ self.assertEqual(Path(result.trajectory_path).read_text(encoding="utf-8"), '{"partial":true}') ++ + def test_headless_runner_uses_fake_invoker_and_captures_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" +@@ -56,8 +96,23 @@ class P82BaselineTests(unittest.TestCase): + self.assertEqual(calls, [("task-1", P82Treatment.A, "trajectory.json")]) + self.assertTrue(Path(result.log_path).exists()) + self.assertTrue(Path(result.diff_path).exists()) ++ self.assertTrue(Path(result.artifact_path).exists()) ++ persisted = json.loads(Path(result.artifact_path).read_text(encoding="utf-8")) ++ self.assertEqual(persisted, result.to_dict()) ++ self.assertEqual(result.model_usage, {"calls": 1}) + self.assertEqual(result.treatment, "A") + ++ def test_run_identity_changes_when_frozen_model_configuration_changes(self): ++ with tempfile.TemporaryDirectory() as directory: ++ first = config(directory, model="model-a") ++ second = config(directory, model="model-b") ++ self.assertNotEqual(first.identity_digest(), second.identity_digest()) ++ ++ runner = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)) ++ first_result = runner.run(task(directory), first, attempt=1) ++ second_result = runner.run(task(directory), second, attempt=2) ++ self.assertNotEqual(first_result.run_id, second_result.run_id) ++ + def test_missing_model_or_provider_is_blocked_without_fallback(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner().run(task(directory), config(directory, model=None)) +diff --git a/tools/run_mini_headless.py b/tools/run_mini_headless.py +index eedd60b..d501982 100644 +--- a/tools/run_mini_headless.py ++++ b/tools/run_mini_headless.py +@@ -22,17 +22,37 @@ def main() -> int: + from minisweagent.utils.serialize import recursive_merge, UNSET + + config = get_config_from_spec("mini.yaml") +- config = recursive_merge(config, { +- "run": {"task": payload["task"] or UNSET}, +- "agent": {"mode": "yolo", "confirm_exit": False, "output_path": Path(payload["trajectory"])}, +- "model": {"model_name": payload["model"], "model_class": payload.get("model_class", UNSET), **(payload.get("model_configuration") or payload.get("model_kwargs") or {})}, +- "environment": {"cwd": payload["workspace"]}, +- }) ++ config = recursive_merge( ++ config, ++ { ++ "run": {"task": payload["task"] or UNSET}, ++ "agent": { ++ "mode": "yolo", ++ "confirm_exit": False, ++ "output_path": Path(payload["trajectory"]), ++ "cost_limit": None if payload["max_cost"] is None else float(payload["max_cost"]), ++ "wall_time_limit_seconds": None if payload["wall_time_seconds"] is None else int(payload["wall_time_seconds"]), ++ }, ++ "model": { ++ "model_name": payload["model"], ++ "model_class": payload.get("model_class", UNSET), ++ **(payload.get("model_configuration") or payload.get("model_kwargs") or {}), ++ }, ++ "environment": {"cwd": payload["workspace"]}, ++ }, ++ ) + model = get_model(config=config["model"]) + env = get_environment(config["environment"], default_type="local") + agent = get_agent(model, env, config["agent"], default_type="default") + result = agent.run(payload["task"] or "") +- print(json.dumps(result, ensure_ascii=False, sort_keys=True)) ++ output = { ++ "result": result, ++ "usage": { ++ "model_calls": getattr(agent, "n_calls", None), ++ "cost": getattr(agent, "cost", None), ++ }, ++ } ++ print(json.dumps(output, ensure_ascii=False, sort_keys=True)) + return 0 + + diff --git a/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/execution.json b/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/execution.json new file mode 100644 index 00000000..5f9da5d0 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"logs\\p82-agent-preflight-openrouter-liquid-02\\agent-preflight-openrouter-liquid-02\\attempt-1\\execution.json","attempt":1,"diff_path":"logs\\p82-agent-preflight-openrouter-liquid-02\\agent-preflight-openrouter-liquid-02\\attempt-1\\diff.patch","exit_code":0,"failure":null,"failure_category":null,"finished_at":"2026-09-23T10:20:34.017+00:00","log_path":"logs\\p82-agent-preflight-openrouter-liquid-02\\agent-preflight-openrouter-liquid-02\\attempt-1\\executor.log","model_usage":{"cost":0.0,"model_calls":3},"run_id":"036df2de70cb2612","started_at":"2026-09-23T10:20:20.152+00:00","state":"COMPLETED","task_id":"agent-preflight-openrouter-liquid-02","trajectory_path":"logs\\p82-agent-preflight-openrouter-liquid-02\\agent-preflight-openrouter-liquid-02\\attempt-1\\trajectory.json","treatment":"A"} diff --git a/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/executor.log b/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/executor.log new file mode 100644 index 00000000..ef4528b6 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/executor.log @@ -0,0 +1,15 @@ + +Provider List: https://docs.litellm.ai/docs/providers + + +Provider List: https://docs.litellm.ai/docs/providers + + +Give Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new +LiteLLM.Info: If you need to debug this error, use `litellm._turn_on_debug()'. + + +Provider List: https://docs.litellm.ai/docs/providers + +{"result": {"exit_status": "RepeatedFormatError", "submission": ""}, "usage": {"cost": 0.0, "model_calls": 3}} +Retrying in 4 seconds as it raised RateLimitError: litellm.RateLimitError: RateLimitError: OpenrouterException - {"error":{"message":"Provider returned error","code":429,"metadata":{"raw":"liquid/lfm-2.5-2.6b:free is temporarily rate-limited upstream. Please retry shortly, or add your own key to accumulate your rate limits: https://openrouter.ai/settings/integrations","provider_name":"Liquid","is_byok":false,"provider_error_code":"rate_limit_exceeded","limit_source":"upstream_provider_shared_pool","remedy_hint":"Retry shortly, add your own provider key (https://openrouter.ai/settings/integrations), or route to another provider with provider routing: https://openrouter.ai/docs/features/provider-routing","retry_after_seconds":32,"retry_after_seconds_raw":32,"headers":{"Retry-After":"32"}}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. diff --git a/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/trajectory.json b/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/trajectory.json new file mode 100644 index 00000000..f0c41a11 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-liquid-02/agent-preflight-openrouter-liquid-02/attempt-1/trajectory.json @@ -0,0 +1,279 @@ +{ + "info": { + "model_stats": { + "instance_cost": 0.0, + "api_calls": 3 + }, + "config": { + "agent": { + "system_template": "You are a helpful assistant that can interact with a computer.\n", + "instance_template": "Please solve this issue: {{task}}\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\n{{system}} {{release}} {{version}} {{machine}}\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:\n\n{%- if system == \"Darwin\" -%}\n\nYou are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.\n\n{%- endif -%}\n\n```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```\n", + "step_limit": 0, + "cost_limit": 0.0, + "wall_time_limit_seconds": 180, + "max_consecutive_format_errors": 3, + "output_path": "logs\\p82-agent-preflight-openrouter-liquid-02\\agent-preflight-openrouter-liquid-02\\attempt-1\\trajectory.json" + }, + "agent_type": "minisweagent.agents.default.DefaultAgent", + "model": { + "model_name": "liquid/lfm-2.5-2.6b:free", + "model_kwargs": { + "drop_params": true, + "custom_llm_provider": "openrouter", + "provider": { + "allow_fallbacks": false + }, + "max_tokens": 64 + }, + "litellm_model_registry": null, + "set_cache_control": null, + "cost_tracking": "ignore_errors", + "format_error_template": "{% if finish_reason is defined and (finish_reason == \"length\" or (finish_reason == \"tool_calls\" and not has_tool_calls)) -%}\nYour previous response reached the output token limit (finish_reason={{ finish_reason }}) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.\n{%- else -%}\nTool call error:\n\n\n{{error}}\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.\n{%- endif %}\n", + "observation_template": "{%- if output.output | length < 10000 -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output\": {{ output.output | tojson }}\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- else -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output_head\": {{ output.output[:5000] | tojson }},\n \"output_tail\": {{ output.output[-5000:] | tojson }},\n \"elided_chars\": {{ output.output | length - 10000 }},\n \"warning\": \"Output too long.\"\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- endif -%}\n", + "multimodal_regex": "" + }, + "model_type": "minisweagent.models.litellm_model.LitellmModel", + "environment": { + "cwd": ".", + "env": { + "PAGER": "cat", + "MANPAGER": "cat", + "LESS": "-R", + "PIP_PROGRESS_BAR": "off", + "TQDM_DISABLE": "1" + }, + "timeout": 30 + }, + "environment_type": "minisweagent.environments.local.LocalEnvironment" + }, + "mini_version": "2.4.6", + "exit_status": "RepeatedFormatError", + "submission": "" + }, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: Use the bash tool to run exactly `echo PREFLIGHT_OK`, then finish the task.\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nWindows 10 10.0.26200 AMD64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + }, + { + "role": "user", + "content": "Your previous response reached the output token limit (finish_reason=length) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.", + "extra": { + "interrupt_type": "FormatError", + "cost": 0.0, + "response": { + "id": "gen-1790158826-smO9CNHdgrMrd3lo0Qol", + "created": 1790158826, + "model": "liquid/lfm-2.5-2.6b:free", + "object": "chat.completion", + "system_fingerprint": "vllm-0.29.1rc1.dev9+g2671fedfc-dafbf1d2", + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": null, + "function_call": null, + "reasoning_content": "The user wants me to solve an issue by using the bash tool to run `echo PREFLIGHT_OK`. But I need to first understand the context - there's no actual issue description provided beyond \"Use the bash tool to run exactly `echo PREFLIGHT_OK`, then finish the task.\"\n\nLet me start by", + "provider_specific_fields": { + "reasoning": "The user wants me to solve an issue by using the bash tool to run `echo PREFLIGHT_OK`. But I need to first understand the context - there's no actual issue description provided beyond \"Use the bash tool to run exactly `echo PREFLIGHT_OK`, then finish the task.\"\n\nLet me start by", + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user wants me to solve an issue by using the bash tool to run `echo PREFLIGHT_OK`. But I need to first understand the context - there's no actual issue description provided beyond \"Use the bash tool to run exactly `echo PREFLIGHT_OK`, then finish the task.\"\n\nLet me start by", + "format": "unknown", + "index": 0 + } + ], + "refusal": null + } + }, + "provider_specific_fields": { + "native_finish_reason": "length" + } + } + ], + "usage": { + "completion_tokens": 64, + "prompt_tokens": 794, + "total_tokens": 858, + "completion_tokens_details": { + "audio_tokens": 0, + "reasoning_tokens": 64, + "image_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 0, + "video_tokens": 0, + "cache_creation_tokens": 0 + }, + "cost": 0.0, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0, + "upstream_inference_completions_cost": 0 + } + }, + "provider": "Liquid", + "service_tier": null + } + } + }, + { + "role": "user", + "content": "Your previous response reached the output token limit (finish_reason=length) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.", + "extra": { + "interrupt_type": "FormatError", + "cost": 0.0, + "response": { + "id": "gen-1790158827-1lrsNys4oKHAFMKZOvTn", + "created": 1790158827, + "model": "liquid/lfm-2.5-2.6b:free", + "object": "chat.completion", + "system_fingerprint": "vllm-0.29.1rc1.dev9+g2671fedfc-dafbf1d2", + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": null, + "function_call": null, + "reasoning_content": "The user wants me to solve an issue by using the bash tool to run `echo PREFLIGHT_OK`. This seems like a simple task - I just need to execute that command. But let me first understand the context better. The user mentioned \"Use the bash tool to run exactly `echo PREFLIGHT_OK`", + "provider_specific_fields": { + "reasoning": "The user wants me to solve an issue by using the bash tool to run `echo PREFLIGHT_OK`. This seems like a simple task - I just need to execute that command. But let me first understand the context better. The user mentioned \"Use the bash tool to run exactly `echo PREFLIGHT_OK`", + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user wants me to solve an issue by using the bash tool to run `echo PREFLIGHT_OK`. This seems like a simple task - I just need to execute that command. But let me first understand the context better. The user mentioned \"Use the bash tool to run exactly `echo PREFLIGHT_OK`", + "format": "unknown", + "index": 0 + } + ], + "refusal": null + } + }, + "provider_specific_fields": { + "native_finish_reason": "length" + } + } + ], + "usage": { + "completion_tokens": 64, + "prompt_tokens": 850, + "total_tokens": 914, + "completion_tokens_details": { + "audio_tokens": 0, + "reasoning_tokens": 64, + "image_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 784, + "video_tokens": 0, + "cache_creation_tokens": 0 + }, + "cost": 0.0, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0, + "upstream_inference_completions_cost": 0 + } + }, + "provider": "Liquid", + "service_tier": null + } + } + }, + { + "role": "user", + "content": "Your previous response reached the output token limit (finish_reason=length) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.", + "extra": { + "interrupt_type": "FormatError", + "cost": 0.0, + "response": { + "id": "gen-1790158833-9uxOlOPJ9RprOzp0fPcV", + "created": 1790158833, + "model": "liquid/lfm-2.5-2.6b:free", + "object": "chat.completion", + "system_fingerprint": "vllm-0.29.1rc1.dev9+g2671fedfc-dafbf1d2", + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": null, + "function_call": null, + "reasoning_content": "The user wants me to solve an issue by using the bash tool to run `echo PREFLIGHT_OK`. This seems like a simple task - just execute that echo command. But I need to follow the workflow properly.\n\nLet me start by understanding what's being asked. The instruction says to use the bash tool to run", + "provider_specific_fields": { + "reasoning": "The user wants me to solve an issue by using the bash tool to run `echo PREFLIGHT_OK`. This seems like a simple task - just execute that echo command. But I need to follow the workflow properly.\n\nLet me start by understanding what's being asked. The instruction says to use the bash tool to run", + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user wants me to solve an issue by using the bash tool to run `echo PREFLIGHT_OK`. This seems like a simple task - just execute that echo command. But I need to follow the workflow properly.\n\nLet me start by understanding what's being asked. The instruction says to use the bash tool to run", + "format": "unknown", + "index": 0 + } + ], + "refusal": null + } + }, + "provider_specific_fields": { + "native_finish_reason": "length" + } + } + ], + "usage": { + "completion_tokens": 64, + "prompt_tokens": 906, + "total_tokens": 970, + "completion_tokens_details": { + "audio_tokens": 0, + "reasoning_tokens": 64, + "image_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 784, + "video_tokens": 0, + "cache_creation_tokens": 0 + }, + "cost": 0.0, + "is_byok": false, + "cost_details": { + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0, + "upstream_inference_completions_cost": 0 + } + }, + "provider": "Liquid", + "service_tier": null + } + } + }, + { + "role": "exit", + "content": "RepeatedFormatError", + "extra": { + "exit_status": "RepeatedFormatError", + "submission": "" + } + } + ], + "trajectory_format": "mini-swe-agent-1.1" +} \ No newline at end of file diff --git a/logs/p82-agent-preflight-openrouter-nemotron-01/agent-preflight-openrouter-nemotron-01/attempt-1/diff.patch b/logs/p82-agent-preflight-openrouter-nemotron-01/agent-preflight-openrouter-nemotron-01/attempt-1/diff.patch new file mode 100644 index 00000000..e69de29b diff --git a/logs/p82-agent-preflight-openrouter-nemotron-01/agent-preflight-openrouter-nemotron-01/attempt-1/execution.json b/logs/p82-agent-preflight-openrouter-nemotron-01/agent-preflight-openrouter-nemotron-01/attempt-1/execution.json new file mode 100644 index 00000000..026378a5 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-01/agent-preflight-openrouter-nemotron-01/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"logs\\p82-agent-preflight-openrouter-nemotron-01\\agent-preflight-openrouter-nemotron-01\\attempt-1\\execution.json","attempt":1,"diff_path":"logs\\p82-agent-preflight-openrouter-nemotron-01\\agent-preflight-openrouter-nemotron-01\\attempt-1\\diff.patch","exit_code":1,"failure":"executor returned non-zero","failure_category":"AGENT_RUN_FAILURE","finished_at":"2026-09-23T09:21:41.108+00:00","log_path":"logs\\p82-agent-preflight-openrouter-nemotron-01\\agent-preflight-openrouter-nemotron-01\\attempt-1\\executor.log","model_usage":null,"run_id":"cb345612f5b127e2","started_at":"2026-09-23T09:21:41.029+00:00","state":"EXECUTOR_FAILED","task_id":"agent-preflight-openrouter-nemotron-01","trajectory_path":"logs\\p82-agent-preflight-openrouter-nemotron-01\\agent-preflight-openrouter-nemotron-01\\attempt-1\\trajectory.json","treatment":"A"} diff --git a/logs/p82-agent-preflight-openrouter-nemotron-01/agent-preflight-openrouter-nemotron-01/attempt-1/executor.log b/logs/p82-agent-preflight-openrouter-nemotron-01/agent-preflight-openrouter-nemotron-01/attempt-1/executor.log new file mode 100644 index 00000000..51fd2d52 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-01/agent-preflight-openrouter-nemotron-01/attempt-1/executor.log @@ -0,0 +1,7 @@ +Traceback (most recent call last): + File "D:\projetos\arkx\tools\run_mini_headless.py", line 40, in + raise SystemExit(main()) + ^^^^^^ + File "D:\projetos\arkx\tools\run_mini_headless.py", line 18, in main + from minisweagent.agents import get_agent +ModuleNotFoundError: No module named 'minisweagent' diff --git a/logs/p82-agent-preflight-openrouter-nemotron-02/agent-preflight-openrouter-nemotron-02/attempt-1/execution.json b/logs/p82-agent-preflight-openrouter-nemotron-02/agent-preflight-openrouter-nemotron-02/attempt-1/execution.json new file mode 100644 index 00000000..4e307f77 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-02/agent-preflight-openrouter-nemotron-02/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"logs\\p82-agent-preflight-openrouter-nemotron-02\\agent-preflight-openrouter-nemotron-02\\attempt-1\\execution.json","attempt":1,"diff_path":null,"exit_code":null,"failure":"Command '['D:\\\\projetos\\\\arkx\\\\.venv-mini\\\\Scripts\\\\python.exe', 'D:\\\\projetos\\\\arkx\\\\tools\\\\run_mini_headless.py']' timed out after 180 seconds","failure_category":"TIMEOUT","finished_at":"2026-09-23T09:30:48.201+00:00","log_path":null,"model_usage":null,"run_id":"0a1785ce99c5e360","started_at":"2026-09-23T09:27:48.183+00:00","state":"EXECUTOR_FAILED","task_id":"agent-preflight-openrouter-nemotron-02","trajectory_path":null,"treatment":"A"} diff --git a/logs/p82-agent-preflight-openrouter-nemotron-03/agent-preflight-openrouter-nemotron-03/attempt-1/execution.json b/logs/p82-agent-preflight-openrouter-nemotron-03/agent-preflight-openrouter-nemotron-03/attempt-1/execution.json new file mode 100644 index 00000000..45f1edd8 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-03/agent-preflight-openrouter-nemotron-03/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"logs\\p82-agent-preflight-openrouter-nemotron-03\\agent-preflight-openrouter-nemotron-03\\attempt-1\\execution.json","attempt":1,"diff_path":null,"exit_code":null,"failure":"Command '['D:\\\\projetos\\\\arkx\\\\.venv-mini\\\\Scripts\\\\python.exe', 'D:\\\\projetos\\\\arkx\\\\tools\\\\run_mini_headless.py']' timed out after 190 seconds","failure_category":"TIMEOUT","finished_at":"2026-09-23T09:54:13.511+00:00","log_path":"logs\\p82-agent-preflight-openrouter-nemotron-03\\agent-preflight-openrouter-nemotron-03\\attempt-1\\executor.log","model_usage":null,"run_id":"01b2b10f27e44813","started_at":"2026-09-23T09:51:03.477+00:00","state":"EXECUTOR_FAILED","task_id":"agent-preflight-openrouter-nemotron-03","trajectory_path":null,"treatment":"A"} diff --git a/logs/p82-agent-preflight-openrouter-nemotron-03/agent-preflight-openrouter-nemotron-03/attempt-1/executor.log b/logs/p82-agent-preflight-openrouter-nemotron-03/agent-preflight-openrouter-nemotron-03/attempt-1/executor.log new file mode 100644 index 00000000..a21753f8 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-03/agent-preflight-openrouter-nemotron-03/attempt-1/executor.log @@ -0,0 +1,16 @@ +Retrying in 4 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 4 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 4 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 8 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 16 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 32 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 60 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 60 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. diff --git a/logs/p82-agent-preflight-openrouter-nemotron-04/agent-preflight-openrouter-nemotron-04/attempt-1/execution.json b/logs/p82-agent-preflight-openrouter-nemotron-04/agent-preflight-openrouter-nemotron-04/attempt-1/execution.json new file mode 100644 index 00000000..214c0683 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-04/agent-preflight-openrouter-nemotron-04/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"logs\\p82-agent-preflight-openrouter-nemotron-04\\agent-preflight-openrouter-nemotron-04\\attempt-1\\execution.json","attempt":1,"diff_path":null,"exit_code":null,"failure":"Command '['D:\\\\projetos\\\\arkx\\\\.venv-mini\\\\Scripts\\\\python.exe', 'D:\\\\projetos\\\\arkx\\\\tools\\\\run_mini_headless.py']' timed out after 190 seconds","failure_category":"TIMEOUT","finished_at":"2026-09-23T09:57:59.349+00:00","log_path":"logs\\p82-agent-preflight-openrouter-nemotron-04\\agent-preflight-openrouter-nemotron-04\\attempt-1\\executor.log","model_usage":null,"run_id":"766d7d896e9b9e05","started_at":"2026-09-23T09:54:49.332+00:00","state":"EXECUTOR_FAILED","task_id":"agent-preflight-openrouter-nemotron-04","trajectory_path":null,"treatment":"A"} diff --git a/logs/p82-agent-preflight-openrouter-nemotron-04/agent-preflight-openrouter-nemotron-04/attempt-1/executor.log b/logs/p82-agent-preflight-openrouter-nemotron-04/agent-preflight-openrouter-nemotron-04/attempt-1/executor.log new file mode 100644 index 00000000..a21753f8 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-04/agent-preflight-openrouter-nemotron-04/attempt-1/executor.log @@ -0,0 +1,16 @@ +Retrying in 4 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 4 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 4 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 8 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 16 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 32 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 60 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. +Retrying in 60 seconds as it raised BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=nvidia/nemotron-3-ultra-550b-a55b:free + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers. diff --git a/logs/p82-agent-preflight-openrouter-nemotron-05/agent-preflight-openrouter-nemotron-05/attempt-1/execution.json b/logs/p82-agent-preflight-openrouter-nemotron-05/agent-preflight-openrouter-nemotron-05/attempt-1/execution.json new file mode 100644 index 00000000..acda41fc --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-05/agent-preflight-openrouter-nemotron-05/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"logs\\p82-agent-preflight-openrouter-nemotron-05\\agent-preflight-openrouter-nemotron-05\\attempt-1\\execution.json","attempt":1,"diff_path":null,"exit_code":null,"failure":"Command '['D:\\\\projetos\\\\arkx\\\\.venv-mini\\\\Scripts\\\\python.exe', 'D:\\\\projetos\\\\arkx\\\\tools\\\\run_mini_headless.py']' timed out after 190 seconds","failure_category":"TIMEOUT","finished_at":"2026-09-23T10:01:47.168+00:00","log_path":"logs\\p82-agent-preflight-openrouter-nemotron-05\\agent-preflight-openrouter-nemotron-05\\attempt-1\\executor.log","model_usage":null,"run_id":"f081fe6789bb90a9","started_at":"2026-09-23T09:58:37.154+00:00","state":"EXECUTOR_FAILED","task_id":"agent-preflight-openrouter-nemotron-05","trajectory_path":null,"treatment":"A"} diff --git a/logs/p82-agent-preflight-openrouter-nemotron-05/agent-preflight-openrouter-nemotron-05/attempt-1/executor.log b/logs/p82-agent-preflight-openrouter-nemotron-05/agent-preflight-openrouter-nemotron-05/attempt-1/executor.log new file mode 100644 index 00000000..e69de29b diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/diff.patch b/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/diff.patch new file mode 100644 index 00000000..5cc4cdda --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/diff.patch @@ -0,0 +1,619 @@ +diff --git a/README.md b/README.md +index 13041af..c729c17 100644 +--- a/README.md ++++ b/README.md +@@ -1,26 +1,29 @@ + # Arkx + +-Arkx is a greenfield project for empirical, incremental development of reliable capability. ++Arkx is a repository for incremental implementation and empirical evaluation of software capability. + + > **MINIMUM SUFFICIENT ARCHITECTURE** + > **FOR MAXIMUM RELIABLE CAPABILITY** + +-This repository is the canonical root for the project. It intentionally contains only the foundation in this bootstrap: the project contract, experimental protocol, structural placeholders, and a minimal CI check. ++This repository is the canonical root for the project. It contains runtime ++contracts, an execution harness, the project contract, the experimental ++protocol, and structural checks. + +-## Scope of this bootstrap ++## Current scope + + Included: + + - explicit project invariants; +-- a small, evidence-oriented experimental protocol; +-- stable top-level boundaries for future work; +-- a deterministic structural check and minimal CI. ++- an evidence-oriented experimental protocol; ++- stable top-level boundaries; ++- a deterministic structural check, executable contracts, and a Treatment A ++ baseline harness. + +-Deferred: ++Still deferred or unqualified: + +-- routing and planning; + - multi-agent orchestration; +-- OpenHands, ReX, mini-SWE-agent, and SWE-agent integrations; ++- OpenHands, ReX, and SWE-agent integrations; ++- real provider/model adoption and the A/B/C/D mechanism trial; + - product dependencies or automatic migration from `smag-rex`. + + ## Development posture +@@ -44,9 +47,9 @@ P1 adds deterministic task characterization from explicit signals only. See [the + + P2 adds deterministic progress/stagnation assessment from explicit snapshots only. See [the P2 contract](docs/progress-stagnation.md). It does not control execution or authorize retry/replan. + +-P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke or select concrete executors. ++P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke concrete executors. + +-Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. None executes a plan, owns acceptance, or creates orchestration. ++Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. P8.2a adds a Treatment A baseline runner and official SWE-bench acceptance adapter; neither owns general orchestration or promotion. + + ## Checks + +diff --git a/docs/architecture.md b/docs/architecture.md +index 17bbba6..98d1e6a 100644 +--- a/docs/architecture.md ++++ b/docs/architecture.md +@@ -1,24 +1,75 @@ + # Architecture boundary + +-The bootstrap architecture is intentionally small: ++Arkx separates contracts, execution, verification, acceptance, and promotion. ++The executable contracts cover the request-to-acceptance path. Runtime ++capability is qualified per provider, model, executor, and treatment; the ++presence of a contract does not imply adoption. + + ```text + Arkx +-├── docs/ contracts, protocol, and decisions +-├── experiments/ experiment records and fixtures; no runtime yet +-├── src/ reserved product boundary; no implementation yet +-├── tests/ reserved verification boundary; no test suite yet +-└── tools/ dependency-free repository checks ++├── docs/ contracts, protocol, decisions, and evidence boundaries ++├── experiments/ experiment records and synthetic fixtures; not orchestration ++├── src/arkx/ executable contracts and bounded policy/runtime surfaces ++├── tests/ executable contract and behavior verification ++└── tools/ repository checks and narrow execution adapters + ``` + +-P0 adds a deliberately small executable core under `src/arkx/`: versioned contracts, evidence status resolution, telemetry collection, and a deterministic baseline fixture. P1 adds explicit-signal task characterization, P2 adds snapshot-based progress assessment, P3 adds bounded routing/escalation policy, and Block B adds independent P4/P5/P6 artifacts in the same boundary. No layer has a concrete executor interface or executor orchestration. ++The current executable boundary is layered as follows: ++ ++- P0: versioned contracts, evidence status resolution, telemetry collection, ++ and a deterministic baseline fixture; ++- P1: explicit-signal task characterization; ++- P2/P3: progress assessment, bounded routing, and evidence-based escalation; ++- P4/P5/P6: repository planning/state, patch verification, and handoff/context ++ accounting; ++- P7/P8.1/P8.2: composition records, executor qualification contracts, and ++ controlled treatment plans; ++- P8.2a: a narrow Treatment A baseline runner and the official SWE-bench ++ acceptance adapter; ++- current orchestration surfaces: request governance, treatment/executor ++ selection, sequential execution, recovery, independent acceptance, and ++ promotion gates. ++ ++The canonical flow is: ++ ++```text ++request ++ -> governance / authority ++ -> task characterization ++ -> treatment route ++ -> sufficient executor ++ -> progress / recovery ++ -> verification ++ -> independent acceptance ++ -> evidence for possible promotion ++``` ++ ++Routing selects the minimum sufficient treatment before selecting an executor. ++Simple, localized, and repository-wide paths are capabilities/regimes, not ++automatic executor cascades. A fallback, executor switch, scope expansion, ++acceptance, or promotion must remain explicit and evidence-backed. ++ ++The official SWE-bench Docker authority has been qualified on the frozen Gold ++tasks. That validates the acceptance path and infrastructure, not a real ++provider/model, Treatment A execution, or A/B/C/D mechanism evidence. + + ## Boundary rules + +-- `src/` is reserved for product code and is empty in this bootstrap. ++- `src/arkx/` contains product contracts and bounded policy/runtime code; it ++ must not silently grow into an unqualified general-purpose orchestrator. + - `experiments/` contains evidence-oriented records, not an implicit runtime. +-- `tools/` may validate repository contracts but must not become orchestration. +-- `tests/` contains P0 executable verification and remains the boundary for future tests. +-- Integrations and executors require an explicit decision record and experiment evidence. ++- `tools/` may validate contracts or host a narrow adapter, but must not become ++ an untracked orchestration layer. ++- `tests/` verifies contracts and behavior; passing tests are local evidence, ++ not scientific or upstream evidence. ++- Integrations, providers, models, and executors require explicit identity, ++ qualification evidence, and an acceptance authority. ++- Missing provider/model identity, missing evidence, and blocked execution are ++ preserved as blocked or unknown; they never become success. ++- The working tree may contain an implementation under qualification. Merge ++ readiness requires the corresponding decision record, tests, and evidence to ++ be committed together. + +-An empty boundary is intentional. Creating a directory does not imply that the corresponding capability exists. ++The boundary is explicit: creating a module, fixture, or directory does not by ++itself prove that the corresponding capability exists or is suitable for ++promotion. +diff --git a/docs/experimental-protocol.md b/docs/experimental-protocol.md +index f24bf94..2b0f92d 100644 +--- a/docs/experimental-protocol.md ++++ b/docs/experimental-protocol.md +@@ -1,6 +1,7 @@ + # Experimental protocol + +-Arkx develops through small, empirical increments. The protocol below is the minimum record required before a capability can be considered for promotion. ++Arkx develops through empirical increments. The protocol below defines the ++minimum record required before a capability can be considered for promotion. + + ## Required record + +@@ -14,6 +15,9 @@ Each experiment records: + - `procedure`: reproducible steps; + - `expected_signal`: measurable success and failure criteria; + - `result`: `PASS`, `FAIL`, `BLOCKED`, or `NOT_EXECUTED`; ++- `evidence`: links or paths to raw evidence and environment facts; ++- `acceptance`: explicit reviewer decision and rationale; ++- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. + + P7 composition trials use their own comparability namespace: `COMPARABLE`, `INCOMPARABLE`, and `UNKNOWN`. P7 records externally supplied measurements and does not convert them into acceptance results. + +@@ -26,9 +30,14 @@ environment, budget, verification, and independent acceptance authority. It + does not invoke an executor or mechanism and does not manufacture acceptance; + results remain P8.1 observations. Missing controls and measurements remain + `UNKNOWN`, and the only intended treatment difference is the declared arm. +-- `evidence`: links or paths to raw evidence and environment facts; +-- `acceptance`: explicit reviewer decision and rationale; +-- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. ++ ++P8.2a adds the Treatment A baseline runner. Each attempt is serial, refuses to ++overwrite an existing attempt directory, derives a stable run identity from ++the task and frozen execution configuration, and writes an `execution.json` ++artifact atomically alongside the log, diff, and trajectory. The artifact ++records execution only; verification, independent acceptance, and promotion ++remain separate records. A missing provider, model, mini version, or Python ++executable blocks the attempt instead of selecting a fallback. + + ## State separation + +diff --git a/docs/p82-minimal-mechanism-trial.md b/docs/p82-minimal-mechanism-trial.md +index b05ef93..43225e8 100644 +--- a/docs/p82-minimal-mechanism-trial.md ++++ b/docs/p82-minimal-mechanism-trial.md +@@ -34,8 +34,8 @@ promotion decision. + + ## P8.2a baseline execution path + +-P8.2a is infrastructure qualification, not mechanism evidence. The baseline +-runner accepts treatment A only, invokes mini-SWE-agent through its public ++P8.2a records infrastructure qualification, not mechanism evidence. The ++baseline runner accepts treatment A only, invokes mini-SWE-agent through its public + Python API in a headless worker, and records execution artifacts. It does not + run verification or decide acceptance. Verification and independent + acceptance are explicit, nullable records; missing provider/model identity is +@@ -49,7 +49,9 @@ sample is frozen in `experiments/p82-wave0-task-sample.json` from the official + + The official acceptance adapter is in `arkx.swebench_authority`. It creates a + SWE-bench prediction without gold patch or grading labels and invokes the +-official harness with a unique `run_id`. Gold qualification and real A runs +-remain blocked until `swebench` and Docker are available, the gold harness +-passes, and one exact provider/model/configuration is frozen. B/C/D and any ++official harness with a unique `run_id`. The frozen Docker Gold authority ++returned `resolved: true` for the two Wave 0 tasks recorded under ++`logs/evaluation/`; this qualifies the acceptance path and infrastructure only. ++Real A runs still ++require one exact provider/model/configuration to be frozen. B/C/D and any + mechanism comparison remain out of scope. +diff --git a/docs/project-contract.md b/docs/project-contract.md +index f2f734a..82054a4 100644 +--- a/docs/project-contract.md ++++ b/docs/project-contract.md +@@ -40,7 +40,9 @@ NO_SILENT_SCOPE_EXPANSION + 5. Any fallback, executor change, or scope change is explicit in the record and review. + 6. New architecture requires a decision record describing the observed need, alternatives, evidence, and rollback/removal condition. + +-## Non-goals of this bootstrap +- +-This repository does not yet define routing, planning, multi-agent execution, or integrations with OpenHands, ReX, mini-SWE-agent, or SWE-agent. ++## Current non-goals + ++The repository now defines bounded routing, planning/state, verification, ++recovery, acceptance, promotion gates, and a mini-SWE-agent Treatment A ++baseline path. It does not define general multi-agent execution, automatic ++executor adoption, or integrations with OpenHands, ReX, and SWE-agent. +diff --git a/experiments/README.md b/experiments/README.md +index 551f37e..fe93b83 100644 +--- a/experiments/README.md ++++ b/experiments/README.md +@@ -2,3 +2,4 @@ + + This directory is reserved for explicit experiment records and small fixtures. Use [the experimental protocol](../docs/experimental-protocol.md). No experiment has been executed by the bootstrap. + ++The exploratory local-model qualification wave is documented in [local model qualification](../docs/local-model-qualification.md). It was run outside the Arkx experiment harness and must not be read as executor adoption or acceptance evidence. +diff --git a/src/arkx/__init__.py b/src/arkx/__init__.py +index 39c7d6c..2508056 100644 +--- a/src/arkx/__init__.py ++++ b/src/arkx/__init__.py +@@ -1,6 +1,62 @@ + """Minimal, executor-agnostic Arkx execution contracts and telemetry.""" + + from .contracts import Event, EventType, ExecutionRecord, ExecutionStatus ++from .request import ( ++ ArkxRequest, ++ GovernanceDecision, ++ GovernancePolicy, ++ GovernanceReason, ++ GovernanceStatus, ++ RequestBudget, ++ RequestEnvironment, ++ authorize_request, ++ characterize_authorized_request, ++) ++from .selection import ( ++ ExecutorBinding, ++ ExecutorRegistry, ++ SelectionDecision, ++ SelectionReason, ++ SelectionStatus, ++ TreatmentCatalog, ++ TreatmentDefinition, ++ select_executor, ++ select_treatment, ++) ++from .p82_localization import ( ++ LocalizationArtifact, ++ LocalizationEvidenceBudget, ++ record_localization_artifact, ++) ++from .p82_editing import ( ++ EditOperation, ++ StructuredEditProposal, ++ record_edit_proposal, ++) ++from .orchestration import ( ++ ExecutionOutcome, ++ ExecutionResult, ++ OrchestrationReason, ++ OrchestrationResult, ++ OrchestrationStatus, ++ run_routed_pipeline, ++) ++from .recovery import ( ++ RecoveryAction, ++ RecoveryBudget, ++ RecoveryDecision, ++ RecoveryReason, ++ decide_recovery, ++) ++from .acceptance import AcceptancePolicy, decide_independent_acceptance ++from .promotion import ( ++ PromotionCandidate, ++ PromotionDecision, ++ PromotionPolicy, ++ PromotionReason, ++ PromotionStatus, ++ decide_promotion, ++) + from .characterization import ( + CharacterizationConfig, + Confidence, +@@ -108,6 +164,43 @@ __all__ = [ + "EventType", + "ExecutionRecord", + "ExecutionStatus", ++ "ArkxRequest", ++ "GovernanceDecision", ++ "GovernancePolicy", ++ "GovernanceReason", ++ "GovernanceStatus", ++ "RequestBudget", ++ "RequestEnvironment", ++ "authorize_request", ++ "characterize_authorized_request", ++ "ExecutorBinding", ++ "ExecutorRegistry", ++ "SelectionDecision", ++ "SelectionReason", ++ "SelectionStatus", ++ "TreatmentCatalog", ++ "TreatmentDefinition", ++ "select_executor", ++ "select_treatment", ++ "ExecutionOutcome", ++ "ExecutionResult", ++ "OrchestrationReason", ++ "OrchestrationResult", ++ "OrchestrationStatus", ++ "run_routed_pipeline", ++ "RecoveryAction", ++ "RecoveryBudget", ++ "RecoveryDecision", ++ "RecoveryReason", ++ "decide_recovery", ++ "AcceptancePolicy", ++ "decide_independent_acceptance", ++ "PromotionCandidate", ++ "PromotionDecision", ++ "PromotionPolicy", ++ "PromotionReason", ++ "PromotionStatus", ++ "decide_promotion", + "CharacterizationConfig", + "Confidence", + "RecommendedPath", +diff --git a/src/arkx/p82_baseline.py b/src/arkx/p82_baseline.py +index 9cd9d72..cd192fa 100644 +--- a/src/arkx/p82_baseline.py ++++ b/src/arkx/p82_baseline.py +@@ -138,6 +138,26 @@ class BaselineRunConfig: + def missing_identity(self) -> tuple[str, ...]: + return tuple(name for name in ("provider", "model", "mini_version", "python_executable") if getattr(self, name) is None) + ++ def identity_digest(self) -> str: ++ """Return a stable digest for the frozen execution configuration.""" ++ value = { ++ "provider": self.provider, ++ "model": self.model, ++ "model_version": self.model_version, ++ "temperature": self.temperature, ++ "reasoning_configuration": self.reasoning_configuration, ++ "model_configuration": self.model_configuration, ++ "max_tokens": self.max_tokens, ++ "max_cost": self.max_cost, ++ "wall_time_seconds": self.wall_time_seconds, ++ "mini_version": self.mini_version, ++ "python_executable": self.python_executable, ++ "runner_version": self.runner_version, ++ "treatment": self.treatment.value, ++ } ++ encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:16] ++ + + @dataclass(frozen=True) + class ExecutionArtifact: +@@ -155,16 +175,21 @@ class ExecutionArtifact: + log_path: str | None + trajectory_path: str | None + model_usage: dict[str, Any] | None ++ artifact_path: str | None = None + + def to_dict(self) -> dict[str, Any]: + value = {name: getattr(self, name) for name in ( + "run_id", "state", "task_id", "treatment", "attempt", "started_at", "finished_at", +- "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", "model_usage")} ++ "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", ++ "model_usage", "artifact_path")} + for name in ("state", "failure_category"): + if isinstance(value[name], Enum): + value[name] = value[name].value + return value + ++ def to_json(self) -> str: ++ return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ + + class HeadlessInvoker(Protocol): + def __call__(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: ... +@@ -185,7 +210,9 @@ class MiniSweAgentHeadlessRunner: + raise ValueError("attempt must be positive") + missing = config.missing_identity() + root = Path(config.artifact_root) / task.task_id / f"attempt-{attempt}" +- run_id = hashlib.sha256(f"{task.task_id}|A|{attempt}|{config.runner_version}".encode()).hexdigest()[:16] ++ run_id = hashlib.sha256( ++ f"{task.task_id}|{task.base_commit}|A|{attempt}|{config.identity_digest()}".encode("utf-8") ++ ).hexdigest()[:16] + if root.exists(): + return self._blocked(run_id, task, attempt, "artifact already exists; refusing silent overwrite") + root.mkdir(parents=True) +@@ -204,7 +231,16 @@ class MiniSweAgentHeadlessRunner: + state = BaselineExecutionState.COMPLETED if exit_code == 0 else BaselineExecutionState.EXECUTOR_FAILED + return self._finish(root, run_id, task, attempt, started, state, exit_code, None if exit_code == 0 else "executor returned non-zero", None if exit_code == 0 else RunnerFailureCategory.AGENT_RUN_FAILURE, str(diff_path), str(log_path), str(trajectory), usage) + except subprocess.TimeoutExpired as error: +- return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, str(error), RunnerFailureCategory.TIMEOUT, None, None, None, None) ++ log_path = root / "executor.log" ++ stdout = self._decode_process_output(error.stdout) ++ stderr = self._decode_process_output(error.stderr) ++ log_path.write_text(stdout + stderr, encoding="utf-8", newline="\n") ++ return self._finish( ++ root, run_id, task, attempt, started, ++ BaselineExecutionState.EXECUTOR_FAILED, None, str(error), ++ RunnerFailureCategory.TIMEOUT, None, str(log_path), ++ str(trajectory) if trajectory.exists() else None, None, ++ ) + except Exception as error: + message = f"{type(error).__name__}: {error}" + return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, message, self._classify_failure(message), None, None, None, None) +@@ -228,21 +264,47 @@ class MiniSweAgentHeadlessRunner: + return RunnerFailureCategory.DEADLOCK_HANG + return RunnerFailureCategory.AGENT_RUN_FAILURE + ++ @staticmethod ++ def _decode_process_output(value: str | bytes | None) -> str: ++ if value is None: ++ return "" ++ if isinstance(value, bytes): ++ return value.decode("utf-8", errors="replace") ++ return value ++ + def _subprocess_invoker(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: + worker = Path(__file__).parents[2] / "tools" / "run_mini_headless.py" + payload = {"task": task.problem_statement, "workspace": task.workspace, "model": config.model, + "trajectory": str(trajectory_path), "max_cost": config.max_cost, "wall_time_seconds": config.wall_time_seconds, + "model_configuration": config.model_configuration} +- completed = subprocess.run([config.python_executable, str(worker)], input=json.dumps(payload), text=True, +- capture_output=True, cwd=task.workspace, timeout=config.wall_time_seconds) +- return completed.returncode, completed.stdout + completed.stderr, None ++ timeout = None if config.wall_time_seconds is None else config.wall_time_seconds + 10 ++ completed = subprocess.run( ++ [config.python_executable, str(worker)], input=json.dumps(payload), text=True, ++ capture_output=True, cwd=task.workspace, timeout=timeout, ++ ) ++ combined_log = completed.stdout + completed.stderr ++ usage = None ++ if completed.returncode == 0: ++ lines = [line for line in completed.stdout.splitlines() if line.strip()] ++ if lines: ++ try: ++ worker_result = json.loads(lines[-1]) ++ usage = worker_result.get("usage") ++ except (json.JSONDecodeError, AttributeError): ++ usage = None ++ return completed.returncode, combined_log, usage + + def _blocked(self, run_id: str, task: ProspectiveTask, attempt: int, failure: str) -> ExecutionArtifact: + now = _now() + return ExecutionArtifact(run_id, BaselineExecutionState.BLOCKED, task.task_id, "A", attempt, now, now, None, failure, RunnerFailureCategory.UNKNOWN, None, None, None, None) + + def _finish(self, root: Path, run_id: str, task: ProspectiveTask, attempt: int, started: str, state: BaselineExecutionState, exit_code: int | None, failure: str | None, failure_category: RunnerFailureCategory | None, diff_path: str | None, log_path: str | None, trajectory_path: str | None, usage: dict[str, Any] | None) -> ExecutionArtifact: +- return ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage) ++ artifact_path = root / "execution.json" ++ artifact = ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage, str(artifact_path)) ++ temporary_path = root / "execution.json.tmp" ++ temporary_path.write_text(artifact.to_json() + "\n", encoding="utf-8", newline="\n") ++ temporary_path.replace(artifact_path) ++ return artifact + + + @dataclass(frozen=True) +diff --git a/tests/test_p82_baseline.py b/tests/test_p82_baseline.py +index d7d45ab..8ddf91f 100644 +--- a/tests/test_p82_baseline.py ++++ b/tests/test_p82_baseline.py +@@ -1,6 +1,9 @@ + import tempfile + import unittest ++import json ++import subprocess + from pathlib import Path ++from unittest.mock import patch + + from arkx.p82 import P82RunStatus, P82Treatment, TaskStratum + from arkx.p82_baseline import ( +@@ -40,6 +43,43 @@ def config(root, **overrides): + + + class P82BaselineTests(unittest.TestCase): ++ def test_subprocess_invoker_passes_limits_and_captures_usage(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ runner = MiniSweAgentHeadlessRunner() ++ completed = type("Completed", (), { ++ "returncode": 0, ++ "stdout": '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\n', ++ "stderr": "worker log\n", ++ })() ++ with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: ++ usage = runner._subprocess_invoker(task(workspace), config(directory), Path(directory) / "trajectory.json") ++ self.assertEqual(usage, (0, '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\nworker log\n', {"model_calls": 3, "cost": 0.25})) ++ self.assertEqual(mocked.call_args.kwargs["timeout"], 20) ++ payload = json.loads(mocked.call_args.kwargs["input"]) ++ self.assertEqual(payload["max_cost"], "1") ++ self.assertEqual(payload["wall_time_seconds"], 10) ++ ++ def test_timeout_preserves_partial_output_and_trajectory(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ artifact_root = Path(directory) / "artifacts" ++ trajectory = artifact_root / "task-1" / "attempt-1" / "trajectory.json" ++ ++ def timed_out_invoker(run_task, run_config, trajectory_path): ++ trajectory_path.write_text('{"partial":true}', encoding="utf-8") ++ raise subprocess.TimeoutExpired( ++ cmd="worker", timeout=20, output=b"partial stdout\n", stderr=b"partial stderr\n" ++ ) ++ ++ result = MiniSweAgentHeadlessRunner(timed_out_invoker).run(task(workspace), config(artifact_root)) ++ self.assertEqual(result.failure_category, RunnerFailureCategory.TIMEOUT) ++ self.assertTrue(Path(result.log_path).exists()) ++ self.assertEqual(Path(result.log_path).read_text(encoding="utf-8"), "partial stdout\npartial stderr\n") ++ self.assertEqual(Path(result.trajectory_path).read_text(encoding="utf-8"), '{"partial":true}') ++ + def test_headless_runner_uses_fake_invoker_and_captures_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" +@@ -56,8 +96,23 @@ class P82BaselineTests(unittest.TestCase): + self.assertEqual(calls, [("task-1", P82Treatment.A, "trajectory.json")]) + self.assertTrue(Path(result.log_path).exists()) + self.assertTrue(Path(result.diff_path).exists()) ++ self.assertTrue(Path(result.artifact_path).exists()) ++ persisted = json.loads(Path(result.artifact_path).read_text(encoding="utf-8")) ++ self.assertEqual(persisted, result.to_dict()) ++ self.assertEqual(result.model_usage, {"calls": 1}) + self.assertEqual(result.treatment, "A") + ++ def test_run_identity_changes_when_frozen_model_configuration_changes(self): ++ with tempfile.TemporaryDirectory() as directory: ++ first = config(directory, model="model-a") ++ second = config(directory, model="model-b") ++ self.assertNotEqual(first.identity_digest(), second.identity_digest()) ++ ++ runner = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)) ++ first_result = runner.run(task(directory), first, attempt=1) ++ second_result = runner.run(task(directory), second, attempt=2) ++ self.assertNotEqual(first_result.run_id, second_result.run_id) ++ + def test_missing_model_or_provider_is_blocked_without_fallback(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner().run(task(directory), config(directory, model=None)) +diff --git a/tools/run_mini_headless.py b/tools/run_mini_headless.py +index eedd60b..d501982 100644 +--- a/tools/run_mini_headless.py ++++ b/tools/run_mini_headless.py +@@ -22,17 +22,37 @@ def main() -> int: + from minisweagent.utils.serialize import recursive_merge, UNSET + + config = get_config_from_spec("mini.yaml") +- config = recursive_merge(config, { +- "run": {"task": payload["task"] or UNSET}, +- "agent": {"mode": "yolo", "confirm_exit": False, "output_path": Path(payload["trajectory"])}, +- "model": {"model_name": payload["model"], "model_class": payload.get("model_class", UNSET), **(payload.get("model_configuration") or payload.get("model_kwargs") or {})}, +- "environment": {"cwd": payload["workspace"]}, +- }) ++ config = recursive_merge( ++ config, ++ { ++ "run": {"task": payload["task"] or UNSET}, ++ "agent": { ++ "mode": "yolo", ++ "confirm_exit": False, ++ "output_path": Path(payload["trajectory"]), ++ "cost_limit": None if payload["max_cost"] is None else float(payload["max_cost"]), ++ "wall_time_limit_seconds": None if payload["wall_time_seconds"] is None else int(payload["wall_time_seconds"]), ++ }, ++ "model": { ++ "model_name": payload["model"], ++ "model_class": payload.get("model_class", UNSET), ++ **(payload.get("model_configuration") or payload.get("model_kwargs") or {}), ++ }, ++ "environment": {"cwd": payload["workspace"]}, ++ }, ++ ) + model = get_model(config=config["model"]) + env = get_environment(config["environment"], default_type="local") + agent = get_agent(model, env, config["agent"], default_type="default") + result = agent.run(payload["task"] or "") +- print(json.dumps(result, ensure_ascii=False, sort_keys=True)) ++ output = { ++ "result": result, ++ "usage": { ++ "model_calls": getattr(agent, "n_calls", None), ++ "cost": getattr(agent, "cost", None), ++ }, ++ } ++ print(json.dumps(output, ensure_ascii=False, sort_keys=True)) + return 0 + + diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/execution.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/execution.json new file mode 100644 index 00000000..8945be94 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-retry-01\\agent-preflight-openrouter-nemotron-ultra-retry-01\\attempt-1\\execution.json","attempt":1,"diff_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-retry-01\\agent-preflight-openrouter-nemotron-ultra-retry-01\\attempt-1\\diff.patch","exit_code":1,"failure":"executor returned non-zero","failure_category":"AGENT_RUN_FAILURE","finished_at":"2026-09-23T10:55:08.473+00:00","log_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-retry-01\\agent-preflight-openrouter-nemotron-ultra-retry-01\\attempt-1\\executor.log","model_usage":null,"run_id":"88d482cc66fd8e16","started_at":"2026-09-23T10:54:59.964+00:00","state":"EXECUTOR_FAILED","task_id":"agent-preflight-openrouter-nemotron-ultra-retry-01","trajectory_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-retry-01\\agent-preflight-openrouter-nemotron-ultra-retry-01\\attempt-1\\trajectory.json","treatment":"A"} diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/executor.log b/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/executor.log new file mode 100644 index 00000000..be557a0e --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/executor.log @@ -0,0 +1,22 @@ +Traceback (most recent call last): + File "D:\projetos\arkx\tools\run_mini_headless.py", line 60, in + raise SystemExit(main()) + ^^^^^^ + File "D:\projetos\arkx\tools\run_mini_headless.py", line 47, in main + result = agent.run(payload["task"] or "") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\agents\default.py", line 98, in run + self.step() + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\agents\default.py", line 128, in step + return self.execute_actions(self.query()) + ^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\agents\default.py", line 149, in query + message = self.model.query(self.messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\models\openrouter_model.py", line 104, in query + actions = self._parse_actions(response) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\models\openrouter_model.py", line 134, in _parse_actions + tool_calls = response["choices"][0]["message"].get("tool_calls") or [] + ~~~~~~~~^^^^^^^^^^^ +KeyError: 'choices' diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/trajectory.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/trajectory.json new file mode 100644 index 00000000..73013ffd --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-retry-01/agent-preflight-openrouter-nemotron-ultra-retry-01/attempt-1/trajectory.json @@ -0,0 +1,185 @@ +{ + "info": { + "model_stats": { + "instance_cost": 0.0, + "api_calls": 2 + }, + "config": { + "agent": { + "system_template": "You are a helpful assistant that can interact with a computer.\n", + "instance_template": "Please solve this issue: {{task}}\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\n{{system}} {{release}} {{version}} {{machine}}\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:\n\n{%- if system == \"Darwin\" -%}\n\nYou are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.\n\n{%- endif -%}\n\n```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```\n", + "step_limit": 0, + "cost_limit": 0.0, + "wall_time_limit_seconds": 180, + "max_consecutive_format_errors": 3, + "output_path": "D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-retry-01\\agent-preflight-openrouter-nemotron-ultra-retry-01\\attempt-1\\trajectory.json" + }, + "agent_type": "minisweagent.agents.default.DefaultAgent", + "model": { + "model_name": "nvidia/nemotron-3-ultra-550b-a55b:free", + "model_kwargs": { + "drop_params": true, + "temperature": 0.0, + "max_tokens": 256, + "custom_llm_provider": "openrouter", + "provider": { + "allow_fallbacks": false + } + }, + "set_cache_control": null, + "cost_tracking": "ignore_errors", + "format_error_template": "{% if finish_reason is defined and (finish_reason == \"length\" or (finish_reason == \"tool_calls\" and not has_tool_calls)) -%}\nYour previous response reached the output token limit (finish_reason={{ finish_reason }}) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.\n{%- else -%}\nTool call error:\n\n\n{{error}}\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.\n{%- endif %}\n", + "observation_template": "{%- if output.output | length < 10000 -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output\": {{ output.output | tojson }}\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- else -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output_head\": {{ output.output[:5000] | tojson }},\n \"output_tail\": {{ output.output[-5000:] | tojson }},\n \"elided_chars\": {{ output.output | length - 10000 }},\n \"warning\": \"Output too long.\"\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- endif -%}\n", + "multimodal_regex": "" + }, + "model_type": "minisweagent.models.openrouter_model.OpenRouterModel", + "environment": { + "cwd": "D:\\projetos\\arkx\\.p82-local-preflight-nemotron-ultra-workspace", + "env": { + "PAGER": "cat", + "MANPAGER": "cat", + "LESS": "-R", + "PIP_PROGRESS_BAR": "off", + "TQDM_DISABLE": "1" + }, + "timeout": 30 + }, + "environment_type": "minisweagent.environments.local.LocalEnvironment" + }, + "mini_version": "2.4.6", + "exit_status": "KeyError", + "submission": "" + }, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: Use the bash tool to create target.py containing exactly: def answer():\\n return 'ARKX_PREFLIGHT_OK'\\n. Then finish the task.\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nWindows 10 10.0.26200 AMD64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + }, + { + "role": "assistant", + "content": "I need to create a file called `target.py` with the exact content specified. Let me do this using bash.\n", + "refusal": null, + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "call-de56ecbd-a0ae-4b84-b6a6-f415194b4a41", + "function": { + "name": "bash", + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "format": "unknown", + "index": 0 + } + ], + "extra": { + "actions": [ + { + "command": "cat <<'EOF' > target.py\ndef answer():\n return 'ARKX_PREFLIGHT_OK'\nEOF", + "tool_call_id": "call-de56ecbd-a0ae-4b84-b6a6-f415194b4a41" + } + ], + "response": { + "id": "gen-1790160901-NE5pSsB97pwSThcKLEqm", + "object": "chat.completion", + "created": 1790160901, + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "provider": "Nvidia", + "system_fingerprint": null, + "service_tier": null, + "choices": [ + { + "index": 0, + "logprobs": null, + "finish_reason": "tool_calls", + "native_finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "I need to create a file called `target.py` with the exact content specified. Let me do this using bash.\n", + "refusal": null, + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "call-de56ecbd-a0ae-4b84-b6a6-f415194b4a41", + "function": { + "name": "bash", + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "format": "unknown", + "index": 0 + } + ] + } + } + ], + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 124, + "total_tokens": 1124, + "cost": 0, + "is_byok": false, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0, + "video_tokens": 0 + }, + "cost_details": { + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0, + "upstream_inference_completions_cost": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 62, + "image_tokens": 0, + "audio_tokens": 0 + } + } + }, + "cost": 0, + "timestamp": 1790160907.2642088 + } + }, + { + "content": "{\n \"returncode\": 1,\n \"output\": \"\\u003c\\u003c foi inesperado neste momento.\\n\"\n}", + "extra": { + "raw_output": "<< foi inesperado neste momento.\n", + "returncode": 1, + "timestamp": 1790160907.3063397, + "exception_info": "" + }, + "tool_call_id": "call-de56ecbd-a0ae-4b84-b6a6-f415194b4a41", + "role": "tool" + }, + { + "role": "exit", + "content": "'choices'", + "extra": { + "exit_status": "KeyError", + "submission": "", + "exception_str": "'choices'", + "traceback": "Traceback (most recent call last):\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 98, in run\n self.step()\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 128, in step\n return self.execute_actions(self.query())\n ^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 149, in query\n message = self.model.query(self.messages)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\models\\openrouter_model.py\", line 104, in query\n actions = self._parse_actions(response)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\models\\openrouter_model.py\", line 134, in _parse_actions\n tool_calls = response[\"choices\"][0][\"message\"].get(\"tool_calls\") or []\n ~~~~~~~~^^^^^^^^^^^\nKeyError: 'choices'\n" + } + } + ], + "trajectory_format": "mini-swe-agent-1.1" +} \ No newline at end of file diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/diff.patch b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/diff.patch new file mode 100644 index 00000000..d7b37098 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/diff.patch @@ -0,0 +1,664 @@ +diff --git a/README.md b/README.md +index 13041af..c729c17 100644 +--- a/README.md ++++ b/README.md +@@ -1,26 +1,29 @@ + # Arkx + +-Arkx is a greenfield project for empirical, incremental development of reliable capability. ++Arkx is a repository for incremental implementation and empirical evaluation of software capability. + + > **MINIMUM SUFFICIENT ARCHITECTURE** + > **FOR MAXIMUM RELIABLE CAPABILITY** + +-This repository is the canonical root for the project. It intentionally contains only the foundation in this bootstrap: the project contract, experimental protocol, structural placeholders, and a minimal CI check. ++This repository is the canonical root for the project. It contains runtime ++contracts, an execution harness, the project contract, the experimental ++protocol, and structural checks. + +-## Scope of this bootstrap ++## Current scope + + Included: + + - explicit project invariants; +-- a small, evidence-oriented experimental protocol; +-- stable top-level boundaries for future work; +-- a deterministic structural check and minimal CI. ++- an evidence-oriented experimental protocol; ++- stable top-level boundaries; ++- a deterministic structural check, executable contracts, and a Treatment A ++ baseline harness. + +-Deferred: ++Still deferred or unqualified: + +-- routing and planning; + - multi-agent orchestration; +-- OpenHands, ReX, mini-SWE-agent, and SWE-agent integrations; ++- OpenHands, ReX, and SWE-agent integrations; ++- real provider/model adoption and the A/B/C/D mechanism trial; + - product dependencies or automatic migration from `smag-rex`. + + ## Development posture +@@ -44,9 +47,9 @@ P1 adds deterministic task characterization from explicit signals only. See [the + + P2 adds deterministic progress/stagnation assessment from explicit snapshots only. See [the P2 contract](docs/progress-stagnation.md). It does not control execution or authorize retry/replan. + +-P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke or select concrete executors. ++P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke concrete executors. + +-Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. None executes a plan, owns acceptance, or creates orchestration. ++Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. P8.2a adds a Treatment A baseline runner and official SWE-bench acceptance adapter; neither owns general orchestration or promotion. + + ## Checks + +diff --git a/docs/architecture.md b/docs/architecture.md +index 17bbba6..98d1e6a 100644 +--- a/docs/architecture.md ++++ b/docs/architecture.md +@@ -1,24 +1,75 @@ + # Architecture boundary + +-The bootstrap architecture is intentionally small: ++Arkx separates contracts, execution, verification, acceptance, and promotion. ++The executable contracts cover the request-to-acceptance path. Runtime ++capability is qualified per provider, model, executor, and treatment; the ++presence of a contract does not imply adoption. + + ```text + Arkx +-├── docs/ contracts, protocol, and decisions +-├── experiments/ experiment records and fixtures; no runtime yet +-├── src/ reserved product boundary; no implementation yet +-├── tests/ reserved verification boundary; no test suite yet +-└── tools/ dependency-free repository checks ++├── docs/ contracts, protocol, decisions, and evidence boundaries ++├── experiments/ experiment records and synthetic fixtures; not orchestration ++├── src/arkx/ executable contracts and bounded policy/runtime surfaces ++├── tests/ executable contract and behavior verification ++└── tools/ repository checks and narrow execution adapters + ``` + +-P0 adds a deliberately small executable core under `src/arkx/`: versioned contracts, evidence status resolution, telemetry collection, and a deterministic baseline fixture. P1 adds explicit-signal task characterization, P2 adds snapshot-based progress assessment, P3 adds bounded routing/escalation policy, and Block B adds independent P4/P5/P6 artifacts in the same boundary. No layer has a concrete executor interface or executor orchestration. ++The current executable boundary is layered as follows: ++ ++- P0: versioned contracts, evidence status resolution, telemetry collection, ++ and a deterministic baseline fixture; ++- P1: explicit-signal task characterization; ++- P2/P3: progress assessment, bounded routing, and evidence-based escalation; ++- P4/P5/P6: repository planning/state, patch verification, and handoff/context ++ accounting; ++- P7/P8.1/P8.2: composition records, executor qualification contracts, and ++ controlled treatment plans; ++- P8.2a: a narrow Treatment A baseline runner and the official SWE-bench ++ acceptance adapter; ++- current orchestration surfaces: request governance, treatment/executor ++ selection, sequential execution, recovery, independent acceptance, and ++ promotion gates. ++ ++The canonical flow is: ++ ++```text ++request ++ -> governance / authority ++ -> task characterization ++ -> treatment route ++ -> sufficient executor ++ -> progress / recovery ++ -> verification ++ -> independent acceptance ++ -> evidence for possible promotion ++``` ++ ++Routing selects the minimum sufficient treatment before selecting an executor. ++Simple, localized, and repository-wide paths are capabilities/regimes, not ++automatic executor cascades. A fallback, executor switch, scope expansion, ++acceptance, or promotion must remain explicit and evidence-backed. ++ ++The official SWE-bench Docker authority has been qualified on the frozen Gold ++tasks. That validates the acceptance path and infrastructure, not a real ++provider/model, Treatment A execution, or A/B/C/D mechanism evidence. + + ## Boundary rules + +-- `src/` is reserved for product code and is empty in this bootstrap. ++- `src/arkx/` contains product contracts and bounded policy/runtime code; it ++ must not silently grow into an unqualified general-purpose orchestrator. + - `experiments/` contains evidence-oriented records, not an implicit runtime. +-- `tools/` may validate repository contracts but must not become orchestration. +-- `tests/` contains P0 executable verification and remains the boundary for future tests. +-- Integrations and executors require an explicit decision record and experiment evidence. ++- `tools/` may validate contracts or host a narrow adapter, but must not become ++ an untracked orchestration layer. ++- `tests/` verifies contracts and behavior; passing tests are local evidence, ++ not scientific or upstream evidence. ++- Integrations, providers, models, and executors require explicit identity, ++ qualification evidence, and an acceptance authority. ++- Missing provider/model identity, missing evidence, and blocked execution are ++ preserved as blocked or unknown; they never become success. ++- The working tree may contain an implementation under qualification. Merge ++ readiness requires the corresponding decision record, tests, and evidence to ++ be committed together. + +-An empty boundary is intentional. Creating a directory does not imply that the corresponding capability exists. ++The boundary is explicit: creating a module, fixture, or directory does not by ++itself prove that the corresponding capability exists or is suitable for ++promotion. +diff --git a/docs/experimental-protocol.md b/docs/experimental-protocol.md +index f24bf94..2b0f92d 100644 +--- a/docs/experimental-protocol.md ++++ b/docs/experimental-protocol.md +@@ -1,6 +1,7 @@ + # Experimental protocol + +-Arkx develops through small, empirical increments. The protocol below is the minimum record required before a capability can be considered for promotion. ++Arkx develops through empirical increments. The protocol below defines the ++minimum record required before a capability can be considered for promotion. + + ## Required record + +@@ -14,6 +15,9 @@ Each experiment records: + - `procedure`: reproducible steps; + - `expected_signal`: measurable success and failure criteria; + - `result`: `PASS`, `FAIL`, `BLOCKED`, or `NOT_EXECUTED`; ++- `evidence`: links or paths to raw evidence and environment facts; ++- `acceptance`: explicit reviewer decision and rationale; ++- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. + + P7 composition trials use their own comparability namespace: `COMPARABLE`, `INCOMPARABLE`, and `UNKNOWN`. P7 records externally supplied measurements and does not convert them into acceptance results. + +@@ -26,9 +30,14 @@ environment, budget, verification, and independent acceptance authority. It + does not invoke an executor or mechanism and does not manufacture acceptance; + results remain P8.1 observations. Missing controls and measurements remain + `UNKNOWN`, and the only intended treatment difference is the declared arm. +-- `evidence`: links or paths to raw evidence and environment facts; +-- `acceptance`: explicit reviewer decision and rationale; +-- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. ++ ++P8.2a adds the Treatment A baseline runner. Each attempt is serial, refuses to ++overwrite an existing attempt directory, derives a stable run identity from ++the task and frozen execution configuration, and writes an `execution.json` ++artifact atomically alongside the log, diff, and trajectory. The artifact ++records execution only; verification, independent acceptance, and promotion ++remain separate records. A missing provider, model, mini version, or Python ++executable blocks the attempt instead of selecting a fallback. + + ## State separation + +diff --git a/docs/p82-minimal-mechanism-trial.md b/docs/p82-minimal-mechanism-trial.md +index b05ef93..43225e8 100644 +--- a/docs/p82-minimal-mechanism-trial.md ++++ b/docs/p82-minimal-mechanism-trial.md +@@ -34,8 +34,8 @@ promotion decision. + + ## P8.2a baseline execution path + +-P8.2a is infrastructure qualification, not mechanism evidence. The baseline +-runner accepts treatment A only, invokes mini-SWE-agent through its public ++P8.2a records infrastructure qualification, not mechanism evidence. The ++baseline runner accepts treatment A only, invokes mini-SWE-agent through its public + Python API in a headless worker, and records execution artifacts. It does not + run verification or decide acceptance. Verification and independent + acceptance are explicit, nullable records; missing provider/model identity is +@@ -49,7 +49,9 @@ sample is frozen in `experiments/p82-wave0-task-sample.json` from the official + + The official acceptance adapter is in `arkx.swebench_authority`. It creates a + SWE-bench prediction without gold patch or grading labels and invokes the +-official harness with a unique `run_id`. Gold qualification and real A runs +-remain blocked until `swebench` and Docker are available, the gold harness +-passes, and one exact provider/model/configuration is frozen. B/C/D and any ++official harness with a unique `run_id`. The frozen Docker Gold authority ++returned `resolved: true` for the two Wave 0 tasks recorded under ++`logs/evaluation/`; this qualifies the acceptance path and infrastructure only. ++Real A runs still ++require one exact provider/model/configuration to be frozen. B/C/D and any + mechanism comparison remain out of scope. +diff --git a/docs/project-contract.md b/docs/project-contract.md +index f2f734a..82054a4 100644 +--- a/docs/project-contract.md ++++ b/docs/project-contract.md +@@ -40,7 +40,9 @@ NO_SILENT_SCOPE_EXPANSION + 5. Any fallback, executor change, or scope change is explicit in the record and review. + 6. New architecture requires a decision record describing the observed need, alternatives, evidence, and rollback/removal condition. + +-## Non-goals of this bootstrap +- +-This repository does not yet define routing, planning, multi-agent execution, or integrations with OpenHands, ReX, mini-SWE-agent, or SWE-agent. ++## Current non-goals + ++The repository now defines bounded routing, planning/state, verification, ++recovery, acceptance, promotion gates, and a mini-SWE-agent Treatment A ++baseline path. It does not define general multi-agent execution, automatic ++executor adoption, or integrations with OpenHands, ReX, and SWE-agent. +diff --git a/experiments/README.md b/experiments/README.md +index 551f37e..fe93b83 100644 +--- a/experiments/README.md ++++ b/experiments/README.md +@@ -2,3 +2,4 @@ + + This directory is reserved for explicit experiment records and small fixtures. Use [the experimental protocol](../docs/experimental-protocol.md). No experiment has been executed by the bootstrap. + ++The exploratory local-model qualification wave is documented in [local model qualification](../docs/local-model-qualification.md). It was run outside the Arkx experiment harness and must not be read as executor adoption or acceptance evidence. +diff --git a/src/arkx/__init__.py b/src/arkx/__init__.py +index 39c7d6c..2508056 100644 +--- a/src/arkx/__init__.py ++++ b/src/arkx/__init__.py +@@ -1,6 +1,62 @@ + """Minimal, executor-agnostic Arkx execution contracts and telemetry.""" + + from .contracts import Event, EventType, ExecutionRecord, ExecutionStatus ++from .request import ( ++ ArkxRequest, ++ GovernanceDecision, ++ GovernancePolicy, ++ GovernanceReason, ++ GovernanceStatus, ++ RequestBudget, ++ RequestEnvironment, ++ authorize_request, ++ characterize_authorized_request, ++) ++from .selection import ( ++ ExecutorBinding, ++ ExecutorRegistry, ++ SelectionDecision, ++ SelectionReason, ++ SelectionStatus, ++ TreatmentCatalog, ++ TreatmentDefinition, ++ select_executor, ++ select_treatment, ++) ++from .p82_localization import ( ++ LocalizationArtifact, ++ LocalizationEvidenceBudget, ++ record_localization_artifact, ++) ++from .p82_editing import ( ++ EditOperation, ++ StructuredEditProposal, ++ record_edit_proposal, ++) ++from .orchestration import ( ++ ExecutionOutcome, ++ ExecutionResult, ++ OrchestrationReason, ++ OrchestrationResult, ++ OrchestrationStatus, ++ run_routed_pipeline, ++) ++from .recovery import ( ++ RecoveryAction, ++ RecoveryBudget, ++ RecoveryDecision, ++ RecoveryReason, ++ decide_recovery, ++) ++from .acceptance import AcceptancePolicy, decide_independent_acceptance ++from .promotion import ( ++ PromotionCandidate, ++ PromotionDecision, ++ PromotionPolicy, ++ PromotionReason, ++ PromotionStatus, ++ decide_promotion, ++) + from .characterization import ( + CharacterizationConfig, + Confidence, +@@ -108,6 +164,43 @@ __all__ = [ + "EventType", + "ExecutionRecord", + "ExecutionStatus", ++ "ArkxRequest", ++ "GovernanceDecision", ++ "GovernancePolicy", ++ "GovernanceReason", ++ "GovernanceStatus", ++ "RequestBudget", ++ "RequestEnvironment", ++ "authorize_request", ++ "characterize_authorized_request", ++ "ExecutorBinding", ++ "ExecutorRegistry", ++ "SelectionDecision", ++ "SelectionReason", ++ "SelectionStatus", ++ "TreatmentCatalog", ++ "TreatmentDefinition", ++ "select_executor", ++ "select_treatment", ++ "ExecutionOutcome", ++ "ExecutionResult", ++ "OrchestrationReason", ++ "OrchestrationResult", ++ "OrchestrationStatus", ++ "run_routed_pipeline", ++ "RecoveryAction", ++ "RecoveryBudget", ++ "RecoveryDecision", ++ "RecoveryReason", ++ "decide_recovery", ++ "AcceptancePolicy", ++ "decide_independent_acceptance", ++ "PromotionCandidate", ++ "PromotionDecision", ++ "PromotionPolicy", ++ "PromotionReason", ++ "PromotionStatus", ++ "decide_promotion", + "CharacterizationConfig", + "Confidence", + "RecommendedPath", +diff --git a/src/arkx/p82_baseline.py b/src/arkx/p82_baseline.py +index 9cd9d72..bd37b71 100644 +--- a/src/arkx/p82_baseline.py ++++ b/src/arkx/p82_baseline.py +@@ -126,6 +126,8 @@ class BaselineRunConfig: + runner_version: str + artifact_root: str + treatment: P82Treatment = P82Treatment.A ++ environment: str = "local" ++ bash_executable: str | None = None + + def __post_init__(self) -> None: + if self.treatment is not P82Treatment.A: +@@ -138,6 +140,28 @@ class BaselineRunConfig: + def missing_identity(self) -> tuple[str, ...]: + return tuple(name for name in ("provider", "model", "mini_version", "python_executable") if getattr(self, name) is None) + ++ def identity_digest(self) -> str: ++ """Return a stable digest for the frozen execution configuration.""" ++ value = { ++ "provider": self.provider, ++ "model": self.model, ++ "model_version": self.model_version, ++ "temperature": self.temperature, ++ "reasoning_configuration": self.reasoning_configuration, ++ "model_configuration": self.model_configuration, ++ "max_tokens": self.max_tokens, ++ "max_cost": self.max_cost, ++ "wall_time_seconds": self.wall_time_seconds, ++ "mini_version": self.mini_version, ++ "python_executable": self.python_executable, ++ "runner_version": self.runner_version, ++ "treatment": self.treatment.value, ++ "environment": self.environment, ++ "bash_executable": self.bash_executable, ++ } ++ encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:16] ++ + + @dataclass(frozen=True) + class ExecutionArtifact: +@@ -155,16 +179,21 @@ class ExecutionArtifact: + log_path: str | None + trajectory_path: str | None + model_usage: dict[str, Any] | None ++ artifact_path: str | None = None + + def to_dict(self) -> dict[str, Any]: + value = {name: getattr(self, name) for name in ( + "run_id", "state", "task_id", "treatment", "attempt", "started_at", "finished_at", +- "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", "model_usage")} ++ "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", ++ "model_usage", "artifact_path")} + for name in ("state", "failure_category"): + if isinstance(value[name], Enum): + value[name] = value[name].value + return value + ++ def to_json(self) -> str: ++ return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ + + class HeadlessInvoker(Protocol): + def __call__(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: ... +@@ -185,7 +214,9 @@ class MiniSweAgentHeadlessRunner: + raise ValueError("attempt must be positive") + missing = config.missing_identity() + root = Path(config.artifact_root) / task.task_id / f"attempt-{attempt}" +- run_id = hashlib.sha256(f"{task.task_id}|A|{attempt}|{config.runner_version}".encode()).hexdigest()[:16] ++ run_id = hashlib.sha256( ++ f"{task.task_id}|{task.base_commit}|A|{attempt}|{config.identity_digest()}".encode("utf-8") ++ ).hexdigest()[:16] + if root.exists(): + return self._blocked(run_id, task, attempt, "artifact already exists; refusing silent overwrite") + root.mkdir(parents=True) +@@ -204,7 +235,16 @@ class MiniSweAgentHeadlessRunner: + state = BaselineExecutionState.COMPLETED if exit_code == 0 else BaselineExecutionState.EXECUTOR_FAILED + return self._finish(root, run_id, task, attempt, started, state, exit_code, None if exit_code == 0 else "executor returned non-zero", None if exit_code == 0 else RunnerFailureCategory.AGENT_RUN_FAILURE, str(diff_path), str(log_path), str(trajectory), usage) + except subprocess.TimeoutExpired as error: +- return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, str(error), RunnerFailureCategory.TIMEOUT, None, None, None, None) ++ log_path = root / "executor.log" ++ stdout = self._decode_process_output(error.stdout) ++ stderr = self._decode_process_output(error.stderr) ++ log_path.write_text(stdout + stderr, encoding="utf-8", newline="\n") ++ return self._finish( ++ root, run_id, task, attempt, started, ++ BaselineExecutionState.EXECUTOR_FAILED, None, str(error), ++ RunnerFailureCategory.TIMEOUT, None, str(log_path), ++ str(trajectory) if trajectory.exists() else None, None, ++ ) + except Exception as error: + message = f"{type(error).__name__}: {error}" + return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, message, self._classify_failure(message), None, None, None, None) +@@ -228,21 +268,48 @@ class MiniSweAgentHeadlessRunner: + return RunnerFailureCategory.DEADLOCK_HANG + return RunnerFailureCategory.AGENT_RUN_FAILURE + ++ @staticmethod ++ def _decode_process_output(value: str | bytes | None) -> str: ++ if value is None: ++ return "" ++ if isinstance(value, bytes): ++ return value.decode("utf-8", errors="replace") ++ return value ++ + def _subprocess_invoker(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: + worker = Path(__file__).parents[2] / "tools" / "run_mini_headless.py" + payload = {"task": task.problem_statement, "workspace": task.workspace, "model": config.model, + "trajectory": str(trajectory_path), "max_cost": config.max_cost, "wall_time_seconds": config.wall_time_seconds, +- "model_configuration": config.model_configuration} +- completed = subprocess.run([config.python_executable, str(worker)], input=json.dumps(payload), text=True, +- capture_output=True, cwd=task.workspace, timeout=config.wall_time_seconds) +- return completed.returncode, completed.stdout + completed.stderr, None ++ "model_configuration": config.model_configuration, "environment": config.environment, ++ "bash_executable": config.bash_executable} ++ timeout = None if config.wall_time_seconds is None else config.wall_time_seconds + 10 ++ completed = subprocess.run( ++ [config.python_executable, str(worker)], input=json.dumps(payload), text=True, ++ capture_output=True, cwd=task.workspace, timeout=timeout, ++ ) ++ combined_log = completed.stdout + completed.stderr ++ usage = None ++ if completed.returncode == 0: ++ lines = [line for line in completed.stdout.splitlines() if line.strip()] ++ if lines: ++ try: ++ worker_result = json.loads(lines[-1]) ++ usage = worker_result.get("usage") ++ except (json.JSONDecodeError, AttributeError): ++ usage = None ++ return completed.returncode, combined_log, usage + + def _blocked(self, run_id: str, task: ProspectiveTask, attempt: int, failure: str) -> ExecutionArtifact: + now = _now() + return ExecutionArtifact(run_id, BaselineExecutionState.BLOCKED, task.task_id, "A", attempt, now, now, None, failure, RunnerFailureCategory.UNKNOWN, None, None, None, None) + + def _finish(self, root: Path, run_id: str, task: ProspectiveTask, attempt: int, started: str, state: BaselineExecutionState, exit_code: int | None, failure: str | None, failure_category: RunnerFailureCategory | None, diff_path: str | None, log_path: str | None, trajectory_path: str | None, usage: dict[str, Any] | None) -> ExecutionArtifact: +- return ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage) ++ artifact_path = root / "execution.json" ++ artifact = ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage, str(artifact_path)) ++ temporary_path = root / "execution.json.tmp" ++ temporary_path.write_text(artifact.to_json() + "\n", encoding="utf-8", newline="\n") ++ temporary_path.replace(artifact_path) ++ return artifact + + + @dataclass(frozen=True) +diff --git a/tests/test_p82_baseline.py b/tests/test_p82_baseline.py +index d7d45ab..338b6b6 100644 +--- a/tests/test_p82_baseline.py ++++ b/tests/test_p82_baseline.py +@@ -1,6 +1,9 @@ + import tempfile + import unittest ++import json ++import subprocess + from pathlib import Path ++from unittest.mock import patch + + from arkx.p82 import P82RunStatus, P82Treatment, TaskStratum + from arkx.p82_baseline import ( +@@ -40,6 +43,43 @@ def config(root, **overrides): + + + class P82BaselineTests(unittest.TestCase): ++ def test_subprocess_invoker_passes_limits_and_captures_usage(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ runner = MiniSweAgentHeadlessRunner() ++ completed = type("Completed", (), { ++ "returncode": 0, ++ "stdout": '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\n', ++ "stderr": "worker log\n", ++ })() ++ with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: ++ usage = runner._subprocess_invoker(task(workspace), config(directory), Path(directory) / "trajectory.json") ++ self.assertEqual(usage, (0, '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\nworker log\n', {"model_calls": 3, "cost": 0.25})) ++ self.assertEqual(mocked.call_args.kwargs["timeout"], 20) ++ payload = json.loads(mocked.call_args.kwargs["input"]) ++ self.assertEqual(payload["max_cost"], "1") ++ self.assertEqual(payload["wall_time_seconds"], 10) ++ ++ def test_timeout_preserves_partial_output_and_trajectory(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ artifact_root = Path(directory) / "artifacts" ++ trajectory = artifact_root / "task-1" / "attempt-1" / "trajectory.json" ++ ++ def timed_out_invoker(run_task, run_config, trajectory_path): ++ trajectory_path.write_text('{"partial":true}', encoding="utf-8") ++ raise subprocess.TimeoutExpired( ++ cmd="worker", timeout=20, output=b"partial stdout\n", stderr=b"partial stderr\n" ++ ) ++ ++ result = MiniSweAgentHeadlessRunner(timed_out_invoker).run(task(workspace), config(artifact_root)) ++ self.assertEqual(result.failure_category, RunnerFailureCategory.TIMEOUT) ++ self.assertTrue(Path(result.log_path).exists()) ++ self.assertEqual(Path(result.log_path).read_text(encoding="utf-8"), "partial stdout\npartial stderr\n") ++ self.assertEqual(Path(result.trajectory_path).read_text(encoding="utf-8"), '{"partial":true}') ++ + def test_headless_runner_uses_fake_invoker_and_captures_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" +@@ -56,8 +96,46 @@ class P82BaselineTests(unittest.TestCase): + self.assertEqual(calls, [("task-1", P82Treatment.A, "trajectory.json")]) + self.assertTrue(Path(result.log_path).exists()) + self.assertTrue(Path(result.diff_path).exists()) ++ self.assertTrue(Path(result.artifact_path).exists()) ++ persisted = json.loads(Path(result.artifact_path).read_text(encoding="utf-8")) ++ self.assertEqual(persisted, result.to_dict()) ++ self.assertEqual(result.model_usage, {"calls": 1}) + self.assertEqual(result.treatment, "A") + ++ def test_run_identity_changes_when_frozen_model_configuration_changes(self): ++ with tempfile.TemporaryDirectory() as directory: ++ first = config(directory, model="model-a") ++ second = config(directory, model="model-b") ++ self.assertNotEqual(first.identity_digest(), second.identity_digest()) ++ ++ runner = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)) ++ first_result = runner.run(task(directory), first, attempt=1) ++ second_result = runner.run(task(directory), second, attempt=2) ++ self.assertNotEqual(first_result.run_id, second_result.run_id) ++ ++ def test_run_identity_changes_when_environment_changes(self): ++ with tempfile.TemporaryDirectory() as directory: ++ first = config(directory, environment="local") ++ second = config(directory, environment="git-bash", bash_executable=r"C:\Program Files\Git\bin\bash.exe") ++ self.assertNotEqual(first.identity_digest(), second.identity_digest()) ++ ++ def test_subprocess_invoker_passes_explicit_git_bash_environment(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ runner = MiniSweAgentHeadlessRunner() ++ completed = type("Completed", (), {"returncode": 0, "stdout": "{}\n", "stderr": ""})() ++ run_config = config( ++ directory, ++ environment="git-bash", ++ bash_executable=r"C:\Program Files\Git\bin\bash.exe", ++ ) ++ with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: ++ runner._subprocess_invoker(task(workspace), run_config, Path(directory) / "trajectory.json") ++ payload = json.loads(mocked.call_args.kwargs["input"]) ++ self.assertEqual(payload["environment"], "git-bash") ++ self.assertEqual(payload["bash_executable"], r"C:\Program Files\Git\bin\bash.exe") ++ + def test_missing_model_or_provider_is_blocked_without_fallback(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner().run(task(directory), config(directory, model=None)) +diff --git a/tools/run_mini_headless.py b/tools/run_mini_headless.py +index eedd60b..c772505 100644 +--- a/tools/run_mini_headless.py ++++ b/tools/run_mini_headless.py +@@ -22,17 +22,45 @@ def main() -> int: + from minisweagent.utils.serialize import recursive_merge, UNSET + + config = get_config_from_spec("mini.yaml") +- config = recursive_merge(config, { +- "run": {"task": payload["task"] or UNSET}, +- "agent": {"mode": "yolo", "confirm_exit": False, "output_path": Path(payload["trajectory"])}, +- "model": {"model_name": payload["model"], "model_class": payload.get("model_class", UNSET), **(payload.get("model_configuration") or payload.get("model_kwargs") or {})}, +- "environment": {"cwd": payload["workspace"]}, +- }) ++ config = recursive_merge( ++ config, ++ { ++ "run": {"task": payload["task"] or UNSET}, ++ "agent": { ++ "mode": "yolo", ++ "confirm_exit": False, ++ "output_path": Path(payload["trajectory"]), ++ "cost_limit": None if payload["max_cost"] is None else float(payload["max_cost"]), ++ "wall_time_limit_seconds": None if payload["wall_time_seconds"] is None else int(payload["wall_time_seconds"]), ++ }, ++ "model": { ++ "model_name": payload["model"], ++ "model_class": payload.get("model_class", UNSET), ++ **(payload.get("model_configuration") or payload.get("model_kwargs") or {}), ++ }, ++ "environment": {"cwd": payload["workspace"]}, ++ }, ++ ) + model = get_model(config=config["model"]) +- env = get_environment(config["environment"], default_type="local") ++ if payload.get("environment") == "git-bash": ++ from p82_bash_environment import BashLocalEnvironment ++ ++ environment_config = config["environment"] | { ++ "bash_executable": payload["bash_executable"], ++ } ++ env = BashLocalEnvironment(**environment_config) ++ else: ++ env = get_environment(config["environment"], default_type="local") + agent = get_agent(model, env, config["agent"], default_type="default") + result = agent.run(payload["task"] or "") +- print(json.dumps(result, ensure_ascii=False, sort_keys=True)) ++ output = { ++ "result": result, ++ "usage": { ++ "model_calls": getattr(agent, "n_calls", None), ++ "cost": getattr(agent, "cost", None), ++ }, ++ } ++ print(json.dumps(output, ensure_ascii=False, sort_keys=True)) + return 0 + + diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/execution.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/execution.json new file mode 100644 index 00000000..4cf50940 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-01\\agent-preflight-openrouter-nemotron-ultra-v3-01\\attempt-1\\execution.json","attempt":1,"diff_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-01\\agent-preflight-openrouter-nemotron-ultra-v3-01\\attempt-1\\diff.patch","exit_code":1,"failure":"executor returned non-zero","failure_category":"AGENT_RUN_FAILURE","finished_at":"2026-09-23T12:15:10.295+00:00","log_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-01\\agent-preflight-openrouter-nemotron-ultra-v3-01\\attempt-1\\executor.log","model_usage":null,"run_id":"44e006b26942b1f8","started_at":"2026-09-23T12:15:07.719+00:00","state":"EXECUTOR_FAILED","task_id":"agent-preflight-openrouter-nemotron-ultra-v3-01","trajectory_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-01\\agent-preflight-openrouter-nemotron-ultra-v3-01\\attempt-1\\trajectory.json","treatment":"A"} diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/executor.log b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/executor.log new file mode 100644 index 00000000..79601825 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/executor.log @@ -0,0 +1,22 @@ +Traceback (most recent call last): + File "D:\projetos\arkx\tools\run_mini_headless.py", line 68, in + raise SystemExit(main()) + ^^^^^^ + File "D:\projetos\arkx\tools\run_mini_headless.py", line 55, in main + result = agent.run(payload["task"] or "") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\agents\default.py", line 98, in run + self.step() + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\agents\default.py", line 128, in step + return self.execute_actions(self.query()) + ^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\agents\default.py", line 149, in query + message = self.model.query(self.messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\models\openrouter_model.py", line 104, in query + actions = self._parse_actions(response) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\models\openrouter_model.py", line 134, in _parse_actions + tool_calls = response["choices"][0]["message"].get("tool_calls") or [] + ~~~~~~~~^^^^^^^^^^^ +KeyError: 'choices' diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/trajectory.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/trajectory.json new file mode 100644 index 00000000..a021fd43 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/agent-preflight-openrouter-nemotron-ultra-v3-01/attempt-1/trajectory.json @@ -0,0 +1,74 @@ +{ + "info": { + "model_stats": { + "instance_cost": 0.0, + "api_calls": 1 + }, + "config": { + "agent": { + "system_template": "You are a helpful assistant that can interact with a computer.\n", + "instance_template": "Please solve this issue: {{task}}\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\n{{system}} {{release}} {{version}} {{machine}}\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:\n\n{%- if system == \"Darwin\" -%}\n\nYou are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.\n\n{%- endif -%}\n\n```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```\n", + "step_limit": 0, + "cost_limit": 0.0, + "wall_time_limit_seconds": 180, + "max_consecutive_format_errors": 3, + "output_path": "D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-01\\agent-preflight-openrouter-nemotron-ultra-v3-01\\attempt-1\\trajectory.json" + }, + "agent_type": "minisweagent.agents.default.DefaultAgent", + "model": { + "model_name": "nvidia/nemotron-3-ultra-550b-a55b:free", + "model_kwargs": { + "drop_params": true, + "temperature": 0.0, + "max_tokens": 256, + "custom_llm_provider": "openrouter", + "provider": { + "allow_fallbacks": false + } + }, + "set_cache_control": null, + "cost_tracking": "ignore_errors", + "format_error_template": "{% if finish_reason is defined and (finish_reason == \"length\" or (finish_reason == \"tool_calls\" and not has_tool_calls)) -%}\nYour previous response reached the output token limit (finish_reason={{ finish_reason }}) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.\n{%- else -%}\nTool call error:\n\n\n{{error}}\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.\n{%- endif %}\n", + "observation_template": "{%- if output.output | length < 10000 -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output\": {{ output.output | tojson }}\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- else -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output_head\": {{ output.output[:5000] | tojson }},\n \"output_tail\": {{ output.output[-5000:] | tojson }},\n \"elided_chars\": {{ output.output | length - 10000 }},\n \"warning\": \"Output too long.\"\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- endif -%}\n", + "multimodal_regex": "" + }, + "model_type": "minisweagent.models.openrouter_model.OpenRouterModel", + "environment": { + "cwd": "D:\\projetos\\arkx\\.p82-local-preflight-nemotron-ultra-v3-workspace", + "env": { + "PAGER": "cat", + "MANPAGER": "cat", + "LESS": "-R", + "PIP_PROGRESS_BAR": "off", + "TQDM_DISABLE": "1" + }, + "timeout": 30 + }, + "environment_type": "p82_bash_environment.BashLocalEnvironment" + }, + "mini_version": "2.4.6", + "exit_status": "KeyError", + "submission": "" + }, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: Use the bash tool to create target.py containing exactly: def answer():\\n return 'ARKX_PREFLIGHT_OK'\\n. Then finish the task.\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nWindows 10 10.0.26200 AMD64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + }, + { + "role": "exit", + "content": "'choices'", + "extra": { + "exit_status": "KeyError", + "submission": "", + "exception_str": "'choices'", + "traceback": "Traceback (most recent call last):\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 98, in run\n self.step()\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 128, in step\n return self.execute_actions(self.query())\n ^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 149, in query\n message = self.model.query(self.messages)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\models\\openrouter_model.py\", line 104, in query\n actions = self._parse_actions(response)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\models\\openrouter_model.py\", line 134, in _parse_actions\n tool_calls = response[\"choices\"][0][\"message\"].get(\"tool_calls\") or []\n ~~~~~~~~^^^^^^^^^^^\nKeyError: 'choices'\n" + } + } + ], + "trajectory_format": "mini-swe-agent-1.1" +} \ No newline at end of file diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/raw-openrouter-response.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/raw-openrouter-response.json new file mode 100644 index 00000000..57827987 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-01/raw-openrouter-response.json @@ -0,0 +1 @@ +"ModelResponse(id='gen-1790165873-pKt1zpukvn3ghr3eflaN', created=1790165873, model='nvidia/nemotron-3-ultra-550b-a55b:free', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=\"I'll create the target.py file with the exact content specified in the issue.\\n\", role='assistant', tool_calls=[ChatCompletionMessageToolCall(index=0, function=Function(arguments='{\"command\":\"cat <<\\'EOF\\' > target.py\\\\ndef answer():\\\\n return \\'ARKX_PREFLIGHT_OK\\'\\\\nEOF\"}', name='bash'), id='call-41c82fd6-c664-42b7-b44f-1da362d8b918', type='function')], function_call=None, reasoning_content=\"The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\\n\\nLet me create the file using bash.\", provider_specific_fields={'reasoning': \"The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\\n\\nLet me create the file using bash.\", 'reasoning_details': [{'type': 'reasoning.text', 'text': \"The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\\n\\nLet me create the file using bash.\", 'format': 'unknown', 'index': 0}], 'refusal': None}), provider_specific_fields={'native_finish_reason': 'tool_calls'})], usage=Usage(completion_tokens=116, prompt_tokens=1000, total_tokens=1116, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=None, audio_tokens=0, reasoning_tokens=62, rejected_prediction_tokens=None, text_tokens=None, image_tokens=0, video_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=0, cache_write_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None, video_tokens=0, cache_creation_tokens=0), cost=0, is_byok=False, cost_details={'upstream_inference_cost': 0, 'upstream_inference_prompt_cost': 0, 'upstream_inference_completions_cost': 0}), provider='Nvidia', service_tier=None)" \ No newline at end of file diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-02/agent-preflight-openrouter-nemotron-ultra-v3-02/attempt-1/diff.patch b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-02/agent-preflight-openrouter-nemotron-ultra-v3-02/attempt-1/diff.patch new file mode 100644 index 00000000..e69de29b diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-02/agent-preflight-openrouter-nemotron-ultra-v3-02/attempt-1/execution.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-02/agent-preflight-openrouter-nemotron-ultra-v3-02/attempt-1/execution.json new file mode 100644 index 00000000..71de7648 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-02/agent-preflight-openrouter-nemotron-ultra-v3-02/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-02\\agent-preflight-openrouter-nemotron-ultra-v3-02\\attempt-1\\execution.json","attempt":1,"diff_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-02\\agent-preflight-openrouter-nemotron-ultra-v3-02\\attempt-1\\diff.patch","exit_code":1,"failure":"executor returned non-zero","failure_category":"AGENT_RUN_FAILURE","finished_at":"2026-09-23T12:26:09.913+00:00","log_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-02\\agent-preflight-openrouter-nemotron-ultra-v3-02\\attempt-1\\executor.log","model_usage":null,"run_id":"ca9486456fa479f5","started_at":"2026-09-23T12:26:08.277+00:00","state":"EXECUTOR_FAILED","task_id":"agent-preflight-openrouter-nemotron-ultra-v3-02","trajectory_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-02\\agent-preflight-openrouter-nemotron-ultra-v3-02\\attempt-1\\trajectory.json","treatment":"A"} diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-02/agent-preflight-openrouter-nemotron-ultra-v3-02/attempt-1/executor.log b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-02/agent-preflight-openrouter-nemotron-ultra-v3-02/attempt-1/executor.log new file mode 100644 index 00000000..79601825 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-02/agent-preflight-openrouter-nemotron-ultra-v3-02/attempt-1/executor.log @@ -0,0 +1,22 @@ +Traceback (most recent call last): + File "D:\projetos\arkx\tools\run_mini_headless.py", line 68, in + raise SystemExit(main()) + ^^^^^^ + File "D:\projetos\arkx\tools\run_mini_headless.py", line 55, in main + result = agent.run(payload["task"] or "") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\agents\default.py", line 98, in run + self.step() + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\agents\default.py", line 128, in step + return self.execute_actions(self.query()) + ^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\agents\default.py", line 149, in query + message = self.model.query(self.messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\models\openrouter_model.py", line 104, in query + actions = self._parse_actions(response) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\projetos\arkx\.venv-mini\Lib\site-packages\minisweagent\models\openrouter_model.py", line 134, in _parse_actions + tool_calls = response["choices"][0]["message"].get("tool_calls") or [] + ~~~~~~~~^^^^^^^^^^^ +KeyError: 'choices' diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-02/agent-preflight-openrouter-nemotron-ultra-v3-02/attempt-1/trajectory.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-02/agent-preflight-openrouter-nemotron-ultra-v3-02/attempt-1/trajectory.json new file mode 100644 index 00000000..892cdcaf --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-02/agent-preflight-openrouter-nemotron-ultra-v3-02/attempt-1/trajectory.json @@ -0,0 +1,75 @@ +{ + "info": { + "model_stats": { + "instance_cost": 0.0, + "api_calls": 1 + }, + "config": { + "agent": { + "system_template": "You are a helpful assistant that can interact with a computer.\n", + "instance_template": "Please solve this issue: {{task}}\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\n{{system}} {{release}} {{version}} {{machine}}\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:\n\n{%- if system == \"Darwin\" -%}\n\nYou are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.\n\n{%- endif -%}\n\n```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```\n", + "step_limit": 0, + "cost_limit": 0.0, + "wall_time_limit_seconds": 180, + "max_consecutive_format_errors": 3, + "output_path": "D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-02\\agent-preflight-openrouter-nemotron-ultra-v3-02\\attempt-1\\trajectory.json" + }, + "agent_type": "minisweagent.agents.default.DefaultAgent", + "model": { + "model_name": "nvidia/nemotron-3-ultra-550b-a55b:free", + "model_kwargs": { + "drop_params": true, + "temperature": 0.0, + "max_tokens": 256, + "custom_llm_provider": "openrouter", + "provider": { + "allow_fallbacks": false + } + }, + "set_cache_control": null, + "cost_tracking": "ignore_errors", + "format_error_template": "{% if finish_reason is defined and (finish_reason == \"length\" or (finish_reason == \"tool_calls\" and not has_tool_calls)) -%}\nYour previous response reached the output token limit (finish_reason={{ finish_reason }}) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.\n{%- else -%}\nTool call error:\n\n\n{{error}}\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.\n{%- endif %}\n", + "observation_template": "{%- if output.output | length < 10000 -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output\": {{ output.output | tojson }}\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- else -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output_head\": {{ output.output[:5000] | tojson }},\n \"output_tail\": {{ output.output[-5000:] | tojson }},\n \"elided_chars\": {{ output.output | length - 10000 }},\n \"warning\": \"Output too long.\"\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- endif -%}\n", + "multimodal_regex": "" + }, + "model_type": "minisweagent.models.openrouter_model.OpenRouterModel", + "environment": { + "cwd": "D:\\projetos\\arkx\\.p82-local-preflight-nemotron-ultra-v3-02-workspace", + "env": { + "PAGER": "cat", + "MANPAGER": "cat", + "LESS": "-R", + "PIP_PROGRESS_BAR": "off", + "TQDM_DISABLE": "1" + }, + "timeout": 30, + "bash_executable": "C:\\Program Files\\Git\\bin\\bash.exe" + }, + "environment_type": "p82_bash_environment.BashLocalEnvironment" + }, + "mini_version": "2.4.6", + "exit_status": "KeyError", + "submission": "" + }, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: Use the bash tool to create target.py containing exactly: def answer():\n return 'ARKX_PREFLIGHT_OK'\n. Then finish the task.\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nWindows 10 10.0.26200 AMD64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + }, + { + "role": "exit", + "content": "'choices'", + "extra": { + "exit_status": "KeyError", + "submission": "", + "exception_str": "'choices'", + "traceback": "Traceback (most recent call last):\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 98, in run\n self.step()\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 128, in step\n return self.execute_actions(self.query())\n ^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 149, in query\n message = self.model.query(self.messages)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\models\\openrouter_model.py\", line 104, in query\n actions = self._parse_actions(response)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\models\\openrouter_model.py\", line 134, in _parse_actions\n tool_calls = response[\"choices\"][0][\"message\"].get(\"tool_calls\") or []\n ~~~~~~~~^^^^^^^^^^^\nKeyError: 'choices'\n" + } + } + ], + "trajectory_format": "mini-swe-agent-1.1" +} \ No newline at end of file diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-diagnostic-01/raw-response.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-diagnostic-01/raw-response.json new file mode 100644 index 00000000..78da89bf --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-diagnostic-01/raw-response.json @@ -0,0 +1,10 @@ +{ + "error": { + "code": 503, + "message": "Upstream error from Nvidia: Service temporarily overloaded", + "metadata": { + "error_type": "provider_overloaded" + } + }, + "id": "gen-1790166542-dZrcZ5dpD1xQXtjn0sfg" +} \ No newline at end of file diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-diagnostic-01/trajectory.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-diagnostic-01/trajectory.json new file mode 100644 index 00000000..5e9a43d9 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-diagnostic-01/trajectory.json @@ -0,0 +1,75 @@ +{ + "info": { + "model_stats": { + "instance_cost": 0.0, + "api_calls": 1 + }, + "config": { + "agent": { + "system_template": "You are a helpful assistant that can interact with a computer.\n", + "instance_template": "Please solve this issue: {{task}}\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\n{{system}} {{release}} {{version}} {{machine}}\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:\n\n{%- if system == \"Darwin\" -%}\n\nYou are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.\n\n{%- endif -%}\n\n```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```\n", + "step_limit": 0, + "cost_limit": 0.0, + "wall_time_limit_seconds": 180, + "max_consecutive_format_errors": 3, + "output_path": "D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-diagnostic-01\\trajectory.json" + }, + "agent_type": "minisweagent.agents.default.DefaultAgent", + "model": { + "model_name": "nvidia/nemotron-3-ultra-550b-a55b:free", + "model_kwargs": { + "drop_params": true, + "temperature": 0.0, + "max_tokens": 256, + "custom_llm_provider": "openrouter", + "provider": { + "allow_fallbacks": false + } + }, + "set_cache_control": null, + "cost_tracking": "ignore_errors", + "format_error_template": "{% if finish_reason is defined and (finish_reason == \"length\" or (finish_reason == \"tool_calls\" and not has_tool_calls)) -%}\nYour previous response reached the output token limit (finish_reason={{ finish_reason }}) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.\n{%- else -%}\nTool call error:\n\n\n{{error}}\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.\n{%- endif %}\n", + "observation_template": "{%- if output.output | length < 10000 -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output\": {{ output.output | tojson }}\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- else -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output_head\": {{ output.output[:5000] | tojson }},\n \"output_tail\": {{ output.output[-5000:] | tojson }},\n \"elided_chars\": {{ output.output | length - 10000 }},\n \"warning\": \"Output too long.\"\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- endif -%}\n", + "multimodal_regex": "" + }, + "model_type": "minisweagent.models.openrouter_model.OpenRouterModel", + "environment": { + "cwd": "D:\\projetos\\arkx\\.p82-local-preflight-nemotron-ultra-v3-diagnostic-01-workspace", + "env": { + "PAGER": "cat", + "MANPAGER": "cat", + "LESS": "-R", + "PIP_PROGRESS_BAR": "off", + "TQDM_DISABLE": "1" + }, + "timeout": 30, + "bash_executable": "C:\\Program Files\\Git\\bin\\bash.exe" + }, + "environment_type": "p82_bash_environment.BashLocalEnvironment" + }, + "mini_version": "2.4.6", + "exit_status": "KeyError", + "submission": "" + }, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: Use the bash tool to create target.py containing exactly: def answer():\n return 'ARKX_PREFLIGHT_OK'\n. Then finish the task.\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nWindows 10 10.0.26200 AMD64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + }, + { + "role": "exit", + "content": "'choices'", + "extra": { + "exit_status": "KeyError", + "submission": "", + "exception_str": "'choices'", + "traceback": "Traceback (most recent call last):\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 98, in run\n self.step()\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 128, in step\n return self.execute_actions(self.query())\n ^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 149, in query\n message = self.model.query(self.messages)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\models\\openrouter_model.py\", line 104, in query\n actions = self._parse_actions(response)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\", line 121, in diagnostic_parse\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\models\\openrouter_model.py\", line 134, in _parse_actions\n tool_calls = response[\"choices\"][0][\"message\"].get(\"tool_calls\") or []\n ~~~~~~~~^^^^^^^^^^^\nKeyError: 'choices'\n" + } + } + ], + "trajectory_format": "mini-swe-agent-1.1" +} \ No newline at end of file diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/diff.patch b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/diff.patch new file mode 100644 index 00000000..46879949 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/diff.patch @@ -0,0 +1,736 @@ +diff --git a/README.md b/README.md +index 13041af..c729c17 100644 +--- a/README.md ++++ b/README.md +@@ -1,26 +1,29 @@ + # Arkx + +-Arkx is a greenfield project for empirical, incremental development of reliable capability. ++Arkx is a repository for incremental implementation and empirical evaluation of software capability. + + > **MINIMUM SUFFICIENT ARCHITECTURE** + > **FOR MAXIMUM RELIABLE CAPABILITY** + +-This repository is the canonical root for the project. It intentionally contains only the foundation in this bootstrap: the project contract, experimental protocol, structural placeholders, and a minimal CI check. ++This repository is the canonical root for the project. It contains runtime ++contracts, an execution harness, the project contract, the experimental ++protocol, and structural checks. + +-## Scope of this bootstrap ++## Current scope + + Included: + + - explicit project invariants; +-- a small, evidence-oriented experimental protocol; +-- stable top-level boundaries for future work; +-- a deterministic structural check and minimal CI. ++- an evidence-oriented experimental protocol; ++- stable top-level boundaries; ++- a deterministic structural check, executable contracts, and a Treatment A ++ baseline harness. + +-Deferred: ++Still deferred or unqualified: + +-- routing and planning; + - multi-agent orchestration; +-- OpenHands, ReX, mini-SWE-agent, and SWE-agent integrations; ++- OpenHands, ReX, and SWE-agent integrations; ++- real provider/model adoption and the A/B/C/D mechanism trial; + - product dependencies or automatic migration from `smag-rex`. + + ## Development posture +@@ -44,9 +47,9 @@ P1 adds deterministic task characterization from explicit signals only. See [the + + P2 adds deterministic progress/stagnation assessment from explicit snapshots only. See [the P2 contract](docs/progress-stagnation.md). It does not control execution or authorize retry/replan. + +-P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke or select concrete executors. ++P3 adds bounded rule-based routing and evidence-based escalation decisions. See [the P3 contract](docs/routing-escalation.md). It does not invoke concrete executors. + +-Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. None executes a plan, owns acceptance, or creates orchestration. ++Block B adds independently removable P4 repository planning/state, P5 patch verification, and P6 handoff/context accounting. P8.2a adds a Treatment A baseline runner and official SWE-bench acceptance adapter; neither owns general orchestration or promotion. + + ## Checks + +diff --git a/docs/architecture.md b/docs/architecture.md +index 17bbba6..98d1e6a 100644 +--- a/docs/architecture.md ++++ b/docs/architecture.md +@@ -1,24 +1,75 @@ + # Architecture boundary + +-The bootstrap architecture is intentionally small: ++Arkx separates contracts, execution, verification, acceptance, and promotion. ++The executable contracts cover the request-to-acceptance path. Runtime ++capability is qualified per provider, model, executor, and treatment; the ++presence of a contract does not imply adoption. + + ```text + Arkx +-├── docs/ contracts, protocol, and decisions +-├── experiments/ experiment records and fixtures; no runtime yet +-├── src/ reserved product boundary; no implementation yet +-├── tests/ reserved verification boundary; no test suite yet +-└── tools/ dependency-free repository checks ++├── docs/ contracts, protocol, decisions, and evidence boundaries ++├── experiments/ experiment records and synthetic fixtures; not orchestration ++├── src/arkx/ executable contracts and bounded policy/runtime surfaces ++├── tests/ executable contract and behavior verification ++└── tools/ repository checks and narrow execution adapters + ``` + +-P0 adds a deliberately small executable core under `src/arkx/`: versioned contracts, evidence status resolution, telemetry collection, and a deterministic baseline fixture. P1 adds explicit-signal task characterization, P2 adds snapshot-based progress assessment, P3 adds bounded routing/escalation policy, and Block B adds independent P4/P5/P6 artifacts in the same boundary. No layer has a concrete executor interface or executor orchestration. ++The current executable boundary is layered as follows: ++ ++- P0: versioned contracts, evidence status resolution, telemetry collection, ++ and a deterministic baseline fixture; ++- P1: explicit-signal task characterization; ++- P2/P3: progress assessment, bounded routing, and evidence-based escalation; ++- P4/P5/P6: repository planning/state, patch verification, and handoff/context ++ accounting; ++- P7/P8.1/P8.2: composition records, executor qualification contracts, and ++ controlled treatment plans; ++- P8.2a: a narrow Treatment A baseline runner and the official SWE-bench ++ acceptance adapter; ++- current orchestration surfaces: request governance, treatment/executor ++ selection, sequential execution, recovery, independent acceptance, and ++ promotion gates. ++ ++The canonical flow is: ++ ++```text ++request ++ -> governance / authority ++ -> task characterization ++ -> treatment route ++ -> sufficient executor ++ -> progress / recovery ++ -> verification ++ -> independent acceptance ++ -> evidence for possible promotion ++``` ++ ++Routing selects the minimum sufficient treatment before selecting an executor. ++Simple, localized, and repository-wide paths are capabilities/regimes, not ++automatic executor cascades. A fallback, executor switch, scope expansion, ++acceptance, or promotion must remain explicit and evidence-backed. ++ ++The official SWE-bench Docker authority has been qualified on the frozen Gold ++tasks. That validates the acceptance path and infrastructure, not a real ++provider/model, Treatment A execution, or A/B/C/D mechanism evidence. + + ## Boundary rules + +-- `src/` is reserved for product code and is empty in this bootstrap. ++- `src/arkx/` contains product contracts and bounded policy/runtime code; it ++ must not silently grow into an unqualified general-purpose orchestrator. + - `experiments/` contains evidence-oriented records, not an implicit runtime. +-- `tools/` may validate repository contracts but must not become orchestration. +-- `tests/` contains P0 executable verification and remains the boundary for future tests. +-- Integrations and executors require an explicit decision record and experiment evidence. ++- `tools/` may validate contracts or host a narrow adapter, but must not become ++ an untracked orchestration layer. ++- `tests/` verifies contracts and behavior; passing tests are local evidence, ++ not scientific or upstream evidence. ++- Integrations, providers, models, and executors require explicit identity, ++ qualification evidence, and an acceptance authority. ++- Missing provider/model identity, missing evidence, and blocked execution are ++ preserved as blocked or unknown; they never become success. ++- The working tree may contain an implementation under qualification. Merge ++ readiness requires the corresponding decision record, tests, and evidence to ++ be committed together. + +-An empty boundary is intentional. Creating a directory does not imply that the corresponding capability exists. ++The boundary is explicit: creating a module, fixture, or directory does not by ++itself prove that the corresponding capability exists or is suitable for ++promotion. +diff --git a/docs/experimental-protocol.md b/docs/experimental-protocol.md +index f24bf94..2b0f92d 100644 +--- a/docs/experimental-protocol.md ++++ b/docs/experimental-protocol.md +@@ -1,6 +1,7 @@ + # Experimental protocol + +-Arkx develops through small, empirical increments. The protocol below is the minimum record required before a capability can be considered for promotion. ++Arkx develops through empirical increments. The protocol below defines the ++minimum record required before a capability can be considered for promotion. + + ## Required record + +@@ -14,6 +15,9 @@ Each experiment records: + - `procedure`: reproducible steps; + - `expected_signal`: measurable success and failure criteria; + - `result`: `PASS`, `FAIL`, `BLOCKED`, or `NOT_EXECUTED`; ++- `evidence`: links or paths to raw evidence and environment facts; ++- `acceptance`: explicit reviewer decision and rationale; ++- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. + + P7 composition trials use their own comparability namespace: `COMPARABLE`, `INCOMPARABLE`, and `UNKNOWN`. P7 records externally supplied measurements and does not convert them into acceptance results. + +@@ -26,9 +30,14 @@ environment, budget, verification, and independent acceptance authority. It + does not invoke an executor or mechanism and does not manufacture acceptance; + results remain P8.1 observations. Missing controls and measurements remain + `UNKNOWN`, and the only intended treatment difference is the declared arm. +-- `evidence`: links or paths to raw evidence and environment facts; +-- `acceptance`: explicit reviewer decision and rationale; +-- `promotion`: explicit decision to make the change durable, or `NOT_PROMOTED`. ++ ++P8.2a adds the Treatment A baseline runner. Each attempt is serial, refuses to ++overwrite an existing attempt directory, derives a stable run identity from ++the task and frozen execution configuration, and writes an `execution.json` ++artifact atomically alongside the log, diff, and trajectory. The artifact ++records execution only; verification, independent acceptance, and promotion ++remain separate records. A missing provider, model, mini version, or Python ++executable blocks the attempt instead of selecting a fallback. + + ## State separation + +diff --git a/docs/p82-minimal-mechanism-trial.md b/docs/p82-minimal-mechanism-trial.md +index b05ef93..43225e8 100644 +--- a/docs/p82-minimal-mechanism-trial.md ++++ b/docs/p82-minimal-mechanism-trial.md +@@ -34,8 +34,8 @@ promotion decision. + + ## P8.2a baseline execution path + +-P8.2a is infrastructure qualification, not mechanism evidence. The baseline +-runner accepts treatment A only, invokes mini-SWE-agent through its public ++P8.2a records infrastructure qualification, not mechanism evidence. The ++baseline runner accepts treatment A only, invokes mini-SWE-agent through its public + Python API in a headless worker, and records execution artifacts. It does not + run verification or decide acceptance. Verification and independent + acceptance are explicit, nullable records; missing provider/model identity is +@@ -49,7 +49,9 @@ sample is frozen in `experiments/p82-wave0-task-sample.json` from the official + + The official acceptance adapter is in `arkx.swebench_authority`. It creates a + SWE-bench prediction without gold patch or grading labels and invokes the +-official harness with a unique `run_id`. Gold qualification and real A runs +-remain blocked until `swebench` and Docker are available, the gold harness +-passes, and one exact provider/model/configuration is frozen. B/C/D and any ++official harness with a unique `run_id`. The frozen Docker Gold authority ++returned `resolved: true` for the two Wave 0 tasks recorded under ++`logs/evaluation/`; this qualifies the acceptance path and infrastructure only. ++Real A runs still ++require one exact provider/model/configuration to be frozen. B/C/D and any + mechanism comparison remain out of scope. +diff --git a/docs/project-contract.md b/docs/project-contract.md +index f2f734a..82054a4 100644 +--- a/docs/project-contract.md ++++ b/docs/project-contract.md +@@ -40,7 +40,9 @@ NO_SILENT_SCOPE_EXPANSION + 5. Any fallback, executor change, or scope change is explicit in the record and review. + 6. New architecture requires a decision record describing the observed need, alternatives, evidence, and rollback/removal condition. + +-## Non-goals of this bootstrap +- +-This repository does not yet define routing, planning, multi-agent execution, or integrations with OpenHands, ReX, mini-SWE-agent, or SWE-agent. ++## Current non-goals + ++The repository now defines bounded routing, planning/state, verification, ++recovery, acceptance, promotion gates, and a mini-SWE-agent Treatment A ++baseline path. It does not define general multi-agent execution, automatic ++executor adoption, or integrations with OpenHands, ReX, and SWE-agent. +diff --git a/experiments/README.md b/experiments/README.md +index 551f37e..fe93b83 100644 +--- a/experiments/README.md ++++ b/experiments/README.md +@@ -2,3 +2,4 @@ + + This directory is reserved for explicit experiment records and small fixtures. Use [the experimental protocol](../docs/experimental-protocol.md). No experiment has been executed by the bootstrap. + ++The exploratory local-model qualification wave is documented in [local model qualification](../docs/local-model-qualification.md). It was run outside the Arkx experiment harness and must not be read as executor adoption or acceptance evidence. +diff --git a/src/arkx/__init__.py b/src/arkx/__init__.py +index 39c7d6c..2508056 100644 +--- a/src/arkx/__init__.py ++++ b/src/arkx/__init__.py +@@ -1,6 +1,62 @@ + """Minimal, executor-agnostic Arkx execution contracts and telemetry.""" + + from .contracts import Event, EventType, ExecutionRecord, ExecutionStatus ++from .request import ( ++ ArkxRequest, ++ GovernanceDecision, ++ GovernancePolicy, ++ GovernanceReason, ++ GovernanceStatus, ++ RequestBudget, ++ RequestEnvironment, ++ authorize_request, ++ characterize_authorized_request, ++) ++from .selection import ( ++ ExecutorBinding, ++ ExecutorRegistry, ++ SelectionDecision, ++ SelectionReason, ++ SelectionStatus, ++ TreatmentCatalog, ++ TreatmentDefinition, ++ select_executor, ++ select_treatment, ++) ++from .p82_localization import ( ++ LocalizationArtifact, ++ LocalizationEvidenceBudget, ++ record_localization_artifact, ++) ++from .p82_editing import ( ++ EditOperation, ++ StructuredEditProposal, ++ record_edit_proposal, ++) ++from .orchestration import ( ++ ExecutionOutcome, ++ ExecutionResult, ++ OrchestrationReason, ++ OrchestrationResult, ++ OrchestrationStatus, ++ run_routed_pipeline, ++) ++from .recovery import ( ++ RecoveryAction, ++ RecoveryBudget, ++ RecoveryDecision, ++ RecoveryReason, ++ decide_recovery, ++) ++from .acceptance import AcceptancePolicy, decide_independent_acceptance ++from .promotion import ( ++ PromotionCandidate, ++ PromotionDecision, ++ PromotionPolicy, ++ PromotionReason, ++ PromotionStatus, ++ decide_promotion, ++) + from .characterization import ( + CharacterizationConfig, + Confidence, +@@ -108,6 +164,43 @@ __all__ = [ + "EventType", + "ExecutionRecord", + "ExecutionStatus", ++ "ArkxRequest", ++ "GovernanceDecision", ++ "GovernancePolicy", ++ "GovernanceReason", ++ "GovernanceStatus", ++ "RequestBudget", ++ "RequestEnvironment", ++ "authorize_request", ++ "characterize_authorized_request", ++ "ExecutorBinding", ++ "ExecutorRegistry", ++ "SelectionDecision", ++ "SelectionReason", ++ "SelectionStatus", ++ "TreatmentCatalog", ++ "TreatmentDefinition", ++ "select_executor", ++ "select_treatment", ++ "ExecutionOutcome", ++ "ExecutionResult", ++ "OrchestrationReason", ++ "OrchestrationResult", ++ "OrchestrationStatus", ++ "run_routed_pipeline", ++ "RecoveryAction", ++ "RecoveryBudget", ++ "RecoveryDecision", ++ "RecoveryReason", ++ "decide_recovery", ++ "AcceptancePolicy", ++ "decide_independent_acceptance", ++ "PromotionCandidate", ++ "PromotionDecision", ++ "PromotionPolicy", ++ "PromotionReason", ++ "PromotionStatus", ++ "decide_promotion", + "CharacterizationConfig", + "Confidence", + "RecommendedPath", +diff --git a/src/arkx/p82_baseline.py b/src/arkx/p82_baseline.py +index 9cd9d72..26ec39a 100644 +--- a/src/arkx/p82_baseline.py ++++ b/src/arkx/p82_baseline.py +@@ -54,6 +54,7 @@ class RunnerFailureCategory(str, Enum): + TIMEOUT = "TIMEOUT" + DEADLOCK_HANG = "DEADLOCK/HANG" + OUTPUT_CAPTURE_FAILURE = "OUTPUT_CAPTURE_FAILURE" ++ PROVIDER_AVAILABILITY_FAILURE = "PROVIDER_AVAILABILITY_FAILURE" + UNKNOWN = "UNKNOWN" + + +@@ -126,6 +127,10 @@ class BaselineRunConfig: + runner_version: str + artifact_root: str + treatment: P82Treatment = P82Treatment.A ++ environment: str = "local" ++ bash_executable: str | None = None ++ provider_retry_attempts: int = 0 ++ provider_retry_backoff_seconds: float = 0.0 + + def __post_init__(self) -> None: + if self.treatment is not P82Treatment.A: +@@ -134,10 +139,36 @@ class BaselineRunConfig: + value = getattr(self, name) + if value is not None and value < 0: + raise ValueError(f"{name} must be non-negative when present") ++ if self.provider_retry_attempts < 0 or self.provider_retry_backoff_seconds < 0: ++ raise ValueError("provider retry limits must be non-negative") + + def missing_identity(self) -> tuple[str, ...]: + return tuple(name for name in ("provider", "model", "mini_version", "python_executable") if getattr(self, name) is None) + ++ def identity_digest(self) -> str: ++ """Return a stable digest for the frozen execution configuration.""" ++ value = { ++ "provider": self.provider, ++ "model": self.model, ++ "model_version": self.model_version, ++ "temperature": self.temperature, ++ "reasoning_configuration": self.reasoning_configuration, ++ "model_configuration": self.model_configuration, ++ "max_tokens": self.max_tokens, ++ "max_cost": self.max_cost, ++ "wall_time_seconds": self.wall_time_seconds, ++ "mini_version": self.mini_version, ++ "python_executable": self.python_executable, ++ "runner_version": self.runner_version, ++ "treatment": self.treatment.value, ++ "environment": self.environment, ++ "bash_executable": self.bash_executable, ++ "provider_retry_attempts": self.provider_retry_attempts, ++ "provider_retry_backoff_seconds": self.provider_retry_backoff_seconds, ++ } ++ encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:16] ++ + + @dataclass(frozen=True) + class ExecutionArtifact: +@@ -155,16 +186,21 @@ class ExecutionArtifact: + log_path: str | None + trajectory_path: str | None + model_usage: dict[str, Any] | None ++ artifact_path: str | None = None + + def to_dict(self) -> dict[str, Any]: + value = {name: getattr(self, name) for name in ( + "run_id", "state", "task_id", "treatment", "attempt", "started_at", "finished_at", +- "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", "model_usage")} ++ "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", ++ "model_usage", "artifact_path")} + for name in ("state", "failure_category"): + if isinstance(value[name], Enum): + value[name] = value[name].value + return value + ++ def to_json(self) -> str: ++ return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) ++ + + class HeadlessInvoker(Protocol): + def __call__(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: ... +@@ -185,7 +221,9 @@ class MiniSweAgentHeadlessRunner: + raise ValueError("attempt must be positive") + missing = config.missing_identity() + root = Path(config.artifact_root) / task.task_id / f"attempt-{attempt}" +- run_id = hashlib.sha256(f"{task.task_id}|A|{attempt}|{config.runner_version}".encode()).hexdigest()[:16] ++ run_id = hashlib.sha256( ++ f"{task.task_id}|{task.base_commit}|A|{attempt}|{config.identity_digest()}".encode("utf-8") ++ ).hexdigest()[:16] + if root.exists(): + return self._blocked(run_id, task, attempt, "artifact already exists; refusing silent overwrite") + root.mkdir(parents=True) +@@ -201,10 +239,24 @@ class MiniSweAgentHeadlessRunner: + diff_path = root / "diff.patch" + diff = subprocess.run(["git", "diff", "--binary"], cwd=task.workspace, capture_output=True, text=True, check=False) + diff_path.write_text(diff.stdout, encoding="utf-8") ++ if exit_code == 75: ++ return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.BLOCKED, exit_code, ++ "provider availability failure after bounded retries", ++ RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE, ++ str(diff_path), str(log_path), str(trajectory), usage) + state = BaselineExecutionState.COMPLETED if exit_code == 0 else BaselineExecutionState.EXECUTOR_FAILED + return self._finish(root, run_id, task, attempt, started, state, exit_code, None if exit_code == 0 else "executor returned non-zero", None if exit_code == 0 else RunnerFailureCategory.AGENT_RUN_FAILURE, str(diff_path), str(log_path), str(trajectory), usage) + except subprocess.TimeoutExpired as error: +- return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, str(error), RunnerFailureCategory.TIMEOUT, None, None, None, None) ++ log_path = root / "executor.log" ++ stdout = self._decode_process_output(error.stdout) ++ stderr = self._decode_process_output(error.stderr) ++ log_path.write_text(stdout + stderr, encoding="utf-8", newline="\n") ++ return self._finish( ++ root, run_id, task, attempt, started, ++ BaselineExecutionState.EXECUTOR_FAILED, None, str(error), ++ RunnerFailureCategory.TIMEOUT, None, str(log_path), ++ str(trajectory) if trajectory.exists() else None, None, ++ ) + except Exception as error: + message = f"{type(error).__name__}: {error}" + return self._finish(root, run_id, task, attempt, started, BaselineExecutionState.EXECUTOR_FAILED, None, message, self._classify_failure(message), None, None, None, None) +@@ -228,21 +280,52 @@ class MiniSweAgentHeadlessRunner: + return RunnerFailureCategory.DEADLOCK_HANG + return RunnerFailureCategory.AGENT_RUN_FAILURE + ++ @staticmethod ++ def _decode_process_output(value: str | bytes | None) -> str: ++ if value is None: ++ return "" ++ if isinstance(value, bytes): ++ return value.decode("utf-8", errors="replace") ++ return value ++ + def _subprocess_invoker(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: + worker = Path(__file__).parents[2] / "tools" / "run_mini_headless.py" + payload = {"task": task.problem_statement, "workspace": task.workspace, "model": config.model, + "trajectory": str(trajectory_path), "max_cost": config.max_cost, "wall_time_seconds": config.wall_time_seconds, +- "model_configuration": config.model_configuration} +- completed = subprocess.run([config.python_executable, str(worker)], input=json.dumps(payload), text=True, +- capture_output=True, cwd=task.workspace, timeout=config.wall_time_seconds) +- return completed.returncode, completed.stdout + completed.stderr, None ++ "model_configuration": config.model_configuration, "environment": config.environment, ++ "bash_executable": config.bash_executable, "provider_retry_attempts": config.provider_retry_attempts, ++ "provider_retry_backoff_seconds": config.provider_retry_backoff_seconds} ++ timeout = None if config.wall_time_seconds is None else config.wall_time_seconds + 10 ++ completed = subprocess.run( ++ [config.python_executable, str(worker)], input=json.dumps(payload), text=True, ++ capture_output=True, cwd=task.workspace, timeout=timeout, ++ ) ++ combined_log = completed.stdout + completed.stderr ++ usage = None ++ if completed.returncode == 0: ++ lines = [line for line in completed.stdout.splitlines() if line.strip()] ++ if lines: ++ try: ++ worker_result = json.loads(lines[-1]) ++ usage = worker_result.get("usage") ++ result = worker_result.get("result", {}) ++ if isinstance(result, dict) and result.get("failure_category") == RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE.value: ++ return 75, combined_log, usage ++ except (json.JSONDecodeError, AttributeError): ++ usage = None ++ return completed.returncode, combined_log, usage + + def _blocked(self, run_id: str, task: ProspectiveTask, attempt: int, failure: str) -> ExecutionArtifact: + now = _now() + return ExecutionArtifact(run_id, BaselineExecutionState.BLOCKED, task.task_id, "A", attempt, now, now, None, failure, RunnerFailureCategory.UNKNOWN, None, None, None, None) + + def _finish(self, root: Path, run_id: str, task: ProspectiveTask, attempt: int, started: str, state: BaselineExecutionState, exit_code: int | None, failure: str | None, failure_category: RunnerFailureCategory | None, diff_path: str | None, log_path: str | None, trajectory_path: str | None, usage: dict[str, Any] | None) -> ExecutionArtifact: +- return ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage) ++ artifact_path = root / "execution.json" ++ artifact = ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage, str(artifact_path)) ++ temporary_path = root / "execution.json.tmp" ++ temporary_path.write_text(artifact.to_json() + "\n", encoding="utf-8", newline="\n") ++ temporary_path.replace(artifact_path) ++ return artifact + + + @dataclass(frozen=True) +diff --git a/tests/test_p82_baseline.py b/tests/test_p82_baseline.py +index d7d45ab..97070ae 100644 +--- a/tests/test_p82_baseline.py ++++ b/tests/test_p82_baseline.py +@@ -1,6 +1,9 @@ + import tempfile + import unittest ++import json ++import subprocess + from pathlib import Path ++from unittest.mock import patch + + from arkx.p82 import P82RunStatus, P82Treatment, TaskStratum + from arkx.p82_baseline import ( +@@ -40,6 +43,43 @@ def config(root, **overrides): + + + class P82BaselineTests(unittest.TestCase): ++ def test_subprocess_invoker_passes_limits_and_captures_usage(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ runner = MiniSweAgentHeadlessRunner() ++ completed = type("Completed", (), { ++ "returncode": 0, ++ "stdout": '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\n', ++ "stderr": "worker log\n", ++ })() ++ with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: ++ usage = runner._subprocess_invoker(task(workspace), config(directory), Path(directory) / "trajectory.json") ++ self.assertEqual(usage, (0, '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\nworker log\n', {"model_calls": 3, "cost": 0.25})) ++ self.assertEqual(mocked.call_args.kwargs["timeout"], 20) ++ payload = json.loads(mocked.call_args.kwargs["input"]) ++ self.assertEqual(payload["max_cost"], "1") ++ self.assertEqual(payload["wall_time_seconds"], 10) ++ ++ def test_timeout_preserves_partial_output_and_trajectory(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ artifact_root = Path(directory) / "artifacts" ++ trajectory = artifact_root / "task-1" / "attempt-1" / "trajectory.json" ++ ++ def timed_out_invoker(run_task, run_config, trajectory_path): ++ trajectory_path.write_text('{"partial":true}', encoding="utf-8") ++ raise subprocess.TimeoutExpired( ++ cmd="worker", timeout=20, output=b"partial stdout\n", stderr=b"partial stderr\n" ++ ) ++ ++ result = MiniSweAgentHeadlessRunner(timed_out_invoker).run(task(workspace), config(artifact_root)) ++ self.assertEqual(result.failure_category, RunnerFailureCategory.TIMEOUT) ++ self.assertTrue(Path(result.log_path).exists()) ++ self.assertEqual(Path(result.log_path).read_text(encoding="utf-8"), "partial stdout\npartial stderr\n") ++ self.assertEqual(Path(result.trajectory_path).read_text(encoding="utf-8"), '{"partial":true}') ++ + def test_headless_runner_uses_fake_invoker_and_captures_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" +@@ -56,8 +96,53 @@ class P82BaselineTests(unittest.TestCase): + self.assertEqual(calls, [("task-1", P82Treatment.A, "trajectory.json")]) + self.assertTrue(Path(result.log_path).exists()) + self.assertTrue(Path(result.diff_path).exists()) ++ self.assertTrue(Path(result.artifact_path).exists()) ++ persisted = json.loads(Path(result.artifact_path).read_text(encoding="utf-8")) ++ self.assertEqual(persisted, result.to_dict()) ++ self.assertEqual(result.model_usage, {"calls": 1}) + self.assertEqual(result.treatment, "A") + ++ def test_run_identity_changes_when_frozen_model_configuration_changes(self): ++ with tempfile.TemporaryDirectory() as directory: ++ first = config(directory, model="model-a") ++ second = config(directory, model="model-b") ++ self.assertNotEqual(first.identity_digest(), second.identity_digest()) ++ ++ runner = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)) ++ first_result = runner.run(task(directory), first, attempt=1) ++ second_result = runner.run(task(directory), second, attempt=2) ++ self.assertNotEqual(first_result.run_id, second_result.run_id) ++ ++ def test_run_identity_changes_when_environment_changes(self): ++ with tempfile.TemporaryDirectory() as directory: ++ first = config(directory, environment="local") ++ second = config(directory, environment="git-bash", bash_executable=r"C:\Program Files\Git\bin\bash.exe") ++ self.assertNotEqual(first.identity_digest(), second.identity_digest()) ++ ++ def test_subprocess_invoker_passes_explicit_git_bash_environment(self): ++ with tempfile.TemporaryDirectory() as directory: ++ workspace = Path(directory) / "workspace" ++ workspace.mkdir() ++ runner = MiniSweAgentHeadlessRunner() ++ completed = type("Completed", (), {"returncode": 0, "stdout": "{}\n", "stderr": ""})() ++ run_config = config( ++ directory, ++ environment="git-bash", ++ bash_executable=r"C:\Program Files\Git\bin\bash.exe", ++ ) ++ with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: ++ runner._subprocess_invoker(task(workspace), run_config, Path(directory) / "trajectory.json") ++ payload = json.loads(mocked.call_args.kwargs["input"]) ++ self.assertEqual(payload["environment"], "git-bash") ++ self.assertEqual(payload["bash_executable"], r"C:\Program Files\Git\bin\bash.exe") ++ ++ def test_provider_availability_exit_is_blocked_not_agent_failed(self): ++ with tempfile.TemporaryDirectory() as directory: ++ runner = MiniSweAgentHeadlessRunner(lambda *_: (75, '{"provider_error":{"code":503}}', {"model_calls": 3})) ++ result = runner.run(task(directory), config(directory)) ++ self.assertEqual(result.state, BaselineExecutionState.BLOCKED) ++ self.assertEqual(result.failure_category, RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE) ++ + def test_missing_model_or_provider_is_blocked_without_fallback(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner().run(task(directory), config(directory, model=None)) +diff --git a/tools/run_mini_headless.py b/tools/run_mini_headless.py +index eedd60b..0e095e0 100644 +--- a/tools/run_mini_headless.py ++++ b/tools/run_mini_headless.py +@@ -9,6 +9,7 @@ from __future__ import annotations + import json + import os + import sys ++import time + from pathlib import Path + + +@@ -22,17 +23,70 @@ def main() -> int: + from minisweagent.utils.serialize import recursive_merge, UNSET + + config = get_config_from_spec("mini.yaml") +- config = recursive_merge(config, { +- "run": {"task": payload["task"] or UNSET}, +- "agent": {"mode": "yolo", "confirm_exit": False, "output_path": Path(payload["trajectory"])}, +- "model": {"model_name": payload["model"], "model_class": payload.get("model_class", UNSET), **(payload.get("model_configuration") or payload.get("model_kwargs") or {})}, +- "environment": {"cwd": payload["workspace"]}, +- }) ++ config = recursive_merge( ++ config, ++ { ++ "run": {"task": payload["task"] or UNSET}, ++ "agent": { ++ "mode": "yolo", ++ "confirm_exit": False, ++ "output_path": Path(payload["trajectory"]), ++ "cost_limit": None if payload["max_cost"] is None else float(payload["max_cost"]), ++ "wall_time_limit_seconds": None if payload["wall_time_seconds"] is None else int(payload["wall_time_seconds"]), ++ }, ++ "model": { ++ "model_name": payload["model"], ++ "model_class": payload.get("model_class", UNSET), ++ **(payload.get("model_configuration") or payload.get("model_kwargs") or {}), ++ }, ++ "environment": {"cwd": payload["workspace"]}, ++ }, ++ ) + model = get_model(config=config["model"]) +- env = get_environment(config["environment"], default_type="local") +- agent = get_agent(model, env, config["agent"], default_type="default") +- result = agent.run(payload["task"] or "") +- print(json.dumps(result, ensure_ascii=False, sort_keys=True)) ++ if payload.get("environment") == "git-bash": ++ from p82_bash_environment import BashLocalEnvironment ++ ++ environment_config = config["environment"] | { ++ "bash_executable": payload["bash_executable"], ++ } ++ env = BashLocalEnvironment(**environment_config) ++ else: ++ env = get_environment(config["environment"], default_type="local") ++ retry_limit = int(payload.get("provider_retry_attempts", 0)) ++ retry_backoff = float(payload.get("provider_retry_backoff_seconds", 0)) ++ result = None ++ agent = None ++ for retry_index in range(retry_limit + 1): ++ try: ++ agent = get_agent(model, env, config["agent"], default_type="default") ++ result = agent.run(payload["task"] or "") ++ break ++ except Exception as error: ++ from p82_openrouter_model import ProviderAvailabilityError ++ ++ if not isinstance(error, ProviderAvailabilityError): ++ raise ++ if retry_index >= retry_limit: ++ result = { ++ "exit_status": "ProviderAvailabilityFailure", ++ "submission": "", ++ "failure_category": "PROVIDER_AVAILABILITY_FAILURE", ++ "provider_error": error.error, ++ "provider_retry_attempts": retry_index, ++ } ++ break ++ if retry_backoff > 0: ++ time.sleep(retry_backoff) ++ model = get_model(config=config["model"]) ++ env = get_environment(config["environment"], default_type="local") if payload.get("environment") != "git-bash" else BashLocalEnvironment(**environment_config) ++ output = { ++ "result": result, ++ "usage": { ++ "model_calls": getattr(agent, "n_calls", None), ++ "cost": getattr(agent, "cost", None), ++ }, ++ } ++ print(json.dumps(output, ensure_ascii=False, sort_keys=True)) + return 0 + + diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/execution.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/execution.json new file mode 100644 index 00000000..121431af --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01\\agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01\\attempt-1\\execution.json","attempt":1,"diff_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01\\agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01\\attempt-1\\diff.patch","exit_code":0,"failure":null,"failure_category":null,"finished_at":"2026-09-23T12:34:08.066+00:00","log_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01\\agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01\\attempt-1\\executor.log","model_usage":{"cost":0.0,"model_calls":4},"run_id":"74c9c58b62e6f23c","started_at":"2026-09-23T12:33:32.614+00:00","state":"COMPLETED","task_id":"agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01","trajectory_path":"D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01\\agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01\\attempt-1\\trajectory.json","treatment":"A"} diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/executor.log b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/executor.log new file mode 100644 index 00000000..7cf4b259 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/executor.log @@ -0,0 +1 @@ +{"result": {"exit_status": "Submitted", "submission": ""}, "usage": {"cost": 0.0, "model_calls": 4}} diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/trajectory.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/trajectory.json new file mode 100644 index 00000000..8885f57f --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01/attempt-1/trajectory.json @@ -0,0 +1,506 @@ +{ + "info": { + "model_stats": { + "instance_cost": 0.0, + "api_calls": 4 + }, + "config": { + "agent": { + "system_template": "You are a helpful assistant that can interact with a computer.\n", + "instance_template": "Please solve this issue: {{task}}\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\n{{system}} {{release}} {{version}} {{machine}}\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:\n\n{%- if system == \"Darwin\" -%}\n\nYou are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.\n\n{%- endif -%}\n\n```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```\n", + "step_limit": 0, + "cost_limit": 0.0, + "wall_time_limit_seconds": 180, + "max_consecutive_format_errors": 3, + "output_path": "D:\\projetos\\arkx\\logs\\p82-agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01\\agent-preflight-openrouter-nemotron-ultra-v3-provider-retry-01\\attempt-1\\trajectory.json" + }, + "agent_type": "minisweagent.agents.default.DefaultAgent", + "model": { + "model_name": "nvidia/nemotron-3-ultra-550b-a55b:free", + "model_kwargs": { + "drop_params": true, + "temperature": 0.0, + "max_tokens": 256, + "custom_llm_provider": "openrouter", + "provider": { + "allow_fallbacks": false + } + }, + "set_cache_control": null, + "cost_tracking": "ignore_errors", + "format_error_template": "{% if finish_reason is defined and (finish_reason == \"length\" or (finish_reason == \"tool_calls\" and not has_tool_calls)) -%}\nYour previous response reached the output token limit (finish_reason={{ finish_reason }}) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.\n{%- else -%}\nTool call error:\n\n\n{{error}}\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.\n{%- endif %}\n", + "observation_template": "{%- if output.output | length < 10000 -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output\": {{ output.output | tojson }}\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- else -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output_head\": {{ output.output[:5000] | tojson }},\n \"output_tail\": {{ output.output[-5000:] | tojson }},\n \"elided_chars\": {{ output.output | length - 10000 }},\n \"warning\": \"Output too long.\"\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- endif -%}\n", + "multimodal_regex": "" + }, + "model_type": "p82_openrouter_model.ArkxOpenRouterModel", + "environment": { + "cwd": "D:\\projetos\\arkx\\.p82-local-preflight-nemotron-ultra-v3-retry-workspace", + "env": { + "PAGER": "cat", + "MANPAGER": "cat", + "LESS": "-R", + "PIP_PROGRESS_BAR": "off", + "TQDM_DISABLE": "1" + }, + "timeout": 30, + "bash_executable": "C:\\Program Files\\Git\\bin\\bash.exe" + }, + "environment_type": "p82_bash_environment.BashLocalEnvironment" + }, + "mini_version": "2.4.6", + "exit_status": "Submitted", + "submission": "" + }, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: Use the bash tool to create target.py containing exactly: def answer():\\n return 'ARKX_PREFLIGHT_OK'\\n. Then finish the task.\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nWindows 10 10.0.26200 AMD64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + }, + { + "role": "assistant", + "content": "I need to create a file called `target.py` with the exact content specified. Let me do this using bash.\n", + "refusal": null, + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "call-1f71a4b3-696d-45c9-8ae6-238ec911465b", + "function": { + "name": "bash", + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "format": "unknown", + "index": 0 + } + ], + "extra": { + "actions": [ + { + "command": "cat <<'EOF' > target.py\ndef answer():\n return 'ARKX_PREFLIGHT_OK'\nEOF", + "tool_call_id": "call-1f71a4b3-696d-45c9-8ae6-238ec911465b" + } + ], + "response": { + "id": "gen-1790166817-flHkA8myklVauthIeIa8", + "object": "chat.completion", + "created": 1790166817, + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "provider": "Nvidia", + "system_fingerprint": null, + "service_tier": null, + "choices": [ + { + "index": 0, + "logprobs": null, + "finish_reason": "tool_calls", + "native_finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "I need to create a file called `target.py` with the exact content specified. Let me do this using bash.\n", + "refusal": null, + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "call-1f71a4b3-696d-45c9-8ae6-238ec911465b", + "function": { + "name": "bash", + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "format": "unknown", + "index": 0 + } + ] + } + } + ], + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 124, + "total_tokens": 1124, + "cost": 0, + "is_byok": false, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0, + "video_tokens": 0 + }, + "cost_details": { + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0, + "upstream_inference_completions_cost": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 62, + "image_tokens": 0, + "audio_tokens": 0 + } + } + }, + "cost": 0, + "timestamp": 1790166824.7014902 + } + }, + { + "content": "{\n \"returncode\": 0,\n \"output\": \"\"\n}", + "extra": { + "raw_output": "", + "returncode": 0, + "timestamp": 1790166824.9906704, + "exception_info": "" + }, + "tool_call_id": "call-1f71a4b3-696d-45c9-8ae6-238ec911465b", + "role": "tool" + }, + { + "role": "assistant", + "content": "Let me verify the file was created with the correct content.\n\n", + "refusal": null, + "reasoning": "The file was created. Let me verify the content is correct.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "call-c4cd0c36-9822-4045-a886-a722a1dda0ce", + "function": { + "name": "bash", + "arguments": "{\"command\":\"cat target.py\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The file was created. Let me verify the content is correct.", + "format": "unknown", + "index": 0 + } + ], + "extra": { + "actions": [ + { + "command": "cat target.py", + "tool_call_id": "call-c4cd0c36-9822-4045-a886-a722a1dda0ce" + } + ], + "response": { + "id": "gen-1790166826-ODuOSLePErV2I11ZFy2A", + "object": "chat.completion", + "created": 1790166826, + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "provider": "Nvidia", + "system_fingerprint": null, + "service_tier": null, + "choices": [ + { + "index": 0, + "logprobs": null, + "finish_reason": "tool_calls", + "native_finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "Let me verify the file was created with the correct content.\n\n", + "refusal": null, + "reasoning": "The file was created. Let me verify the content is correct.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "call-c4cd0c36-9822-4045-a886-a722a1dda0ce", + "function": { + "name": "bash", + "arguments": "{\"command\":\"cat target.py\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The file was created. Let me verify the content is correct.", + "format": "unknown", + "index": 0 + } + ] + } + } + ], + "usage": { + "prompt_tokens": 1155, + "completion_tokens": 51, + "total_tokens": 1206, + "cost": 0, + "is_byok": false, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0, + "video_tokens": 0 + }, + "cost_details": { + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0, + "upstream_inference_completions_cost": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 15, + "image_tokens": 0, + "audio_tokens": 0 + } + } + }, + "cost": 0, + "timestamp": 1790166836.666582 + } + }, + { + "content": "{\n \"returncode\": 0,\n \"output\": \"def answer():\\n return \\u0027ARKX_PREFLIGHT_OK\\u0027\\n\"\n}", + "extra": { + "raw_output": "def answer():\n return 'ARKX_PREFLIGHT_OK'\n", + "returncode": 0, + "timestamp": 1790166837.0052812, + "exception_info": "" + }, + "tool_call_id": "call-c4cd0c36-9822-4045-a886-a722a1dda0ce", + "role": "tool" + }, + { + "role": "assistant", + "content": "The file `target.py` has been created with the exact content specified. Let me verify it works by running the function.\n\n", + "refusal": null, + "reasoning": "The file has been created with the exact content requested. The task is complete. Let me submit the final output.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "call-41652755-a53f-40c1-81a6-ea5b2d039612", + "function": { + "name": "bash", + "arguments": "{\"command\":\"python3 -c \\\"import target; print(target.answer())\\\"\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The file has been created with the exact content requested. The task is complete. Let me submit the final output.", + "format": "unknown", + "index": 0 + } + ], + "extra": { + "actions": [ + { + "command": "python3 -c \"import target; print(target.answer())\"", + "tool_call_id": "call-41652755-a53f-40c1-81a6-ea5b2d039612" + } + ], + "response": { + "id": "gen-1790166838-hOlHqGkgd6johYeBOacg", + "object": "chat.completion", + "created": 1790166838, + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "provider": "Nvidia", + "system_fingerprint": null, + "service_tier": null, + "choices": [ + { + "index": 0, + "logprobs": null, + "finish_reason": "tool_calls", + "native_finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "The file `target.py` has been created with the exact content specified. Let me verify it works by running the function.\n\n", + "refusal": null, + "reasoning": "The file has been created with the exact content requested. The task is complete. Let me submit the final output.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "call-41652755-a53f-40c1-81a6-ea5b2d039612", + "function": { + "name": "bash", + "arguments": "{\"command\":\"python3 -c \\\"import target; print(target.answer())\\\"\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "The file has been created with the exact content requested. The task is complete. Let me submit the final output.", + "format": "unknown", + "index": 0 + } + ] + } + } + ], + "usage": { + "prompt_tokens": 1263, + "completion_tokens": 84, + "total_tokens": 1347, + "cost": 0, + "is_byok": false, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0, + "video_tokens": 0 + }, + "cost_details": { + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0, + "upstream_inference_completions_cost": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 29, + "image_tokens": 0, + "audio_tokens": 0 + } + } + }, + "cost": 0, + "timestamp": 1790166843.0645485 + } + }, + { + "content": "{\n \"returncode\": 49,\n \"output\": \"Python was not found; run without arguments to install from the Microsoft Store, or disable this shortcut from Settings \\u003e Apps \\u003e Advanced app settings \\u003e App execution aliases.\\n\"\n}", + "extra": { + "raw_output": "Python was not found; run without arguments to install from the Microsoft Store, or disable this shortcut from Settings > Apps > Advanced app settings > App execution aliases.\n", + "returncode": 49, + "timestamp": 1790166843.5110028, + "exception_info": "" + }, + "tool_call_id": "call-41652755-a53f-40c1-81a6-ea5b2d039612", + "role": "tool" + }, + { + "role": "assistant", + "content": "The file has been created with the exact content requested. The task is complete.\n\n", + "refusal": null, + "reasoning": "Python isn't available in this environment, but the file has been created correctly with the exact content requested. The task is complete - I just needed to create the file with the specified content. Let me submit the final output.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "call-77dd9443-fbcb-44ac-ad8a-e0cfde3b2b60", + "function": { + "name": "bash", + "arguments": "{\"command\":\"echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "Python isn't available in this environment, but the file has been created correctly with the exact content requested. The task is complete - I just needed to create the file with the specified content. Let me submit the final output.", + "format": "unknown", + "index": 0 + } + ], + "extra": { + "actions": [ + { + "command": "echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT", + "tool_call_id": "call-77dd9443-fbcb-44ac-ad8a-e0cfde3b2b60" + } + ], + "response": { + "id": "gen-1790166844-Cv4TPgPQdb6n6WzozwJU", + "object": "chat.completion", + "created": 1790166844, + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "provider": "Nvidia", + "system_fingerprint": null, + "service_tier": null, + "choices": [ + { + "index": 0, + "logprobs": null, + "finish_reason": "tool_calls", + "native_finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "The file has been created with the exact content requested. The task is complete.\n\n", + "refusal": null, + "reasoning": "Python isn't available in this environment, but the file has been created correctly with the exact content requested. The task is complete - I just needed to create the file with the specified content. Let me submit the final output.", + "tool_calls": [ + { + "type": "function", + "index": 0, + "id": "call-77dd9443-fbcb-44ac-ad8a-e0cfde3b2b60", + "function": { + "name": "bash", + "arguments": "{\"command\":\"echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\"}" + } + } + ], + "reasoning_details": [ + { + "type": "reasoning.text", + "text": "Python isn't available in this environment, but the file has been created correctly with the exact content requested. The task is complete - I just needed to create the file with the specified content. Let me submit the final output.", + "format": "unknown", + "index": 0 + } + ] + } + } + ], + "usage": { + "prompt_tokens": 1428, + "completion_tokens": 95, + "total_tokens": 1523, + "cost": 0, + "is_byok": false, + "prompt_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 0, + "audio_tokens": 0, + "video_tokens": 0 + }, + "cost_details": { + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0, + "upstream_inference_completions_cost": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 59, + "image_tokens": 0, + "audio_tokens": 0 + } + } + }, + "cost": 0, + "timestamp": 1790166847.7033691 + } + }, + { + "role": "exit", + "content": "", + "extra": { + "exit_status": "Submitted", + "submission": "" + } + } + ], + "trajectory_format": "mini-swe-agent-1.1" +} \ No newline at end of file diff --git a/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-replicates.json b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-replicates.json new file mode 100644 index 00000000..459445c0 --- /dev/null +++ b/logs/p82-agent-preflight-openrouter-nemotron-ultra-v3-replicates.json @@ -0,0 +1,578 @@ +[ + { + "elapsed_seconds": 22.844, + "finish_reason": "tool_calls", + "has_choices": true, + "has_tool_calls": true, + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "query_result": { + "content": "I'll create the target.py file with the exact content specified in the issue.\n", + "extra": { + "actions": [ + { + "command": "cat <<'EOF' > target.py\ndef answer():\n return 'ARKX_PREFLIGHT_OK'\nEOF", + "tool_call_id": "call-d0dd6cb7-a067-4f6c-9a9d-14f53ce88ce0" + } + ], + "cost": 0.0, + "response": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll create the target.py file with the exact content specified in the issue.\n", + "function_call": null, + "provider_specific_fields": { + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "reasoning_details": [ + { + "format": "unknown", + "index": 0, + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "type": "reasoning.text" + } + ], + "refusal": null + }, + "reasoning_content": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}", + "name": "bash" + }, + "id": "call-d0dd6cb7-a067-4f6c-9a9d-14f53ce88ce0", + "index": 0, + "type": "function" + } + ] + }, + "provider_specific_fields": { + "native_finish_reason": "tool_calls" + } + } + ], + "created": 1790166149, + "id": "gen-1790166149-PgqDUvpBDERt1Yekjuu7", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "object": "chat.completion", + "provider": "Nvidia", + "service_tier": null, + "system_fingerprint": null, + "usage": { + "completion_tokens": 116, + "completion_tokens_details": { + "audio_tokens": 0, + "image_tokens": 0, + "reasoning_tokens": 62 + }, + "cost": 0, + "cost_details": { + "upstream_inference_completions_cost": 0, + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0 + }, + "is_byok": false, + "prompt_tokens": 1000, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_creation_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 0, + "video_tokens": 0 + }, + "total_tokens": 1116 + } + }, + "timestamp": 1790166161.9773188 + }, + "function_call": null, + "provider_specific_fields": { + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "reasoning_details": [ + { + "format": "unknown", + "index": 0, + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "type": "reasoning.text" + } + ], + "refusal": null + }, + "reasoning_content": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}", + "name": "bash" + }, + "id": "call-d0dd6cb7-a067-4f6c-9a9d-14f53ce88ce0", + "index": 0, + "type": "function" + } + ] + }, + "raw_response": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll create the target.py file with the exact content specified in the issue.\n", + "function_call": null, + "provider_specific_fields": { + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "reasoning_details": [ + { + "format": "unknown", + "index": 0, + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "type": "reasoning.text" + } + ], + "refusal": null + }, + "reasoning_content": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}", + "name": "bash" + }, + "id": "call-d0dd6cb7-a067-4f6c-9a9d-14f53ce88ce0", + "index": 0, + "type": "function" + } + ] + }, + "provider_specific_fields": { + "native_finish_reason": "tool_calls" + } + } + ], + "created": 1790166149, + "id": "gen-1790166149-PgqDUvpBDERt1Yekjuu7", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "object": "chat.completion", + "provider": "Nvidia", + "service_tier": null, + "system_fingerprint": null, + "usage": { + "completion_tokens": 116, + "completion_tokens_details": { + "audio_tokens": 0, + "image_tokens": 0, + "reasoning_tokens": 62 + }, + "cost": 0, + "cost_details": { + "upstream_inference_completions_cost": 0, + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0 + }, + "is_byok": false, + "prompt_tokens": 1000, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_creation_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 0, + "video_tokens": 0 + }, + "total_tokens": 1116 + } + }, + "response_type": "ModelResponse", + "run": 1, + "status": "PASS", + "top_level_keys": null + }, + { + "elapsed_seconds": 14.203, + "finish_reason": "tool_calls", + "has_choices": true, + "has_tool_calls": true, + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "query_result": { + "content": "I need to create a file called `target.py` with the exact content specified. Let me do this using bash.\n", + "extra": { + "actions": [ + { + "command": "cat <<'EOF' > target.py\ndef answer():\n return 'ARKX_PREFLIGHT_OK'\nEOF", + "tool_call_id": "call-5d82ed14-3d52-46bf-9c38-03dc2287f1b1" + } + ], + "cost": 0.0, + "response": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I need to create a file called `target.py` with the exact content specified. Let me do this using bash.\n", + "function_call": null, + "provider_specific_fields": { + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "reasoning_details": [ + { + "format": "unknown", + "index": 0, + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "type": "reasoning.text" + } + ], + "refusal": null + }, + "reasoning_content": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}", + "name": "bash" + }, + "id": "call-5d82ed14-3d52-46bf-9c38-03dc2287f1b1", + "index": 0, + "type": "function" + } + ] + }, + "provider_specific_fields": { + "native_finish_reason": "tool_calls" + } + } + ], + "created": 1790166162, + "id": "gen-1790166162-a4Pz0ock4X3igVfzocqe", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "object": "chat.completion", + "provider": "Nvidia", + "service_tier": null, + "system_fingerprint": null, + "usage": { + "completion_tokens": 124, + "completion_tokens_details": { + "audio_tokens": 0, + "image_tokens": 0, + "reasoning_tokens": 62 + }, + "cost": 0, + "cost_details": { + "upstream_inference_completions_cost": 0, + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0 + }, + "is_byok": false, + "prompt_tokens": 1000, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_creation_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 0, + "video_tokens": 0 + }, + "total_tokens": 1124 + } + }, + "timestamp": 1790166176.1769083 + }, + "function_call": null, + "provider_specific_fields": { + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "reasoning_details": [ + { + "format": "unknown", + "index": 0, + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "type": "reasoning.text" + } + ], + "refusal": null + }, + "reasoning_content": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}", + "name": "bash" + }, + "id": "call-5d82ed14-3d52-46bf-9c38-03dc2287f1b1", + "index": 0, + "type": "function" + } + ] + }, + "raw_response": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I need to create a file called `target.py` with the exact content specified. Let me do this using bash.\n", + "function_call": null, + "provider_specific_fields": { + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "reasoning_details": [ + { + "format": "unknown", + "index": 0, + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "type": "reasoning.text" + } + ], + "refusal": null + }, + "reasoning_content": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}", + "name": "bash" + }, + "id": "call-5d82ed14-3d52-46bf-9c38-03dc2287f1b1", + "index": 0, + "type": "function" + } + ] + }, + "provider_specific_fields": { + "native_finish_reason": "tool_calls" + } + } + ], + "created": 1790166162, + "id": "gen-1790166162-a4Pz0ock4X3igVfzocqe", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "object": "chat.completion", + "provider": "Nvidia", + "service_tier": null, + "system_fingerprint": null, + "usage": { + "completion_tokens": 124, + "completion_tokens_details": { + "audio_tokens": 0, + "image_tokens": 0, + "reasoning_tokens": 62 + }, + "cost": 0, + "cost_details": { + "upstream_inference_completions_cost": 0, + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0 + }, + "is_byok": false, + "prompt_tokens": 1000, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_creation_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 0, + "video_tokens": 0 + }, + "total_tokens": 1124 + } + }, + "response_type": "ModelResponse", + "run": 2, + "status": "PASS", + "top_level_keys": null + }, + { + "elapsed_seconds": 3.141, + "finish_reason": "tool_calls", + "has_choices": true, + "has_tool_calls": true, + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "query_result": { + "content": "I'll create the target.py file with the exact content specified in the issue.\n", + "extra": { + "actions": [ + { + "command": "cat <<'EOF' > target.py\ndef answer():\n return 'ARKX_PREFLIGHT_OK'\nEOF", + "tool_call_id": "call-16a1801f-c971-4154-9d82-1e1ae0090238" + } + ], + "cost": 0.0, + "response": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll create the target.py file with the exact content specified in the issue.\n", + "function_call": null, + "provider_specific_fields": { + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "reasoning_details": [ + { + "format": "unknown", + "index": 0, + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "type": "reasoning.text" + } + ], + "refusal": null + }, + "reasoning_content": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}", + "name": "bash" + }, + "id": "call-16a1801f-c971-4154-9d82-1e1ae0090238", + "index": 0, + "type": "function" + } + ] + }, + "provider_specific_fields": { + "native_finish_reason": "tool_calls" + } + } + ], + "created": 1790166176, + "id": "gen-1790166176-jfr9ZmScZutl95APfLL4", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "object": "chat.completion", + "provider": "Nvidia", + "service_tier": null, + "system_fingerprint": null, + "usage": { + "completion_tokens": 116, + "completion_tokens_details": { + "audio_tokens": 0, + "image_tokens": 0, + "reasoning_tokens": 62 + }, + "cost": 0, + "cost_details": { + "upstream_inference_completions_cost": 0, + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0 + }, + "is_byok": false, + "prompt_tokens": 1000, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_creation_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 0, + "video_tokens": 0 + }, + "total_tokens": 1116 + } + }, + "timestamp": 1790166179.318726 + }, + "function_call": null, + "provider_specific_fields": { + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "reasoning_details": [ + { + "format": "unknown", + "index": 0, + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "type": "reasoning.text" + } + ], + "refusal": null + }, + "reasoning_content": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}", + "name": "bash" + }, + "id": "call-16a1801f-c971-4154-9d82-1e1ae0090238", + "index": 0, + "type": "function" + } + ] + }, + "raw_response": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll create the target.py file with the exact content specified in the issue.\n", + "function_call": null, + "provider_specific_fields": { + "reasoning": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "reasoning_details": [ + { + "format": "unknown", + "index": 0, + "text": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "type": "reasoning.text" + } + ], + "refusal": null + }, + "reasoning_content": "The user wants me to create a file called `target.py` with a specific function that returns 'ARKX_PREFLIGHT_OK'. This is a straightforward task - I just need to create the file with the exact content specified.\n\nLet me create the file using bash.", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"cat <<'EOF' > target.py\\ndef answer():\\n return 'ARKX_PREFLIGHT_OK'\\nEOF\"}", + "name": "bash" + }, + "id": "call-16a1801f-c971-4154-9d82-1e1ae0090238", + "index": 0, + "type": "function" + } + ] + }, + "provider_specific_fields": { + "native_finish_reason": "tool_calls" + } + } + ], + "created": 1790166176, + "id": "gen-1790166176-jfr9ZmScZutl95APfLL4", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "object": "chat.completion", + "provider": "Nvidia", + "service_tier": null, + "system_fingerprint": null, + "usage": { + "completion_tokens": 116, + "completion_tokens_details": { + "audio_tokens": 0, + "image_tokens": 0, + "reasoning_tokens": 62 + }, + "cost": 0, + "cost_details": { + "upstream_inference_completions_cost": 0, + "upstream_inference_cost": 0, + "upstream_inference_prompt_cost": 0 + }, + "is_byok": false, + "prompt_tokens": 1000, + "prompt_tokens_details": { + "audio_tokens": 0, + "cache_creation_tokens": 0, + "cache_write_tokens": 0, + "cached_tokens": 0, + "video_tokens": 0 + }, + "total_tokens": 1116 + } + }, + "response_type": "ModelResponse", + "run": 3, + "status": "PASS", + "top_level_keys": null + } +] \ No newline at end of file diff --git a/logs/p82-block1-compatibility-20260923/probe-result.json b/logs/p82-block1-compatibility-20260923/probe-result.json new file mode 100644 index 00000000..c17a7bed --- /dev/null +++ b/logs/p82-block1-compatibility-20260923/probe-result.json @@ -0,0 +1,64 @@ +{ + "schema_version": "p8.2-block1-probe-v1", + "block": "BLOCK_1", + "status": "QUALIFIED_FOR_NEXT_BLOCK", + "timestamp_local": "2026-09-23", + "scope": "bounded compatibility probes only; no SWE-bench execution", + "candidate_identity": { + "provider": "OpenRouter", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "runner_version": "p8.2a-runner-v3", + "environment": "git-bash", + "bash_executable": "C:\\Program Files\\Git\\bin\\bash.exe", + "docker_context": "desktop-linux", + "fallbacks": false + }, + "probes": { + "git_bash": { + "command": "printf 'ARKX_BLOCK1_GIT_BASH_OK\\n'", + "exit_code": 0, + "stdout": "ARKX_BLOCK1_GIT_BASH_OK", + "status": "PASS" + }, + "docker_context": { + "command": "docker context show", + "stdout": "desktop-linux", + "status": "PASS" + }, + "docker_version": { + "client": "29.7.2", + "server": "29.7.2", + "os": "linux", + "architecture": "amd64", + "status": "PASS" + }, + "docker_info": { + "server_version": "29.7.2", + "os_type": "linux", + "architecture": "x86_64", + "status": "PASS" + }, + "adapter_and_parser": { + "tests": [ + "tests.test_p82_openrouter_adapter", + "tests.test_p82_block_plan" + ], + "tests_passed": 6, + "status": "PASS" + } + }, + "sandbox_observation": { + "initial_git_bash_probe": "CreateFileMapping Win32 error 5", + "initial_docker_probe": "permission denied on dockerDesktopLinuxEngine named pipe", + "interpretation": "sandbox access restriction; both probes passed with elevated access" + }, + "execution_boundary": { + "swebench_executed": false, + "wave0_executed": false, + "b1_executed": false, + "c1_executed": false, + "d1_executed": false, + "protocol_changed": false + }, + "next_gate": "BLOCK_2_DIFFICULTY_SMOKE_REQUIRES_EXPLICIT_FIXTURE_SET" +} diff --git a/logs/p82-block2-smoke-20260923/smoke-result.json b/logs/p82-block2-smoke-20260923/smoke-result.json new file mode 100644 index 00000000..a7e32a5e --- /dev/null +++ b/logs/p82-block2-smoke-20260923/smoke-result.json @@ -0,0 +1,34 @@ +{ + "schema_version": "p8.2-block2-smoke-result-v1", + "manifest": "experiments/p82-block2-smoke-manifest.json", + "status": "PASS_CONTRACT_SMOKE", + "cells": { + "T0-headless": "PASS", + "T1-localized": "PASS", + "T2-localization-contract": "PASS" + }, + "tests": { + "count": 42, + "result": "PASS", + "compileall": "PASS", + "diff_check": "PASS" + }, + "model_calls": 0, + "provider_calls": 0, + "swebench_executed": false, + "interpretation": { + "proves": [ + "deterministic fixture and contract behavior", + "bounded localization contract behavior", + "local test harness consistency" + ], + "does_not_prove": [ + "model capability", + "task resolution", + "B1 effectiveness", + "C1 effectiveness", + "Wave 0 readiness" + ] + }, + "next_gate": "BLOCK_3_REQUIRES_EXPLICIT_CANDIDATE_MATRIX_AND_QUALIFIED_CELLS" +} diff --git a/logs/p82-block3-local-t0-20260923/expanded-result.json b/logs/p82-block3-local-t0-20260923/expanded-result.json new file mode 100644 index 00000000..db5e9836 --- /dev/null +++ b/logs/p82-block3-local-t0-20260923/expanded-result.json @@ -0,0 +1,20 @@ +{ + "schema_version": "p8.2-block3-t0-expanded-v1", + "harness": "ollama-api-json-local", + "prompt": "Respond with exactly ARKX_T0_OK and nothing else.", + "generation": {"temperature": 0, "num_predict": 32, "stream": false}, + "results": [ + {"candidate": "local/gemma3-4b", "runtime_model": "gemma3:4b", "status": "PASS_T0_FORMAT", "elapsed_seconds": 21.34, "eval_count": 9, "replicate": true}, + {"candidate": "local/phi4-mini", "runtime_model": "phi4-mini:latest", "status": "PASS_T0_FORMAT", "elapsed_seconds": 11.515, "eval_count": 6}, + {"candidate": "local/granite3.3-8b", "runtime_model": "granite3.3:8b", "status": "PASS_T0_FORMAT", "elapsed_seconds": 29.266, "eval_count": 8}, + {"candidate": "local/qwen2.5-coder-3b", "runtime_model": "qwen2.5-coder:3b", "status": "PASS_T0_FORMAT", "elapsed_seconds": 25.89, "eval_count": 6}, + {"candidate": "local/qwen2.5-coder-7b", "runtime_model": "qwen2.5-coder:7b", "status": "INDETERMINATE_CAPTURE", "reason": "no auditable JSON returned within bounded command window"} + ], + "qualification_boundary": { + "t0_format_pass_count": 4, + "t1_or_t2_qualified_count": 0, + "resolution_evidence": false, + "mechanism_evidence": false + }, + "next_gate": "run the fixed functional JSON probe on T1 candidates only" +} diff --git a/logs/p82-block3-local-t0-20260923/probe-result.json b/logs/p82-block3-local-t0-20260923/probe-result.json new file mode 100644 index 00000000..2d452e96 --- /dev/null +++ b/logs/p82-block3-local-t0-20260923/probe-result.json @@ -0,0 +1,45 @@ +{ + "schema_version": "p8.2-block3-t0-result-v1", + "matrix": "experiments/p82-candidate-matrix.json", + "harness": "ollama-api-json-local", + "prompt": "Respond with exactly ARKX_T0_OK and nothing else.", + "generation": {"temperature": 0, "num_predict": 32, "stream": false}, + "results": [ + { + "candidate": "local/gemma3-4b", + "runtime_model": "gemma3:4b", + "status": "PASS_T0_FORMAT", + "response": "ARKX_T0_OK", + "done": true, + "elapsed_seconds": 21.34, + "eval_count": 9 + }, + { + "candidates": [ + "local/granite3.3-8b", + "local/phi4-mini", + "local/qwen2.5-coder-3b", + "local/qwen2.5-coder-7b", + "local/qwen2.5-coder-14b", + "local/glm4-9b", + "local/starcoder2-7b", + "local/nemotron-nano-4b", + "local/deepseek-r1-7b" + ], + "status": "INDETERMINATE_CAPTURE", + "reason": "probe command did not return auditable JSON within the bounded command window; no failure is inferred" + } + ], + "runtime_inventory": { + "ollama_version": "0.34.3", + "api_tags_count": 11, + "excluded": {"qwen3.6:35b": "outside recorded 16 GB envelope"} + }, + "interpretation": { + "qualifies_for_next_t0_adapter_work": ["local/gemma3-4b"], + "qualifies_for_t1_or_t2": [], + "model_capability_evidence": false, + "task_resolution_evidence": false + }, + "next_gate": "stabilize bounded local JSON collector before additional candidate probes" +} diff --git a/logs/p82-block3-local-t1-20260923-v2/gemma-result.json b/logs/p82-block3-local-t1-20260923-v2/gemma-result.json new file mode 100644 index 00000000..9b7d06ef --- /dev/null +++ b/logs/p82-block3-local-t1-20260923-v2/gemma-result.json @@ -0,0 +1,20 @@ +{ + "schema": "experiments/p82-block3-t1-functional-v2.json", + "candidate": "local/gemma3-4b", + "runtime_model": "gemma3:4b", + "status": "FORMAT_OR_CONTENT_FAIL", + "functional_status": "FORMAT_OR_CONTENT_FAIL", + "num_predict": 128, + "elapsed_seconds": 6.468, + "eval_count": 71, + "done": true, + "validation_error": "Expecting value: line 1 column 1 (char 0)", + "raw_response": "```python\\nimport pytest\\n\\ndef validate_value(value):\\n if value is None:\\n raise ValueError('value is required')\\n return value\\n\\ndef test_validate_value():\\n pytest.raises(ValueError, validate_value, None)\\n assert validate_value(7) == 7\\n```", + "diagnosis": "complete response, but output contract requested JSON and model returned a Python code block", + "interpretation": { + "truncation": false, + "model_capability_proven": false, + "functional_output_contract_under_v2": "FAIL", + "semantic_acceptance": "NOT_RUN" + } +} diff --git a/logs/p82-block3-local-t1-20260923-v2/ollama-runtime-diagnostic.json b/logs/p82-block3-local-t1-20260923-v2/ollama-runtime-diagnostic.json new file mode 100644 index 00000000..2132551d --- /dev/null +++ b/logs/p82-block3-local-t1-20260923-v2/ollama-runtime-diagnostic.json @@ -0,0 +1,22 @@ +{ + "schema": "p8.2-local-runtime-diagnostic-v1", + "source": "C:\\Users\\igorb\\AppData\\Local\\Ollama\\server.log", + "observed": [ + "Load failed: timed out waiting for llama-server to start: context canceled", + "waiting for llama-server to become available: status=llm server error", + "repeated llama-server started cycles followed by availability errors" + ], + "affected_probe_group": [ + "local/phi4-mini", + "local/granite3.3-8b", + "local/qwen2.5-coder-3b" + ], + "classification": "LOCAL_RUNTIME_MODEL_LOAD_FAILURE_OR_AVAILABILITY", + "confidence": "supported_for_runtime_layer_only", + "not_concluded": [ + "individual model incapability", + "T1 semantic failure", + "candidate rejection" + ], + "decision": "pause additional local T1 probes until one stable model load and response capture are observed" +} diff --git a/logs/p82-block3-local-t1-20260923-v2/round-summary.json b/logs/p82-block3-local-t1-20260923-v2/round-summary.json new file mode 100644 index 00000000..550614b6 --- /dev/null +++ b/logs/p82-block3-local-t1-20260923-v2/round-summary.json @@ -0,0 +1,11 @@ +{ + "schema": "experiments/p82-block3-t1-functional-v2.json", + "status": "INCOMPLETE_OBSERVABILITY", + "results": [ + {"candidate": "local/gemma3-4b", "status": "FORMAT_OR_CONTENT_FAIL", "evidence": "gemma-result.json"}, + {"candidate": "local/phi4-mini", "status": "INDETERMINATE_CAPTURE", "reason": "no auditable stdout returned"}, + {"candidate": "local/granite3.3-8b", "status": "INDETERMINATE_CAPTURE", "reason": "no auditable stdout returned"}, + {"candidate": "local/qwen2.5-coder-3b", "status": "INDETERMINATE_CAPTURE", "reason": "no auditable stdout returned"} + ], + "interpretation": "Only Gemma produced a classified functional response; no other candidate is promoted or rejected from this round." +} diff --git a/logs/p82-block3-local-t1-20260923/gemma-result.json b/logs/p82-block3-local-t1-20260923/gemma-result.json new file mode 100644 index 00000000..3057ceb8 --- /dev/null +++ b/logs/p82-block3-local-t1-20260923/gemma-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "p8.2-block3-t1-result-v1", + "candidate": "local/gemma3-4b", + "runtime_model": "gemma3:4b", + "harness": "ollama-api-json-local", + "temperature": 0, + "num_predict": 32, + "status": "FORMAT_OR_CONTENT_FAIL", + "functional_status": "FORMAT_OR_CONTENT_FAIL", + "elapsed_seconds": 21.188, + "eval_count": 32, + "done": true, + "validation_error": "Expecting value: line 1 column 1 (char 0)", + "raw_response_prefix": "```python\\nimport pytest\\n\\ndef validate_value(value):\\n if value is None:\\n raise ValueError('value is required')\\n return", + "diagnosis": "response exceeded the 32-token output bound and was truncated before valid JSON and test content completed", + "interpretation": { + "model_capability_proven": false, + "semantic_task_failure_proven": false, + "output_contract_under_this_budget": "FAIL" + }, + "next_step": "predeclare a separate functional-probe output budget before any repeat" +} diff --git a/logs/p82-provider-gate-nemotron-ultra-v3-20260923/model-observability.json b/logs/p82-provider-gate-nemotron-ultra-v3-20260923/model-observability.json new file mode 100644 index 00000000..dc03cb5a --- /dev/null +++ b/logs/p82-provider-gate-nemotron-ultra-v3-20260923/model-observability.json @@ -0,0 +1 @@ +{"events": [{"call_index": null, "duration_seconds": 11.235, "event": "model_call", "finished_at": "2026-09-23T13:02:49.792+00:00", "model": "nvidia/nemotron-3-ultra-550b-a55b:free", "outcome": "response", "started_at": "2026-09-23T13:02:38.549+00:00", "timeout_origin": "provider_request"}], "updated_at": "2026-09-23T13:02:49.792+00:00"} diff --git a/logs/p82-provider-gate-nemotron-ultra-v3-20260923/probe-result.json b/logs/p82-provider-gate-nemotron-ultra-v3-20260923/probe-result.json new file mode 100644 index 00000000..1f411bd7 --- /dev/null +++ b/logs/p82-provider-gate-nemotron-ultra-v3-20260923/probe-result.json @@ -0,0 +1,13 @@ +{ + "adapter": "tools.p82_openrouter_model.ArkxOpenRouterModel", + "cost": null, + "elapsed_seconds": 11.235, + "fallbacks": false, + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "model_calls": null, + "outcome": "normal_response", + "provider": "OpenRouter", + "provider_ready": true, + "response": "{'role': 'assistant', 'content': None, 'refusal': None, 'reasoning': 'The user wants me to respond with exactly one executable shell action and no prose. This means I should use the bash tool to execute a command, and not provide any additional text or explanation.\\n\\nLet me think about what command to run. Since no specific task was given, I should probably run a simple, harmless command like `ls` or `pwd` to show the current directory.', 'tool_calls': [{'type': 'function', 'index': 0, 'id': 'call-3a4f686e-662d-453c-9506-ec6610b368e8', 'function': {'name': 'bash', 'arguments': '{\"command\":\"ls -la\"}'}}], 'reasoning_details': [{'type': 'reasoning.text', 'text': 'The user wants me to respond with exactly one executable shell action and no prose. This means I should use the bash tool to execute a command, and not provide any additional text or explanation.\\n\\nLet me think about what command to run. Since no specific task was given, I should probably run a simple, harmless command like `ls` or `pwd` to show the current directory.', 'format': 'unknown', 'index': 0}], 'extra': {'actions': [{'command': 'ls -la', 'tool_call_id': 'call-3a4f686e-662d-453c-9506-ec6610b368e8'}], 'response': {'id': 'gen-1790168559-p3H3CygU3h9vFZX1CXL8', 'object': 'chat.completion', 'created': 1790168559, 'model': 'nvidia/nemotron-3-ultra-550b-a55b:free', 'provider': 'Nvidia', 'system_fingerprint': None, 'service_tier': None, 'choices': [{'index': 0, 'logprobs': None, 'finish_reason': 'tool_calls', 'native_finish_reason': 'tool_calls', 'message': {'role': 'assistant', 'content': None, 'refusal': None, 'reasoning': 'The user wants me to respond with exactly one executable shell action and no prose. This means I should use the bash tool to execute a command, and not provide any additional text or explanation.\\n\\nLet me think about what command to run. Since no specific task was given, I should probably run a simple, harmless command like `ls` or `pwd` to show the current directory.', 'tool_calls': [{'type': 'function', 'index': 0, 'id': 'call-3a4f686e-662d-453c-9506-ec6610b368e8', 'function': {'name': 'bash', 'arguments': '{\"command\":\"ls -la\"}'}}], 'reasoning_details': [{'type': 'reasoning.text', 'text': 'The user wants me to respond with exactly one executable shell action and no prose. This means I should use the bash tool to execute a command, and not provide any additional text or explanation.\\n\\nLet me think about what command to run. Since no specific task was given, I should probably run a simple, harmless command like `ls` or `pwd` to show the current directory.', 'format': 'unknown', 'index': 0}]}}], 'usage': {'prompt_tokens': 286, 'completion_tokens': 104, 'total_tokens': 390, 'cost': 0, 'is_byok': False, 'prompt_tokens_details': {'cached_tokens': 0, 'cache_write_tokens': 0, 'audio_tokens': 0, 'video_tokens': 0}, 'cost_details': {'upstream_inference_cost': 0, 'upstream_inference_prompt_cost': 0, 'upstream_inference_completions_cost': 0}, 'completion_tokens_details': {'reasoning_tokens': 93, 'image_tokens': 0, 'audio_tokens': 0}}}, 'cost': 0, 'timestamp': 1790168569.7933867}}", + "started_at": 1790168558.5498474 +} diff --git a/logs/p82-wave0-treatment-a-v3-20260923/docker-diagnostic-02.json b/logs/p82-wave0-treatment-a-v3-20260923/docker-diagnostic-02.json new file mode 100644 index 00000000..aa5243c8 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-20260923/docker-diagnostic-02.json @@ -0,0 +1,23 @@ +{ + "diagnostic": "docker-authority-preflight", + "state": "BLOCKED", + "context_active": "desktop-linux", + "contexts": { + "default": "npipe:////./pipe/docker_engine", + "desktop-linux": "npipe:////./pipe/dockerDesktopLinuxEngine" + }, + "client": { + "version": "29.7.2", + "api_version": "1.55", + "os_arch": "windows/amd64" + }, + "server": null, + "processes": [], + "service": null, + "failure_category": "DOCKER_DAEMON_UNAVAILABLE", + "raw_failure": "failed to connect to the docker API at npipe:////./pipe/dockerDesktopLinuxEngine; check if the path is correct and if the daemon is running: open //./pipe/dockerDesktopLinuxEngine: O sistema não pode encontrar o arquivo especificado.", + "wave0_executed": false, + "fallback_used": false, + "protocol_changed": false, + "required_manual_action": "Start Docker Desktop and wait until the Linux engine reports Running, then rerun docker version and docker info in context desktop-linux." +} diff --git a/logs/p82-wave0-treatment-a-v3-20260923/run.json b/logs/p82-wave0-treatment-a-v3-20260923/run.json new file mode 100644 index 00000000..c4f8e497 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-20260923/run.json @@ -0,0 +1,64 @@ +{ + "schema_version": "p8.2a-wave0-execution-v1", + "run_id": "p82-wave0-treatment-a-v3-20260923", + "treatment": "A", + "wave": "wave-0-baseline-smoke", + "state": "BLOCKED", + "executor_authority": "Docker", + "identity": { + "provider": "OpenRouter", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "runner_version": "p8.2a-runner-v3", + "environment": "git-bash", + "bash_executable": "C:\\Program Files\\Git\\bin\\bash.exe", + "mini_version": "2.4.6", + "temperature": 0, + "reasoning": "disabled", + "provider_allow_fallbacks": false, + "provider_retry_attempts": 3, + "provider_retry_backoff_seconds": 2.0, + "wall_time_seconds": 180, + "max_cost": "0", + "tasks_manifest": "experiments/p82-wave0-task-sample.json", + "protocol": "experiments/p82-wave0-comparison-protocol.json" + }, + "blocker": { + "category": "DOCKER_DAEMON_UNAVAILABLE", + "command": "docker version --format '{{.Server.Version}}'", + "raw_output": "failed to connect to the docker API at npipe:////./pipe/dockerDesktopLinuxEngine; check if the path is correct and if the daemon is running: open //./pipe/dockerDesktopLinuxEngine: O sistema não pode encontrar o arquivo especificado." + }, + "tasks": [ + { + "instance_id": "sympy__sympy-14711", + "terminal_state": "BLOCKED", + "classification": "BLOCKED", + "failure_category": "DOCKER_DAEMON_UNAVAILABLE", + "model_calls": null, + "tokens": null, + "cost": null, + "latency_seconds": null, + "executor_result": null, + "patch": null, + "artifacts": [] + }, + { + "instance_id": "sympy__sympy-24443", + "terminal_state": "BLOCKED", + "classification": "BLOCKED", + "failure_category": "DOCKER_DAEMON_UNAVAILABLE", + "model_calls": null, + "tokens": null, + "cost": null, + "latency_seconds": null, + "executor_result": null, + "patch": null, + "artifacts": [] + } + ], + "notes": [ + "No model execution was started.", + "No local evaluation substituted for Docker.", + "B1, C1, and D1 were not implemented or executed.", + "No protocol, commit, push, or merge was performed." + ] +} diff --git a/logs/p82-wave0-treatment-a-v3-20260923/sympy-repo-cache b/logs/p82-wave0-treatment-a-v3-20260923/sympy-repo-cache new file mode 160000 index 00000000..f14b5cd2 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-20260923/sympy-repo-cache @@ -0,0 +1 @@ +Subproject commit f14b5cd2d13c0145521eccec991c315794952117 diff --git a/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/execution.json b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/execution.json new file mode 100644 index 00000000..26876f67 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"D:\\projetos\\arkx\\logs\\p82-wave0-treatment-a-v3-20260923\\sympy__sympy-14711\\sympy__sympy-14711\\attempt-1\\execution.json","attempt":1,"diff_path":null,"exit_code":null,"failure":"Command '['D:\\\\projetos\\\\arkx\\\\.venv-mini\\\\Scripts\\\\python.exe', 'D:\\\\projetos\\\\arkx\\\\tools\\\\run_mini_headless.py']' timed out after 190 seconds","failure_category":"TIMEOUT","finished_at":"2026-09-23T12:51:25.594+00:00","log_path":"D:\\projetos\\arkx\\logs\\p82-wave0-treatment-a-v3-20260923\\sympy__sympy-14711\\sympy__sympy-14711\\attempt-1\\executor.log","model_usage":null,"run_id":"c427ddd8fe0034bd","started_at":"2026-09-23T12:48:15.577+00:00","state":"EXECUTOR_FAILED","task_id":"sympy__sympy-14711","trajectory_path":"D:\\projetos\\arkx\\logs\\p82-wave0-treatment-a-v3-20260923\\sympy__sympy-14711\\sympy__sympy-14711\\attempt-1\\trajectory.json","treatment":"A"} diff --git a/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/executor.log b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/executor.log new file mode 100644 index 00000000..54674903 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/executor.log @@ -0,0 +1 @@ +Retrying in 4 seconds as it raised OpenRouterAPIError: Request failed: HTTPSConnectionPool(host='openrouter.ai', port=443): Read timed out. (read timeout=60). diff --git a/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/trajectory.json b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/trajectory.json new file mode 100644 index 00000000..f12d5cdd --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/trajectory.json @@ -0,0 +1,75 @@ +{ + "info": { + "model_stats": { + "instance_cost": 0.0, + "api_calls": 1 + }, + "config": { + "agent": { + "system_template": "You are a helpful assistant that can interact with a computer.\n", + "instance_template": "Please solve this issue: {{task}}\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\n{{system}} {{release}} {{version}} {{machine}}\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:\n\n{%- if system == \"Darwin\" -%}\n\nYou are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`.\n\n{%- endif -%}\n\n```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```\n", + "step_limit": 0, + "cost_limit": 0.0, + "wall_time_limit_seconds": 180, + "max_consecutive_format_errors": 3, + "output_path": "D:\\projetos\\arkx\\logs\\p82-wave0-treatment-a-v3-20260923\\sympy__sympy-14711\\sympy__sympy-14711\\attempt-1\\trajectory.json" + }, + "agent_type": "minisweagent.agents.default.DefaultAgent", + "model": { + "model_name": "nvidia/nemotron-3-ultra-550b-a55b:free", + "model_kwargs": { + "drop_params": true, + "temperature": 0.0, + "max_tokens": 256, + "custom_llm_provider": "openrouter", + "provider": { + "allow_fallbacks": false + } + }, + "set_cache_control": null, + "cost_tracking": "ignore_errors", + "format_error_template": "{% if finish_reason is defined and (finish_reason == \"length\" or (finish_reason == \"tool_calls\" and not has_tool_calls)) -%}\nYour previous response reached the output token limit (finish_reason={{ finish_reason }}) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly.\n{%- else -%}\nTool call error:\n\n\n{{error}}\n\n\nHere is general guidance on how to submit correct toolcalls:\n\nEvery response needs to use the 'bash' tool at least once to execute commands.\n\nCall the bash tool with your command as the argument:\n- Tool: bash\n- Arguments: {\"command\": \"your_command_here\"}\n\nIf you want to end the task, please issue the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\nwithout any other command.\n{%- endif %}\n", + "observation_template": "{%- if output.output | length < 10000 -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output\": {{ output.output | tojson }}\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- else -%}\n{\n \"returncode\": {{ output.returncode }},\n \"output_head\": {{ output.output[:5000] | tojson }},\n \"output_tail\": {{ output.output[-5000:] | tojson }},\n \"elided_chars\": {{ output.output | length - 10000 }},\n \"warning\": \"Output too long.\"\n {%- if output.exception_info %}, \"exception_info\": {{ output.exception_info | tojson }}{% endif %}\n}\n{%- endif -%}\n", + "multimodal_regex": "" + }, + "model_type": "p82_openrouter_model.ArkxOpenRouterModel", + "environment": { + "cwd": "D:\\projetos\\arkx\\logs\\p82-wave0-treatment-a-v3-20260923\\sympy__sympy-14711\\workspace", + "env": { + "PAGER": "cat", + "MANPAGER": "cat", + "LESS": "-R", + "PIP_PROGRESS_BAR": "off", + "TQDM_DISABLE": "1" + }, + "timeout": 30, + "bash_executable": "C:\\Program Files\\Git\\bin\\bash.exe" + }, + "environment_type": "p82_bash_environment.BashLocalEnvironment" + }, + "mini_version": "2.4.6", + "exit_status": "ProviderAvailabilityError", + "submission": "" + }, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: vector add 0 error\n```python\nfrom sympy.physics.vector import ReferenceFrame, Vector\nfrom sympy import symbols\nsum([N.x, (0 * N.x)])\n```\ngives a TypeError because adding zero to a vector is not accepted.\n\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nWindows 10 10.0.26200 AMD64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + }, + { + "role": "exit", + "content": "provider_error code=503 error_type=provider_overloaded: Upstream error from Nvidia: Service temporarily overloaded", + "extra": { + "exit_status": "ProviderAvailabilityError", + "submission": "", + "exception_str": "provider_error code=503 error_type=provider_overloaded: Upstream error from Nvidia: Service temporarily overloaded", + "traceback": "Traceback (most recent call last):\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 98, in run\n self.step()\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 128, in step\n return self.execute_actions(self.query())\n ^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\agents\\default.py\", line 149, in query\n message = self.model.query(self.messages)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\.venv-mini\\Lib\\site-packages\\minisweagent\\models\\openrouter_model.py\", line 104, in query\n actions = self._parse_actions(response)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"D:\\projetos\\arkx\\tools\\p82_openrouter_model.py\", line 30, in _parse_actions\n raise ProviderAvailabilityError(response[\"error\"])\np82_openrouter_model.ProviderAvailabilityError: provider_error code=503 error_type=provider_overloaded: Upstream error from Nvidia: Service temporarily overloaded\n" + } + } + ], + "trajectory_format": "mini-swe-agent-1.1" +} \ No newline at end of file diff --git a/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/workspace b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/workspace new file mode 160000 index 00000000..c6753448 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/workspace @@ -0,0 +1 @@ +Subproject commit c6753448b5c34f95e250105d76709fe4d349ca1f diff --git a/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-24443/sympy__sympy-24443/attempt-1/execution.json b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-24443/sympy__sympy-24443/attempt-1/execution.json new file mode 100644 index 00000000..27b6d9b0 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-24443/sympy__sympy-24443/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"D:\\projetos\\arkx\\logs\\p82-wave0-treatment-a-v3-20260923\\sympy__sympy-24443\\sympy__sympy-24443\\attempt-1\\execution.json","attempt":1,"diff_path":null,"exit_code":null,"failure":"Command '['D:\\\\projetos\\\\arkx\\\\.venv-mini\\\\Scripts\\\\python.exe', 'D:\\\\projetos\\\\arkx\\\\tools\\\\run_mini_headless.py']' timed out after 190 seconds","failure_category":"TIMEOUT","finished_at":"2026-09-23T12:54:35.622+00:00","log_path":"D:\\projetos\\arkx\\logs\\p82-wave0-treatment-a-v3-20260923\\sympy__sympy-24443\\sympy__sympy-24443\\attempt-1\\executor.log","model_usage":null,"run_id":"27bc4904c2120fd6","started_at":"2026-09-23T12:51:25.599+00:00","state":"EXECUTOR_FAILED","task_id":"sympy__sympy-24443","trajectory_path":null,"treatment":"A"} diff --git a/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-24443/sympy__sympy-24443/attempt-1/executor.log b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-24443/sympy__sympy-24443/attempt-1/executor.log new file mode 100644 index 00000000..e69de29b diff --git a/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-24443/workspace b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-24443/workspace new file mode 160000 index 00000000..809c53c0 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-24443/workspace @@ -0,0 +1 @@ +Subproject commit 809c53c077485ca48a206cee78340389cb83b7f1 diff --git a/logs/p82-wave0-treatment-a-v3-20260923/wave0-report.json b/logs/p82-wave0-treatment-a-v3-20260923/wave0-report.json new file mode 100644 index 00000000..2aca2534 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-20260923/wave0-report.json @@ -0,0 +1,94 @@ +{ + "schema_version": "p8.2a-wave0-report-v1", + "treatment": "A", + "wave": "wave-0-baseline-smoke", + "aggregate": { + "terminal_state": "BLOCKED", + "resolved": 0, + "unresolved": 0, + "blocked": 2, + "docker_evaluation_invoked": false, + "docker_evaluation_reason": "No trustworthy patch was produced by either agent execution." + }, + "identity": { + "provider": "OpenRouter", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "runner_version": "p8.2a-runner-v3", + "environment": "git-bash", + "bash_executable": "C:\\Program Files\\Git\\bin\\bash.exe", + "model_adapter": "p82_openrouter_model.ArkxOpenRouterModel", + "temperature": 0, + "reasoning": "disabled", + "provider_allow_fallbacks": false, + "provider_retry_attempts": 3, + "provider_retry_backoff_seconds": 2.0, + "wall_time_seconds": 180, + "max_cost": "0", + "mini_version": "2.4.6", + "task_manifest": "experiments/p82-wave0-task-sample.json", + "protocol": "experiments/p82-wave0-comparison-protocol.json" + }, + "tasks": [ + { + "instance_id": "sympy__sympy-14711", + "terminal_state": "BLOCKED", + "classification": "BLOCKED", + "failure_category": "PROVIDER_AVAILABILITY_FAILURE", + "raw_runner_state": "EXECUTOR_FAILED", + "raw_runner_failure_category": "TIMEOUT", + "model_calls": 1, + "model_calls_note": "One call is preserved in the final partial trajectory; the process timed out before aggregate retry usage was emitted.", + "tokens": null, + "cost": 0.0, + "latency_seconds": 190.017, + "retries_policy": 3, + "retries_observed": null, + "executor_result": "ProviderAvailabilityError: provider_error code=503 error_type=provider_overloaded: Upstream error from Nvidia: Service temporarily overloaded", + "swebench_result": "NOT_EXECUTED", + "patch": null, + "artifacts": { + "execution": "logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/execution.json", + "executor_log": "logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/executor.log", + "trajectory": "logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-14711/sympy__sympy-14711/attempt-1/trajectory.json", + "diff": null + }, + "workspace_diff": "empty" + }, + { + "instance_id": "sympy__sympy-24443", + "terminal_state": "BLOCKED", + "classification": "BLOCKED", + "failure_category": "PROVIDER_OR_RUNTIME_TIMEOUT", + "raw_runner_state": "EXECUTOR_FAILED", + "raw_runner_failure_category": "TIMEOUT", + "model_calls": null, + "tokens": null, + "cost": null, + "latency_seconds": 190.023, + "retries_policy": 3, + "retries_observed": null, + "executor_result": "No structured executor result before wall-time timeout", + "swebench_result": "NOT_EXECUTED", + "patch": null, + "artifacts": { + "execution": "logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-24443/sympy__sympy-24443/attempt-1/execution.json", + "executor_log": "logs/p82-wave0-treatment-a-v3-20260923/sympy__sympy-24443/sympy__sympy-24443/attempt-1/executor.log", + "trajectory": null, + "diff": null + }, + "workspace_diff": "empty" + } + ], + "gate": { + "b1_c1": "CLOSED", + "evidence_sufficient_to_open": false, + "reason": "Wave 0 produced no authoritative task acceptance results." + }, + "divergence": [], + "notes": [ + "Docker context desktop-linux was validated with Client and Server before execution.", + "No fallback executor was used.", + "B1, C1, and D1 were not executed.", + "No commit, push, or merge was performed." + ] +} diff --git a/logs/p82-wave0-treatment-a-v3-gated-20260923/runner-report.json b/logs/p82-wave0-treatment-a-v3-gated-20260923/runner-report.json new file mode 100644 index 00000000..5d0863c7 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-gated-20260923/runner-report.json @@ -0,0 +1,77 @@ +{ + "identity": { + "artifact_root": "logs\\p82-wave0-treatment-a-v3-gated-20260923", + "bash_executable": "C:\\Program Files\\Git\\bin\\bash.exe", + "environment": "git-bash", + "max_cost": "0", + "max_tokens": 256, + "mini_version": "2.4.6", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "model_configuration": { + "cost_tracking": "ignore_errors", + "model_class": "p82_openrouter_model.ArkxOpenRouterModel", + "model_kwargs": { + "custom_llm_provider": "openrouter", + "drop_params": true, + "max_tokens": 256, + "provider": { + "allow_fallbacks": false + }, + "temperature": 0.0 + } + }, + "model_version": "nvidia/nemotron-3-ultra-550b-a55b:free", + "provider": "OpenRouter", + "provider_retry_attempts": 3, + "provider_retry_backoff_seconds": 2.0, + "python_executable": "D:\\projetos\\arkx\\.venv-mini\\Scripts\\python.exe", + "reasoning_configuration": "disabled", + "runner_version": "p8.2a-runner-v3", + "temperature": "0", + "treatment": "A", + "wall_time_seconds": 180 + }, + "notes": [ + "No SWE-bench evaluator invoked without a reliable patch.", + "B1/C1/D1 not executed.", + "No commit, push, or merge performed." + ], + "run_id": "p82-wave0-treatment-a-v3-gated-20260923", + "tasks": [ + { + "artifact_path": "logs\\p82-wave0-treatment-a-v3-gated-20260923\\sympy__sympy-14711\\attempt-1\\execution.json", + "attempt": 1, + "diff_path": null, + "exit_code": null, + "failure": "NotADirectoryError: [WinError 267] O nome do diretório é inválido", + "failure_category": "AGENT_RUN_FAILURE", + "finished_at": "2026-09-23T13:05:56.016+00:00", + "log_path": null, + "model_usage": null, + "run_id": "cbf6881e8022c65e", + "started_at": "2026-09-23T13:05:56.016+00:00", + "state": "EXECUTOR_FAILED", + "task_id": "sympy__sympy-14711", + "trajectory_path": null, + "treatment": "A" + }, + { + "artifact_path": "logs\\p82-wave0-treatment-a-v3-gated-20260923\\sympy__sympy-24443\\attempt-1\\execution.json", + "attempt": 1, + "diff_path": null, + "exit_code": null, + "failure": "NotADirectoryError: [WinError 267] O nome do diretório é inválido", + "failure_category": "AGENT_RUN_FAILURE", + "finished_at": "2026-09-23T13:05:56.026+00:00", + "log_path": null, + "model_usage": null, + "run_id": "b85696f5e6b56f90", + "started_at": "2026-09-23T13:05:56.026+00:00", + "state": "EXECUTOR_FAILED", + "task_id": "sympy__sympy-24443", + "trajectory_path": null, + "treatment": "A" + } + ], + "treatment": "A" +} diff --git a/logs/p82-wave0-treatment-a-v3-gated-20260923/sympy__sympy-14711/attempt-1/execution.json b/logs/p82-wave0-treatment-a-v3-gated-20260923/sympy__sympy-14711/attempt-1/execution.json new file mode 100644 index 00000000..9313f836 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-gated-20260923/sympy__sympy-14711/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"logs\\p82-wave0-treatment-a-v3-gated-20260923\\sympy__sympy-14711\\attempt-1\\execution.json","attempt":1,"diff_path":null,"exit_code":null,"failure":"NotADirectoryError: [WinError 267] O nome do diretório é inválido","failure_category":"AGENT_RUN_FAILURE","finished_at":"2026-09-23T13:05:56.016+00:00","log_path":null,"model_usage":null,"run_id":"cbf6881e8022c65e","started_at":"2026-09-23T13:05:56.016+00:00","state":"EXECUTOR_FAILED","task_id":"sympy__sympy-14711","trajectory_path":null,"treatment":"A"} diff --git a/logs/p82-wave0-treatment-a-v3-gated-20260923/sympy__sympy-24443/attempt-1/execution.json b/logs/p82-wave0-treatment-a-v3-gated-20260923/sympy__sympy-24443/attempt-1/execution.json new file mode 100644 index 00000000..b0b11f4d --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-gated-20260923/sympy__sympy-24443/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"logs\\p82-wave0-treatment-a-v3-gated-20260923\\sympy__sympy-24443\\attempt-1\\execution.json","attempt":1,"diff_path":null,"exit_code":null,"failure":"NotADirectoryError: [WinError 267] O nome do diretório é inválido","failure_category":"AGENT_RUN_FAILURE","finished_at":"2026-09-23T13:05:56.026+00:00","log_path":null,"model_usage":null,"run_id":"b85696f5e6b56f90","started_at":"2026-09-23T13:05:56.026+00:00","state":"EXECUTOR_FAILED","task_id":"sympy__sympy-24443","trajectory_path":null,"treatment":"A"} diff --git a/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/runner-report.json b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/runner-report.json new file mode 100644 index 00000000..cc6cd030 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/runner-report.json @@ -0,0 +1,80 @@ +{ + "identity": { + "artifact_root": "logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923", + "bash_executable": "C:\\Program Files\\Git\\bin\\bash.exe", + "environment": "git-bash", + "max_cost": "0", + "max_tokens": 256, + "mini_version": "2.4.6", + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "model_configuration": { + "cost_tracking": "ignore_errors", + "model_class": "p82_openrouter_model.ArkxOpenRouterModel", + "model_kwargs": { + "custom_llm_provider": "openrouter", + "drop_params": true, + "max_tokens": 256, + "provider": { + "allow_fallbacks": false + }, + "temperature": 0.0 + } + }, + "model_version": "nvidia/nemotron-3-ultra-550b-a55b:free", + "provider": "OpenRouter", + "provider_retry_attempts": 3, + "provider_retry_backoff_seconds": 2.0, + "python_executable": "D:\\projetos\\arkx\\.venv-mini\\Scripts\\python.exe", + "reasoning_configuration": "disabled", + "runner_version": "p8.2a-runner-v3", + "temperature": "0", + "treatment": "A", + "wall_time_seconds": 180 + }, + "notes": [ + "No SWE-bench evaluator invoked without a reliable patch.", + "B1/C1/D1 not executed.", + "No commit, push, or merge performed." + ], + "run_id": "p82-wave0-treatment-a-v3-gated-rerun-20260923", + "tasks": [ + { + "artifact_path": "logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-14711\\attempt-1\\execution.json", + "attempt": 1, + "diff_path": "logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-14711\\attempt-1\\diff.patch", + "exit_code": 75, + "failure": "provider availability failure after bounded retries", + "failure_category": "PROVIDER_AVAILABILITY_FAILURE", + "finished_at": "2026-09-23T13:07:48.907+00:00", + "log_path": "logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-14711\\attempt-1\\executor.log", + "model_usage": { + "cost": 0.0, + "model_calls": 2 + }, + "run_id": "cbf6881e8022c65e", + "started_at": "2026-09-23T13:06:36.247+00:00", + "state": "BLOCKED", + "task_id": "sympy__sympy-14711", + "trajectory_path": "logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-14711\\attempt-1\\trajectory.json", + "treatment": "A" + }, + { + "artifact_path": "logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-24443\\attempt-1\\execution.json", + "attempt": 1, + "diff_path": null, + "exit_code": null, + "failure": "Command '['D:\\\\projetos\\\\arkx\\\\.venv-mini\\\\Scripts\\\\python.exe', 'D:\\\\projetos\\\\arkx\\\\tools\\\\run_mini_headless.py']' timed out after 190 seconds", + "failure_category": "TIMEOUT", + "finished_at": "2026-09-23T13:10:58.921+00:00", + "log_path": "logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-24443\\attempt-1\\executor.log", + "model_usage": null, + "run_id": "b85696f5e6b56f90", + "started_at": "2026-09-23T13:07:48.911+00:00", + "state": "EXECUTOR_FAILED", + "task_id": "sympy__sympy-24443", + "trajectory_path": null, + "treatment": "A" + } + ], + "treatment": "A" +} diff --git a/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-14711/attempt-1/diff.patch b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-14711/attempt-1/diff.patch new file mode 100644 index 00000000..e69de29b diff --git a/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-14711/attempt-1/execution.json b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-14711/attempt-1/execution.json new file mode 100644 index 00000000..75bd6db6 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-14711/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-14711\\attempt-1\\execution.json","attempt":1,"diff_path":"logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-14711\\attempt-1\\diff.patch","exit_code":75,"failure":"provider availability failure after bounded retries","failure_category":"PROVIDER_AVAILABILITY_FAILURE","finished_at":"2026-09-23T13:07:48.907+00:00","log_path":"logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-14711\\attempt-1\\executor.log","model_usage":{"cost":0.0,"model_calls":2},"run_id":"cbf6881e8022c65e","started_at":"2026-09-23T13:06:36.247+00:00","state":"BLOCKED","task_id":"sympy__sympy-14711","trajectory_path":"logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-14711\\attempt-1\\trajectory.json","treatment":"A"} diff --git a/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-14711/attempt-1/executor.log b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-14711/attempt-1/executor.log new file mode 100644 index 00000000..7c12679e --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-14711/attempt-1/executor.log @@ -0,0 +1,7 @@ +{"result": {"exit_status": "ProviderAvailabilityFailure", "failure_category": "PROVIDER_AVAILABILITY_FAILURE", "provider_error": {"code": 503, "message": "Upstream error from Nvidia: Service temporarily overloaded", "metadata": {"error_type": "provider_overloaded"}}, "provider_retry_attempts": 3, "submission": ""}, "usage": {"cost": 0.0, "model_calls": 2}} +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. diff --git a/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-14711/workspace b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-14711/workspace new file mode 160000 index 00000000..c6753448 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-14711/workspace @@ -0,0 +1 @@ +Subproject commit c6753448b5c34f95e250105d76709fe4d349ca1f diff --git a/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/attempt-1/execution.json b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/attempt-1/execution.json new file mode 100644 index 00000000..7e10a9eb --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/attempt-1/execution.json @@ -0,0 +1 @@ +{"artifact_path":"logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-24443\\attempt-1\\execution.json","attempt":1,"diff_path":null,"exit_code":null,"failure":"Command '['D:\\\\projetos\\\\arkx\\\\.venv-mini\\\\Scripts\\\\python.exe', 'D:\\\\projetos\\\\arkx\\\\tools\\\\run_mini_headless.py']' timed out after 190 seconds","failure_category":"TIMEOUT","finished_at":"2026-09-23T13:10:58.921+00:00","log_path":"logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-24443\\attempt-1\\executor.log","model_usage":null,"run_id":"b85696f5e6b56f90","started_at":"2026-09-23T13:07:48.911+00:00","state":"EXECUTOR_FAILED","task_id":"sympy__sympy-24443","trajectory_path":null,"treatment":"A"} diff --git a/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/attempt-1/executor.log b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/attempt-1/executor.log new file mode 100644 index 00000000..14548811 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/attempt-1/executor.log @@ -0,0 +1,9 @@ +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 8 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: HTTP 404: {"error":{"message":"Provider returned error","code":404,"metadata":{"raw":"","provider_name":"Nvidia","is_byok":false}},"user_id":"user_3JSWe2YRaf17sei49utyTsy3tHu"}. +Retrying in 4 seconds as it raised OpenRouterAPIError: Request failed: HTTPSConnectionPool(host='openrouter.ai', port=443): Read timed out. (read timeout=60). diff --git a/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/attempt-1/timeout-observability.json b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/attempt-1/timeout-observability.json new file mode 100644 index 00000000..cb0d7dce --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/attempt-1/timeout-observability.json @@ -0,0 +1 @@ +{"event": "runner_timeout", "exception": "Command '['D:\\\\projetos\\\\arkx\\\\.venv-mini\\\\Scripts\\\\python.exe', 'D:\\\\projetos\\\\arkx\\\\tools\\\\run_mini_headless.py']' timed out after 190 seconds", "exception_type": "TimeoutExpired", "finished_at": "2026-09-23T13:10:58.921+00:00", "model_diagnostics_path": "logs\\p82-wave0-treatment-a-v3-gated-rerun-20260923\\sympy__sympy-24443\\attempt-1\\model-observability.json", "started_at": "2026-09-23T13:07:48.911+00:00", "timeout_origin": "runner_subprocess_watchdog", "timeout_seconds": 190} diff --git a/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/workspace b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/workspace new file mode 160000 index 00000000..809c53c0 --- /dev/null +++ b/logs/p82-wave0-treatment-a-v3-gated-rerun-20260923/sympy__sympy-24443/workspace @@ -0,0 +1 @@ +Subproject commit 809c53c077485ca48a206cee78340389cb83b7f1 diff --git a/logs/qualification/executor-qualification-v1-20260924.json b/logs/qualification/executor-qualification-v1-20260924.json new file mode 100644 index 00000000..7996c8c8 --- /dev/null +++ b/logs/qualification/executor-qualification-v1-20260924.json @@ -0,0 +1,23 @@ +{ + "project": "Codepro", + "run_id": "executor-qualification-v1-20260924", + "protocol": "0035-executor-qualification-v1-paired-protocol", + "status": "BLOCKED", + "executed": false, + "reason": "Second Executor Enablement did not complete.", + "required_pair": { + "executors": 2, + "task_set": "frozen", + "verifier": "same verifier identity", + "budget": "same budget", + "instrumentation": "same instrumentation", + "fallback": false + }, + "evidence_ref": "logs/architecture/second-executor-preflight-20260924.json", + "non_promotions": [ + "Baseline v1", + "routing", + "comparative benchmark", + "release qualification" + ] +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..e326946e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "codepro" +dynamic = ["version"] +description = "Executor-agnostic software-engineering chassis with empirical evidence contracts." +readme = "README.md" +requires-python = ">=3.12" +dependencies = [] + +[project.scripts] +codepro = "codepro.cli:main" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.dynamic] +version = {attr = "codepro._version.__version__"} + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] diff --git a/src/README.md b/src/README.md index 266a4f11..9f8964ef 100644 --- a/src/README.md +++ b/src/README.md @@ -1,4 +1,4 @@ # Product boundary -Reserved for Arkx product code. No runtime implementation is part of the foundation bootstrap. - +Reserved for Codepro product code. The `arkx` directory is the current Python +namespace; no runtime implementation is part of the foundation bootstrap. diff --git a/src/arkx/__init__.py b/src/arkx/__init__.py new file mode 100644 index 00000000..c1d7333a --- /dev/null +++ b/src/arkx/__init__.py @@ -0,0 +1,114 @@ +"""Stable CodePro package boundary with lazy legacy compatibility. + +The neutral result contracts are loaded eagerly. Older root-level exports are +resolved only when explicitly requested, preserving existing imports without +reintroducing the eager experimental import graph. +""" + +from __future__ import annotations + +from importlib import import_module + +__version__ = "0.3.0.dev0" + +from .execution import ExecutionResult +from .orchestration_result import ExecutionResult as OrchestrationExecutionResult + +_LEGACY_MODULES = ( + "contracts", "configuration", "event_log", "integration", "execution", + "harness", "outcomes", "verifier", "request", "selection", + "p82_localization", "p82_editing", "orchestration", "recovery", + "acceptance", "promotion", "characterization", "progress", "routing", + "planning", "verification", "handoff", "composition", + "executor_qualification", "p82", "p82_baseline", "swebench_authority", +) + + +def __getattr__(name: str): + """Resolve a legacy root export without eager subsystem imports.""" + if name == "OrchestrationExecutionResult": + return OrchestrationExecutionResult + if name == "compare_qualification_trials": + value = getattr(import_module(f"{__name__}.executor_qualification"), "compare_trials") + globals()[name] = value + return value + for module_name in _LEGACY_MODULES: + module = import_module(f"{__name__}.{module_name}") + if hasattr(module, name): + value = getattr(module, name) + globals()[name] = value + return value + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__() -> list[str]: + return sorted(set(globals()) | {"OrchestrationExecutionResult"}) + + +# Keep wildcard imports compatible with the pre-lazy public boundary. The +# values remain lazy; this is only the stable name manifest. +__all__ = tuple(""" +Event EventType ExecutionRecord ExecutionStatus ConfigurationSnapshot EventLog +AdapterIdentity AdapterPreflight CapabilityProvenance DependencyObservation +IntegrationKind PreflightStatus assess_preflight exception_error_envelope +preflight_error process_error_envelope provider_error_envelope ReplayState +RestoredChainAudit append_promotion_decision ReplayChain ReferenceReport +ChainIntegrity assess_chain_integrity replay_chain classify_chain_references +classify_snapshot_chain_references replay_events audit_restored_chain +validate_attempt_lineage_chain validate_replay_against_manifest +validate_replay_outcomes validate_replay_telemetry validate_final_chain +validate_outcome_chain ExecutionRequest ExecutionPlan CapabilityProfile +CapabilityRegistry ExecutionBudgetSpec BudgetConsumption BudgetLedger RetryDecision +RetryPolicy RetryAttemptPlan plan_retry_attempt execution_reference +validate_execution_chain ExecutionResult ContractRun Executor Provider Sandbox +ProviderRequest ProviderResponse validate_provider_identity +validate_sandbox_identity SandboxRequest FakeExecutor FakeProvider FakeSandbox +ErrorDomain ArtifactStore AuditedComparison ArtifactRecord ArtifactStatus +AttemptSnapshot AttemptStore AttemptComparison AttemptEvidence VerificationEvidence +AcceptanceEvidence ExperimentRecord ExperimentSnapshot validate_experiment_record +validate_experiment_record_audited write_experiment_record load_experiment_record +ComparabilityStatus compare_attempts ComparisonReport PairComparison +compare_attempt_collection evidence_from_snapshot LifecycleEvent ErrorEnvelope +Retryability RetryLineage RecoveryLineage RunManifest RunState derive_attempt_id +derive_experiment_identity_digest validate_transition +validate_execution_artifact_consistency load_attempt ArkxRequest GovernanceDecision +GovernancePolicy GovernanceReason GovernanceStatus RequestBudget RequestEnvironment +authorize_request characterize_authorized_request ExecutorBinding ExecutorRegistry +SelectionDecision SelectionReason SelectionStatus TreatmentCatalog +TreatmentDefinition select_executor select_treatment ExecutionOutcome +BudgetEnforcedExecutorRunner ContractExecutorRunner OrchestrationExecutionResult +OrchestrationReason OrchestrationResult OrchestrationStatus +orchestration_status_for_acceptance orchestration_status_for_execution +run_routed_pipeline RecoveryAction RecoveryBudget RecoveryDecision RecoveryPlan +RecoveryAttemptPlan RecoveryReason decide_recovery build_recovery_plan +plan_recovery_attempt validate_recovery_reference AcceptancePolicy +decide_independent_acceptance PromotionCandidate PromotionDecision PromotionPolicy +PromotionReason PromotionStatus decide_outcome_promotion decide_promotion +CharacterizationConfig Confidence RecommendedPath Scope TaskCharacterization +TaskSignals characterize characterize_timed ProgressAssessment ProgressConfig +ProgressEvidence ProgressSnapshot ProgressStatus assess_progress assess_progress_timed +BudgetState CapabilityPolicyDecision CapabilityPolicyReason EscalationAction +EscalationDecision EvidenceSufficiency RoutingBudget RoutingDecision +RoutingDecisionType assess_escalation assess_capability_policy route_characterization +PlanStep ReplanRequest ReplanResult ReplanTrigger RepositoryPlan RepositoryState +StepStatus build_plan plan_repository PatchVerificationInput PatchVerificationResult +PatchVerificationStatus TestResult TestResultStatus verify_patch HandoffBudgetStatus +HandoffPolicy HandoffRecord HandoffSummary summarize_handoffs ExperimentIdentity +ComparabilityStatus ExperimentManifest ExperimentSummary Mechanism Treatment TrialObservation +compare_trials initial_manifest summarize_trials ComparisonAxis ExecutionEnvironment +ExecutionOutcome ExecutionBudget ExecutorIdentity ExecutorObservation ExecutorTrial +QualificationExperiment QualificationStatus PairedQualificationReport +validate_paired_executor_experiment QualificationSummary QualificationTask +assess_trial compare_qualification_trials summarize_experiment P82ExperimentConfig +P82ExperimentPlan P82Comparability P82Criteria P82Observation P82ObservationRecord +P82RunManifest P82RunStatus P82Treatment P82TreatmentSummary P82TrialPlan +TaskSampleManifest TaskSampleTask TaskStratum build_p82_plan compare_run_manifests +p82_manifest plan_is_comparable summarize_p82 AcceptanceDecision AcceptanceResult +BaselineExecutionState BaselineRunConfig BaselineSampleManifest +ExplicitVerificationAcceptance ExecutionArtifact MiniSweAgentHeadlessRunner +ProspectiveTask RunnerFailureCategory VerificationResult VerificationState +validate_outcome_references baseline_observation AUTHORITY_IDENTITY +DATASET_IDENTITY DATASET_SPLIT OfficialEvaluationResult OfficialEvaluationStatus +SWEbenchOfficialAuthority SWEbenchPrediction SWEbenchRecord load_verified_records +qualify_gold +""".split()) diff --git a/src/arkx/acceptance.py b/src/arkx/acceptance.py new file mode 100644 index 00000000..d4759e45 --- /dev/null +++ b/src/arkx/acceptance.py @@ -0,0 +1,38 @@ +"""Independent acceptance boundary for verified execution artifacts.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from .outcomes import AcceptanceDecision, AcceptanceResult, VerificationResult, VerificationState + + +@dataclass(frozen=True) +class AcceptancePolicy: + identity: str + require_verification_evidence: bool = True + require_verification_authority: bool = True + + def __post_init__(self) -> None: + if not self.identity: + raise ValueError("acceptance authority identity must be explicit") + + +def decide_independent_acceptance(verification: VerificationResult, policy: AcceptancePolicy) -> AcceptanceResult: + """Map verification to acceptance only under an explicit independent policy.""" + + evidence = tuple(sorted(set((verification.reference,) + verification.evidence))) + + if policy.require_verification_authority and not verification.authority: + return AcceptanceResult(AcceptanceDecision.BLOCKED, policy.identity, "verification authority missing", evidence) + if policy.require_verification_evidence and not verification.evidence: + return AcceptanceResult(AcceptanceDecision.BLOCKED, policy.identity, "verification evidence missing", evidence) + mapping = { + VerificationState.PASS: AcceptanceDecision.ACCEPTED, + VerificationState.FAIL: AcceptanceDecision.REJECTED, + VerificationState.INDETERMINATE: AcceptanceDecision.INDETERMINATE, + VerificationState.NOT_EXECUTED: AcceptanceDecision.NOT_EXECUTED, + VerificationState.BLOCKED: AcceptanceDecision.BLOCKED, + } + return AcceptanceResult(mapping[verification.state], policy.identity, verification.state.value, evidence) diff --git a/src/arkx/baseline.py b/src/arkx/baseline.py new file mode 100644 index 00000000..e7a6baf1 --- /dev/null +++ b/src/arkx/baseline.py @@ -0,0 +1,38 @@ +"""Deterministic, no-network P0 baseline fixture.""" + +from .contracts import Event, EventType +from .telemetry import TelemetryCollector + + +RUN_ID = "baseline-fixture-run" + + +def baseline_events() -> list[Event]: + return [ + Event("2026-01-01T00:00:00+00:00", EventType.TASK_STARTED, RUN_ID, {"task_id": "baseline-task"}), + Event("2026-01-01T00:00:00.100000+00:00", EventType.EXECUTOR_STARTED, RUN_ID, {"executor_id": "fixture"}), + Event("2026-01-01T00:00:00.200000+00:00", EventType.EVIDENCE_ADDED, RUN_ID, {"ref": "fixture://evidence/1"}), + Event("2026-01-01T00:00:00.300000+00:00", EventType.EXECUTOR_FINISHED, RUN_ID, {"total_tokens": 12}), + Event( + "2026-01-01T00:00:00.500000+00:00", + EventType.TASK_FINISHED, + RUN_ID, + {"status": "PASS", "decision_reason": "deterministic fixture completed"}, + ), + ] + + +def build_baseline_record(): + collector = TelemetryCollector(task_id="baseline-task", run_id=RUN_ID) + collector.extend(baseline_events()) + return collector.summarize() + + +def main() -> int: + print(build_baseline_record().to_json()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/src/arkx/characterization.py b/src/arkx/characterization.py new file mode 100644 index 00000000..88da3ae2 --- /dev/null +++ b/src/arkx/characterization.py @@ -0,0 +1,352 @@ +"""Deterministic, conservative task characterization for P1. + +This module consumes explicit signals only. It does not inspect natural +language, invoke a model, select an executor, or perform routing. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from enum import Enum +import json +from time import perf_counter_ns +from typing import Any, Mapping + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class Scope(_ValueEnum): + SIMPLE = "SIMPLE" + LOCALIZED = "LOCALIZED" + REPOSITORY_WIDE = "REPOSITORY_WIDE" + UNKNOWN = "UNKNOWN" + + +class RecommendedPath(_ValueEnum): + SIMPLE_PATH = "SIMPLE_PATH" + LOCALIZED_PATH = "LOCALIZED_PATH" + REPOSITORY_WIDE_PATH = "REPOSITORY_WIDE_PATH" + QUALIFICATION_REQUIRED = "QUALIFICATION_REQUIRED" + + +class Confidence(_ValueEnum): + HIGH = "HIGH" + MEDIUM = "MEDIUM" + LOW = "LOW" + + +class SignalLevel(_ValueEnum): + LOW = "LOW" + MEDIUM = "MEDIUM" + HIGH = "HIGH" + UNKNOWN = "UNKNOWN" + + +class TestSurface(_ValueEnum): + NONE = "NONE" + LIMITED = "LIMITED" + BROAD = "BROAD" + UNKNOWN = "UNKNOWN" + + +class ReasonCode(_ValueEnum): + SINGLE_FILE = "SINGLE_FILE" + BOUNDED_FILE_SET = "BOUNDED_FILE_SET" + MULTI_COMPONENT = "MULTI_COMPONENT" + SINGLE_COMPONENT = "SINGLE_COMPONENT" + LOW_DEPENDENCY_FANOUT = "LOW_DEPENDENCY_FANOUT" + HIGH_DEPENDENCY_FANOUT = "HIGH_DEPENDENCY_FANOUT" + BROAD_TEST_SURFACE = "BROAD_TEST_SURFACE" + AMBIGUOUS_ACCEPTANCE = "AMBIGUOUS_ACCEPTANCE" + RISK_MARKER = "RISK_MARKER" + ARCHITECTURAL_CHANGE = "ARCHITECTURAL_CHANGE" + SHARED_STATE = "SHARED_STATE" + INSUFFICIENT_FILE_SIGNAL = "INSUFFICIENT_FILE_SIGNAL" + INSUFFICIENT_COMPONENT_SIGNAL = "INSUFFICIENT_COMPONENT_SIGNAL" + INSUFFICIENT_ACCEPTANCE_SIGNAL = "INSUFFICIENT_ACCEPTANCE_SIGNAL" + CONTRADICTORY_SIGNALS = "CONTRADICTORY_SIGNALS" + + +@dataclass(frozen=True) +class CharacterizationConfig: + """Centralized, reviewable thresholds for the initial heuristic.""" + + max_simple_files: int = 1 + max_localized_files: int = 5 + max_localized_components: int = 2 + max_localized_fanout: int = 3 + max_simple_acceptance_checks: int = 1 + + +DEFAULT_CONFIG = CharacterizationConfig() + + +def _sorted_unique(values: tuple[str, ...] | None) -> list[str] | None: + if values is None: + return None + return sorted(set(values)) + + +@dataclass(frozen=True) +class TaskSignals: + """Explicit signals supplied by an upstream observer or fixture.""" + + candidate_files: tuple[str, ...] | None = None + dependency_edges: tuple[tuple[str, str], ...] | None = None + affected_components: tuple[str, ...] | None = None + known_tests: tuple[str, ...] | None = None + ambiguity_markers: tuple[str, ...] | None = None + risk_markers: tuple[str, ...] | None = None + acceptance_checks: tuple[str, ...] | None = None + state_shared: bool | None = None + architectural_change: bool | None = None + declared_scope: Scope | None = None + + def normalized(self) -> "TaskSignals": + edges = None if self.dependency_edges is None else tuple(sorted(set(self.dependency_edges))) + return replace( + self, + candidate_files=None if self.candidate_files is None else tuple(sorted(set(self.candidate_files))), + dependency_edges=edges, + affected_components=None if self.affected_components is None else tuple(sorted(set(self.affected_components))), + known_tests=None if self.known_tests is None else tuple(sorted(set(self.known_tests))), + ambiguity_markers=None if self.ambiguity_markers is None else tuple(sorted(set(self.ambiguity_markers))), + risk_markers=None if self.risk_markers is None else tuple(sorted(set(self.risk_markers))), + acceptance_checks=None if self.acceptance_checks is None else tuple(sorted(set(self.acceptance_checks))), + ) + + def to_dict(self) -> dict[str, Any]: + value = self.normalized() + return { + "candidate_files": _sorted_unique(value.candidate_files), + "dependency_edges": None if value.dependency_edges is None else [list(edge) for edge in value.dependency_edges], + "affected_components": _sorted_unique(value.affected_components), + "known_tests": _sorted_unique(value.known_tests), + "ambiguity_markers": _sorted_unique(value.ambiguity_markers), + "risk_markers": _sorted_unique(value.risk_markers), + "acceptance_checks": _sorted_unique(value.acceptance_checks), + "state_shared": value.state_shared, + "architectural_change": value.architectural_change, + "declared_scope": None if value.declared_scope is None else value.declared_scope.value, + } + + +@dataclass(frozen=True) +class TaskCharacterization: + schema_version: int + scope: Scope + estimated_files: int | None + dependency_fanout: int | None + test_surface: TestSurface + ambiguity: SignalLevel + risk: SignalLevel + acceptance_complexity: SignalLevel + confidence: Confidence + signals: tuple[ReasonCode, ...] + recommended_path: RecommendedPath + input_signals: dict[str, Any] + derived_signals: dict[str, Any] + telemetry: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "scope": self.scope.value, + "estimated_files": self.estimated_files, + "dependency_fanout": self.dependency_fanout, + "test_surface": self.test_surface.value, + "ambiguity": self.ambiguity.value, + "risk": self.risk.value, + "acceptance_complexity": self.acceptance_complexity.value, + "confidence": self.confidence.value, + "signals": [signal.value for signal in self.signals], + "recommended_path": self.recommended_path.value, + "input_signals": self.input_signals, + "derived_signals": self.derived_signals, + "telemetry": self.telemetry, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _test_surface(known_tests: tuple[str, ...] | None) -> TestSurface: + if known_tests is None: + return TestSurface.UNKNOWN + if not known_tests: + return TestSurface.NONE + return TestSurface.LIMITED if len(known_tests) <= 2 else TestSurface.BROAD + + +def _level_for_count(count: int | None, *, unknown: SignalLevel = SignalLevel.UNKNOWN) -> SignalLevel: + if count is None: + return unknown + if count == 0: + return SignalLevel.LOW + if count <= 2: + return SignalLevel.MEDIUM + return SignalLevel.HIGH + + +def characterize( + task_signals: TaskSignals, + *, + config: CharacterizationConfig = DEFAULT_CONFIG, + characterization_duration_ms: int | None = None, +) -> TaskCharacterization: + """Classify explicit signals conservatively and without side effects.""" + + signals = task_signals.normalized() + file_count = None if signals.candidate_files is None else len(signals.candidate_files) + fanout = None if signals.dependency_edges is None else len(signals.dependency_edges) + component_count = None if signals.affected_components is None else len(signals.affected_components) + acceptance_count = None if signals.acceptance_checks is None else len(signals.acceptance_checks) + ambiguity_count = None if signals.ambiguity_markers is None else len(signals.ambiguity_markers) + risk_count = None if signals.risk_markers is None else len(signals.risk_markers) + + reason_codes: list[ReasonCode] = [] + if file_count == 1: + reason_codes.append(ReasonCode.SINGLE_FILE) + elif file_count is not None and file_count > 1: + reason_codes.append(ReasonCode.BOUNDED_FILE_SET) + if component_count == 1: + reason_codes.append(ReasonCode.SINGLE_COMPONENT) + elif component_count is not None and component_count > 1: + reason_codes.append(ReasonCode.MULTI_COMPONENT) + if fanout is not None and fanout <= config.max_localized_fanout: + reason_codes.append(ReasonCode.LOW_DEPENDENCY_FANOUT) + elif fanout is not None: + reason_codes.append(ReasonCode.HIGH_DEPENDENCY_FANOUT) + if signals.known_tests is not None and len(signals.known_tests) > 2: + reason_codes.append(ReasonCode.BROAD_TEST_SURFACE) + if ambiguity_count: + reason_codes.append(ReasonCode.AMBIGUOUS_ACCEPTANCE) + if risk_count: + reason_codes.append(ReasonCode.RISK_MARKER) + if signals.architectural_change: + reason_codes.append(ReasonCode.ARCHITECTURAL_CHANGE) + if signals.state_shared: + reason_codes.append(ReasonCode.SHARED_STATE) + + incomplete = ( + file_count in (None, 0) + or component_count in (None, 0) + or acceptance_count in (None, 0) + ) + if file_count in (None, 0): + reason_codes.append(ReasonCode.INSUFFICIENT_FILE_SIGNAL) + if component_count in (None, 0): + reason_codes.append(ReasonCode.INSUFFICIENT_COMPONENT_SIGNAL) + if acceptance_count in (None, 0): + reason_codes.append(ReasonCode.INSUFFICIENT_ACCEPTANCE_SIGNAL) + + contradictory = ( + (file_count == 0 and fanout not in (None, 0)) + or (signals.declared_scope is Scope.SIMPLE and file_count not in (None, 1)) + or (signals.declared_scope is Scope.LOCALIZED and file_count is not None and file_count > config.max_localized_files) + or (signals.declared_scope is Scope.REPOSITORY_WIDE and file_count == 1 and component_count == 1 and fanout == 0) + ) + if contradictory: + reason_codes.append(ReasonCode.CONTRADICTORY_SIGNALS) + + ambiguity = _level_for_count(ambiguity_count) + risk = _level_for_count(risk_count) + acceptance = _level_for_count(acceptance_count) + derived = { + "component_count": component_count, + "acceptance_check_count": acceptance_count, + "ambiguity_marker_count": ambiguity_count, + "risk_marker_count": risk_count, + "has_shared_state": signals.state_shared, + "has_architectural_change": signals.architectural_change, + } + + if incomplete or contradictory or ambiguity_count: + scope = Scope.UNKNOWN + path = RecommendedPath.QUALIFICATION_REQUIRED + confidence = Confidence.LOW + elif ( + file_count <= config.max_simple_files + and component_count == 1 + and fanout == 0 + and acceptance_count <= config.max_simple_acceptance_checks + and not risk_count + ): + scope = Scope.SIMPLE + path = RecommendedPath.SIMPLE_PATH + confidence = Confidence.HIGH + elif ( + signals.architectural_change + or signals.state_shared + or component_count > config.max_localized_components + or file_count > config.max_localized_files + or fanout > config.max_localized_fanout + ): + scope = Scope.REPOSITORY_WIDE + path = RecommendedPath.REPOSITORY_WIDE_PATH + confidence = Confidence.MEDIUM + elif file_count >= 1 and component_count >= 1: + scope = Scope.LOCALIZED + path = RecommendedPath.LOCALIZED_PATH + confidence = Confidence.HIGH + else: + scope = Scope.UNKNOWN + path = RecommendedPath.QUALIFICATION_REQUIRED + confidence = Confidence.LOW + + # A risk marker is observable, but P1 does not turn it into escalation. + # It only prevents the narrow SIMPLE classification. + if scope is Scope.SIMPLE and risk_count: + scope = Scope.UNKNOWN + path = RecommendedPath.QUALIFICATION_REQUIRED + confidence = Confidence.LOW + + telemetry = { + "characterization_duration_ms": characterization_duration_ms, + "characterization_scope": scope.value, + "characterization_confidence": confidence.value, + "recommended_path": path.value, + "signal_count": sum( + value is not None + for value in ( + signals.candidate_files, + signals.dependency_edges, + signals.affected_components, + signals.known_tests, + signals.ambiguity_markers, + signals.risk_markers, + signals.acceptance_checks, + ) + ), + } + return TaskCharacterization( + schema_version=SCHEMA_VERSION, + scope=scope, + estimated_files=file_count, + dependency_fanout=fanout, + test_surface=_test_surface(signals.known_tests), + ambiguity=ambiguity, + risk=risk, + acceptance_complexity=acceptance, + confidence=confidence, + signals=tuple(dict.fromkeys(reason_codes)), + recommended_path=path, + input_signals=signals.to_dict(), + derived_signals=derived, + telemetry=telemetry, + ) + + +def characterize_timed(task_signals: TaskSignals, *, config: CharacterizationConfig = DEFAULT_CONFIG) -> TaskCharacterization: + """Measure local characterization overhead; no model or executor is called.""" + + started = perf_counter_ns() + result = characterize(task_signals, config=config) + duration_ms = max(0, (perf_counter_ns() - started) // 1_000_000) + return replace(result, telemetry={**result.telemetry, "characterization_duration_ms": duration_ms}) diff --git a/src/arkx/cli.py b/src/arkx/cli.py new file mode 100644 index 00000000..4643fad4 --- /dev/null +++ b/src/arkx/cli.py @@ -0,0 +1,65 @@ +"""Dependency-free public CLI for the packaged CodePro chassis.""" + +from __future__ import annotations + +import argparse +import json +import platform +import sys +from collections.abc import Sequence + +from . import __version__ +from .baseline import main as baseline_main + + +SUPPORTED_PYTHON_MIN = (3, 12) +SUPPORTED_PYTHON_MAX = (3, 14) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="codepro", + description="Empirical, executor-agnostic CodePro chassis.", + ) + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + commands = parser.add_subparsers(dest="command") + + doctor = commands.add_parser("doctor", help="Check the installed package and Python runtime.") + doctor.add_argument("--json", action="store_true", dest="as_json") + commands.add_parser("baseline", help="Run the deterministic, no-network baseline fixture.") + return parser + + +def _doctor(*, as_json: bool) -> int: + python_version = sys.version_info[:2] + supported = SUPPORTED_PYTHON_MIN <= python_version <= SUPPORTED_PYTHON_MAX + payload = { + "package": "codepro", + "version": __version__, + "python": platform.python_version(), + "python_supported": supported, + "runtime_dependencies": [], + "status": "PASS" if supported else "FAIL", + } + if as_json: + print(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + else: + print(f"package: PASS (codepro {__version__})") + state = "PASS" if supported else "FAIL" + print(f"python: {state} ({platform.python_version()})") + print(f"status: {payload['status']}") + return 0 if supported else 1 + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == "doctor": + return _doctor(as_json=args.as_json) + if args.command == "baseline": + return baseline_main() + build_parser().print_help() + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/arkx/composition.py b/src/arkx/composition.py new file mode 100644 index 00000000..d60162d7 --- /dev/null +++ b/src/arkx/composition.py @@ -0,0 +1,216 @@ +"""Pure experimental composition contracts for P7. + +P7 describes treatment arms and records externally supplied observations. It +does not execute mechanisms, select executors, or make acceptance decisions. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +import json +from typing import Any, Mapping + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class Mechanism(_ValueEnum): + P1_CHARACTERIZATION = "P1_CHARACTERIZATION" + P2_PROGRESS = "P2_PROGRESS" + P3_ROUTING = "P3_ROUTING" + P4_PLANNING_STATE = "P4_PLANNING_STATE" + P5_PATCH_VERIFICATION = "P5_PATCH_VERIFICATION" + P6_HANDOFF_ACCOUNTING = "P6_HANDOFF_ACCOUNTING" + P8_SAFE_EDITOR = "P8_SAFE_EDITOR" + P8_ENHANCED_REPOSITORY_CONTEXT = "P8_ENHANCED_REPOSITORY_CONTEXT" + + +_MECHANISM_ORDER = {mechanism: index for index, mechanism in enumerate(Mechanism)} + + +class ComparabilityStatus(_ValueEnum): + COMPARABLE = "COMPARABLE" + INCOMPARABLE = "INCOMPARABLE" + UNKNOWN = "UNKNOWN" + + +def _json(value: Mapping[str, Any]) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _measurement_items(items: tuple[tuple[str, Any], ...]) -> tuple[tuple[str, Any], ...]: + if any(not key for key, _ in items): + raise ValueError("Measurement keys must be non-empty") + keys = [key for key, _ in items] + if len(set(keys)) != len(keys): + raise ValueError("Measurement keys must be unique") + return tuple(sorted(items, key=lambda item: item[0])) + + +@dataclass(frozen=True) +class Treatment: + name: str + enabled_mechanisms: tuple[Mechanism, ...] + + def __post_init__(self) -> None: + if not self.name: + raise ValueError("Treatment names must be non-empty") + mechanisms = tuple(self.enabled_mechanisms) + if len(set(mechanisms)) != len(mechanisms): + raise ValueError("A treatment cannot enable a mechanism twice") + object.__setattr__(self, "enabled_mechanisms", tuple(sorted(mechanisms, key=_MECHANISM_ORDER.__getitem__))) + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "enabled_mechanisms": [mechanism.value for mechanism in self.enabled_mechanisms], + } + + +@dataclass(frozen=True) +class ExperimentIdentity: + task: str | None + executor: str | None + model: str | None + environment: str | None + budget: str | None + + def to_dict(self) -> dict[str, str | None]: + return { + "task": self.task, + "executor": self.executor, + "model": self.model, + "environment": self.environment, + "budget": self.budget, + } + + +@dataclass(frozen=True) +class TrialObservation: + identity: ExperimentIdentity + treatment: Treatment + measurements: tuple[tuple[str, Any], ...] = () + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + object.__setattr__(self, "measurements", _measurement_items(tuple(self.measurements))) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "identity": self.identity.to_dict(), + "treatment": self.treatment.to_dict(), + "measurements": {key: value for key, value in self.measurements}, + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class ExperimentManifest: + treatments: tuple[Treatment, ...] + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + treatments = tuple(sorted(self.treatments, key=lambda treatment: treatment.name)) + names = [treatment.name for treatment in treatments] + if len(set(names)) != len(names): + raise ValueError("Treatment names must be unique within a manifest") + compositions = [treatment.enabled_mechanisms for treatment in treatments] + if len(set(compositions)) != len(compositions): + raise ValueError("Semantically identical treatments are not allowed in one manifest") + object.__setattr__(self, "treatments", treatments) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "treatments": [treatment.to_dict() for treatment in self.treatments], + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class ExperimentSummary: + trial_count: int + treatment_names: tuple[str, ...] + comparability: ComparabilityStatus + measurement_keys: tuple[str, ...] + measurement_counts: tuple[tuple[str, int], ...] + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "trial_count": self.trial_count, + "treatment_names": list(self.treatment_names), + "comparability": self.comparability.value, + "measurement_keys": list(self.measurement_keys), + "measurement_counts": {key: count for key, count in self.measurement_counts}, + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + +def initial_manifest() -> ExperimentManifest: + """Return only the P7 A-F treatment arms; P8.2 is a separate manifest.""" + + return ExperimentManifest( + treatments=( + Treatment("A", ()), + Treatment("B", (Mechanism.P1_CHARACTERIZATION,)), + Treatment("C", (Mechanism.P1_CHARACTERIZATION, Mechanism.P2_PROGRESS)), + Treatment("D", (Mechanism.P1_CHARACTERIZATION, Mechanism.P2_PROGRESS, Mechanism.P3_ROUTING)), + Treatment("E", (Mechanism.P4_PLANNING_STATE,)), + Treatment("F", (Mechanism.P5_PATCH_VERIFICATION,)), + ) + ) + + +def compare_trials(left: TrialObservation, right: TrialObservation) -> ComparabilityStatus: + """Compare only controlled identity dimensions; treatment remains declarative.""" + + left_values = left.identity.to_dict() + right_values = right.identity.to_dict() + if any(left_values[key] is None or right_values[key] is None for key in left_values): + return ComparabilityStatus.UNKNOWN + if left_values != right_values: + return ComparabilityStatus.INCOMPARABLE + return ComparabilityStatus.COMPARABLE + + +def summarize_trials(trials: tuple[TrialObservation, ...]) -> ExperimentSummary: + """Aggregate observations without imputing absent measurements or accepting work.""" + + trials = tuple(trials) + if not trials: + comparability = ComparabilityStatus.UNKNOWN + else: + statuses = tuple(compare_trials(trials[0], trial) for trial in trials[1:]) + comparability = ( + ComparabilityStatus.INCOMPARABLE + if ComparabilityStatus.INCOMPARABLE in statuses + else ComparabilityStatus.UNKNOWN + if ComparabilityStatus.UNKNOWN in statuses + else ComparabilityStatus.COMPARABLE + ) + counts: dict[str, int] = {} + for trial in trials: + for key, _ in trial.measurements: + counts[key] = counts.get(key, 0) + 1 + return ExperimentSummary( + trial_count=len(trials), + treatment_names=tuple(sorted({trial.treatment.name for trial in trials})), + comparability=comparability, + measurement_keys=tuple(sorted(counts)), + measurement_counts=tuple(sorted(counts.items())), + ) diff --git a/src/arkx/configuration.py b/src/arkx/configuration.py new file mode 100644 index 00000000..641a7c69 --- /dev/null +++ b/src/arkx/configuration.py @@ -0,0 +1,52 @@ +"""Canonical, secret-safe configuration identity for reproducible runs.""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +from typing import Any, Mapping + + +@dataclass(frozen=True) +class ConfigurationSnapshot: + """Reproducibility snapshot without storing secret material.""" + + component: str + version: str + public_values: Mapping[str, Any] + secret_digests: Mapping[str, str] = () + schema_version: int = 1 + + def __post_init__(self) -> None: + if not self.component.strip() or not self.version.strip(): + raise ValueError("configuration component and version must be non-empty") + if self.schema_version < 1: + raise ValueError("configuration schema_version must be positive") + public = dict(self.public_values) + secrets = dict(self.secret_digests) + if any(not str(key).strip() for key in public) or any(not str(key).strip() for key in secrets): + raise ValueError("configuration keys must be non-empty") + if any(not str(value).strip() for value in secrets.values()): + raise ValueError("secret digests must be non-empty") + object.__setattr__(self, "public_values", public) + object.__setattr__(self, "secret_digests", secrets) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "component": self.component, + "version": self.version, + "public_values": self.public_values, + "secret_digests": self.secret_digests, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + def digest(self) -> str: + return hashlib.sha256(self.to_json().encode("utf-8")).hexdigest()[:16] + + @property + def reference(self) -> str: + return f"config://{self.digest()}" diff --git a/src/arkx/contracts.py b/src/arkx/contracts.py new file mode 100644 index 00000000..24c32fef --- /dev/null +++ b/src/arkx/contracts.py @@ -0,0 +1,149 @@ +"""Versioned, serializable contracts for P0 execution telemetry.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +import json +from pathlib import Path +from typing import Any, Mapping + + +SCHEMA_VERSION = 1 + + +class ExecutionStatus(str, Enum): + PASS = "PASS" + FAILED = "FAILED" + BLOCKED = "BLOCKED" + INVALID = "INVALID" + NOT_EXECUTED = "NOT_EXECUTED" + UNVERIFIED = "UNVERIFIED" + + +class EventType(str, Enum): + TASK_STARTED = "TASK_STARTED" + EXECUTOR_STARTED = "EXECUTOR_STARTED" + EXECUTOR_FINISHED = "EXECUTOR_FINISHED" + RETRY = "RETRY" + REPLAN = "REPLAN" + HANDOFF = "HANDOFF" + EVIDENCE_ADDED = "EVIDENCE_ADDED" + HUMAN_INTERVENTION = "HUMAN_INTERVENTION" + TASK_FINISHED = "TASK_FINISHED" + + +@dataclass(frozen=True) +class Event: + timestamp: str + type: EventType + run_id: str + data: Mapping[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "timestamp": self.timestamp, + "type": self.type.value, + "run_id": self.run_id, + "data": dict(self.data), + } + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "Event": + return cls( + timestamp=str(value["timestamp"]), + type=EventType(value["type"]), + run_id=str(value["run_id"]), + data=dict(value.get("data", {})), + ) + + +@dataclass +class ExecutionRecord: + task_id: str + run_id: str + executor_id: str | None + started_at: str | None + finished_at: str | None + status: ExecutionStatus + decision_reason: str | None + input_tokens: int | None = None + output_tokens: int | None = None + total_tokens: int | None = None + monetary_cost: float | None = None + wall_time_ms: int | None = None + retry_count: int = 0 + replan_count: int = 0 + executor_invocations: int = 0 + handoff_count: int = 0 + human_interventions: int = 0 + evidence_refs: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schemaVersion": self.schema_version, + "task_id": self.task_id, + "run_id": self.run_id, + "executor_id": self.executor_id, + "started_at": self.started_at, + "finished_at": self.finished_at, + "status": self.status.value, + "decision_reason": self.decision_reason, + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "total_tokens": self.total_tokens, + "monetary_cost": self.monetary_cost, + "wall_time_ms": self.wall_time_ms, + "retry_count": self.retry_count, + "replan_count": self.replan_count, + "executor_invocations": self.executor_invocations, + "handoff_count": self.handoff_count, + "human_interventions": self.human_interventions, + "evidence_refs": list(self.evidence_refs), + "errors": list(self.errors), + "metadata": dict(self.metadata), + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + def write_json(self, path: str | Path) -> None: + """Write one deterministic record; callers choose the experiment path.""" + + Path(path).write_text(self.to_json() + "\n", encoding="utf-8") + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "ExecutionRecord": + schema_version = int(value.get("schemaVersion", 0)) + if schema_version != SCHEMA_VERSION: + raise ValueError(f"Unsupported execution record schema: {schema_version}") + return cls( + task_id=str(value["task_id"]), + run_id=str(value["run_id"]), + executor_id=value.get("executor_id"), + started_at=value.get("started_at"), + finished_at=value.get("finished_at"), + status=ExecutionStatus(value["status"]), + decision_reason=value.get("decision_reason"), + input_tokens=value.get("input_tokens"), + output_tokens=value.get("output_tokens"), + total_tokens=value.get("total_tokens"), + monetary_cost=value.get("monetary_cost"), + wall_time_ms=value.get("wall_time_ms"), + retry_count=int(value.get("retry_count", 0)), + replan_count=int(value.get("replan_count", 0)), + executor_invocations=int(value.get("executor_invocations", 0)), + handoff_count=int(value.get("handoff_count", 0)), + human_interventions=int(value.get("human_interventions", 0)), + evidence_refs=list(value.get("evidence_refs", [])), + errors=list(value.get("errors", [])), + metadata=dict(value.get("metadata", {})), + schema_version=schema_version, + ) + + @classmethod + def from_json(cls, value: str) -> "ExecutionRecord": + return cls.from_dict(json.loads(value)) diff --git a/src/arkx/event_log.py b/src/arkx/event_log.py new file mode 100644 index 00000000..54bb3dba --- /dev/null +++ b/src/arkx/event_log.py @@ -0,0 +1,561 @@ +"""Persistent, append-only event log for executor-neutral lifecycle telemetry.""" + +from __future__ import annotations + +from datetime import datetime +import json +from pathlib import Path +from typing import Any, Mapping + +from dataclasses import dataclass +from enum import Enum + +from .contracts import Event, EventType +from .harness import AcceptanceEvidence, AttemptSnapshot, RunManifest, VerificationEvidence +from .outcomes import AcceptanceResult, VerificationResult, validate_outcome_references +from .promotion import PromotionDecision +from .recovery import RecoveryPlan + + +EVENT_SCHEMA_VERSION = 1 + + +def _time(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +class EventLog: + """Serial event log; it does not interpret events or choose outcomes.""" + + def __init__(self, path: str | Path, *, run_id: str) -> None: + if not run_id.strip(): + raise ValueError("run_id must be non-empty") + self.path = Path(path).resolve() + self.run_id = run_id + + def read(self) -> tuple[Event, ...]: + if not self.path.exists(): + return () + events: list[Event] = [] + for line in self.path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + event = Event.from_dict(json.loads(line)) + self._validate_event(event, events[-1] if events else None) + events.append(event) + return tuple(events) + + def append(self, event: Event) -> None: + events = list(self.read()) + self._validate_event(event, events[-1] if events else None) + self.path.parent.mkdir(parents=True, exist_ok=True) + content = "".join(json.dumps(item.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n" for item in (*events, event)) + temporary = self.path.with_name(self.path.name + ".tmp") + temporary.write_text(content, encoding="utf-8", newline="\n") + temporary.replace(self.path) + + def append_stage(self, *, timestamp: str, event_type: EventType, stage: str, ref: str, data: dict | None = None) -> bool: + """Persist one causal stage reference; identical repeats are idempotent.""" + if stage not in {"request", "plan", "preflight", "provider_preflight", "sandbox_preflight", "execution", "recovery", "verification", "acceptance", "promotion"}: + raise ValueError("unknown lifecycle stage") + if not ref.strip(): + raise ValueError("stage reference must be non-empty") + current = replay_chain(self.read(), run_id=self.run_id) + current_ref = current.to_dict()[f"{stage}_ref"] + if current_ref == ref: + return False + if current_ref is not None: + raise ValueError("stage reference changed during append") + payload = dict(data or {}) + payload.update({"stage": stage, "ref": ref}) + self.append(Event(timestamp, event_type, self.run_id, payload)) + return True + + def append_terminal(self, *, timestamp: str, status: str, error_code: str | None = None, data: dict | None = None) -> bool: + """Persist one terminal lifecycle fact without inferring later stages.""" + if not status.strip(): + raise ValueError("terminal status must be non-empty") + current = self.read() + terminals = [event for event in current if event.type is EventType.TASK_FINISHED] + if terminals: + previous = terminals[-1].data.get("status") + if previous != status: + raise ValueError("terminal status changed during append") + previous_error = terminals[-1].data.get("error_code") + if previous_error != error_code: + raise ValueError("terminal error code changed during append") + return False + payload = dict(data or {}) + payload["status"] = status + if error_code is not None: + payload["error_code"] = error_code + self.append(Event(timestamp, EventType.TASK_FINISHED, self.run_id, payload)) + return True + + def _validate_event(self, event: Event, previous: Event | None) -> None: + if event.run_id != self.run_id: + raise ValueError("event run_id does not match log") + current_time = _time(event.timestamp) + if previous is not None and current_time < _time(previous.timestamp): + raise ValueError("event timestamps must be monotonic") + + +@dataclass(frozen=True) +class ReplayState: + run_id: str + event_count: int + declared_status: str | None = None + executor_invocations: int = 0 + retry_count: int = 0 + evidence_refs: tuple[str, ...] = () + verification_state: str | None = None + acceptance_decision: str | None = None + execution_outcome: str | None = None + preflight_status: str | None = None + promotion_status: str | None = None + error_codes: tuple[str, ...] = () + budget_ledger: Mapping[str, Any] | None = None + recovery_source_attempt_id: str | None = None + + +@dataclass(frozen=True) +class ReplayChain: + request_ref: str | None = None + plan_ref: str | None = None + execution_ref: str | None = None + verification_ref: str | None = None + acceptance_ref: str | None = None + promotion_ref: str | None = None + recovery_ref: str | None = None + preflight_ref: str | None = None + provider_preflight_ref: str | None = None + sandbox_preflight_ref: str | None = None + + def to_dict(self) -> dict[str, str | None]: + return { + "request_ref": self.request_ref, + "plan_ref": self.plan_ref, + "preflight_ref": self.preflight_ref, + "provider_preflight_ref": self.provider_preflight_ref, + "sandbox_preflight_ref": self.sandbox_preflight_ref, + "execution_ref": self.execution_ref, + "recovery_ref": self.recovery_ref, + "verification_ref": self.verification_ref, + "acceptance_ref": self.acceptance_ref, + "promotion_ref": self.promotion_ref, + } + + +@dataclass(frozen=True) +class ReferenceReport: + present: tuple[str, ...] = () + missing: tuple[str, ...] = () + indeterminate: tuple[str, ...] = () + + def status(self) -> str: + if self.missing: + return "MISSING" + if self.indeterminate: + return "INDETERMINATE" + return "PRESENT" + + +@dataclass(frozen=True) +class RestoredChainAudit: + chain: ReplayChain + replay: ReplayState + references: ReferenceReport + integrity: ChainIntegrity + protocol_version: str | None = None + schema_version: int | None = None + configuration_digest: str | None = None + budget_digest: str | None = None + treatment: str | None = None + capability_digest: str | None = None + resume_decision: ResumeDecision | None = None + budget_ledger: Mapping[str, Any] | None = None + + +class ChainIntegrity(str, Enum): + COMPLETE = "COMPLETE" + INCOMPLETE = "INCOMPLETE" + INCONSISTENT = "INCONSISTENT" + + +class ResumeAction(str, Enum): + TERMINAL = "TERMINAL" + RESUME_VERIFICATION = "RESUME_VERIFICATION" + RESUME_ACCEPTANCE = "RESUME_ACCEPTANCE" + RESUME_PROMOTION = "RESUME_PROMOTION" + RESTART_REQUIRED = "RESTART_REQUIRED" + REPLAN_REQUIRED = "REPLAN_REQUIRED" + + +@dataclass(frozen=True) +class ResumeDecision: + action: ResumeAction + reason: str + source_ref: str | None = None + + +def decide_resume(chain: ReplayChain, replay: ReplayState) -> ResumeDecision: + """Decide a safe resume boundary without executing or inferring missing facts.""" + if replay.declared_status is not None: + return ResumeDecision(ResumeAction.TERMINAL, "terminal event already persisted") + if chain.recovery_ref is not None and chain.execution_ref is None: + return ResumeDecision(ResumeAction.REPLAN_REQUIRED, "recovery exists without a new execution attempt", chain.recovery_ref) + if chain.execution_ref is None: + return ResumeDecision(ResumeAction.RESTART_REQUIRED, "execution evidence is absent; prior execution cannot be inferred", chain.plan_ref) + if chain.verification_ref is None: + return ResumeDecision(ResumeAction.RESUME_VERIFICATION, "execution is persisted without verification", chain.execution_ref) + if chain.acceptance_ref is None: + return ResumeDecision(ResumeAction.RESUME_ACCEPTANCE, "verification is persisted without acceptance", chain.verification_ref) + if chain.promotion_ref is None: + return ResumeDecision(ResumeAction.RESUME_PROMOTION, "acceptance is persisted without promotion", chain.acceptance_ref) + return ResumeDecision(ResumeAction.TERMINAL, "complete outcome chain is persisted", chain.promotion_ref) + + +def assess_chain_integrity(chain: ReplayChain, references: ReferenceReport) -> ChainIntegrity: + values = chain.to_dict() + if references.missing: + return ChainIntegrity.INCONSISTENT + required = (chain.request_ref, chain.plan_ref, chain.execution_ref, chain.verification_ref, chain.acceptance_ref, chain.promotion_ref) + if any(value is None for value in required) or references.indeterminate: + return ChainIntegrity.INCOMPLETE + return ChainIntegrity.COMPLETE + + +def replay_events(events: tuple[Event, ...] | list[Event], *, run_id: str) -> ReplayState: + """Reduce persisted facts without executing or evaluating the task.""" + log = EventLog(".", run_id=run_id) + previous = None + refs: list[str] = [] + invocations = 0 + retries = 0 + declared_status = None + verification_state = None + acceptance_decision = None + execution_outcome = None + preflight_status = None + promotion_status = None + error_codes: list[str] = [] + budget_ledger: Mapping[str, Any] | None = None + recovery_source_attempt_id: str | None = None + for event in events: + log._validate_event(event, previous) + previous = event + if event.type is EventType.EXECUTOR_STARTED: + invocations += 1 + elif event.type is EventType.RETRY: + retries += 1 + elif event.type is EventType.EVIDENCE_ADDED: + if event.data.get("ref") is not None: + refs.append(str(event.data["ref"])) + refs.extend(str(ref) for ref in event.data.get("refs", [])) + if event.data.get("verification_state") is not None: + verification_state = str(event.data["verification_state"]) + if event.data.get("acceptance_decision") is not None: + acceptance_decision = str(event.data["acceptance_decision"]) + if event.data.get("execution_outcome") is not None: + execution_outcome = str(event.data["execution_outcome"]) + if event.data.get("preflight_status") is not None: + preflight_status = str(event.data["preflight_status"]) + if event.data.get("promotion_status") is not None: + promotion_status = str(event.data["promotion_status"]) + if event.data.get("error_code") is not None: + error_codes.append(str(event.data["error_code"])) + if event.data.get("budget_ledger") is not None: + candidate = event.data["budget_ledger"] + if not isinstance(candidate, Mapping): + raise ValueError("budget_ledger event data must be a mapping") + if budget_ledger is not None and dict(budget_ledger) != dict(candidate): + raise ValueError("budget ledger changed during replay") + budget_ledger = dict(candidate) + elif event.type is EventType.TASK_FINISHED: + value = event.data.get("status") + declared_status = None if value is None else str(value) + if event.data.get("error_code") is not None: + error_codes.append(str(event.data["error_code"])) + if event.type is EventType.REPLAN and event.data.get("source_attempt_id") is not None: + candidate_source = str(event.data["source_attempt_id"]) + if recovery_source_attempt_id is not None and recovery_source_attempt_id != candidate_source: + raise ValueError("recovery source attempt changed during replay") + recovery_source_attempt_id = candidate_source + return ReplayState(run_id, len(events), declared_status, invocations, retries, tuple(refs), verification_state, acceptance_decision, execution_outcome, preflight_status, promotion_status, tuple(sorted(set(error_codes))), budget_ledger, recovery_source_attempt_id) + + +def replay_chain(events: tuple[Event, ...] | list[Event], *, run_id: str) -> ReplayChain: + """Reconstruct explicit stage references without filling missing stages.""" + replay_events(events, run_id=run_id) + values: dict[str, str | None] = {stage: None for stage in ("request", "plan", "preflight", "provider_preflight", "sandbox_preflight", "execution", "recovery", "verification", "acceptance", "promotion")} + order = {stage: index for index, stage in enumerate(values)} + highest = -1 + for event in events: + stage = event.data.get("stage") + ref = event.data.get("ref") + if stage is None: + continue + if stage not in order or ref is None or not str(ref).strip(): + raise ValueError("stage events require a known stage and non-empty ref") + if order[stage] < highest: + raise ValueError("stage references are out of causal order") + if values[stage] is not None and values[stage] != str(ref): + raise ValueError("stage reference changed during replay") + highest = order[stage] + values[stage] = str(ref) + return ReplayChain(**{f"{stage}_ref": values[stage] for stage in values}) + + +def classify_chain_references(chain: ReplayChain, *, available_refs: set[str] | None = None) -> ReferenceReport: + refs = tuple(ref for ref in chain.to_dict().values() if ref is not None) + if available_refs is None: + return ReferenceReport(indeterminate=tuple(sorted(set(refs)))) + present = tuple(sorted(set(ref for ref in refs if ref in available_refs))) + missing = tuple(sorted(set(ref for ref in refs if ref not in available_refs))) + return ReferenceReport(present=present, missing=missing) + + +def classify_snapshot_chain_references(snapshot: AttemptSnapshot, chain: ReplayChain) -> ReferenceReport: + """Classify chain refs only against artifacts validated by an attempt snapshot.""" + available = {str(path) for path in snapshot.artifact_paths} + available.update(snapshot.manifest.artifact_refs) + return classify_chain_references(chain, available_refs=available) + + +def validate_replay_against_manifest(manifest: RunManifest, replay: ReplayState) -> None: + """Reject divergence between persisted manifest state and replayed facts.""" + if manifest.attempt_id != replay.run_id: + raise ValueError("replay run_id does not match manifest attempt_id") + if replay.declared_status is None: + return + if replay.declared_status != manifest.state.value: + raise ValueError("replayed declared status does not match manifest state") + + +def validate_terminal_replay( + replay: ReplayState, + *, + expected_status: str, + expected_error_code: str | None = None, +) -> None: + """Check the terminal fact without inferring a missing terminal event.""" + if not expected_status.strip(): + raise ValueError("expected terminal status must be non-empty") + if replay.declared_status is None: + raise ValueError("terminal status is missing from replay") + if replay.declared_status != expected_status: + raise ValueError("replayed terminal status does not match expected status") + if expected_error_code is not None and expected_error_code not in replay.error_codes: + raise ValueError("replayed terminal error code does not match expected error") + + +def validate_replay_outcomes( + replay: ReplayState, + *, + verification: VerificationEvidence | None = None, + acceptance: AcceptanceEvidence | None = None, +) -> None: + """Check explicitly replayed outcomes against independent evidence only.""" + if verification is not None and replay.verification_state is not None and replay.verification_state != verification.state: + raise ValueError("replayed verification state does not match evidence") + if acceptance is not None and replay.acceptance_decision is not None and replay.acceptance_decision != acceptance.decision: + raise ValueError("replayed acceptance decision does not match evidence") + + +def validate_replay_telemetry( + replay: ReplayState, + *, + execution: object | None = None, + verification: VerificationResult | None = None, + acceptance: AcceptanceResult | None = None, + promotion: PromotionDecision | None = None, +) -> None: + """Reject persisted telemetry that diverges from loaded result objects.""" + if execution is not None: + state = getattr(execution, "state", None) + expected = getattr(state, "value", state) + if replay.execution_outcome is not None and replay.execution_outcome not in (str(expected), "COMPLETED" if str(expected) == "COMPLETED" else str(expected)): + raise ValueError("replayed execution outcome does not match execution result") + error = getattr(execution, "error", None) + if error is not None and replay.error_codes: + code = getattr(error, "code", None) + if code is not None and code not in replay.error_codes: + raise ValueError("replayed error code does not match execution result") + if verification is not None and replay.verification_state is not None and replay.verification_state != verification.state.value: + raise ValueError("replayed verification state does not match verification result") + if acceptance is not None and replay.acceptance_decision is not None and replay.acceptance_decision != acceptance.decision.value: + raise ValueError("replayed acceptance decision does not match acceptance result") + if promotion is not None and replay.promotion_status is not None and replay.promotion_status != promotion.status.value: + raise ValueError("replayed promotion status does not match promotion decision") + + +def validate_final_chain( + *, + attempt_id: str, + verification_ref: str, + acceptance_ref: str, + promotion_ref: str, + promotion: PromotionDecision, +) -> None: + if promotion.attempt_id != attempt_id: + raise ValueError("promotion decision does not match attempt") + if promotion.reference != promotion_ref: + raise ValueError("promotion reference does not match decision") + if promotion.acceptance is not None and promotion.acceptance.reference != acceptance_ref: + raise ValueError("promotion acceptance does not match acceptance reference") + if not verification_ref.strip() or not acceptance_ref.strip(): + raise ValueError("verification and acceptance references must be non-empty") + + +def validate_outcome_chain( + chain: ReplayChain, + *, + verification: VerificationResult, + acceptance: AcceptanceResult, + promotion: PromotionDecision, +) -> None: + """Validate that replayed outcome references identify the supplied objects.""" + if chain.verification_ref != verification.reference: + raise ValueError("replay verification reference does not match result") + if chain.acceptance_ref != acceptance.reference: + raise ValueError("replay acceptance reference does not match result") + if chain.promotion_ref != promotion.reference: + raise ValueError("replay promotion reference does not match decision") + validate_outcome_references( + verification, + acceptance, + verification_ref=chain.verification_ref, + acceptance_ref=chain.acceptance_ref, + ) + if verification.reference not in acceptance.evidence: + raise ValueError("acceptance does not carry the verification reference") + if promotion.acceptance is None or promotion.acceptance.reference != acceptance.reference: + raise ValueError("promotion does not carry the acceptance reference") + + +def audit_restored_chain( + event_log_path: str | Path, + snapshot: AttemptSnapshot, + *, + verification: VerificationResult | None = None, + acceptance: AcceptanceResult | None = None, + promotion: PromotionDecision | None = None, + protocol_version: str | None = None, + schema_version: int = EVENT_SCHEMA_VERSION, + configuration_digest: str | None = None, + budget_digest: str | None = None, + treatment: str | None = None, + capability_digest: str | None = None, +) -> RestoredChainAudit: + """Audit a restored attempt without rerunning any execution stage.""" + log = EventLog(event_log_path, run_id=snapshot.manifest.attempt_id) + events = log.read() + protocols = {str(event.data["protocol_version"]) for event in events if event.data.get("protocol_version") is not None} + schemas = {int(event.data["schema_version"]) for event in events if event.data.get("schema_version") is not None} + if len(protocols) > 1 or len(schemas) > 1: + raise ValueError("restored event log contains incompatible protocol or schema versions") + observed_protocol = next(iter(protocols), None) + observed_schema = next(iter(schemas), None) + def observed_identity(name: str) -> str | None: + values = {str(event.data[name]) for event in events if event.data.get(name) is not None} + if len(values) > 1: + raise ValueError(f"restored event log contains inconsistent {name}") + return next(iter(values), None) + observed_configuration = observed_identity("configuration_digest") + observed_budget = observed_identity("budget_digest") + observed_treatment = observed_identity("treatment") + observed_capability = observed_identity("capability_digest") + if protocol_version is not None: + if observed_protocol != protocol_version: + raise ValueError("restored event log protocol_version does not match expected protocol") + if observed_schema != schema_version: + raise ValueError("restored event log schema_version does not match expected schema") + for name, expected, observed in ( + ("configuration_digest", configuration_digest, observed_configuration), + ("budget_digest", budget_digest, observed_budget), + ("treatment", treatment, observed_treatment), + ("capability_digest", capability_digest, observed_capability), + ): + if expected is not None and observed != expected: + raise ValueError(f"restored event log {name} does not match expected identity") + replay = replay_events(events, run_id=snapshot.manifest.attempt_id) + validate_replay_against_manifest(snapshot.manifest, replay) + chain = replay_chain(events, run_id=snapshot.manifest.attempt_id) + available = {str(path) for path in snapshot.artifact_paths} + available.update(snapshot.manifest.artifact_refs) + if verification is not None: + available.add(verification.reference) + if acceptance is not None: + available.add(acceptance.reference) + if promotion is not None: + available.add(promotion.reference) + references = classify_chain_references(chain, available_refs=available) + integrity = assess_chain_integrity(chain, references) + if verification is not None and acceptance is not None and promotion is not None: + validate_outcome_chain(chain, verification=verification, acceptance=acceptance, promotion=promotion) + if snapshot.manifest.budget_ledger is not None and replay.budget_ledger != snapshot.manifest.budget_ledger: + raise ValueError("replayed budget ledger does not match attempt manifest") + validate_attempt_lineage_chain(snapshot, chain, replay) + return RestoredChainAudit(chain, replay, references, integrity, observed_protocol, observed_schema, observed_configuration, observed_budget, observed_treatment, observed_capability, decide_resume(chain, replay), replay.budget_ledger) + + +def validate_attempt_lineage_chain(snapshot: AttemptSnapshot, chain: ReplayChain, replay: ReplayState) -> None: + """Require persisted event references to agree with attempt lineage facts.""" + lineage = snapshot.retry_lineage or snapshot.recovery_lineage + if lineage is None: + return + expected_execution = f"execution://{snapshot.manifest.attempt_id}" + if chain.execution_ref != expected_execution: + raise ValueError("attempt lineage execution reference does not match manifest attempt") + recovery_lineage = snapshot.recovery_lineage + if recovery_lineage is not None: + if chain.recovery_ref != recovery_lineage.recovery_reference: + raise ValueError("attempt lineage recovery reference does not match recovery lineage") + if replay.recovery_source_attempt_id != recovery_lineage.source_attempt_id: + raise ValueError("replayed recovery source does not match recovery lineage") + if snapshot.manifest.budget_ledger is not None and replay.budget_ledger != snapshot.manifest.budget_ledger: + raise ValueError("attempt lineage budget ledger does not match replay") + + +def append_promotion_decision( + log: EventLog, + decision: PromotionDecision, + *, + verification_ref: str, + timestamp: str, +) -> bool: + """Persist promotion only after the same log contains verification and acceptance.""" + if not verification_ref.strip(): + raise ValueError("verification reference must be non-empty") + if decision.acceptance is None: + raise ValueError("promotion decision requires persisted acceptance") + chain = replay_chain(log.read(), run_id=log.run_id) + if chain.verification_ref != verification_ref: + raise ValueError("promotion verification reference does not match event log") + if chain.acceptance_ref != decision.acceptance.reference: + raise ValueError("promotion acceptance does not match event log") + return log.append_stage( + timestamp=timestamp, + event_type=EventType.EVIDENCE_ADDED, + stage="promotion", + ref=decision.reference, + data={"promotion_status": decision.status.value, "authority_id": decision.authority_id}, + ) + + +def append_recovery_plan(log: EventLog, plan: RecoveryPlan, execution_ref: str, timestamp: str) -> bool: + """Persist recovery only when causally bound to the recorded execution.""" + if not plan.source_attempt_id.strip(): + raise ValueError("recovery plan source attempt must be non-empty") + if execution_ref != f"execution://{plan.source_attempt_id}": + raise ValueError("recovery plan source attempt does not match execution reference") + chain = replay_chain(log.read(), run_id=log.run_id) + if chain.execution_ref != execution_ref: + raise ValueError("recovery requires a matching persisted execution stage") + return log.append_stage( + timestamp=timestamp, + event_type=EventType.REPLAN, + stage="recovery", + ref=plan.reference, + data={"source_attempt_id": plan.source_attempt_id, "recovery_action": plan.action.value}, + ) diff --git a/src/arkx/evidence.py b/src/arkx/evidence.py new file mode 100644 index 00000000..2dc0b18b --- /dev/null +++ b/src/arkx/evidence.py @@ -0,0 +1,25 @@ +"""Fail-closed separation between execution result, evidence, and final status.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from .contracts import ExecutionStatus + + +def resolve_status( + requested: ExecutionStatus | str | None, + evidence_refs: Iterable[str], +) -> ExecutionStatus: + """Resolve a status without allowing missing evidence to become PASS. + + This function does not verify evidence quality. It only enforces the P0 + boundary: a requested PASS requires at least one explicit evidence ref. + """ + + status = ExecutionStatus(requested) if requested is not None else ExecutionStatus.NOT_EXECUTED + refs = list(evidence_refs) + if status is ExecutionStatus.PASS and not refs: + return ExecutionStatus.UNVERIFIED + return status + diff --git a/src/arkx/execution.py b/src/arkx/execution.py new file mode 100644 index 00000000..54b55460 --- /dev/null +++ b/src/arkx/execution.py @@ -0,0 +1,590 @@ +"""Executor-neutral execution boundary contracts. + +These protocols describe capability boundaries only. They do not select a +provider, invoke a model, create a sandbox, verify a patch, or accept a task. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +import hashlib +import json +from typing import Any, Mapping, Protocol + +from .harness import ErrorDomain, ErrorEnvelope, Retryability, RunState, derive_attempt_id, validate_transition +from .integration import AdapterIdentity, AdapterPreflight, CapabilityProvenance, IntegrationKind +from .harness import AcceptanceEvidence, VerificationEvidence +from .configuration import ConfigurationSnapshot + + +@dataclass(frozen=True) +class ExecutionBudgetSpec: + """Executor-neutral budget declaration carried by an execution plan.""" + + max_attempts: int | None = None + max_tokens: int | None = None + max_wall_time_ms: int | None = None + max_cost: Decimal | None = None + max_executor_invocations: int | None = None + + def __post_init__(self) -> None: + for name in ("max_attempts", "max_tokens", "max_wall_time_ms", "max_executor_invocations"): + value = getattr(self, name) + if value is not None and value < 0: + raise ValueError(f"{name} must be non-negative") + if self.max_cost is not None and self.max_cost < 0: + raise ValueError("max_cost must be non-negative") + + def to_dict(self) -> dict[str, Any]: + return { + "max_attempts": self.max_attempts, + "max_tokens": self.max_tokens, + "max_wall_time_ms": self.max_wall_time_ms, + "max_cost": None if self.max_cost is None else str(self.max_cost), + "max_executor_invocations": self.max_executor_invocations, + } + + def digest(self) -> str: + payload = json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16] + + +@dataclass(frozen=True) +class BudgetConsumption: + accepted: bool + reason: str + ledger: "BudgetLedger" + + +@dataclass(frozen=True) +class BudgetLedger: + """Immutable execution-budget ledger with idempotent attempt consumption.""" + + budget: ExecutionBudgetSpec + consumed_attempts: tuple[str, ...] = () + tokens_used: int = 0 + wall_time_ms_used: int = 0 + cost_used: Decimal = Decimal("0") + executor_invocations_used: int = 0 + + def __post_init__(self) -> None: + if any(not attempt.strip() for attempt in self.consumed_attempts): + raise ValueError("consumed attempt ids must be non-empty") + if len(set(self.consumed_attempts)) != len(self.consumed_attempts): + raise ValueError("consumed attempt ids must be unique") + if min(self.tokens_used, self.wall_time_ms_used, self.executor_invocations_used) < 0 or self.cost_used < 0: + raise ValueError("budget usage must be non-negative") + object.__setattr__(self, "consumed_attempts", tuple(sorted(self.consumed_attempts))) + + def consume(self, attempt_id: str, *, tokens: int = 0, wall_time_ms: int = 0, cost: Decimal = Decimal("0"), executor_invocations: int = 1) -> BudgetConsumption: + if not attempt_id.strip(): + raise ValueError("attempt_id must be non-empty") + if min(tokens, wall_time_ms, executor_invocations) < 0 or cost < 0: + raise ValueError("consumption values must be non-negative") + if attempt_id in self.consumed_attempts: + return BudgetConsumption(True, "ALREADY_CONSUMED", self) + next_values = { + "tokens": self.tokens_used + tokens, + "wall_time_ms": self.wall_time_ms_used + wall_time_ms, + "cost": self.cost_used + cost, + "executor_invocations": self.executor_invocations_used + executor_invocations, + } + limits = (("tokens", self.budget.max_tokens), ("wall_time_ms", self.budget.max_wall_time_ms), ("cost", self.budget.max_cost), ("executor_invocations", self.budget.max_executor_invocations), ("attempts", self.budget.max_attempts)) + for name, limit in limits: + value = len(self.consumed_attempts) + 1 if name == "attempts" else next_values[name] + if limit is not None and value > limit: + return BudgetConsumption(False, f"{name.upper()}_EXCEEDED", self) + ledger = BudgetLedger(self.budget, self.consumed_attempts + (attempt_id,), next_values["tokens"], next_values["wall_time_ms"], next_values["cost"], next_values["executor_invocations"]) + return BudgetConsumption(True, "CONSUMED", ledger) + + def record_usage(self, attempt_id: str, *, tokens: int = 0, wall_time_ms: int = 0, cost: Decimal = Decimal("0")) -> BudgetConsumption: + """Record measured post-execution usage for an already consumed attempt.""" + if attempt_id not in self.consumed_attempts: + raise ValueError("usage must reference a consumed attempt") + if min(tokens, wall_time_ms) < 0 or cost < 0: + raise ValueError("usage values must be non-negative") + next_values = {"tokens": self.tokens_used + tokens, "wall_time_ms": self.wall_time_ms_used + wall_time_ms, "cost": self.cost_used + cost} + for name, limit in (("tokens", self.budget.max_tokens), ("wall_time_ms", self.budget.max_wall_time_ms), ("cost", self.budget.max_cost)): + if limit is not None and next_values[name] > limit: + return BudgetConsumption(False, f"{name.upper()}_EXCEEDED", self) + ledger = BudgetLedger(self.budget, self.consumed_attempts, next_values["tokens"], next_values["wall_time_ms"], next_values["cost"], self.executor_invocations_used) + return BudgetConsumption(True, "USAGE_RECORDED", ledger) + + def to_dict(self) -> dict[str, Any]: + return {"budget_digest": self.budget.digest(), "budget": self.budget.to_dict(), "consumed_attempts": list(self.consumed_attempts), "tokens_used": self.tokens_used, "wall_time_ms_used": self.wall_time_ms_used, "cost_used": str(self.cost_used), "executor_invocations_used": self.executor_invocations_used} + + +@dataclass(frozen=True) +class RetryDecision: + allowed: bool + reason: str + next_attempt: int | None = None + + +@dataclass(frozen=True) +class RetryAttemptPlan: + trial_id: str + previous_attempt_id: str + previous_attempt_number: int + next_attempt_id: str + next_attempt_number: int + configuration_digest: str + + def __post_init__(self) -> None: + if self.previous_attempt_number < 1 or self.next_attempt_number != self.previous_attempt_number + 1: + raise ValueError("retry attempt numbers must be consecutive and positive") + expected_previous = derive_attempt_id(trial_id=self.trial_id, attempt_number=self.previous_attempt_number, configuration_digest=self.configuration_digest) + expected_next = derive_attempt_id(trial_id=self.trial_id, attempt_number=self.next_attempt_number, configuration_digest=self.configuration_digest) + if self.previous_attempt_id != expected_previous or self.next_attempt_id != expected_next: + raise ValueError("retry attempt identity does not match trial and configuration") + + def to_dict(self) -> dict[str, Any]: + return { + "trial_id": self.trial_id, + "previous_attempt_id": self.previous_attempt_id, + "previous_attempt_number": self.previous_attempt_number, + "next_attempt_id": self.next_attempt_id, + "next_attempt_number": self.next_attempt_number, + "configuration_digest": self.configuration_digest, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +@dataclass(frozen=True) +class RetryPolicy: + """Pure retry authorization; it never performs a retry or changes state.""" + + max_retries: int = 0 + allowed_domains: tuple[ErrorDomain, ...] = () + + def __post_init__(self) -> None: + if self.max_retries < 0: + raise ValueError("max_retries must be non-negative") + object.__setattr__(self, "allowed_domains", tuple(sorted(set(self.allowed_domains), key=lambda item: item.value))) + + def decide(self, error: ErrorEnvelope | None, *, retries_used: int, max_attempts: int | None = None) -> RetryDecision: + if retries_used < 0: + raise ValueError("retries_used must be non-negative") + if error is None: + return RetryDecision(False, "NO_ERROR") + if error.retryability is not Retryability.RETRYABLE: + return RetryDecision(False, "ERROR_NOT_RETRYABLE") + if self.allowed_domains and error.domain not in self.allowed_domains: + return RetryDecision(False, "ERROR_DOMAIN_NOT_ALLOWED") + if retries_used >= self.max_retries: + return RetryDecision(False, "RETRY_POLICY_EXHAUSTED") + next_attempt = retries_used + 2 + if max_attempts is not None and next_attempt > max_attempts: + return RetryDecision(False, "EXECUTION_BUDGET_EXHAUSTED") + return RetryDecision(True, "RETRY_AUTHORIZED", next_attempt) + + +def plan_retry_attempt( + *, trial_id: str, previous_attempt_id: str, previous_attempt_number: int, + configuration_digest: str, decision: RetryDecision, +) -> RetryAttemptPlan | None: + if not decision.allowed: + return None + if decision.next_attempt is None: + raise ValueError("authorized retry must contain next_attempt") + return RetryAttemptPlan( + trial_id, previous_attempt_id, previous_attempt_number, + derive_attempt_id(trial_id=trial_id, attempt_number=decision.next_attempt, configuration_digest=configuration_digest), + decision.next_attempt, configuration_digest, + ) + + +@dataclass(frozen=True) +class CapabilityProfile: + """Evidence-bearing, versioned capability declaration for one component.""" + + subject_type: str + subject_id: str + version: str | None = None + configuration_digest: str | None = None + capabilities: tuple[str, ...] = () + evidence_refs: tuple[str, ...] = () + + def __post_init__(self) -> None: + if not self.subject_type.strip() or not self.subject_id.strip(): + raise ValueError("capability profile identity must be non-empty") + if any(not value.strip() for value in self.capabilities): + raise ValueError("capability names must be non-empty") + if any(not value.strip() for value in self.evidence_refs): + raise ValueError("capability evidence references must be non-empty") + object.__setattr__(self, "capabilities", tuple(sorted(set(self.capabilities)))) + object.__setattr__(self, "evidence_refs", tuple(sorted(set(self.evidence_refs)))) + + def supports(self, required: tuple[str, ...]) -> bool: + return set(required).issubset(self.capabilities) + + def to_dict(self) -> dict[str, Any]: + return {"subject_type": self.subject_type, "subject_id": self.subject_id, "version": self.version, "configuration_digest": self.configuration_digest, "capabilities": list(self.capabilities), "evidence_refs": list(self.evidence_refs)} + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +@dataclass(frozen=True) +class CapabilityRegistry: + profiles: tuple[CapabilityProfile, ...] = () + + def __post_init__(self) -> None: + values = tuple(sorted(self.profiles, key=lambda item: (item.subject_type, item.subject_id, item.version or ""))) + identities = [(item.subject_type, item.subject_id, item.version, item.configuration_digest) for item in values] + if len(set(identities)) != len(identities): + raise ValueError("capability profiles must have unique identities") + object.__setattr__(self, "profiles", values) + + def find(self, subject_type: str, subject_id: str) -> tuple[CapabilityProfile, ...]: + return tuple(item for item in self.profiles if item.subject_type == subject_type and item.subject_id == subject_id) + + def to_dict(self) -> dict[str, Any]: + return {"profiles": [item.to_dict() for item in self.profiles]} + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + def digest(self) -> str: + return hashlib.sha256(self.to_json().encode("utf-8")).hexdigest()[:16] + + @property + def reference(self) -> str: + return f"capability://{self.digest()}" + + +@dataclass(frozen=True) +class ExecutionRequest: + task_id: str + task_revision: str | None + treatment: str + prompt: str + executor_id: str + provider_id: str | None + model_id: str | None + sandbox_id: str | None + budget_digest: str | None + configuration_digest: str | None + + +@dataclass(frozen=True) +class ExecutionPlan: + plan_id: str + task_id: str + task_revision: str | None + treatment: str + executor_id: str + provider_id: str | None + model_id: str | None + sandbox_id: str | None + budget_digest: str | None + configuration_digest: str | None + steps: tuple[str, ...] = () + budget: ExecutionBudgetSpec | None = None + required_capabilities: tuple[str, ...] = () + minimum_capability_provenance: CapabilityProvenance = CapabilityProvenance.DECLARED + configuration_snapshot: ConfigurationSnapshot | None = None + provider_identity: AdapterIdentity | None = None + sandbox_identity: AdapterIdentity | None = None + + def __post_init__(self) -> None: + for value, name in ((self.plan_id, "plan_id"), (self.task_id, "task_id"), (self.treatment, "treatment"), (self.executor_id, "executor_id")): + if not value or not value.strip(): + raise ValueError(f"{name} must be non-empty") + if any(not step.strip() for step in self.steps): + raise ValueError("execution plan steps must be non-empty") + object.__setattr__(self, "steps", tuple(self.steps)) + if any(not capability.strip() for capability in self.required_capabilities): + raise ValueError("execution plan capabilities must be non-empty") + object.__setattr__(self, "required_capabilities", tuple(sorted(set(self.required_capabilities)))) + if self.budget is not None: + expected_digest = self.budget.digest() + if self.budget_digest is None: + object.__setattr__(self, "budget_digest", expected_digest) + elif self.budget_digest != expected_digest: + raise ValueError("execution plan budget_digest does not match budget") + if self.configuration_snapshot is not None: + expected_digest = self.configuration_snapshot.digest() + if self.configuration_digest is None: + object.__setattr__(self, "configuration_digest", expected_digest) + elif self.configuration_digest != expected_digest: + raise ValueError("execution plan configuration_digest does not match snapshot") + for identity, expected_kind, expected_id, label in ( + (self.provider_identity, IntegrationKind.PROVIDER, self.provider_id, "provider"), + (self.sandbox_identity, IntegrationKind.SANDBOX, self.sandbox_id, "sandbox"), + ): + if identity is None: + continue + if identity.kind is not expected_kind: + raise ValueError(f"{label} identity has wrong integration kind") + if expected_id is not None and identity.name != expected_id: + raise ValueError(f"{label} identity does not match plan id") + if identity.version is None or identity.configuration_digest is None: + raise ValueError(f"{label} identity requires version and configuration_digest") + + def to_dict(self) -> dict[str, Any]: + return { + "plan_id": self.plan_id, + "task_id": self.task_id, + "task_revision": self.task_revision, + "treatment": self.treatment, + "executor_id": self.executor_id, + "provider_id": self.provider_id, + "model_id": self.model_id, + "sandbox_id": self.sandbox_id, + "budget_digest": self.budget_digest, + "configuration_digest": self.configuration_digest, + "steps": list(self.steps), + "budget": None if self.budget is None else self.budget.to_dict(), + "required_capabilities": list(self.required_capabilities), + "minimum_capability_provenance": self.minimum_capability_provenance.value, + "configuration_snapshot": None if self.configuration_snapshot is None else self.configuration_snapshot.to_dict(), + "provider_identity": None if self.provider_identity is None else self.provider_identity.to_dict(), + "sandbox_identity": None if self.sandbox_identity is None else self.sandbox_identity.to_dict(), + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + def to_request(self, prompt: str) -> ExecutionRequest: + return ExecutionRequest( + self.task_id, self.task_revision, self.treatment, prompt, + self.executor_id, self.provider_id, self.model_id, self.sandbox_id, + self.budget_digest, self.configuration_digest, + ) + + def validate_capabilities(self, preflight: AdapterPreflight): + """Apply routing capability policy before execution starts.""" + from .routing import assess_capability_policy + return assess_capability_policy( + preflight, + self.required_capabilities, + minimum_provenance=self.minimum_capability_provenance, + ) + + def preflight_matches_identity(self, preflight: AdapterPreflight) -> bool: + """Check that a preflight describes this plan's executor/configuration.""" + return ( + preflight.identity.name == self.executor_id + and preflight.identity.kind.value == "EXECUTOR" + and ( + self.configuration_digest is None + or preflight.identity.configuration_digest == self.configuration_digest + ) + ) + + def preflight_matches_integration(self, preflight: AdapterPreflight, kind: IntegrationKind) -> bool: + """Check a provider or sandbox preflight against the plan binding.""" + expected = self.provider_identity if kind is IntegrationKind.PROVIDER else self.sandbox_identity if kind is IntegrationKind.SANDBOX else None + expected_id = self.provider_id if kind is IntegrationKind.PROVIDER else self.sandbox_id if kind is IntegrationKind.SANDBOX else None + return expected is not None and preflight.identity.kind is kind and preflight.identity.name == expected_id and preflight.identity.version == expected.version and preflight.identity.configuration_digest == expected.configuration_digest + + @property + def reference(self) -> str: + return f"plan://{self.plan_id}" + + +@dataclass(frozen=True) +class ExecutionResult: + state: RunState + artifact_refs: tuple[str, ...] = () + output: str | None = None + usage: Mapping[str, Any] | None = None + error: ErrorEnvelope | None = None + + def __post_init__(self) -> None: + if self.state in {RunState.FAILED, RunState.BLOCKED} and self.error is None: + raise ValueError("failed or blocked execution requires an error envelope") + if self.state is RunState.COMPLETED and self.error is not None: + raise ValueError("completed execution cannot contain an error envelope") + refs = tuple(sorted(set(self.artifact_refs))) + if any(not ref.strip() for ref in refs): + raise ValueError("execution artifact references must be non-empty") + object.__setattr__(self, "artifact_refs", refs) + + +def execution_reference(attempt_id: str) -> str: + if not attempt_id.strip(): + raise ValueError("attempt_id must be non-empty") + return f"execution://{attempt_id}" + + +def validate_execution_chain(plan: ExecutionPlan, *, plan_reference: str, execution_ref: str, attempt_id: str) -> None: + if plan_reference != plan.reference: + raise ValueError("plan reference does not match execution plan") + if execution_ref != execution_reference(attempt_id): + raise ValueError("execution reference does not match attempt") + + +@dataclass(frozen=True) +class ContractRun: + request: ExecutionRequest + result: ExecutionResult + state_history: tuple[RunState, ...] + verification: VerificationEvidence | None = None + acceptance: AcceptanceEvidence | None = None + budget_ledger: BudgetLedger | None = None + budget_reason: str | None = None + + @classmethod + def execute(cls, request: ExecutionRequest, executor: "Executor") -> "ContractRun": + result = executor.execute(request) + terminal = result.state + history = (RunState.PLANNED, RunState.STARTED, RunState.EXECUTING, terminal) + for previous, current in zip(history, history[1:]): + validate_transition(previous, current) + return cls(request, result, history) + + @classmethod + def execute_with_budget( + cls, + request: ExecutionRequest, + executor: "Executor", + ledger: BudgetLedger, + *, + attempt_id: str, + tokens: int = 0, + wall_time_ms: int = 0, + cost: Decimal = Decimal("0"), + executor_invocations: int = 1, + ) -> "ContractRun": + consumption = ledger.consume( + attempt_id, + tokens=tokens, + wall_time_ms=wall_time_ms, + cost=cost, + executor_invocations=executor_invocations, + ) + if not consumption.accepted: + error = ErrorEnvelope(ErrorDomain.HARNESS, consumption.reason, "execution budget rejected attempt", Retryability.NOT_RETRYABLE, attempt_number=None) + result = ExecutionResult(RunState.BLOCKED, error=error) + return cls(request, result, (RunState.PLANNED, RunState.BLOCKED), budget_ledger=consumption.ledger, budget_reason=consumption.reason) + run = cls.execute(request, executor) + return cls(run.request, run.result, run.state_history, run.verification, run.acceptance, consumption.ledger, consumption.reason) + + def with_verification(self, evidence: VerificationEvidence) -> "ContractRun": + return ContractRun(self.request, self.result, self.state_history, evidence, self.acceptance) + + def with_acceptance(self, evidence: AcceptanceEvidence) -> "ContractRun": + return ContractRun(self.request, self.result, self.state_history, self.verification, evidence) + + def is_accepted(self) -> bool: + return ( + self.result.state is RunState.COMPLETED + and self.verification is not None + and self.verification.state == "PASS" + and self.acceptance is not None + and self.acceptance.decision == "ACCEPTED" + ) + + +@dataclass(frozen=True) +class ProviderRequest: + prompt: str + model_id: str + configuration_digest: str | None + budget_digest: str | None + + +@dataclass(frozen=True) +class ProviderResponse: + output: str | None + usage: Mapping[str, Any] | None = None + error: ErrorEnvelope | None = None + + +@dataclass(frozen=True) +class SandboxRequest: + task_id: str + task_revision: str | None + sandbox_id: str + policy_digest: str | None + + +class Executor(Protocol): + executor_id: str + + def execute(self, request: ExecutionRequest) -> ExecutionResult: + """Execute one request without deciding verification or acceptance.""" + + +class Provider(Protocol): + provider_identity: AdapterIdentity + + def complete(self, request: ProviderRequest) -> ProviderResponse: + """Return provider output or a structured provider error.""" + + +def validate_provider_identity(provider: Provider) -> AdapterIdentity: + """Require a versioned, configuration-bound provider identity.""" + identity = getattr(provider, "provider_identity", None) + if not isinstance(identity, AdapterIdentity) or identity.kind is not IntegrationKind.PROVIDER: + raise ValueError("provider must expose an AdapterIdentity with PROVIDER kind") + if identity.version is None or identity.configuration_digest is None: + raise ValueError("provider identity requires version and configuration_digest") + return identity + + +class Sandbox(Protocol): + sandbox_identity: AdapterIdentity + + def prepare(self, request: SandboxRequest) -> None: + """Prepare an isolated workspace according to the declared policy.""" + + def collect_artifacts(self) -> tuple[str, ...]: + """Return references to artifacts produced by the sandbox.""" + + +def validate_sandbox_identity(sandbox: Sandbox) -> AdapterIdentity: + """Require a versioned, configuration-bound sandbox identity.""" + identity = getattr(sandbox, "sandbox_identity", None) + if not isinstance(identity, AdapterIdentity) or identity.kind is not IntegrationKind.SANDBOX: + raise ValueError("sandbox must expose an AdapterIdentity with SANDBOX kind") + if identity.version is None or identity.configuration_digest is None: + raise ValueError("sandbox identity requires version and configuration_digest") + return identity + + +class FakeExecutor: + """Deterministic contract-test executor; it has no external dependencies.""" + + executor_id = "fake-executor" + + def __init__(self, result: ExecutionResult) -> None: + self.result = result + self.requests: list[ExecutionRequest] = [] + + def execute(self, request: ExecutionRequest) -> ExecutionResult: + self.requests.append(request) + return self.result + + +class FakeProvider: + provider_identity = AdapterIdentity(IntegrationKind.PROVIDER, "fake-provider", "1", "fake-config") + provider_id = "fake-provider" + + def __init__(self, response: ProviderResponse) -> None: + self.response = response + self.requests: list[ProviderRequest] = [] + + def complete(self, request: ProviderRequest) -> ProviderResponse: + self.requests.append(request) + return self.response + + +class FakeSandbox: + sandbox_identity = AdapterIdentity(IntegrationKind.SANDBOX, "fake-sandbox", "1", "fake-config") + sandbox_id = "fake-sandbox" + + def __init__(self, artifacts: tuple[str, ...] = ()) -> None: + self.artifacts = artifacts + self.requests: list[SandboxRequest] = [] + + def prepare(self, request: SandboxRequest) -> None: + self.requests.append(request) + + def collect_artifacts(self) -> tuple[str, ...]: + return self.artifacts diff --git a/src/arkx/executor_qualification.py b/src/arkx/executor_qualification.py new file mode 100644 index 00000000..1bc8f621 --- /dev/null +++ b/src/arkx/executor_qualification.py @@ -0,0 +1,419 @@ +"""Executor qualification contracts for P8.1. + +This module records externally supplied executor observations. It does not +execute an executor, select one, accept a result, or promote a component. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +from enum import Enum +import json +from statistics import median +from typing import Any + +from .composition import Treatment + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class ExecutionOutcome(_ValueEnum): + COMPLETED = "COMPLETED" + EXECUTOR_FAILED = "EXECUTOR_FAILED" + INFRASTRUCTURE_FAILED = "INFRASTRUCTURE_FAILED" + BUDGET_EXHAUSTED = "BUDGET_EXHAUSTED" + BLOCKED = "BLOCKED" + UNKNOWN = "UNKNOWN" + + +class QualificationStatus(_ValueEnum): + QUALIFIABLE = "QUALIFIABLE" + INCOMPARABLE = "INCOMPARABLE" + BLOCKED = "BLOCKED" + UNKNOWN = "UNKNOWN" + + +class ComparisonAxis(_ValueEnum): + EXECUTOR = "EXECUTOR" + TREATMENT = "TREATMENT" + + +def _jsonable(value: Any) -> Any: + if isinstance(value, Decimal): + return str(value) + if isinstance(value, Enum): + return value.value + if isinstance(value, tuple): + return [_jsonable(item) for item in value] + if isinstance(value, list): + return [_jsonable(item) for item in value] + if isinstance(value, dict): + return {key: _jsonable(item) for key, item in value.items()} + return value + + +def _json(value: dict[str, Any]) -> str: + return json.dumps(_jsonable(value), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _nonnegative(value: int | Decimal | None, field_name: str) -> None: + if value is not None and value < 0: + raise ValueError(f"{field_name} must be non-negative when present") + + +def _sorted_refs(refs: tuple[str, ...] | None) -> list[str] | None: + return None if refs is None else sorted(set(refs)) + + +@dataclass(frozen=True) +class ExecutorIdentity: + name: str + version: str | None + integration_kind: str + configuration_digest: str | None + advertised_capabilities: tuple[str, ...] = () + + def __post_init__(self) -> None: + if not self.name or not self.integration_kind: + raise ValueError("Executor name and integration kind must be non-empty") + capabilities = tuple(self.advertised_capabilities) + if any(not capability for capability in capabilities): + raise ValueError("Advertised capability names must be non-empty") + object.__setattr__(self, "advertised_capabilities", tuple(sorted(set(capabilities)))) + + def to_dict(self) -> dict[str, str | None]: + return { + "name": self.name, + "version": self.version, + "integration_kind": self.integration_kind, + "configuration_digest": self.configuration_digest, + "advertised_capabilities": list(self.advertised_capabilities), + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class ExecutionEnvironment: + repository_revision: str | None + runtime: str | None + platform: str | None + toolchain: str | None + environment_digest: str | None + + def to_dict(self) -> dict[str, str | None]: + return { + "repository_revision": self.repository_revision, + "runtime": self.runtime, + "platform": self.platform, + "toolchain": self.toolchain, + "environment_digest": self.environment_digest, + } + + +@dataclass(frozen=True) +class ExecutionBudget: + max_tokens: int | None + max_cost: Decimal | None + max_wall_time: Decimal | None + max_attempts: int | None + max_executor_invocations: int | None + + def __post_init__(self) -> None: + for name in ("max_tokens", "max_attempts", "max_executor_invocations"): + _nonnegative(getattr(self, name), name) + _nonnegative(self.max_cost, "max_cost") + _nonnegative(self.max_wall_time, "max_wall_time") + + def to_dict(self) -> dict[str, Any]: + return { + "max_tokens": self.max_tokens, + "max_cost": self.max_cost, + "max_wall_time": self.max_wall_time, + "max_attempts": self.max_attempts, + "max_executor_invocations": self.max_executor_invocations, + } + + +@dataclass(frozen=True) +class QualificationTask: + task_id: str + task_revision: str | None + acceptance_definition: str | None + + def __post_init__(self) -> None: + if not self.task_id: + raise ValueError("task_id must be non-empty") + + def to_dict(self) -> dict[str, str | None]: + return { + "task_id": self.task_id, + "task_revision": self.task_revision, + "acceptance_definition": self.acceptance_definition, + } + + +@dataclass(frozen=True) +class ExecutorObservation: + accepted: bool | None = None + patch_verified: bool | None = None + false_pass: bool | None = None + human_interventions: int | None = None + input_tokens: int | None = None + output_tokens: int | None = None + total_tokens: int | None = None + cost: Decimal | None = None + wall_time_seconds: Decimal | None = None + attempts: int | None = None + retries: int | None = None + replans: int | None = None + handoffs: int | None = None + executor_invocations: int | None = None + unnecessary_changes: int | None = None + regressions: int | None = None + error_propagation_observed: bool | None = None + external_evidence: tuple[str, ...] | None = None + + def __post_init__(self) -> None: + for field_name in ( + "human_interventions", "input_tokens", "output_tokens", "total_tokens", + "attempts", "retries", "replans", "handoffs", "executor_invocations", + "unnecessary_changes", "regressions", + ): + _nonnegative(getattr(self, field_name), field_name) + _nonnegative(self.cost, "cost") + _nonnegative(self.wall_time_seconds, "wall_time_seconds") + + def is_consistent(self) -> bool: + return not (self.false_pass is True and (self.accepted is not True or self.patch_verified is not False)) + + def to_dict(self) -> dict[str, Any]: + return { + "accepted": self.accepted, + "patch_verified": self.patch_verified, + "false_pass": self.false_pass, + "human_interventions": self.human_interventions, + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "total_tokens": self.total_tokens, + "cost": self.cost, + "wall_time_seconds": self.wall_time_seconds, + "attempts": self.attempts, + "retries": self.retries, + "replans": self.replans, + "handoffs": self.handoffs, + "executor_invocations": self.executor_invocations, + "unnecessary_changes": self.unnecessary_changes, + "regressions": self.regressions, + "error_propagation_observed": self.error_propagation_observed, + "external_evidence": _sorted_refs(self.external_evidence), + } + + +@dataclass(frozen=True) +class ExecutorTrial: + executor: ExecutorIdentity + task: QualificationTask + model: str | None + environment: ExecutionEnvironment + budget: ExecutionBudget + treatment: Treatment + replicate_id: str + outcome: ExecutionOutcome + observation: ExecutorObservation + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + if not self.replicate_id: + raise ValueError("replicate_id must be non-empty") + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "executor": self.executor.to_dict(), + "task": self.task.to_dict(), + "model": self.model, + "environment": self.environment.to_dict(), + "budget": self.budget.to_dict(), + "treatment": self.treatment.to_dict(), + "replicate_id": self.replicate_id, + "outcome": self.outcome.value, + "observation": self.observation.to_dict(), + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class QualificationExperiment: + trials: tuple[ExecutorTrial, ...] + comparison_axis: ComparisonAxis + schema_version: int = SCHEMA_VERSION + verifier_identity: str | None = None + instrumentation_identity: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "comparison_axis": self.comparison_axis.value, + "verifier_identity": self.verifier_identity, + "instrumentation_identity": self.instrumentation_identity, + "trials": [trial.to_dict() for trial in sorted(self.trials, key=lambda item: (item.replicate_id, item.executor.name, item.treatment.name))], + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class PairedQualificationReport: + status: QualificationStatus + executor_names: tuple[str, ...] + task_count: int + replicate_count: int + reason_codes: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "status": self.status.value, + "executor_names": list(self.executor_names), + "task_count": self.task_count, + "replicate_count": self.replicate_count, + "reason_codes": list(self.reason_codes), + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class QualificationSummary: + trial_count: int + qualifiable_trial_count: int + status_counts: tuple[tuple[str, int], ...] + outcome_counts: tuple[tuple[str, int], ...] + acceptance_rate: Decimal | None + false_pass_rate: Decimal | None + median_cost: Decimal | None + median_wall_time_seconds: Decimal | None + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "trial_count": self.trial_count, + "qualifiable_trial_count": self.qualifiable_trial_count, + "status_counts": dict(self.status_counts), + "outcome_counts": dict(self.outcome_counts), + "acceptance_rate": self.acceptance_rate, + "false_pass_rate": self.false_pass_rate, + "median_cost": self.median_cost, + "median_wall_time_seconds": self.median_wall_time_seconds, + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + +def assess_trial(trial: ExecutorTrial) -> QualificationStatus: + if not trial.observation.is_consistent(): + return QualificationStatus.BLOCKED + if trial.outcome is ExecutionOutcome.UNKNOWN: + return QualificationStatus.UNKNOWN + if trial.outcome is not ExecutionOutcome.COMPLETED: + return QualificationStatus.BLOCKED + if trial.observation.accepted is None: + return QualificationStatus.UNKNOWN + return QualificationStatus.QUALIFIABLE + + +def compare_trials(left: ExecutorTrial, right: ExecutorTrial, axis: ComparisonAxis) -> QualificationStatus: + if left.task != right.task or left.model != right.model or left.environment != right.environment or left.budget != right.budget or left.treatment != right.treatment and axis is ComparisonAxis.EXECUTOR: + return QualificationStatus.INCOMPARABLE + if axis is ComparisonAxis.EXECUTOR: + return QualificationStatus.INCOMPARABLE if left.executor == right.executor else _pair_status(left, right) + if left.executor != right.executor: + return QualificationStatus.INCOMPARABLE + return QualificationStatus.INCOMPARABLE if left.treatment == right.treatment else _pair_status(left, right) + + +def validate_paired_executor_experiment(experiment: QualificationExperiment) -> PairedQualificationReport: + """Validate the structural controls for an executor-only paired run. + + This function never ranks executors and never infers a performance result. + It only decides whether the submitted observations form a comparable pair + set for a later analysis. + """ + trials = tuple(experiment.trials) + executors = tuple(sorted({trial.executor.name for trial in trials})) + reasons: list[str] = [] + if experiment.comparison_axis is not ComparisonAxis.EXECUTOR: + reasons.append("COMPARISON_AXIS_NOT_EXECUTOR") + if len(executors) < 2: + reasons.append("AT_LEAST_TWO_EXECUTORS_REQUIRED") + if not experiment.verifier_identity: + reasons.append("VERIFIER_IDENTITY_MISSING") + if not experiment.instrumentation_identity: + reasons.append("INSTRUMENTATION_IDENTITY_MISSING") + cells: dict[tuple[str, str | None, str | None, str], list[ExecutorTrial]] = {} + for trial in trials: + key = (trial.task.task_id, trial.task.task_revision, trial.task.acceptance_definition, trial.replicate_id) + cells.setdefault(key, []).append(trial) + if any(len(cell) != len(executors) for cell in cells.values()): + reasons.append("PAIRED_CELL_INCOMPLETE") + if any(len({trial.executor.name for trial in cell}) != len(cell) for cell in cells.values()): + reasons.append("DUPLICATE_EXECUTOR_CELL") + for cell in cells.values(): + first = cell[0] + for other in cell[1:]: + if compare_trials(first, other, ComparisonAxis.EXECUTOR) is QualificationStatus.INCOMPARABLE: + reasons.append("CONTROL_DIMENSION_MISMATCH") + break + if any(assess_trial(trial) is QualificationStatus.BLOCKED for trial in trials): + reasons.append("OBSERVED_BLOCKED_TRIAL") + elif any(assess_trial(trial) is QualificationStatus.UNKNOWN for trial in trials): + reasons.append("OBSERVATION_INCOMPLETE") + status = QualificationStatus.QUALIFIABLE if not reasons else ( + QualificationStatus.BLOCKED if "OBSERVED_BLOCKED_TRIAL" in reasons else QualificationStatus.UNKNOWN + ) + if any(reason in reasons for reason in ("COMPARISON_AXIS_NOT_EXECUTOR", "AT_LEAST_TWO_EXECUTORS_REQUIRED", "PAIRED_CELL_INCOMPLETE", "DUPLICATE_EXECUTOR_CELL", "CONTROL_DIMENSION_MISMATCH")): + status = QualificationStatus.INCOMPARABLE + return PairedQualificationReport(status, executors, len({key[:3] for key in cells}), len({key[3] for key in cells}), tuple(sorted(set(reasons)))) + + +def _pair_status(left: ExecutorTrial, right: ExecutorTrial) -> QualificationStatus: + statuses = (assess_trial(left), assess_trial(right)) + if QualificationStatus.BLOCKED in statuses: + return QualificationStatus.BLOCKED + if QualificationStatus.UNKNOWN in statuses: + return QualificationStatus.UNKNOWN + return QualificationStatus.QUALIFIABLE + + +def summarize_experiment(experiment: QualificationExperiment) -> QualificationSummary: + trials = tuple(experiment.trials) + statuses = [assess_trial(trial) for trial in trials] + qualifiable = [trial for trial, status in zip(trials, statuses) if status is QualificationStatus.QUALIFIABLE] + accepted = [trial.observation.accepted for trial in qualifiable] + false_pass_values = [trial.observation.false_pass for trial in trials if trial.observation.false_pass is not None] + costs = [trial.observation.cost for trial in qualifiable if trial.observation.cost is not None] + wall_times = [trial.observation.wall_time_seconds for trial in qualifiable if trial.observation.wall_time_seconds is not None] + return QualificationSummary( + trial_count=len(trials), + qualifiable_trial_count=len(qualifiable), + status_counts=tuple(sorted((status.value, statuses.count(status)) for status in set(statuses))), + outcome_counts=tuple(sorted((outcome.value, sum(trial.outcome is outcome for trial in trials)) for outcome in set(trial.outcome for trial in trials))), + acceptance_rate=None if not accepted else Decimal(sum(accepted)) / Decimal(len(accepted)), + false_pass_rate=None if not false_pass_values else Decimal(sum(false_pass_values)) / Decimal(len(false_pass_values)), + median_cost=None if not costs else median(costs), + median_wall_time_seconds=None if not wall_times else median(wall_times), + ) diff --git a/src/arkx/handoff.py b/src/arkx/handoff.py new file mode 100644 index 00000000..f9e25496 --- /dev/null +++ b/src/arkx/handoff.py @@ -0,0 +1,139 @@ +"""Explicit handoff and context accounting records for P6.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +import json +from typing import Any + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class HandoffBudgetStatus(_ValueEnum): + WITHIN_BUDGET = "WITHIN_BUDGET" + BUDGET_EXCEEDED = "BUDGET_EXCEEDED" + UNKNOWN = "UNKNOWN" + + +def _values(items: tuple[str, ...] | None) -> list[str] | None: + return None if items is None else sorted(set(items)) + + +@dataclass(frozen=True) +class HandoffRecord: + handoff_id: str + source_executor: str | None + target_executor: str | None + reason: str | None + evidence_refs: tuple[str, ...] | None + context_summary: str | None + context_bytes_in: int | None + context_bytes_out: int | None + duplicated_instructions: int | None + duplicated_exploration: int | None + discarded_context: int | None + lost_information: tuple[str, ...] | None + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "handoff_id": self.handoff_id, + "source_executor": self.source_executor, + "target_executor": self.target_executor, + "reason": self.reason, + "evidence_refs": _values(self.evidence_refs), + "context_summary": self.context_summary, + "context_bytes_in": self.context_bytes_in, + "context_bytes_out": self.context_bytes_out, + "duplicated_instructions": self.duplicated_instructions, + "duplicated_exploration": self.duplicated_exploration, + "discarded_context": self.discarded_context, + "lost_information": _values(self.lost_information), + } + + +@dataclass(frozen=True) +class HandoffPolicy: + max_handoffs: int = 2 + max_executor_transitions: int = 2 + + +@dataclass(frozen=True) +class HandoffSummary: + handoffs: int + executor_transitions: tuple[tuple[str, str], ...] + context_bytes_in: int | None + context_bytes_out: int | None + duplicated_instructions: int | None + duplicated_exploration: int | None + discarded_context: int | None + lost_information_count: int | None + budget_status: HandoffBudgetStatus + schema_version: int = SCHEMA_VERSION + telemetry: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "handoffs": self.handoffs, + "executor_transitions": [list(edge) for edge in self.executor_transitions], + "context_bytes_in": self.context_bytes_in, + "context_bytes_out": self.context_bytes_out, + "duplicated_instructions": self.duplicated_instructions, + "duplicated_exploration": self.duplicated_exploration, + "discarded_context": self.discarded_context, + "lost_information_count": self.lost_information_count, + "budget_status": self.budget_status.value, + "telemetry": self.telemetry, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def summarize_handoffs(records: tuple[HandoffRecord, ...], policy: HandoffPolicy | None = None) -> HandoffSummary: + policy = policy or HandoffPolicy() + transitions = tuple(sorted({(record.source_executor, record.target_executor) for record in records if record.source_executor and record.target_executor})) + + def total(field: str) -> int | None: + values = [getattr(record, field) for record in records] + return None if any(value is None for value in values) else sum(values) + + lost_values = [record.lost_information for record in records] + lost_count = None if any(value is None for value in lost_values) else sum(len(value) for value in lost_values) + if len(records) > policy.max_handoffs or len(transitions) > policy.max_executor_transitions: + budget_status = HandoffBudgetStatus.BUDGET_EXCEEDED + elif any(record.source_executor is None or record.target_executor is None for record in records): + budget_status = HandoffBudgetStatus.UNKNOWN + else: + budget_status = HandoffBudgetStatus.WITHIN_BUDGET + return HandoffSummary( + handoffs=len(records), + executor_transitions=transitions, + context_bytes_in=total("context_bytes_in"), + context_bytes_out=total("context_bytes_out"), + duplicated_instructions=total("duplicated_instructions"), + duplicated_exploration=total("duplicated_exploration"), + discarded_context=total("discarded_context"), + lost_information_count=lost_count, + budget_status=budget_status, + telemetry={ + "handoffs": len(records), + "executor_transitions": len(transitions), + "context_bytes_in": total("context_bytes_in"), + "context_bytes_out": total("context_bytes_out"), + "duplicated_instructions": total("duplicated_instructions"), + "duplicated_exploration": total("duplicated_exploration"), + "discarded_context": total("discarded_context"), + "lost_information_count": lost_count, + }, + ) + diff --git a/src/arkx/harness.py b/src/arkx/harness.py new file mode 100644 index 00000000..e36f0774 --- /dev/null +++ b/src/arkx/harness.py @@ -0,0 +1,953 @@ +"""Executor-neutral contracts for reproducible harness runs. + +This module records identity and failure boundaries. It deliberately does not +execute providers, executors, sandboxes, verification, or acceptance. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from datetime import datetime +from itertools import combinations +from enum import Enum +import hashlib +import json +from pathlib import Path +from typing import Any, Mapping + + +SCHEMA_VERSION = 1 + + +class ErrorDomain(str, Enum): + HARNESS = "HARNESS" + PROVIDER = "PROVIDER" + EXECUTOR = "EXECUTOR" + SANDBOX = "SANDBOX" + VERIFICATION = "VERIFICATION" + ACCEPTANCE = "ACCEPTANCE" + + +class Retryability(str, Enum): + RETRYABLE = "RETRYABLE" + NOT_RETRYABLE = "NOT_RETRYABLE" + UNKNOWN = "UNKNOWN" + + +class ArtifactStatus(str, Enum): + PRESENT = "PRESENT" + MISSING = "MISSING" + + +class ComparabilityStatus(str, Enum): + COMPARABLE = "COMPARABLE" + INCOMPARABLE = "INCOMPARABLE" + INDETERMINATE = "INDETERMINATE" + + +@dataclass(frozen=True) +class ArtifactRecord: + kind: str + status: ArtifactStatus + path: str | None + required: bool = False + reason: str | None = None + + def __post_init__(self) -> None: + _require(self.kind, "artifact kind") + if self.status is ArtifactStatus.PRESENT and not self.path: + raise ValueError("present artifacts require a path") + if self.status is ArtifactStatus.MISSING and not self.reason: + raise ValueError("missing artifacts require a reason") + + def to_dict(self) -> dict[str, Any]: + return { + "kind": self.kind, + "status": self.status.value, + "path": self.path, + "required": self.required, + "reason": self.reason, + } + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "ArtifactRecord": + return cls( + kind=str(value["kind"]), + status=ArtifactStatus(value["status"]), + path=value.get("path"), + required=bool(value.get("required", False)), + reason=value.get("reason"), + ) + + +class RunState(str, Enum): + PLANNED = "PLANNED" + STARTED = "STARTED" + EXECUTING = "EXECUTING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + BLOCKED = "BLOCKED" + UNKNOWN = "UNKNOWN" + + +_ALLOWED_TRANSITIONS: dict[RunState, frozenset[RunState]] = { + RunState.PLANNED: frozenset({RunState.STARTED, RunState.BLOCKED}), + RunState.STARTED: frozenset({RunState.EXECUTING, RunState.BLOCKED, RunState.FAILED}), + RunState.EXECUTING: frozenset({RunState.COMPLETED, RunState.BLOCKED, RunState.FAILED, RunState.UNKNOWN}), + RunState.COMPLETED: frozenset(), + RunState.FAILED: frozenset(), + RunState.BLOCKED: frozenset(), + RunState.UNKNOWN: frozenset(), +} + + +def _json(value: dict[str, Any]) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _require(value: str, name: str) -> None: + if not value or not value.strip(): + raise ValueError(f"{name} must be non-empty") + + +def validate_transition(previous: RunState, current: RunState) -> None: + if current not in _ALLOWED_TRANSITIONS[previous]: + raise ValueError(f"invalid harness transition: {previous.value} -> {current.value}") + + +def validate_execution_artifact_consistency(manifest: "RunManifest", execution: Mapping[str, Any]) -> None: + """Validate the legacy execution record against its common manifest.""" + + if execution.get("run_id") != manifest.attempt_id: + raise ValueError("execution run_id does not match manifest attempt_id") + if execution.get("trial_id") != manifest.trial_id: + raise ValueError("execution trial_id does not match manifest trial_id") + if int(execution.get("attempt", 0)) != manifest.attempt_number: + raise ValueError("execution attempt does not match manifest attempt_number") + state_map = {"COMPLETED": RunState.COMPLETED, "EXECUTOR_FAILED": RunState.FAILED, "BLOCKED": RunState.BLOCKED} + if state_map.get(execution.get("state")) is not manifest.state: + raise ValueError("execution state does not match manifest state") + execution_error = execution.get("error") + if bool(manifest.errors) != bool(execution_error): + raise ValueError("manifest and execution disagree about terminal error") + if execution.get("manifest_path") is None: + raise ValueError("execution artifact must point to its manifest") + + +@dataclass(frozen=True) +class RetryLineage: + trial_id: str + previous_attempt_id: str + previous_attempt_number: int + attempt_id: str + attempt_number: int + configuration_digest: str + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "RetryLineage": + lineage = cls( + trial_id=str(value["trial_id"]), + previous_attempt_id=str(value["previous_attempt_id"]), + previous_attempt_number=int(value["previous_attempt_number"]), + attempt_id=str(value["attempt_id"]), + attempt_number=int(value["attempt_number"]), + configuration_digest=str(value["configuration_digest"]), + ) + if lineage.attempt_number != lineage.previous_attempt_number + 1: + raise ValueError("retry lineage attempt numbers must be consecutive") + expected_previous = derive_attempt_id(trial_id=lineage.trial_id, attempt_number=lineage.previous_attempt_number, configuration_digest=lineage.configuration_digest) + expected_current = derive_attempt_id(trial_id=lineage.trial_id, attempt_number=lineage.attempt_number, configuration_digest=lineage.configuration_digest) + if lineage.previous_attempt_id != expected_previous or lineage.attempt_id != expected_current: + raise ValueError("retry lineage identities do not match its inputs") + return lineage + + def to_dict(self) -> dict[str, Any]: + return { + "trial_id": self.trial_id, + "previous_attempt_id": self.previous_attempt_id, + "previous_attempt_number": self.previous_attempt_number, + "attempt_id": self.attempt_id, + "attempt_number": self.attempt_number, + "configuration_digest": self.configuration_digest, + } + + +@dataclass(frozen=True) +class RecoveryLineage: + trial_id: str + source_attempt_id: str + source_attempt_number: int + attempt_id: str + attempt_number: int + configuration_digest: str + recovery_reference: str + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "RecoveryLineage": + lineage = cls(str(value["trial_id"]), str(value["source_attempt_id"]), int(value["source_attempt_number"]), str(value["attempt_id"]), int(value["attempt_number"]), str(value["configuration_digest"]), str(value["recovery_reference"])) + if not lineage.recovery_reference.strip() or lineage.attempt_number != lineage.source_attempt_number + 1: + raise ValueError("recovery lineage is incomplete or non-consecutive") + expected_source = derive_attempt_id(trial_id=lineage.trial_id, attempt_number=lineage.source_attempt_number, configuration_digest=lineage.configuration_digest) + expected_current = derive_attempt_id(trial_id=lineage.trial_id, attempt_number=lineage.attempt_number, configuration_digest=lineage.configuration_digest) + if lineage.source_attempt_id != expected_source or lineage.attempt_id != expected_current: + raise ValueError("recovery lineage identities do not match its inputs") + return lineage + + def to_dict(self) -> dict[str, Any]: + return {"trial_id": self.trial_id, "source_attempt_id": self.source_attempt_id, "source_attempt_number": self.source_attempt_number, "attempt_id": self.attempt_id, "attempt_number": self.attempt_number, "configuration_digest": self.configuration_digest, "recovery_reference": self.recovery_reference} + + +@dataclass(frozen=True) +class AttemptSnapshot: + manifest: "RunManifest" + execution: Mapping[str, Any] + artifact_paths: tuple[Path, ...] + retry_lineage: RetryLineage | None = None + recovery_lineage: RecoveryLineage | None = None + + +@dataclass(frozen=True) +class AttemptComparison: + status: ComparabilityStatus + differing_fields: tuple[str, ...] = () + missing_fields: tuple[str, ...] = () + + +@dataclass(frozen=True) +class PairComparison: + left_attempt_id: str + right_attempt_id: str + comparison: AttemptComparison + + +@dataclass(frozen=True) +class ComparisonReport: + pairs: tuple[PairComparison, ...] + + @property + def comparable_pairs(self) -> tuple[PairComparison, ...]: + return tuple(pair for pair in self.pairs if pair.comparison.status is ComparabilityStatus.COMPARABLE) + + @property + def indeterminate_pairs(self) -> tuple[PairComparison, ...]: + return tuple(pair for pair in self.pairs if pair.comparison.status is ComparabilityStatus.INDETERMINATE) + + @property + def incomparable_pairs(self) -> tuple[PairComparison, ...]: + return tuple(pair for pair in self.pairs if pair.comparison.status is ComparabilityStatus.INCOMPARABLE) + + +@dataclass(frozen=True) +class AuditedComparison: + comparison: AttemptComparison + left_audit: Any + right_audit: Any + + +@dataclass(frozen=True) +class AttemptEvidence: + attempt_id: str + execution_state: RunState + verification_evidence: "VerificationEvidence | None" = None + acceptance_evidence: "AcceptanceEvidence | None" = None + verification_state: str | None = None + acceptance_decision: str | None = None + accepted: bool | None = None + verified: bool | None = None + input_tokens: int | None = None + output_tokens: int | None = None + total_tokens: int | None = None + cost: float | None = None + wall_time_seconds: float | None = None + + +@dataclass(frozen=True) +class ExperimentRecord: + experiment_id: str + protocol_version: str + input_manifest_digest: str + trial_ids: tuple[str, ...] = () + attempt_ids: tuple[str, ...] = () + verification_refs: tuple[str, ...] = () + acceptance_refs: tuple[str, ...] = () + promotion_decision_ref: str | None = None + status: str = "PLANNED" + identity_digest: str | None = None + capability_digest: str | None = None + + def __post_init__(self) -> None: + for value, name in ((self.experiment_id, "experiment_id"), (self.protocol_version, "protocol_version"), (self.input_manifest_digest, "input_manifest_digest")): + _require(value, name) + for values, name in ((self.trial_ids, "trial_ids"), (self.attempt_ids, "attempt_ids"), (self.verification_refs, "verification_refs"), (self.acceptance_refs, "acceptance_refs")): + if any(not value.strip() for value in values): + raise ValueError(f"{name} must not contain empty references") + if len(set(values)) != len(values): + raise ValueError(f"{name} must be unique") + if self.promotion_decision_ref and not self.attempt_ids: + raise ValueError("promotion decision requires at least one attempt") + if self.identity_digest is not None: + _require(self.identity_digest, "identity_digest") + if self.capability_digest is not None: + _require(self.capability_digest, "capability_digest") + + def to_dict(self) -> dict[str, Any]: + return { + "experiment_id": self.experiment_id, + "protocol_version": self.protocol_version, + "input_manifest_digest": self.input_manifest_digest, + "trial_ids": list(self.trial_ids), + "attempt_ids": list(self.attempt_ids), + "verification_refs": list(self.verification_refs), + "acceptance_refs": list(self.acceptance_refs), + "promotion_decision_ref": self.promotion_decision_ref, + "status": self.status, + "identity_digest": self.identity_digest, + "capability_digest": self.capability_digest, + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + @classmethod + def from_json(cls, value: str) -> "ExperimentRecord": + payload = json.loads(value) + return cls(**payload) + + +@dataclass(frozen=True) +class ExperimentSnapshot: + record: ExperimentRecord + attempts: tuple[AttemptSnapshot, ...] + + +def derive_experiment_identity_digest(attempts: tuple[AttemptSnapshot, ...]) -> str: + """Derive a stable identity over execution-relevant attempt fields.""" + values = [] + for attempt in sorted(attempts, key=lambda item: item.manifest.attempt_id): + manifest = attempt.manifest + values.append({ + "attempt_id": manifest.attempt_id, + "task_id": manifest.task_id, + "task_revision": manifest.task_revision, + "treatment": manifest.treatment, + "executor": manifest.executor, + "provider": manifest.provider, + "model": manifest.model, + "sandbox": manifest.sandbox, + "repository_revision": manifest.repository_revision, + "configuration_digest": manifest.configuration_digest, + "budget_digest": manifest.budget_digest, + "protocol_version": manifest.protocol_version, + }) + return hashlib.sha256(_json({"attempts": values}).encode("utf-8")).hexdigest()[:16] + + +def _validate_experiment_reference(reference: str, attempts: tuple[AttemptSnapshot, ...]) -> None: + if reference.startswith(("evidence://", "experiment://", "acceptance://", "verification://", "promotion://")): + return + available = {str(path) for attempt in attempts for path in attempt.artifact_paths} + if reference not in available: + raise ValueError(f"experiment reference is not present in validated artifacts: {reference}") + + +def validate_experiment_record(record: ExperimentRecord, attempts: tuple[AttemptSnapshot, ...], *, promotion_attempt_id: str | None = None) -> None: + attempt_ids = {attempt.manifest.attempt_id for attempt in attempts} + trial_ids = {attempt.manifest.trial_id for attempt in attempts} + if set(record.attempt_ids) != attempt_ids: + raise ValueError("experiment record attempt_ids do not match loaded attempts") + if not set(record.trial_ids).issuperset(trial_ids): + raise ValueError("experiment record is missing a loaded trial_id") + for reference in record.verification_refs + record.acceptance_refs: + _validate_experiment_reference(reference, attempts) + if record.promotion_decision_ref: + if promotion_attempt_id is None or promotion_attempt_id not in attempt_ids: + raise ValueError("promotion decision is not bound to a recorded attempt") + + +def validate_experiment_record_audited( + record: ExperimentRecord, + attempts: tuple[AttemptSnapshot, ...], + audits: tuple[Any, ...], + *, + promotion_attempt_id: str | None = None, +) -> None: + """Require complete restored chains before treating an experiment as reproducible.""" + validate_experiment_record(record, attempts, promotion_attempt_id=promotion_attempt_id) + if not record.identity_digest: + raise ValueError("audited experiment requires identity_digest") + if not record.capability_digest: + raise ValueError("audited experiment requires capability_digest") + if record.identity_digest != derive_experiment_identity_digest(attempts): + raise ValueError("experiment identity_digest does not match audited attempts") + if len(attempts) != len(audits): + raise ValueError("audited experiment attempts and audits must have equal cardinality") + chain_refs: set[str] = set() + for attempt, audit in zip(attempts, audits): + if audit.replay.run_id != attempt.manifest.attempt_id: + raise ValueError("audited chain run_id does not match experiment attempt") + if audit.protocol_version != record.protocol_version: + raise ValueError("audited chain protocol_version does not match experiment") + if audit.schema_version is None: + raise ValueError("audited chain requires explicit event schema_version") + manifest = attempt.manifest + if audit.configuration_digest != manifest.configuration_digest: + raise ValueError("audited chain configuration_digest does not match attempt") + if audit.budget_digest != manifest.budget_digest: + raise ValueError("audited chain budget_digest does not match attempt") + if audit.treatment != manifest.treatment: + raise ValueError("audited chain treatment does not match attempt") + if audit.capability_digest != record.capability_digest: + raise ValueError("audited chain capability_digest does not match experiment") + if str(audit.integrity) != "ChainIntegrity.COMPLETE" and getattr(audit.integrity, "value", audit.integrity) != "COMPLETE": + raise ValueError("experiment requires complete audited attempt chains") + chain_refs.update(ref for ref in audit.chain.to_dict().values() if ref is not None) + required_refs = set(record.verification_refs) | set(record.acceptance_refs) + if record.promotion_decision_ref: + required_refs.add(record.promotion_decision_ref) + missing = sorted(required_refs - chain_refs) + if missing: + raise ValueError(f"experiment references are absent from audited chains: {','.join(missing)}") + + +def write_experiment_record(record: ExperimentRecord, path: str | Path) -> Path: + target = Path(path) + if target.exists(): + raise FileExistsError(f"experiment record already exists: {target}") + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + ".tmp") + temporary.write_text(record.to_json() + "\n", encoding="utf-8", newline="\n") + temporary.replace(target) + return target + + +def load_experiment_record(path: str | Path, artifact_root: str | Path, *, promotion_attempt_id: str | None = None) -> ExperimentSnapshot: + record = ExperimentRecord.from_json(Path(path).read_text(encoding="utf-8")) + attempts = tuple( + attempt for attempt in AttemptStore(artifact_root).list_attempts() + if attempt.manifest.attempt_id in set(record.attempt_ids) + ) + validate_experiment_record(record, attempts, promotion_attempt_id=promotion_attempt_id) + return ExperimentSnapshot(record, attempts) + + +@dataclass(frozen=True) +class VerificationEvidence: + authority_identity: str | None + state: str | None + evidence_refs: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AcceptanceEvidence: + authority_identity: str | None + decision: str | None + evidence_refs: tuple[str, ...] = () + + +def evidence_from_snapshot(snapshot: "AttemptSnapshot") -> AttemptEvidence: + usage = snapshot.execution.get("model_usage") + usage = usage if isinstance(usage, Mapping) else {} + verification = snapshot.execution.get("verification") + verification = verification if isinstance(verification, Mapping) else None + acceptance = snapshot.execution.get("acceptance") + acceptance = acceptance if isinstance(acceptance, Mapping) else None + return AttemptEvidence( + attempt_id=snapshot.manifest.attempt_id, + execution_state=snapshot.manifest.state, + verification_evidence=None if verification is None else VerificationEvidence(verification.get("authority_identity"), verification.get("state"), tuple(verification.get("evidence_refs", ()))), + acceptance_evidence=None if acceptance is None else AcceptanceEvidence(acceptance.get("authority_identity"), acceptance.get("decision"), tuple(acceptance.get("evidence_refs", ()))), + verification_state=snapshot.execution.get("verification_state") if verification is None else verification.get("state"), + acceptance_decision=snapshot.execution.get("acceptance_decision") if acceptance is None else acceptance.get("decision"), + accepted=snapshot.execution.get("accepted"), + verified=snapshot.execution.get("verified"), + input_tokens=usage.get("input_tokens"), + output_tokens=usage.get("output_tokens"), + total_tokens=usage.get("total_tokens"), + cost=usage.get("cost"), + wall_time_seconds=usage.get("wall_time_seconds"), + ) + + +def compare_attempts(left: "RunManifest", right: "RunManifest") -> AttemptComparison: + fields = ( + "task_id", "task_revision", "treatment", "executor", "provider", "model", + "sandbox", "configuration_digest", "budget_digest", "protocol_version", + ) + differing = tuple(field for field in fields if getattr(left, field) is not None and getattr(right, field) is not None and getattr(left, field) != getattr(right, field)) + missing = tuple(field for field in fields if getattr(left, field) is None or getattr(right, field) is None) + if differing: + return AttemptComparison(ComparabilityStatus.INCOMPARABLE, differing_fields=differing, missing_fields=missing) + if missing: + return AttemptComparison(ComparabilityStatus.INDETERMINATE, missing_fields=missing) + return AttemptComparison(ComparabilityStatus.COMPARABLE) + + +def compare_attempt_collection(manifests: tuple["RunManifest", ...]) -> ComparisonReport: + pairs = tuple( + PairComparison(left.attempt_id, right.attempt_id, compare_attempts(left, right)) + for left, right in combinations(manifests, 2) + ) + return ComparisonReport(pairs) + + +def load_attempt(attempt_root: str | Path) -> AttemptSnapshot: + root = Path(attempt_root).resolve() + manifest_path = root / "manifest.json" + execution_path = root / "execution.json" + if not manifest_path.is_file() or not execution_path.is_file(): + raise FileNotFoundError("attempt requires manifest.json and execution.json") + manifest = RunManifest.from_json(manifest_path.read_text(encoding="utf-8")) + execution = json.loads(execution_path.read_text(encoding="utf-8")) + manifest.validate_attempt_identity() + manifest.validate_lifecycle() + validate_execution_artifact_consistency(manifest, execution) + lineage_path = root / "retry-lineage.json" + lineage = None + if lineage_path.is_file(): + lineage = RetryLineage.from_dict(json.loads(lineage_path.read_text(encoding="utf-8"))) + if lineage.trial_id != manifest.trial_id or lineage.attempt_id != manifest.attempt_id or lineage.attempt_number != manifest.attempt_number: + raise ValueError("retry lineage does not match attempt manifest") + if lineage.configuration_digest != manifest.configuration_digest: + raise ValueError("retry lineage configuration does not match attempt manifest") + recovery_path = root / "recovery-lineage.json" + recovery_lineage = None + if recovery_path.is_file(): + if lineage is not None: + raise ValueError("attempt cannot contain both retry and recovery lineage") + recovery_lineage = RecoveryLineage.from_dict(json.loads(recovery_path.read_text(encoding="utf-8"))) + if recovery_lineage.trial_id != manifest.trial_id or recovery_lineage.attempt_id != manifest.attempt_id or recovery_lineage.attempt_number != manifest.attempt_number: + raise ValueError("recovery lineage does not match attempt manifest") + if recovery_lineage.configuration_digest != manifest.configuration_digest: + raise ValueError("recovery lineage configuration does not match attempt manifest") + store = ArtifactStore(root.parents[1]) + artifact_paths = store.validate_manifest_references(manifest) + return AttemptSnapshot(manifest, execution, artifact_paths, lineage, recovery_lineage) + + +class AttemptStore: + """Read-only index over validated attempts; it never selects a winner.""" + + def __init__(self, root: str | Path) -> None: + self.root = Path(root).resolve() + + def list_attempts(self, task_id: str | None = None) -> tuple[AttemptSnapshot, ...]: + task_roots = [self.root / task_id] if task_id else [path for path in self.root.iterdir() if path.is_dir()] + snapshots: list[AttemptSnapshot] = [] + for task_root in sorted(task_roots, key=lambda path: path.name): + if not task_root.is_dir(): + continue + for attempt_root in sorted(task_root.glob("attempt-*"), key=lambda path: path.name): + if attempt_root.is_dir(): + snapshots.append(load_attempt(attempt_root)) + return tuple(sorted(snapshots, key=lambda item: (item.manifest.task_id, item.manifest.attempt_number))) + + def load_audited(self, attempt_root: str | Path, event_log_path: str | Path, **outcomes: Any) -> Any: + """Load an attempt and audit its event/outcome chain before consumption.""" + snapshot = load_attempt(attempt_root) + from .event_log import audit_restored_chain + return audit_restored_chain(event_log_path, snapshot, **outcomes) + + def compare_audited( + self, + left_root: str | Path, + left_event_log: str | Path, + right_root: str | Path, + right_event_log: str | Path, + ) -> AuditedComparison: + """Compare attempts only after both persisted chains are complete.""" + from .event_log import ChainIntegrity + left = load_attempt(left_root) + right = load_attempt(right_root) + left_audit = self.load_audited(left_root, left_event_log) + right_audit = self.load_audited(right_root, right_event_log) + if left_audit.integrity is not ChainIntegrity.COMPLETE or right_audit.integrity is not ChainIntegrity.COMPLETE: + raise ValueError("audited comparison requires complete persisted chains") + return AuditedComparison(compare_attempts(left.manifest, right.manifest), left_audit, right_audit) + + +class ArtifactStore: + """Small append-only store for one harness artifact root.""" + + def __init__(self, root: str | Path) -> None: + self.root = Path(root).resolve() + + def reserve_attempt(self, task_id: str, attempt_number: int) -> Path: + _require(task_id, "task_id") + if attempt_number < 1: + raise ValueError("attempt_number must be positive") + path = (self.root / task_id / f"attempt-{attempt_number}").resolve() + if self.root not in path.parents: + raise ValueError("task_id escapes artifact root") + if path.exists(): + raise FileExistsError(f"attempt artifact already exists: {path}") + path.mkdir(parents=True) + return path + + def reserve_retry_attempt( + self, + task_id: str, + *, + trial_id: str, + previous_attempt_id: str, + previous_attempt_number: int, + configuration_digest: str, + budget_ledger: Mapping[str, Any] | None = None, + ) -> Path: + """Reserve a new retry directory and persist its immutable lineage.""" + if previous_attempt_number < 1: + raise ValueError("previous_attempt_number must be positive") + expected_previous = derive_attempt_id(trial_id=trial_id, attempt_number=previous_attempt_number, configuration_digest=configuration_digest) + if previous_attempt_id != expected_previous: + raise ValueError("previous attempt identity does not match lineage inputs") + if budget_ledger is not None: + consumed = tuple(str(value) for value in budget_ledger.get("consumed_attempts", ())) + budget = budget_ledger.get("budget", {}) + if previous_attempt_id not in consumed: + raise ValueError("budget ledger does not contain previous retry attempt") + max_attempts = budget.get("max_attempts") if isinstance(budget, Mapping) else None + if max_attempts is not None and len(consumed) + 1 > int(max_attempts): + raise ValueError("retry attempt exceeds execution budget") + next_number = previous_attempt_number + 1 + path = self.reserve_attempt(task_id, next_number) + next_id = derive_attempt_id(trial_id=trial_id, attempt_number=next_number, configuration_digest=configuration_digest) + self.write_text_atomic(path / "retry-lineage.json", _json({ + "trial_id": trial_id, + "previous_attempt_id": previous_attempt_id, + "previous_attempt_number": previous_attempt_number, + "attempt_id": next_id, + "attempt_number": next_number, + "configuration_digest": configuration_digest, + })) + if budget_ledger is not None: + updated_ledger = dict(budget_ledger) + updated_ledger["consumed_attempts"] = [*tuple(str(value) for value in budget_ledger.get("consumed_attempts", ())), next_id] + self.write_text_atomic(path / "budget-ledger.json", _json(updated_ledger)) + return path + + def reserve_recovery_attempt( + self, + task_id: str, + *, + trial_id: str, + source_attempt_id: str, + source_attempt_number: int, + configuration_digest: str, + recovery_reference: str, + budget_ledger: Mapping[str, Any] | None = None, + ) -> Path: + """Reserve a fresh recovery attempt and persist its causal lineage.""" + if not recovery_reference.strip(): + raise ValueError("recovery_reference must be non-empty") + if source_attempt_number < 1: + raise ValueError("source_attempt_number must be positive") + expected_source = derive_attempt_id(trial_id=trial_id, attempt_number=source_attempt_number, configuration_digest=configuration_digest) + if source_attempt_id != expected_source: + raise ValueError("source attempt identity does not match recovery lineage") + if budget_ledger is not None: + consumed = tuple(str(value) for value in budget_ledger.get("consumed_attempts", ())) + budget = budget_ledger.get("budget", {}) + if source_attempt_id not in consumed: + raise ValueError("budget ledger does not contain source recovery attempt") + max_attempts = budget.get("max_attempts") if isinstance(budget, Mapping) else None + if max_attempts is not None and len(consumed) + 1 > int(max_attempts): + raise ValueError("recovery attempt exceeds execution budget") + next_number = source_attempt_number + 1 + path = self.reserve_attempt(task_id, next_number) + next_id = derive_attempt_id(trial_id=trial_id, attempt_number=next_number, configuration_digest=configuration_digest) + self.write_text_atomic(path / "recovery-lineage.json", _json({ + "trial_id": trial_id, + "source_attempt_id": source_attempt_id, + "source_attempt_number": source_attempt_number, + "attempt_id": next_id, + "attempt_number": next_number, + "configuration_digest": configuration_digest, + "recovery_reference": recovery_reference, + })) + if budget_ledger is not None: + updated_ledger = dict(budget_ledger) + updated_ledger["consumed_attempts"] = [*tuple(str(value) for value in budget_ledger.get("consumed_attempts", ())), next_id] + self.write_text_atomic(path / "budget-ledger.json", _json(updated_ledger)) + return path + + def write_text_atomic(self, path: str | Path, content: str) -> Path: + target = Path(path).resolve() + if self.root not in target.parents: + raise ValueError("artifact path escapes artifact root") + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + ".tmp") + temporary.write_text(content, encoding="utf-8", newline="\n") + temporary.replace(target) + return target + + def write_manifest(self, attempt_root: str | Path, manifest: "RunManifest") -> Path: + """Validate and atomically persist the current manifest snapshot.""" + root = Path(attempt_root).resolve() + if self.root not in root.parents or not root.is_dir(): + raise ValueError("attempt root is outside artifact store") + manifest.validate_attempt_identity() + manifest.validate_lifecycle() + recovery_path = root / "recovery-lineage.json" + if recovery_path.is_file(): + lineage = RecoveryLineage.from_dict(json.loads(recovery_path.read_text(encoding="utf-8"))) + if lineage.trial_id != manifest.trial_id or lineage.attempt_id != manifest.attempt_id or lineage.attempt_number != manifest.attempt_number or lineage.configuration_digest != manifest.configuration_digest: + raise ValueError("recovery lineage does not match manifest being written") + budget_path = root / "budget-ledger.json" + if budget_path.is_file(): + persisted_ledger = json.loads(budget_path.read_text(encoding="utf-8")) + if manifest.budget_ledger != persisted_ledger: + raise ValueError("manifest budget ledger does not match persisted attempt ledger") + return self.write_text_atomic(root / "manifest.json", manifest.to_json()) + + def validate_manifest_references(self, manifest: "RunManifest") -> tuple[Path, ...]: + if manifest.state is not RunState.PLANNED and not manifest.artifact_refs: + raise ValueError("non-planned manifests require artifact references") + resolved: list[Path] = [] + for reference in manifest.artifact_refs: + path = Path(reference).resolve() + if self.root not in path.parents: + raise ValueError("manifest artifact reference escapes artifact root") + if not path.is_file(): + raise FileNotFoundError(f"manifest artifact is missing: {path}") + resolved.append(path) + return tuple(resolved) + + +@dataclass(frozen=True) +class ErrorEnvelope: + domain: ErrorDomain + code: str + message: str + retryability: Retryability = Retryability.UNKNOWN + raw_evidence_refs: tuple[str, ...] = () + attempt_number: int | None = None + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + _require(self.code, "error code") + _require(self.message, "error message") + if self.attempt_number is not None and self.attempt_number < 1: + raise ValueError("attempt_number must be positive when present") + refs = tuple(sorted(set(self.raw_evidence_refs))) + if any(not ref.strip() for ref in refs): + raise ValueError("raw evidence references must be non-empty") + object.__setattr__(self, "raw_evidence_refs", refs) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "domain": self.domain.value, + "code": self.code, + "message": self.message, + "retryability": self.retryability.value, + "raw_evidence_refs": list(self.raw_evidence_refs), + "attempt_number": self.attempt_number, + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "ErrorEnvelope": + return cls( + domain=ErrorDomain(value["domain"]), + code=str(value["code"]), + message=str(value["message"]), + retryability=Retryability(value.get("retryability", Retryability.UNKNOWN.value)), + raw_evidence_refs=tuple(value.get("raw_evidence_refs", ())), + attempt_number=value.get("attempt_number"), + schema_version=int(value.get("schema_version", 0)), + ) + + +@dataclass(frozen=True) +class LifecycleEvent: + state: RunState + timestamp: str + evidence_refs: tuple[str, ...] = () + + def __post_init__(self) -> None: + datetime.fromisoformat(self.timestamp.replace("Z", "+00:00")) + refs = tuple(sorted(set(self.evidence_refs))) + if any(not ref.strip() for ref in refs): + raise ValueError("lifecycle evidence references must be non-empty") + object.__setattr__(self, "evidence_refs", refs) + + def to_dict(self) -> dict[str, Any]: + return {"state": self.state.value, "timestamp": self.timestamp, "evidence_refs": list(self.evidence_refs)} + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "LifecycleEvent": + return cls(RunState(value["state"]), str(value["timestamp"]), tuple(value.get("evidence_refs", ()))) + + +@dataclass(frozen=True) +class RunManifest: + """One immutable execution attempt, distinct from its experiment/trial.""" + + experiment_id: str + trial_id: str + attempt_id: str + attempt_number: int + task_id: str + task_revision: str | None + treatment: str + executor: str | None + provider: str | None + model: str | None + sandbox: str | None + repository_revision: str | None + configuration_digest: str | None + protocol_version: str | None + budget_digest: str | None = None + state: RunState = RunState.PLANNED + state_history: tuple[RunState, ...] = () + lifecycle_events: tuple[LifecycleEvent, ...] = () + artifacts: tuple[ArtifactRecord, ...] = () + artifact_refs: tuple[str, ...] = () + errors: tuple[ErrorEnvelope, ...] = () + schema_version: int = SCHEMA_VERSION + budget_ledger: Mapping[str, Any] | None = None + + def __post_init__(self) -> None: + for name in ( + "experiment_id", "trial_id", "attempt_id", "task_id", "treatment", + ): + _require(getattr(self, name), name) + if self.attempt_number < 1: + raise ValueError("attempt_number must be positive") + refs = tuple(sorted(set(self.artifact_refs))) + if any(not ref.strip() for ref in refs): + raise ValueError("artifact references must be non-empty") + object.__setattr__(self, "artifact_refs", refs) + object.__setattr__(self, "errors", tuple(self.errors)) + object.__setattr__(self, "state_history", tuple(self.state_history)) + object.__setattr__(self, "lifecycle_events", tuple(self.lifecycle_events)) + object.__setattr__(self, "artifacts", tuple(self.artifacts)) + if self.budget_ledger is not None: + object.__setattr__(self, "budget_ledger", dict(self.budget_ledger)) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "experiment_id": self.experiment_id, + "trial_id": self.trial_id, + "attempt_id": self.attempt_id, + "attempt_number": self.attempt_number, + "task_id": self.task_id, + "task_revision": self.task_revision, + "treatment": self.treatment, + "executor": self.executor, + "provider": self.provider, + "model": self.model, + "sandbox": self.sandbox, + "repository_revision": self.repository_revision, + "configuration_digest": self.configuration_digest, + "protocol_version": self.protocol_version, + "budget_digest": self.budget_digest, + "state": self.state.value, + "state_history": [state.value for state in self.state_history], + "lifecycle_events": [event.to_dict() for event in self.lifecycle_events], + "artifacts": [artifact.to_dict() for artifact in self.artifacts], + "artifact_refs": list(self.artifact_refs), + "errors": [error.to_dict() for error in self.errors], + "budget_ledger": self.budget_ledger, + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "RunManifest": + schema_version = int(value.get("schema_version", 0)) + if schema_version != SCHEMA_VERSION: + raise ValueError(f"Unsupported harness manifest schema: {schema_version}") + errors = tuple(ErrorEnvelope.from_dict(item) for item in value.get("errors", ())) + return cls( + experiment_id=str(value["experiment_id"]), + trial_id=str(value["trial_id"]), + attempt_id=str(value["attempt_id"]), + attempt_number=int(value["attempt_number"]), + task_id=str(value["task_id"]), + task_revision=value.get("task_revision"), + treatment=str(value["treatment"]), + executor=value.get("executor"), + provider=value.get("provider"), + model=value.get("model"), + sandbox=value.get("sandbox"), + repository_revision=value.get("repository_revision"), + configuration_digest=value.get("configuration_digest"), + protocol_version=value.get("protocol_version"), + budget_digest=value.get("budget_digest"), + state=RunState(value["state"]), + state_history=tuple(RunState(item) for item in value.get("state_history", ())), + lifecycle_events=tuple(LifecycleEvent.from_dict(item) for item in value.get("lifecycle_events", ())), + artifacts=tuple(ArtifactRecord.from_dict(item) for item in value.get("artifacts", ())), + artifact_refs=tuple(value.get("artifact_refs", ())), + errors=errors, + schema_version=schema_version, + budget_ledger=value.get("budget_ledger"), + ) + + @classmethod + def from_json(cls, value: str) -> "RunManifest": + return cls.from_dict(json.loads(value)) + + def validate_attempt_identity(self) -> None: + if self.configuration_digest is None: + raise ValueError("configuration_digest is required to validate attempt identity") + expected = derive_attempt_id( + trial_id=self.trial_id, + attempt_number=self.attempt_number, + configuration_digest=self.configuration_digest, + ) + if self.attempt_id != expected: + raise ValueError("attempt_id does not match trial, attempt number, and configuration") + + def with_budget_ledger(self, ledger: Mapping[str, Any]) -> "RunManifest": + """Return a manifest snapshot bound to this attempt's budget ledger.""" + consumed = tuple(ledger.get("consumed_attempts", ())) + if self.attempt_id not in consumed: + raise ValueError("budget ledger does not contain manifest attempt_id") + ledger_digest = ledger.get("budget_digest") + if self.budget_digest is not None and ledger_digest != self.budget_digest: + raise ValueError("budget ledger digest does not match manifest budget_digest") + return replace(self, budget_ledger=dict(ledger)) + + def validate_lifecycle(self) -> None: + terminal = {RunState.COMPLETED, RunState.FAILED, RunState.BLOCKED, RunState.UNKNOWN} + if self.state is RunState.COMPLETED and self.errors: + raise ValueError("completed runs cannot contain terminal errors") + if self.state in {RunState.FAILED, RunState.BLOCKED} and not self.errors: + raise ValueError(f"{self.state.value.lower()} runs require structured errors") + if self.state is RunState.PLANNED and self.artifact_refs: + raise ValueError("planned runs cannot reference execution artifacts") + if self.state not in terminal and self.errors: + raise ValueError(f"{self.state.value.lower()} runs cannot contain terminal errors") + if self.state_history: + if self.state_history[0] is not RunState.PLANNED: + raise ValueError("state history must start at PLANNED") + if self.state_history[-1] is not self.state: + raise ValueError("state history must end at the manifest state") + for previous, current in zip(self.state_history, self.state_history[1:]): + validate_transition(previous, current) + if self.lifecycle_events: + states = tuple(event.state for event in self.lifecycle_events) + if states != self.state_history: + raise ValueError("lifecycle event states must match state history") + timestamps = [datetime.fromisoformat(event.timestamp.replace("Z", "+00:00")) for event in self.lifecycle_events] + if timestamps != sorted(timestamps): + raise ValueError("lifecycle event timestamps must be monotonic") + for artifact in self.artifacts: + if artifact.required and artifact.status is not ArtifactStatus.PRESENT: + raise ValueError(f"required artifact is not present: {artifact.kind}") + + +def derive_attempt_id(*, trial_id: str, attempt_number: int, configuration_digest: str) -> str: + """Derive a stable, collision-resistant id without timestamps or fallbacks.""" + + _require(trial_id, "trial_id") + _require(configuration_digest, "configuration_digest") + if attempt_number < 1: + raise ValueError("attempt_number must be positive") + payload = f"{trial_id}\n{attempt_number}\n{configuration_digest}".encode("utf-8") + return hashlib.sha256(payload).hexdigest()[:24] diff --git a/src/arkx/integration.py b/src/arkx/integration.py new file mode 100644 index 00000000..3e0756ac --- /dev/null +++ b/src/arkx/integration.py @@ -0,0 +1,198 @@ +"""Executor-neutral adapter qualification and preflight contracts.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +import hashlib +import json +from typing import Any + +from .harness import ErrorDomain, ErrorEnvelope, Retryability + + +class IntegrationKind(str, Enum): + EXECUTOR = "EXECUTOR" + PROVIDER = "PROVIDER" + SANDBOX = "SANDBOX" + EVALUATOR = "EVALUATOR" + + +class PreflightStatus(str, Enum): + READY = "READY" + BLOCKED = "BLOCKED" + UNKNOWN = "UNKNOWN" + FAILED = "FAILED" + + +class CapabilityProvenance(str, Enum): + DECLARED = "DECLARED" + OBSERVED = "OBSERVED" + QUALIFIED = "QUALIFIED" + + +@dataclass(frozen=True) +class AdapterIdentity: + kind: IntegrationKind + name: str + version: str | None = None + configuration_digest: str | None = None + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("adapter name must be non-empty") + + def to_dict(self) -> dict[str, str | None]: + return {"kind": self.kind.value, "name": self.name, "version": self.version, "configuration_digest": self.configuration_digest} + + +@dataclass(frozen=True) +class DependencyObservation: + name: str + available: bool | None + version: str | None = None + reason: str | None = None + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("dependency name must be non-empty") + + def to_dict(self) -> dict[str, Any]: + return {"name": self.name, "available": self.available, "version": self.version, "reason": self.reason} + + +@dataclass(frozen=True) +class AdapterPreflight: + identity: AdapterIdentity + status: PreflightStatus + dependencies: tuple[DependencyObservation, ...] = () + capabilities: tuple[str, ...] = () + evidence_refs: tuple[str, ...] = () + reason: str | None = None + capability_digest: str | None = None + capability_provenance: CapabilityProvenance = CapabilityProvenance.DECLARED + + def __post_init__(self) -> None: + if any(not value.strip() for value in self.capabilities + self.evidence_refs): + raise ValueError("preflight capabilities and evidence refs must be non-empty") + if self.capability_provenance is CapabilityProvenance.QUALIFIED and not self.evidence_refs: + raise ValueError("qualified capabilities require evidence references") + object.__setattr__(self, "dependencies", tuple(sorted(self.dependencies, key=lambda item: item.name))) + object.__setattr__(self, "capabilities", tuple(sorted(set(self.capabilities)))) + object.__setattr__(self, "evidence_refs", tuple(sorted(set(self.evidence_refs)))) + + def to_dict(self) -> dict[str, Any]: + return { + "identity": self.identity.to_dict(), + "status": self.status.value, + "dependencies": [item.to_dict() for item in self.dependencies], + "capabilities": list(self.capabilities), + "evidence_refs": list(self.evidence_refs), + "reason": self.reason, + "capability_digest": self.capability_digest, + "capability_provenance": self.capability_provenance.value, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + @property + def reference(self) -> str: + digest = hashlib.sha256(self.to_json().encode("utf-8")).hexdigest()[:16] + return f"preflight://{digest}" + + +def preflight_error(preflight: AdapterPreflight) -> ErrorEnvelope | None: + """Map non-ready preflight state without guessing retryability.""" + if preflight.status is PreflightStatus.READY: + return None + domains = { + IntegrationKind.EXECUTOR: ErrorDomain.EXECUTOR, + IntegrationKind.PROVIDER: ErrorDomain.PROVIDER, + IntegrationKind.SANDBOX: ErrorDomain.SANDBOX, + IntegrationKind.EVALUATOR: ErrorDomain.VERIFICATION, + } + code = f"PREFLIGHT_{preflight.status.value}" + message = preflight.reason or f"adapter {preflight.identity.name} is not ready" + return ErrorEnvelope(domains[preflight.identity.kind], code, message, Retryability.UNKNOWN, (preflight.reference,)) + + +def provider_error_envelope(payload: Any, *, raw_evidence_refs: tuple[str, ...] = ()) -> ErrorEnvelope: + """Normalize a provider error payload without importing a provider SDK.""" + if not isinstance(payload, dict) or not isinstance(payload.get("error"), dict): + raise ValueError("provider payload must contain an error object") + error = payload["error"] + code = str(error.get("code", "UNKNOWN")) + message = str(error.get("message", "provider returned an error")) + return ErrorEnvelope( + ErrorDomain.PROVIDER, + f"PROVIDER_{code}", + message, + Retryability.UNKNOWN, + raw_evidence_refs, + ) + + +def process_error_envelope( + kind: IntegrationKind, + *, + returncode: int | None, + message: str, + raw_evidence_refs: tuple[str, ...] = (), + attempt_number: int | None = None, +) -> ErrorEnvelope: + """Normalize a process boundary failure while retaining its integration kind.""" + if kind not in (IntegrationKind.EXECUTOR, IntegrationKind.SANDBOX, IntegrationKind.EVALUATOR): + raise ValueError("process failures require executor, sandbox, or evaluator kind") + domains = { + IntegrationKind.EXECUTOR: ErrorDomain.EXECUTOR, + IntegrationKind.SANDBOX: ErrorDomain.SANDBOX, + IntegrationKind.EVALUATOR: ErrorDomain.VERIFICATION, + } + suffix = "UNKNOWN" if returncode is None else str(returncode) + return ErrorEnvelope(domains[kind], f"{kind.value}_PROCESS_{suffix}", message, Retryability.UNKNOWN, raw_evidence_refs, attempt_number) + + +def exception_error_envelope( + kind: IntegrationKind, + error: BaseException, + *, + raw_evidence_refs: tuple[str, ...] = (), +) -> ErrorEnvelope: + """Normalize an adapter exception without inferring retryability.""" + domains = { + IntegrationKind.EXECUTOR: ErrorDomain.EXECUTOR, + IntegrationKind.PROVIDER: ErrorDomain.PROVIDER, + IntegrationKind.SANDBOX: ErrorDomain.SANDBOX, + IntegrationKind.EVALUATOR: ErrorDomain.VERIFICATION, + } + return ErrorEnvelope( + domains[kind], + f"{kind.value}_{type(error).__name__.upper()}", + str(error) or type(error).__name__, + Retryability.UNKNOWN, + raw_evidence_refs, + ) + + +def assess_preflight( + identity: AdapterIdentity, + dependencies: tuple[DependencyObservation, ...], + *, + capabilities: tuple[str, ...] = (), + evidence_refs: tuple[str, ...] = (), + failure_reason: str | None = None, + capability_digest: str | None = None, + capability_provenance: CapabilityProvenance = CapabilityProvenance.DECLARED, +) -> AdapterPreflight: + if failure_reason: + status = PreflightStatus.FAILED + elif any(item.available is False for item in dependencies): + status = PreflightStatus.BLOCKED + elif any(item.available is None for item in dependencies): + status = PreflightStatus.UNKNOWN + elif identity.version is None or identity.configuration_digest is None: + status = PreflightStatus.UNKNOWN + else: + status = PreflightStatus.READY + return AdapterPreflight(identity, status, dependencies, capabilities, evidence_refs, failure_reason, capability_digest, capability_provenance) diff --git a/src/arkx/orchestration.py b/src/arkx/orchestration.py new file mode 100644 index 00000000..b78b168b --- /dev/null +++ b/src/arkx/orchestration.py @@ -0,0 +1,468 @@ +"""Sequential routed-composition orchestration contracts for P8.4. + +The orchestrator wires already-qualified boundaries together. It does not +choose an executor, reinterpret verification, or cascade to another executor. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from datetime import datetime, timezone +from enum import Enum +import json +import hashlib +from typing import Any, Protocol + +from .characterization import TaskCharacterization +from .outcomes import AcceptanceDecision, AcceptanceResult, VerificationResult, VerificationState +from .progress import ProgressAssessment, ProgressStatus +from .request import GovernanceDecision, GovernanceStatus +from .routing import RoutingDecision, RoutingDecisionType +from .selection import SelectionDecision, SelectionStatus +from .execution import BudgetLedger, ExecutionBudgetSpec, ExecutionPlan, ExecutionRequest, ExecutionResult as ContractExecutionResult, Executor, execution_reference +from .integration import AdapterIdentity, AdapterPreflight, IntegrationKind, PreflightStatus +from .configuration import ConfigurationSnapshot +from .contracts import EventType +from .harness import RunState + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class ExecutionOutcome(_ValueEnum): + COMPLETED = "COMPLETED" + FAILED = "FAILED" + BLOCKED = "BLOCKED" + NOT_EXECUTED = "NOT_EXECUTED" + + +class OrchestrationStatus(_ValueEnum): + BLOCKED = "BLOCKED" + NOT_EXECUTED = "NOT_EXECUTED" + EXECUTED = "EXECUTED" + VERIFIED = "VERIFIED" + ACCEPTED = "ACCEPTED" + REJECTED = "REJECTED" + INDETERMINATE = "INDETERMINATE" + + +class OrchestrationReason(_ValueEnum): + GOVERNANCE_REQUIRED = "GOVERNANCE_REQUIRED" + CHARACTERIZATION_REQUIRED = "CHARACTERIZATION_REQUIRED" + ROUTING_REQUIRED = "ROUTING_REQUIRED" + SELECTION_REQUIRED = "SELECTION_REQUIRED" + ACCEPTANCE_AUTHORITY_MISMATCH = "ACCEPTANCE_AUTHORITY_MISMATCH" + EXECUTOR_FAILED = "EXECUTOR_FAILED" + EXECUTOR_BLOCKED = "EXECUTOR_BLOCKED" + EXECUTOR_NOT_EXECUTED = "EXECUTOR_NOT_EXECUTED" + NO_PROGRESS = "NO_PROGRESS" + PROGRESS_UNKNOWN = "PROGRESS_UNKNOWN" + VERIFICATION_BLOCKED = "VERIFICATION_BLOCKED" + VERIFICATION_UNKNOWN = "VERIFICATION_UNKNOWN" + ACCEPTANCE_DECISION = "ACCEPTANCE_DECISION" + CAPABILITY_PREFLIGHT_REQUIRED = "CAPABILITY_PREFLIGHT_REQUIRED" + CAPABILITY_POLICY_BLOCKED = "CAPABILITY_POLICY_BLOCKED" + CAPABILITY_PREFLIGHT_IDENTITY_MISMATCH = "CAPABILITY_PREFLIGHT_IDENTITY_MISMATCH" + CAPABILITY_PREFLIGHT_PERSISTENCE_REQUIRED = "CAPABILITY_PREFLIGHT_PERSISTENCE_REQUIRED" + LIFECYCLE_PERSISTENCE_FAILED = "LIFECYCLE_PERSISTENCE_FAILED" + EXECUTION_BUDGET_EXHAUSTED = "EXECUTION_BUDGET_EXHAUSTED" + BUDGET_USAGE_UNAVAILABLE = "BUDGET_USAGE_UNAVAILABLE" + PROVIDER_PREFLIGHT_REQUIRED = "PROVIDER_PREFLIGHT_REQUIRED" + SANDBOX_PREFLIGHT_REQUIRED = "SANDBOX_PREFLIGHT_REQUIRED" + PROVIDER_PREFLIGHT_IDENTITY_MISMATCH = "PROVIDER_PREFLIGHT_IDENTITY_MISMATCH" + SANDBOX_PREFLIGHT_IDENTITY_MISMATCH = "SANDBOX_PREFLIGHT_IDENTITY_MISMATCH" + INTEGRATION_PREFLIGHT_BLOCKED = "INTEGRATION_PREFLIGHT_BLOCKED" + INTEGRATION_PREFLIGHT_PERSISTENCE_REQUIRED = "INTEGRATION_PREFLIGHT_PERSISTENCE_REQUIRED" + BUDGET_ENFORCEMENT_REQUIRED = "BUDGET_ENFORCEMENT_REQUIRED" + + +def orchestration_status_for_execution(outcome: ExecutionOutcome) -> OrchestrationStatus: + """Map execution outcome into the orchestration result vocabulary.""" + return { + ExecutionOutcome.COMPLETED: OrchestrationStatus.EXECUTED, + ExecutionOutcome.FAILED: OrchestrationStatus.BLOCKED, + ExecutionOutcome.BLOCKED: OrchestrationStatus.BLOCKED, + ExecutionOutcome.NOT_EXECUTED: OrchestrationStatus.NOT_EXECUTED, + }[outcome] + + +def orchestration_status_for_acceptance(decision: AcceptanceDecision) -> OrchestrationStatus: + """Map acceptance decision into the orchestration result vocabulary.""" + return { + AcceptanceDecision.ACCEPTED: OrchestrationStatus.ACCEPTED, + AcceptanceDecision.REJECTED: OrchestrationStatus.REJECTED, + AcceptanceDecision.BLOCKED: OrchestrationStatus.BLOCKED, + AcceptanceDecision.NOT_EXECUTED: OrchestrationStatus.NOT_EXECUTED, + AcceptanceDecision.INDETERMINATE: OrchestrationStatus.INDETERMINATE, + }[decision] + + +@dataclass(frozen=True) +class ExecutionResult: + run_id: str + outcome: ExecutionOutcome + evidence_refs: tuple[str, ...] = () + error: str | None = None + telemetry: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "run_id": self.run_id, + "outcome": self.outcome.value, + "evidence_refs": sorted(set(self.evidence_refs)), + "error": self.error, + "telemetry": self.telemetry, + } + + +def legacy_execution_from_contract(result: "ContractExecutionResult") -> ExecutionResult: + outcome = { + RunState.COMPLETED: ExecutionOutcome.COMPLETED, + RunState.FAILED: ExecutionOutcome.FAILED, + RunState.BLOCKED: ExecutionOutcome.BLOCKED, + RunState.UNKNOWN: ExecutionOutcome.NOT_EXECUTED, + }.get(result.state, ExecutionOutcome.NOT_EXECUTED) + return ExecutionResult( + result.artifact_refs[0] if result.artifact_refs else "contract-run", + outcome, + result.artifact_refs, + None if result.error is None else result.error.message, + dict(result.usage or {}), + ) + + +class ExecutorRunner(Protocol): + def run(self, request: GovernanceDecision, selection: SelectionDecision) -> ExecutionResult: ... + + +class BudgetEnforcedExecutorRunner(Protocol): + """Legacy seam capability required before a budgeted execution.""" + + def run_with_budget( + self, + request: GovernanceDecision, + selection: SelectionDecision, + *, + attempt_id: str, + budget_ledger: BudgetLedger, + ) -> ExecutionResult: ... + + +class ContractExecutorRunner: + """Bridge the neutral Executor contract into the legacy orchestration seam.""" + + def __init__(self, executor: Executor) -> None: + self.executor = executor + + def run(self, request: GovernanceDecision, selection: SelectionDecision) -> ExecutionResult: + arkx_request = request.request + if arkx_request is None or selection.executor is None: + return ExecutionResult("unbound", ExecutionOutcome.NOT_EXECUTED, error="missing request or selected executor") + environment = arkx_request.environment + budget = arkx_request.budget + execution_request = ExecutionRequest( + task_id=arkx_request.task_id or "", + task_revision=None if environment is None else environment.revision, + treatment=selection.treatment.name if selection.treatment is not None else "", + prompt=f"task:{arkx_request.task_id}", + executor_id=selection.executor.executor.name, + provider_id=None, + model_id=None, + sandbox_id=None if environment is None else environment.environment_id, + budget_digest=None if budget is None else json.dumps(budget.to_dict(), sort_keys=True, separators=(",", ":")), + configuration_digest=selection.executor.executor.configuration_digest, + ) + return legacy_execution_from_contract(self.executor.execute(execution_request)) + + def run_with_budget( + self, + request: GovernanceDecision, + selection: SelectionDecision, + *, + attempt_id: str, + budget_ledger: BudgetLedger, + ) -> ExecutionResult: + """Run only after orchestration has reserved this attempt in the ledger.""" + if attempt_id not in budget_ledger.consumed_attempts: + return ExecutionResult("unbound", ExecutionOutcome.BLOCKED, error="budget attempt was not reserved") + result = self.run(request, selection) + telemetry = dict(result.telemetry) + telemetry["budget_enforcement"] = "RESERVED_AND_MEASURED" + return replace(result, telemetry=telemetry) + + +class ProgressObserver(Protocol): + def assess(self, execution: ExecutionResult) -> ProgressAssessment: ... + + +class Verifier(Protocol): + def verify(self, execution: ExecutionResult) -> VerificationResult: ... + + +class AcceptanceAuthority(Protocol): + identity: str + + def decide(self, verification: VerificationResult) -> AcceptanceResult: ... + + +@dataclass(frozen=True) +class OrchestrationResult: + status: OrchestrationStatus + reason_codes: tuple[OrchestrationReason, ...] + run_id: str | None + execution: ExecutionResult | None = None + plan: ExecutionPlan | None = None + progress: ProgressAssessment | None = None + verification: VerificationResult | None = None + acceptance: AcceptanceResult | None = None + telemetry: dict[str, Any] = field(default_factory=dict) + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "status": self.status.value, + "reason_codes": [reason.value for reason in self.reason_codes], + "run_id": self.run_id, + "execution": None if self.execution is None else self.execution.to_dict(), + "plan": None if self.plan is None else self.plan.to_dict(), + "progress": None if self.progress is None else self.progress.to_dict(), + "verification": None if self.verification is None else self.verification.to_dict(), + "acceptance": None if self.acceptance is None else self.acceptance.to_dict(), + "telemetry": self.telemetry, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _blocked(reason: OrchestrationReason, *, run_id: str | None = None) -> OrchestrationResult: + return OrchestrationResult(OrchestrationStatus.BLOCKED, (reason,), run_id, telemetry={"orchestration_status": OrchestrationStatus.BLOCKED.value}) + + +def build_execution_plan( + governance: GovernanceDecision, + routing: RoutingDecision, + selection: SelectionDecision, + *, + provider_identity: AdapterIdentity | None = None, + sandbox_identity: AdapterIdentity | None = None, + configuration_snapshot: ConfigurationSnapshot | None = None, +) -> ExecutionPlan: + request = governance.request + if request is None or selection.executor is None or selection.treatment is None: + raise ValueError("governance request, treatment and executor are required for an execution plan") + identity = selection.executor.executor + configuration_digest = identity.configuration_digest + if configuration_snapshot is not None: + if configuration_digest is not None and configuration_digest != configuration_snapshot.digest(): + raise ValueError("configuration snapshot does not match executor identity") + configuration_digest = configuration_snapshot.digest() + budget = None if request.budget is None else ExecutionBudgetSpec( + max_attempts=request.budget.max_attempts, + max_tokens=request.budget.max_tokens, + max_wall_time_ms=request.budget.max_wall_time_ms, + ) + payload = json.dumps({ + "request_id": request.request_id, + "task_id": request.task_id, + "path": routing.selected_path.value if routing.selected_path else None, + "treatment": selection.treatment.name, + "executor": identity.to_dict(), + "provider": None if provider_identity is None else provider_identity.to_dict(), + "sandbox": None if sandbox_identity is None else sandbox_identity.to_dict(), + "budget": None if request.budget is None else request.budget.to_dict(), + "configuration_snapshot": None if configuration_snapshot is None else configuration_snapshot.to_dict(), + }, sort_keys=True, separators=(",", ":")) + return ExecutionPlan( + plan_id=hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16], + task_id=request.task_id or "", + task_revision=None if request.environment is None else request.environment.revision, + treatment=selection.treatment.name, + executor_id=identity.name, + provider_id=None if provider_identity is None else provider_identity.name, + model_id=None, + sandbox_id=sandbox_identity.name if sandbox_identity is not None else (None if request.environment is None else request.environment.environment_id), + budget_digest=None if budget is None else budget.digest(), + configuration_digest=configuration_digest, + steps=("execute",), + budget=budget, + required_capabilities=selection.treatment.required_capabilities, + provider_identity=provider_identity, + sandbox_identity=sandbox_identity, + configuration_snapshot=configuration_snapshot, + ) + + +def run_routed_pipeline( + governance: GovernanceDecision, + characterization: TaskCharacterization | None, + routing: RoutingDecision | None, + selection: SelectionDecision | None, + executor: ExecutorRunner | Executor, + progress_observer: ProgressObserver, + verifier: Verifier, + acceptance_authority: AcceptanceAuthority, + capability_preflight: AdapterPreflight | None = None, + event_log: Any | None = None, + provider_identity: AdapterIdentity | None = None, + sandbox_identity: AdapterIdentity | None = None, + provider_preflight: AdapterPreflight | None = None, + sandbox_preflight: AdapterPreflight | None = None, + configuration_snapshot: ConfigurationSnapshot | None = None, +) -> OrchestrationResult: + """Run one selected treatment sequentially through independent gates.""" + + def terminal(result: OrchestrationResult) -> OrchestrationResult: + if event_log is None or result.run_id is None: + return result + try: + event_log.append_terminal( + timestamp=datetime.now(timezone.utc).isoformat(timespec="milliseconds"), + status=result.status.value, + error_code=None if not result.reason_codes else result.reason_codes[0].value, + ) + return result + except Exception: + return replace(result, status=OrchestrationStatus.BLOCKED, reason_codes=(OrchestrationReason.LIFECYCLE_PERSISTENCE_FAILED,)) + + if governance.status is not GovernanceStatus.AUTHORIZED or governance.request is None: + return _blocked(OrchestrationReason.GOVERNANCE_REQUIRED) + if characterization is None: + return _blocked(OrchestrationReason.CHARACTERIZATION_REQUIRED) + if routing is None or routing.decision is not RoutingDecisionType.ROUTE or routing.selected_path is None: + return _blocked(OrchestrationReason.ROUTING_REQUIRED) + if selection is None or selection.status is not SelectionStatus.SELECTED or selection.executor is None: + return _blocked(OrchestrationReason.SELECTION_REQUIRED) + if acceptance_authority.identity != governance.request.acceptance_authority: + return _blocked(OrchestrationReason.ACCEPTANCE_AUTHORITY_MISMATCH) + + if hasattr(executor, "execute") and not hasattr(executor, "run"): + executor = ContractExecutorRunner(executor) + plan = build_execution_plan(governance, routing, selection, provider_identity=provider_identity, sandbox_identity=sandbox_identity, configuration_snapshot=configuration_snapshot) + if plan.budget is not None and not hasattr(executor, "run_with_budget"): + return _blocked(OrchestrationReason.BUDGET_ENFORCEMENT_REQUIRED) + def persist_stage(stage: str, ref: str, data: dict[str, Any] | None = None) -> bool: + if event_log is None: + return True + try: + event_log.append_stage( + timestamp=datetime.now(timezone.utc).isoformat(timespec="milliseconds"), + event_type=EventType.EVIDENCE_ADDED, + stage=stage, + ref=ref, + data=data, + ) + return True + except Exception: + return False + + if not persist_stage("plan", plan.reference): + return _blocked(OrchestrationReason.LIFECYCLE_PERSISTENCE_FAILED) + if plan.required_capabilities: + if capability_preflight is None: + return _blocked(OrchestrationReason.CAPABILITY_PREFLIGHT_REQUIRED) + if not plan.preflight_matches_identity(capability_preflight): + return _blocked(OrchestrationReason.CAPABILITY_PREFLIGHT_IDENTITY_MISMATCH) + capability_decision = plan.validate_capabilities(capability_preflight) + if not capability_decision.allowed: + return _blocked(OrchestrationReason.CAPABILITY_POLICY_BLOCKED) + if event_log is None: + return _blocked(OrchestrationReason.CAPABILITY_PREFLIGHT_PERSISTENCE_REQUIRED) + if not persist_stage("preflight", capability_preflight.reference, { + "plan_ref": plan.reference, + "capability_digest": capability_preflight.capability_digest, + "capability_provenance": capability_preflight.capability_provenance.value, + "preflight_status": capability_preflight.status.value, + }): + return _blocked(OrchestrationReason.CAPABILITY_PREFLIGHT_PERSISTENCE_REQUIRED) + for kind, identity, preflight, required_reason, mismatch_reason in ( + (IntegrationKind.PROVIDER, plan.provider_identity, provider_preflight, OrchestrationReason.PROVIDER_PREFLIGHT_REQUIRED, OrchestrationReason.PROVIDER_PREFLIGHT_IDENTITY_MISMATCH), + (IntegrationKind.SANDBOX, plan.sandbox_identity, sandbox_preflight, OrchestrationReason.SANDBOX_PREFLIGHT_REQUIRED, OrchestrationReason.SANDBOX_PREFLIGHT_IDENTITY_MISMATCH), + ): + if identity is None: + continue + if preflight is None: + return _blocked(required_reason) + if not plan.preflight_matches_integration(preflight, kind): + return _blocked(mismatch_reason) + if preflight.status is not PreflightStatus.READY: + return _blocked(OrchestrationReason.INTEGRATION_PREFLIGHT_BLOCKED) + if event_log is None: + return _blocked(OrchestrationReason.INTEGRATION_PREFLIGHT_PERSISTENCE_REQUIRED) + stage = "provider_preflight" if kind is IntegrationKind.PROVIDER else "sandbox_preflight" + if not persist_stage(stage, preflight.reference, {"integration_kind": kind.value, "preflight_status": preflight.status.value}): + return _blocked(OrchestrationReason.INTEGRATION_PREFLIGHT_PERSISTENCE_REQUIRED) + budget_ledger = None if plan.budget is None else BudgetLedger(plan.budget) + if budget_ledger is not None: + consumption = budget_ledger.consume(plan.reference, executor_invocations=1) + if not consumption.accepted: + return _blocked(OrchestrationReason.EXECUTION_BUDGET_EXHAUSTED) + budget_ledger = consumption.ledger + if plan.budget is not None: + execution = executor.run_with_budget(governance, selection, attempt_id=plan.reference, budget_ledger=budget_ledger) + else: + execution = executor.run(governance, selection) + execution_ref = execution_reference(execution.run_id) + execution_data = {"artifact_refs": list(execution.evidence_refs), "execution_outcome": execution.outcome.value} + if budget_ledger is not None: + usage = execution.telemetry + required_usage = {} + if plan.budget.max_tokens is not None: + required_usage["tokens"] = usage.get("total_tokens", usage.get("tokens")) + if plan.budget.max_wall_time_ms is not None: + required_usage["wall_time_ms"] = usage.get("wall_time_ms") + if plan.budget.max_cost is not None: + required_usage["cost"] = usage.get("cost") + missing_usage = tuple(name for name, value in required_usage.items() if value is None) + if missing_usage: + execution_data["budget_usage_status"] = "UNAVAILABLE" + execution_data["budget_usage_missing"] = list(missing_usage) + budget_error = OrchestrationReason.BUDGET_USAGE_UNAVAILABLE + else: + measured = budget_ledger.record_usage(plan.reference, tokens=int(required_usage.get("tokens", 0)), wall_time_ms=int(required_usage.get("wall_time_ms", 0)), cost=required_usage.get("cost", 0)) + budget_ledger = measured.ledger + budget_error = None if measured.accepted else OrchestrationReason.EXECUTION_BUDGET_EXHAUSTED + execution_data["budget_usage_status"] = "RECORDED" if measured.accepted else "EXCEEDED" + execution_data["budget_ledger"] = budget_ledger.to_dict() + else: + budget_error = None + if execution.error: + execution_data["error_code"] = getattr(execution.error, "code", type(execution.error).__name__.upper()) + if not persist_stage("execution", execution_ref, execution_data): + return terminal(OrchestrationResult(OrchestrationStatus.BLOCKED, (OrchestrationReason.LIFECYCLE_PERSISTENCE_FAILED,), execution.run_id, execution=execution, plan=plan)) + if budget_error is not None: + return terminal(OrchestrationResult(OrchestrationStatus.BLOCKED, (budget_error,), execution.run_id, execution=execution, plan=plan)) + if execution.outcome is ExecutionOutcome.NOT_EXECUTED: + return terminal(OrchestrationResult(OrchestrationStatus.NOT_EXECUTED, (OrchestrationReason.EXECUTOR_NOT_EXECUTED,), execution.run_id, execution=execution, plan=plan)) + if execution.outcome is ExecutionOutcome.BLOCKED: + return terminal(OrchestrationResult(OrchestrationStatus.BLOCKED, (OrchestrationReason.EXECUTOR_BLOCKED,), execution.run_id, execution=execution, plan=plan)) + if execution.outcome is ExecutionOutcome.FAILED: + return terminal(OrchestrationResult(OrchestrationStatus.BLOCKED, (OrchestrationReason.EXECUTOR_FAILED,), execution.run_id, execution=execution, plan=plan)) + + progress = progress_observer.assess(execution) + if progress.status is ProgressStatus.NO_PROGRESS: + return terminal(OrchestrationResult(OrchestrationStatus.BLOCKED, (OrchestrationReason.NO_PROGRESS,), execution.run_id, execution=execution, plan=plan, progress=progress)) + if progress.status is ProgressStatus.UNKNOWN: + return terminal(OrchestrationResult(OrchestrationStatus.INDETERMINATE, (OrchestrationReason.PROGRESS_UNKNOWN,), execution.run_id, execution=execution, plan=plan, progress=progress)) + + verification = verifier.verify(execution) + if not persist_stage("verification", verification.reference, {"verification_state": verification.state.value}): + return terminal(OrchestrationResult(OrchestrationStatus.BLOCKED, (OrchestrationReason.LIFECYCLE_PERSISTENCE_FAILED,), execution.run_id, execution=execution, plan=plan, progress=progress, verification=verification)) + if verification.state is VerificationState.BLOCKED: + return terminal(OrchestrationResult(OrchestrationStatus.BLOCKED, (OrchestrationReason.VERIFICATION_BLOCKED,), execution.run_id, execution=execution, plan=plan, progress=progress, verification=verification)) + if verification.state in (VerificationState.INDETERMINATE, VerificationState.NOT_EXECUTED): + return terminal(OrchestrationResult(OrchestrationStatus.INDETERMINATE, (OrchestrationReason.VERIFICATION_UNKNOWN,), execution.run_id, execution=execution, plan=plan, progress=progress, verification=verification)) + if verification.state is VerificationState.FAIL: + return terminal(OrchestrationResult(OrchestrationStatus.REJECTED, (OrchestrationReason.ACCEPTANCE_DECISION,), execution.run_id, execution=execution, plan=plan, progress=progress, verification=verification)) + + acceptance = acceptance_authority.decide(verification) + if not persist_stage("acceptance", acceptance.reference, {"acceptance_decision": acceptance.decision.value}): + return terminal(OrchestrationResult(OrchestrationStatus.BLOCKED, (OrchestrationReason.LIFECYCLE_PERSISTENCE_FAILED,), execution.run_id, execution=execution, plan=plan, progress=progress, verification=verification, acceptance=acceptance)) + status = orchestration_status_for_acceptance(acceptance.decision) + return terminal(OrchestrationResult(status, (OrchestrationReason.ACCEPTANCE_DECISION,), execution.run_id, execution=execution, plan=plan, progress=progress, verification=verification, acceptance=acceptance)) diff --git a/src/arkx/outcomes.py b/src/arkx/outcomes.py new file mode 100644 index 00000000..dc41ca7e --- /dev/null +++ b/src/arkx/outcomes.py @@ -0,0 +1,86 @@ +"""Executor-neutral verification and acceptance outcome contracts. + +These types belong to the product boundary. They intentionally contain no +executor, provider, model, sandbox, or experiment-specific behavior. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +import hashlib +import json +from typing import Any + + +class VerificationState(str, Enum): + PASS = "PASS" + FAIL = "FAIL" + INDETERMINATE = "INDETERMINATE" + NOT_EXECUTED = "NOT_EXECUTED" + BLOCKED = "BLOCKED" + + +class AcceptanceDecision(str, Enum): + ACCEPTED = "ACCEPTED" + REJECTED = "REJECTED" + INDETERMINATE = "INDETERMINATE" + NOT_EXECUTED = "NOT_EXECUTED" + BLOCKED = "BLOCKED" + + +@dataclass(frozen=True) +class VerificationResult: + state: VerificationState + authority: str + commands: tuple[str, ...] + outcomes: tuple[str, ...] + evidence: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "state": self.state.value, + "authority": self.authority, + "commands": list(self.commands), + "outcomes": list(self.outcomes), + "evidence": list(self.evidence), + } + + @property + def reference(self) -> str: + payload = json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return f"verification://{hashlib.sha256(payload.encode('utf-8')).hexdigest()[:16]}" + + +@dataclass(frozen=True) +class AcceptanceResult: + decision: AcceptanceDecision + authority: str + raw_outcome: str | None + evidence: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "decision": self.decision.value, + "authority": self.authority, + "raw_outcome": self.raw_outcome, + "evidence": list(self.evidence), + } + + @property + def reference(self) -> str: + payload = json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return f"acceptance://{hashlib.sha256(payload.encode('utf-8')).hexdigest()[:16]}" + + +def validate_outcome_references( + verification: VerificationResult, + acceptance: AcceptanceResult, + *, + verification_ref: str, + acceptance_ref: str, +) -> None: + if verification_ref != verification.reference: + raise ValueError("verification reference does not match result") + if acceptance_ref != acceptance.reference: + raise ValueError("acceptance reference does not match result") diff --git a/src/arkx/p82.py b/src/arkx/p82.py new file mode 100644 index 00000000..c6c27c16 --- /dev/null +++ b/src/arkx/p82.py @@ -0,0 +1,486 @@ +"""P8.2 minimal mechanism trial design. + +This module defines the controlled A/B/C/D design only. It never invokes an +executor, applies a mechanism, verifies a patch, or makes an acceptance +decision. Results must be recorded later as :class:`ExecutorTrial` records. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from decimal import Decimal +import json +from statistics import median +from typing import Any, Iterable + +from .composition import Mechanism, Treatment +from .executor_qualification import ( + ExecutionBudget, + ExecutionEnvironment, + ExecutorIdentity, + QualificationTask, +) + + +SCHEMA_VERSION = 1 + + +class P82Treatment(str, Enum): + A = "A" + B = "B" + C = "C" + D = "D" + + +class TaskStratum(str, Enum): + CHALLENGER_RECOVERY = "challenger_recovery" + LOCALIZATION_CONTEXT = "localization_context" + MINI_CONTROL = "mini_control" + HARD_ALL_FAIL = "hard_all_fail" + + +class P82RunStatus(str, Enum): + NOT_EXECUTED = "NOT_EXECUTED" + BLOCKED = "BLOCKED" + EXECUTED = "EXECUTED" + + +class P82Comparability(str, Enum): + COMPARABLE = "COMPARABLE" + NOT_COMPARABLE = "NOT_COMPARABLE" + INDETERMINATE = "INDETERMINATE" + + +@dataclass(frozen=True) +class P82Criteria: + primary_metrics: tuple[str, ...] = ("verified_acceptance", "cost_to_acceptance") + secondary_metrics: tuple[str, ...] = ( + "tokens_to_acceptance", "wall_time_seconds", "model_calls", "turns", + "repeated_exploration", "repeated_file_reads_searches", "edit_failures", + "retries", "replans", "unnecessary_diff", "regressions", "human_intervention", + ) + guardrail: str = "false_pass_must_not_increase" + + def to_dict(self) -> dict[str, Any]: + return {"primary_metrics": list(self.primary_metrics), "secondary_metrics": list(self.secondary_metrics), "guardrail": self.guardrail} + + def to_json(self) -> str: + return _json(self.to_dict()) + + +def _json(value: Any) -> str: + return json.dumps(_jsonable(value), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _jsonable(value: Any) -> Any: + if isinstance(value, Decimal): + return str(value) + if isinstance(value, Enum): + return value.value + if hasattr(value, "to_dict"): + return _jsonable(value.to_dict()) + if isinstance(value, dict): + return {key: _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + return value + + +def _treatment(name: P82Treatment) -> Treatment: + mechanisms = { + P82Treatment.A: (), + P82Treatment.B: (Mechanism.P8_SAFE_EDITOR,), + P82Treatment.C: (Mechanism.P8_ENHANCED_REPOSITORY_CONTEXT,), + P82Treatment.D: (Mechanism.P8_SAFE_EDITOR, Mechanism.P8_ENHANCED_REPOSITORY_CONTEXT), + } + return Treatment(name.value, mechanisms[name]) + + +@dataclass(frozen=True) +class P82ExperimentConfig: + """Frozen controls shared by every planned arm.""" + + executor: ExecutorIdentity + model: str | None + environment: ExecutionEnvironment + budget: ExecutionBudget + tasks: tuple[QualificationTask, ...] + replicate_ids: tuple[str, ...] = ("rep-1",) + verification_contract: str | None = None + acceptance_authority: str | None = None + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + tasks = tuple(self.tasks) + if not tasks: + raise ValueError("P8.2 requires at least one task") + if len({task.task_id for task in tasks}) != len(tasks): + raise ValueError("P8.2 task identifiers must be unique") + replicates = tuple(self.replicate_ids) + if not replicates or any(not replicate for replicate in replicates): + raise ValueError("P8.2 requires non-empty replicate identifiers") + if len(set(replicates)) != len(replicates): + raise ValueError("P8.2 replicate identifiers must be unique") + object.__setattr__(self, "tasks", tuple(sorted(tasks, key=lambda task: task.task_id))) + object.__setattr__(self, "replicate_ids", tuple(sorted(replicates))) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "executor": self.executor.to_dict(), + "model": self.model, + "environment": self.environment.to_dict(), + "budget": self.budget.to_dict(), + "tasks": [task.to_dict() for task in self.tasks], + "replicate_ids": list(self.replicate_ids), + "verification_contract": self.verification_contract, + "acceptance_authority": self.acceptance_authority, + } + + +@dataclass(frozen=True) +class P82TrialPlan: + treatment: Treatment + task: QualificationTask + replicate_id: str + + def to_dict(self) -> dict[str, Any]: + return { + "treatment": self.treatment.to_dict(), + "task": self.task.to_dict(), + "replicate_id": self.replicate_id, + } + + +@dataclass(frozen=True) +class P82ExperimentPlan: + config: P82ExperimentConfig + treatments: tuple[Treatment, ...] + trials: tuple[P82TrialPlan, ...] + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "config": self.config.to_dict(), + "treatments": [treatment.to_dict() for treatment in self.treatments], + "trials": [trial.to_dict() for trial in self.trials], + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class TaskSampleTask: + task: QualificationTask + stratum: TaskStratum + provenance: tuple[str, ...] | None = None + evidence: tuple[str, ...] | None = None + + def __post_init__(self) -> None: + for name, values in (("provenance", self.provenance), ("evidence", self.evidence)): + if values is not None and any(not value for value in values): + raise ValueError(f"{name} references must be non-empty when present") + + def to_dict(self) -> dict[str, Any]: + return { + "task": self.task.to_dict(), + "stratum": self.stratum.value, + "provenance": None if self.provenance is None else sorted(set(self.provenance)), + "evidence": None if self.evidence is None else sorted(set(self.evidence)), + } + + +@dataclass(frozen=True) +class TaskSampleManifest: + sample_id: str + tasks: tuple[TaskSampleTask, ...] + sample_revision: str | None = None + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + if not self.sample_id: + raise ValueError("sample_id must be non-empty") + tasks = tuple(sorted(self.tasks, key=lambda item: item.task.task_id)) + if not tasks: + raise ValueError("task sample must contain at least one task") + if len({item.task.task_id for item in tasks}) != len(tasks): + raise ValueError("task sample task identifiers must be unique") + object.__setattr__(self, "tasks", tasks) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "sample_id": self.sample_id, + "sample_revision": self.sample_revision, + "tasks": [task.to_dict() for task in self.tasks], + } + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "TaskSampleManifest": + tasks = tuple( + TaskSampleTask( + QualificationTask(**item["task"]), + TaskStratum(item["stratum"]), + None if item.get("provenance") is None else tuple(item["provenance"]), + None if item.get("evidence") is None else tuple(item["evidence"]), + ) + for item in value["tasks"] + ) + return cls(value["sample_id"], tasks, value.get("sample_revision"), value.get("schema_version", SCHEMA_VERSION)) + + +@dataclass(frozen=True) +class P82RunManifest: + """All run identity needed to establish treatment-only comparability.""" + + protocol_version: str | None + treatment: Treatment + executor: ExecutorIdentity + model: str | None + temperature: str | None + reasoning_configuration: str | None + environment: ExecutionEnvironment + task_sample: TaskSampleManifest + attempts: int | None + token_budget: int | None + cost_budget: Decimal | None + wall_time_limit: Decimal | None + verification_identity: str | None + acceptance_identity: str | None + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + for name, value in (("attempts", self.attempts), ("token_budget", self.token_budget), + ("cost_budget", self.cost_budget), ("wall_time_limit", self.wall_time_limit)): + if value is not None and value < 0: + raise ValueError(f"{name} must be non-negative when present") + if self.treatment.name not in {item.value for item in P82Treatment}: + raise ValueError("P8.2 run treatment must be one of A, B, C, or D") + + def _control_dict(self) -> dict[str, Any]: + value = self.to_dict() + value.pop("treatment") + return value + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "protocol_version": self.protocol_version, + "treatment": self.treatment.to_dict(), + "executor": self.executor.to_dict(), + "model": self.model, + "temperature": self.temperature, + "reasoning_configuration": self.reasoning_configuration, + "environment": self.environment.to_dict(), + "task_sample": self.task_sample.to_dict(), + "attempts": self.attempts, + "token_budget": self.token_budget, + "cost_budget": self.cost_budget, + "wall_time_limit": self.wall_time_limit, + "verification_identity": self.verification_identity, + "acceptance_identity": self.acceptance_identity, + } + + def to_json(self) -> str: + return _json(self.to_dict()) + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "P82RunManifest": + executor = ExecutorIdentity(**value["executor"]) + environment = ExecutionEnvironment(**value["environment"]) + budget_fields = { + "max_tokens": value["token_budget"], "max_cost": value["cost_budget"], + "max_wall_time": value["wall_time_limit"], "max_attempts": value["attempts"], + "max_executor_invocations": None, + } + # P8.2 records its budget fields explicitly; qualification can still + # receive the resulting ExecutionBudget without inventing a limit. + budget_fields["max_cost"] = None if budget_fields["max_cost"] is None else Decimal(str(budget_fields["max_cost"])) + budget_fields["max_wall_time"] = None if budget_fields["max_wall_time"] is None else Decimal(str(budget_fields["max_wall_time"])) + treatment = Treatment(value["treatment"]["name"], tuple(Mechanism(item) for item in value["treatment"]["enabled_mechanisms"])) + return cls( + protocol_version=value.get("protocol_version"), treatment=treatment, executor=executor, + model=value.get("model"), temperature=value.get("temperature"), + reasoning_configuration=value.get("reasoning_configuration"), environment=environment, + task_sample=TaskSampleManifest.from_dict(value["task_sample"]), attempts=value.get("attempts"), + token_budget=value.get("token_budget"), cost_budget=budget_fields["max_cost"], + wall_time_limit=budget_fields["max_wall_time"], verification_identity=value.get("verification_identity"), + acceptance_identity=value.get("acceptance_identity"), schema_version=value.get("schema_version", SCHEMA_VERSION), + ) + + +@dataclass(frozen=True) +class P82Observation: + status: P82RunStatus + verified_acceptance: bool | None = None + cost_to_acceptance: Decimal | None = None + tokens_to_acceptance: int | None = None + wall_time_seconds: Decimal | None = None + model_calls: int | None = None + turns: int | None = None + repeated_exploration: int | None = None + repeated_file_reads_searches: int | None = None + edit_failures: int | None = None + retries: int | None = None + replans: int | None = None + unnecessary_diff: int | None = None + regressions: int | None = None + human_intervention: int | None = None + false_pass_evidence: bool | None = None + verification_outcome: str | None = None + + def __post_init__(self) -> None: + for name in ("tokens_to_acceptance", "model_calls", "turns", "repeated_exploration", + "repeated_file_reads_searches", "edit_failures", "retries", "replans", + "unnecessary_diff", "regressions", "human_intervention"): + value = getattr(self, name) + if value is not None and value < 0: + raise ValueError(f"{name} must be non-negative when present") + for name in ("cost_to_acceptance", "wall_time_seconds"): + value = getattr(self, name) + if value is not None and value < 0: + raise ValueError(f"{name} must be non-negative when present") + + def to_dict(self) -> dict[str, Any]: + return {"status": self.status.value, **{name: getattr(self, name) for name in ( + "verified_acceptance", "cost_to_acceptance", "tokens_to_acceptance", "wall_time_seconds", + "model_calls", "turns", "repeated_exploration", "repeated_file_reads_searches", + "edit_failures", "retries", "replans", "unnecessary_diff", "regressions", + "human_intervention", "false_pass_evidence", "verification_outcome")}} + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "P82Observation": + values = dict(value) + values["status"] = P82RunStatus(values["status"]) + for name in ("cost_to_acceptance", "wall_time_seconds"): + if values.get(name) is not None: + values[name] = Decimal(str(values[name])) + return cls(**values) + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class P82ObservationRecord: + run: P82RunManifest + observation: P82Observation + + def to_dict(self) -> dict[str, Any]: + return {"run": self.run.to_dict(), "observation": self.observation.to_dict()} + + def to_json(self) -> str: + return _json(self.to_dict()) + + +@dataclass(frozen=True) +class P82TreatmentSummary: + treatment: str + record_count: int + executed_count: int + not_executed_count: int + blocked_count: int + verified_acceptance_rate: Decimal | None + median_cost_to_acceptance: Decimal | None + median_tokens_to_acceptance: Decimal | None + false_pass_rate: Decimal | None + + def to_dict(self) -> dict[str, Any]: + return {name: getattr(self, name) for name in ( + "treatment", "record_count", "executed_count", "not_executed_count", "blocked_count", + "verified_acceptance_rate", "median_cost_to_acceptance", "median_tokens_to_acceptance", + "false_pass_rate")} + + +def p82_manifest() -> tuple[Treatment, ...]: + """Return the canonical A/B/C/D arms in experimental order.""" + + return tuple(_treatment(name) for name in P82Treatment) + + +def build_p82_plan(config: P82ExperimentConfig) -> P82ExperimentPlan: + """Create deterministic plans without executing or fabricating results.""" + + treatments = p82_manifest() + trials = tuple( + P82TrialPlan(treatment, task, replicate) + for task in config.tasks + for replicate in config.replicate_ids + for treatment in treatments + ) + return P82ExperimentPlan(config, treatments, trials) + + +def plan_is_comparable(plan: P82ExperimentPlan) -> bool | None: + """Return True when all shared controls are known, otherwise None. + + There is no False result: a generated plan has one frozen control set, so + missing evidence remains unknown rather than being treated as a mismatch. + """ + + values = ( + plan.config.executor.version, + plan.config.executor.configuration_digest, + plan.config.model, + plan.config.environment.repository_revision, + plan.config.environment.runtime, + plan.config.environment.platform, + plan.config.environment.toolchain, + plan.config.environment.environment_digest, + plan.config.budget.max_tokens, + plan.config.budget.max_cost, + plan.config.budget.max_wall_time, + plan.config.budget.max_attempts, + plan.config.budget.max_executor_invocations, + plan.config.verification_contract, + plan.config.acceptance_authority, + ) + return None if any(value is None for value in values) else True + + +def compare_run_manifests(left: P82RunManifest, right: P82RunManifest) -> P82Comparability: + """Compare runs while excluding treatment and preserving insufficient data.""" + + left_values, right_values = left._control_dict(), right._control_dict() + if any(value is None for value in _flatten_values(left_values)) or any(value is None for value in _flatten_values(right_values)): + return P82Comparability.INDETERMINATE + return P82Comparability.COMPARABLE if left_values == right_values else P82Comparability.NOT_COMPARABLE + + +def _flatten_values(value: Any) -> Iterable[Any]: + if isinstance(value, dict): + for item in value.values(): + yield from _flatten_values(item) + elif isinstance(value, list): + for item in value: + yield from _flatten_values(item) + else: + yield value + + +def summarize_p82(records: tuple[P82ObservationRecord, ...]) -> tuple[P82TreatmentSummary, ...]: + """Aggregate observations by arm without treating unavailable data as zero.""" + + result = [] + for treatment in P82Treatment: + selected = tuple(record for record in records if record.run.treatment.name == treatment.value) + executed = tuple(record for record in selected if record.observation.status is P82RunStatus.EXECUTED) + accepted = tuple(record.observation.verified_acceptance for record in executed if record.observation.verified_acceptance is not None) + accepted_costs = tuple(record.observation.cost_to_acceptance for record in executed if record.observation.verified_acceptance is True and record.observation.cost_to_acceptance is not None) + accepted_tokens = tuple(record.observation.tokens_to_acceptance for record in executed if record.observation.verified_acceptance is True and record.observation.tokens_to_acceptance is not None) + false_pass = tuple(record.observation.false_pass_evidence for record in selected if record.observation.false_pass_evidence is not None) + result.append(P82TreatmentSummary( + treatment=treatment.value, + record_count=len(selected), + executed_count=len(executed), + not_executed_count=sum(record.observation.status is P82RunStatus.NOT_EXECUTED for record in selected), + blocked_count=sum(record.observation.status is P82RunStatus.BLOCKED for record in selected), + verified_acceptance_rate=None if not accepted else Decimal(sum(accepted)) / Decimal(len(accepted)), + median_cost_to_acceptance=None if not accepted_costs else median(accepted_costs), + median_tokens_to_acceptance=None if not accepted_tokens else Decimal(str(median(accepted_tokens))), + false_pass_rate=None if not false_pass else Decimal(sum(false_pass)) / Decimal(len(false_pass)), + )) + return tuple(result) diff --git a/src/arkx/p82_baseline.py b/src/arkx/p82_baseline.py new file mode 100644 index 00000000..15b7aa96 --- /dev/null +++ b/src/arkx/p82_baseline.py @@ -0,0 +1,521 @@ +"""P8.2a baseline execution path. + +This is infrastructure qualification, not mechanism evidence. The runner +captures executor output only; verification and independent acceptance are +separate contracts and are never inferred from an exit status or a diff. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +import hashlib +import importlib.util +import json +from pathlib import Path +import os +import subprocess +from typing import Any, Callable, Protocol + +from .outcomes import AcceptanceDecision, AcceptanceResult, VerificationResult, VerificationState +from .harness import ArtifactRecord, ArtifactStatus, ArtifactStore, ErrorDomain, ErrorEnvelope, LifecycleEvent, Retryability, RunManifest, RunState, derive_attempt_id, validate_execution_artifact_consistency +from .execution import ExecutionRequest, ExecutionResult +from .configuration import ConfigurationSnapshot +from .integration import IntegrationKind, process_error_envelope +from .integration import AdapterIdentity, DependencyObservation, CapabilityProvenance, assess_preflight, AdapterPreflight +from .p82 import P82Observation, P82RunStatus, P82Treatment, TaskStratum + + +PROTOCOL_VERSION = "p8.2a-v1" + + +class BaselineExecutionState(str, Enum): + COMPLETED = "COMPLETED" + EXECUTOR_FAILED = "EXECUTOR_FAILED" + BLOCKED = "BLOCKED" + + +class RunnerFailureCategory(str, Enum): + PROCESS_START_FAILURE = "PROCESS_START_FAILURE" + ENVIRONMENT_FAILURE = "ENVIRONMENT_FAILURE" + MODEL_INITIALIZATION_FAILURE = "MODEL_INITIALIZATION_FAILURE" + AGENT_INITIALIZATION_FAILURE = "AGENT_INITIALIZATION_FAILURE" + AGENT_RUN_FAILURE = "AGENT_RUN_FAILURE" + TIMEOUT = "TIMEOUT" + DEADLOCK_HANG = "DEADLOCK/HANG" + OUTPUT_CAPTURE_FAILURE = "OUTPUT_CAPTURE_FAILURE" + PROVIDER_AVAILABILITY_FAILURE = "PROVIDER_AVAILABILITY_FAILURE" + UNKNOWN = "UNKNOWN" + + +@dataclass(frozen=True) +class ProspectiveTask: + task_id: str + source_dataset: str | None + repository: str | None + base_commit: str | None + problem_statement: str | None + stratum: TaskStratum + rationale: str | None + selection_timestamp: str | None + protocol_version: str | None + verification_identity: str | None + acceptance_authority_identity: str | None + workspace: str + + def __post_init__(self) -> None: + if not self.task_id or not self.workspace: + raise ValueError("task_id and workspace must be non-empty") + + def to_dict(self) -> dict[str, Any]: + value = {name: getattr(self, name) for name in ( + "task_id", "source_dataset", "repository", "base_commit", "problem_statement", + "stratum", "rationale", "selection_timestamp", "protocol_version", + "verification_identity", "acceptance_authority_identity", "workspace")} + value["stratum"] = self.stratum.value + return value + + +@dataclass(frozen=True) +class BaselineSampleManifest: + sample_id: str + wave: str + selection_rule: str + tasks: tuple[ProspectiveTask, ...] + protocol_version: str = PROTOCOL_VERSION + + def __post_init__(self) -> None: + if not self.sample_id or not self.wave or not self.selection_rule: + raise ValueError("sample identity, wave, and selection rule are required") + if not self.tasks: + raise ValueError("prospective sample cannot be empty") + if len({task.task_id for task in self.tasks}) != len(self.tasks): + raise ValueError("sample task identifiers must be unique") + object.__setattr__(self, "tasks", tuple(sorted(self.tasks, key=lambda task: task.task_id))) + + def to_dict(self) -> dict[str, Any]: + return {"sample_id": self.sample_id, "wave": self.wave, "selection_rule": self.selection_rule, + "protocol_version": self.protocol_version, "tasks": [task.to_dict() for task in self.tasks]} + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +@dataclass(frozen=True) +class BaselineRunConfig: + provider: str | None + model: str | None + model_version: str | None + temperature: str | None + reasoning_configuration: str | None + model_configuration: dict[str, Any] | None + max_tokens: int | None + max_cost: str | None + wall_time_seconds: int | None + mini_version: str | None + python_executable: str | None + runner_version: str + artifact_root: str + treatment: P82Treatment = P82Treatment.A + environment: str = "local" + bash_executable: str | None = None + provider_retry_attempts: int = 0 + provider_retry_backoff_seconds: float = 0.0 + + def __post_init__(self) -> None: + if self.treatment is not P82Treatment.A: + raise ValueError("P8.2a baseline runner accepts treatment A only") + for name in ("max_tokens", "wall_time_seconds"): + value = getattr(self, name) + if value is not None and value < 0: + raise ValueError(f"{name} must be non-negative when present") + if self.provider_retry_attempts < 0 or self.provider_retry_backoff_seconds < 0: + raise ValueError("provider retry limits must be non-negative") + + def missing_identity(self) -> tuple[str, ...]: + return tuple(name for name in ("provider", "model", "mini_version", "python_executable") if getattr(self, name) is None) + + def identity_digest(self) -> str: + """Return a stable digest for the frozen execution configuration.""" + return self.configuration_snapshot().digest() + + def configuration_snapshot(self) -> ConfigurationSnapshot: + """Return the canonical, secret-safe configuration identity.""" + value = { + "provider": self.provider, + "model": self.model, + "model_version": self.model_version, + "temperature": self.temperature, + "reasoning_configuration": self.reasoning_configuration, + "model_configuration": self.model_configuration, + "max_tokens": self.max_tokens, + "max_cost": self.max_cost, + "wall_time_seconds": self.wall_time_seconds, + "mini_version": self.mini_version, + "python_executable": self.python_executable, + "runner_version": self.runner_version, + "treatment": self.treatment.value, + "environment": self.environment, + "bash_executable": self.bash_executable, + "provider_retry_attempts": self.provider_retry_attempts, + "provider_retry_backoff_seconds": self.provider_retry_backoff_seconds, + } + return ConfigurationSnapshot("p82-baseline-executor", self.runner_version, value) + + def budget_digest(self) -> str: + value = { + "max_tokens": self.max_tokens, + "max_cost": self.max_cost, + "wall_time_seconds": self.wall_time_seconds, + "provider_retry_attempts": self.provider_retry_attempts, + "provider_retry_backoff_seconds": self.provider_retry_backoff_seconds, + } + encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:16] + + +@dataclass(frozen=True) +class ExecutionArtifact: + run_id: str + state: BaselineExecutionState + task_id: str + treatment: str + attempt: int + started_at: str + finished_at: str + exit_code: int | None + failure: str | None + failure_category: RunnerFailureCategory | None + diff_path: str | None + log_path: str | None + trajectory_path: str | None + model_usage: dict[str, Any] | None + artifact_path: str | None = None + trial_id: str | None = None + attempt_id: str | None = None + manifest_path: str | None = None + error: ErrorEnvelope | None = None + + def to_dict(self) -> dict[str, Any]: + value = {name: getattr(self, name) for name in ( + "run_id", "state", "task_id", "treatment", "attempt", "started_at", "finished_at", + "exit_code", "failure", "failure_category", "diff_path", "log_path", "trajectory_path", + "model_usage", "artifact_path", "trial_id", "attempt_id", "manifest_path", "error")} + if isinstance(value.get("error"), ErrorEnvelope): + value["error"] = value["error"].to_dict() + for name in ("state", "failure_category"): + if isinstance(value[name], Enum): + value[name] = value[name].value + return value + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +class HeadlessInvoker(Protocol): + def __call__(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: ... + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="milliseconds") + + +class MiniSweAgentHeadlessRunner: + """Narrow runner. It never calls verification or acceptance.""" + + def __init__(self, invoker: HeadlessInvoker | None = None) -> None: + self.invoker = invoker or self._subprocess_invoker + + def run(self, task: ProspectiveTask, config: BaselineRunConfig, attempt: int = 1) -> ExecutionArtifact: + if attempt < 1: + raise ValueError("attempt must be positive") + missing = config.missing_identity() + store = ArtifactStore(config.artifact_root) + trial_id = hashlib.sha256( + f"{task.task_id}|{task.base_commit}|A|{config.identity_digest()}".encode("utf-8") + ).hexdigest()[:16] + run_id = derive_attempt_id( + trial_id=trial_id, + attempt_number=attempt, + configuration_digest=config.identity_digest(), + ) + try: + root = store.reserve_attempt(task.task_id, attempt) + except FileExistsError: + return self._blocked(run_id, task, attempt, "artifact already exists; refusing silent overwrite") + started = _now() + trajectory = root / "trajectory.json" + if missing: + return self._finish(root, run_id, trial_id, task, config, attempt, started, False, BaselineExecutionState.BLOCKED, None, + f"missing explicit identity: {','.join(missing)}", RunnerFailureCategory.UNKNOWN, None, None, None, None) + try: + exit_code, log, usage = self.invoker(task, config, trajectory) + log_path = root / "executor.log" + log_path.write_text(log, encoding="utf-8") + diff_path = root / "diff.patch" + diff = subprocess.run(["git", "diff", "--binary"], cwd=task.workspace, capture_output=True, text=True, check=False) + diff_path.write_text(diff.stdout, encoding="utf-8") + if exit_code == 75: + return self._finish(root, run_id, trial_id, task, config, attempt, started, True, BaselineExecutionState.BLOCKED, exit_code, + "provider availability failure after bounded retries", + RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE, + str(diff_path), str(log_path), str(trajectory), usage) + state = BaselineExecutionState.COMPLETED if exit_code == 0 else BaselineExecutionState.EXECUTOR_FAILED + return self._finish(root, run_id, trial_id, task, config, attempt, started, True, state, exit_code, None if exit_code == 0 else "executor returned non-zero", None if exit_code == 0 else RunnerFailureCategory.AGENT_RUN_FAILURE, str(diff_path), str(log_path), str(trajectory), usage) + except subprocess.TimeoutExpired as error: + log_path = root / "executor.log" + stdout = self._decode_process_output(error.stdout) + stderr = self._decode_process_output(error.stderr) + log_path.write_text(stdout + stderr, encoding="utf-8", newline="\n") + diagnostics_path = root / "timeout-observability.json" + diagnostics_path.write_text(json.dumps({ + "event": "runner_timeout", + "timeout_origin": "runner_subprocess_watchdog", + "timeout_seconds": error.timeout, + "started_at": started, + "finished_at": _now(), + "exception_type": type(error).__name__, + "exception": str(error), + "model_diagnostics_path": str(root / "model-observability.json"), + }, ensure_ascii=False, sort_keys=True) + "\n", encoding="utf-8") + return self._finish( + root, run_id, trial_id, task, config, attempt, started, True, + BaselineExecutionState.EXECUTOR_FAILED, None, str(error), + RunnerFailureCategory.TIMEOUT, None, str(log_path), + str(trajectory) if trajectory.exists() else None, None, + ) + except Exception as error: + message = f"{type(error).__name__}: {error}" + return self._finish(root, run_id, trial_id, task, config, attempt, started, True, BaselineExecutionState.EXECUTOR_FAILED, None, message, self._classify_failure(message), None, None, None, None) + + @staticmethod + def _classify_failure(message: str) -> RunnerFailureCategory: + lowered = message.lower() + if "filenotfound" in lowered or "process" in lowered and "start" in lowered: + return RunnerFailureCategory.PROCESS_START_FAILURE + if "timeout" in lowered or "timed out" in lowered: + return RunnerFailureCategory.TIMEOUT + if "environment" in lowered or "docker" in lowered or "workspace" in lowered: + return RunnerFailureCategory.ENVIRONMENT_FAILURE + if "model" in lowered and ("validation" in lowered or "initial" in lowered or "config" in lowered): + return RunnerFailureCategory.MODEL_INITIALIZATION_FAILURE + if "agent" in lowered and ("initial" in lowered or "config" in lowered): + return RunnerFailureCategory.AGENT_INITIALIZATION_FAILURE + if "trajectory" in lowered or "capture" in lowered: + return RunnerFailureCategory.OUTPUT_CAPTURE_FAILURE + if "deadlock" in lowered or "hang" in lowered: + return RunnerFailureCategory.DEADLOCK_HANG + return RunnerFailureCategory.AGENT_RUN_FAILURE + + @staticmethod + def _decode_process_output(value: str | bytes | None) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return value + + def _subprocess_invoker(self, task: ProspectiveTask, config: BaselineRunConfig, trajectory_path: Path) -> tuple[int, str, dict[str, Any] | None]: + worker = Path(__file__).parents[2] / "tools" / "run_mini_headless.py" + payload = {"task": task.problem_statement, "workspace": task.workspace, "model": config.model, + "trajectory": str(trajectory_path), "max_cost": config.max_cost, "wall_time_seconds": config.wall_time_seconds, + "model_configuration": config.model_configuration, "environment": config.environment, + "bash_executable": config.bash_executable, "provider_retry_attempts": config.provider_retry_attempts, + "provider_retry_backoff_seconds": config.provider_retry_backoff_seconds, + "diagnostics_path": str(trajectory_path.with_name("model-observability.json").resolve())} + timeout = None if config.wall_time_seconds is None else config.wall_time_seconds + 10 + completed = subprocess.run( + [config.python_executable, str(worker)], input=json.dumps(payload), text=True, + capture_output=True, cwd=task.workspace, timeout=timeout, + ) + combined_log = completed.stdout + completed.stderr + usage = None + if completed.returncode == 0: + lines = [line for line in completed.stdout.splitlines() if line.strip()] + if lines: + try: + worker_result = json.loads(lines[-1]) + usage = worker_result.get("usage") + result = worker_result.get("result", {}) + if isinstance(result, dict) and result.get("failure_category") == RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE.value: + return 75, combined_log, usage + except (json.JSONDecodeError, AttributeError): + usage = None + return completed.returncode, combined_log, usage + + def _blocked(self, run_id: str, task: ProspectiveTask, attempt: int, failure: str) -> ExecutionArtifact: + now = _now() + return ExecutionArtifact(run_id, BaselineExecutionState.BLOCKED, task.task_id, "A", attempt, now, now, None, failure, RunnerFailureCategory.UNKNOWN, None, None, None, None) + + def _finish(self, root: Path, run_id: str, trial_id: str, task: ProspectiveTask, config: BaselineRunConfig, attempt: int, started: str, execution_started: bool, state: BaselineExecutionState, exit_code: int | None, failure: str | None, failure_category: RunnerFailureCategory | None, diff_path: str | None, log_path: str | None, trajectory_path: str | None, usage: dict[str, Any] | None) -> ExecutionArtifact: + store = ArtifactStore(root.parents[1]) + artifact_path = root / "execution.json" + manifest_path = root / "manifest.json" + error = None + if failure: + raw_refs = tuple(path for path in (log_path, trajectory_path) if path) + if failure_category is RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE: + # This is the one legacy classification with an explicit retry policy. + error = ErrorEnvelope( + domain=ErrorDomain.PROVIDER, + code=failure_category.value, + message=failure, + retryability=Retryability.RETRYABLE, + raw_evidence_refs=raw_refs, + attempt_number=attempt, + ) + elif failure_category is RunnerFailureCategory.UNKNOWN: + # Missing identity is a harness precondition failure, not an + # executor invocation failure. + error = ErrorEnvelope( + domain=ErrorDomain.HARNESS, + code=RunnerFailureCategory.UNKNOWN.value, + message=failure, + retryability=Retryability.UNKNOWN, + raw_evidence_refs=raw_refs, + attempt_number=attempt, + ) + else: + kind = IntegrationKind.SANDBOX if failure_category in ( + RunnerFailureCategory.ENVIRONMENT_FAILURE, + RunnerFailureCategory.PROCESS_START_FAILURE, + ) else IntegrationKind.EXECUTOR + error = process_error_envelope( + kind, + returncode=exit_code, + message=failure, + raw_evidence_refs=raw_refs, + attempt_number=attempt, + ) + manifest = RunManifest( + experiment_id=f"p82a:{config.runner_version}", trial_id=trial_id, attempt_id=run_id, + attempt_number=attempt, task_id=task.task_id, task_revision=task.base_commit, + treatment="A", executor="mini-swe-agent", provider=config.provider, + model=config.model, sandbox=config.environment, repository_revision=task.base_commit, + configuration_digest=config.identity_digest(), protocol_version=task.protocol_version or PROTOCOL_VERSION, + budget_digest=config.budget_digest(), + state={BaselineExecutionState.COMPLETED: RunState.COMPLETED, BaselineExecutionState.EXECUTOR_FAILED: RunState.FAILED, BaselineExecutionState.BLOCKED: RunState.BLOCKED}[state], + state_history=(RunState.PLANNED, RunState.STARTED) + ((RunState.EXECUTING,) if execution_started else ()) + ({BaselineExecutionState.COMPLETED: (RunState.COMPLETED,), BaselineExecutionState.EXECUTOR_FAILED: (RunState.FAILED,), BaselineExecutionState.BLOCKED: (RunState.BLOCKED,)}[state]), + lifecycle_events=tuple( + LifecycleEvent(current, started if index < 2 else _now()) + for index, current in enumerate((RunState.PLANNED, RunState.STARTED) + ((RunState.EXECUTING,) if execution_started else ()) + ({BaselineExecutionState.COMPLETED: (RunState.COMPLETED,), BaselineExecutionState.EXECUTOR_FAILED: (RunState.FAILED,), BaselineExecutionState.BLOCKED: (RunState.BLOCKED,)}[state])) + ), + artifacts=tuple( + ArtifactRecord( + kind=kind, + status=ArtifactStatus.PRESENT if path == str(artifact_path) or (path is not None and Path(path).is_file()) else ArtifactStatus.MISSING, + path=path if path == str(artifact_path) or (path is not None and Path(path).is_file()) else None, + required=kind == "execution", + reason=None if path == str(artifact_path) or (path is not None and Path(path).is_file()) else "not produced by this attempt", + ) + for kind, path in (("execution", str(artifact_path)), ("diff", diff_path), ("executor_log", log_path), ("trajectory", trajectory_path)) + ), + artifact_refs=tuple( + [str(artifact_path)] + [ + path for path in (diff_path, log_path, trajectory_path) + if path and Path(path).is_file() + ] + ), + errors=(error,) if error else (), + ) + manifest.validate_attempt_identity() + manifest.validate_lifecycle() + artifact = ExecutionArtifact(run_id, state, task.task_id, "A", attempt, started, _now(), exit_code, failure, failure_category, diff_path, log_path, trajectory_path, usage, str(artifact_path), trial_id, run_id, str(manifest_path), error) + store.write_text_atomic(manifest_path, manifest.to_json() + "\n") + store.write_text_atomic(artifact_path, artifact.to_json() + "\n") + validate_execution_artifact_consistency(manifest, artifact.to_dict()) + store.validate_manifest_references(manifest) + return artifact + + @staticmethod + def _error_domain(category: RunnerFailureCategory | None) -> ErrorDomain: + if category is RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE: + return ErrorDomain.PROVIDER + if category in (RunnerFailureCategory.ENVIRONMENT_FAILURE, RunnerFailureCategory.PROCESS_START_FAILURE): + return ErrorDomain.SANDBOX + if category is RunnerFailureCategory.UNKNOWN or category is None: + return ErrorDomain.HARNESS + return ErrorDomain.EXECUTOR + + +class P82BaselineExecutorAdapter: + """Translate the legacy P8.2 runner into the neutral Executor contract.""" + + executor_id = "p82-baseline-mini-swe-agent" + + def __init__(self, task: ProspectiveTask, config: BaselineRunConfig, runner: MiniSweAgentHeadlessRunner | None = None, attempt: int = 1) -> None: + self.task = task + self.config = config + self.runner = runner or MiniSweAgentHeadlessRunner() + self.attempt = attempt + + def preflight(self) -> AdapterPreflight: + """Describe concrete readiness without invoking the runner.""" + available = importlib.util.find_spec("minisweagent") is not None + return assess_preflight( + AdapterIdentity( + IntegrationKind.EXECUTOR, + self.executor_id, + self.config.mini_version, + self.config.configuration_snapshot().digest(), + ), + (DependencyObservation("minisweagent", available, reason=None if available else "python module unavailable"),), + capabilities=("edit",), + capability_digest=self.config.configuration_snapshot().digest(), + capability_provenance=CapabilityProvenance.DECLARED, + ) + + def execute(self, request: ExecutionRequest) -> ExecutionResult: + if request.task_id != self.task.task_id: + raise ValueError("execution request task_id does not match bound baseline task") + if request.treatment != "A": + raise ValueError("P8.2 baseline adapter accepts treatment A only") + artifact = self.runner.run(self.task, self.config, attempt=self.attempt) + state = { + BaselineExecutionState.COMPLETED: RunState.COMPLETED, + BaselineExecutionState.EXECUTOR_FAILED: RunState.FAILED, + BaselineExecutionState.BLOCKED: RunState.BLOCKED, + }[artifact.state] + return ExecutionResult( + state=state, + artifact_refs=tuple(ref for ref in (artifact.artifact_path, artifact.diff_path, artifact.log_path, artifact.trajectory_path, artifact.manifest_path) if ref), + usage=artifact.model_usage, + error=artifact.error, + ) + + +class IndependentAcceptanceAuthority(Protocol): + identity: str + + def decide(self, verification: VerificationResult) -> AcceptanceResult: ... + + +class ExplicitVerificationAcceptance: + """An explicit policy: verification PASS maps to acceptance only by declaration.""" + + def __init__(self, identity: str) -> None: + if not identity: + raise ValueError("acceptance authority identity must be explicit") + self.identity = identity + + def decide(self, verification: VerificationResult) -> AcceptanceResult: + mapping = { + VerificationState.PASS: AcceptanceDecision.ACCEPTED, + VerificationState.FAIL: AcceptanceDecision.REJECTED, + VerificationState.INDETERMINATE: AcceptanceDecision.INDETERMINATE, + VerificationState.NOT_EXECUTED: AcceptanceDecision.NOT_EXECUTED, + VerificationState.BLOCKED: AcceptanceDecision.BLOCKED, + } + return AcceptanceResult(mapping[verification.state], self.identity, verification.state.value, verification.evidence) + + +def baseline_observation(execution: ExecutionArtifact, verification: VerificationResult | None, acceptance: AcceptanceResult | None) -> P82Observation: + if execution.state is BaselineExecutionState.BLOCKED: + status = P82RunStatus.BLOCKED + elif execution.state is not BaselineExecutionState.COMPLETED: + status = P82RunStatus.EXECUTED + else: + status = P82RunStatus.EXECUTED + accepted = None if acceptance is None else acceptance.decision is AcceptanceDecision.ACCEPTED + return P82Observation(status=status, verified_acceptance=accepted) diff --git a/src/arkx/p82_editing.py b/src/arkx/p82_editing.py new file mode 100644 index 00000000..1f09a6b5 --- /dev/null +++ b/src/arkx/p82_editing.py @@ -0,0 +1,102 @@ +"""B1 structured-editing contract. + +This module represents an edit proposal only. It never writes files, applies a +patch, runs validation, selects a target, retries, or changes acceptance. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +import json +from typing import Any + + +SCHEMA_VERSION = 1 +TREATMENT = "B1" + + +class EditOperation(str, Enum): + REPLACE_RANGE = "REPLACE_RANGE" + APPLY_PATCH = "APPLY_PATCH" + + +@dataclass(frozen=True) +class StructuredEditProposal: + """A syntactically bounded edit proposal with no execution semantics.""" + + task_id: str + target_file: str + operation: EditOperation | str + replacement_content: str | None = None + patch_content: str | None = None + start_line: int | None = None + end_line: int | None = None + treatment: str = TREATMENT + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + for name in ("task_id", "target_file"): + if not getattr(self, name).strip(): + raise ValueError(f"{name} must be non-empty") + if self.treatment != TREATMENT: + raise ValueError(f"edit proposal treatment must be {TREATMENT}") + try: + operation = EditOperation(self.operation) + except ValueError as error: + raise ValueError("unsupported structured edit operation") from error + object.__setattr__(self, "operation", operation) + + has_replacement = self.replacement_content is not None + has_patch = self.patch_content is not None + if has_replacement == has_patch: + raise ValueError("exactly one edit representation must be provided") + if operation is EditOperation.REPLACE_RANGE: + if not has_replacement or self.start_line is None or self.end_line is None: + raise ValueError("REPLACE_RANGE requires replacement content and line range") + if self.start_line < 1 or self.end_line < self.start_line: + raise ValueError("edit line range is invalid") + elif operation is EditOperation.APPLY_PATCH: + if not has_patch: + raise ValueError("APPLY_PATCH requires patch content") + if self.start_line is not None or self.end_line is not None: + raise ValueError("APPLY_PATCH cannot include a line range") + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "treatment": self.treatment, + "task_id": self.task_id, + "target_file": self.target_file, + "operation": self.operation.value, + "replacement_content": self.replacement_content, + "patch_content": self.patch_content, + "start_line": self.start_line, + "end_line": self.end_line, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def record_edit_proposal( + *, + task_id: str, + target_file: str, + operation: EditOperation | str, + replacement_content: str | None = None, + patch_content: str | None = None, + start_line: int | None = None, + end_line: int | None = None, +) -> StructuredEditProposal: + """Validate and record B1 without applying or evaluating the proposal.""" + + return StructuredEditProposal( + task_id=task_id, + target_file=target_file, + operation=operation, + replacement_content=replacement_content, + patch_content=patch_content, + start_line=start_line, + end_line=end_line, + ) diff --git a/src/arkx/p82_localization.py b/src/arkx/p82_localization.py new file mode 100644 index 00000000..34566589 --- /dev/null +++ b/src/arkx/p82_localization.py @@ -0,0 +1,121 @@ +"""C1 repository-localization contract. + +This module specifies the artifact boundary for Treatment C1. It deliberately +does not implement repository search, ranking, prompting, or execution. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from typing import Any + + +SCHEMA_VERSION = 1 +TREATMENT = "C1" + + +def _sorted_unique(values: tuple[str, ...]) -> tuple[str, ...]: + if any(not value or not value.strip() for value in values): + raise ValueError("localization values must be non-empty") + return tuple(sorted(set(values))) + + +@dataclass(frozen=True) +class LocalizationEvidenceBudget: + """Hard cap for evidence emitted by one localization pass.""" + + max_files: int + max_symbols: int + max_context_lines: int + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + for name in ("max_files", "max_symbols", "max_context_lines"): + if getattr(self, name) < 0: + raise ValueError(f"{name} must be non-negative") + + def to_dict(self) -> dict[str, int]: + return { + "max_files": self.max_files, + "max_symbols": self.max_symbols, + "max_context_lines": self.max_context_lines, + } + + +@dataclass(frozen=True) +class LocalizationArtifact: + """Bounded, non-authoritative output of a future C1 localization pass.""" + + task_id: str + repository_revision: str + localization_method: str + candidate_files: tuple[str, ...] + candidate_symbols: tuple[str, ...] + context_lines_used: int + evidence_budget: LocalizationEvidenceBudget + candidates_are_non_authoritative: bool = True + treatment: str = TREATMENT + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + for name in ("task_id", "repository_revision", "localization_method"): + if not getattr(self, name).strip(): + raise ValueError(f"{name} must be non-empty") + if self.treatment != TREATMENT: + raise ValueError(f"localization artifact treatment must be {TREATMENT}") + if not self.candidates_are_non_authoritative: + raise ValueError("C1 candidates must be explicitly non-authoritative") + if self.context_lines_used < 0: + raise ValueError("context_lines_used must be non-negative") + + files = _sorted_unique(self.candidate_files) + symbols = _sorted_unique(self.candidate_symbols) + if len(files) > self.evidence_budget.max_files: + raise ValueError("candidate file count exceeds evidence budget") + if len(symbols) > self.evidence_budget.max_symbols: + raise ValueError("candidate symbol count exceeds evidence budget") + if self.context_lines_used > self.evidence_budget.max_context_lines: + raise ValueError("context line count exceeds evidence budget") + object.__setattr__(self, "candidate_files", files) + object.__setattr__(self, "candidate_symbols", symbols) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "treatment": self.treatment, + "task_id": self.task_id, + "repository_revision": self.repository_revision, + "localization_method": self.localization_method, + "candidate_files": list(self.candidate_files), + "candidate_symbols": list(self.candidate_symbols), + "context_lines_used": self.context_lines_used, + "evidence_budget": self.evidence_budget.to_dict(), + "candidates_are_non_authoritative": self.candidates_are_non_authoritative, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def record_localization_artifact( + *, + task_id: str, + repository_revision: str, + localization_method: str, + candidate_files: tuple[str, ...] = (), + candidate_symbols: tuple[str, ...] = (), + context_lines_used: int = 0, + evidence_budget: LocalizationEvidenceBudget, +) -> LocalizationArtifact: + """Validate and record C1 output without performing localization.""" + + return LocalizationArtifact( + task_id=task_id, + repository_revision=repository_revision, + localization_method=localization_method, + candidate_files=candidate_files, + candidate_symbols=candidate_symbols, + context_lines_used=context_lines_used, + evidence_budget=evidence_budget, + ) diff --git a/src/arkx/planning.py b/src/arkx/planning.py new file mode 100644 index 00000000..fd526e5a --- /dev/null +++ b/src/arkx/planning.py @@ -0,0 +1,231 @@ +"""Deterministic repository state and plan artifacts for P4.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +import json +from typing import Any + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class StepStatus(_ValueEnum): + PENDING = "PENDING" + IN_PROGRESS = "IN_PROGRESS" + COMPLETED = "COMPLETED" + BLOCKED = "BLOCKED" + + +class ReplanTrigger(_ValueEnum): + DEPENDENCY_DISCOVERED = "DEPENDENCY_DISCOVERED" + EXPECTED_FILE_MISSING = "EXPECTED_FILE_MISSING" + TEST_SURFACE_CHANGED = "TEST_SURFACE_CHANGED" + ASSUMPTION_INVALIDATED = "ASSUMPTION_INVALIDATED" + BLOCKED_STEP = "BLOCKED_STEP" + SCOPE_EXPANDED = "SCOPE_EXPANDED" + + +def _values(items: tuple[str, ...] | None) -> list[str] | None: + return None if items is None else sorted(set(items)) + + +@dataclass(frozen=True) +class RepositoryState: + relevant_files: tuple[str, ...] | None = None + dependency_edges: tuple[tuple[str, str], ...] | None = None + impacted_components: tuple[str, ...] | None = None + affected_tests: tuple[str, ...] | None = None + assumptions: tuple[str, ...] | None = None + unresolved_questions: tuple[str, ...] | None = None + completed_steps: tuple[str, ...] | None = None + remaining_steps: tuple[str, ...] | None = None + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + edges = None if self.dependency_edges is None else [list(edge) for edge in sorted(set(self.dependency_edges))] + return { + "schema_version": self.schema_version, + "relevant_files": _values(self.relevant_files), + "dependency_edges": edges, + "impacted_components": _values(self.impacted_components), + "affected_tests": _values(self.affected_tests), + "assumptions": _values(self.assumptions), + "unresolved_questions": _values(self.unresolved_questions), + "completed_steps": _values(self.completed_steps), + "remaining_steps": _values(self.remaining_steps), + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +@dataclass(frozen=True) +class PlanStep: + step_id: str + goal: str + status: StepStatus = StepStatus.PENDING + depends_on: tuple[str, ...] = () + expected_files: tuple[str, ...] = () + + def to_dict(self) -> dict[str, Any]: + return { + "step_id": self.step_id, + "goal": self.goal, + "status": self.status.value, + "depends_on": sorted(set(self.depends_on)), + "expected_files": sorted(set(self.expected_files)), + } + + +@dataclass(frozen=True) +class RepositoryPlan: + goal: str + steps: tuple[PlanStep, ...] + dependencies: tuple[tuple[str, str], ...] + acceptance_criteria: tuple[str, ...] + expected_files: tuple[str, ...] + replan_triggers: tuple[ReplanTrigger, ...] + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "goal": self.goal, + "steps": [step.to_dict() for step in sorted(self.steps, key=lambda item: item.step_id)], + "dependencies": [list(edge) for edge in sorted(set(self.dependencies))], + "acceptance_criteria": _values(self.acceptance_criteria), + "expected_files": _values(self.expected_files), + "replan_triggers": sorted(trigger.value for trigger in set(self.replan_triggers)), + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def build_plan( + goal: str, + steps: tuple[PlanStep, ...], + *, + dependencies: tuple[tuple[str, str], ...] = (), + acceptance_criteria: tuple[str, ...] = (), + expected_files: tuple[str, ...] = (), + replan_triggers: tuple[ReplanTrigger, ...] = (), +) -> RepositoryPlan: + """Build and validate a plan without marking steps complete.""" + + ids = {step.step_id for step in steps} + if len(ids) != len(steps) or "" in ids: + raise ValueError("Plan step identities must be unique and non-empty") + edges = set(dependencies) + for step in steps: + unknown = set(step.depends_on) - ids + if unknown: + raise ValueError(f"Unknown step dependencies: {sorted(unknown)}") + edges.update((dependency, step.step_id) for dependency in step.depends_on) + _assert_acyclic(ids, edges) + return RepositoryPlan( + goal=goal, + steps=tuple(steps), + dependencies=tuple(edges), + acceptance_criteria=acceptance_criteria, + expected_files=expected_files, + replan_triggers=replan_triggers, + ) + + +def _assert_acyclic(nodes: set[str], edges: set[tuple[str, str]]) -> None: + graph = {node: [] for node in nodes} + for source, target in edges: + if source not in nodes or target not in nodes: + raise ValueError("Plan dependency references an unknown step") + graph[source].append(target) + visiting: set[str] = set() + visited: set[str] = set() + + def visit(node: str) -> None: + if node in visiting: + raise ValueError("Plan dependencies contain a cycle") + if node in visited: + return + visiting.add(node) + for child in graph[node]: + visit(child) + visiting.remove(node) + visited.add(node) + + for node in sorted(nodes): + visit(node) + + +def plan_repository(goal: str, state: RepositoryState) -> RepositoryPlan: + """Create a small deterministic plan from explicit repository state.""" + + files = tuple(state.relevant_files or ()) + components = tuple(state.impacted_components or ()) + steps = tuple( + PlanStep( + step_id=f"inspect-{index:02d}", + goal=f"Inspect component {component}", + expected_files=tuple(file for file in files if component in file), + ) + for index, component in enumerate(sorted(set(components)), start=1) + ) + test_step = PlanStep( + step_id="verify-affected-tests", + goal="Verify affected tests", + depends_on=tuple(step.step_id for step in steps), + expected_files=tuple(state.affected_tests or ()), + ) + all_steps = steps + (test_step,) + return build_plan( + goal, + all_steps, + acceptance_criteria=("all planned steps have explicit outcomes",), + expected_files=files, + replan_triggers=(ReplanTrigger.DEPENDENCY_DISCOVERED, ReplanTrigger.BLOCKED_STEP), + ) + + +@dataclass(frozen=True) +class ReplanRequest: + trigger: ReplanTrigger + previous_plan_ref: str + evidence_refs: tuple[str, ...] + remaining_budget: int | None + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "trigger": self.trigger.value, + "previous_plan_ref": self.previous_plan_ref, + "evidence_refs": _values(self.evidence_refs), + "remaining_budget": self.remaining_budget, + } + + +@dataclass(frozen=True) +class ReplanResult: + previous_plan_ref: str + new_plan: RepositoryPlan + trigger: ReplanTrigger + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "previous_plan_ref": self.previous_plan_ref, + "trigger": self.trigger.value, + "new_plan": self.new_plan.to_dict(), + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + diff --git a/src/arkx/progress.py b/src/arkx/progress.py new file mode 100644 index 00000000..00fc8c76 --- /dev/null +++ b/src/arkx/progress.py @@ -0,0 +1,341 @@ +"""Deterministic progress and stagnation assessment for P2. + +P2 observes snapshots. It never controls execution, selects an executor, or +turns an assessment into an acceptance status. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from enum import Enum +import json +from time import perf_counter_ns +from typing import Any + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class ProgressStatus(_ValueEnum): + PROGRESS_PROVEN = "PROGRESS_PROVEN" + PROGRESS_NOT_PROVEN = "PROGRESS_NOT_PROVEN" + NO_PROGRESS = "NO_PROGRESS" + UNKNOWN = "UNKNOWN" + + +class Confidence(_ValueEnum): + HIGH = "HIGH" + MEDIUM = "MEDIUM" + LOW = "LOW" + + +class ReasonCode(_ValueEnum): + NEW_RELEVANT_FILE = "NEW_RELEVANT_FILE" + NEW_PASSING_TEST = "NEW_PASSING_TEST" + FAILURE_EXPLAINED = "FAILURE_EXPLAINED" + ACCEPTANCE_DISTANCE_REDUCED = "ACCEPTANCE_DISTANCE_REDUCED" + DIFF_DISTANCE_CHANGED = "DIFF_DISTANCE_CHANGED" + REPEATED_ACTION = "REPEATED_ACTION" + REPEATED_FAILURE_SIGNATURE = "REPEATED_FAILURE_SIGNATURE" + NO_NEW_EVIDENCE = "NO_NEW_EVIDENCE" + INSUFFICIENT_MEASUREMENT = "INSUFFICIENT_MEASUREMENT" + CONTRADICTORY_SIGNALS = "CONTRADICTORY_SIGNALS" + + +@dataclass(frozen=True) +class ProgressConfig: + """Explicit fail-closed measurement requirements.""" + + required_fields: tuple[str, ...] = ( + "useful_files", + "passing_tests", + "explained_failures", + "diff_distance", + "acceptance_distance", + "recent_actions", + "failure_signatures", + ) + + +DEFAULT_CONFIG = ProgressConfig() + + +def _sorted_unique(values: tuple[str, ...] | None) -> list[str] | None: + if values is None: + return None + return sorted(set(values)) + + +@dataclass(frozen=True) +class ProgressSnapshot: + useful_files: tuple[str, ...] | None = None + passing_tests: tuple[str, ...] | None = None + explained_failures: tuple[str, ...] | None = None + diff_distance: int | None = None + acceptance_distance: int | None = None + recent_actions: tuple[str, ...] | None = None + failure_signatures: tuple[str, ...] | None = None + + def normalized(self) -> "ProgressSnapshot": + return replace( + self, + useful_files=None if self.useful_files is None else tuple(sorted(set(self.useful_files))), + passing_tests=None if self.passing_tests is None else tuple(sorted(set(self.passing_tests))), + explained_failures=None if self.explained_failures is None else tuple(sorted(set(self.explained_failures))), + recent_actions=None if self.recent_actions is None else tuple(sorted(set(self.recent_actions))), + failure_signatures=None if self.failure_signatures is None else tuple(sorted(set(self.failure_signatures))), + ) + + def to_dict(self) -> dict[str, Any]: + value = self.normalized() + return { + "useful_files": _sorted_unique(value.useful_files), + "passing_tests": _sorted_unique(value.passing_tests), + "explained_failures": _sorted_unique(value.explained_failures), + "diff_distance": value.diff_distance, + "acceptance_distance": value.acceptance_distance, + "recent_actions": _sorted_unique(value.recent_actions), + "failure_signatures": _sorted_unique(value.failure_signatures), + } + + +@dataclass(frozen=True) +class ProgressEvidence: + useful_files_discovered: tuple[str, ...] | None + tests_passed: tuple[str, ...] | None + failures_explained: tuple[str, ...] | None + diff_distance: int | None + acceptance_distance: int | None + repeated_actions: int | None + failure_signatures: tuple[str, ...] | None + + def to_dict(self) -> dict[str, Any]: + return { + "useful_files_discovered": _sorted_unique(self.useful_files_discovered), + "tests_passed": _sorted_unique(self.tests_passed), + "failures_explained": _sorted_unique(self.failures_explained), + "diff_distance": self.diff_distance, + "acceptance_distance": self.acceptance_distance, + "repeated_actions": self.repeated_actions, + "failure_signatures": _sorted_unique(self.failure_signatures), + } + + +@dataclass(frozen=True) +class ProgressAssessment: + schema_version: int + status: ProgressStatus + confidence: Confidence + signals: tuple[str, ...] + reason_codes: tuple[ReasonCode, ...] + previous_snapshot: dict[str, Any] | None + current_snapshot: dict[str, Any] + evidence: ProgressEvidence + telemetry: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "status": self.status.value, + "confidence": self.confidence.value, + "signals": list(self.signals), + "reason_codes": [code.value for code in self.reason_codes], + "previous_snapshot": self.previous_snapshot, + "current_snapshot": self.current_snapshot, + "evidence": self.evidence.to_dict(), + "telemetry": self.telemetry, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _missing(snapshot: ProgressSnapshot, config: ProgressConfig) -> bool: + values = snapshot.to_dict() + return any(values.get(field) is None for field in config.required_fields) + + +def _contradictory(previous: ProgressSnapshot, current: ProgressSnapshot) -> bool: + distances = (previous.diff_distance, previous.acceptance_distance, current.diff_distance, current.acceptance_distance) + return any(isinstance(value, bool) or (value is not None and value < 0) for value in distances) + + +def assess_progress( + previous: ProgressSnapshot | None, + current: ProgressSnapshot, + *, + config: ProgressConfig = DEFAULT_CONFIG, + assessment_duration_ms: int | None = None, +) -> ProgressAssessment: + """Compare explicit snapshots without making any execution decision.""" + + current = current.normalized() + previous = None if previous is None else previous.normalized() + reason_codes: list[ReasonCode] = [] + + if previous is None or _missing(current, config) or _missing(previous, config): + reason_codes.append(ReasonCode.INSUFFICIENT_MEASUREMENT) + status = ProgressStatus.UNKNOWN + confidence = Confidence.LOW + evidence = ProgressEvidence( + useful_files_discovered=current.useful_files, + tests_passed=current.passing_tests, + failures_explained=current.explained_failures, + diff_distance=current.diff_distance, + acceptance_distance=current.acceptance_distance, + repeated_actions=None, + failure_signatures=current.failure_signatures, + ) + return _assessment( + status=status, + confidence=confidence, + reason_codes=reason_codes, + previous=previous, + current=current, + evidence=evidence, + duration_ms=assessment_duration_ms, + ) + + if _contradictory(previous, current): + reason_codes.append(ReasonCode.CONTRADICTORY_SIGNALS) + evidence = ProgressEvidence( + useful_files_discovered=current.useful_files, + tests_passed=current.passing_tests, + failures_explained=current.explained_failures, + diff_distance=current.diff_distance, + acceptance_distance=current.acceptance_distance, + repeated_actions=0, + failure_signatures=current.failure_signatures, + ) + return _assessment( + status=ProgressStatus.UNKNOWN, + confidence=Confidence.LOW, + reason_codes=reason_codes, + previous=previous, + current=current, + evidence=evidence, + duration_ms=assessment_duration_ms, + ) + + previous_files = set(previous.useful_files or ()) + current_files = set(current.useful_files or ()) + previous_tests = set(previous.passing_tests or ()) + current_tests = set(current.passing_tests or ()) + previous_explained = set(previous.explained_failures or ()) + current_explained = set(current.explained_failures or ()) + previous_failures = set(previous.failure_signatures or ()) + current_failures = set(current.failure_signatures or ()) + previous_actions = set(previous.recent_actions or ()) + current_actions = set(current.recent_actions or ()) + + new_files = current_files - previous_files + new_tests = current_tests - previous_tests + new_explained = current_explained - previous_explained + acceptance_reduced = current.acceptance_distance < previous.acceptance_distance + diff_changed = current.diff_distance != previous.diff_distance + diff_reduced = current.diff_distance < previous.diff_distance + repeated_actions = len(previous_actions & current_actions) + repeated_failure = bool(previous_failures & current_failures) + new_failure = bool(current_failures - previous_failures) + new_evidence = bool(new_files or new_tests or new_explained or acceptance_reduced or diff_reduced) + + if new_files: + reason_codes.append(ReasonCode.NEW_RELEVANT_FILE) + if new_tests: + reason_codes.append(ReasonCode.NEW_PASSING_TEST) + if new_explained: + reason_codes.append(ReasonCode.FAILURE_EXPLAINED) + if acceptance_reduced: + reason_codes.append(ReasonCode.ACCEPTANCE_DISTANCE_REDUCED) + if diff_changed: + reason_codes.append(ReasonCode.DIFF_DISTANCE_CHANGED) + if repeated_actions: + reason_codes.append(ReasonCode.REPEATED_ACTION) + if repeated_failure: + reason_codes.append(ReasonCode.REPEATED_FAILURE_SIGNATURE) + if not new_evidence: + reason_codes.append(ReasonCode.NO_NEW_EVIDENCE) + + if new_evidence: + status = ProgressStatus.PROGRESS_PROVEN + confidence = Confidence.HIGH if (new_tests or acceptance_reduced) else Confidence.MEDIUM + elif repeated_failure and not new_evidence: + status = ProgressStatus.NO_PROGRESS + confidence = Confidence.HIGH + elif repeated_actions and current.acceptance_distance == previous.acceptance_distance: + status = ProgressStatus.NO_PROGRESS + confidence = Confidence.HIGH + else: + status = ProgressStatus.PROGRESS_NOT_PROVEN + confidence = Confidence.MEDIUM if new_failure else Confidence.LOW + + evidence = ProgressEvidence( + useful_files_discovered=current.useful_files, + tests_passed=current.passing_tests, + failures_explained=current.explained_failures, + diff_distance=current.diff_distance, + acceptance_distance=current.acceptance_distance, + repeated_actions=repeated_actions, + failure_signatures=current.failure_signatures, + ) + return _assessment( + status=status, + confidence=confidence, + reason_codes=reason_codes, + previous=previous, + current=current, + evidence=evidence, + duration_ms=assessment_duration_ms, + ) + + +def _assessment( + *, + status: ProgressStatus, + confidence: Confidence, + reason_codes: list[ReasonCode], + previous: ProgressSnapshot | None, + current: ProgressSnapshot, + evidence: ProgressEvidence, + duration_ms: int | None, +) -> ProgressAssessment: + codes = tuple(dict.fromkeys(reason_codes)) + telemetry = { + "progress_status": status.value, + "progress_confidence": confidence.value, + "progress_signal_count": len(codes), + "repeated_action_count": evidence.repeated_actions, + "failure_signature_count": None if current.failure_signatures is None else len(current.failure_signatures), + "assessment_duration_ms": duration_ms, + } + return ProgressAssessment( + schema_version=SCHEMA_VERSION, + status=status, + confidence=confidence, + signals=[code.value for code in codes], + reason_codes=codes, + previous_snapshot=None if previous is None else previous.to_dict(), + current_snapshot=current.to_dict(), + evidence=evidence, + telemetry=telemetry, + ) + + +def assess_progress_timed( + previous: ProgressSnapshot | None, + current: ProgressSnapshot, + *, + config: ProgressConfig = DEFAULT_CONFIG, +) -> ProgressAssessment: + """Measure local assessment overhead; no action is taken.""" + + started = perf_counter_ns() + result = assess_progress(previous, current, config=config) + duration_ms = max(0, (perf_counter_ns() - started) // 1_000_000) + return replace(result, telemetry={**result.telemetry, "assessment_duration_ms": duration_ms}) + diff --git a/src/arkx/promotion.py b/src/arkx/promotion.py new file mode 100644 index 00000000..c9c0c929 --- /dev/null +++ b/src/arkx/promotion.py @@ -0,0 +1,238 @@ +"""Explicit evidence-backed promotion decisions.""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from enum import Enum +import hashlib +import json +from pathlib import Path +from typing import Any + +from .outcomes import AcceptanceDecision, AcceptanceResult, VerificationResult +from .harness import AttemptSnapshot, RunState, VerificationEvidence + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class PromotionStatus(_ValueEnum): + PROMOTED = "PROMOTED" + NOT_PROMOTED = "NOT_PROMOTED" + BLOCKED = "BLOCKED" + INDETERMINATE = "INDETERMINATE" + + +class PromotionReason(_ValueEnum): + PROMOTED_WITH_INDEPENDENT_EVIDENCE = "PROMOTED_WITH_INDEPENDENT_EVIDENCE" + ACCEPTANCE_REQUIRED = "ACCEPTANCE_REQUIRED" + ACCEPTANCE_AUTHORITY_MISMATCH = "ACCEPTANCE_AUTHORITY_MISMATCH" + EVIDENCE_MISSING = "EVIDENCE_MISSING" + EVIDENCE_INDETERMINATE = "EVIDENCE_INDETERMINATE" + CANDIDATE_IDENTITY_MISSING = "CANDIDATE_IDENTITY_MISSING" + VERIFICATION_REQUIRED = "VERIFICATION_REQUIRED" + VERIFICATION_EVIDENCE_MISSING = "VERIFICATION_EVIDENCE_MISSING" + VERIFICATION_INDETERMINATE = "VERIFICATION_INDETERMINATE" + EXECUTION_NOT_COMPLETED = "EXECUTION_NOT_COMPLETED" + SNAPSHOT_IDENTITY_MISMATCH = "SNAPSHOT_IDENTITY_MISMATCH" + ACCEPTANCE_VERIFICATION_MISMATCH = "ACCEPTANCE_VERIFICATION_MISMATCH" + + +@dataclass(frozen=True) +class PromotionCandidate: + component_type: str + component_id: str + version: str | None + configuration_digest: str | None + + def valid(self) -> bool: + return bool(self.component_type and self.component_id) + + def to_dict(self) -> dict[str, str | None]: + return { + "component_type": self.component_type, + "component_id": self.component_id, + "version": self.version, + "configuration_digest": self.configuration_digest, + } + + +@dataclass(frozen=True) +class PromotionPolicy: + authority_id: str + require_acceptance: bool = True + require_evidence: bool = True + require_verification: bool = True + + def __post_init__(self) -> None: + if not self.authority_id: + raise ValueError("promotion authority_id must be explicit") + + +@dataclass(frozen=True) +class PromotionDecision: + status: PromotionStatus + reason_codes: tuple[PromotionReason, ...] + candidate: PromotionCandidate + authority_id: str + acceptance: AcceptanceResult | None + evidence_refs: tuple[str, ...] + telemetry: dict[str, Any] = field(default_factory=dict) + attempt_id: str | None = None + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "status": self.status.value, + "reason_codes": [reason.value for reason in self.reason_codes], + "candidate": self.candidate.to_dict(), + "authority_id": self.authority_id, + "acceptance": None if self.acceptance is None else self.acceptance.to_dict(), + "evidence_refs": list(self.evidence_refs), + "telemetry": self.telemetry, + "attempt_id": self.attempt_id, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + @property + def reference(self) -> str: + payload = json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return f"promotion://{hashlib.sha256(payload.encode('utf-8')).hexdigest()[:16]}" + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "PromotionDecision": + acceptance_value = value.get("acceptance") + acceptance = None if acceptance_value is None else AcceptanceResult( + decision=AcceptanceDecision(acceptance_value["decision"]), + authority=str(acceptance_value["authority"]), + raw_outcome=acceptance_value.get("raw_outcome"), + evidence=tuple(acceptance_value.get("evidence", ())), + ) + candidate_value = value["candidate"] + return cls( + status=PromotionStatus(value["status"]), + reason_codes=tuple(PromotionReason(item) for item in value.get("reason_codes", ())), + candidate=PromotionCandidate(candidate_value["component_type"], candidate_value["component_id"], candidate_value.get("version"), candidate_value.get("configuration_digest")), + authority_id=str(value["authority_id"]), + acceptance=acceptance, + evidence_refs=tuple(value.get("evidence_refs", ())), + telemetry=dict(value.get("telemetry", {})), + attempt_id=value.get("attempt_id"), + schema_version=int(value.get("schema_version", 0)), + ) + + @classmethod + def from_json(cls, value: str) -> "PromotionDecision": + return cls.from_dict(json.loads(value)) + + +def decide_promotion( + candidate: PromotionCandidate, + acceptance: AcceptanceResult | None, + evidence_refs: tuple[str, ...] | None, + policy: PromotionPolicy, + verification: VerificationEvidence | None = None, +) -> PromotionDecision: + """Promote only an explicitly identified, independently accepted candidate.""" + + refs = tuple(sorted(set(evidence_refs or ()))) + if not candidate.valid(): + return PromotionDecision(PromotionStatus.BLOCKED, (PromotionReason.CANDIDATE_IDENTITY_MISSING,), candidate, policy.authority_id, acceptance, refs) + if policy.require_verification and verification is None: + return PromotionDecision(PromotionStatus.BLOCKED, (PromotionReason.VERIFICATION_REQUIRED,), candidate, policy.authority_id, acceptance, refs) + if verification is not None and verification.state != "PASS": + status = PromotionStatus.INDETERMINATE if verification.state in ("INDETERMINATE", "UNKNOWN") else PromotionStatus.NOT_PROMOTED + reason = PromotionReason.VERIFICATION_INDETERMINATE if status is PromotionStatus.INDETERMINATE else PromotionReason.VERIFICATION_REQUIRED + return PromotionDecision(status, (reason,), candidate, policy.authority_id, acceptance, refs) + if policy.require_verification and verification is not None and not verification.evidence_refs: + return PromotionDecision(PromotionStatus.BLOCKED, (PromotionReason.VERIFICATION_EVIDENCE_MISSING,), candidate, policy.authority_id, acceptance, refs) + if policy.require_acceptance and acceptance is None: + return PromotionDecision(PromotionStatus.BLOCKED, (PromotionReason.ACCEPTANCE_REQUIRED,), candidate, policy.authority_id, None, refs) + if acceptance is not None and acceptance.authority != policy.authority_id: + return PromotionDecision(PromotionStatus.BLOCKED, (PromotionReason.ACCEPTANCE_AUTHORITY_MISMATCH,), candidate, policy.authority_id, acceptance, refs) + if acceptance is not None and acceptance.decision is AcceptanceDecision.INDETERMINATE: + return PromotionDecision(PromotionStatus.INDETERMINATE, (PromotionReason.EVIDENCE_INDETERMINATE,), candidate, policy.authority_id, acceptance, refs) + if acceptance is not None and acceptance.decision is not AcceptanceDecision.ACCEPTED: + return PromotionDecision(PromotionStatus.NOT_PROMOTED, (PromotionReason.ACCEPTANCE_REQUIRED,), candidate, policy.authority_id, acceptance, refs) + if policy.require_evidence and not refs: + return PromotionDecision(PromotionStatus.BLOCKED, (PromotionReason.EVIDENCE_MISSING,), candidate, policy.authority_id, acceptance, refs) + return PromotionDecision(PromotionStatus.PROMOTED, (PromotionReason.PROMOTED_WITH_INDEPENDENT_EVIDENCE,), candidate, policy.authority_id, acceptance, refs) + + +def decide_outcome_promotion( + candidate: PromotionCandidate, + verification: VerificationResult, + acceptance: AcceptanceResult, + evidence_refs: tuple[str, ...] | None, + policy: PromotionPolicy, +) -> PromotionDecision: + """Promote only when acceptance explicitly carries its verification reference.""" + if verification.reference not in acceptance.evidence: + refs = tuple(sorted(set(evidence_refs or ()))) + return PromotionDecision( + PromotionStatus.BLOCKED, + (PromotionReason.ACCEPTANCE_VERIFICATION_MISMATCH,), + candidate, + policy.authority_id, + acceptance, + refs, + ) + verification_evidence = VerificationEvidence( + authority_identity=verification.authority, + state=verification.state.value, + evidence_refs=verification.evidence, + ) + refs = tuple(sorted(set((evidence_refs or ()) + verification.evidence + (verification.reference,)))) + return decide_promotion(candidate, acceptance, refs, policy, verification_evidence) + + +def decide_snapshot_promotion( + candidate: PromotionCandidate, + snapshot: AttemptSnapshot, + acceptance: AcceptanceResult | None, + evidence_refs: tuple[str, ...] | None, + policy: PromotionPolicy, + verification: VerificationEvidence | None = None, +) -> PromotionDecision: + """Apply promotion policy only to a validated, concrete attempt snapshot.""" + + if snapshot.manifest.state is not RunState.COMPLETED: + return PromotionDecision(PromotionStatus.BLOCKED, (PromotionReason.EXECUTION_NOT_COMPLETED,), candidate, policy.authority_id, acceptance, tuple(sorted(set(evidence_refs or ())))) + if candidate.configuration_digest and snapshot.manifest.configuration_digest and candidate.configuration_digest != snapshot.manifest.configuration_digest: + return PromotionDecision(PromotionStatus.BLOCKED, (PromotionReason.SNAPSHOT_IDENTITY_MISMATCH,), candidate, policy.authority_id, acceptance, tuple(sorted(set(evidence_refs or ())))) + refs = tuple(sorted(set((evidence_refs or ()) + (() if verification is None else verification.evidence_refs)))) + decision = decide_promotion(candidate, acceptance, refs, policy, verification) + return replace(decision, attempt_id=snapshot.manifest.attempt_id) + + +def write_promotion_decision(decision: PromotionDecision, path: str | Path) -> Path: + target = Path(path) + if target.exists(): + raise FileExistsError(f"promotion decision already exists: {target}") + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + ".tmp") + temporary.write_text(decision.to_json() + "\n", encoding="utf-8", newline="\n") + temporary.replace(target) + return target + + +def load_promotion_decision(path: str | Path, snapshot: AttemptSnapshot) -> PromotionDecision: + decision = PromotionDecision.from_json(Path(path).read_text(encoding="utf-8")) + if decision.attempt_id != snapshot.manifest.attempt_id: + raise ValueError("promotion decision attempt_id does not match snapshot") + if decision.candidate.configuration_digest and snapshot.manifest.configuration_digest and decision.candidate.configuration_digest != snapshot.manifest.configuration_digest: + raise ValueError("promotion decision candidate does not match snapshot configuration") + local_refs = set(snapshot.manifest.artifact_refs) + for reference in decision.evidence_refs: + if reference in local_refs or reference.startswith(("evidence://", "experiment://")): + continue + raise ValueError(f"promotion evidence reference is not bound to snapshot: {reference}") + return decision diff --git a/src/arkx/recovery.py b/src/arkx/recovery.py new file mode 100644 index 00000000..acd9e450 --- /dev/null +++ b/src/arkx/recovery.py @@ -0,0 +1,194 @@ +"""Evidence-driven recovery decisions for stagnation and failed progress.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +import hashlib +import json +from typing import Any + +from .handoff import HandoffBudgetStatus, HandoffRecord, HandoffSummary, summarize_handoffs +from .planning import ReplanRequest, ReplanTrigger +from .progress import ProgressAssessment, ProgressStatus +from .routing import EscalationAction, EscalationDecision, RecommendedPath +from .harness import derive_attempt_id + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class RecoveryAction(_ValueEnum): + CONTINUE = "CONTINUE" + REPLAN = "REPLAN" + ESCALATE = "ESCALATE" + HANDOFF = "HANDOFF" + BLOCK = "BLOCK" + + +class RecoveryReason(_ValueEnum): + PROGRESS_CONFIRMED = "PROGRESS_CONFIRMED" + NO_PROGRESS_REQUIRES_REPLAN = "NO_PROGRESS_REQUIRES_REPLAN" + PATH_ESCALATION_AUTHORIZED = "PATH_ESCALATION_AUTHORIZED" + QUALIFICATION_REQUIRED = "QUALIFICATION_REQUIRED" + INSUFFICIENT_EVIDENCE = "INSUFFICIENT_EVIDENCE" + HANDOFF_AUTHORIZED = "HANDOFF_AUTHORIZED" + HANDOFF_BUDGET_EXCEEDED = "HANDOFF_BUDGET_EXCEEDED" + HANDOFF_EVIDENCE_MISSING = "HANDOFF_EVIDENCE_MISSING" + RECOVERY_BUDGET_EXHAUSTED = "RECOVERY_BUDGET_EXHAUSTED" + + +@dataclass(frozen=True) +class RecoveryBudget: + max_replans: int = 1 + max_escalations: int = 2 + max_handoffs: int = 2 + + def valid(self) -> bool: + return all(value >= 0 for value in (self.max_replans, self.max_escalations, self.max_handoffs)) + + +@dataclass(frozen=True) +class RecoveryDecision: + action: RecoveryAction + reason_codes: tuple[RecoveryReason, ...] + current_path: RecommendedPath | None + target_path: RecommendedPath | None = None + replan_request: ReplanRequest | None = None + handoff_summary: HandoffSummary | None = None + telemetry: dict[str, Any] = field(default_factory=dict) + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "action": self.action.value, + "reason_codes": [reason.value for reason in self.reason_codes], + "current_path": None if self.current_path is None else self.current_path.value, + "target_path": None if self.target_path is None else self.target_path.value, + "replan_request": None if self.replan_request is None else self.replan_request.to_dict(), + "handoff_summary": None if self.handoff_summary is None else self.handoff_summary.to_dict(), + "telemetry": self.telemetry, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +@dataclass(frozen=True) +class RecoveryPlan: + plan_id: str + source_attempt_id: str + action: RecoveryAction + reason_codes: tuple[RecoveryReason, ...] + current_path: RecommendedPath | None + target_path: RecommendedPath | None + + def to_dict(self) -> dict[str, Any]: + return { + "plan_id": self.plan_id, + "source_attempt_id": self.source_attempt_id, + "action": self.action.value, + "reason_codes": [reason.value for reason in self.reason_codes], + "current_path": None if self.current_path is None else self.current_path.value, + "target_path": None if self.target_path is None else self.target_path.value, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + @property + def reference(self) -> str: + return f"recovery://{self.plan_id}" + + +@dataclass(frozen=True) +class RecoveryAttemptPlan: + trial_id: str + source_attempt_id: str + source_attempt_number: int + next_attempt_id: str + next_attempt_number: int + configuration_digest: str + recovery_reference: str + + def __post_init__(self) -> None: + if not self.trial_id.strip() or not self.source_attempt_id.strip() or not self.recovery_reference.strip() or not self.configuration_digest.strip(): + raise ValueError("recovery attempt identity fields must be non-empty") + if self.source_attempt_number < 1 or self.next_attempt_number != self.source_attempt_number + 1: + raise ValueError("recovery attempt numbers must be consecutive and positive") + expected_source = derive_attempt_id(trial_id=self.trial_id, attempt_number=self.source_attempt_number, configuration_digest=self.configuration_digest) + expected_next = derive_attempt_id(trial_id=self.trial_id, attempt_number=self.next_attempt_number, configuration_digest=self.configuration_digest) + if self.source_attempt_id != expected_source or self.next_attempt_id != expected_next: + raise ValueError("recovery attempt identity does not match trial and configuration") + + def to_dict(self) -> dict[str, Any]: + return {"trial_id": self.trial_id, "source_attempt_id": self.source_attempt_id, "source_attempt_number": self.source_attempt_number, "next_attempt_id": self.next_attempt_id, "next_attempt_number": self.next_attempt_number, "configuration_digest": self.configuration_digest, "recovery_reference": self.recovery_reference} + + +def plan_recovery_attempt(*, trial_id: str, source_attempt_id: str, source_attempt_number: int, configuration_digest: str, recovery_reference: str) -> RecoveryAttemptPlan: + """Derive a new attempt identity without executing or selecting an executor.""" + return RecoveryAttemptPlan( + trial_id, source_attempt_id, source_attempt_number, + derive_attempt_id(trial_id=trial_id, attempt_number=source_attempt_number + 1, configuration_digest=configuration_digest), + source_attempt_number + 1, configuration_digest, recovery_reference, + ) + + +def validate_recovery_reference(plan: RecoveryPlan, reference: str) -> None: + if reference != plan.reference: + raise ValueError("recovery reference does not match recovery plan") + + +def build_recovery_plan(source_attempt_id: str, decision: RecoveryDecision) -> RecoveryPlan: + if not source_attempt_id.strip(): + raise ValueError("source_attempt_id must be non-empty") + payload = json.dumps({"source_attempt_id": source_attempt_id, **decision.to_dict()}, sort_keys=True, separators=(",", ":")) + return RecoveryPlan( + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16], source_attempt_id, + decision.action, decision.reason_codes, decision.current_path, decision.target_path, + ) + + +def decide_recovery( + escalation: EscalationDecision, + progress: ProgressAssessment, + *, + budget: RecoveryBudget | None = None, + replans_used: int = 0, + escalations_used: int = 0, + replan_request: ReplanRequest | None = None, + handoffs: tuple[HandoffRecord, ...] = (), +) -> RecoveryDecision: + """Convert an escalation proposal into an explicit recovery artifact.""" + + budget = budget or RecoveryBudget() + if not budget.valid() or replans_used < 0 or escalations_used < 0: + return RecoveryDecision(RecoveryAction.BLOCK, (RecoveryReason.RECOVERY_BUDGET_EXHAUSTED,), escalation.current_path, telemetry={"invalid_budget": True}) + + handoff_summary = summarize_handoffs(handoffs) + if handoffs and handoff_summary.budget_status is HandoffBudgetStatus.BUDGET_EXCEEDED: + return RecoveryDecision(RecoveryAction.BLOCK, (RecoveryReason.HANDOFF_BUDGET_EXCEEDED,), escalation.current_path, handoff_summary=handoff_summary) + if handoffs and any(record.reason is None or not record.evidence_refs for record in handoffs): + return RecoveryDecision(RecoveryAction.BLOCK, (RecoveryReason.HANDOFF_EVIDENCE_MISSING,), escalation.current_path, handoff_summary=handoff_summary) + + if escalation.action is EscalationAction.STOP_SUFFICIENT_EVIDENCE or progress.status is ProgressStatus.PROGRESS_PROVEN: + return RecoveryDecision(RecoveryAction.CONTINUE, (RecoveryReason.PROGRESS_CONFIRMED,), escalation.current_path, escalation.current_path, handoff_summary=handoff_summary) + if escalation.action is EscalationAction.REQUIRE_QUALIFICATION: + return RecoveryDecision(RecoveryAction.BLOCK, (RecoveryReason.QUALIFICATION_REQUIRED,), escalation.current_path, handoff_summary=handoff_summary) + if escalation.action is EscalationAction.BLOCK: + return RecoveryDecision(RecoveryAction.BLOCK, (RecoveryReason.INSUFFICIENT_EVIDENCE,), escalation.current_path, escalation.target_path, handoff_summary=handoff_summary) + if escalation.action is EscalationAction.ESCALATE: + if escalations_used >= budget.max_escalations: + return RecoveryDecision(RecoveryAction.BLOCK, (RecoveryReason.RECOVERY_BUDGET_EXHAUSTED,), escalation.current_path, escalation.target_path, handoff_summary=handoff_summary) + return RecoveryDecision(RecoveryAction.ESCALATE, (RecoveryReason.PATH_ESCALATION_AUTHORIZED,), escalation.current_path, escalation.target_path, handoff_summary=handoff_summary) + if progress.status is ProgressStatus.NO_PROGRESS: + if replans_used >= budget.max_replans or replan_request is None: + return RecoveryDecision(RecoveryAction.BLOCK, (RecoveryReason.RECOVERY_BUDGET_EXHAUSTED,), escalation.current_path, handoff_summary=handoff_summary) + return RecoveryDecision(RecoveryAction.REPLAN, (RecoveryReason.NO_PROGRESS_REQUIRES_REPLAN,), escalation.current_path, escalation.target_path, replan_request, handoff_summary) + return RecoveryDecision(RecoveryAction.CONTINUE, (RecoveryReason.PROGRESS_CONFIRMED,), escalation.current_path, escalation.current_path, handoff_summary=handoff_summary) diff --git a/src/arkx/request.py b/src/arkx/request.py new file mode 100644 index 00000000..d9346b09 --- /dev/null +++ b/src/arkx/request.py @@ -0,0 +1,232 @@ +"""Request and governance contracts for the pre-characterization boundary. + +Governance authorizes the conditions under which a request may be analyzed. +It does not characterize the task, select an executor, verify a patch, or +accept a result. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +import json +from typing import Any + +from .characterization import CharacterizationConfig, TaskCharacterization, TaskSignals, characterize + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class GovernanceStatus(_ValueEnum): + AUTHORIZED = "AUTHORIZED" + BLOCKED = "BLOCKED" + UNKNOWN = "UNKNOWN" + + +class GovernanceReason(_ValueEnum): + AUTHORIZED = "AUTHORIZED" + REQUEST_ID_MISSING = "REQUEST_ID_MISSING" + TASK_ID_MISSING = "TASK_ID_MISSING" + REQUESTER_MISSING = "REQUESTER_MISSING" + AUTHORIZED_SCOPE_MISSING = "AUTHORIZED_SCOPE_MISSING" + ACCEPTANCE_AUTHORITY_MISSING = "ACCEPTANCE_AUTHORITY_MISSING" + ENVIRONMENT_MISSING = "ENVIRONMENT_MISSING" + WORKSPACE_MISSING = "WORKSPACE_MISSING" + REVISION_MISSING = "REVISION_MISSING" + ENVIRONMENT_NOT_ALLOWED = "ENVIRONMENT_NOT_ALLOWED" + BUDGET_MISSING = "BUDGET_MISSING" + BUDGET_INVALID = "BUDGET_INVALID" + BUDGET_EXCEEDED = "BUDGET_EXCEEDED" + REQUESTER_NOT_ALLOWED = "REQUESTER_NOT_ALLOWED" + PATH_OUTSIDE_AUTHORIZED_SCOPE = "PATH_OUTSIDE_AUTHORIZED_SCOPE" + + +@dataclass(frozen=True) +class RequestBudget: + max_attempts: int | None = None + max_tokens: int | None = None + max_wall_time_ms: int | None = None + + def valid(self) -> bool: + return all(value is None or value >= 0 for value in (self.max_attempts, self.max_tokens, self.max_wall_time_ms)) + + def to_dict(self) -> dict[str, int | None]: + return { + "max_attempts": self.max_attempts, + "max_tokens": self.max_tokens, + "max_wall_time_ms": self.max_wall_time_ms, + } + + +@dataclass(frozen=True) +class RequestEnvironment: + environment_id: str | None = None + workspace: str | None = None + revision: str | None = None + + def to_dict(self) -> dict[str, str | None]: + return { + "environment_id": self.environment_id, + "workspace": self.workspace, + "revision": self.revision, + } + + +@dataclass(frozen=True) +class ArkxRequest: + request_id: str | None + task_id: str | None + requester: str | None + signals: TaskSignals + authorized_paths: tuple[str, ...] | None + budget: RequestBudget | None + environment: RequestEnvironment | None + acceptance_authority: str | None + evidence_refs: tuple[str, ...] | None = None + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "request_id": self.request_id, + "task_id": self.task_id, + "requester": self.requester, + "signals": self.signals.to_dict(), + "authorized_paths": None if self.authorized_paths is None else sorted(set(self.authorized_paths)), + "budget": None if self.budget is None else self.budget.to_dict(), + "environment": None if self.environment is None else self.environment.to_dict(), + "acceptance_authority": self.acceptance_authority, + "evidence_refs": None if self.evidence_refs is None else sorted(set(self.evidence_refs)), + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +@dataclass(frozen=True) +class GovernancePolicy: + authority_id: str + allowed_requesters: tuple[str, ...] | None = None + allowed_environments: tuple[str, ...] | None = None + max_attempts: int | None = None + max_tokens: int | None = None + max_wall_time_ms: int | None = None + require_acceptance_authority: bool = True + + def to_dict(self) -> dict[str, Any]: + return { + "authority_id": self.authority_id, + "allowed_requesters": None if self.allowed_requesters is None else sorted(set(self.allowed_requesters)), + "allowed_environments": None if self.allowed_environments is None else sorted(set(self.allowed_environments)), + "max_attempts": self.max_attempts, + "max_tokens": self.max_tokens, + "max_wall_time_ms": self.max_wall_time_ms, + "require_acceptance_authority": self.require_acceptance_authority, + } + + +@dataclass(frozen=True) +class GovernanceDecision: + request_id: str | None + status: GovernanceStatus + authority_id: str + reason_codes: tuple[GovernanceReason, ...] + request: ArkxRequest | None = None + telemetry: dict[str, Any] = field(default_factory=dict) + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "request_id": self.request_id, + "status": self.status.value, + "authority_id": self.authority_id, + "reason_codes": [reason.value for reason in self.reason_codes], + "request": None if self.request is None else self.request.to_dict(), + "telemetry": self.telemetry, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _blocked(request: ArkxRequest, policy: GovernancePolicy, reasons: list[GovernanceReason]) -> GovernanceDecision: + unique = tuple(dict.fromkeys(reasons)) + return GovernanceDecision( + request_id=request.request_id, + status=GovernanceStatus.BLOCKED, + authority_id=policy.authority_id, + reason_codes=unique, + request=None, + telemetry={"governance_status": GovernanceStatus.BLOCKED.value, "reason_codes": [reason.value for reason in unique]}, + ) + + +def authorize_request(request: ArkxRequest, policy: GovernancePolicy) -> GovernanceDecision: + """Authorize processing boundaries without characterizing or executing a task.""" + + reasons: list[GovernanceReason] = [] + if not policy.authority_id: + raise ValueError("governance authority_id must be explicit") + if not request.request_id: + reasons.append(GovernanceReason.REQUEST_ID_MISSING) + if not request.task_id: + reasons.append(GovernanceReason.TASK_ID_MISSING) + if not request.requester: + reasons.append(GovernanceReason.REQUESTER_MISSING) + if request.authorized_paths is None or not request.authorized_paths: + reasons.append(GovernanceReason.AUTHORIZED_SCOPE_MISSING) + if policy.require_acceptance_authority and not request.acceptance_authority: + reasons.append(GovernanceReason.ACCEPTANCE_AUTHORITY_MISSING) + if request.environment is None or not request.environment.environment_id: + reasons.append(GovernanceReason.ENVIRONMENT_MISSING) + else: + if not request.environment.workspace: + reasons.append(GovernanceReason.WORKSPACE_MISSING) + if not request.environment.revision: + reasons.append(GovernanceReason.REVISION_MISSING) + if policy.allowed_environments and request.environment.environment_id not in policy.allowed_environments: + reasons.append(GovernanceReason.ENVIRONMENT_NOT_ALLOWED) + if request.budget is None: + reasons.append(GovernanceReason.BUDGET_MISSING) + elif not request.budget.valid(): + reasons.append(GovernanceReason.BUDGET_INVALID) + else: + limits = ((request.budget.max_attempts, policy.max_attempts), (request.budget.max_tokens, policy.max_tokens), (request.budget.max_wall_time_ms, policy.max_wall_time_ms)) + if any(value is not None and limit is not None and value > limit for value, limit in limits): + reasons.append(GovernanceReason.BUDGET_EXCEEDED) + if policy.allowed_requesters and request.requester not in policy.allowed_requesters: + reasons.append(GovernanceReason.REQUESTER_NOT_ALLOWED) + if request.authorized_paths and request.signals.candidate_files: + prefixes = tuple(path.rstrip("/") + "/" for path in request.authorized_paths) + if any(not any(file == path.rstrip("/") or file.startswith(path_prefix) for path, path_prefix in zip(request.authorized_paths, prefixes)) for file in request.signals.candidate_files): + reasons.append(GovernanceReason.PATH_OUTSIDE_AUTHORIZED_SCOPE) + + if reasons: + return _blocked(request, policy, reasons) + return GovernanceDecision( + request_id=request.request_id, + status=GovernanceStatus.AUTHORIZED, + authority_id=policy.authority_id, + reason_codes=(GovernanceReason.AUTHORIZED,), + request=request, + telemetry={"governance_status": GovernanceStatus.AUTHORIZED.value, "reason_codes": [GovernanceReason.AUTHORIZED.value]}, + ) + + +def characterize_authorized_request( + decision: GovernanceDecision, + *, + config: CharacterizationConfig | None = None, +) -> TaskCharacterization | None: + """Characterize only an authorized request; governance remains authoritative.""" + + if decision.status is not GovernanceStatus.AUTHORIZED or decision.request is None: + return None + return characterize(decision.request.signals, config=config or CharacterizationConfig()) diff --git a/src/arkx/routing.py b/src/arkx/routing.py new file mode 100644 index 00000000..78e5a6f7 --- /dev/null +++ b/src/arkx/routing.py @@ -0,0 +1,386 @@ +"""Bounded, rule-based routing and escalation policy for P3. + +The policy produces decisions only. A future layer may execute a decision; +this module does not know concrete executors and never produces PASS. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +import json +from typing import Any + +from .characterization import Confidence, RecommendedPath, Scope, TaskCharacterization +from .progress import ProgressAssessment, ProgressStatus +from .integration import AdapterPreflight, CapabilityProvenance, PreflightStatus + + +SCHEMA_VERSION = 1 +POLICY_VERSION = "p3.rule-based.v1" + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class RoutingDecisionType(_ValueEnum): + ROUTE = "ROUTE" + REQUIRE_QUALIFICATION = "REQUIRE_QUALIFICATION" + REJECT_INVALID_INPUT = "REJECT_INVALID_INPUT" + + +class EscalationAction(_ValueEnum): + CONTINUE = "CONTINUE" + STOP_SUFFICIENT_EVIDENCE = "STOP_SUFFICIENT_EVIDENCE" + ESCALATE = "ESCALATE" + BLOCK = "BLOCK" + REQUIRE_QUALIFICATION = "REQUIRE_QUALIFICATION" + + +class EvidenceSufficiency(_ValueEnum): + SUFFICIENT = "SUFFICIENT" + INSUFFICIENT = "INSUFFICIENT" + UNKNOWN = "UNKNOWN" + + +class ReasonCode(_ValueEnum): + CHARACTERIZED_SIMPLE = "CHARACTERIZED_SIMPLE" + CHARACTERIZED_LOCALIZED = "CHARACTERIZED_LOCALIZED" + CHARACTERIZED_REPOSITORY_WIDE = "CHARACTERIZED_REPOSITORY_WIDE" + CHARACTERIZATION_UNCERTAIN = "CHARACTERIZATION_UNCERTAIN" + PROGRESS_CONFIRMED = "PROGRESS_CONFIRMED" + NO_PROGRESS_DETECTED = "NO_PROGRESS_DETECTED" + PROGRESS_UNKNOWN = "PROGRESS_UNKNOWN" + EVIDENCE_SUFFICIENT = "EVIDENCE_SUFFICIENT" + EVIDENCE_INSUFFICIENT = "EVIDENCE_INSUFFICIENT" + EVIDENCE_UNKNOWN = "EVIDENCE_UNKNOWN" + ESCALATION_BUDGET_AVAILABLE = "ESCALATION_BUDGET_AVAILABLE" + ESCALATION_BUDGET_EXHAUSTED = "ESCALATION_BUDGET_EXHAUSTED" + ATTEMPT_BUDGET_AVAILABLE = "ATTEMPT_BUDGET_AVAILABLE" + ATTEMPT_BUDGET_EXHAUSTED = "ATTEMPT_BUDGET_EXHAUSTED" + NO_HIGHER_PATH = "NO_HIGHER_PATH" + INVALID_PATH_TRANSITION = "INVALID_PATH_TRANSITION" + INVALID_CHARACTERIZATION = "INVALID_CHARACTERIZATION" + + +class CapabilityPolicyReason(_ValueEnum): + CAPABILITY_READY = "CAPABILITY_READY" + PREFLIGHT_NOT_READY = "PREFLIGHT_NOT_READY" + CAPABILITY_PROVENANCE_INSUFFICIENT = "CAPABILITY_PROVENANCE_INSUFFICIENT" + CAPABILITY_MISSING = "CAPABILITY_MISSING" + CAPABILITY_IDENTITY_MISSING = "CAPABILITY_IDENTITY_MISSING" + + +@dataclass(frozen=True) +class CapabilityPolicyDecision: + allowed: bool + reason: CapabilityPolicyReason + required: tuple[str, ...] + missing: tuple[str, ...] = () + required_provenance: CapabilityProvenance = CapabilityProvenance.DECLARED + observed_provenance: CapabilityProvenance | None = None + + +def assess_capability_policy( + preflight: AdapterPreflight, + required: tuple[str, ...], + *, + minimum_provenance: CapabilityProvenance = CapabilityProvenance.DECLARED, +) -> CapabilityPolicyDecision: + """Authorize capability use without selecting an executor or fallback.""" + required = tuple(sorted(set(required))) + if preflight.status is not PreflightStatus.READY: + return CapabilityPolicyDecision(False, CapabilityPolicyReason.PREFLIGHT_NOT_READY, required, required_provenance=minimum_provenance, observed_provenance=preflight.capability_provenance) + if not preflight.capability_digest: + return CapabilityPolicyDecision(False, CapabilityPolicyReason.CAPABILITY_IDENTITY_MISSING, required, required_provenance=minimum_provenance, observed_provenance=preflight.capability_provenance) + rank = {CapabilityProvenance.DECLARED: 0, CapabilityProvenance.OBSERVED: 1, CapabilityProvenance.QUALIFIED: 2} + if rank[preflight.capability_provenance] < rank[minimum_provenance]: + return CapabilityPolicyDecision(False, CapabilityPolicyReason.CAPABILITY_PROVENANCE_INSUFFICIENT, required, required_provenance=minimum_provenance, observed_provenance=preflight.capability_provenance) + missing = tuple(sorted(set(required) - set(preflight.capabilities))) + if missing: + return CapabilityPolicyDecision(False, CapabilityPolicyReason.CAPABILITY_MISSING, required, missing, minimum_provenance, preflight.capability_provenance) + return CapabilityPolicyDecision(True, CapabilityPolicyReason.CAPABILITY_READY, required, required_provenance=minimum_provenance, observed_provenance=preflight.capability_provenance) + + +PATH_LADDER: tuple[RecommendedPath, ...] = ( + RecommendedPath.SIMPLE_PATH, + RecommendedPath.LOCALIZED_PATH, + RecommendedPath.REPOSITORY_WIDE_PATH, +) + + +@dataclass(frozen=True) +class RoutingBudget: + max_path_escalations: int = 2 + max_attempts: int = 3 + + def to_dict(self) -> dict[str, int]: + return { + "max_path_escalations": self.max_path_escalations, + "max_attempts": self.max_attempts, + } + + +@dataclass(frozen=True) +class BudgetState: + budget: RoutingBudget = field(default_factory=RoutingBudget) + attempts_used: int = 0 + path_escalations_used: int = 0 + + @property + def attempts_available(self) -> bool: + return self.attempts_used < self.budget.max_attempts + + @property + def escalation_available(self) -> bool: + return self.path_escalations_used < self.budget.max_path_escalations + + def after(self, action: EscalationAction) -> "BudgetState": + if action is EscalationAction.ESCALATE: + return BudgetState( + budget=self.budget, + attempts_used=self.attempts_used, + path_escalations_used=self.path_escalations_used + 1, + ) + return self + + def to_dict(self) -> dict[str, Any]: + return { + **self.budget.to_dict(), + "attempts_used": self.attempts_used, + "path_escalations_used": self.path_escalations_used, + } + + +@dataclass(frozen=True) +class RoutingDecision: + schema_version: int + task_id: str + initial_path: RecommendedPath | None + selected_path: RecommendedPath | None + decision: RoutingDecisionType + confidence: Confidence + reason_codes: tuple[ReasonCode, ...] + characterization_ref: str | None + policy_version: str + budget: BudgetState + telemetry: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "task_id": self.task_id, + "initial_path": None if self.initial_path is None else self.initial_path.value, + "selected_path": None if self.selected_path is None else self.selected_path.value, + "decision": self.decision.value, + "confidence": self.confidence.value, + "reason_codes": [code.value for code in self.reason_codes], + "characterization_ref": self.characterization_ref, + "policy_version": self.policy_version, + "budget": self.budget.to_dict(), + "telemetry": self.telemetry, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +@dataclass(frozen=True) +class EscalationDecision: + schema_version: int + current_path: RecommendedPath | None + action: EscalationAction + target_path: RecommendedPath | None + reason_codes: tuple[ReasonCode, ...] + progress_status: ProgressStatus + evidence_sufficiency: EvidenceSufficiency + budget_state: BudgetState + telemetry: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "current_path": None if self.current_path is None else self.current_path.value, + "action": self.action.value, + "target_path": None if self.target_path is None else self.target_path.value, + "reason_codes": [code.value for code in self.reason_codes], + "progress_status": self.progress_status.value, + "evidence_sufficiency": self.evidence_sufficiency.value, + "budget_state": self.budget_state.to_dict(), + "telemetry": self.telemetry, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def route_characterization( + task_id: str, + characterization: TaskCharacterization | None, + *, + budget_state: BudgetState | None = None, + characterization_ref: str | None = None, +) -> RoutingDecision: + """Map P1's closed classification to a path without hidden inference.""" + + budget_state = budget_state or BudgetState() + if characterization is None: + return _routing( + task_id=task_id, + initial_path=None, + selected_path=None, + decision=RoutingDecisionType.REQUIRE_QUALIFICATION, + confidence=Confidence.LOW, + reason_codes=(ReasonCode.CHARACTERIZATION_UNCERTAIN,), + characterization_ref=characterization_ref, + budget=budget_state, + ) + + expected = { + Scope.SIMPLE: RecommendedPath.SIMPLE_PATH, + Scope.LOCALIZED: RecommendedPath.LOCALIZED_PATH, + Scope.REPOSITORY_WIDE: RecommendedPath.REPOSITORY_WIDE_PATH, + Scope.UNKNOWN: RecommendedPath.QUALIFICATION_REQUIRED, + }[characterization.scope] + if expected is RecommendedPath.QUALIFICATION_REQUIRED: + return _routing( + task_id=task_id, + initial_path=expected, + selected_path=expected, + decision=RoutingDecisionType.REQUIRE_QUALIFICATION, + confidence=Confidence.LOW, + reason_codes=(ReasonCode.CHARACTERIZATION_UNCERTAIN,), + characterization_ref=characterization_ref, + budget=budget_state, + ) + if characterization.recommended_path is not expected: + return _routing( + task_id=task_id, + initial_path=expected, + selected_path=None, + decision=RoutingDecisionType.REJECT_INVALID_INPUT, + confidence=Confidence.LOW, + reason_codes=(ReasonCode.INVALID_CHARACTERIZATION,), + characterization_ref=characterization_ref, + budget=budget_state, + ) + + reason = { + Scope.SIMPLE: ReasonCode.CHARACTERIZED_SIMPLE, + Scope.LOCALIZED: ReasonCode.CHARACTERIZED_LOCALIZED, + Scope.REPOSITORY_WIDE: ReasonCode.CHARACTERIZED_REPOSITORY_WIDE, + }[characterization.scope] + return _routing( + task_id=task_id, + initial_path=expected, + selected_path=expected, + decision=RoutingDecisionType.ROUTE, + confidence=characterization.confidence, + reason_codes=(reason,), + characterization_ref=characterization_ref, + budget=budget_state, + ) + + +def _routing(**values: Any) -> RoutingDecision: + decision = values["decision"] + return RoutingDecision( + schema_version=SCHEMA_VERSION, + policy_version=POLICY_VERSION, + telemetry={ + "initial_path": None if values["initial_path"] is None else values["initial_path"].value, + "selected_path": None if values["selected_path"] is None else values["selected_path"].value, + "routing_decision": decision.value, + "escalation_action": None, + "escalation_count": values["budget"].path_escalations_used, + "attempt_count": values["budget"].attempts_used, + "routing_reason_codes": [code.value for code in values["reason_codes"]], + }, + **values, + ) + + +def _next_path(path: RecommendedPath | None) -> RecommendedPath | None: + if path not in PATH_LADDER: + return None + index = PATH_LADDER.index(path) + return PATH_LADDER[index + 1] if index + 1 < len(PATH_LADDER) else None + + +def assess_escalation( + current_path: RecommendedPath | None, + progress: ProgressAssessment, + evidence: EvidenceSufficiency, + *, + budget_state: BudgetState | None = None, +) -> EscalationDecision: + """Produce a bounded action proposal; never execute the proposal.""" + + budget_state = budget_state or BudgetState() + reasons: list[ReasonCode] = [] + target_path: RecommendedPath | None = None + + if current_path not in PATH_LADDER: + reasons.append(ReasonCode.INVALID_PATH_TRANSITION) + return _escalation(current_path, EscalationAction.BLOCK, target_path, reasons, progress, evidence, budget_state) + + if evidence is EvidenceSufficiency.SUFFICIENT and progress.status is ProgressStatus.PROGRESS_PROVEN: + reasons.extend((ReasonCode.PROGRESS_CONFIRMED, ReasonCode.EVIDENCE_SUFFICIENT)) + return _escalation(current_path, EscalationAction.STOP_SUFFICIENT_EVIDENCE, current_path, reasons, progress, evidence, budget_state) + + if evidence is EvidenceSufficiency.UNKNOWN: + reasons.append(ReasonCode.EVIDENCE_UNKNOWN) + return _escalation(current_path, EscalationAction.REQUIRE_QUALIFICATION, target_path, reasons, progress, evidence, budget_state) + if progress.status is ProgressStatus.UNKNOWN: + reasons.append(ReasonCode.PROGRESS_UNKNOWN) + return _escalation(current_path, EscalationAction.REQUIRE_QUALIFICATION, target_path, reasons, progress, evidence, budget_state) + + if evidence is EvidenceSufficiency.SUFFICIENT: + reasons.append(ReasonCode.EVIDENCE_SUFFICIENT) + else: + reasons.append(ReasonCode.EVIDENCE_INSUFFICIENT) + + if progress.status is ProgressStatus.NO_PROGRESS: + target_path = _next_path(current_path) + if target_path is None: + reasons.append(ReasonCode.NO_HIGHER_PATH) + return _escalation(current_path, EscalationAction.BLOCK, target_path, reasons, progress, evidence, budget_state) + if not budget_state.escalation_available: + reasons.append(ReasonCode.ESCALATION_BUDGET_EXHAUSTED) + return _escalation(current_path, EscalationAction.BLOCK, target_path, reasons, progress, evidence, budget_state) + reasons.extend((ReasonCode.NO_PROGRESS_DETECTED, ReasonCode.ESCALATION_BUDGET_AVAILABLE)) + return _escalation(current_path, EscalationAction.ESCALATE, target_path, reasons, progress, evidence, budget_state) + + if progress.status is ProgressStatus.PROGRESS_PROVEN: + reasons.append(ReasonCode.PROGRESS_CONFIRMED) + if not budget_state.attempts_available: + reasons.append(ReasonCode.ATTEMPT_BUDGET_EXHAUSTED) + return _escalation(current_path, EscalationAction.BLOCK, target_path, reasons, progress, evidence, budget_state) + reasons.append(ReasonCode.ATTEMPT_BUDGET_AVAILABLE) + return _escalation(current_path, EscalationAction.CONTINUE, current_path, reasons, progress, evidence, budget_state) + + +def _escalation(current_path, action, target_path, reasons, progress, evidence, budget_state): + result_state = budget_state.after(action) + codes = tuple(dict.fromkeys(reasons)) + return EscalationDecision( + schema_version=SCHEMA_VERSION, + current_path=current_path, + action=action, + target_path=target_path, + reason_codes=codes, + progress_status=progress.status, + evidence_sufficiency=evidence, + budget_state=result_state, + telemetry={ + "initial_path": None if current_path is None else current_path.value, + "selected_path": None if target_path is None else target_path.value, + "routing_decision": None, + "escalation_action": action.value, + "escalation_count": result_state.path_escalations_used, + "attempt_count": result_state.attempts_used, + "routing_reason_codes": [code.value for code in codes], + }, + ) diff --git a/src/arkx/selection.py b/src/arkx/selection.py new file mode 100644 index 00000000..5b6a2b2b --- /dev/null +++ b/src/arkx/selection.py @@ -0,0 +1,196 @@ +"""Explicit treatment, capability and executor selection contracts. + +Selection happens after routing and never falls back silently. Qualification +evidence is required before an executor can be selected for a treatment. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +import json +from typing import Any + +from .characterization import RecommendedPath +from .composition import Mechanism, Treatment +from .executor_qualification import ExecutorIdentity, QualificationStatus + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class SelectionStatus(_ValueEnum): + SELECTED = "SELECTED" + BLOCKED = "BLOCKED" + REQUIRE_QUALIFICATION = "REQUIRE_QUALIFICATION" + UNKNOWN = "UNKNOWN" + + +class SelectionReason(_ValueEnum): + SELECTED = "SELECTED" + NO_TREATMENT_FOR_PATH = "NO_TREATMENT_FOR_PATH" + AMBIGUOUS_TREATMENT = "AMBIGUOUS_TREATMENT" + TREATMENT_NOT_FOUND = "TREATMENT_NOT_FOUND" + NO_EXECUTOR_FOR_CAPABILITY = "NO_EXECUTOR_FOR_CAPABILITY" + EXECUTOR_NOT_QUALIFIED = "EXECUTOR_NOT_QUALIFIED" + QUALIFICATION_EVIDENCE_MISSING = "QUALIFICATION_EVIDENCE_MISSING" + AMBIGUOUS_EXECUTOR = "AMBIGUOUS_EXECUTOR" + CAPABILITY_MISMATCH = "CAPABILITY_MISMATCH" + + +def _sorted_unique(values: tuple[str, ...]) -> tuple[str, ...]: + return tuple(sorted(set(values))) + + +@dataclass(frozen=True) +class TreatmentDefinition: + name: str + path: RecommendedPath + required_capabilities: tuple[str, ...] = () + mechanisms: tuple[Mechanism, ...] = () + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + if not self.name: + raise ValueError("Treatment name must be non-empty") + if any(not value for value in self.required_capabilities): + raise ValueError("Required capability names must be non-empty") + object.__setattr__(self, "required_capabilities", _sorted_unique(self.required_capabilities)) + object.__setattr__(self, "mechanisms", tuple(sorted(set(self.mechanisms), key=lambda item: item.value))) + + def to_treatment(self) -> Treatment: + return Treatment(self.name, self.mechanisms) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "name": self.name, + "path": self.path.value, + "required_capabilities": list(self.required_capabilities), + "mechanisms": [mechanism.value for mechanism in self.mechanisms], + } + + +@dataclass(frozen=True) +class TreatmentCatalog: + treatments: tuple[TreatmentDefinition, ...] + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + values = tuple(sorted(self.treatments, key=lambda item: item.name)) + if len({item.name for item in values}) != len(values): + raise ValueError("Treatment names must be unique") + object.__setattr__(self, "treatments", values) + + def to_dict(self) -> dict[str, Any]: + return {"schema_version": self.schema_version, "treatments": [item.to_dict() for item in self.treatments]} + + +@dataclass(frozen=True) +class ExecutorBinding: + executor: ExecutorIdentity + capabilities: tuple[str, ...] + qualification_status: QualificationStatus + evidence_refs: tuple[str, ...] | None + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + if any(not value for value in self.capabilities): + raise ValueError("Executor capability names must be non-empty") + object.__setattr__(self, "capabilities", _sorted_unique(self.capabilities)) + if self.evidence_refs is not None: + object.__setattr__(self, "evidence_refs", _sorted_unique(self.evidence_refs)) + + def supports(self, required: tuple[str, ...]) -> bool: + return set(required).issubset(self.capabilities) + + def empirically_qualified(self) -> bool: + return self.qualification_status is QualificationStatus.QUALIFIABLE and bool(self.evidence_refs) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "executor": self.executor.to_dict(), + "capabilities": list(self.capabilities), + "qualification_status": self.qualification_status.value, + "evidence_refs": None if self.evidence_refs is None else list(self.evidence_refs), + } + + +@dataclass(frozen=True) +class ExecutorRegistry: + bindings: tuple[ExecutorBinding, ...] + schema_version: int = SCHEMA_VERSION + + def __post_init__(self) -> None: + values = tuple(sorted(self.bindings, key=lambda item: (item.executor.name, item.executor.version or ""))) + identities = [(item.executor.name, item.executor.version, item.executor.configuration_digest) for item in values] + if len(set(identities)) != len(identities): + raise ValueError("Executor bindings must have unique identities") + object.__setattr__(self, "bindings", values) + + def to_dict(self) -> dict[str, Any]: + return {"schema_version": self.schema_version, "bindings": [item.to_dict() for item in self.bindings]} + + +@dataclass(frozen=True) +class SelectionDecision: + status: SelectionStatus + reason_codes: tuple[SelectionReason, ...] + path: RecommendedPath | None = None + treatment: TreatmentDefinition | None = None + executor: ExecutorBinding | None = None + telemetry: dict[str, Any] = field(default_factory=dict) + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "status": self.status.value, + "reason_codes": [reason.value for reason in self.reason_codes], + "path": None if self.path is None else self.path.value, + "treatment": None if self.treatment is None else self.treatment.to_dict(), + "executor": None if self.executor is None else self.executor.to_dict(), + "telemetry": self.telemetry, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def select_treatment( + path: RecommendedPath, + catalog: TreatmentCatalog, + *, + treatment_name: str | None = None, +) -> SelectionDecision: + candidates = tuple(item for item in catalog.treatments if item.path is path) + if treatment_name is not None: + candidates = tuple(item for item in candidates if item.name == treatment_name) + if not candidates: + return SelectionDecision(SelectionStatus.BLOCKED, (SelectionReason.TREATMENT_NOT_FOUND,), path=path) + if not candidates: + return SelectionDecision(SelectionStatus.BLOCKED, (SelectionReason.NO_TREATMENT_FOR_PATH,), path=path) + if len(candidates) > 1: + return SelectionDecision(SelectionStatus.REQUIRE_QUALIFICATION, (SelectionReason.AMBIGUOUS_TREATMENT,), path=path) + selected = candidates[0] + return SelectionDecision(SelectionStatus.SELECTED, (SelectionReason.SELECTED,), path=path, treatment=selected) + + +def select_executor(treatment: TreatmentDefinition, registry: ExecutorRegistry) -> SelectionDecision: + candidates = tuple(binding for binding in registry.bindings if binding.supports(treatment.required_capabilities)) + if not candidates: + return SelectionDecision(SelectionStatus.BLOCKED, (SelectionReason.NO_EXECUTOR_FOR_CAPABILITY,), path=treatment.path, treatment=treatment) + qualified = tuple(binding for binding in candidates if binding.empirically_qualified()) + if not qualified: + reason = SelectionReason.QUALIFICATION_EVIDENCE_MISSING if any(binding.evidence_refs in (None, ()) for binding in candidates) else SelectionReason.EXECUTOR_NOT_QUALIFIED + return SelectionDecision(SelectionStatus.REQUIRE_QUALIFICATION, (reason,), path=treatment.path, treatment=treatment) + if len(qualified) > 1: + return SelectionDecision(SelectionStatus.REQUIRE_QUALIFICATION, (SelectionReason.AMBIGUOUS_EXECUTOR,), path=treatment.path, treatment=treatment) + selected = qualified[0] + return SelectionDecision(SelectionStatus.SELECTED, (SelectionReason.SELECTED,), path=treatment.path, treatment=treatment, executor=selected) diff --git a/src/arkx/swebench_authority.py b/src/arkx/swebench_authority.py new file mode 100644 index 00000000..e079e339 --- /dev/null +++ b/src/arkx/swebench_authority.py @@ -0,0 +1,255 @@ +"""Official SWE-bench Verified acquisition and acceptance adapter. + +The dataset loader is the only source of task metadata. The gold ``patch`` +field is retained only in a provenance record and is never included in the +executor payload or prediction produced by this module. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +import json +from pathlib import Path +from typing import Any, Callable, Iterable + +from .harness import ErrorDomain, ErrorEnvelope, Retryability +from .outcomes import AcceptanceDecision, AcceptanceResult, VerificationResult, VerificationState + + +DATASET_IDENTITY = "SWE-bench/SWE-bench_Verified" +DATASET_SPLIT = "test" +AUTHORITY_IDENTITY = "swebench.harness.run_evaluation" + + +class OfficialEvaluationStatus(str, Enum): + RESOLVED = "RESOLVED" + TESTS_FAILED = "TESTS_FAILED" + INFRASTRUCTURE_ERROR = "INFRASTRUCTURE_ERROR" + AMBIGUOUS = "AMBIGUOUS" + NOT_EXECUTED = "NOT_EXECUTED" + BLOCKED = "BLOCKED" + + +@dataclass(frozen=True) +class SWEbenchRecord: + instance_id: str + repo: str + base_commit: str + problem_statement: str + version: str + environment_setup_commit: str | None + fail_to_pass: tuple[str, ...] + pass_to_pass: tuple[str, ...] + test_patch: str + gold_patch: str | None + dataset_revision: str | None + retrieved_at: str + + @classmethod + def from_row(cls, row: dict[str, Any], *, revision: str | None, retrieved_at: str) -> "SWEbenchRecord": + required = ("instance_id", "repo", "base_commit", "problem_statement", "version", "test_patch") + missing = [name for name in required if row.get(name) is None] + if missing: + raise ValueError(f"SWE-bench record missing fields: {','.join(missing)}") + return cls( + instance_id=row["instance_id"], repo=row["repo"], base_commit=row["base_commit"], + problem_statement=row["problem_statement"], version=row["version"], + environment_setup_commit=row.get("environment_setup_commit"), + fail_to_pass=tuple(row.get("FAIL_TO_PASS") or ()), + pass_to_pass=tuple(row.get("PASS_TO_PASS") or ()), test_patch=row["test_patch"], + gold_patch=row.get("patch"), dataset_revision=revision, retrieved_at=retrieved_at, + ) + + def executor_payload(self) -> dict[str, Any]: + """Return inference input; gold patch and grading labels are excluded.""" + return {"instance_id": self.instance_id, "repo": self.repo, "base_commit": self.base_commit, + "problem_statement": self.problem_statement, "version": self.version, + "environment_setup_commit": self.environment_setup_commit} + + def to_dict(self, include_gold_provenance: bool = True) -> dict[str, Any]: + value = {"instance_id": self.instance_id, "repo": self.repo, "base_commit": self.base_commit, + "problem_statement": self.problem_statement, "version": self.version, + "environment_setup_commit": self.environment_setup_commit, + "FAIL_TO_PASS": list(self.fail_to_pass), "PASS_TO_PASS": list(self.pass_to_pass), + "test_patch": self.test_patch, "dataset_revision": self.dataset_revision, + "retrieved_at": self.retrieved_at} + if include_gold_provenance: + value["gold_patch_sha256"] = None if self.gold_patch is None else _sha256(self.gold_patch) + return value + + +def _sha256(value: str) -> str: + import hashlib + return hashlib.sha256(value.encode()).hexdigest() + + +def load_verified_records(instance_ids: Iterable[str], *, revision: str | None = None) -> tuple[SWEbenchRecord, ...]: + """Load exact public records from Hugging Face; no local dataset is invented.""" + from datasets import load_dataset + wanted = set(instance_ids) + if not wanted: + raise ValueError("at least one instance_id is required") + retrieved_at = datetime.now(timezone.utc).isoformat(timespec="seconds") + found: list[SWEbenchRecord] = [] + for row in load_dataset(DATASET_IDENTITY, split=DATASET_SPLIT, streaming=True): + if row["instance_id"] in wanted: + found.append(SWEbenchRecord.from_row(row, revision=revision, retrieved_at=retrieved_at)) + if len(found) == len(wanted): + break + if {item.instance_id for item in found} != wanted: + raise LookupError("requested instance_ids were not all found in official test split") + return tuple(sorted(found, key=lambda item: item.instance_id)) + + +@dataclass(frozen=True) +class SWEbenchPrediction: + instance_id: str + model_name_or_path: str + model_patch: str + + def to_dict(self) -> dict[str, str]: + return {"instance_id": self.instance_id, "model_name_or_path": self.model_name_or_path, + "model_patch": self.model_patch} + + +@dataclass(frozen=True) +class OfficialEvaluationResult: + status: OfficialEvaluationStatus + authority: str + run_id: str + instance_id: str + raw_outcome: str | None + report_path: str | None + log_paths: tuple[str, ...] + error: str | None = None + + def verification_result(self) -> VerificationResult: + """Convert the evaluator observation into a neutral verification outcome.""" + mapping = { + OfficialEvaluationStatus.RESOLVED: VerificationState.PASS, + OfficialEvaluationStatus.TESTS_FAILED: VerificationState.FAIL, + OfficialEvaluationStatus.INFRASTRUCTURE_ERROR: VerificationState.INDETERMINATE, + OfficialEvaluationStatus.AMBIGUOUS: VerificationState.INDETERMINATE, + OfficialEvaluationStatus.NOT_EXECUTED: VerificationState.NOT_EXECUTED, + OfficialEvaluationStatus.BLOCKED: VerificationState.BLOCKED, + } + evidence = tuple(sorted(path for path in (self.report_path, *self.log_paths) if path)) + outcomes = tuple(value for value in (self.raw_outcome, self.error) if value) + return VerificationResult( + mapping[self.status], + self.authority, + ("swebench.harness.run_evaluation",), + outcomes or (self.status.value,), + evidence, + ) + + def error_envelope(self) -> ErrorEnvelope | None: + """Expose evaluator/infrastructure failure without turning test failure into an error.""" + if self.status in (OfficialEvaluationStatus.RESOLVED, OfficialEvaluationStatus.TESTS_FAILED): + return None + references = tuple(path for path in (self.report_path, *self.log_paths) if path) + message = self.error or self.raw_outcome or f"official evaluation status: {self.status.value}" + return ErrorEnvelope( + ErrorDomain.VERIFICATION, + f"EVALUATOR_{self.status.value}", + message, + Retryability.UNKNOWN, + references, + ) + + def acceptance(self) -> AcceptanceResult: + mapping = { + OfficialEvaluationStatus.RESOLVED: AcceptanceDecision.ACCEPTED, + OfficialEvaluationStatus.TESTS_FAILED: AcceptanceDecision.REJECTED, + OfficialEvaluationStatus.INFRASTRUCTURE_ERROR: AcceptanceDecision.INDETERMINATE, + OfficialEvaluationStatus.AMBIGUOUS: AcceptanceDecision.INDETERMINATE, + OfficialEvaluationStatus.NOT_EXECUTED: AcceptanceDecision.NOT_EXECUTED, + OfficialEvaluationStatus.BLOCKED: AcceptanceDecision.BLOCKED, + } + return AcceptanceResult(mapping[self.status], self.authority, self.raw_outcome, + tuple(path for path in (self.report_path, *self.log_paths) if path)) + + def to_dict(self) -> dict[str, Any]: + return {"status": self.status.value, "authority": self.authority, "run_id": self.run_id, + "instance_id": self.instance_id, "raw_outcome": self.raw_outcome, + "report_path": self.report_path, "log_paths": list(self.log_paths), "error": self.error} + + +OfficialEvaluator = Callable[[dict[str, Any]], OfficialEvaluationResult] + + +class SWEbenchOfficialAuthority: + """Adapter boundary around the official harness, with explicit run identity.""" + + def __init__(self, artifact_root: str, evaluator: OfficialEvaluator | None = None) -> None: + self.artifact_root = Path(artifact_root) + self.evaluator = evaluator or self._invoke_harness + self.identity = AUTHORITY_IDENTITY + + def evaluate(self, *, record: SWEbenchRecord, model_name_or_path: str, model_patch: str, run_id: str) -> OfficialEvaluationResult: + if not model_name_or_path or not run_id: + return OfficialEvaluationResult(OfficialEvaluationStatus.BLOCKED, self.identity, run_id, + record.instance_id, None, None, (), "model and run_id are required") + root = self.artifact_root / run_id / record.instance_id + if root.exists(): + return OfficialEvaluationResult(OfficialEvaluationStatus.BLOCKED, self.identity, run_id, + record.instance_id, None, None, (), "evaluation artifact already exists") + root.mkdir(parents=True) + prediction = SWEbenchPrediction(record.instance_id, model_name_or_path, model_patch) + prediction_path = root / "prediction.jsonl" + prediction_path.write_text(json.dumps(prediction.to_dict(), ensure_ascii=False) + "\n", encoding="utf-8") + payload = {"dataset_name": DATASET_IDENTITY, "split": DATASET_SPLIT, "instance_ids": [record.instance_id], + "predictions_path": str(prediction_path), "run_id": run_id, "artifact_root": str(root)} + try: + result = self.evaluator(payload) + if result.authority != self.identity or result.run_id != run_id: + raise ValueError("official evaluation identity mismatch") + return result + except Exception as error: + return OfficialEvaluationResult(OfficialEvaluationStatus.INFRASTRUCTURE_ERROR, self.identity, run_id, + record.instance_id, None, None, (), f"{type(error).__name__}: {error}") + + def evaluate_gold(self, *, record: SWEbenchRecord, run_id: str) -> OfficialEvaluationResult: + """Run the harness' supported ``predictions_path=gold`` qualification.""" + if not run_id: + return OfficialEvaluationResult(OfficialEvaluationStatus.BLOCKED, self.identity, run_id, + record.instance_id, None, None, (), "run_id is required") + root = self.artifact_root / run_id / record.instance_id + if root.exists(): + return OfficialEvaluationResult(OfficialEvaluationStatus.BLOCKED, self.identity, run_id, + record.instance_id, None, None, (), "evaluation artifact already exists") + root.mkdir(parents=True) + payload = {"dataset_name": DATASET_IDENTITY, "split": DATASET_SPLIT, "instance_ids": [record.instance_id], + "predictions_path": "gold", "run_id": run_id, "artifact_root": str(root)} + try: + result = self.evaluator(payload) + if result.authority != self.identity or result.run_id != run_id: + raise ValueError("official evaluation identity mismatch") + return result + except Exception as error: + return OfficialEvaluationResult(OfficialEvaluationStatus.INFRASTRUCTURE_ERROR, self.identity, run_id, + record.instance_id, None, None, (), f"{type(error).__name__}: {error}") + + @staticmethod + def _invoke_harness(payload: dict[str, Any]) -> OfficialEvaluationResult: + from swebench.harness.run_evaluation import main + main(dataset_name=payload["dataset_name"], split=payload["split"], instance_ids=payload["instance_ids"], + predictions_path=payload["predictions_path"], max_workers=1, open_file_limit=4096, + run_id=payload["run_id"], timeout=1800, rewrite_reports=True, modal=False) + report = Path("logs") / "evaluation" / payload["run_id"] / "results.json" + if not report.exists(): + return OfficialEvaluationResult(OfficialEvaluationStatus.AMBIGUOUS, AUTHORITY_IDENTITY, payload["run_id"], + payload["instance_ids"][0], None, str(report), ()) + data = json.loads(report.read_text(encoding="utf-8")) + outcome = str(data.get(payload["instance_ids"][0], data.get("resolved", "UNKNOWN"))) + status = OfficialEvaluationStatus.RESOLVED if outcome.lower() in {"true", "resolved", "pass"} else OfficialEvaluationStatus.TESTS_FAILED + return OfficialEvaluationResult(status, AUTHORITY_IDENTITY, payload["run_id"], payload["instance_ids"][0], outcome, str(report), ()) + + +def qualify_gold(*, instance_id: str, run_id: str, artifact_root: str) -> OfficialEvaluationResult: + """Describe the official gold qualification call; caller must have swebench + Docker.""" + authority = SWEbenchOfficialAuthority(artifact_root) + record = load_verified_records((instance_id,))[0] + return authority.evaluate_gold(record=record, run_id=run_id) diff --git a/src/arkx/telemetry.py b/src/arkx/telemetry.py new file mode 100644 index 00000000..4063261f --- /dev/null +++ b/src/arkx/telemetry.py @@ -0,0 +1,124 @@ +"""Small deterministic event collector for executor-agnostic P0 metrics.""" + +from __future__ import annotations + +from datetime import datetime +from numbers import Real +from typing import Any + +from .contracts import Event, EventType, ExecutionRecord, ExecutionStatus +from .evidence import resolve_status + + +def _timestamp(value: str) -> datetime: + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError(f"Invalid event timestamp: {value!r}") from exc + + +def _sum_optional(events: list[Event], key: str, *, numeric_type: type) -> int | float | None: + values: list[int | float] = [] + for event in events: + value = event.data.get(key) + if value is None: + continue + if isinstance(value, bool) or not isinstance(value, numeric_type): + raise ValueError(f"Event field {key!r} must be numeric when present") + values.append(value) + return sum(values) if values else None + + +class TelemetryCollector: + """Collect events without making execution or policy decisions.""" + + def __init__(self, *, task_id: str, run_id: str): + self.task_id = task_id + self.run_id = run_id + self._events: list[Event] = [] + + def add(self, event: Event) -> None: + if event.run_id != self.run_id: + raise ValueError(f"Event run_id {event.run_id!r} does not match {self.run_id!r}") + _timestamp(event.timestamp) + self._events.append(event) + + def extend(self, events: list[Event]) -> None: + for event in events: + self.add(event) + + @property + def events(self) -> tuple[Event, ...]: + return tuple(self._events) + + def summarize(self) -> ExecutionRecord: + events = list(self._events) + task_starts = [event for event in events if event.type is EventType.TASK_STARTED] + task_finishes = [event for event in events if event.type is EventType.TASK_FINISHED] + evidence_refs: list[str] = [] + errors: list[str] = [] + executor_id: str | None = None + + for event in events: + if event.type is EventType.EXECUTOR_STARTED and executor_id is None: + value = event.data.get("executor_id") + executor_id = str(value) if value is not None else None + if event.type is EventType.EVIDENCE_ADDED: + ref = event.data.get("ref") + refs = event.data.get("refs", []) + if ref is not None: + evidence_refs.append(str(ref)) + if not isinstance(refs, list): + raise ValueError("EVIDENCE_ADDED refs must be a list when present") + evidence_refs.extend(str(value) for value in refs) + if event.data.get("error") is not None: + errors.append(str(event.data["error"])) + + started_at = task_starts[0].timestamp if task_starts else None + finished_at = task_finishes[-1].timestamp if task_finishes else None + wall_time_ms: int | None = None + if started_at is not None and finished_at is not None: + wall_time_ms = int((_timestamp(finished_at) - _timestamp(started_at)).total_seconds() * 1000) + + requested = None + reason = None + if task_finishes: + requested = task_finishes[-1].data.get("status") + reason_value = task_finishes[-1].data.get("decision_reason") + reason = str(reason_value) if reason_value is not None else None + status = resolve_status(requested, evidence_refs) if task_finishes else ExecutionStatus.NOT_EXECUTED + + handoff_metadata = [ + { + "source_executor": event.data.get("source_executor"), + "target_executor": event.data.get("target_executor"), + "reason": event.data.get("reason"), + } + for event in events + if event.type is EventType.HANDOFF + ] + metadata: dict[str, Any] = {"handoffs": handoff_metadata} + + return ExecutionRecord( + task_id=self.task_id, + run_id=self.run_id, + executor_id=executor_id, + started_at=started_at, + finished_at=finished_at, + status=status, + decision_reason=reason, + input_tokens=_sum_optional(events, "input_tokens", numeric_type=int), + output_tokens=_sum_optional(events, "output_tokens", numeric_type=int), + total_tokens=_sum_optional(events, "total_tokens", numeric_type=int), + monetary_cost=_sum_optional(events, "monetary_cost", numeric_type=Real), + wall_time_ms=wall_time_ms, + retry_count=sum(event.type is EventType.RETRY for event in events), + replan_count=sum(event.type is EventType.REPLAN for event in events), + executor_invocations=sum(event.type is EventType.EXECUTOR_STARTED for event in events), + handoff_count=sum(event.type is EventType.HANDOFF for event in events), + human_interventions=sum(event.type is EventType.HUMAN_INTERVENTION for event in events), + evidence_refs=evidence_refs, + errors=errors, + metadata=metadata, + ) + diff --git a/src/arkx/verification.py b/src/arkx/verification.py new file mode 100644 index 00000000..d5ca8a52 --- /dev/null +++ b/src/arkx/verification.py @@ -0,0 +1,179 @@ +"""Deterministic patch verification, separate from final acceptance (P5).""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +import json +from typing import Any + + +SCHEMA_VERSION = 1 + + +class _ValueEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class TestResultStatus(_ValueEnum): + PASSED = "PASSED" + FAILED = "FAILED" + NOT_EXECUTED = "NOT_EXECUTED" + UNKNOWN = "UNKNOWN" + + +class PatchVerificationStatus(_ValueEnum): + VERIFIED = "VERIFIED" + REJECTED = "REJECTED" + BLOCKED = "BLOCKED" + UNKNOWN = "UNKNOWN" + + +class ReasonCode(_ValueEnum): + REPRODUCTION_REQUIRED_MISSING = "REPRODUCTION_REQUIRED_MISSING" + ISSUE_NOT_REPRODUCED = "ISSUE_NOT_REPRODUCED" + PATCH_NOT_APPLIED = "PATCH_NOT_APPLIED" + REQUIRED_REGRESSION_FAILED = "REQUIRED_REGRESSION_FAILED" + REQUIRED_TEST_NOT_EXECUTED = "REQUIRED_TEST_NOT_EXECUTED" + POST_PATCH_REPRODUCTION_FAILED = "POST_PATCH_REPRODUCTION_FAILED" + SCOPE_CHANGED = "SCOPE_CHANGED" + EVIDENCE_MISSING = "EVIDENCE_MISSING" + ALL_GATES_SATISFIED = "ALL_GATES_SATISFIED" + + +def _values(items: tuple[str, ...] | None) -> list[str] | None: + return None if items is None else sorted(set(items)) + + +@dataclass(frozen=True) +class TestResult: + test_id: str + status: TestResultStatus + required: bool + evidence_ref: str | None = None + command: tuple[str, ...] | None = None + exit_code: int | None = None + duration_ms: int | None = None + + def __post_init__(self) -> None: + if not self.test_id.strip(): + raise ValueError("test_id must be non-empty") + if self.command is not None: + if not self.command or any(not item.strip() for item in self.command): + raise ValueError("command must contain non-empty argv items") + object.__setattr__(self, "command", tuple(self.command)) + if self.duration_ms is not None and self.duration_ms < 0: + raise ValueError("duration_ms must be non-negative") + + def to_dict(self) -> dict[str, Any]: + return { + "test_id": self.test_id, + "status": self.status.value, + "required": self.required, + "evidence_ref": self.evidence_ref, + "command": None if self.command is None else list(self.command), + "exit_code": self.exit_code, + "duration_ms": self.duration_ms, + } + + +@dataclass(frozen=True) +class PatchVerificationInput: + task_id: str + issue_reproduction_required: bool + reproduces_issue_before_patch: bool | None + patch_applied: bool | None + regression_results: tuple[TestResult, ...] | None + reproduces_issue_after_patch: bool | None + changed_files: tuple[str, ...] | None + expected_scope: tuple[str, ...] | None + evidence_refs: tuple[str, ...] | None + schema_version: int = SCHEMA_VERSION + + +@dataclass(frozen=True) +class PatchVerificationResult: + task_id: str + reproduces_issue_before_patch: bool | None + patch_applied: bool | None + regression_tests_passed: bool | None + reproduces_issue_after_patch: bool | None + scope_changed: bool | None + evidence_sufficient: bool | None + status: PatchVerificationStatus + reason_codes: tuple[ReasonCode, ...] + telemetry: dict[str, Any] = field(default_factory=dict) + schema_version: int = SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "task_id": self.task_id, + "reproduces_issue_before_patch": self.reproduces_issue_before_patch, + "patch_applied": self.patch_applied, + "regression_tests_passed": self.regression_tests_passed, + "reproduces_issue_after_patch": self.reproduces_issue_after_patch, + "scope_changed": self.scope_changed, + "evidence_sufficient": self.evidence_sufficient, + "status": self.status.value, + "reason_codes": [code.value for code in self.reason_codes], + "telemetry": self.telemetry, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def verify_patch(value: PatchVerificationInput) -> PatchVerificationResult: + reasons: list[ReasonCode] = [] + if value.issue_reproduction_required and value.reproduces_issue_before_patch is None: + return _result(value, PatchVerificationStatus.BLOCKED, None, None, None, None, None, [ReasonCode.REPRODUCTION_REQUIRED_MISSING]) + if value.issue_reproduction_required and value.reproduces_issue_before_patch is False: + return _result(value, PatchVerificationStatus.REJECTED, False, value.patch_applied, None, None, None, [ReasonCode.ISSUE_NOT_REPRODUCED]) + if value.patch_applied is None or value.regression_results is None or value.reproduces_issue_after_patch is None or value.changed_files is None or value.expected_scope is None: + return _result(value, PatchVerificationStatus.UNKNOWN, value.reproduces_issue_before_patch, value.patch_applied, None, value.reproduces_issue_after_patch, None, [ReasonCode.EVIDENCE_MISSING]) + if value.patch_applied is False: + return _result(value, PatchVerificationStatus.REJECTED, value.reproduces_issue_before_patch, False, None, value.reproduces_issue_after_patch, False, [ReasonCode.PATCH_NOT_APPLIED]) + + required = [test for test in value.regression_results if test.required] + if any(test.status is TestResultStatus.FAILED for test in required): + reasons.append(ReasonCode.REQUIRED_REGRESSION_FAILED) + return _result(value, PatchVerificationStatus.REJECTED, value.reproduces_issue_before_patch, True, False, value.reproduces_issue_after_patch, None, reasons) + if any(test.status in (TestResultStatus.NOT_EXECUTED, TestResultStatus.UNKNOWN) for test in required): + return _result(value, PatchVerificationStatus.BLOCKED, value.reproduces_issue_before_patch, True, None, value.reproduces_issue_after_patch, None, [ReasonCode.REQUIRED_TEST_NOT_EXECUTED]) + if value.reproduces_issue_after_patch is True: + return _result(value, PatchVerificationStatus.REJECTED, value.reproduces_issue_before_patch, True, True, True, None, [ReasonCode.POST_PATCH_REPRODUCTION_FAILED]) + + scope_changed = bool(set(value.changed_files) - set(value.expected_scope)) + if scope_changed: + return _result(value, PatchVerificationStatus.REJECTED, value.reproduces_issue_before_patch, True, True, False, True, [ReasonCode.SCOPE_CHANGED]) + evidence_sufficient = bool(value.evidence_refs) + if not evidence_sufficient: + return _result(value, PatchVerificationStatus.BLOCKED, value.reproduces_issue_before_patch, True, True, False, False, [ReasonCode.EVIDENCE_MISSING]) + reasons.append(ReasonCode.ALL_GATES_SATISFIED) + return _result(value, PatchVerificationStatus.VERIFIED, value.reproduces_issue_before_patch, True, True, False, False, reasons) + + +def _result(value, status, reproduces_before, applied, regressions, reproduces_after, scope_changed, reasons): + return PatchVerificationResult( + task_id=value.task_id, + reproduces_issue_before_patch=reproduces_before, + patch_applied=applied, + regression_tests_passed=regressions, + reproduces_issue_after_patch=reproduces_after, + scope_changed=scope_changed, + evidence_sufficient=(bool(value.evidence_refs) if value.evidence_refs is not None else None), + status=status, + reason_codes=tuple(reasons), + telemetry={ + "patch_verification_status": status.value, + "required_tests": None if value.regression_results is None else sum(test.required for test in value.regression_results), + "passed_tests": None if value.regression_results is None else sum(test.status is TestResultStatus.PASSED for test in value.regression_results), + "failed_tests": None if value.regression_results is None else sum(test.status is TestResultStatus.FAILED for test in value.regression_results), + "scope_changed": scope_changed, + "reproduction_required": value.issue_reproduction_required, + "reproduces_issue_before_patch": reproduces_before, + "reproduces_issue_after_patch": reproduces_after, + }, + ) diff --git a/src/arkx/verifier.py b/src/arkx/verifier.py new file mode 100644 index 00000000..a21d0a4e --- /dev/null +++ b/src/arkx/verifier.py @@ -0,0 +1,192 @@ +"""Small, executor-neutral verification command boundary.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +import hashlib +import json +import os +from pathlib import Path +import subprocess +import time +from collections.abc import Sequence + +from .outcomes import VerificationResult, VerificationState +from .verification import TestResult, TestResultStatus + + +class VerificationErrorKind(str, Enum): + TIMEOUT = "TIMEOUT" + EXECUTABLE_NOT_FOUND = "EXECUTABLE_NOT_FOUND" + INVOCATION_ERROR = "INVOCATION_ERROR" + + +@dataclass(frozen=True) +class VerificationObservation: + result: TestResult + stdout: str + stderr: str + error_kind: VerificationErrorKind | None = None + + +class VerificationEvidenceStore: + """Persist raw verifier observations without silently replacing attempts.""" + + def __init__(self, root: str | Path) -> None: + self.root = Path(root).resolve() + + def persist(self, run_id: str, observation: VerificationObservation) -> str: + if not run_id.strip() or Path(run_id).name != run_id: + raise ValueError("run_id must be a single non-empty path component") + payload = { + "run_id": run_id, + "result": observation.result.to_dict(), + "stdout": observation.stdout, + "stderr": observation.stderr, + "error_kind": None if observation.error_kind is None else observation.error_kind.value, + } + content = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n" + identity = json.dumps({"run_id": run_id, "test_id": observation.result.test_id}, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16] + target = (self.root / run_id / f"verification-{digest}.json").resolve() + if self.root not in target.parents: + raise ValueError("verification evidence path escapes store root") + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + if target.read_text(encoding="utf-8") != content: + raise FileExistsError(f"verification evidence collision: {target}") + return f"evidence://{target.relative_to(self.root).as_posix()}" + temporary = target.with_name(target.name + f".tmp-{os.getpid()}") + if temporary.exists(): + raise FileExistsError(f"temporary verification evidence already exists: {temporary}") + temporary.write_text(content, encoding="utf-8", newline="\n") + try: + if target.exists(): + if target.read_text(encoding="utf-8") != content: + raise FileExistsError(f"verification evidence collision: {target}") + else: + temporary.replace(target) + finally: + if temporary.exists(): + temporary.unlink() + return f"evidence://{target.relative_to(self.root).as_posix()}" + + +class CommandVerifier: + """Adapt one declared local verifier command to the orchestration seam.""" + + def __init__( + self, + *, + authority: str, + workspace: str | Path, + command: Sequence[str], + test_id: str, + evidence_store: VerificationEvidenceStore, + required: bool = True, + timeout_seconds: float = 30.0, + ) -> None: + if not authority.strip(): + raise ValueError("authority must be non-empty") + self.authority = authority + self.workspace = workspace + self.command = tuple(command) + self.test_id = test_id + self.evidence_store = evidence_store + self.required = required + self.timeout_seconds = timeout_seconds + + def verify(self, execution) -> VerificationResult: + if getattr(execution.outcome, "value", execution.outcome) != "COMPLETED": + return VerificationResult(VerificationState.NOT_EXECUTED, self.authority, (), ("execution-not-completed",), ()) + observation = run_verification_command( + self.command, + workspace=self.workspace, + test_id=self.test_id, + required=self.required, + timeout_seconds=self.timeout_seconds, + ) + try: + evidence_ref = self.evidence_store.persist(execution.run_id, observation) + except (OSError, ValueError, FileExistsError) as error: + return VerificationResult( + VerificationState.BLOCKED, + self.authority, + (" ".join(self.command),), + (f"evidence-persistence-error:{type(error).__name__}",), + (), + ) + state = { + TestResultStatus.PASSED: VerificationState.PASS, + TestResultStatus.FAILED: VerificationState.FAIL, + TestResultStatus.UNKNOWN: VerificationState.INDETERMINATE, + TestResultStatus.NOT_EXECUTED: VerificationState.NOT_EXECUTED, + }[observation.result.status] + return VerificationResult( + state, + self.authority, + (" ".join(self.command),), + (f"{self.test_id}:{observation.result.status.value}",), + (evidence_ref,), + ) + + +def run_verification_command( + command: Sequence[str], + *, + workspace: str | Path, + test_id: str, + required: bool = True, + timeout_seconds: float = 30.0, + evidence_ref: str | None = None, +) -> VerificationObservation: + """Run one declared verifier command without shell interpretation.""" + + argv = tuple(command) + if not argv or any(not item.strip() for item in argv): + raise ValueError("verification command must contain non-empty argv items") + if not test_id.strip(): + raise ValueError("test_id must be non-empty") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + root = Path(workspace).resolve() + if not root.is_dir(): + raise ValueError("verification workspace must be an existing directory") + + started = time.monotonic() + try: + completed = subprocess.run( + list(argv), + cwd=root, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout_seconds, + shell=False, + check=False, + ) + except subprocess.TimeoutExpired as error: + duration_ms = int((time.monotonic() - started) * 1000) + result = TestResult(test_id, TestResultStatus.UNKNOWN, required, evidence_ref, argv, None, duration_ms) + return VerificationObservation(result, _text(error.stdout), _text(error.stderr), VerificationErrorKind.TIMEOUT) + except FileNotFoundError as error: + duration_ms = int((time.monotonic() - started) * 1000) + result = TestResult(test_id, TestResultStatus.UNKNOWN, required, evidence_ref, argv, None, duration_ms) + return VerificationObservation(result, "", str(error), VerificationErrorKind.EXECUTABLE_NOT_FOUND) + except OSError as error: + duration_ms = int((time.monotonic() - started) * 1000) + result = TestResult(test_id, TestResultStatus.UNKNOWN, required, evidence_ref, argv, None, duration_ms) + return VerificationObservation(result, "", str(error), VerificationErrorKind.INVOCATION_ERROR) + + duration_ms = int((time.monotonic() - started) * 1000) + status = TestResultStatus.PASSED if completed.returncode == 0 else TestResultStatus.FAILED + result = TestResult(test_id, status, required, evidence_ref, argv, completed.returncode, duration_ms) + return VerificationObservation(result, completed.stdout, completed.stderr) + + +def _text(value: str | bytes | None) -> str: + if value is None: + return "" + return value.decode("utf-8", errors="replace") if isinstance(value, bytes) else value diff --git a/src/codepro/__init__.py b/src/codepro/__init__.py new file mode 100644 index 00000000..5a578eb5 --- /dev/null +++ b/src/codepro/__init__.py @@ -0,0 +1,23 @@ +"""Canonical CodePro namespace during the staged ``arkx`` migration. + +The implementation remains in ``arkx`` for this migration slice. This +facade gives new consumers the product namespace while keeping the legacy +namespace available until the supported API has been migrated and tested. +""" + +from __future__ import annotations + +import arkx as _legacy + +from ._version import __version__ + +__all__ = tuple(_legacy.__all__) + + +def __getattr__(name: str): + """Resolve a supported public symbol from the legacy implementation.""" + return getattr(_legacy, name) + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/src/codepro/_legacy_module.py b/src/codepro/_legacy_module.py new file mode 100644 index 00000000..355b61ab --- /dev/null +++ b/src/codepro/_legacy_module.py @@ -0,0 +1,13 @@ +"""Helpers for staged module-level compatibility with the ``arkx`` namespace.""" + +from __future__ import annotations + +from types import ModuleType + + +def public_names(module: ModuleType) -> tuple[str, ...]: + return tuple(sorted(name for name in dir(module) if not name.startswith("_"))) + + +def resolve(module: ModuleType, name: str): + return getattr(module, name) diff --git a/src/codepro/_version.py b/src/codepro/_version.py new file mode 100644 index 00000000..db90c5d0 --- /dev/null +++ b/src/codepro/_version.py @@ -0,0 +1,3 @@ +"""Build-time version metadata without importing the compatibility facade.""" + +__version__ = "0.3.0.dev0" diff --git a/src/codepro/baseline.py b/src/codepro/baseline.py new file mode 100644 index 00000000..9bca6dc6 --- /dev/null +++ b/src/codepro/baseline.py @@ -0,0 +1,10 @@ +"""Canonical facade for the deterministic baseline fixture.""" + +from arkx import baseline as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/cli.py b/src/codepro/cli.py new file mode 100644 index 00000000..a7589545 --- /dev/null +++ b/src/codepro/cli.py @@ -0,0 +1,74 @@ +"""Canonical CodePro CLI facade during the staged namespace migration.""" + +import argparse +import json +import platform +import sys +from collections.abc import Sequence + +from . import __version__ +from .baseline import main as baseline_main +from .performance_baseline import main as performance_baseline_main + + +SUPPORTED_PYTHON_MIN = (3, 12) +SUPPORTED_PYTHON_MAX = (3, 14) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="codepro", + description="Empirical, executor-agnostic CodePro chassis.", + ) + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + commands = parser.add_subparsers(dest="command") + doctor = commands.add_parser("doctor", help="Check the installed package and Python runtime.") + doctor.add_argument("--json", action="store_true", dest="as_json") + commands.add_parser("baseline", help="Run the deterministic, no-network baseline fixture.") + performance = commands.add_parser("baseline-cost", help="Measure local overhead without changing behavior.") + performance.add_argument("--workspace", default=".") + performance.add_argument("--output", required=True) + performance.add_argument("--samples", type=int, default=5) + return parser + + +def _doctor(*, as_json: bool) -> int: + supported = SUPPORTED_PYTHON_MIN <= sys.version_info[:2] <= SUPPORTED_PYTHON_MAX + payload = { + "package": "codepro", + "version": __version__, + "python": platform.python_version(), + "python_supported": supported, + "runtime_dependencies": [], + "status": "PASS" if supported else "FAIL", + } + if as_json: + print(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + else: + print(f"package: PASS (codepro {__version__})") + state = "PASS" if supported else "FAIL" + print(f"python: {state} ({platform.python_version()})") + print(f"status: {payload['status']}") + return 0 if supported else 1 + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == "doctor": + return _doctor(as_json=args.as_json) + if args.command == "baseline": + return baseline_main() + if args.command == "baseline-cost": + return performance_baseline_main([ + "--workspace", args.workspace, + "--output", args.output, + "--samples", str(args.samples), + ]) + build_parser().print_help() + return 2 + +__all__ = ("build_parser", "main") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/codepro/contracts.py b/src/codepro/contracts.py new file mode 100644 index 00000000..ae9bab00 --- /dev/null +++ b/src/codepro/contracts.py @@ -0,0 +1,10 @@ +"""Canonical facade for the current neutral contracts.""" + +from arkx import contracts as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/orchestration.py b/src/codepro/orchestration.py new file mode 100644 index 00000000..d44d7183 --- /dev/null +++ b/src/codepro/orchestration.py @@ -0,0 +1,10 @@ +"""Canonical facade for the orchestration compatibility boundary.""" + +from arkx import orchestration as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/p82_localization.py b/src/codepro/p82_localization.py new file mode 100644 index 00000000..4c1bfc82 --- /dev/null +++ b/src/codepro/p82_localization.py @@ -0,0 +1,10 @@ +"""Canonical facade for P8.2 localization contracts.""" + +from arkx import p82_localization as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/performance_baseline.py b/src/codepro/performance_baseline.py new file mode 100644 index 00000000..67b3e909 --- /dev/null +++ b/src/codepro/performance_baseline.py @@ -0,0 +1,10 @@ +"""Canonical facade for local overhead measurement.""" + +from arkx import performance_baseline as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/verification.py b/src/codepro/verification.py new file mode 100644 index 00000000..dddab8e7 --- /dev/null +++ b/src/codepro/verification.py @@ -0,0 +1,10 @@ +"""Canonical facade for patch-verification contracts.""" + +from arkx import verification as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/src/codepro/verifier.py b/src/codepro/verifier.py new file mode 100644 index 00000000..4ac7c45e --- /dev/null +++ b/src/codepro/verifier.py @@ -0,0 +1,10 @@ +"""Canonical facade for the verifier boundary.""" + +from arkx import verifier as _legacy +from ._legacy_module import public_names, resolve + +__all__ = public_names(_legacy) + + +def __getattr__(name: str): + return resolve(_legacy, name) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..af71cece --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +"""P0 test package for standard-library unittest discovery.""" + diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py new file mode 100644 index 00000000..41364107 --- /dev/null +++ b/tests/test_acceptance.py @@ -0,0 +1,28 @@ +import unittest + +from arkx.acceptance import AcceptancePolicy, decide_independent_acceptance +from arkx.p82_baseline import AcceptanceDecision, VerificationResult, VerificationState + + +class AcceptanceTests(unittest.TestCase): + def test_verified_with_evidence_is_accepted_by_explicit_authority(self): + result = decide_independent_acceptance(VerificationResult(VerificationState.PASS, "verifier", ("pytest",), ("0",), ("evidence://log",)), AcceptancePolicy("authority")) + self.assertEqual(result.decision, AcceptanceDecision.ACCEPTED) + self.assertEqual(result.authority, "authority") + self.assertTrue(any(item.startswith("verification://") for item in result.evidence)) + + def test_verified_without_evidence_is_blocked(self): + result = decide_independent_acceptance(VerificationResult(VerificationState.PASS, "verifier", (), (), ()), AcceptancePolicy("authority")) + self.assertEqual(result.decision, AcceptanceDecision.BLOCKED) + + def test_unknown_verification_is_not_accepted(self): + result = decide_independent_acceptance(VerificationResult(VerificationState.INDETERMINATE, "verifier", (), (), ("evidence://unknown",)), AcceptancePolicy("authority")) + self.assertEqual(result.decision, AcceptanceDecision.INDETERMINATE) + + def test_missing_verification_authority_is_blocked(self): + result = decide_independent_acceptance(VerificationResult(VerificationState.PASS, "", (), (), ("evidence://log",)), AcceptancePolicy("authority")) + self.assertEqual(result.decision, AcceptanceDecision.BLOCKED) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_adapter_import_boundaries.py b/tests/test_adapter_import_boundaries.py new file mode 100644 index 00000000..13262b5b --- /dev/null +++ b/tests/test_adapter_import_boundaries.py @@ -0,0 +1,22 @@ +import importlib +import unittest + + +class AdapterImportBoundaryTests(unittest.TestCase): + def test_openrouter_module_import_does_not_select_a_fallback(self): + module = importlib.import_module("tools.p82_openrouter_model") + self.assertTrue(hasattr(module, "MINISWEAGENT_AVAILABLE")) + if not module.MINISWEAGENT_AVAILABLE: + with self.assertRaises(RuntimeError): + module.ArkxOpenRouterModel() + + def test_bash_module_import_does_not_select_a_fallback(self): + module = importlib.import_module("tools.p82_bash_environment") + self.assertTrue(hasattr(module, "MINISWEAGENT_AVAILABLE")) + if not module.MINISWEAGENT_AVAILABLE: + with self.assertRaises(RuntimeError): + module.BashLocalEnvironment() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_architecture_boundaries.py b/tests/test_architecture_boundaries.py new file mode 100644 index 00000000..6b37ff50 --- /dev/null +++ b/tests/test_architecture_boundaries.py @@ -0,0 +1,83 @@ +from pathlib import Path +import os +import subprocess +import sys +import unittest + + +class ArchitectureBoundaryTests(unittest.TestCase): + def test_legacy_orchestration_result_is_confined_to_compatibility_modules(self): + source_root = Path(__file__).parents[1] / "src" / "arkx" + violations = [] + for path in source_root.glob("*.py"): + if path.name in {"orchestration.py", "__init__.py"}: + continue + text = path.read_text(encoding="utf-8") + if "from .orchestration import ExecutionResult" in text or "from arkx.orchestration import ExecutionResult" in text: + violations.append(path.name) + self.assertEqual(violations, []) + + def test_public_api_exposes_distinct_neutral_and_legacy_names(self): + import arkx + self.assertIsNot(arkx.ExecutionResult, arkx.OrchestrationExecutionResult) + self.assertIn("ExecutionResult", arkx.__all__) + self.assertIn("OrchestrationExecutionResult", arkx.__all__) + + def test_root_import_does_not_eagerly_load_experimental_subsystems(self): + probe = subprocess.run( + [ + sys.executable, + "-c", + "import sys; import arkx; print('\\n'.join(sorted(k for k in sys.modules if k.startswith('arkx.'))))", + ], + check=True, + capture_output=True, + text=True, + env=os.environ.copy(), + ) + loaded = set(probe.stdout.splitlines()) + self.assertNotIn("arkx.p82", loaded) + self.assertNotIn("arkx.p82_baseline", loaded) + self.assertNotIn("arkx.composition", loaded) + self.assertNotIn("arkx.executor_qualification", loaded) + self.assertNotIn("arkx.swebench_authority", loaded) + + def test_legacy_root_export_is_lazy_and_compatible(self): + probe = subprocess.run( + [ + sys.executable, + "-c", + "import sys; import arkx; from arkx import Event; print(Event.__module__); print('arkx.contracts' in sys.modules); print('arkx.event_log' in sys.modules)", + ], + check=True, + capture_output=True, + text=True, + env=os.environ.copy(), + ) + self.assertEqual(probe.stdout.splitlines(), ["arkx.contracts", "True", "False"]) + + def test_legacy_public_manifest_resolves_every_declared_name(self): + import arkx + + self.assertEqual(len(arkx.__all__), 259) + for name in arkx.__all__: + with self.subTest(name=name): + self.assertIsNotNone(getattr(arkx, name)) + + def test_legacy_wildcard_import_preserves_public_manifest(self): + probe = subprocess.run( + [ + sys.executable, + "-c", + "import arkx; namespace={}; exec('from arkx import *', namespace); print(len(namespace)); print('compare_qualification_trials' in namespace)", + ], + check=True, + capture_output=True, + text=True, + env=os.environ.copy(), + ) + self.assertEqual(probe.stdout.splitlines(), ["258", "True"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_audited_comparison.py b/tests/test_audited_comparison.py new file mode 100644 index 00000000..885d3e4d --- /dev/null +++ b/tests/test_audited_comparison.py @@ -0,0 +1,18 @@ +import unittest +from unittest.mock import patch + +from arkx.event_log import ChainIntegrity +from arkx.harness import AttemptStore + + +class AuditedComparisonTests(unittest.TestCase): + def test_comparison_rejects_incomplete_audited_chain(self): + audit = type("Audit", (), {"integrity": ChainIntegrity.INCOMPLETE})() + with patch("arkx.harness.load_attempt", side_effect=["left", "right"]): + with patch.object(AttemptStore, "load_audited", side_effect=[audit, audit]): + with self.assertRaises(ValueError): + AttemptStore("artifacts").compare_audited("left", "left-events", "right", "right-events") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_audited_experiment.py b/tests/test_audited_experiment.py new file mode 100644 index 00000000..9c95e654 --- /dev/null +++ b/tests/test_audited_experiment.py @@ -0,0 +1,71 @@ +import unittest +from types import SimpleNamespace + +from arkx.harness import ( + AttemptSnapshot, + ExperimentRecord, + RunManifest, + RunState, + derive_experiment_identity_digest, + validate_experiment_record_audited, +) + + +def attempt(): + manifest = RunManifest( + experiment_id="exp", trial_id="trial", attempt_id="attempt", attempt_number=1, + task_id="task", task_revision="rev", treatment="A", executor="executor", + provider="provider", model="model", sandbox="sandbox", repository_revision="rev", + configuration_digest="cfg", protocol_version="protocol", state=RunState.COMPLETED, + ) + return AttemptSnapshot(manifest, {}, ()) + + +class AuditedExperimentTests(unittest.TestCase): + def test_experiment_requires_complete_audited_chain(self): + record = ExperimentRecord( + experiment_id="exp", protocol_version="protocol", input_manifest_digest="input", + trial_ids=("trial",), attempt_ids=("attempt",), verification_refs=("verification://1",), + acceptance_refs=("acceptance://1",), promotion_decision_ref="promotion://1", + identity_digest=derive_experiment_identity_digest((attempt(),)), + capability_digest="capabilities-1", + ) + audit = SimpleNamespace( + replay=SimpleNamespace(run_id="attempt"), + integrity=SimpleNamespace(value="COMPLETE"), + protocol_version="protocol", + schema_version=1, + configuration_digest="cfg", + budget_digest=None, + treatment="A", + capability_digest="capabilities-1", + chain=SimpleNamespace(to_dict=lambda: { + "verification_ref": "verification://1", "acceptance_ref": "acceptance://1", "promotion_ref": "promotion://1", + }), + ) + validate_experiment_record_audited(record, (attempt(),), (audit,), promotion_attempt_id="attempt") + + def test_incomplete_audit_blocks_reproducible_experiment(self): + record = ExperimentRecord( + experiment_id="exp", protocol_version="protocol", input_manifest_digest="input", + trial_ids=("trial",), attempt_ids=("attempt",), verification_refs=("verification://1",), + identity_digest="wrong", + capability_digest="capabilities-1", + ) + audit = SimpleNamespace( + replay=SimpleNamespace(run_id="attempt"), + integrity=SimpleNamespace(value="INCOMPLETE"), + protocol_version="protocol", + schema_version=1, + configuration_digest="cfg", + budget_digest=None, + treatment="A", + capability_digest="capabilities-1", + chain=SimpleNamespace(to_dict=lambda: {"verification_ref": "verification://1"}), + ) + with self.assertRaises(ValueError): + validate_experiment_record_audited(record, (attempt(),), (audit,)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_characterization.py b/tests/test_characterization.py new file mode 100644 index 00000000..f8446389 --- /dev/null +++ b/tests/test_characterization.py @@ -0,0 +1,140 @@ +import unittest +import json +from pathlib import Path + +from arkx.characterization import ( + Confidence, + RecommendedPath, + Scope, + TaskSignals, + characterize, + characterize_timed, +) +from arkx.contracts import ExecutionStatus +from arkx.telemetry import TelemetryCollector + + +def complete_signals(**overrides): + values = { + "candidate_files": ("src/one.py",), + "dependency_edges": (), + "affected_components": ("one",), + "known_tests": ("tests/test_one.py",), + "ambiguity_markers": (), + "risk_markers": (), + "acceptance_checks": ("tests pass",), + "state_shared": False, + "architectural_change": False, + } + values.update(overrides) + return TaskSignals(**values) + + +class ClassificationTests(unittest.TestCase): + def test_single_file_is_simple(self): + result = characterize(complete_signals()) + self.assertEqual(result.scope, Scope.SIMPLE) + self.assertEqual(result.recommended_path, RecommendedPath.SIMPLE_PATH) + self.assertEqual(result.confidence, Confidence.HIGH) + + def test_bounded_few_file_task_is_localized(self): + result = characterize( + complete_signals( + candidate_files=("src/one.py", "tests/test_one.py"), + dependency_edges=(("one", "contracts"),), + acceptance_checks=("tests pass", "round-trip passes"), + ) + ) + self.assertEqual(result.scope, Scope.LOCALIZED) + self.assertEqual(result.recommended_path, RecommendedPath.LOCALIZED_PATH) + + def test_cross_component_change_is_repository_wide(self): + result = characterize( + complete_signals( + candidate_files=("a.py", "b.py", "c.py", "d.py", "e.py", "f.py"), + dependency_edges=(("a", "b"), ("b", "c"), ("c", "shared"), ("tests", "a")), + affected_components=("a", "b", "c"), + known_tests=("a", "b", "c"), + state_shared=True, + architectural_change=True, + ) + ) + self.assertEqual(result.scope, Scope.REPOSITORY_WIDE) + self.assertEqual(result.recommended_path, RecommendedPath.REPOSITORY_WIDE_PATH) + self.assertEqual(result.confidence, Confidence.MEDIUM) + + def test_insufficient_signals_are_unknown_and_require_qualification(self): + result = characterize(TaskSignals(ambiguity_markers=("unclear",))) + self.assertEqual(result.scope, Scope.UNKNOWN) + self.assertEqual(result.recommended_path, RecommendedPath.QUALIFICATION_REQUIRED) + self.assertEqual(result.confidence, Confidence.LOW) + + def test_contradictory_signals_require_qualification(self): + result = characterize(complete_signals(declared_scope=Scope.REPOSITORY_WIDE)) + self.assertEqual(result.scope, Scope.UNKNOWN) + self.assertEqual(result.recommended_path, RecommendedPath.QUALIFICATION_REQUIRED) + self.assertIn("CONTRADICTORY_SIGNALS", [value.value for value in result.signals]) + + +class EvidenceAndDeterminismTests(unittest.TestCase): + def test_signal_order_does_not_change_serialization(self): + first = complete_signals( + candidate_files=("b.py", "a.py"), + dependency_edges=(("b", "c"), ("a", "b")), + affected_components=("two", "one"), + known_tests=("z", "a"), + acceptance_checks=("second", "first"), + ) + second = complete_signals( + candidate_files=("a.py", "b.py"), + dependency_edges=(("a", "b"), ("b", "c")), + affected_components=("one", "two"), + known_tests=("a", "z"), + acceptance_checks=("first", "second"), + ) + self.assertEqual(characterize(first).to_json(), characterize(second).to_json()) + + def test_controlled_duration_is_serialized_as_telemetry(self): + result = characterize(complete_signals(), characterization_duration_ms=7) + self.assertEqual(result.telemetry["characterization_duration_ms"], 7) + self.assertEqual(result.telemetry["signal_count"], 7) + + def test_timed_characterization_reports_nonnegative_duration(self): + result = characterize_timed(complete_signals()) + self.assertGreaterEqual(result.telemetry["characterization_duration_ms"], 0) + + def test_characterization_alone_cannot_produce_pass(self): + result = characterize(complete_signals()) + self.assertNotIn("status", result.to_dict()) + execution = TelemetryCollector(task_id="task", run_id="run").summarize() + self.assertEqual(execution.status, ExecutionStatus.NOT_EXECUTED) + + def test_synthetic_fixture_covers_the_four_declared_categories(self): + manifest_path = Path(__file__).parents[1] / "experiments" / "characterization-fixture.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + self.assertEqual(manifest["fixture_type"], "CHARACTERIZATION_FIXTURE") + observed = {} + for task in manifest["tasks"]: + raw = task["signals"] + signals = TaskSignals( + candidate_files=None if raw["candidate_files"] is None else tuple(raw["candidate_files"]), + dependency_edges=None if raw["dependency_edges"] is None else tuple(tuple(edge) for edge in raw["dependency_edges"]), + affected_components=None if raw["affected_components"] is None else tuple(raw["affected_components"]), + known_tests=None if raw["known_tests"] is None else tuple(raw["known_tests"]), + ambiguity_markers=None if raw["ambiguity_markers"] is None else tuple(raw["ambiguity_markers"]), + risk_markers=None if raw["risk_markers"] is None else tuple(raw["risk_markers"]), + acceptance_checks=None if raw["acceptance_checks"] is None else tuple(raw["acceptance_checks"]), + state_shared=raw.get("state_shared"), + architectural_change=raw.get("architectural_change"), + ) + observed[task["id"]] = characterize(signals).scope.value + self.assertEqual(observed, { + "simple": "SIMPLE", + "localized": "LOCALIZED", + "repository-wide": "REPOSITORY_WIDE", + "ambiguous": "UNKNOWN", + }) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..0b255b78 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,57 @@ +import contextlib +import io +import json +import unittest +from unittest.mock import patch + +from arkx.cli import main + + +class CliTests(unittest.TestCase): + def test_doctor_json_reports_packaged_runtime(self): + output = io.StringIO() + with contextlib.redirect_stdout(output): + status = main(["doctor", "--json"]) + payload = json.loads(output.getvalue()) + self.assertEqual(status, 0) + self.assertEqual(payload["package"], "codepro") + self.assertEqual(payload["version"], "0.3.0.dev0") + self.assertTrue(payload["python_supported"]) + self.assertEqual(payload["status"], "PASS") + + def test_doctor_fails_outside_supported_python_range(self): + output = io.StringIO() + with patch("arkx.cli.sys.version_info", (3, 11)), contextlib.redirect_stdout(output): + status = main(["doctor", "--json"]) + payload = json.loads(output.getvalue()) + self.assertEqual(status, 1) + self.assertFalse(payload["python_supported"]) + self.assertEqual(payload["status"], "FAIL") + + def test_baseline_delegates_to_deterministic_fixture(self): + output = io.StringIO() + with contextlib.redirect_stdout(output): + status = main(["baseline"]) + payload = json.loads(output.getvalue()) + self.assertEqual(status, 0) + self.assertEqual(payload["status"], "PASS") + self.assertEqual(payload["executor_id"], "fixture") + + def test_no_command_is_not_a_product_execution(self): + output = io.StringIO() + with contextlib.redirect_stdout(output): + status = main([]) + self.assertEqual(status, 2) + self.assertIn("baseline", output.getvalue()) + + def test_task_submission_command_is_not_available(self): + error = io.StringIO() + with contextlib.redirect_stderr(error): + with self.assertRaises(SystemExit) as raised: + main(["run"]) + self.assertEqual(raised.exception.code, 2) + self.assertIn("invalid choice", error.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_codepro_namespace.py b/tests/test_codepro_namespace.py new file mode 100644 index 00000000..b4173208 --- /dev/null +++ b/tests/test_codepro_namespace.py @@ -0,0 +1,91 @@ +import contextlib +import io +import json +import subprocess +import sys +import unittest + +import arkx +import codepro +import arkx.contracts as arkx_contracts +import arkx.orchestration as arkx_orchestration +import arkx.verification as arkx_verification +import arkx.verifier as arkx_verifier +import codepro.contracts as codepro_contracts +import codepro.orchestration as codepro_orchestration +import codepro.verification as codepro_verification +import codepro.verifier as codepro_verifier +import codepro.baseline as codepro_baseline +import codepro.performance_baseline as codepro_performance_baseline +import codepro.p82_localization as codepro_localization +from codepro.cli import main as codepro_main +from arkx.cli import main as arkx_main + + +class CodeproNamespaceTests(unittest.TestCase): + def test_codepro_is_the_canonical_facade_for_the_current_public_api(self): + self.assertEqual(codepro.__version__, arkx.__version__) + self.assertEqual(codepro.__all__, tuple(arkx.__all__)) + for name in codepro.__all__: + self.assertIs(getattr(codepro, name), getattr(arkx, name)) + + def test_legacy_namespace_remains_available(self): + self.assertIs(codepro.ExecutionResult, arkx.ExecutionResult) + self.assertIs(codepro.OrchestrationExecutionResult, arkx.OrchestrationExecutionResult) + + def test_codepro_cli_is_the_canonical_entrypoint_facade(self): + self.assertIsNot(codepro_main, arkx_main) + + canonical_output = io.StringIO() + with contextlib.redirect_stdout(canonical_output): + self.assertEqual(codepro_main(["doctor", "--json"]), 0) + + legacy_output = io.StringIO() + with contextlib.redirect_stdout(legacy_output): + self.assertEqual(arkx_main(["doctor", "--json"]), 0) + + canonical_payload = json.loads(canonical_output.getvalue()) + legacy_payload = json.loads(legacy_output.getvalue()) + self.assertEqual(canonical_payload, legacy_payload) + + def test_core_submodules_preserve_object_identity_during_migration(self): + pairs = ( + (codepro_contracts, arkx_contracts), + (codepro_orchestration, arkx_orchestration), + (codepro_verification, arkx_verification), + (codepro_verifier, arkx_verifier), + ) + for canonical, legacy in pairs: + self.assertEqual(canonical.__all__, tuple(sorted(name for name in dir(legacy) if not name.startswith("_")))) + for name in canonical.__all__: + self.assertIs(getattr(canonical, name), getattr(legacy, name)) + + def test_operational_submodules_preserve_object_identity(self): + import arkx.baseline as arkx_baseline + import arkx.performance_baseline as arkx_performance_baseline + import arkx.p82_localization as arkx_localization + + for canonical, legacy in ( + (codepro_baseline, arkx_baseline), + (codepro_performance_baseline, arkx_performance_baseline), + (codepro_localization, arkx_localization), + ): + for name in canonical.__all__: + self.assertIs(getattr(canonical, name), getattr(legacy, name)) + + def test_import_does_not_claim_migration_of_unsupported_submodules(self): + probe = subprocess.run( + [ + sys.executable, + "-c", + "import codepro, sys; print('codepro.cli' in sys.modules); print(codepro.__version__)", + ], + check=True, + capture_output=True, + text=True, + ) + self.assertEqual(probe.stdout.splitlines(), ["False", arkx.__version__]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_composition.py b/tests/test_composition.py new file mode 100644 index 00000000..d52c7ff6 --- /dev/null +++ b/tests/test_composition.py @@ -0,0 +1,95 @@ +import json +import unittest +from pathlib import Path + +from arkx.composition import ( + ComparabilityStatus, + ExperimentIdentity, + ExperimentManifest, + Mechanism, + Treatment, + TrialObservation, + compare_trials, + initial_manifest, + summarize_trials, +) + + +def identity(**overrides): + values = { + "task": "task-1", + "executor": "executor-1", + "model": "model-1", + "environment": "env-1", + "budget": "budget-1", + } + values.update(overrides) + return ExperimentIdentity(**values) + + +class CompositionTests(unittest.TestCase): + def test_mechanism_order_is_canonical(self): + treatment = Treatment("D", (Mechanism.P3_ROUTING, Mechanism.P1_CHARACTERIZATION, Mechanism.P2_PROGRESS)) + self.assertEqual( + treatment.enabled_mechanisms, + (Mechanism.P1_CHARACTERIZATION, Mechanism.P2_PROGRESS, Mechanism.P3_ROUTING), + ) + + def test_initial_manifest_contains_only_a_to_f(self): + self.assertEqual(tuple(t.name for t in initial_manifest().treatments), ("A", "B", "C", "D", "E", "F")) + + def test_duplicate_names_are_invalid(self): + with self.assertRaises(ValueError): + ExperimentManifest((Treatment("A", ()), Treatment("A", (Mechanism.P1_CHARACTERIZATION,)))) + + def test_duplicate_semantic_compositions_are_invalid(self): + with self.assertRaises(ValueError): + ExperimentManifest((Treatment("A", ()), Treatment("another-name", ()))) + + def test_input_order_has_same_canonical_json(self): + left = Treatment("C", (Mechanism.P2_PROGRESS, Mechanism.P1_CHARACTERIZATION)) + right = Treatment("C", (Mechanism.P1_CHARACTERIZATION, Mechanism.P2_PROGRESS)) + self.assertEqual(left.to_dict(), right.to_dict()) + + def test_each_control_dimension_divergence_is_incomparable(self): + base = identity() + trial = TrialObservation(base, Treatment("A", ())) + for field in ("task", "executor", "model", "environment", "budget"): + divergent = identity(**{field: f"different-{field}"}) + self.assertEqual(compare_trials(trial, TrialObservation(divergent, Treatment("B", ()))), ComparabilityStatus.INCOMPARABLE) + + def test_missing_control_dimension_is_unknown(self): + trial = TrialObservation(identity(), Treatment("A", ())) + missing = TrialObservation(identity(model=None), Treatment("B", ())) + self.assertEqual(compare_trials(trial, missing), ComparabilityStatus.UNKNOWN) + + def test_summary_does_not_impute_missing_measurements(self): + summary = summarize_trials((TrialObservation(identity(), Treatment("A", ()), (("tokens", 12),)),)) + self.assertEqual(summary.measurement_counts, (("tokens", 1),)) + self.assertNotIn("cost", summary.to_json()) + + def test_summary_is_deterministic_and_never_pass(self): + trials = ( + TrialObservation(identity(), Treatment("B", (Mechanism.P1_CHARACTERIZATION,)), (("wall_time_ms", 4),)), + TrialObservation(identity(), Treatment("A", ()), (("wall_time_ms", 5),)), + ) + first = summarize_trials(trials) + second = summarize_trials(tuple(reversed(trials))) + self.assertEqual(first.to_json(), second.to_json()) + self.assertNotIn("PASS", first.to_json()) + + def test_measurements_are_sorted_and_unique(self): + trial = TrialObservation(identity(), Treatment("A", ()), (("z", 1), ("a", 2))) + self.assertEqual(tuple(key for key, _ in trial.measurements), ("a", "z")) + with self.assertRaises(ValueError): + TrialObservation(identity(), Treatment("A", ()), (("a", 1), ("a", 2))) + + def test_fixture_is_classified_as_composition_fixture(self): + path = Path(__file__).parents[1] / "experiments" / "composition-harness-fixture.json" + value = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(value["fixture_type"], "EXPERIMENTAL_COMPOSITION_FIXTURE") + self.assertEqual(value["treatments"], ["A", "B", "C", "D", "E", "F"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_configuration.py b/tests/test_configuration.py new file mode 100644 index 00000000..dd49176f --- /dev/null +++ b/tests/test_configuration.py @@ -0,0 +1,23 @@ +import unittest + +from arkx.configuration import ConfigurationSnapshot + + +class ConfigurationSnapshotTests(unittest.TestCase): + def test_digest_is_stable_and_secret_material_is_not_serialized(self): + first = ConfigurationSnapshot("executor", "1.0", {"temperature": 0, "mode": "strict"}, {"api_key": "sha256:key"}) + second = ConfigurationSnapshot("executor", "1.0", {"mode": "strict", "temperature": 0}, {"api_key": "sha256:key"}) + self.assertEqual(first.digest(), second.digest()) + self.assertNotIn("secret-value", first.to_json()) + self.assertEqual(first.reference, f"config://{first.digest()}") + + def test_configuration_identity_changes_with_public_or_secret_digest(self): + base = ConfigurationSnapshot("executor", "1.0", {"mode": "strict"}, {"api_key": "a"}) + self.assertNotEqual(base.digest(), ConfigurationSnapshot("executor", "1.0", {"mode": "relaxed"}, {"api_key": "a"}).digest()) + self.assertNotEqual(base.digest(), ConfigurationSnapshot("executor", "1.0", {"mode": "strict"}, {"api_key": "b"}).digest()) + + def test_empty_component_version_or_secret_digest_is_rejected(self): + with self.assertRaises(ValueError): + ConfigurationSnapshot("", "1.0", {}) + with self.assertRaises(ValueError): + ConfigurationSnapshot("executor", "1.0", {}, {"api_key": ""}) diff --git a/tests/test_core_outcomes.py b/tests/test_core_outcomes.py new file mode 100644 index 00000000..10e8d764 --- /dev/null +++ b/tests/test_core_outcomes.py @@ -0,0 +1,57 @@ +import unittest + +from arkx.outcomes import AcceptanceDecision, AcceptanceResult, VerificationResult, VerificationState, validate_outcome_references +from arkx.p82_baseline import ( + AcceptanceDecision as BaselineAcceptanceDecision, + AcceptanceResult as BaselineAcceptanceResult, + VerificationResult as BaselineVerificationResult, + VerificationState as BaselineVerificationState, +) + + +class CoreOutcomeBoundaryTests(unittest.TestCase): + def test_baseline_reexports_are_compatibility_aliases(self): + self.assertIs(AcceptanceDecision, BaselineAcceptanceDecision) + self.assertIs(AcceptanceResult, BaselineAcceptanceResult) + self.assertIs(VerificationResult, BaselineVerificationResult) + self.assertIs(VerificationState, BaselineVerificationState) + + def test_core_outcome_serialization_is_executor_neutral(self): + verification = VerificationResult( + VerificationState.PASS, + "verifier-v1", + ("pytest",), + ("passed",), + ("evidence://verification/1",), + ) + acceptance = AcceptanceResult( + AcceptanceDecision.ACCEPTED, + "authority-v1", + "verified", + ("evidence://acceptance/1",), + ) + self.assertEqual(verification.to_dict(), { + "state": "PASS", + "authority": "verifier-v1", + "commands": ["pytest"], + "outcomes": ["passed"], + "evidence": ["evidence://verification/1"], + }) + self.assertEqual(acceptance.to_dict(), { + "decision": "ACCEPTED", + "authority": "authority-v1", + "raw_outcome": "verified", + "evidence": ["evidence://acceptance/1"], + }) + + +if __name__ == "__main__": + unittest.main() + def test_outcome_references_are_deterministic_and_independent(self): + verification = VerificationResult(VerificationState.PASS, "verifier-1", ("pytest",), ("PASS",), ("artifact://v",)) + acceptance = AcceptanceResult(AcceptanceDecision.ACCEPTED, "authority-1", "accepted", ("acceptance://evidence",)) + self.assertEqual(verification.reference, VerificationResult(VerificationState.PASS, "verifier-1", ("pytest",), ("PASS",), ("artifact://v",)).reference) + self.assertNotEqual(verification.reference, acceptance.reference) + validate_outcome_references(verification, acceptance, verification_ref=verification.reference, acceptance_ref=acceptance.reference) + with self.assertRaises(ValueError): + validate_outcome_references(verification, acceptance, verification_ref="verification://wrong", acceptance_ref=acceptance.reference) diff --git a/tests/test_event_log.py b/tests/test_event_log.py new file mode 100644 index 00000000..11cdcfe1 --- /dev/null +++ b/tests/test_event_log.py @@ -0,0 +1,36 @@ +import unittest +from tempfile import TemporaryDirectory +from pathlib import Path + +from arkx.contracts import Event, EventType +from arkx.event_log import EventLog + + +class EventLogTests(unittest.TestCase): + def test_event_log_round_trips_and_appends_in_order(self): + with TemporaryDirectory() as directory: + log = EventLog(Path(directory) / "events.jsonl", run_id="run-1") + log.append(Event("2026-01-01T00:00:00+00:00", EventType.TASK_STARTED, "run-1")) + log.append(Event("2026-01-01T00:00:01+00:00", EventType.TASK_FINISHED, "run-1", {"status": "BLOCKED"})) + self.assertEqual([event.type for event in log.read()], [EventType.TASK_STARTED, EventType.TASK_FINISHED]) + + def test_event_log_rejects_wrong_run_and_time_regression(self): + with TemporaryDirectory() as directory: + log = EventLog(Path(directory) / "events.jsonl", run_id="run-1") + log.append(Event("2026-01-01T00:00:01+00:00", EventType.TASK_STARTED, "run-1")) + with self.assertRaises(ValueError): + log.append(Event("2026-01-01T00:00:02+00:00", EventType.RETRY, "run-2")) + with self.assertRaises(ValueError): + log.append(Event("2025-01-01T00:00:00+00:00", EventType.RETRY, "run-1")) + + def test_stage_append_is_causal_and_idempotent(self): + with TemporaryDirectory() as directory: + log = EventLog(Path(directory) / "events.jsonl", run_id="run-1") + self.assertTrue(log.append_stage(timestamp="2026-01-01T00:00:00+00:00", event_type=EventType.EVIDENCE_ADDED, stage="verification", ref="verification://1")) + self.assertFalse(log.append_stage(timestamp="2026-01-01T00:00:01+00:00", event_type=EventType.EVIDENCE_ADDED, stage="verification", ref="verification://1")) + with self.assertRaises(ValueError): + log.append_stage(timestamp="2026-01-01T00:00:02+00:00", event_type=EventType.EVIDENCE_ADDED, stage="verification", ref="verification://2") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_event_replay.py b/tests/test_event_replay.py new file mode 100644 index 00000000..c67a90bb --- /dev/null +++ b/tests/test_event_replay.py @@ -0,0 +1,80 @@ +import unittest + +from arkx.contracts import Event, EventType +from arkx.event_log import replay_events, validate_replay_against_manifest, validate_replay_outcomes, validate_replay_telemetry +from arkx.harness import AcceptanceEvidence, RunManifest, RunState, VerificationEvidence, derive_attempt_id + + +class EventReplayTests(unittest.TestCase): + def test_replay_restores_budget_ledger_and_rejects_drift(self): + ledger = {"budget_digest": "b", "consumed_attempts": ["attempt"]} + event = Event("2026-01-01T00:00:00+00:00", EventType.EVIDENCE_ADDED, "run-1", {"budget_ledger": ledger}) + replay = replay_events((event,), run_id="run-1") + self.assertEqual(replay.budget_ledger, ledger) + drift = Event("2026-01-01T00:00:01+00:00", EventType.EVIDENCE_ADDED, "run-1", {"budget_ledger": {"budget_digest": "other"}}) + with self.assertRaises(ValueError): + replay_events((event, drift), run_id="run-1") + def test_replay_reconstructs_declared_facts_without_execution(self): + events = ( + Event("2026-01-01T00:00:00+00:00", EventType.TASK_STARTED, "run-1"), + Event("2026-01-01T00:00:01+00:00", EventType.EXECUTOR_STARTED, "run-1"), + Event("2026-01-01T00:00:02+00:00", EventType.RETRY, "run-1"), + Event("2026-01-01T00:00:03+00:00", EventType.EVIDENCE_ADDED, "run-1", {"ref": "artifact://1"}), + Event("2026-01-01T00:00:04+00:00", EventType.TASK_FINISHED, "run-1", {"status": "BLOCKED"}), + ) + state = replay_events(events, run_id="run-1") + self.assertEqual((state.event_count, state.executor_invocations, state.retry_count), (5, 1, 1)) + self.assertEqual(state.declared_status, "BLOCKED") + self.assertEqual(state.evidence_refs, ("artifact://1",)) + + def test_replay_rejects_wrong_run_id(self): + with self.assertRaises(ValueError): + replay_events((Event("2026-01-01T00:00:00+00:00", EventType.TASK_STARTED, "other"),), run_id="run-1") + + def test_replay_manifest_consistency_is_validated(self): + trial_id = "trial-task-treatment-1" + manifest = RunManifest( + experiment_id="exp-1", trial_id=trial_id, + attempt_id=derive_attempt_id(trial_id=trial_id, attempt_number=1, configuration_digest="cfg-a"), + attempt_number=1, task_id="task-1", task_revision="task-rev-1", treatment="A", + executor="executor-x", provider="provider-x", model="model-x", sandbox="sandbox-x", + repository_revision="repo-rev-1", configuration_digest="cfg-a", protocol_version="protocol-1", + state=RunState.BLOCKED, + ) + replay = replay_events((Event("2026-01-01T00:00:00+00:00", EventType.TASK_FINISHED, manifest.attempt_id, {"status": "BLOCKED"}),), run_id=manifest.attempt_id) + validate_replay_against_manifest(manifest, replay) + inconsistent = replay_events((Event("2026-01-01T00:00:00+00:00", EventType.TASK_FINISHED, manifest.attempt_id, {"status": "COMPLETED"}),), run_id=manifest.attempt_id) + with self.assertRaises(ValueError): + validate_replay_against_manifest(manifest, inconsistent) + + def test_replay_keeps_verification_and_acceptance_separate(self): + events = ( + Event("2026-01-01T00:00:00+00:00", EventType.EVIDENCE_ADDED, "run-1", {"verification_state": "PASS", "acceptance_decision": "REJECTED"}), + ) + replay = replay_events(events, run_id="run-1") + self.assertEqual((replay.verification_state, replay.acceptance_decision), ("PASS", "REJECTED")) + validate_replay_outcomes(replay, verification=VerificationEvidence("v", "PASS"), acceptance=AcceptanceEvidence("a", "REJECTED")) + with self.assertRaises(ValueError): + validate_replay_outcomes(replay, acceptance=AcceptanceEvidence("a", "ACCEPTED")) + + def test_replay_reconstructs_operational_outcome_fields(self): + event = Event( + "2026-01-01T00:00:00+00:00", + EventType.EVIDENCE_ADDED, + "run-1", + {"stage": "execution", "ref": "execution://1", "execution_outcome": "BLOCKED", "error_code": "SANDBOX_PROCESS_126"}, + ) + replay = replay_events((event,), run_id="run-1") + self.assertEqual(replay.execution_outcome, "BLOCKED") + self.assertEqual(replay.error_codes, ("SANDBOX_PROCESS_126",)) + + def test_replay_telemetry_rejects_verification_drift(self): + event = Event("2026-01-01T00:00:00+00:00", EventType.EVIDENCE_ADDED, "run-1", {"verification_state": "PASS"}) + replay = replay_events((event,), run_id="run-1") + from arkx.outcomes import VerificationResult, VerificationState + with self.assertRaises(ValueError): + validate_replay_telemetry(replay, verification=VerificationResult(VerificationState.FAIL, "v", (), (), ("e",))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_execution.py b/tests/test_execution.py new file mode 100644 index 00000000..6af7f9ee --- /dev/null +++ b/tests/test_execution.py @@ -0,0 +1,252 @@ +import unittest +from decimal import Decimal + +from arkx.execution import ( + BudgetLedger, + CapabilityProfile, + CapabilityRegistry, + ContractRun, + ExecutionBudgetSpec, + ExecutionRequest, + ExecutionPlan, + ExecutionResult, + FakeExecutor, + FakeProvider, + FakeSandbox, + ProviderRequest, + ProviderResponse, + RetryPolicy, + RetryDecision, + plan_retry_attempt, + validate_execution_chain, + validate_provider_identity, + validate_sandbox_identity, + SandboxRequest, +) +from arkx.harness import ErrorDomain, ErrorEnvelope, Retryability, RunState, derive_attempt_id +from arkx.integration import AdapterIdentity, CapabilityProvenance, DependencyObservation, IntegrationKind, assess_preflight +from arkx.configuration import ConfigurationSnapshot + + +class ExecutionContractTests(unittest.TestCase): + def test_budget_ledger_consumes_each_attempt_once_and_is_immutable(self): + ledger = BudgetLedger(ExecutionBudgetSpec(max_attempts=1, max_tokens=10)) + first = ledger.consume("attempt-1", tokens=4) + duplicate = first.ledger.consume("attempt-1", tokens=4) + self.assertTrue(first.accepted) + self.assertEqual(duplicate.reason, "ALREADY_CONSUMED") + self.assertEqual(duplicate.ledger.tokens_used, 4) + self.assertEqual(ledger.tokens_used, 0) + + def test_budget_ledger_rejects_without_mutating_when_limit_exceeded(self): + ledger = BudgetLedger(ExecutionBudgetSpec(max_attempts=1, max_tokens=100)) + first = ledger.consume("attempt-1", tokens=10) + second = first.ledger.consume("attempt-2", tokens=1) + self.assertFalse(second.accepted) + self.assertEqual(second.reason, "ATTEMPTS_EXCEEDED") + self.assertEqual(second.ledger.consumed_attempts, ("attempt-1",)) + + def test_budget_ledger_records_measured_usage_after_consumption(self): + ledger = BudgetLedger(ExecutionBudgetSpec(max_attempts=1, max_tokens=10)) + consumed = ledger.consume("attempt-1") + measured = consumed.ledger.record_usage("attempt-1", tokens=4) + self.assertTrue(measured.accepted) + self.assertEqual(measured.ledger.tokens_used, 4) + + def test_budget_ledger_rejects_usage_without_consumed_attempt(self): + ledger = BudgetLedger(ExecutionBudgetSpec(max_tokens=10)) + with self.assertRaises(ValueError): + ledger.record_usage("missing", tokens=1) + + def test_contract_run_consumes_budget_before_executor(self): + executor = FakeExecutor(ExecutionResult(RunState.COMPLETED)) + ledger = BudgetLedger(ExecutionBudgetSpec(max_attempts=1)) + run = ContractRun.execute_with_budget(self.request(), executor, ledger, attempt_id="attempt-1") + self.assertEqual(run.result.state, RunState.COMPLETED) + self.assertEqual(run.budget_ledger.consumed_attempts, ("attempt-1",)) + self.assertEqual(len(executor.requests), 1) + + def test_contract_run_blocks_before_executor_when_budget_is_exhausted(self): + executor = FakeExecutor(ExecutionResult(RunState.COMPLETED)) + ledger = BudgetLedger(ExecutionBudgetSpec(max_attempts=0)) + run = ContractRun.execute_with_budget(self.request(), executor, ledger, attempt_id="attempt-1") + self.assertEqual(run.result.state, RunState.BLOCKED) + self.assertEqual(run.budget_reason, "ATTEMPTS_EXCEEDED") + self.assertEqual(len(executor.requests), 0) + + def test_retry_policy_requires_retryable_error_and_respects_limits(self): + policy = RetryPolicy(max_retries=1, allowed_domains=(ErrorDomain.PROVIDER,)) + error = ErrorEnvelope(ErrorDomain.PROVIDER, "TEMP", "temporary", Retryability.RETRYABLE) + decision = policy.decide(error, retries_used=0, max_attempts=2) + self.assertEqual((decision.allowed, decision.next_attempt), (True, 2)) + self.assertFalse(policy.decide(error, retries_used=1, max_attempts=2).allowed) + + def test_retry_policy_rejects_unknown_nonretryable_and_wrong_domain(self): + policy = RetryPolicy(max_retries=2, allowed_domains=(ErrorDomain.PROVIDER,)) + unknown = ErrorEnvelope(ErrorDomain.PROVIDER, "UNKNOWN", "unknown") + other = ErrorEnvelope(ErrorDomain.SANDBOX, "TEMP", "temporary", Retryability.RETRYABLE) + self.assertEqual(policy.decide(unknown, retries_used=0).reason, "ERROR_NOT_RETRYABLE") + self.assertEqual(policy.decide(other, retries_used=0).reason, "ERROR_DOMAIN_NOT_ALLOWED") + + def test_retry_attempt_plan_is_deterministic_and_preserves_previous_attempt(self): + policy = RetryPolicy(max_retries=1) + error = ErrorEnvelope(ErrorDomain.PROVIDER, "TEMP", "temporary", Retryability.RETRYABLE) + decision = policy.decide(error, retries_used=0, max_attempts=2) + previous = derive_attempt_id(trial_id="trial", attempt_number=1, configuration_digest="cfg") + plan = plan_retry_attempt(trial_id="trial", previous_attempt_id=previous, previous_attempt_number=1, configuration_digest="cfg", decision=decision) + self.assertEqual(plan.next_attempt_number, 2) + self.assertEqual(plan.to_json(), plan_retry_attempt(trial_id="trial", previous_attempt_id=previous, previous_attempt_number=1, configuration_digest="cfg", decision=decision).to_json()) + + def test_retry_attempt_plan_is_absent_when_retry_is_not_authorized(self): + self.assertIsNone(plan_retry_attempt(trial_id="trial", previous_attempt_id="ignored", previous_attempt_number=1, configuration_digest="cfg", decision=RetryDecision(False, "BLOCKED"))) + + def test_execution_budget_spec_is_validated_and_digest_is_stable(self): + budget = ExecutionBudgetSpec(max_attempts=2, max_tokens=100, max_cost=Decimal("1.25")) + equivalent = ExecutionBudgetSpec(max_attempts=2, max_tokens=100, max_cost=Decimal("1.25")) + self.assertEqual(budget.digest(), equivalent.digest()) + with self.assertRaises(ValueError): + ExecutionBudgetSpec(max_attempts=-1) + + def test_capability_profile_is_deterministic_and_evidence_bearing(self): + profile = CapabilityProfile("provider", "provider-a", "v1", "cfg", ("completion", "completion"), ("artifact://cap",)) + equivalent = CapabilityProfile("provider", "provider-a", "v1", "cfg", ("completion",), ("artifact://cap",)) + self.assertEqual(profile.capabilities, ("completion",)) + self.assertEqual(profile.to_json(), equivalent.to_json()) + + def test_capability_registry_digest_is_stable_and_referenced(self): + first = CapabilityRegistry((CapabilityProfile("provider", "a", "v1", "cfg", ("completion",), ("evidence://cap",)),)) + second = CapabilityRegistry((CapabilityProfile("provider", "a", "v1", "cfg", ("completion",), ("evidence://cap",)),)) + self.assertEqual(first.digest(), second.digest()) + self.assertEqual(first.reference, f"capability://{first.digest()}") + + def test_capability_registry_orders_and_finds_profiles(self): + registry = CapabilityRegistry((CapabilityProfile("sandbox", "b"), CapabilityProfile("provider", "a"))) + self.assertEqual([item.subject_type for item in registry.profiles], ["provider", "sandbox"]) + self.assertEqual(registry.find("provider", "a")[0].subject_id, "a") + + def test_capability_registry_rejects_duplicate_identity(self): + with self.assertRaises(ValueError): + CapabilityRegistry((CapabilityProfile("provider", "a"), CapabilityProfile("provider", "a"))) + + def test_execution_plan_applies_capability_policy_before_execution(self): + plan = ExecutionPlan( + "plan-cap", "task", "rev", "A", "executor", None, None, "sandbox", None, "cfg", + ("execute",), required_capabilities=("patch",), + minimum_capability_provenance=CapabilityProvenance.OBSERVED, + ) + preflight = assess_preflight( + AdapterIdentity(IntegrationKind.EXECUTOR, "executor", "1", "cfg"), + (DependencyObservation("runtime", True, "1"),), + capabilities=("patch",), capability_digest="capability-digest", + capability_provenance=CapabilityProvenance.DECLARED, + ) + decision = plan.validate_capabilities(preflight) + self.assertFalse(decision.allowed) + + def test_execution_plan_rejects_preflight_from_another_executor(self): + plan = ExecutionPlan("plan-id", "task", "rev", "A", "executor", None, None, "sandbox", None, "cfg") + preflight = assess_preflight( + AdapterIdentity(IntegrationKind.EXECUTOR, "other", "1", "other-cfg"), + (DependencyObservation("runtime", True, "1"),), + ) + self.assertFalse(plan.preflight_matches_identity(preflight)) + + def request(self): + return ExecutionRequest("task", "rev", "A", "prompt", "executor", "provider", "model", "sandbox", "budget", "config") + + def test_fake_executor_is_replaceable_and_does_not_accept(self): + result = ExecutionResult(RunState.COMPLETED, ("diff.patch",), output="done") + executor = FakeExecutor(result) + self.assertEqual(executor.execute(self.request()), result) + self.assertEqual(len(executor.requests), 1) + + def test_failed_and_blocked_results_require_structured_error(self): + error = ErrorEnvelope(ErrorDomain.PROVIDER, "UNAVAILABLE", "provider unavailable", Retryability.RETRYABLE) + self.assertEqual(ExecutionResult(RunState.BLOCKED, error=error).error, error) + with self.assertRaises(ValueError): + ExecutionResult(RunState.FAILED) + + def test_provider_and_sandbox_are_independent_boundaries(self): + provider = FakeProvider(ProviderResponse("answer", {"total_tokens": 3})) + sandbox = FakeSandbox(("diff.patch",)) + provider.complete(ProviderRequest("prompt", "model", "config", "budget")) + sandbox.prepare(SandboxRequest("task", "rev", "sandbox", "policy")) + self.assertEqual(provider.response.output, "answer") + self.assertEqual(sandbox.collect_artifacts(), ("diff.patch",)) + + def test_provider_requires_versioned_configuration_bound_identity(self): + provider = FakeProvider(ProviderResponse("answer")) + self.assertEqual(validate_provider_identity(provider).name, "fake-provider") + class AnonymousProvider: + def complete(self, request): + return ProviderResponse("answer") + with self.assertRaises(ValueError): + validate_provider_identity(AnonymousProvider()) + + def test_sandbox_requires_versioned_configuration_bound_identity(self): + sandbox = FakeSandbox() + self.assertEqual(validate_sandbox_identity(sandbox).name, "fake-sandbox") + class AnonymousSandbox: + def prepare(self, request): + return None + def collect_artifacts(self): + return () + with self.assertRaises(ValueError): + validate_sandbox_identity(AnonymousSandbox()) + + def test_unknown_provider_identity_is_preserved(self): + request = self.request() + value = ExecutionRequest(request.task_id, request.task_revision, request.treatment, request.prompt, request.executor_id, None, None, request.sandbox_id, request.budget_digest, request.configuration_digest) + self.assertIsNone(value.provider_id) + self.assertIsNone(value.model_id) + + def test_execution_plan_is_serializable_and_separate_from_result(self): + plan = ExecutionPlan("plan-1", "task", "rev", "A", "executor", "provider", "model", "sandbox", "budget", "config", ("prepare", "execute")) + request = plan.to_request("prompt") + self.assertEqual(request.task_id, plan.task_id) + self.assertEqual(plan.to_json(), plan.to_json()) + self.assertNotIn("state", plan.to_dict()) + + def test_execution_plan_requires_identity(self): + with self.assertRaises(ValueError): + ExecutionPlan("", "task", None, "A", "executor", None, None, None, None, None) + + def test_execution_plan_binds_budget_digest(self): + budget = ExecutionBudgetSpec(max_attempts=2, max_tokens=10) + plan = ExecutionPlan("plan-1", "task", None, "A", "executor", None, None, None, None, None, (), budget) + self.assertEqual(plan.budget_digest, budget.digest()) + with self.assertRaises(ValueError): + ExecutionPlan("plan-2", "task", None, "A", "executor", None, None, None, "wrong", None, (), budget) + + def test_execution_plan_binds_configuration_snapshot_digest(self): + snapshot = ConfigurationSnapshot("executor", "1.0", {"mode": "strict"}) + plan = ExecutionPlan("plan-config", "task", None, "A", "executor", None, None, None, None, None, (), configuration_snapshot=snapshot) + self.assertEqual(plan.configuration_digest, snapshot.digest()) + with self.assertRaises(ValueError): + ExecutionPlan("plan-config-bad", "task", None, "A", "executor", None, None, None, None, "other", (), configuration_snapshot=snapshot) + + def test_execution_plan_binds_provider_and_sandbox_identities(self): + provider = AdapterIdentity(IntegrationKind.PROVIDER, "provider-a", "1", "provider-config") + sandbox = AdapterIdentity(IntegrationKind.SANDBOX, "sandbox-a", "1", "sandbox-config") + plan = ExecutionPlan("plan-integrations", "task", None, "A", "executor", "provider-a", None, "sandbox-a", None, None, (), provider_identity=provider, sandbox_identity=sandbox) + self.assertEqual(plan.to_dict()["provider_identity"]["name"], "provider-a") + with self.assertRaises(ValueError): + ExecutionPlan("plan-bad-provider", "task", None, "A", "executor", "other", None, None, None, None, (), provider_identity=provider) + + def test_execution_chain_references_bind_plan_and_attempt(self): + plan = ExecutionPlan("plan-1", "task", None, "A", "executor", None, None, None, None, None) + validate_execution_chain(plan, plan_reference=plan.reference, execution_ref="execution://attempt-1", attempt_id="attempt-1") + with self.assertRaises(ValueError): + validate_execution_chain(plan, plan_reference="plan://other", execution_ref="execution://attempt-1", attempt_id="attempt-1") + + def test_public_execution_result_is_the_neutral_contract(self): + import arkx + from arkx.execution import ExecutionResult as NeutralExecutionResult + from arkx.orchestration import ExecutionResult as LegacyExecutionResult + self.assertIs(arkx.ExecutionResult, NeutralExecutionResult) + self.assertIs(arkx.OrchestrationExecutionResult, LegacyExecutionResult) + self.assertIsNot(arkx.ExecutionResult, arkx.OrchestrationExecutionResult) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_execution_lifecycle.py b/tests/test_execution_lifecycle.py new file mode 100644 index 00000000..32d6bae5 --- /dev/null +++ b/tests/test_execution_lifecycle.py @@ -0,0 +1,31 @@ +import unittest + +from arkx.execution import ContractRun, ExecutionRequest, ExecutionResult, FakeExecutor +from arkx.harness import AcceptanceEvidence, ErrorDomain, ErrorEnvelope, Retryability, RunState, VerificationEvidence + + +class ContractLifecycleTests(unittest.TestCase): + def request(self): + return ExecutionRequest("task", "rev", "A", "prompt", "executor", "provider", "model", "sandbox", "budget", "config") + + def test_fake_flow_preserves_lifecycle_and_does_not_infer_acceptance(self): + run = ContractRun.execute(self.request(), FakeExecutor(ExecutionResult(RunState.COMPLETED, ("diff",)))) + self.assertEqual(run.state_history, (RunState.PLANNED, RunState.STARTED, RunState.EXECUTING, RunState.COMPLETED)) + self.assertFalse(run.is_accepted()) + + def test_verification_and_acceptance_are_attached_independently(self): + run = ContractRun.execute(self.request(), FakeExecutor(ExecutionResult(RunState.COMPLETED, ("diff",)))) + run = run.with_verification(VerificationEvidence("verifier", "PASS", ("verification://1",))) + self.assertFalse(run.is_accepted()) + run = run.with_acceptance(AcceptanceEvidence("acceptance", "ACCEPTED", ("acceptance://1",))) + self.assertTrue(run.is_accepted()) + + def test_blocked_flow_requires_error_and_is_never_accepted(self): + error = ErrorEnvelope(ErrorDomain.SANDBOX, "UNAVAILABLE", "sandbox unavailable", Retryability.NOT_RETRYABLE) + run = ContractRun.execute(self.request(), FakeExecutor(ExecutionResult(RunState.BLOCKED, error=error))) + self.assertEqual(run.state_history[-1], RunState.BLOCKED) + self.assertFalse(run.is_accepted()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_executor_qualification.py b/tests/test_executor_qualification.py new file mode 100644 index 00000000..2b0b231c --- /dev/null +++ b/tests/test_executor_qualification.py @@ -0,0 +1,159 @@ +import json +import unittest +from decimal import Decimal +from pathlib import Path + +from arkx.composition import Mechanism, Treatment +from arkx.executor_qualification import ( + ComparisonAxis, + ExecutionEnvironment, + ExecutionOutcome, + ExecutionBudget, + ExecutorIdentity, + ExecutorObservation, + ExecutorTrial, + QualificationExperiment, + QualificationStatus, + QualificationTask, + assess_trial, + compare_trials, + summarize_experiment, + validate_paired_executor_experiment, +) + + +def trial(**overrides): + values = { + "executor": ExecutorIdentity("executor", "1.0", "external", "config-a"), + "task": QualificationTask("task", "rev-1", "acceptance-v1"), + "model": "model-1", + "environment": ExecutionEnvironment("repo-1", "python-3.11", "windows", "toolchain-1", "env-a"), + "budget": ExecutionBudget(1000, Decimal("2.00"), Decimal("60"), 3, 3), + "treatment": Treatment("A", ()), + "replicate_id": "rep-1", + "outcome": ExecutionOutcome.COMPLETED, + "observation": ExecutorObservation(accepted=True, patch_verified=True, cost=Decimal("1.20"), wall_time_seconds=Decimal("4")), + } + values.update(overrides) + return ExecutorTrial(**values) + + +class ExecutorQualificationTests(unittest.TestCase): + def test_executor_identity_serialization_is_deterministic(self): + identity = ExecutorIdentity("mini", "1.2", "adapter", "digest", ("shell", "file_editing")) + self.assertEqual(identity.to_json(), '{"advertised_capabilities":["file_editing","shell"],"configuration_digest":"digest","integration_kind":"adapter","name":"mini","version":"1.2"}') + self.assertEqual(identity.to_json(), ExecutorIdentity("mini", "1.2", "adapter", "digest", ("file_editing", "shell")).to_json()) + self.assertNotEqual(identity, ExecutorIdentity("mini", "1.3", "adapter", "digest", ("shell", "file_editing"))) + + def test_advertised_capabilities_are_metadata_not_qualification(self): + advertised = ExecutorIdentity("mini", "1.2", "adapter", "digest", ("supports_subagents",)) + trial_with_unknown_result = trial(executor=advertised, observation=ExecutorObservation(accepted=None)) + self.assertEqual(assess_trial(trial_with_unknown_result), QualificationStatus.UNKNOWN) + + def test_unknown_metrics_stay_none_and_explicit_zero_stays_zero(self): + observation = ExecutorObservation(human_interventions=0, retries=None, accepted=None) + value = observation.to_dict() + self.assertEqual(value["human_interventions"], 0) + self.assertIsNone(value["retries"]) + self.assertIsNone(value["accepted"]) + + def test_executor_axis_requires_same_controls_and_different_executor(self): + left = trial() + right = trial(executor=ExecutorIdentity("other", "1.0", "external", "config-b"), replicate_id="rep-2") + self.assertEqual(compare_trials(left, right, ComparisonAxis.EXECUTOR), QualificationStatus.QUALIFIABLE) + changed = trial(task=QualificationTask("other-task", "rev-1", "acceptance-v1"), replicate_id="rep-2") + self.assertEqual(compare_trials(left, changed, ComparisonAxis.EXECUTOR), QualificationStatus.INCOMPARABLE) + + def test_treatment_axis_requires_same_executor_and_different_treatment(self): + left = trial() + right = trial(treatment=Treatment("B", (Mechanism.P1_CHARACTERIZATION,)), replicate_id="rep-2") + self.assertEqual(compare_trials(left, right, ComparisonAxis.TREATMENT), QualificationStatus.QUALIFIABLE) + changed = trial(executor=ExecutorIdentity("other", "1.0", "external", "config-b"), replicate_id="rep-2") + self.assertEqual(compare_trials(left, changed, ComparisonAxis.TREATMENT), QualificationStatus.INCOMPARABLE) + + def test_infrastructure_failure_is_blocked_not_rejected(self): + failed = trial(outcome=ExecutionOutcome.INFRASTRUCTURE_FAILED, observation=ExecutorObservation(accepted=None)) + self.assertEqual(assess_trial(failed), QualificationStatus.BLOCKED) + self.assertIsNone(failed.observation.accepted) + + def test_completed_trial_does_not_infer_acceptance(self): + unknown = trial(observation=ExecutorObservation(accepted=None)) + self.assertEqual(assess_trial(unknown), QualificationStatus.UNKNOWN) + + def test_patch_verification_does_not_infer_acceptance(self): + unknown = trial(observation=ExecutorObservation(accepted=None, patch_verified=True)) + self.assertIsNone(unknown.observation.accepted) + self.assertEqual(assess_trial(unknown), QualificationStatus.UNKNOWN) + + def test_inconsistent_false_pass_observation_is_blocked(self): + invalid = trial(observation=ExecutorObservation(accepted=False, patch_verified=False, false_pass=True)) + self.assertEqual(assess_trial(invalid), QualificationStatus.BLOCKED) + + def test_summary_has_no_winner_and_preserves_missing_metrics(self): + experiment = QualificationExperiment((trial(), trial(observation=ExecutorObservation(accepted=None))), ComparisonAxis.EXECUTOR) + summary = summarize_experiment(experiment) + self.assertEqual(summary.qualifiable_trial_count, 1) + self.assertEqual(summary.acceptance_rate, Decimal("1")) + self.assertIsNone(summary.false_pass_rate) + self.assertEqual(summary.median_wall_time_seconds, Decimal("4")) + self.assertNotIn("winner", summary.to_json()) + self.assertNotIn("PASS", summary.to_json()) + + def test_zero_metric_is_included_in_summary(self): + zero = trial(observation=ExecutorObservation(accepted=False, cost=Decimal("0"), wall_time_seconds=Decimal("0"))) + summary = summarize_experiment(QualificationExperiment((zero,), ComparisonAxis.EXECUTOR)) + self.assertEqual(summary.median_cost, Decimal("0")) + self.assertEqual(summary.median_wall_time_seconds, Decimal("0")) + + def test_replicates_are_preserved_and_serialized_deterministically(self): + experiment = QualificationExperiment((trial(replicate_id="rep-2"), trial(replicate_id="rep-1")), ComparisonAxis.EXECUTOR) + payload = json.loads(experiment.to_json()) + self.assertEqual([item["replicate_id"] for item in payload["trials"]], ["rep-1", "rep-2"]) + + def test_module_has_no_execution_integration_imports(self): + source = (Path(__file__).parents[1] / "src" / "arkx" / "executor_qualification.py").read_text(encoding="utf-8") + for forbidden in ("subprocess", "openhands", "swe-agent", "mini-swe-agent", "requests", "httpx"): + self.assertNotIn(forbidden, source.lower()) + + def test_fixture_is_classified_as_qualification_fixture(self): + path = Path(__file__).parents[1] / "experiments" / "executor-qualification-fixture.json" + value = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(value["fixture_type"], "EXECUTOR_QUALIFICATION_FIXTURE") + self.assertEqual(value["comparison_axes"], ["EXECUTOR", "TREATMENT"]) + + def test_paired_protocol_requires_shared_verifier_and_instrumentation(self): + experiment = QualificationExperiment( + (trial(), trial(executor=ExecutorIdentity("other", "1.0", "external", "config-b"), replicate_id="rep-1")), + ComparisonAxis.EXECUTOR, + ) + report = validate_paired_executor_experiment(experiment) + self.assertEqual(report.status, QualificationStatus.UNKNOWN) + self.assertIn("VERIFIER_IDENTITY_MISSING", report.reason_codes) + self.assertIn("INSTRUMENTATION_IDENTITY_MISSING", report.reason_codes) + + def test_paired_protocol_accepts_complete_controlled_cells_without_ranking(self): + experiment = QualificationExperiment( + (trial(), trial(executor=ExecutorIdentity("other", "1.0", "external", "config-b"), replicate_id="rep-1")), + ComparisonAxis.EXECUTOR, + verifier_identity="verifier-v1", + instrumentation_identity="telemetry-v1", + ) + report = validate_paired_executor_experiment(experiment) + self.assertEqual(report.status, QualificationStatus.QUALIFIABLE) + self.assertEqual(report.executor_names, ("executor", "other")) + self.assertNotIn("winner", report.to_json()) + + def test_paired_protocol_rejects_missing_executor_cell(self): + experiment = QualificationExperiment( + (trial(),), + ComparisonAxis.EXECUTOR, + verifier_identity="verifier-v1", + instrumentation_identity="telemetry-v1", + ) + report = validate_paired_executor_experiment(experiment) + self.assertEqual(report.status, QualificationStatus.INCOMPARABLE) + self.assertIn("AT_LEAST_TWO_EXECUTORS_REQUIRED", report.reason_codes) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_handoff.py b/tests/test_handoff.py new file mode 100644 index 00000000..6cf22137 --- /dev/null +++ b/tests/test_handoff.py @@ -0,0 +1,74 @@ +import json +import unittest +from pathlib import Path + +from arkx.handoff import HandoffBudgetStatus, HandoffPolicy, HandoffRecord, summarize_handoffs + + +def record(identifier="h1", **overrides): + values = { + "handoff_id": identifier, + "source_executor": "a", + "target_executor": "b", + "reason": "capacity", + "evidence_refs": ("e://handoff",), + "context_summary": "summary", + "context_bytes_in": 100, + "context_bytes_out": 80, + "duplicated_instructions": 1, + "duplicated_exploration": 2, + "discarded_context": 3, + "lost_information": (), + } + values.update(overrides) + return HandoffRecord(**values) + + +class HandoffTests(unittest.TestCase): + def test_count_and_transition_are_deterministic(self): + summary = summarize_handoffs((record(),)) + self.assertEqual(summary.handoffs, 1) + self.assertEqual(summary.executor_transitions, (("a", "b"),)) + + def test_context_bytes_and_duplicate_metrics_are_preserved(self): + summary = summarize_handoffs((record(),)) + self.assertEqual(summary.context_bytes_in, 100) + self.assertEqual(summary.context_bytes_out, 80) + self.assertEqual(summary.duplicated_instructions, 1) + self.assertEqual(summary.duplicated_exploration, 2) + + def test_unknown_measurements_are_not_zero(self): + summary = summarize_handoffs((record(context_bytes_in=None, context_bytes_out=None),)) + self.assertIsNone(summary.context_bytes_in) + self.assertIsNone(summary.context_bytes_out) + + def test_transitions_require_known_source_and_target(self): + summary = summarize_handoffs((record(source_executor=None),)) + self.assertEqual(summary.executor_transitions, ()) + self.assertEqual(summary.budget_status, HandoffBudgetStatus.UNKNOWN) + + def test_lost_information_is_explicit(self): + explicit = summarize_handoffs((record(lost_information=("constraint",)),)) + unknown = summarize_handoffs((record(lost_information=None),)) + self.assertEqual(explicit.lost_information_count, 1) + self.assertIsNone(unknown.lost_information_count) + + def test_budget_exceeded_is_observable(self): + summary = summarize_handoffs((record("h1"), record("h2"), record("h3")), HandoffPolicy(max_handoffs=2)) + self.assertEqual(summary.budget_status, HandoffBudgetStatus.BUDGET_EXCEEDED) + + def test_same_records_have_same_summary(self): + first = summarize_handoffs((record(),)).to_json() + second = summarize_handoffs((record(),)).to_json() + self.assertEqual(first, second) + + def test_fixture_is_classified_as_handoff_fixture(self): + path = Path(__file__).parents[1] / "experiments" / "handoff-accounting-fixture.json" + value = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(value["fixture_type"], "HANDOFF_ACCOUNTING_FIXTURE") + self.assertEqual(len(value["scenarios"]), 7) + + +if __name__ == "__main__": + unittest.main() + diff --git a/tests/test_harness.py b/tests/test_harness.py new file mode 100644 index 00000000..2fbe0e11 --- /dev/null +++ b/tests/test_harness.py @@ -0,0 +1,475 @@ +import unittest +import json +from pathlib import Path +from tempfile import TemporaryDirectory + +from arkx.harness import ( + ArtifactStore, + ArtifactRecord, + ArtifactStatus, + ErrorDomain, + ErrorEnvelope, + LifecycleEvent, + Retryability, + RecoveryLineage, + RetryLineage, + RunManifest, + RunState, + derive_attempt_id, + validate_execution_artifact_consistency, + load_attempt, + AcceptanceEvidence, + VerificationEvidence, + ExperimentRecord, + load_experiment_record, + write_experiment_record, + AttemptStore, + ComparabilityStatus, + compare_attempts, + compare_attempt_collection, + evidence_from_snapshot, + validate_transition, +) + + +class HarnessContractTests(unittest.TestCase): + def manifest(self, **changes): + values = { + "experiment_id": "exp-1", + "trial_id": "trial-task-treatment-1", + "attempt_id": derive_attempt_id( + trial_id="trial-task-treatment-1", + attempt_number=1, + configuration_digest="cfg-a", + ), + "attempt_number": 1, + "task_id": "task-1", + "task_revision": "task-rev-1", + "treatment": "A", + "executor": "executor-x", + "provider": "provider-x", + "model": "model-x", + "sandbox": "sandbox-x", + "repository_revision": "repo-rev-1", + "configuration_digest": "cfg-a", + "protocol_version": "protocol-1", + } + values.update(changes) + return RunManifest(**values) + + def test_manifest_has_distinct_trial_and_attempt_identity(self): + first = self.manifest() + second = self.manifest( + attempt_number=2, + attempt_id=derive_attempt_id( + trial_id=first.trial_id, + attempt_number=2, + configuration_digest=first.configuration_digest, + ), + ) + self.assertNotEqual(first.trial_id, first.attempt_id) + self.assertNotEqual(first.attempt_id, second.attempt_id) + + def test_attempt_identity_is_stable_and_configuration_sensitive(self): + first = derive_attempt_id(trial_id="t", attempt_number=1, configuration_digest="a") + self.assertEqual(first, derive_attempt_id(trial_id="t", attempt_number=1, configuration_digest="a")) + self.assertNotEqual(first, derive_attempt_id(trial_id="t", attempt_number=1, configuration_digest="b")) + + def test_error_taxonomy_preserves_retryability_and_raw_evidence(self): + error = ErrorEnvelope( + domain=ErrorDomain.PROVIDER, + code="PROVIDER_OVERLOADED", + message="temporary overload", + retryability=Retryability.RETRYABLE, + raw_evidence_refs=("raw-b", "raw-a", "raw-a"), + attempt_number=2, + ) + self.assertEqual(error.to_dict()["raw_evidence_refs"], ["raw-a", "raw-b"]) + self.assertEqual(error.to_dict()["retryability"], "RETRYABLE") + + def test_manifest_serialization_is_deterministic(self): + first = self.manifest( + state=RunState.BLOCKED, + artifact_refs=("z", "a", "a"), + errors=(ErrorEnvelope(ErrorDomain.SANDBOX, "SANDBOX_UNAVAILABLE", "daemon unavailable"),), + ) + second = self.manifest( + state=RunState.BLOCKED, + artifact_refs=("a", "z"), + errors=(ErrorEnvelope(ErrorDomain.SANDBOX, "SANDBOX_UNAVAILABLE", "daemon unavailable"),), + ) + self.assertEqual(first.to_json(), second.to_json()) + + def test_manifest_round_trip_supports_replay_validation(self): + original = self.manifest(state=RunState.BLOCKED) + restored = RunManifest.from_json(original.to_json()) + restored.validate_attempt_identity() + self.assertEqual(restored.to_json(), original.to_json()) + + def test_replay_validation_rejects_identity_drift(self): + original = self.manifest() + drifted = self.manifest(attempt_id="different-attempt") + with self.assertRaises(ValueError): + drifted.validate_attempt_identity() + + def test_lifecycle_validation_requires_error_for_blocked(self): + blocked = self.manifest(state=RunState.BLOCKED) + with self.assertRaises(ValueError): + blocked.validate_lifecycle() + valid = self.manifest( + state=RunState.BLOCKED, + errors=(ErrorEnvelope(ErrorDomain.SANDBOX, "SANDBOX_UNAVAILABLE", "daemon unavailable"),), + ) + valid.validate_lifecycle() + + def test_lifecycle_validation_rejects_completed_with_error(self): + completed = self.manifest( + state=RunState.COMPLETED, + errors=(ErrorEnvelope(ErrorDomain.EXECUTOR, "EXECUTOR_FAILED", "failed"),), + ) + with self.assertRaises(ValueError): + completed.validate_lifecycle() + + def test_lifecycle_transitions_are_explicit(self): + validate_transition(RunState.PLANNED, RunState.STARTED) + validate_transition(RunState.STARTED, RunState.EXECUTING) + validate_transition(RunState.EXECUTING, RunState.BLOCKED) + with self.assertRaises(ValueError): + validate_transition(RunState.PLANNED, RunState.COMPLETED) + with self.assertRaises(ValueError): + validate_transition(RunState.COMPLETED, RunState.EXECUTING) + + def test_state_history_is_validated(self): + valid = self.manifest( + state=RunState.COMPLETED, + state_history=(RunState.PLANNED, RunState.STARTED, RunState.EXECUTING, RunState.COMPLETED), + ) + valid.validate_lifecycle() + invalid = self.manifest( + state=RunState.COMPLETED, + state_history=(RunState.PLANNED, RunState.COMPLETED), + ) + with self.assertRaises(ValueError): + invalid.validate_lifecycle() + + def test_lifecycle_events_match_history_and_are_monotonic(self): + events = ( + LifecycleEvent(RunState.PLANNED, "2026-01-01T00:00:00+00:00"), + LifecycleEvent(RunState.STARTED, "2026-01-01T00:00:01+00:00"), + ) + manifest = self.manifest(state=RunState.STARTED, state_history=(RunState.PLANNED, RunState.STARTED), lifecycle_events=events) + manifest.validate_lifecycle() + invalid = self.manifest(state=RunState.STARTED, state_history=(RunState.PLANNED, RunState.STARTED), lifecycle_events=tuple(reversed(events))) + with self.assertRaises(ValueError): + invalid.validate_lifecycle() + + def test_missing_identity_is_preserved_for_blocked_runs(self): + manifest = self.manifest(provider=None, state=RunState.BLOCKED) + self.assertIsNone(manifest.to_dict()["provider"]) + self.assertEqual(manifest.to_dict()["state"], "BLOCKED") + + def test_invalid_attempt_number_is_rejected(self): + with self.assertRaises(ValueError): + derive_attempt_id(trial_id="t", attempt_number=0, configuration_digest="cfg") + + def test_artifact_store_reserves_once_and_writes_atomically(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + attempt = store.reserve_attempt("task-1", 1) + target = store.write_text_atomic(attempt / "execution.json", "{}\n") + self.assertEqual(Path(target).read_text(encoding="utf-8"), "{}\n") + with self.assertRaises(FileExistsError): + store.reserve_attempt("task-1", 1) + + def test_artifact_store_rejects_path_escape(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + with self.assertRaises(ValueError): + store.reserve_attempt("../escape", 1) + + def test_artifact_store_reserves_retry_and_persists_lineage(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + previous = derive_attempt_id(trial_id="trial", attempt_number=1, configuration_digest="cfg") + path = store.reserve_retry_attempt("task-1", trial_id="trial", previous_attempt_id=previous, previous_attempt_number=1, configuration_digest="cfg") + lineage = json.loads((path / "retry-lineage.json").read_text(encoding="utf-8")) + self.assertEqual(lineage["previous_attempt_id"], previous) + self.assertEqual(lineage["attempt_number"], 2) + with self.assertRaises(FileExistsError): + store.reserve_retry_attempt("task-1", trial_id="trial", previous_attempt_id=previous, previous_attempt_number=1, configuration_digest="cfg") + + def test_artifact_store_rejects_retry_identity_drift(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + with self.assertRaises(ValueError): + store.reserve_retry_attempt("task-1", trial_id="trial", previous_attempt_id="wrong", previous_attempt_number=1, configuration_digest="cfg") + + def test_artifact_store_reserves_recovery_attempt_with_causal_lineage(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + source = derive_attempt_id(trial_id="trial", attempt_number=1, configuration_digest="cfg") + path = store.reserve_recovery_attempt("task-1", trial_id="trial", source_attempt_id=source, source_attempt_number=1, configuration_digest="cfg", recovery_reference="recovery://1") + lineage = json.loads((path / "recovery-lineage.json").read_text(encoding="utf-8")) + self.assertEqual(lineage["source_attempt_id"], source) + self.assertNotEqual(lineage["attempt_id"], source) + with self.assertRaises(FileExistsError): + store.reserve_recovery_attempt("task-1", trial_id="trial", source_attempt_id=source, source_attempt_number=1, configuration_digest="cfg", recovery_reference="recovery://1") + + def test_retry_lineage_round_trips_and_validates_manifest_link(self): + lineage = RetryLineage.from_dict({ + "trial_id": "trial", "previous_attempt_id": derive_attempt_id(trial_id="trial", attempt_number=1, configuration_digest="cfg"), + "previous_attempt_number": 1, "attempt_id": derive_attempt_id(trial_id="trial", attempt_number=2, configuration_digest="cfg"), + "attempt_number": 2, "configuration_digest": "cfg", + }) + self.assertEqual(RetryLineage.from_dict(lineage.to_dict()), lineage) + with self.assertRaises(ValueError): + RetryLineage.from_dict({**lineage.to_dict(), "attempt_id": "wrong"}) + + def test_recovery_lineage_round_trips_and_validates_identity(self): + lineage = RecoveryLineage.from_dict({ + "trial_id": "trial", "source_attempt_id": derive_attempt_id(trial_id="trial", attempt_number=1, configuration_digest="cfg"), + "source_attempt_number": 1, "attempt_id": derive_attempt_id(trial_id="trial", attempt_number=2, configuration_digest="cfg"), + "attempt_number": 2, "configuration_digest": "cfg", "recovery_reference": "recovery://1", + }) + self.assertEqual(RecoveryLineage.from_dict(lineage.to_dict()), lineage) + with self.assertRaises(ValueError): + RecoveryLineage.from_dict({**lineage.to_dict(), "source_attempt_id": "wrong"}) + + def test_manifest_binds_budget_ledger_to_attempt_and_digest(self): + manifest = self.manifest(budget_digest="budget-digest") + ledger = {"budget_digest": "budget-digest", "consumed_attempts": [manifest.attempt_id], "tokens_used": 2} + bound = manifest.with_budget_ledger(ledger) + self.assertEqual(bound.budget_ledger["consumed_attempts"], [manifest.attempt_id]) + with self.assertRaises(ValueError): + manifest.with_budget_ledger({"budget_digest": "other", "consumed_attempts": [manifest.attempt_id]}) + + def test_artifact_store_writes_bound_manifest_atomically(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + attempt_root = store.reserve_attempt("task-1", 1) + manifest = self.manifest(budget_digest="budget-digest") + bound = manifest.with_budget_ledger({"budget_digest": "budget-digest", "consumed_attempts": [manifest.attempt_id]}) + target = store.write_manifest(attempt_root, bound) + loaded = RunManifest.from_json(Path(target).read_text(encoding="utf-8")) + self.assertEqual(loaded.budget_ledger["budget_digest"], "budget-digest") + + def test_artifact_store_manifest_must_match_recovery_lineage(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + source = derive_attempt_id(trial_id="trial", attempt_number=1, configuration_digest="cfg") + attempt_root = store.reserve_recovery_attempt("task-1", trial_id="trial", source_attempt_id=source, source_attempt_number=1, configuration_digest="cfg", recovery_reference="recovery://1") + with self.assertRaises(ValueError): + store.write_manifest(attempt_root, self.manifest(attempt_id="wrong", trial_id="trial", configuration_digest="cfg")) + + def test_artifact_store_recovery_respects_persisted_budget_ledger(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + source = derive_attempt_id(trial_id="trial", attempt_number=1, configuration_digest="cfg") + ledger = {"consumed_attempts": [source], "budget": {"max_attempts": 1}} + with self.assertRaises(ValueError): + store.reserve_recovery_attempt("task-1", trial_id="trial", source_attempt_id=source, source_attempt_number=1, configuration_digest="cfg", recovery_reference="recovery://1", budget_ledger=ledger) + self.assertFalse((Path(directory) / "task-1" / "attempt-2").exists()) + + def test_artifact_store_retry_respects_persisted_budget_ledger(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + source = derive_attempt_id(trial_id="trial", attempt_number=1, configuration_digest="cfg") + ledger = {"consumed_attempts": [source], "budget": {"max_attempts": 1}} + with self.assertRaises(ValueError): + store.reserve_retry_attempt("task-1", trial_id="trial", previous_attempt_id=source, previous_attempt_number=1, configuration_digest="cfg", budget_ledger=ledger) + self.assertFalse((Path(directory) / "task-1" / "attempt-2").exists()) + + def test_artifact_store_persists_updated_budget_ledger_for_retry(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + source = derive_attempt_id(trial_id="trial", attempt_number=1, configuration_digest="cfg") + ledger = {"consumed_attempts": [source], "budget": {"max_attempts": 2}} + path = store.reserve_retry_attempt("task-1", trial_id="trial", previous_attempt_id=source, previous_attempt_number=1, configuration_digest="cfg", budget_ledger=ledger) + persisted = json.loads((path / "budget-ledger.json").read_text(encoding="utf-8")) + self.assertEqual(len(persisted["consumed_attempts"]), 2) + self.assertNotEqual(persisted, ledger) + + def test_manifest_must_match_persisted_budget_ledger(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + source = derive_attempt_id(trial_id="trial", attempt_number=1, configuration_digest="cfg") + ledger = {"consumed_attempts": [source], "budget": {"max_attempts": 2}} + path = store.reserve_retry_attempt("task-1", trial_id="trial", previous_attempt_id=source, previous_attempt_number=1, configuration_digest="cfg", budget_ledger=ledger) + attempt_id = derive_attempt_id(trial_id="trial", attempt_number=2, configuration_digest="cfg") + manifest = self.manifest(trial_id="trial", attempt_id=attempt_id, attempt_number=2, configuration_digest="cfg", budget_ledger=ledger) + with self.assertRaises(ValueError): + store.write_manifest(path, manifest) + + def test_artifact_store_validates_manifest_references(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + attempt = store.reserve_attempt("task-1", 1) + artifact = store.write_text_atomic(attempt / "execution.json", "{}\n") + manifest = self.manifest(artifact_refs=(str(artifact),), state=RunState.COMPLETED) + self.assertEqual(store.validate_manifest_references(manifest), (artifact.resolve(),)) + missing = self.manifest(artifact_refs=(str(attempt / "missing.json"),), state=RunState.COMPLETED) + with self.assertRaises(FileNotFoundError): + store.validate_manifest_references(missing) + + def test_required_artifacts_cannot_be_missing(self): + manifest = self.manifest( + state=RunState.BLOCKED, + artifacts=(ArtifactRecord("execution", ArtifactStatus.MISSING, None, True, "not written"),), + errors=(ErrorEnvelope(ErrorDomain.HARNESS, "WRITE_FAILED", "artifact write failed"),), + ) + with self.assertRaises(ValueError): + manifest.validate_lifecycle() + + def test_execution_artifact_must_match_manifest(self): + manifest = self.manifest(state=RunState.COMPLETED) + execution = { + "run_id": manifest.attempt_id, + "trial_id": manifest.trial_id, + "attempt": manifest.attempt_number, + "state": "COMPLETED", + "error": None, + "manifest_path": "manifest.json", + } + validate_execution_artifact_consistency(manifest, execution) + execution["trial_id"] = "other-trial" + with self.assertRaises(ValueError): + validate_execution_artifact_consistency(manifest, execution) + + def test_load_attempt_requires_both_canonical_records(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + attempt = store.reserve_attempt("task-1", 1) + with self.assertRaises(FileNotFoundError): + load_attempt(attempt) + + def test_attempt_store_lists_validated_attempts_without_selecting_one(self): + with TemporaryDirectory() as directory: + store = ArtifactStore(directory) + for number in (1, 2): + attempt = store.reserve_attempt("task-1", number) + execution = attempt / "execution.json" + store.write_text_atomic(execution, "{}\n") + manifest = self.manifest( + attempt_number=number, + attempt_id=derive_attempt_id( + trial_id="trial-task-treatment-1", + attempt_number=number, + configuration_digest="cfg-a", + ), + state=RunState.COMPLETED, + artifact_refs=(str(execution),), + ) + # The index test uses canonical records with matching identity. + payload = { + "run_id": manifest.attempt_id, + "trial_id": manifest.trial_id, + "attempt": number, + "state": "COMPLETED", + "error": None, + "manifest_path": str(attempt / "manifest.json"), + "trial_id": manifest.trial_id, + } + store.write_text_atomic(execution, json.dumps(payload)) + store.write_text_atomic(attempt / "manifest.json", manifest.to_json()) + snapshots = AttemptStore(directory).list_attempts("task-1") + self.assertEqual([item.manifest.attempt_number for item in snapshots], [1, 2]) + + def test_attempt_comparison_is_tri_state(self): + left = self.manifest(budget_digest="budget-a") + right = self.manifest(budget_digest="budget-a") + self.assertEqual(compare_attempts(left, right).status, ComparabilityStatus.COMPARABLE) + different = self.manifest(model="model-y", budget_digest="budget-a") + self.assertEqual(compare_attempts(left, different).status, ComparabilityStatus.INCOMPARABLE) + unknown = self.manifest(provider=None, budget_digest="budget-a") + self.assertEqual(compare_attempts(left, unknown).status, ComparabilityStatus.INDETERMINATE) + + def test_comparison_report_does_not_aggregate_ineligible_pairs(self): + first = self.manifest(budget_digest="budget-a") + second = self.manifest( + attempt_number=2, + attempt_id=derive_attempt_id(trial_id=first.trial_id, attempt_number=2, configuration_digest="cfg-a"), + budget_digest="budget-a", + ) + third = self.manifest(model="model-y", budget_digest="budget-a") + report = compare_attempt_collection((first, second, third)) + self.assertEqual(len(report.pairs), 3) + self.assertEqual(len(report.comparable_pairs), 1) + self.assertEqual(len(report.incomparable_pairs), 2) + self.assertEqual(report.indeterminate_pairs, ()) + + def test_evidence_extraction_preserves_unknown_metrics(self): + snapshot = type("Snapshot", (), { + "manifest": self.manifest(state=RunState.BLOCKED), + "execution": {"model_usage": {"cost": 0.0}}, + })() + evidence = evidence_from_snapshot(snapshot) + self.assertEqual(evidence.execution_state, RunState.BLOCKED) + self.assertIsNone(evidence.verification_state) + self.assertIsNone(evidence.acceptance_decision) + self.assertEqual(evidence.cost, 0.0) + self.assertIsNone(evidence.total_tokens) + self.assertIsNone(evidence.accepted) + + def test_execution_completion_does_not_imply_verification_or_acceptance(self): + snapshot = type("Snapshot", (), { + "manifest": self.manifest(state=RunState.COMPLETED), + "execution": {"state": "COMPLETED"}, + })() + evidence = evidence_from_snapshot(snapshot) + self.assertEqual(evidence.execution_state, RunState.COMPLETED) + self.assertIsNone(evidence.verification_state) + self.assertIsNone(evidence.acceptance_decision) + + def test_verification_and_acceptance_evidence_have_independent_authorities(self): + verification = VerificationEvidence("verifier-v1", "PASS", ("verification.json",)) + acceptance = AcceptanceEvidence("acceptance-v1", "ACCEPTED", ("acceptance.json",)) + self.assertNotEqual(verification.authority_identity, acceptance.authority_identity) + self.assertEqual(verification.evidence_refs, ("verification.json",)) + self.assertEqual(acceptance.evidence_refs, ("acceptance.json",)) + + def test_experiment_record_links_protocol_attempts_and_promotion(self): + record = ExperimentRecord( + experiment_id="exp-1", + protocol_version="protocol-1", + input_manifest_digest="input-digest", + trial_ids=("trial-1",), + attempt_ids=("attempt-1",), + verification_refs=("verification://1",), + acceptance_refs=("acceptance://1",), + promotion_decision_ref="promotion://1", + status="COMPLETED", + ) + self.assertEqual(ExperimentRecord.from_json(record.to_json()).to_dict(), record.to_dict()) + with self.assertRaises(ValueError): + ExperimentRecord("exp", "protocol", "digest", promotion_decision_ref="promotion://orphan") + + def test_experiment_record_loader_validates_attempts_and_refs(self): + with TemporaryDirectory() as directory: + artifact_store = ArtifactStore(directory) + attempt_root = artifact_store.reserve_attempt("task-1", 1) + execution_path = artifact_store.write_text_atomic(attempt_root / "execution.json", "{}\n") + manifest = self.manifest( + state=RunState.COMPLETED, + artifact_refs=(str(execution_path),), + state_history=(RunState.PLANNED, RunState.STARTED, RunState.EXECUTING, RunState.COMPLETED), + ) + execution = { + "run_id": manifest.attempt_id, + "trial_id": manifest.trial_id, + "attempt": 1, + "state": "COMPLETED", + "error": None, + "manifest_path": str(attempt_root / "manifest.json"), + } + artifact_store.write_text_atomic(attempt_root / "execution.json", json.dumps(execution)) + artifact_store.write_text_atomic(attempt_root / "manifest.json", manifest.to_json()) + record = ExperimentRecord("exp-1", "protocol-1", "input", (manifest.trial_id,), (manifest.attempt_id,), ("verification://1",), ("acceptance://1",), "promotion://1", "COMPLETED") + record_path = write_experiment_record(record, Path(directory) / "experiment.json") + loaded = load_experiment_record(record_path, directory, promotion_attempt_id=manifest.attempt_id) + self.assertEqual(len(loaded.attempts), 1) + with self.assertRaises(FileExistsError): + write_experiment_record(record, record_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 00000000..c96ecb5c --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,82 @@ +import unittest + +from arkx.integration import AdapterIdentity, CapabilityProvenance, DependencyObservation, IntegrationKind, PreflightStatus, assess_preflight, exception_error_envelope, preflight_error, process_error_envelope, provider_error_envelope +from arkx.harness import ErrorDomain, Retryability + + +class IntegrationContractTests(unittest.TestCase): + def test_ready_preflight_is_deterministic_and_evidence_bearing(self): + identity = AdapterIdentity(IntegrationKind.PROVIDER, "openrouter", "1", "cfg") + dependencies = (DependencyObservation("runtime", True, "3.11"),) + first = assess_preflight(identity, dependencies, capabilities=("completion",), evidence_refs=("evidence://preflight",)) + second = assess_preflight(identity, dependencies, capabilities=("completion",), evidence_refs=("evidence://preflight",)) + self.assertEqual(first.status, PreflightStatus.READY) + self.assertEqual(first.to_json(), second.to_json()) + self.assertTrue(first.reference.startswith("preflight://")) + + def test_preflight_carries_capability_identity_and_provenance(self): + identity = AdapterIdentity(IntegrationKind.EXECUTOR, "executor", "1", "cfg") + result = assess_preflight( + identity, + (DependencyObservation("runtime", True, "1"),), + capabilities=("patch",), + capability_digest="capability-digest", + capability_provenance=CapabilityProvenance.OBSERVED, + ) + self.assertEqual(result.capability_digest, "capability-digest") + self.assertEqual(result.capability_provenance, CapabilityProvenance.OBSERVED) + with self.assertRaises(ValueError): + assess_preflight( + identity, + (DependencyObservation("runtime", True, "1"),), + capabilities=("patch",), + capability_provenance=CapabilityProvenance.QUALIFIED, + ) + + def test_missing_dependency_blocks_without_fallback(self): + identity = AdapterIdentity(IntegrationKind.SANDBOX, "docker") + result = assess_preflight(identity, (DependencyObservation("daemon", False, reason="permission denied"),)) + self.assertEqual(result.status, PreflightStatus.BLOCKED) + self.assertEqual(result.dependencies[0].reason, "permission denied") + + def test_missing_identity_or_unknown_dependency_stays_unknown(self): + unknown_identity = AdapterIdentity(IntegrationKind.EXECUTOR, "runner") + self.assertEqual(assess_preflight(unknown_identity, ()).status, PreflightStatus.UNKNOWN) + identity = AdapterIdentity(IntegrationKind.EVALUATOR, "swebench", "1", "cfg") + self.assertEqual(assess_preflight(identity, (DependencyObservation("docker", None),)).status, PreflightStatus.UNKNOWN) + + def test_preflight_error_preserves_boundary_and_unknown_retryability(self): + identity = AdapterIdentity(IntegrationKind.PROVIDER, "openrouter") + preflight = assess_preflight(identity, (DependencyObservation("minisweagent", False),)) + error = preflight_error(preflight) + self.assertEqual(error.domain, ErrorDomain.PROVIDER) + self.assertEqual(error.code, "PREFLIGHT_BLOCKED") + self.assertEqual(error.retryability, Retryability.UNKNOWN) + self.assertEqual(error.raw_evidence_refs, (preflight.reference,)) + + def test_provider_payload_normalization_preserves_raw_reference(self): + error = provider_error_envelope( + {"error": {"code": 503, "message": "overloaded", "metadata": {"error_type": "busy"}}}, + raw_evidence_refs=("artifact://provider.json",), + ) + self.assertEqual(error.domain, ErrorDomain.PROVIDER) + self.assertEqual(error.code, "PROVIDER_503") + self.assertEqual(error.retryability, Retryability.UNKNOWN) + self.assertEqual(error.raw_evidence_refs, ("artifact://provider.json",)) + + def test_process_and_exception_normalization_preserve_boundary(self): + process = process_error_envelope( + IntegrationKind.SANDBOX, + returncode=126, + message="permission denied", + raw_evidence_refs=("artifact://sandbox.log",), + ) + exception = exception_error_envelope(IntegrationKind.EVALUATOR, RuntimeError("daemon unavailable")) + self.assertEqual(process.code, "SANDBOX_PROCESS_126") + self.assertEqual(process.domain, ErrorDomain.SANDBOX) + self.assertEqual(exception.domain, ErrorDomain.VERIFICATION) + self.assertEqual(exception.retryability, Retryability.UNKNOWN) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_orchestration.py b/tests/test_orchestration.py new file mode 100644 index 00000000..999895b7 --- /dev/null +++ b/tests/test_orchestration.py @@ -0,0 +1,309 @@ +import unittest +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +from arkx.characterization import RecommendedPath, TaskCharacterization, TaskSignals, characterize +from arkx.orchestration import ( + ExecutionOutcome, + ExecutionResult, + OrchestrationReason, + OrchestrationStatus, + orchestration_status_for_acceptance, + orchestration_status_for_execution, + run_routed_pipeline, +) +from arkx.p82_baseline import AcceptanceDecision, AcceptanceResult, VerificationResult, VerificationState +from arkx.progress import ProgressAssessment, ProgressEvidence, ProgressStatus, Confidence +from arkx.request import ArkxRequest, GovernancePolicy, RequestBudget, RequestEnvironment, authorize_request +from arkx.routing import RoutingDecision, RoutingDecisionType, RoutingBudget +from arkx.selection import ExecutorBinding, ExecutorRegistry, SelectionStatus, TreatmentDefinition, select_executor, select_treatment +from arkx.executor_qualification import ExecutorIdentity, QualificationStatus +from arkx.execution import ExecutionResult as ContractExecutionResult, FakeExecutor as ContractFakeExecutor +from arkx.harness import RunState +from arkx.orchestration import ContractExecutorRunner +from arkx.orchestration import legacy_execution_from_contract +from arkx.orchestration import build_execution_plan +from arkx.configuration import ConfigurationSnapshot +from arkx.verifier import CommandVerifier, VerificationEvidenceStore + + +def setup(): + request = ArkxRequest( + "req-1", "task-1", "developer", + TaskSignals(("src/a.py",), (), ("a",), ("tests/a.py",), (), (), ("tests",)), + ("src",), RequestBudget(2, 1000, 10000), RequestEnvironment("local", "workspace", "HEAD"), "authority-1", + ) + governance = authorize_request(request, GovernancePolicy("gov", ("developer",), ("local",), 3, 2000, 20000)) + characterization = characterize(request.signals) + treatment = TreatmentDefinition("simple", RecommendedPath.SIMPLE_PATH, ()) + selection = select_executor(treatment, ExecutorRegistry((ExecutorBinding(ExecutorIdentity("mini", "1", "adapter", "digest"), ("edit",), QualificationStatus.QUALIFIABLE, ("evidence://1",)),))) + routing = RoutingDecision(1, "task-1", RecommendedPath.SIMPLE_PATH, RecommendedPath.SIMPLE_PATH, RoutingDecisionType.ROUTE, characterization.confidence, (), None, "test", RoutingBudget()) + return governance, characterization, routing, selection + + +class FakeExecutor: + def __init__(self, outcome=ExecutionOutcome.COMPLETED): self.outcome = outcome + def run(self, request, selection): return ExecutionResult("run-1", self.outcome, telemetry={"total_tokens": 0, "wall_time_ms": 0, "cost": 0}) + def run_with_budget(self, request, selection, *, attempt_id, budget_ledger): return self.run(request, selection) + + +class FakeProgress: + def __init__(self, status=ProgressStatus.PROGRESS_PROVEN): self.status = status + def assess(self, execution): + evidence = ProgressEvidence((), (), (), None, None, None, None) + return ProgressAssessment(1, self.status, Confidence.HIGH, (), (), None, {}, evidence) + + +class FakeVerifier: + def __init__(self, state=VerificationState.PASS): self.state = state + def verify(self, execution): return VerificationResult(self.state, "verifier-1", (), (), ("evidence://verify",)) + + +class FakeAcceptance: + identity = "authority-1" + def __init__(self, decision=AcceptanceDecision.ACCEPTED): self.decision = decision + def decide(self, verification): return AcceptanceResult(self.decision, self.identity, "raw", verification.evidence) + + +class OrchestrationTests(unittest.TestCase): + def test_state_crosswalk_is_total_and_explicit(self): + self.assertEqual(orchestration_status_for_execution(ExecutionOutcome.COMPLETED), OrchestrationStatus.EXECUTED) + self.assertEqual(orchestration_status_for_execution(ExecutionOutcome.FAILED), OrchestrationStatus.BLOCKED) + self.assertEqual(orchestration_status_for_execution(ExecutionOutcome.BLOCKED), OrchestrationStatus.BLOCKED) + self.assertEqual(orchestration_status_for_execution(ExecutionOutcome.NOT_EXECUTED), OrchestrationStatus.NOT_EXECUTED) + for decision, expected in ((AcceptanceDecision.ACCEPTED, OrchestrationStatus.ACCEPTED), (AcceptanceDecision.REJECTED, OrchestrationStatus.REJECTED), (AcceptanceDecision.BLOCKED, OrchestrationStatus.BLOCKED), (AcceptanceDecision.NOT_EXECUTED, OrchestrationStatus.NOT_EXECUTED), (AcceptanceDecision.INDETERMINATE, OrchestrationStatus.INDETERMINATE)): + self.assertEqual(orchestration_status_for_acceptance(decision), expected) + def test_pipeline_runs_in_order_to_acceptance(self): + governance, characterization, routing, selection = setup() + result = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(), FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertEqual(result.status, OrchestrationStatus.ACCEPTED) + self.assertIsNotNone(result.verification) + self.assertIsNotNone(result.acceptance) + + def test_pipeline_uses_persisted_command_verifier_before_acceptance(self): + governance, characterization, routing, selection = setup() + with TemporaryDirectory() as directory: + verifier = CommandVerifier( + authority="authority-1", + workspace=directory, + command=(sys.executable, "-c", "print('pipeline-verified')"), + test_id="pipeline-smoke", + evidence_store=VerificationEvidenceStore(directory + "/evidence"), + ) + result = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(), FakeProgress(), verifier, FakeAcceptance()) + files = list((Path(directory) / "evidence" / "run-1").glob("verification-*.json")) + content = files[0].read_text(encoding="utf-8") + self.assertEqual(result.status, OrchestrationStatus.ACCEPTED) + self.assertEqual(len(files), 1) + self.assertIn("pipeline-verified", content) + + def test_invalid_governance_blocks_before_executor(self): + governance, characterization, routing, selection = setup() + blocked = governance.__class__(governance.request_id, governance.status.BLOCKED, governance.authority_id, (), None) + result = run_routed_pipeline(blocked, characterization, routing, selection, FakeExecutor(), FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertEqual(result.status, OrchestrationStatus.BLOCKED) + self.assertIn(OrchestrationReason.GOVERNANCE_REQUIRED, result.reason_codes) + + def test_no_progress_blocks_without_verification(self): + governance, characterization, routing, selection = setup() + result = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(), FakeProgress(ProgressStatus.NO_PROGRESS), FakeVerifier(), FakeAcceptance()) + self.assertEqual(result.status, OrchestrationStatus.BLOCKED) + self.assertIsNone(result.verification) + + def test_acceptance_authority_mismatch_blocks(self): + governance, characterization, routing, selection = setup() + authority = FakeAcceptance(); authority.identity = "other" + result = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(), FakeProgress(), FakeVerifier(), authority) + self.assertIn(OrchestrationReason.ACCEPTANCE_AUTHORITY_MISMATCH, result.reason_codes) + + def test_executor_failure_is_not_acceptance_rejection(self): + governance, characterization, routing, selection = setup() + result = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(ExecutionOutcome.FAILED), FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertEqual(result.status, OrchestrationStatus.BLOCKED) + + def test_serialization_is_deterministic(self): + governance, characterization, routing, selection = setup() + first = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(), FakeProgress(), FakeVerifier(), FakeAcceptance()).to_json() + second = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(), FakeProgress(), FakeVerifier(), FakeAcceptance()).to_json() + self.assertEqual(first, second) + + def test_contract_executor_runner_bridges_neutral_executor(self): + governance, characterization, routing, selection = setup() + runner = ContractExecutorRunner(ContractFakeExecutor(ContractExecutionResult(RunState.COMPLETED, ("artifact://1",), usage={"total_tokens": 0, "wall_time_ms": 0, "cost": 0}))) + result = run_routed_pipeline(governance, characterization, routing, selection, runner, FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertEqual(result.status, OrchestrationStatus.ACCEPTED) + self.assertEqual(result.execution.outcome, ExecutionOutcome.COMPLETED) + + def test_contract_executor_runner_preserves_blocked(self): + governance, characterization, routing, selection = setup() + from arkx.harness import ErrorDomain, ErrorEnvelope, Retryability + error = ErrorEnvelope(ErrorDomain.SANDBOX, "UNAVAILABLE", "sandbox unavailable", Retryability.NOT_RETRYABLE) + runner = ContractExecutorRunner(ContractFakeExecutor(ContractExecutionResult(RunState.BLOCKED, error=error))) + result = run_routed_pipeline(governance, characterization, routing, selection, runner, FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertEqual(result.status, OrchestrationStatus.BLOCKED) + + def test_orchestration_accepts_neutral_executor_directly(self): + governance, characterization, routing, selection = setup() + executor = ContractFakeExecutor(ContractExecutionResult(RunState.COMPLETED, ("artifact://direct",), usage={"total_tokens": 0, "wall_time_ms": 0, "cost": 0})) + result = run_routed_pipeline(governance, characterization, routing, selection, executor, FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertEqual(result.status, OrchestrationStatus.ACCEPTED) + self.assertEqual(result.execution.outcome, ExecutionOutcome.COMPLETED) + + def test_neutral_and_legacy_executor_paths_are_behaviorally_equivalent(self): + governance, characterization, routing, selection = setup() + neutral = ContractFakeExecutor(ContractExecutionResult(RunState.COMPLETED, ("artifact://same",), usage={"total_tokens": 0, "wall_time_ms": 0, "cost": 0})) + bridged = ContractExecutorRunner(ContractFakeExecutor(ContractExecutionResult(RunState.COMPLETED, ("artifact://same",), usage={"total_tokens": 0, "wall_time_ms": 0, "cost": 0}))) + direct_result = run_routed_pipeline(governance, characterization, routing, selection, neutral, FakeProgress(), FakeVerifier(), FakeAcceptance()) + bridge_result = run_routed_pipeline(governance, characterization, routing, selection, bridged, FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertEqual(direct_result.status, bridge_result.status) + self.assertEqual(direct_result.execution.to_dict(), bridge_result.execution.to_dict()) + + def test_orchestration_records_deterministic_execution_plan_before_execution(self): + governance, characterization, routing, selection = setup() + first = build_execution_plan(governance, routing, selection) + second = build_execution_plan(governance, routing, selection) + self.assertEqual(first.to_json(), second.to_json()) + result = run_routed_pipeline(governance, characterization, routing, selection, ContractFakeExecutor(ContractExecutionResult(RunState.COMPLETED, usage={"total_tokens": 0, "wall_time_ms": 0, "cost": 0})), FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertEqual(result.plan, first) + self.assertEqual(result.plan.budget_digest, first.budget_digest) + self.assertEqual(first.required_capabilities, ()) + + def test_orchestration_persists_optional_provider_and_sandbox_identities_in_plan(self): + from arkx.integration import AdapterIdentity, IntegrationKind + governance, characterization, routing, selection = setup() + provider = AdapterIdentity(IntegrationKind.PROVIDER, "provider-a", "1", "provider-config") + sandbox = AdapterIdentity(IntegrationKind.SANDBOX, "local", "1", "sandbox-config") + plan = build_execution_plan(governance, routing, selection, provider_identity=provider, sandbox_identity=sandbox) + self.assertEqual(plan.provider_identity, provider) + self.assertEqual(plan.sandbox_identity, sandbox) + + def test_orchestration_requires_and_persists_provider_and_sandbox_preflights(self): + from tempfile import TemporaryDirectory + from pathlib import Path + from arkx.event_log import EventLog, replay_chain + from arkx.integration import AdapterIdentity, DependencyObservation, IntegrationKind, assess_preflight + governance, characterization, routing, selection = setup() + provider = AdapterIdentity(IntegrationKind.PROVIDER, "provider-a", "1", "provider-config") + sandbox = AdapterIdentity(IntegrationKind.SANDBOX, "local", "1", "sandbox-config") + provider_preflight = assess_preflight(provider, (DependencyObservation("provider", True, "1"),)) + sandbox_preflight = assess_preflight(sandbox, (DependencyObservation("runtime", True, "1"),)) + with TemporaryDirectory() as directory: + log = EventLog(Path(directory) / "events.jsonl", run_id="run-1") + result = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(), FakeProgress(), FakeVerifier(), FakeAcceptance(), event_log=log, provider_identity=provider, sandbox_identity=sandbox, provider_preflight=provider_preflight, sandbox_preflight=sandbox_preflight) + chain = replay_chain(log.read(), run_id="run-1") + self.assertEqual(result.status, OrchestrationStatus.ACCEPTED) + self.assertEqual(chain.provider_preflight_ref, provider_preflight.reference) + self.assertEqual(chain.sandbox_preflight_ref, sandbox_preflight.reference) + + def test_orchestration_blocks_when_bound_provider_preflight_is_missing(self): + from arkx.integration import AdapterIdentity, IntegrationKind + governance, characterization, routing, selection = setup() + provider = AdapterIdentity(IntegrationKind.PROVIDER, "provider-a", "1", "provider-config") + result = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(), FakeProgress(), FakeVerifier(), FakeAcceptance(), provider_identity=provider) + self.assertIn(OrchestrationReason.PROVIDER_PREFLIGHT_REQUIRED, result.reason_codes) + + def test_capability_required_plan_blocks_before_executor_without_preflight(self): + governance, characterization, routing, selection = setup() + treatment = TreatmentDefinition("capability", RecommendedPath.SIMPLE_PATH, ("edit",)) + selection = selection.__class__(selection.status, selection.reason_codes, selection.path, treatment, selection.executor, selection.telemetry, selection.schema_version) + executor = FakeExecutor() + result = run_routed_pipeline(governance, characterization, routing, selection, executor, FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertIn(OrchestrationReason.CAPABILITY_PREFLIGHT_REQUIRED, result.reason_codes) + self.assertEqual(result.status, OrchestrationStatus.BLOCKED) + + def test_capability_required_plan_requires_preflight_event_log(self): + governance, characterization, routing, selection = setup() + treatment = TreatmentDefinition("capability", RecommendedPath.SIMPLE_PATH, ("edit",)) + selection = selection.__class__(selection.status, selection.reason_codes, selection.path, treatment, selection.executor, selection.telemetry, selection.schema_version) + from arkx.integration import AdapterIdentity, CapabilityProvenance, DependencyObservation, IntegrationKind, assess_preflight + preflight = assess_preflight( + AdapterIdentity(IntegrationKind.EXECUTOR, "mini", "1", selection.executor.executor.configuration_digest), + (DependencyObservation("runtime", True, "1"),), + capabilities=("edit",), capability_digest="capability", capability_provenance=CapabilityProvenance.OBSERVED, + ) + result = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(), FakeProgress(), FakeVerifier(), FakeAcceptance(), capability_preflight=preflight) + self.assertIn(OrchestrationReason.CAPABILITY_PREFLIGHT_PERSISTENCE_REQUIRED, result.reason_codes) + + def test_event_log_records_plan_execution_verification_and_acceptance(self): + from tempfile import TemporaryDirectory + from pathlib import Path + from arkx.event_log import EventLog, replay_chain, replay_events + governance, characterization, routing, selection = setup() + with TemporaryDirectory() as directory: + log = EventLog(Path(directory) / "events.jsonl", run_id="run-1") + result = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(), FakeProgress(), FakeVerifier(), FakeAcceptance(), event_log=log) + chain = replay_chain(log.read(), run_id="run-1") + replay = replay_events(log.read(), run_id="run-1") + self.assertEqual(result.status, OrchestrationStatus.ACCEPTED) + self.assertIsNotNone(chain.plan_ref) + self.assertEqual(chain.execution_ref, "execution://run-1") + self.assertEqual(chain.verification_ref, result.verification.reference) + self.assertEqual(chain.acceptance_ref, result.acceptance.reference) + self.assertEqual(replay.declared_status, "ACCEPTED") + + def test_lifecycle_persistence_failure_has_neutral_reason(self): + governance, characterization, routing, selection = setup() + class FailingLog: + def append_stage(self, **kwargs): + raise OSError("read-only event log") + result = run_routed_pipeline(governance, characterization, routing, selection, FakeExecutor(), FakeProgress(), FakeVerifier(), FakeAcceptance(), event_log=FailingLog()) + self.assertIn(OrchestrationReason.LIFECYCLE_PERSISTENCE_FAILED, result.reason_codes) + + def test_legacy_result_is_only_a_deterministic_boundary_conversion(self): + from arkx.harness import ErrorDomain, ErrorEnvelope, Retryability + neutral = ContractExecutionResult( + RunState.BLOCKED, + ("artifact://blocked",), + error=ErrorEnvelope(ErrorDomain.SANDBOX, "BLOCKED", "blocked", Retryability.UNKNOWN), + ) + legacy = legacy_execution_from_contract(neutral) + self.assertEqual(legacy.outcome, ExecutionOutcome.BLOCKED) + self.assertEqual(legacy.evidence_refs, ("artifact://blocked",)) + + def test_pipeline_blocks_before_runner_when_invocation_budget_is_zero(self): + governance, characterization, routing, selection = setup() + request = governance.request.__class__( + governance.request.request_id, governance.request.task_id, governance.request.requester, + governance.request.signals, governance.request.authorized_paths, + RequestBudget(0, 1000, 10000), + governance.request.environment, governance.request.acceptance_authority, + ) + governance = authorize_request(request, GovernancePolicy("gov", ("developer",), ("local",), 3, 2000, 20000)) + class MustNotRun(FakeExecutor): + def run(self, request, selection): + raise AssertionError("runner must not be called after budget rejection") + result = run_routed_pipeline(governance, characterization, routing, selection, MustNotRun(), FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertIn(OrchestrationReason.EXECUTION_BUDGET_EXHAUSTED, result.reason_codes) + + def test_pipeline_blocks_after_execution_when_budget_measurement_is_missing(self): + governance, characterization, routing, selection = setup() + class NoUsage: + def run(self, request, selection): + return ExecutionResult("run-1", ExecutionOutcome.COMPLETED) + def run_with_budget(self, request, selection, *, attempt_id, budget_ledger): + return self.run(request, selection) + result = run_routed_pipeline(governance, characterization, routing, selection, NoUsage(), FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertIn(OrchestrationReason.BUDGET_USAGE_UNAVAILABLE, result.reason_codes) + + def test_pipeline_blocks_legacy_runner_without_budget_enforcement(self): + governance, characterization, routing, selection = setup() + class LegacyRunner: + def run(self, request, selection): + raise AssertionError("legacy runner must be blocked before execution") + result = run_routed_pipeline(governance, characterization, routing, selection, LegacyRunner(), FakeProgress(), FakeVerifier(), FakeAcceptance()) + self.assertIn(OrchestrationReason.BUDGET_ENFORCEMENT_REQUIRED, result.reason_codes) + + def test_execution_plan_persists_typed_configuration_snapshot(self): + governance, characterization, routing, selection = setup() + snapshot = ConfigurationSnapshot("executor", "1", {"mode": "fake"}) + binding = selection.executor + binding_executor = binding.executor.__class__(binding.executor.name, binding.executor.version, binding.executor.integration_kind, snapshot.digest(), binding.executor.advertised_capabilities) + selection = selection.__class__(selection.status, selection.reason_codes, path=selection.path, treatment=selection.treatment, executor=binding.__class__(binding_executor, binding.capabilities, binding.qualification_status, binding.evidence_refs), telemetry=selection.telemetry) + plan = build_execution_plan(governance, routing, selection, configuration_snapshot=snapshot) + self.assertEqual(plan.configuration_snapshot, snapshot) + self.assertEqual(plan.configuration_digest, snapshot.digest()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_outcome_chain_validation.py b/tests/test_outcome_chain_validation.py new file mode 100644 index 00000000..2c06a032 --- /dev/null +++ b/tests/test_outcome_chain_validation.py @@ -0,0 +1,49 @@ +import unittest + +from arkx.acceptance import AcceptancePolicy, decide_independent_acceptance +from arkx.event_log import ReplayChain, validate_outcome_chain +from arkx.outcomes import VerificationResult, VerificationState +from arkx.promotion import PromotionCandidate, PromotionPolicy, decide_outcome_promotion + + +class OutcomeChainValidationTests(unittest.TestCase): + def _objects(self): + verification = VerificationResult( + VerificationState.PASS, + "verifier", + ("pytest",), + ("0",), + ("evidence://verification",), + ) + acceptance = decide_independent_acceptance(verification, AcceptancePolicy("authority")) + promotion = decide_outcome_promotion( + PromotionCandidate("executor", "candidate", "1", "cfg"), + verification, + acceptance, + ("experiment://1",), + PromotionPolicy("authority"), + ) + return verification, acceptance, promotion + + def test_replay_chain_must_match_all_outcome_objects(self): + verification, acceptance, promotion = self._objects() + chain = ReplayChain( + verification_ref=verification.reference, + acceptance_ref=acceptance.reference, + promotion_ref=promotion.reference, + ) + validate_outcome_chain(chain, verification=verification, acceptance=acceptance, promotion=promotion) + + def test_replay_chain_rejects_object_reference_drift(self): + verification, acceptance, promotion = self._objects() + chain = ReplayChain( + verification_ref="verification://different", + acceptance_ref=acceptance.reference, + promotion_ref=promotion.reference, + ) + with self.assertRaises(ValueError): + validate_outcome_chain(chain, verification=verification, acceptance=acceptance, promotion=promotion) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p0_telemetry.py b/tests/test_p0_telemetry.py new file mode 100644 index 00000000..196c5ddd --- /dev/null +++ b/tests/test_p0_telemetry.py @@ -0,0 +1,105 @@ +import unittest +from tempfile import TemporaryDirectory +from pathlib import Path + +from arkx.baseline import RUN_ID, baseline_events, build_baseline_record +from arkx.contracts import Event, EventType, ExecutionStatus +from arkx.evidence import resolve_status +from arkx.telemetry import TelemetryCollector + + +def collector(events, *, task_id="task", run_id="run"): + value = TelemetryCollector(task_id=task_id, run_id=run_id) + value.extend(events) + return value + + +def event(event_type, data=None, *, timestamp="2026-01-01T00:00:00+00:00", run_id="run"): + return Event(timestamp, event_type, run_id, data or {}) + + +class StatusTests(unittest.TestCase): + def test_missing_evidence_cannot_become_pass(self): + self.assertEqual(resolve_status("PASS", []), ExecutionStatus.UNVERIFIED) + record = collector([event(EventType.TASK_FINISHED, {"status": "PASS"})]).summarize() + self.assertEqual(record.status, ExecutionStatus.UNVERIFIED) + + def test_blocked_remains_blocked(self): + record = collector([event(EventType.TASK_FINISHED, {"status": "BLOCKED"})]).summarize() + self.assertEqual(record.status, ExecutionStatus.BLOCKED) + + def test_not_executed_is_distinct_from_failed(self): + not_executed = collector([]).summarize() + failed = collector([event(EventType.TASK_FINISHED, {"status": "FAILED"})]).summarize() + self.assertEqual(not_executed.status, ExecutionStatus.NOT_EXECUTED) + self.assertEqual(failed.status, ExecutionStatus.FAILED) + + +class MetricsTests(unittest.TestCase): + def test_event_metrics(self): + events = [ + event(EventType.RETRY, {"reason": "transient", "attempt": 2}), + event(EventType.REPLAN, {"reason": "new constraint", "attempt": 1}), + event(EventType.EXECUTOR_STARTED, {"executor_id": "a"}), + event(EventType.EXECUTOR_STARTED, {"executor_id": "a"}), + event(EventType.HANDOFF, {"source_executor": "a", "target_executor": "b", "reason": "capacity"}), + event(EventType.HUMAN_INTERVENTION, {"reason": "approval"}), + ] + record = collector(events).summarize() + self.assertEqual(record.retry_count, 1) + self.assertEqual(record.replan_count, 1) + self.assertEqual(record.executor_invocations, 2) + self.assertEqual(record.handoff_count, 1) + self.assertEqual(record.human_interventions, 1) + self.assertEqual(record.metadata["handoffs"][0]["target_executor"], "b") + + +class DeterminismTests(unittest.TestCase): + def test_same_controlled_sequence_has_same_summary(self): + first = build_baseline_record().to_json() + second = build_baseline_record().to_json() + self.assertEqual(first, second) + + +class UnknownMeasurementTests(unittest.TestCase): + def test_unknown_tokens_and_cost_are_not_zero(self): + record = collector([]).summarize() + self.assertIsNone(record.total_tokens) + self.assertIsNone(record.monetary_cost) + + def test_explicit_zero_is_preserved_as_zero(self): + record = collector([event(EventType.EXECUTOR_FINISHED, {"total_tokens": 0, "monetary_cost": 0.0})]).summarize() + self.assertEqual(record.total_tokens, 0) + self.assertEqual(record.monetary_cost, 0.0) + + +class SerializationTests(unittest.TestCase): + def test_record_json_round_trip_preserves_semantics(self): + record = build_baseline_record() + restored = type(record).from_json(record.to_json()) + self.assertEqual(restored.to_dict(), record.to_dict()) + + def test_schema_is_versioned(self): + self.assertEqual(build_baseline_record().to_dict()["schemaVersion"], 1) + + def test_record_can_be_written_to_a_local_artifact_path(self): + record = build_baseline_record() + with TemporaryDirectory() as directory: + path = Path(directory) / "run.json" + record.write_json(path) + self.assertEqual(type(record).from_json(path.read_text(encoding="utf-8").strip()).to_dict(), record.to_dict()) + + +class ValidationTests(unittest.TestCase): + def test_events_from_another_run_are_rejected(self): + value = TelemetryCollector(task_id="task", run_id=RUN_ID) + with self.assertRaises(ValueError): + value.add(event(EventType.TASK_STARTED, run_id="other")) + + def test_invalid_timestamps_are_rejected(self): + with self.assertRaises(ValueError): + collector([event(EventType.TASK_STARTED, timestamp="not-a-timestamp")]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p82.py b/tests/test_p82.py new file mode 100644 index 00000000..68e5090b --- /dev/null +++ b/tests/test_p82.py @@ -0,0 +1,121 @@ +import json +import unittest +from decimal import Decimal +from pathlib import Path + +from arkx.composition import Mechanism +from arkx.executor_qualification import ExecutionBudget, ExecutionEnvironment, ExecutorIdentity, QualificationTask +from arkx.p82 import ( + P82Comparability, P82ExperimentConfig, P82Observation, P82ObservationRecord, + P82RunManifest, P82RunStatus, P82Treatment, TaskSampleManifest, TaskSampleTask, + TaskStratum, build_p82_plan, compare_run_manifests, p82_manifest, plan_is_comparable, + summarize_p82, +) + + +def config(**overrides): + values = { + "executor": ExecutorIdentity("mini-swe-agent", "1.0", "adapter", "mini-config"), + "model": "claude-4-sonnet-20250514", + "environment": ExecutionEnvironment("repo-rev", "python-3.11", "windows", "toolchain", "env-digest"), + "budget": ExecutionBudget(1000, Decimal("2"), Decimal("60"), 1, 1), + "tasks": (QualificationTask("task-1", "task-rev", "acceptance-v1"),), + } + values.update(overrides) + return P82ExperimentConfig(**values) + + +class P82Tests(unittest.TestCase): + def sample(self, task_revision="task-rev"): + task = QualificationTask("task-1", task_revision, "acceptance-v1") + return TaskSampleManifest("sample-1", (TaskSampleTask(task, TaskStratum.MINI_CONTROL, ("upstream",), ("evidence",)),), "sample-rev") + + def make_run(self, treatment="A", **overrides): + values = { + "protocol_version": "p8.2-v1", "treatment": p82_manifest()[ord(treatment) - ord("A")], + "executor": ExecutorIdentity("mini", "1", "adapter", "cfg"), "model": "model-v1", + "temperature": "0", "reasoning_configuration": "standard", + "environment": ExecutionEnvironment("repo", "python", "windows", "tools", "env"), + "task_sample": self.sample(), "attempts": 1, "token_budget": 1000, + "cost_budget": Decimal("2"), "wall_time_limit": Decimal("60"), + "verification_identity": "verification-v1", "acceptance_identity": "acceptance-v1", + } + values.update(overrides) + return P82RunManifest(**values) + + def test_manifest_is_exactly_abcd(self): + manifest = p82_manifest() + self.assertEqual(tuple(t.name for t in manifest), tuple(item.value for item in P82Treatment)) + self.assertEqual(manifest[0].enabled_mechanisms, ()) + self.assertEqual(manifest[1].enabled_mechanisms, (Mechanism.P8_SAFE_EDITOR,)) + self.assertEqual(manifest[2].enabled_mechanisms, (Mechanism.P8_ENHANCED_REPOSITORY_CONTEXT,)) + self.assertEqual(manifest[3].enabled_mechanisms, (Mechanism.P8_SAFE_EDITOR, Mechanism.P8_ENHANCED_REPOSITORY_CONTEXT)) + + def test_plan_is_cartesian_product_without_execution(self): + plan = build_p82_plan(config(tasks=(QualificationTask("a", "r", "v"), QualificationTask("b", "r", "v")), replicate_ids=("r2", "r1"))) + self.assertEqual(len(plan.trials), 16) + self.assertEqual([trial.replicate_id for trial in plan.trials[:4]], ["r1"] * 4) + self.assertNotIn("accepted", plan.to_json()) + self.assertNotIn("outcome", plan.to_json()) + + def test_missing_control_evidence_stays_unknown(self): + self.assertIsNone(plan_is_comparable(build_p82_plan(config(model=None)))) + self.assertTrue(plan_is_comparable(build_p82_plan(config(verification_contract="verification-v1", acceptance_authority="independent-v1")))) + + def test_treatment_only_change_is_comparable(self): + self.assertEqual(compare_run_manifests(self.make_run("A"), self.make_run("D")), P82Comparability.COMPARABLE) + + def test_model_budget_and_environment_changes_are_not_comparable(self): + self.assertEqual(compare_run_manifests(self.make_run(), self.make_run(model="other")), P82Comparability.NOT_COMPARABLE) + self.assertEqual(compare_run_manifests(self.make_run(), self.make_run(token_budget=2000)), P82Comparability.NOT_COMPARABLE) + self.assertEqual(compare_run_manifests(self.make_run(), self.make_run(environment=ExecutionEnvironment("other", "python", "windows", "tools", "env"))), P82Comparability.NOT_COMPARABLE) + + def test_insufficient_identity_is_indeterminate(self): + self.assertEqual(compare_run_manifests(self.make_run(model=None), self.make_run("B", model=None)), P82Comparability.INDETERMINATE) + + def test_observation_preserves_missing_metric_and_boolean(self): + value = P82Observation(P82RunStatus.EXECUTED).to_dict() + self.assertIsNone(value["cost_to_acceptance"]) + self.assertIsNone(value["verified_acceptance"]) + + def test_manifest_and_observation_round_trip_deterministically(self): + manifest = self.make_run("C") + self.assertEqual(manifest.to_json(), P82RunManifest.from_dict(json.loads(manifest.to_json())).to_json()) + observation = P82Observation(P82RunStatus.EXECUTED, cost_to_acceptance=Decimal("1.25")) + self.assertEqual(observation.to_dict(), P82Observation.from_dict(json.loads(json.dumps(observation.to_dict(), default=str))).to_dict()) + + def test_execution_does_not_imply_acceptance_or_promotion(self): + record = P82ObservationRecord(self.make_run(), P82Observation(P82RunStatus.EXECUTED)) + self.assertIsNone(record.observation.verified_acceptance) + self.assertNotIn("promotion", record.to_dict()) + + def test_summary_is_order_independent_and_excludes_non_executed_from_acceptance(self): + records = ( + P82ObservationRecord(self.make_run("A"), P82Observation(P82RunStatus.NOT_EXECUTED, verified_acceptance=True)), + P82ObservationRecord(self.make_run("A"), P82Observation(P82RunStatus.EXECUTED, verified_acceptance=False)), + P82ObservationRecord(self.make_run("D"), P82Observation(P82RunStatus.BLOCKED, verified_acceptance=True)), + ) + left = tuple(item.to_dict() for item in summarize_p82(records)) + right = tuple(item.to_dict() for item in summarize_p82(tuple(reversed(records)))) + self.assertEqual(left, right) + self.assertIsNotNone(left[0]["verified_acceptance_rate"]) + self.assertEqual(left[0]["not_executed_count"], 1) + self.assertEqual(left[3]["blocked_count"], 1) + self.assertIsNone(left[3]["verified_acceptance_rate"]) + + def test_config_rejects_duplicate_tasks_and_replicates(self): + task = QualificationTask("a", "r", "v") + with self.assertRaises(ValueError): + config(tasks=(task, task)) + with self.assertRaises(ValueError): + config(replicate_ids=("r1", "r1")) + + def test_fixture_is_p82_design(self): + path = Path(__file__).parents[1] / "experiments" / "p82-minimal-mechanism-trial-fixture.json" + value = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(value["fixture_type"], "P82_MINIMAL_MECHANISM_TRIAL") + self.assertEqual(value["treatments"], ["A", "B", "C", "D"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p82_baseline.py b/tests/test_p82_baseline.py new file mode 100644 index 00000000..8517d300 --- /dev/null +++ b/tests/test_p82_baseline.py @@ -0,0 +1,273 @@ +import tempfile +import unittest +import json +import subprocess +from pathlib import Path +from unittest.mock import patch + +from arkx.p82 import P82RunStatus, P82Treatment, TaskStratum +from arkx.harness import RunManifest, RunState +from arkx.harness import load_attempt +from arkx.p82_baseline import ( + AcceptanceDecision, + BaselineExecutionState, + BaselineRunConfig, + BaselineSampleManifest, + ExplicitVerificationAcceptance, + MiniSweAgentHeadlessRunner, + P82BaselineExecutorAdapter, + ProspectiveTask, + RunnerFailureCategory, + VerificationResult, + VerificationState, + baseline_observation, +) +from arkx.execution import ExecutionRequest + + +def task(workspace): + return ProspectiveTask( + "task-1", "swebench-verified@v1", "owner/repo", "abc123", "fix the bug", + TaskStratum.MINI_CONTROL, "stable control", "2026-09-23T00:00:00Z", "p8.2a-v1", + "verification-v1", "acceptance-v1", str(workspace), + ) + + +def config(root, **overrides): + values = { + "provider": "test-provider", "model": "test-model", "model_version": "v1", + "temperature": "0", "reasoning_configuration": "none", "max_tokens": 100, + "model_configuration": None, + "max_cost": "1", "wall_time_seconds": 10, "mini_version": "2.4.6", + "python_executable": "python", "runner_version": "p8.2a-runner-v1", + "artifact_root": str(root), "treatment": P82Treatment.A, + } + values.update(overrides) + return BaselineRunConfig(**values) + + +class P82BaselineTests(unittest.TestCase): + def test_subprocess_invoker_passes_limits_and_captures_usage(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" + workspace.mkdir() + runner = MiniSweAgentHeadlessRunner() + completed = type("Completed", (), { + "returncode": 0, + "stdout": '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\n', + "stderr": "worker log\n", + })() + with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: + usage = runner._subprocess_invoker(task(workspace), config(directory), Path(directory) / "trajectory.json") + self.assertEqual(usage, (0, '{"result":"done","usage":{"model_calls":3,"cost":0.25}}\nworker log\n', {"model_calls": 3, "cost": 0.25})) + self.assertEqual(mocked.call_args.kwargs["timeout"], 20) + payload = json.loads(mocked.call_args.kwargs["input"]) + self.assertEqual(payload["max_cost"], "1") + self.assertEqual(payload["wall_time_seconds"], 10) + + def test_timeout_preserves_partial_output_and_trajectory(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" + workspace.mkdir() + artifact_root = Path(directory) / "artifacts" + trajectory = artifact_root / "task-1" / "attempt-1" / "trajectory.json" + + def timed_out_invoker(run_task, run_config, trajectory_path): + trajectory_path.write_text('{"partial":true}', encoding="utf-8") + raise subprocess.TimeoutExpired( + cmd="worker", timeout=20, output=b"partial stdout\n", stderr=b"partial stderr\n" + ) + + result = MiniSweAgentHeadlessRunner(timed_out_invoker).run(task(workspace), config(artifact_root)) + self.assertEqual(result.failure_category, RunnerFailureCategory.TIMEOUT) + self.assertTrue(Path(result.log_path).exists()) + self.assertEqual(Path(result.log_path).read_text(encoding="utf-8"), "partial stdout\npartial stderr\n") + self.assertEqual(Path(result.trajectory_path).read_text(encoding="utf-8"), '{"partial":true}') + + def test_headless_runner_uses_fake_invoker_and_captures_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" + workspace.mkdir() + calls = [] + + def fake_invoker(run_task, run_config, trajectory): + calls.append((run_task.task_id, run_config.treatment, trajectory.name)) + trajectory.write_text("{}", encoding="utf-8") + return 0, "executor-log", {"calls": 1} + + result = MiniSweAgentHeadlessRunner(fake_invoker).run(task(workspace), config(Path(directory) / "artifacts")) + self.assertEqual(result.state, BaselineExecutionState.COMPLETED) + self.assertEqual(calls, [("task-1", P82Treatment.A, "trajectory.json")]) + self.assertTrue(Path(result.log_path).exists()) + self.assertTrue(Path(result.diff_path).exists()) + self.assertTrue(Path(result.artifact_path).exists()) + persisted = json.loads(Path(result.artifact_path).read_text(encoding="utf-8")) + self.assertEqual(persisted, result.to_dict()) + self.assertEqual(result.model_usage, {"calls": 1}) + self.assertEqual(result.treatment, "A") + + def test_run_identity_changes_when_frozen_model_configuration_changes(self): + with tempfile.TemporaryDirectory() as directory: + first = config(directory, model="model-a") + second = config(directory, model="model-b") + self.assertNotEqual(first.identity_digest(), second.identity_digest()) + + runner = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)) + first_result = runner.run(task(directory), first, attempt=1) + second_result = runner.run(task(directory), second, attempt=2) + self.assertNotEqual(first_result.run_id, second_result.run_id) + + def test_run_identity_changes_when_environment_changes(self): + with tempfile.TemporaryDirectory() as directory: + first = config(directory, environment="local") + second = config(directory, environment="git-bash", bash_executable=r"C:\Program Files\Git\bin\bash.exe") + self.assertNotEqual(first.identity_digest(), second.identity_digest()) + + def test_runner_persists_common_manifest_with_distinct_trial_and_attempt(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)).run(task(directory), config(directory)) + self.assertTrue(Path(result.manifest_path).exists()) + manifest = json.loads(Path(result.manifest_path).read_text(encoding="utf-8")) + self.assertEqual(manifest["trial_id"], result.trial_id) + self.assertEqual(manifest["attempt_id"], result.attempt_id) + self.assertNotEqual(manifest["trial_id"], manifest["attempt_id"]) + restored = RunManifest.from_dict(manifest) + restored.validate_attempt_identity() + restored.validate_lifecycle() + self.assertTrue(all(Path(reference).exists() for reference in manifest["artifact_refs"])) + snapshot = load_attempt(Path(result.manifest_path).parent) + self.assertEqual(snapshot.manifest.attempt_id, result.attempt_id) + self.assertEqual(manifest["state_history"], ["PLANNED", "STARTED", "EXECUTING", "COMPLETED"]) + self.assertEqual(manifest["artifacts"][0]["status"], "PRESENT") + + def test_missing_identity_is_preserved_in_common_manifest(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner().run(task(directory), config(directory, model=None)) + manifest = json.loads(Path(result.manifest_path).read_text(encoding="utf-8")) + self.assertEqual(manifest["state"], "BLOCKED") + self.assertIsNone(manifest["model"]) + self.assertEqual(manifest["errors"][0]["domain"], "HARNESS") + self.assertEqual(manifest["state_history"], ["PLANNED", "STARTED", "BLOCKED"]) + self.assertEqual(manifest["artifacts"][1]["status"], "MISSING") + RunManifest.from_dict(manifest).validate_lifecycle() + + def test_subprocess_invoker_passes_explicit_git_bash_environment(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) / "workspace" + workspace.mkdir() + runner = MiniSweAgentHeadlessRunner() + completed = type("Completed", (), {"returncode": 0, "stdout": "{}\n", "stderr": ""})() + run_config = config( + directory, + environment="git-bash", + bash_executable=r"C:\Program Files\Git\bin\bash.exe", + ) + with patch("arkx.p82_baseline.subprocess.run", return_value=completed) as mocked: + runner._subprocess_invoker(task(workspace), run_config, Path(directory) / "trajectory.json") + payload = json.loads(mocked.call_args.kwargs["input"]) + self.assertEqual(payload["environment"], "git-bash") + self.assertEqual(payload["bash_executable"], r"C:\Program Files\Git\bin\bash.exe") + + def test_provider_availability_exit_is_blocked_not_agent_failed(self): + with tempfile.TemporaryDirectory() as directory: + runner = MiniSweAgentHeadlessRunner(lambda *_: (75, '{"provider_error":{"code":503}}', {"model_calls": 3})) + result = runner.run(task(directory), config(directory)) + self.assertEqual(result.state, BaselineExecutionState.BLOCKED) + self.assertEqual(result.failure_category, RunnerFailureCategory.PROVIDER_AVAILABILITY_FAILURE) + self.assertEqual(result.error.domain.value, "PROVIDER") + self.assertEqual(result.error.retryability.value, "RETRYABLE") + + def test_missing_model_or_provider_is_blocked_without_fallback(self): + with tempfile.TemporaryDirectory() as directory: + result = MiniSweAgentHeadlessRunner().run(task(directory), config(directory, model=None)) + self.assertEqual(result.state, BaselineExecutionState.BLOCKED) + self.assertIn("model", result.failure) + self.assertEqual(result.error.domain.value, "HARNESS") + + def test_rerun_does_not_overwrite_attempt(self): + with tempfile.TemporaryDirectory() as directory: + runner = MiniSweAgentHeadlessRunner(lambda *_: (0, "", None)) + first = runner.run(task(directory), config(directory), attempt=1) + second = runner.run(task(directory), config(directory), attempt=1) + third = runner.run(task(directory), config(directory), attempt=2) + self.assertEqual(first.state, BaselineExecutionState.COMPLETED) + self.assertEqual(second.state, BaselineExecutionState.BLOCKED) + self.assertEqual(third.state, BaselineExecutionState.COMPLETED) + + def test_acceptance_is_explicit_and_preserves_indeterminate(self): + authority = ExplicitVerificationAcceptance("acceptance-v1") + result = authority.decide(VerificationResult(VerificationState.INDETERMINATE, "verification-v1", (), (), ())) + self.assertEqual(result.decision, AcceptanceDecision.INDETERMINATE) + self.assertEqual(result.authority, "acceptance-v1") + + def test_verification_pass_is_not_execution_success_or_runner_acceptance(self): + execution = type("Execution", (), {"state": BaselineExecutionState.COMPLETED})() + verification = VerificationResult(VerificationState.PASS, "verification-v1", ("pytest",), ("0",), ("log",)) + observation = baseline_observation(execution, verification, None) + self.assertEqual(observation.status, P82RunStatus.EXECUTED) + self.assertIsNone(observation.verified_acceptance) + + def test_executor_failure_is_not_task_rejection(self): + execution = type("Execution", (), {"state": BaselineExecutionState.EXECUTOR_FAILED})() + observation = baseline_observation(execution, None, None) + self.assertIsNone(observation.verified_acceptance) + self.assertEqual(observation.status, P82RunStatus.EXECUTED) + + def test_headless_failure_is_classified_without_becoming_rejection(self): + with tempfile.TemporaryDirectory() as directory: + def failing_invoker(*_): + raise ValueError("model configuration is invalid") + result = MiniSweAgentHeadlessRunner(failing_invoker).run(task(directory), config(directory)) + self.assertEqual(result.failure_category, RunnerFailureCategory.MODEL_INITIALIZATION_FAILURE) + self.assertEqual(result.state, BaselineExecutionState.EXECUTOR_FAILED) + self.assertEqual(result.error.domain.value, "EXECUTOR") + self.assertEqual(result.error.code, "EXECUTOR_PROCESS_UNKNOWN") + self.assertEqual(result.error.retryability.value, "UNKNOWN") + + def test_baseline_rejects_non_a_treatment(self): + with self.assertRaises(ValueError): + config("artifacts", treatment=P82Treatment.B) + + def test_prospective_sample_manifest_is_deterministic(self): + first = task("workspace-a") + second = ProspectiveTask( + "task-2", "dataset", "repo", "def456", "another task", TaskStratum.LOCALIZATION_CONTEXT, + "localization diversity", "2026-09-23T00:00:00Z", "p8.2a-v1", "verification-v1", "acceptance-v1", "workspace-b", + ) + left = BaselineSampleManifest("wave-0", "baseline-smoke", "predeclared diversity rule", (second, first)) + right = BaselineSampleManifest("wave-0", "baseline-smoke", "predeclared diversity rule", (first, second)) + self.assertEqual(left.to_json(), right.to_json()) + + def test_baseline_executor_adapter_returns_neutral_execution_result(self): + with tempfile.TemporaryDirectory() as directory: + adapter = P82BaselineExecutorAdapter( + task(directory), + config(directory), + MiniSweAgentHeadlessRunner(lambda *_: (0, "log", None)), + ) + request = ExecutionRequest("task-1", "abc123", "A", "prompt", adapter.executor_id, "test-provider", "test-model", "local", "budget", "config") + result = adapter.execute(request) + self.assertEqual(result.state, RunState.COMPLETED) + self.assertIn("execution.json", " ".join(result.artifact_refs)) + + def test_baseline_executor_adapter_preflight_is_read_only_and_identity_bound(self): + with tempfile.TemporaryDirectory() as directory: + adapter = P82BaselineExecutorAdapter(task(directory), config(directory)) + preflight = adapter.preflight() + self.assertEqual(preflight.identity.name, adapter.executor_id) + self.assertEqual(preflight.identity.configuration_digest, adapter.config.identity_digest()) + self.assertEqual(preflight.capabilities, ("edit",)) + self.assertEqual(preflight.capability_provenance.value, "DECLARED") + self.assertEqual(adapter.config.identity_digest(), adapter.config.configuration_snapshot().digest()) + self.assertNotIn("secret-value", adapter.config.configuration_snapshot().to_json()) + + def test_baseline_executor_adapter_rejects_wrong_task(self): + with tempfile.TemporaryDirectory() as directory: + adapter = P82BaselineExecutorAdapter(task(directory), config(directory), MiniSweAgentHeadlessRunner(lambda *_: (0, "", None))) + request = ExecutionRequest("other-task", "abc123", "A", "prompt", adapter.executor_id, "p", "m", "local", "b", "c") + with self.assertRaises(ValueError): + adapter.execute(request) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p82_block2_smoke_manifest.py b/tests/test_p82_block2_smoke_manifest.py new file mode 100644 index 00000000..1888bdd1 --- /dev/null +++ b/tests/test_p82_block2_smoke_manifest.py @@ -0,0 +1,33 @@ +import json +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class P82Block2SmokeManifestTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + path = ROOT / "experiments" / "p82-block2-smoke-manifest.json" + cls.payload = json.loads(path.read_text(encoding="utf-8")) + + def test_manifest_is_predeclared_and_model_free(self): + self.assertEqual(self.payload["status"], "PREDECLARED") + self.assertEqual(self.payload["frozen"]["model_calls"], 0) + self.assertIsNone(self.payload["frozen"]["provider"]) + + def test_has_one_cell_per_initial_stratum(self): + cells = self.payload["cells"] + self.assertEqual([cell["stratum"] for cell in cells], ["T0", "T1", "T2"]) + self.assertEqual(len({cell["cell_id"] for cell in cells}), 3) + + def test_promotion_rule_preserves_evidence_boundary(self): + rule = self.payload["promotion_rule"] + self.assertIn("does not qualify a model", rule) + self.assertIn("B1", rule) + self.assertIn("C1", rule) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p82_block_plan.py b/tests/test_p82_block_plan.py new file mode 100644 index 00000000..78a246e5 --- /dev/null +++ b/tests/test_p82_block_plan.py @@ -0,0 +1,39 @@ +import json +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class P82BlockPlanTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + path = ROOT / "experiments" / "p82-block-plan.json" + cls.plan = json.loads(path.read_text(encoding="utf-8")) + + def test_plan_is_design_only(self): + self.assertEqual(self.plan["status"], "DESIGN_ONLY") + + def test_blocks_are_ordered_and_unique(self): + blocks = self.plan["blocks"] + ids = [block["id"] for block in blocks] + self.assertEqual(ids, [f"BLOCK_{index}" for index in range(7)]) + self.assertEqual(len(ids), len(set(ids))) + + def test_block_four_is_the_only_baseline_execution_block(self): + executable = [ + block["id"] for block in self.plan["blocks"] + if "execution" in block and "only the predeclared tasks" in block["execution"] + ] + self.assertEqual(executable, ["BLOCK_4"]) + + def test_global_rules_keep_mechanisms_closed(self): + rules = " ".join(self.plan["global_rules"]) + self.assertIn("B1 and C1 contracts may be tested", rules) + self.assertIn("their operational mechanisms remain disabled", rules) + self.assertIn("D1 is not executable", rules) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p82_candidate_matrix.py b/tests/test_p82_candidate_matrix.py new file mode 100644 index 00000000..f86b0c96 --- /dev/null +++ b/tests/test_p82_candidate_matrix.py @@ -0,0 +1,38 @@ +import json +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class P82CandidateMatrixTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + path = ROOT / "experiments" / "p82-candidate-matrix.json" + cls.payload = json.loads(path.read_text(encoding="utf-8")) + + def test_matrix_is_design_only_and_broad(self): + self.assertEqual(self.payload["status"], "DESIGN_ONLY") + self.assertGreaterEqual(len(self.payload["candidates"]), 15) + + def test_each_candidate_has_harness_state_and_targets(self): + for candidate in self.payload["candidates"]: + self.assertTrue(candidate["id"]) + self.assertTrue(candidate["harness"]) + self.assertTrue(candidate["prior_state"]) + self.assertTrue(candidate["initial_targets"]) + self.assertTrue(set(candidate["initial_targets"]).issubset({"T0", "T1", "T2", "T3", "T4", "T5"})) + + def test_no_candidate_is_marked_as_resolution_evidence(self): + for candidate in self.payload["candidates"]: + self.assertNotIn("resolved", candidate["prior_state"]) + self.assertNotIn("promoted", candidate["prior_state"]) + + def test_nooa_is_investigation_only(self): + candidate = next(item for item in self.payload["candidates"] if item["id"] == "nooa/0.0.10") + self.assertEqual(candidate["prior_state"], "investigation_only") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p82_editing.py b/tests/test_p82_editing.py new file mode 100644 index 00000000..e90fe573 --- /dev/null +++ b/tests/test_p82_editing.py @@ -0,0 +1,78 @@ +import json +import unittest + +from arkx.p82_editing import EditOperation, TREATMENT, record_edit_proposal + + +class P82EditingTests(unittest.TestCase): + def test_replace_range_is_only_a_proposal(self): + proposal = record_edit_proposal( + task_id="task-1", + target_file="src/example.py", + operation=EditOperation.REPLACE_RANGE, + replacement_content="return value", + start_line=4, + end_line=4, + ) + self.assertEqual(proposal.treatment, TREATMENT) + self.assertEqual(proposal.operation, EditOperation.REPLACE_RANGE) + self.assertEqual(proposal.target_file, "src/example.py") + self.assertEqual(proposal.to_dict()["replacement_content"], "return value") + + def test_patch_representation_is_supported_without_line_semantics(self): + proposal = record_edit_proposal( + task_id="task-1", + target_file="src/example.py", + operation="APPLY_PATCH", + patch_content="@@ -1 +1 @@\n-old\n+new\n", + ) + self.assertEqual(proposal.operation, EditOperation.APPLY_PATCH) + self.assertIsNone(proposal.start_line) + + def test_serialization_is_deterministic(self): + proposal = record_edit_proposal( + task_id="task-1", + target_file="src/example.py", + operation=EditOperation.REPLACE_RANGE, + replacement_content="new", + start_line=1, + end_line=2, + ) + payload = json.loads(proposal.to_json()) + self.assertEqual(proposal.to_json(), proposal.to_json()) + self.assertEqual(payload["operation"], "REPLACE_RANGE") + self.assertEqual(payload["treatment"], "B1") + + def test_unsupported_operations_are_rejected(self): + with self.assertRaisesRegex(ValueError, "unsupported"): + record_edit_proposal( + task_id="task-1", + target_file="src/example.py", + operation="DELETE_FILE", + replacement_content="unused", + ) + + def test_mixed_representations_and_invalid_ranges_are_rejected(self): + with self.assertRaisesRegex(ValueError, "exactly one"): + record_edit_proposal( + task_id="task-1", + target_file="src/example.py", + operation=EditOperation.REPLACE_RANGE, + replacement_content="new", + patch_content="patch", + start_line=1, + end_line=1, + ) + with self.assertRaisesRegex(ValueError, "line range"): + record_edit_proposal( + task_id="task-1", + target_file="src/example.py", + operation=EditOperation.REPLACE_RANGE, + replacement_content="new", + start_line=3, + end_line=2, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p82_localization.py b/tests/test_p82_localization.py new file mode 100644 index 00000000..4491be3c --- /dev/null +++ b/tests/test_p82_localization.py @@ -0,0 +1,81 @@ +import json +import unittest + +from arkx.p82_localization import ( + LocalizationEvidenceBudget, + TREATMENT, + record_localization_artifact, +) + + +class P82LocalizationTests(unittest.TestCase): + def budget(self): + return LocalizationEvidenceBudget(max_files=2, max_symbols=3, max_context_lines=20) + + def test_contract_records_bounded_non_authoritative_candidates(self): + artifact = record_localization_artifact( + task_id="sympy__sympy-14711", + repository_revision="abc123", + localization_method="deterministic-candidate-pass-v1", + candidate_files=("src/z.py", "src/a.py", "src/a.py"), + candidate_symbols=("src/z.py::Z", "src/a.py::A"), + context_lines_used=10, + evidence_budget=self.budget(), + ) + self.assertEqual(artifact.treatment, TREATMENT) + self.assertEqual(artifact.candidate_files, ("src/a.py", "src/z.py")) + self.assertTrue(artifact.candidates_are_non_authoritative) + self.assertEqual(artifact.to_dict()["evidence_budget"]["max_files"], 2) + + def test_serialization_is_deterministic_and_contains_explicit_boundary(self): + artifact = record_localization_artifact( + task_id="task-1", + repository_revision="deadbeef", + localization_method="method-v1", + candidate_files=("b.py", "a.py"), + evidence_budget=self.budget(), + ) + payload = json.loads(artifact.to_json()) + self.assertEqual(artifact.to_json(), artifact.to_json()) + self.assertEqual(payload["treatment"], "C1") + self.assertTrue(payload["candidates_are_non_authoritative"]) + self.assertEqual(payload["candidate_files"], ["a.py", "b.py"]) + + def test_budget_overflow_is_rejected(self): + with self.assertRaisesRegex(ValueError, "file count"): + record_localization_artifact( + task_id="task-1", + repository_revision="rev", + localization_method="method-v1", + candidate_files=("a.py", "b.py", "c.py"), + evidence_budget=self.budget(), + ) + + def test_authoritative_candidates_are_rejected(self): + with self.assertRaisesRegex(ValueError, "non-authoritative"): + from arkx.p82_localization import LocalizationArtifact + + LocalizationArtifact( + task_id="task-1", + repository_revision="rev", + localization_method="method-v1", + candidate_files=(), + candidate_symbols=(), + context_lines_used=0, + evidence_budget=self.budget(), + candidates_are_non_authoritative=False, + ) + + def test_zero_budget_is_a_valid_empty_contract(self): + artifact = record_localization_artifact( + task_id="task-1", + repository_revision="rev", + localization_method="method-v1", + evidence_budget=LocalizationEvidenceBudget(0, 0, 0), + ) + self.assertEqual(artifact.candidate_files, ()) + self.assertEqual(artifact.context_lines_used, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p82_ollama_functional_probe.py b/tests/test_p82_ollama_functional_probe.py new file mode 100644 index 00000000..f3de4134 --- /dev/null +++ b/tests/test_p82_ollama_functional_probe.py @@ -0,0 +1,46 @@ +import json +import unittest + +from tools.p82_ollama_functional_probe import validate_functional_response + + +class P82OllamaFunctionalProbeTests(unittest.TestCase): + def test_valid_functional_response_is_accepted(self): + response = json.dumps({ + "code": "def validate_value(value):\n if value is None:\n raise ValueError('value is required')\n return value", + "test": "with pytest.raises(ValueError, match='value is required'):\n validate_value(None)\nassert validate_value(7) == 7", + }) + result = validate_functional_response(response) + self.assertIn("validate_value", result["code"]) + + def test_wrong_exception_message_is_rejected(self): + response = json.dumps({ + "code": "def validate_value(value):\n if value is None:\n raise ValueError('wrong')\n return value", + "test": "with pytest.raises(ValueError):\n validate_value(None)", + }) + with self.assertRaises(ValueError): + validate_functional_response(response) + + def test_missing_pytest_assertion_is_rejected(self): + response = json.dumps({ + "code": "def validate_value(value):\n return value", + "test": "assert validate_value(7) == 7", + }) + with self.assertRaises(ValueError): + validate_functional_response(response) + + def test_functional_budget_is_explicitly_overridable(self): + from tools.p82_ollama_probe import probe_model + + captured = {} + + def requester(request, timeout): + captured["body"] = json.loads(request.data.decode("utf-8")) + return type("Response", (), {"__enter__": lambda self: self, "__exit__": lambda *args: False, "read": lambda self: json.dumps({"response": "{}", "done": True}).encode("utf-8")})() + + probe_model("model", prompt="functional", num_predict=128, requester=requester) + self.assertEqual(captured["body"]["options"]["num_predict"], 128) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p82_ollama_probe.py b/tests/test_p82_ollama_probe.py new file mode 100644 index 00000000..4929b819 --- /dev/null +++ b/tests/test_p82_ollama_probe.py @@ -0,0 +1,47 @@ +import json +import unittest + +from tools.p82_ollama_probe import probe_model + + +class FakeResponse: + def __init__(self, body): + self.body = json.dumps(body).encode("utf-8") + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self): + return self.body + + +class P82OllamaProbeTests(unittest.TestCase): + def test_exact_response_is_t0_pass(self): + result = probe_model( + "gemma3:4b", + requester=lambda request, timeout: FakeResponse({"response": "ARKX_T0_OK", "done": True, "eval_count": 9}), + ) + self.assertEqual(result["status"], "PASS_T0_FORMAT") + self.assertEqual(result["eval_count"], 9) + + def test_non_exact_response_is_not_pass(self): + result = probe_model( + "model", + requester=lambda request, timeout: FakeResponse({"response": "almost", "done": True}), + ) + self.assertEqual(result["status"], "FORMAT_OR_CONTENT_FAIL") + + def test_timeout_is_explicit(self): + def timeout(request, timeout): + raise TimeoutError("bounded timeout") + + result = probe_model("model", requester=timeout, timeout_seconds=1) + self.assertEqual(result["status"], "TIMEOUT") + self.assertEqual(result["error_type"], "TimeoutError") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p82_openrouter_adapter.py b/tests/test_p82_openrouter_adapter.py new file mode 100644 index 00000000..66057cc0 --- /dev/null +++ b/tests/test_p82_openrouter_adapter.py @@ -0,0 +1,32 @@ +import unittest + +from tools.p82_openrouter_model import ArkxOpenRouterModel, ProviderAvailabilityError + + +class P82OpenRouterAdapterTests(unittest.TestCase): + def test_provider_error_envelope_is_not_parsed_as_success(self): + model = ArkxOpenRouterModel.__new__(ArkxOpenRouterModel) + with self.assertRaises(ProviderAvailabilityError) as raised: + model._parse_actions({ + "error": { + "code": 503, + "message": "Upstream error from Nvidia: Service temporarily overloaded", + "metadata": {"error_type": "provider_overloaded"}, + } + }) + self.assertIn("code=503", str(raised.exception)) + self.assertEqual(raised.exception.provider_error_type, "provider_overloaded") + + def test_missing_choices_without_error_remains_non_provider_failure(self): + model = ArkxOpenRouterModel.__new__(ArkxOpenRouterModel) + with self.assertRaises(KeyError): + model._parse_actions({"unexpected": "payload"}) + + def test_provider_error_can_cross_neutral_error_boundary(self): + model = ArkxOpenRouterModel.__new__(ArkxOpenRouterModel) + with self.assertRaises(ProviderAvailabilityError) as raised: + model._parse_actions({"error": {"code": 429, "message": "rate limited"}}) + envelope = raised.exception.to_error_envelope(raw_evidence_refs=("artifact://provider.json",)) + self.assertEqual(envelope.domain.value, "PROVIDER") + self.assertEqual(envelope.code, "PROVIDER_429") + self.assertEqual(envelope.raw_evidence_refs, ("artifact://provider.json",)) diff --git a/tests/test_p82_task_stratification_evidence.py b/tests/test_p82_task_stratification_evidence.py new file mode 100644 index 00000000..7bddb99f --- /dev/null +++ b/tests/test_p82_task_stratification_evidence.py @@ -0,0 +1,26 @@ +import json +from pathlib import Path +import unittest + + +class P82TaskStratificationEvidenceTests(unittest.TestCase): + def setUp(self): + self.path = Path(__file__).parents[1] / "experiments" / "p82-task-stratification-evidence.json" + self.payload = json.loads(self.path.read_text(encoding="utf-8")) + + def test_design_is_explicitly_non_executed(self): + self.assertEqual(self.payload["status"], "DESIGN_ONLY") + self.assertEqual(self.payload["strata"], ["T0", "T1", "T2", "T3", "T4", "T5"]) + + def test_external_evidence_is_not_labeled_as_arkx(self): + for item in self.payload["external_evidence"]: + self.assertNotEqual(item["class"], "arkx_evidence") + self.assertTrue(item["source"].startswith("https://")) + + def test_wave0_annotations_are_provisional(self): + self.assertEqual(self.payload["wave0"]["sympy__sympy-14711"], "T1_PROVISIONAL") + self.assertEqual(self.payload["wave0"]["sympy__sympy-24443"], "T2_PROVISIONAL") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_planning.py b/tests/test_planning.py new file mode 100644 index 00000000..e5ada9e9 --- /dev/null +++ b/tests/test_planning.py @@ -0,0 +1,83 @@ +import json +import unittest +from pathlib import Path + +from arkx.planning import ( + PlanStep, + ReplanRequest, + ReplanResult, + ReplanTrigger, + RepositoryState, + StepStatus, + build_plan, + plan_repository, +) + + +class PlanningTests(unittest.TestCase): + def state(self): + return RepositoryState( + relevant_files=("src/b.py", "src/a.py"), + dependency_edges=(("a", "b"),), + impacted_components=("b", "a"), + affected_tests=("tests/test_a.py",), + assumptions=("dependency graph is explicit",), + unresolved_questions=(), + completed_steps=(), + remaining_steps=("inspect a", "inspect b"), + ) + + def test_state_serialization_is_deterministic(self): + first = self.state().to_json() + second = RepositoryState( + relevant_files=("src/a.py", "src/b.py"), + dependency_edges=(("a", "b"),), + impacted_components=("a", "b"), + affected_tests=("tests/test_a.py",), + assumptions=("dependency graph is explicit",), + unresolved_questions=(), + completed_steps=(), + remaining_steps=("inspect b", "inspect a"), + ).to_json() + self.assertEqual(first, second) + + def test_plan_is_deterministic_and_does_not_complete_steps(self): + plan = plan_repository("change components", self.state()) + self.assertEqual(plan.to_json(), plan_repository("change components", self.state()).to_json()) + self.assertTrue(all(step.status is StepStatus.PENDING for step in plan.steps)) + self.assertNotIn("PASS", plan.to_json()) + + def test_dependency_order_is_preserved(self): + plan = build_plan( + "goal", + (PlanStep("b", "B", depends_on=("a",)), PlanStep("a", "A")), + ) + self.assertIn(["a", "b"], plan.to_dict()["dependencies"]) + + def test_unknown_dependency_is_rejected(self): + with self.assertRaises(ValueError): + build_plan("goal", (PlanStep("a", "A", depends_on=("missing",)),)) + + def test_cycle_is_rejected(self): + with self.assertRaises(ValueError): + build_plan( + "goal", + (PlanStep("a", "A", depends_on=("b",)), PlanStep("b", "B", depends_on=("a",))), + ) + + def test_replan_preserves_previous_plan_reference(self): + plan = plan_repository("goal", self.state()) + request = ReplanRequest(ReplanTrigger.DEPENDENCY_DISCOVERED, "plan://old", ("evidence://1",), 1) + result = ReplanResult(request.previous_plan_ref, plan, request.trigger) + self.assertEqual(result.to_dict()["previous_plan_ref"], "plan://old") + + def test_fixture_is_classified_as_planning_fixture(self): + path = Path(__file__).parents[1] / "experiments" / "repository-planning-fixture.json" + value = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(value["fixture_type"], "REPOSITORY_PLANNING_FIXTURE") + self.assertEqual(len(value["scenarios"]), 5) + + +if __name__ == "__main__": + unittest.main() + diff --git a/tests/test_progress.py b/tests/test_progress.py new file mode 100644 index 00000000..40e64bb2 --- /dev/null +++ b/tests/test_progress.py @@ -0,0 +1,119 @@ +import json +import unittest +from pathlib import Path + +from arkx.contracts import ExecutionStatus +from arkx.progress import ( + Confidence, + ProgressSnapshot, + ProgressStatus, + ReasonCode, + assess_progress, + assess_progress_timed, +) +from arkx.telemetry import TelemetryCollector + + +def snapshot(**overrides): + values = { + "useful_files": ("src/a.py",), + "passing_tests": (), + "explained_failures": (), + "diff_distance": 4, + "acceptance_distance": 3, + "recent_actions": ("inspect a",), + "failure_signatures": ("failure-a",), + } + values.update(overrides) + return ProgressSnapshot(**values) + + +class StatusTests(unittest.TestCase): + def test_new_passing_test_proves_progress(self): + result = assess_progress(snapshot(), snapshot(passing_tests=("test_a",))) + self.assertEqual(result.status, ProgressStatus.PROGRESS_PROVEN) + self.assertIn(ReasonCode.NEW_PASSING_TEST, result.reason_codes) + + def test_new_relevant_file_proves_progress(self): + result = assess_progress(snapshot(), snapshot(useful_files=("src/a.py", "src/b.py"))) + self.assertEqual(result.status, ProgressStatus.PROGRESS_PROVEN) + + def test_same_failure_and_no_new_evidence_is_no_progress(self): + result = assess_progress(snapshot(), snapshot()) + self.assertEqual(result.status, ProgressStatus.NO_PROGRESS) + self.assertIn(ReasonCode.REPEATED_FAILURE_SIGNATURE, result.reason_codes) + self.assertIn(ReasonCode.NO_NEW_EVIDENCE, result.reason_codes) + + def test_repeated_action_and_unchanged_acceptance_is_no_progress(self): + result = assess_progress(snapshot(failure_signatures=()), snapshot(failure_signatures=())) + self.assertEqual(result.status, ProgressStatus.NO_PROGRESS) + self.assertIn(ReasonCode.REPEATED_ACTION, result.reason_codes) + + def test_new_failure_with_new_evidence_is_not_no_progress(self): + result = assess_progress( + snapshot(failure_signatures=("failure-a",)), + snapshot(useful_files=("src/a.py", "src/b.py"), failure_signatures=("failure-b",)), + ) + self.assertEqual(result.status, ProgressStatus.PROGRESS_PROVEN) + self.assertNotEqual(result.status, ProgressStatus.NO_PROGRESS) + + def test_missing_measurement_is_unknown(self): + result = assess_progress(None, ProgressSnapshot()) + self.assertEqual(result.status, ProgressStatus.UNKNOWN) + self.assertEqual(result.confidence, Confidence.LOW) + + def test_contradictory_distance_is_unknown(self): + result = assess_progress(snapshot(diff_distance=-1), snapshot(diff_distance=0)) + self.assertEqual(result.status, ProgressStatus.UNKNOWN) + self.assertIn(ReasonCode.CONTRADICTORY_SIGNALS, result.reason_codes) + + +class DeterminismTests(unittest.TestCase): + def test_same_snapshots_are_byte_equivalent(self): + first = assess_progress(snapshot(), snapshot(useful_files=("src/a.py", "src/b.py")), assessment_duration_ms=4) + second = assess_progress(snapshot(), snapshot(useful_files=("src/b.py", "src/a.py")), assessment_duration_ms=4) + self.assertEqual(first.to_json(), second.to_json()) + + def test_timed_assessment_reports_duration(self): + result = assess_progress_timed(snapshot(), snapshot()) + self.assertGreaterEqual(result.telemetry["assessment_duration_ms"], 0) + + def test_assessment_does_not_produce_pass_or_change_p0(self): + result = assess_progress(snapshot(), snapshot()) + self.assertNotIn("PASS", result.to_json()) + execution = TelemetryCollector(task_id="task", run_id="run").summarize() + self.assertEqual(execution.status, ExecutionStatus.NOT_EXECUTED) + + def test_synthetic_fixture_covers_all_sequences(self): + path = Path(__file__).parents[1] / "experiments" / "progress-detection-fixture.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(manifest["fixture_type"], "PROGRESS_DETECTION_FIXTURE") + observed = {} + for sequence in manifest["sequences"]: + previous = None if sequence["previous"] is None else ProgressSnapshot(**_snapshot_values(sequence["previous"])) + current = ProgressSnapshot(**_snapshot_values(sequence["current"])) + observed[sequence["id"]] = assess_progress(previous, current).status.value + self.assertEqual(observed, { + "A_productive_progression": "PROGRESS_PROVEN", + "B_repeated_failure_without_new_evidence": "NO_PROGRESS", + "C_partial_progress": "PROGRESS_PROVEN", + "D_unknown_missing_telemetry": "UNKNOWN", + "E_contradictory_signals": "UNKNOWN", + }) + + +def _snapshot_values(value): + return { + "useful_files": None if value["useful_files"] is None else tuple(value["useful_files"]), + "passing_tests": None if value["passing_tests"] is None else tuple(value["passing_tests"]), + "explained_failures": None if value["explained_failures"] is None else tuple(value["explained_failures"]), + "diff_distance": value["diff_distance"], + "acceptance_distance": value["acceptance_distance"], + "recent_actions": None if value["recent_actions"] is None else tuple(value["recent_actions"]), + "failure_signatures": None if value["failure_signatures"] is None else tuple(value["failure_signatures"]), + } + + +if __name__ == "__main__": + unittest.main() + diff --git a/tests/test_promotion.py b/tests/test_promotion.py new file mode 100644 index 00000000..391e5c58 --- /dev/null +++ b/tests/test_promotion.py @@ -0,0 +1,143 @@ +import unittest + +from arkx.p82_baseline import AcceptanceDecision, AcceptanceResult +from arkx.harness import VerificationEvidence +from arkx.harness import AttemptSnapshot, RunManifest, RunState +from arkx.promotion import ( + PromotionCandidate, + PromotionPolicy, + PromotionReason, + PromotionStatus, + decide_snapshot_promotion, + decide_promotion, + decide_outcome_promotion, + load_promotion_decision, + write_promotion_decision, +) +from arkx.outcomes import VerificationResult, VerificationState + + +def candidate(**overrides): + values = {"component_type": "executor", "component_id": "mini", "version": "1.0", "configuration_digest": "digest"} + values.update(overrides) + return PromotionCandidate(**values) + + +def accepted(**overrides): + values = {"decision": AcceptanceDecision.ACCEPTED, "authority": "authority-1", "raw_outcome": "accepted", "evidence": ("evidence://acceptance",)} + values.update(overrides) + return AcceptanceResult(**values) + + +def verified(**overrides): + values = {"authority_identity": "verifier-1", "state": "PASS", "evidence_refs": ("evidence://verification",)} + values.update(overrides) + return VerificationEvidence(**values) + + +class PromotionTests(unittest.TestCase): + def test_accepted_candidate_with_evidence_is_promoted(self): + result = decide_promotion(candidate(), accepted(), ("experiment://p8.3/trial-1",), PromotionPolicy("authority-1"), verified()) + self.assertEqual(result.status, PromotionStatus.PROMOTED) + self.assertIn(PromotionReason.PROMOTED_WITH_INDEPENDENT_EVIDENCE, result.reason_codes) + + def test_missing_acceptance_is_blocked(self): + result = decide_promotion(candidate(), None, ("experiment://1",), PromotionPolicy("authority-1"), verified()) + self.assertEqual(result.status, PromotionStatus.BLOCKED) + self.assertIn(PromotionReason.ACCEPTANCE_REQUIRED, result.reason_codes) + + def test_missing_evidence_is_blocked(self): + result = decide_promotion(candidate(), accepted(), (), PromotionPolicy("authority-1"), verified()) + self.assertEqual(result.status, PromotionStatus.BLOCKED) + self.assertIn(PromotionReason.EVIDENCE_MISSING, result.reason_codes) + + def test_authority_mismatch_is_blocked(self): + result = decide_promotion(candidate(), accepted(authority="other"), ("experiment://1",), PromotionPolicy("authority-1"), verified()) + self.assertEqual(result.status, PromotionStatus.BLOCKED) + + def test_indeterminate_acceptance_is_not_promoted(self): + result = decide_promotion(candidate(), accepted(decision=AcceptanceDecision.INDETERMINATE), ("experiment://1",), PromotionPolicy("authority-1"), verified()) + self.assertEqual(result.status, PromotionStatus.INDETERMINATE) + + def test_rejected_acceptance_is_not_promoted(self): + result = decide_promotion(candidate(), accepted(decision=AcceptanceDecision.REJECTED), ("experiment://1",), PromotionPolicy("authority-1"), verified()) + self.assertEqual(result.status, PromotionStatus.NOT_PROMOTED) + + def test_serialization_is_deterministic(self): + first = decide_promotion(candidate(), accepted(), ("b", "a"), PromotionPolicy("authority-1"), verified()).to_json() + second = decide_promotion(candidate(), accepted(), ("a", "b"), PromotionPolicy("authority-1"), verified()).to_json() + self.assertEqual(first, second) + + def test_verification_is_required_for_promotion(self): + result = decide_promotion(candidate(), accepted(), ("experiment://1",), PromotionPolicy("authority-1")) + self.assertEqual(result.status, PromotionStatus.BLOCKED) + self.assertIn(PromotionReason.VERIFICATION_REQUIRED, result.reason_codes) + + def test_verification_without_evidence_is_blocked(self): + result = decide_promotion(candidate(), accepted(), ("experiment://1",), PromotionPolicy("authority-1"), verified(evidence_refs=())) + self.assertEqual(result.status, PromotionStatus.BLOCKED) + self.assertIn(PromotionReason.VERIFICATION_EVIDENCE_MISSING, result.reason_codes) + + def test_snapshot_promotion_requires_completed_matching_attempt(self): + manifest = RunManifest( + experiment_id="exp", trial_id="trial", attempt_id="attempt", attempt_number=1, + task_id="task", task_revision="rev", treatment="A", executor="executor", + provider="provider", model="model", sandbox="sandbox", repository_revision="rev", + configuration_digest="digest", protocol_version="protocol", state=RunState.BLOCKED, + ) + snapshot = AttemptSnapshot(manifest, {}, ()) + result = decide_snapshot_promotion(candidate(), snapshot, accepted(), ("evidence://1",), PromotionPolicy("authority-1"), verified()) + self.assertEqual(result.status, PromotionStatus.BLOCKED) + self.assertIn(PromotionReason.EXECUTION_NOT_COMPLETED, result.reason_codes) + + def test_snapshot_promotion_binds_decision_to_attempt_and_verification_refs(self): + manifest = RunManifest( + experiment_id="exp", trial_id="trial", attempt_id="attempt", attempt_number=1, + task_id="task", task_revision="rev", treatment="A", executor="executor", + provider="provider", model="model", sandbox="sandbox", repository_revision="rev", + configuration_digest="digest", protocol_version="protocol", state=RunState.COMPLETED, + ) + snapshot = AttemptSnapshot(manifest, {}, ()) + result = decide_snapshot_promotion(candidate(configuration_digest="digest"), snapshot, accepted(), ("evidence://promotion",), PromotionPolicy("authority-1"), verified()) + self.assertEqual(result.status, PromotionStatus.PROMOTED) + self.assertEqual(result.attempt_id, "attempt") + self.assertTrue(result.reference.startswith("promotion://")) + self.assertIn("evidence://verification", result.evidence_refs) + + def test_promotion_decision_persistence_is_immutable_and_bound(self): + import tempfile + from pathlib import Path + manifest = RunManifest( + experiment_id="exp", trial_id="trial", attempt_id="attempt", attempt_number=1, + task_id="task", task_revision="rev", treatment="A", executor="executor", + provider="provider", model="model", sandbox="sandbox", repository_revision="rev", + configuration_digest="digest", protocol_version="protocol", state=RunState.COMPLETED, + ) + snapshot = AttemptSnapshot(manifest, {}, ()) + decision = decide_snapshot_promotion(candidate(configuration_digest="digest"), snapshot, accepted(), ("evidence://promotion",), PromotionPolicy("authority-1"), verified()) + with tempfile.TemporaryDirectory() as directory: + path = write_promotion_decision(decision, Path(directory) / "promotion.json") + restored = load_promotion_decision(path, snapshot) + self.assertEqual(restored.attempt_id, "attempt") + with self.assertRaises(FileExistsError): + write_promotion_decision(decision, path) + + def test_outcome_promotion_requires_acceptance_to_reference_verification(self): + verification = VerificationResult(VerificationState.PASS, "verifier", ("pytest",), ("0",), ("evidence://verification",)) + acceptance = AcceptanceResult(AcceptanceDecision.ACCEPTED, "authority-1", "accepted", ("evidence://acceptance",)) + blocked = decide_outcome_promotion(candidate(), verification, acceptance, ("experiment://1",), PromotionPolicy("authority-1")) + self.assertEqual(blocked.status, PromotionStatus.BLOCKED) + self.assertIn(PromotionReason.ACCEPTANCE_VERIFICATION_MISMATCH, blocked.reason_codes) + + accepted_result = AcceptanceResult( + AcceptanceDecision.ACCEPTED, + "authority-1", + "accepted", + tuple(sorted((verification.reference, "evidence://verification"))), + ) + promoted = decide_outcome_promotion(candidate(), verification, accepted_result, ("experiment://1",), PromotionPolicy("authority-1")) + self.assertEqual(promoted.status, PromotionStatus.PROMOTED) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_promotion_event_persistence.py b/tests/test_promotion_event_persistence.py new file mode 100644 index 00000000..95329272 --- /dev/null +++ b/tests/test_promotion_event_persistence.py @@ -0,0 +1,39 @@ +import tempfile +import unittest +from pathlib import Path + +from arkx.acceptance import AcceptancePolicy, decide_independent_acceptance +from arkx.contracts import EventType +from arkx.event_log import EventLog, append_promotion_decision, replay_chain +from arkx.outcomes import VerificationResult, VerificationState +from arkx.promotion import PromotionCandidate, PromotionPolicy, decide_outcome_promotion + + +class PromotionEventPersistenceTests(unittest.TestCase): + def test_promotion_is_persisted_only_after_acceptance(self): + verification = VerificationResult(VerificationState.PASS, "verifier", ("pytest",), ("0",), ("evidence://verification",)) + acceptance = decide_independent_acceptance(verification, AcceptancePolicy("authority")) + decision = decide_outcome_promotion( + PromotionCandidate("executor", "candidate", "1", "cfg"), verification, acceptance, + ("experiment://1",), PromotionPolicy("authority"), + ) + with tempfile.TemporaryDirectory() as directory: + log = EventLog(Path(directory) / "events.jsonl", run_id="run-1") + log.append_stage(timestamp="2026-01-01T00:00:00+00:00", event_type=EventType.EVIDENCE_ADDED, stage="verification", ref=verification.reference) + log.append_stage(timestamp="2026-01-01T00:00:01+00:00", event_type=EventType.EVIDENCE_ADDED, stage="acceptance", ref=acceptance.reference) + self.assertTrue(append_promotion_decision(log, decision, verification_ref=verification.reference, timestamp="2026-01-01T00:00:02+00:00")) + self.assertEqual(replay_chain(log.read(), run_id="run-1").promotion_ref, decision.reference) + + def test_promotion_rejects_acceptance_from_another_chain(self): + verification = VerificationResult(VerificationState.PASS, "verifier", ("pytest",), ("0",), ("evidence://verification",)) + acceptance = decide_independent_acceptance(verification, AcceptancePolicy("authority")) + decision = decide_outcome_promotion(PromotionCandidate("executor", "candidate", "1", "cfg"), verification, acceptance, ("experiment://1",), PromotionPolicy("authority")) + with tempfile.TemporaryDirectory() as directory: + log = EventLog(Path(directory) / "events.jsonl", run_id="run-1") + log.append_stage(timestamp="2026-01-01T00:00:00+00:00", event_type=EventType.EVIDENCE_ADDED, stage="verification", ref=verification.reference) + with self.assertRaises(ValueError): + append_promotion_decision(log, decision, verification_ref=verification.reference, timestamp="2026-01-01T00:00:01+00:00") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_recovery.py b/tests/test_recovery.py new file mode 100644 index 00000000..5c14ea44 --- /dev/null +++ b/tests/test_recovery.py @@ -0,0 +1,67 @@ +import unittest + +from arkx.handoff import HandoffRecord +from arkx.planning import ReplanRequest, ReplanTrigger +from arkx.progress import Confidence, ProgressAssessment, ProgressEvidence, ProgressStatus +from arkx.recovery import RecoveryAction, RecoveryReason, build_recovery_plan, decide_recovery, plan_recovery_attempt, validate_recovery_reference +from arkx.routing import EscalationAction, EscalationDecision, EvidenceSufficiency, RecommendedPath +from arkx.harness import derive_attempt_id + + +def progress(status): + return ProgressAssessment(1, status, Confidence.HIGH, (), (), None, {}, ProgressEvidence((), (), (), None, None, None, None)) + + +def escalation(action, current=RecommendedPath.LOCALIZED_PATH, target=RecommendedPath.REPOSITORY_WIDE_PATH): + return EscalationDecision(1, current, action, target, (), progress(ProgressStatus.NO_PROGRESS).status, EvidenceSufficiency.INSUFFICIENT, None) # type: ignore[arg-type] + + +class RecoveryTests(unittest.TestCase): + def test_authorized_path_escalation_is_explicit(self): + result = decide_recovery(escalation(EscalationAction.ESCALATE), progress(ProgressStatus.NO_PROGRESS)) + self.assertEqual(result.action, RecoveryAction.ESCALATE) + self.assertIn(RecoveryReason.PATH_ESCALATION_AUTHORIZED, result.reason_codes) + + def test_no_progress_can_replan_when_escalation_is_not_available(self): + request = ReplanRequest(ReplanTrigger.BLOCKED_STEP, "plan-1", ("evidence://stagnation",), 100) + result = decide_recovery(escalation(EscalationAction.CONTINUE), progress(ProgressStatus.NO_PROGRESS), replan_request=request) + self.assertEqual(result.action, RecoveryAction.REPLAN) + self.assertEqual(result.replan_request.previous_plan_ref, "plan-1") + + def test_unknown_requires_qualification(self): + result = decide_recovery(escalation(EscalationAction.REQUIRE_QUALIFICATION), progress(ProgressStatus.UNKNOWN)) + self.assertEqual(result.action, RecoveryAction.BLOCK) + self.assertIn(RecoveryReason.QUALIFICATION_REQUIRED, result.reason_codes) + + def test_handoff_requires_reason_and_evidence(self): + record = HandoffRecord("h-1", "a", "b", None, (), "summary", 1, 1, 0, 0, 0, ()) + result = decide_recovery(escalation(EscalationAction.CONTINUE), progress(ProgressStatus.PROGRESS_PROVEN), handoffs=(record,)) + self.assertEqual(result.action, RecoveryAction.BLOCK) + self.assertIn(RecoveryReason.HANDOFF_EVIDENCE_MISSING, result.reason_codes) + + def test_exhausted_replan_budget_blocks(self): + request = ReplanRequest(ReplanTrigger.BLOCKED_STEP, "plan-1", ("evidence://stagnation",), 100) + result = decide_recovery(escalation(EscalationAction.CONTINUE), progress(ProgressStatus.NO_PROGRESS), replans_used=1, replan_request=request) + self.assertEqual(result.action, RecoveryAction.BLOCK) + + def test_recovery_plan_binds_decision_to_source_attempt_deterministically(self): + decision = decide_recovery(escalation(EscalationAction.ESCALATE), progress(ProgressStatus.NO_PROGRESS)) + first = build_recovery_plan("attempt-1", decision) + second = build_recovery_plan("attempt-1", decision) + self.assertEqual(first.to_json(), second.to_json()) + self.assertEqual(first.source_attempt_id, "attempt-1") + validate_recovery_reference(first, first.reference) + with self.assertRaises(ValueError): + validate_recovery_reference(first, "recovery://other") + + def test_recovery_attempt_plan_derives_a_new_identity(self): + source = derive_attempt_id(trial_id="trial", attempt_number=1, configuration_digest="cfg") + plan = plan_recovery_attempt(trial_id="trial", source_attempt_id=source, source_attempt_number=1, configuration_digest="cfg", recovery_reference="recovery://1") + self.assertNotEqual(plan.next_attempt_id, plan.source_attempt_id) + self.assertEqual(plan.next_attempt_number, 2) + with self.assertRaises(ValueError): + plan_recovery_attempt(trial_id="trial", source_attempt_id="wrong", source_attempt_number=1, configuration_digest="cfg", recovery_reference="recovery://1") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_replay_attempt_inventory.py b/tests/test_replay_attempt_inventory.py new file mode 100644 index 00000000..0ca3ffd7 --- /dev/null +++ b/tests/test_replay_attempt_inventory.py @@ -0,0 +1,28 @@ +import unittest +from pathlib import Path + +from arkx.event_log import ReplayChain, classify_snapshot_chain_references +from arkx.harness import AttemptSnapshot, RunManifest, RunState, derive_attempt_id + + +class ReplayAttemptInventoryTests(unittest.TestCase): + def test_snapshot_inventory_classifies_physical_and_missing_refs(self): + trial_id = "trial-task-treatment-1" + manifest = RunManifest( + experiment_id="exp-1", trial_id=trial_id, + attempt_id=derive_attempt_id(trial_id=trial_id, attempt_number=1, configuration_digest="cfg-a"), + attempt_number=1, task_id="task-1", task_revision="rev", treatment="A", + executor="executor", provider="provider", model="model", sandbox="sandbox", + repository_revision="repo", configuration_digest="cfg-a", protocol_version="p1", + state=RunState.BLOCKED, artifact_refs=("artifact://manifest",), + ) + snapshot = AttemptSnapshot(manifest, {}, (Path("D:/artifacts/execution.json"),)) + physical_ref = str(Path("D:/artifacts/execution.json")) + report = classify_snapshot_chain_references(snapshot, ReplayChain(execution_ref=physical_ref, verification_ref="missing://verification")) + self.assertEqual(report.status(), "MISSING") + self.assertIn(physical_ref, report.present) + self.assertIn("missing://verification", report.missing) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_replay_chain.py b/tests/test_replay_chain.py new file mode 100644 index 00000000..4a8411a5 --- /dev/null +++ b/tests/test_replay_chain.py @@ -0,0 +1,75 @@ +import unittest + +from arkx.contracts import Event, EventType +from arkx.event_log import EventLog, ReplayChain, ReplayState, ResumeAction, append_recovery_plan, decide_resume, replay_chain, validate_attempt_lineage_chain +from arkx.harness import AttemptSnapshot, RecoveryLineage, RunManifest, RunState, derive_attempt_id +from arkx.recovery import RecoveryAction, RecoveryDecision, RecoveryReason, build_recovery_plan +from arkx.characterization import RecommendedPath + + +class ReplayChainTests(unittest.TestCase): + def test_attempt_lineage_requires_matching_event_references(self): + attempt_id = derive_attempt_id(trial_id="trial-1", attempt_number=2, configuration_digest="cfg") + source_id = derive_attempt_id(trial_id="trial-1", attempt_number=1, configuration_digest="cfg") + manifest = RunManifest("exp", "trial-1", attempt_id, 2, "task", "rev", "A", "executor", "provider", "model", "sandbox", "repo", "cfg", "protocol", budget_ledger={"consumed_attempts": [source_id, attempt_id]}) + snapshot = AttemptSnapshot(manifest, {}, (), recovery_lineage=RecoveryLineage("trial-1", source_id, 1, attempt_id, 2, "cfg", "recovery://2")) + replay = ReplayState(attempt_id, 2, budget_ledger=manifest.budget_ledger, recovery_source_attempt_id=source_id) + validate_attempt_lineage_chain(snapshot, ReplayChain(execution_ref=f"execution://{attempt_id}", recovery_ref="recovery://2"), replay) + with self.assertRaises(ValueError): + validate_attempt_lineage_chain(snapshot, ReplayChain(execution_ref=f"execution://{attempt_id}"), replay) + def test_resume_decision_only_returns_safe_persisted_boundary(self): + self.assertEqual(decide_resume(replay_chain((Event("2026-01-01T00:00:00+00:00", EventType.EVIDENCE_ADDED, "run-1", {"stage": "plan", "ref": "plan://1"}),), run_id="run-1"), ReplayState("run-1", 1)).action, ResumeAction.RESTART_REQUIRED) + execution = replay_chain((Event("2026-01-01T00:00:00+00:00", EventType.EVIDENCE_ADDED, "run-1", {"stage": "execution", "ref": "execution://1"}),), run_id="run-1") + self.assertEqual(decide_resume(execution, ReplayState("run-1", 1)).action, ResumeAction.RESUME_VERIFICATION) + verified = ReplayChain(execution_ref="execution://1", verification_ref="verification://1") + self.assertEqual(decide_resume(verified, ReplayState("run-1", 2)).action, ResumeAction.RESUME_ACCEPTANCE) + terminal = ReplayState("run-1", 1, declared_status="ACCEPTED") + self.assertEqual(decide_resume(verified, terminal).action, ResumeAction.TERMINAL) + def test_recovery_persistence_requires_matching_execution(self): + from tempfile import TemporaryDirectory + from pathlib import Path + with TemporaryDirectory() as directory: + log = EventLog(Path(directory) / "events.jsonl", run_id="attempt-1") + log.append_stage(timestamp="2026-01-01T00:00:00+00:00", event_type=EventType.EVIDENCE_ADDED, stage="execution", ref="execution://attempt-1") + decision = RecoveryDecision(RecoveryAction.REPLAN, (RecoveryReason.NO_PROGRESS_REQUIRES_REPLAN,), RecommendedPath.SIMPLE_PATH, RecommendedPath.LOCALIZED_PATH) + plan = build_recovery_plan("attempt-1", decision) + self.assertTrue(append_recovery_plan(log, plan, "execution://attempt-1", "2026-01-01T00:00:01+00:00")) + with self.assertRaises(ValueError): + append_recovery_plan(log, plan, "execution://other", "2026-01-01T00:00:02+00:00") + def test_replay_chain_reconstructs_explicit_stage_references(self): + events = tuple( + Event(f"2026-01-01T00:00:0{index}+00:00", EventType.EVIDENCE_ADDED, "run-1", {"stage": stage, "ref": f"{stage}://1"}) + for index, stage in enumerate(("request", "plan", "execution", "recovery", "verification", "acceptance", "promotion")) + ) + chain = replay_chain(events, run_id="run-1") + self.assertEqual(chain.promotion_ref, "promotion://1") + self.assertIsNotNone(chain.verification_ref) + + def test_recovery_reference_is_optional_but_ordered(self): + events = ( + Event("2026-01-01T00:00:00+00:00", EventType.EVIDENCE_ADDED, "run-1", {"stage": "execution", "ref": "execution://1"}), + Event("2026-01-01T00:00:01+00:00", EventType.REPLAN, "run-1", {"stage": "recovery", "ref": "recovery://1"}), + Event("2026-01-01T00:00:02+00:00", EventType.EVIDENCE_ADDED, "run-1", {"stage": "verification", "ref": "verification://1"}), + ) + chain = replay_chain(events, run_id="run-1") + self.assertEqual(chain.recovery_ref, "recovery://1") + + def test_replay_chain_rejects_out_of_order_stages(self): + events = ( + Event("2026-01-01T00:00:00+00:00", EventType.EVIDENCE_ADDED, "run-1", {"stage": "execution", "ref": "execution://1"}), + Event("2026-01-01T00:00:01+00:00", EventType.EVIDENCE_ADDED, "run-1", {"stage": "plan", "ref": "plan://1"}), + ) + with self.assertRaises(ValueError): + replay_chain(events, run_id="run-1") + + def test_replay_chain_rejects_stage_reference_drift(self): + events = ( + Event("2026-01-01T00:00:00+00:00", EventType.EVIDENCE_ADDED, "run-1", {"stage": "verification", "ref": "verification://1"}), + Event("2026-01-01T00:00:01+00:00", EventType.EVIDENCE_ADDED, "run-1", {"stage": "verification", "ref": "verification://2"}), + ) + with self.assertRaises(ValueError): + replay_chain(events, run_id="run-1") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_replay_integrity.py b/tests/test_replay_integrity.py new file mode 100644 index 00000000..4ab2a490 --- /dev/null +++ b/tests/test_replay_integrity.py @@ -0,0 +1,15 @@ +import unittest + +from arkx.event_log import ChainIntegrity, ReferenceReport, ReplayChain, assess_chain_integrity + + +class ReplayIntegrityTests(unittest.TestCase): + def test_chain_integrity_distinguishes_complete_incomplete_and_inconsistent(self): + complete = ReplayChain("request", "plan", "execution", "verification", "acceptance", "promotion") + self.assertEqual(assess_chain_integrity(complete, ReferenceReport(present=tuple(complete.to_dict().values()))), ChainIntegrity.COMPLETE) + self.assertEqual(assess_chain_integrity(ReplayChain(request_ref="request"), ReferenceReport(indeterminate=("request",))), ChainIntegrity.INCOMPLETE) + self.assertEqual(assess_chain_integrity(complete, ReferenceReport(missing=("promotion",))), ChainIntegrity.INCONSISTENT) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_replay_references.py b/tests/test_replay_references.py new file mode 100644 index 00000000..c51fb518 --- /dev/null +++ b/tests/test_replay_references.py @@ -0,0 +1,21 @@ +import unittest + +from arkx.event_log import ReplayChain, classify_chain_references + + +class ReplayReferenceTests(unittest.TestCase): + def test_reference_classification_preserves_missing_and_present(self): + chain = ReplayChain(request_ref="request://1", execution_ref="execution://1") + report = classify_chain_references(chain, available_refs={"request://1"}) + self.assertEqual(report.status(), "MISSING") + self.assertEqual(report.present, ("request://1",)) + self.assertEqual(report.missing, ("execution://1",)) + + def test_reference_classification_is_indeterminate_without_inventory(self): + report = classify_chain_references(ReplayChain(request_ref="request://1")) + self.assertEqual(report.status(), "INDETERMINATE") + self.assertEqual(report.indeterminate, ("request://1",)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_request.py b/tests/test_request.py new file mode 100644 index 00000000..744a35bf --- /dev/null +++ b/tests/test_request.py @@ -0,0 +1,127 @@ +import unittest + +from arkx.characterization import Scope, TaskSignals +from arkx.request import ( + ArkxRequest, + GovernancePolicy, + GovernanceReason, + GovernanceStatus, + RequestBudget, + RequestEnvironment, + authorize_request, + characterize_authorized_request, +) + + +def valid_request(**overrides): + values = { + "request_id": "req-1", + "task_id": "task-1", + "requester": "developer", + "signals": TaskSignals( + candidate_files=("src/arkx/example.py",), + dependency_edges=(), + affected_components=("example",), + known_tests=("tests/test_example.py",), + ambiguity_markers=(), + risk_markers=(), + acceptance_checks=("tests pass",), + ), + "authorized_paths": ("src/arkx",), + "budget": RequestBudget(max_attempts=2, max_tokens=1000, max_wall_time_ms=10000), + "environment": RequestEnvironment("local", "workspace", "HEAD"), + "acceptance_authority": "authority-1", + } + values.update(overrides) + return ArkxRequest(**values) + + +def policy(**overrides): + values = { + "authority_id": "governance-1", + "allowed_requesters": ("developer",), + "allowed_environments": ("local",), + "max_attempts": 3, + "max_tokens": 2000, + "max_wall_time_ms": 20000, + } + values.update(overrides) + return GovernancePolicy(**values) + + +class GovernanceTests(unittest.TestCase): + def test_valid_request_is_authorized(self): + result = authorize_request(valid_request(), policy()) + self.assertEqual(result.status, GovernanceStatus.AUTHORIZED) + self.assertEqual(result.reason_codes, (GovernanceReason.AUTHORIZED,)) + + def test_missing_authority_scope_or_budget_is_blocked(self): + result = authorize_request( + valid_request(authorized_paths=None, budget=None, acceptance_authority=None), + policy(), + ) + self.assertEqual(result.status, GovernanceStatus.BLOCKED) + self.assertIn(GovernanceReason.AUTHORIZED_SCOPE_MISSING, result.reason_codes) + self.assertIn(GovernanceReason.BUDGET_MISSING, result.reason_codes) + self.assertIn(GovernanceReason.ACCEPTANCE_AUTHORITY_MISSING, result.reason_codes) + self.assertIsNone(result.request) + + def test_out_of_scope_file_is_blocked(self): + result = authorize_request( + valid_request( + signals=valid_request().signals.__class__( + candidate_files=("outside.py",), + dependency_edges=(), + affected_components=("example",), + known_tests=("tests/test_example.py",), + ambiguity_markers=(), + risk_markers=(), + acceptance_checks=("tests pass",), + ) + ), + policy(), + ) + self.assertEqual(result.status, GovernanceStatus.BLOCKED) + self.assertIn(GovernanceReason.PATH_OUTSIDE_AUTHORIZED_SCOPE, result.reason_codes) + + def test_budget_over_policy_is_blocked(self): + result = authorize_request(valid_request(budget=RequestBudget(max_attempts=4)), policy(max_attempts=3)) + self.assertEqual(result.status, GovernanceStatus.BLOCKED) + self.assertIn(GovernanceReason.BUDGET_EXCEEDED, result.reason_codes) + + def test_environment_and_requester_are_policy_bound(self): + result = authorize_request( + valid_request(requester="unknown", environment=RequestEnvironment("remote", "workspace", "HEAD")), + policy(), + ) + self.assertEqual(result.status, GovernanceStatus.BLOCKED) + self.assertIn(GovernanceReason.REQUESTER_NOT_ALLOWED, result.reason_codes) + self.assertIn(GovernanceReason.ENVIRONMENT_NOT_ALLOWED, result.reason_codes) + + def test_environment_requires_workspace_and_revision(self): + result = authorize_request( + valid_request(environment=RequestEnvironment("local", None, None)), + policy(), + ) + self.assertEqual(result.status, GovernanceStatus.BLOCKED) + self.assertIn(GovernanceReason.WORKSPACE_MISSING, result.reason_codes) + self.assertIn(GovernanceReason.REVISION_MISSING, result.reason_codes) + + def test_unauthorized_request_cannot_be_characterized(self): + decision = authorize_request(valid_request(authorized_paths=None), policy()) + self.assertIsNone(characterize_authorized_request(decision)) + + def test_authorized_request_is_characterized_without_selecting_executor(self): + decision = authorize_request(valid_request(), policy()) + result = characterize_authorized_request(decision) + self.assertEqual(result.scope, Scope.SIMPLE) + self.assertNotIn("executor", result.to_dict()) + + def test_serialization_is_deterministic(self): + first = authorize_request(valid_request(), policy()).to_json() + second = authorize_request(valid_request(), policy()).to_json() + self.assertEqual(first, second) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_restored_chain_audit.py b/tests/test_restored_chain_audit.py new file mode 100644 index 00000000..2024cdb9 --- /dev/null +++ b/tests/test_restored_chain_audit.py @@ -0,0 +1,107 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from arkx.acceptance import AcceptancePolicy, decide_independent_acceptance +from arkx.contracts import Event, EventType +from arkx.event_log import ChainIntegrity, EventLog, audit_restored_chain +from arkx.harness import AttemptSnapshot, AttemptStore, RunManifest, RunState, derive_attempt_id +from arkx.event_log import ResumeAction +from arkx.outcomes import VerificationResult, VerificationState +from arkx.promotion import PromotionCandidate, PromotionPolicy, decide_outcome_promotion + + +class RestoredChainAuditTests(unittest.TestCase): + def test_restored_chain_audit_replays_without_execution(self): + trial_id = "trial-1" + attempt_id = derive_attempt_id(trial_id=trial_id, attempt_number=1, configuration_digest="cfg") + manifest = RunManifest( + experiment_id="exp", trial_id=trial_id, attempt_id=attempt_id, attempt_number=1, + task_id="task", task_revision="rev", treatment="A", executor="executor", + provider="provider", model="model", sandbox="sandbox", repository_revision="rev", + configuration_digest="cfg", protocol_version="protocol", state=RunState.COMPLETED, + ) + snapshot = AttemptSnapshot(manifest, {}, ()) + verification = VerificationResult(VerificationState.PASS, "verifier", ("pytest",), ("0",), ("evidence://v",)) + acceptance = decide_independent_acceptance(verification, AcceptancePolicy("authority")) + promotion = decide_outcome_promotion( + PromotionCandidate("executor", "candidate", "1", "cfg"), verification, acceptance, + ("experiment://1",), PromotionPolicy("authority"), + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "events.jsonl" + log = EventLog(path, run_id=attempt_id) + for index, (stage, ref) in enumerate(( + ("verification", verification.reference), + ("acceptance", acceptance.reference), + ("promotion", promotion.reference), + )): + log.append(Event(f"2026-01-01T00:00:0{index}+00:00", EventType.EVIDENCE_ADDED, attempt_id, {"stage": stage, "ref": ref, "protocol_version": "protocol", "schema_version": 1, "configuration_digest": "cfg", "treatment": "A"})) + audit = audit_restored_chain(path, snapshot, verification=verification, acceptance=acceptance, promotion=promotion, protocol_version="protocol") + self.assertEqual(audit.integrity, ChainIntegrity.INCOMPLETE) + self.assertEqual(audit.references.missing, ()) + self.assertEqual(audit.replay.event_count, 3) + self.assertEqual(audit.resume_decision.action, ResumeAction.RESTART_REQUIRED) + + def test_restored_chain_rejects_protocol_drift(self): + manifest = RunManifest( + experiment_id="exp", trial_id="trial", attempt_id="attempt", attempt_number=1, + task_id="task", task_revision="rev", treatment="A", executor="executor", + provider="provider", model="model", sandbox="sandbox", repository_revision="rev", + configuration_digest="cfg", protocol_version="protocol", state=RunState.COMPLETED, + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "events.jsonl" + EventLog(path, run_id="attempt").append(Event("2026-01-01T00:00:00+00:00", EventType.EVIDENCE_ADDED, "attempt", {"stage": "verification", "ref": "verification://1", "protocol_version": "other", "schema_version": 1})) + with self.assertRaises(ValueError): + audit_restored_chain(path, AttemptSnapshot(manifest, {}, ()), protocol_version="protocol") + + def test_restored_chain_rejects_configuration_drift(self): + manifest = RunManifest( + experiment_id="exp", trial_id="trial", attempt_id="attempt", attempt_number=1, + task_id="task", task_revision="rev", treatment="A", executor="executor", + provider="provider", model="model", sandbox="sandbox", repository_revision="rev", + configuration_digest="cfg", protocol_version="protocol", state=RunState.COMPLETED, + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "events.jsonl" + EventLog(path, run_id="attempt").append(Event("2026-01-01T00:00:00+00:00", EventType.EVIDENCE_ADDED, "attempt", {"stage": "verification", "ref": "verification://1", "protocol_version": "protocol", "schema_version": 1, "configuration_digest": "other", "treatment": "A"})) + with self.assertRaises(ValueError): + audit_restored_chain(path, AttemptSnapshot(manifest, {}, ()), protocol_version="protocol", configuration_digest="cfg") + + def test_restored_chain_audit_rejects_manifest_status_divergence(self): + manifest = RunManifest( + experiment_id="exp", trial_id="trial", attempt_id="attempt", attempt_number=1, + task_id="task", task_revision="rev", treatment="A", executor="executor", + provider="provider", model="model", sandbox="sandbox", repository_revision="rev", + configuration_digest="cfg", protocol_version="protocol", state=RunState.COMPLETED, + ) + snapshot = AttemptSnapshot(manifest, {}, ()) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "events.jsonl" + EventLog(path, run_id="attempt").append(Event("2026-01-01T00:00:00+00:00", EventType.TASK_FINISHED, "attempt", {"status": "BLOCKED"})) + with self.assertRaises(ValueError): + audit_restored_chain(path, snapshot) + + def test_attempt_store_exposes_audited_loading(self): + snapshot = AttemptSnapshot( + RunManifest( + experiment_id="exp", trial_id="trial", attempt_id="attempt", attempt_number=1, + task_id="task", task_revision="rev", treatment="A", executor="executor", + provider="provider", model="model", sandbox="sandbox", repository_revision="rev", + configuration_digest="cfg", protocol_version="protocol", state=RunState.COMPLETED, + ), + {}, + (), + ) + with patch("arkx.harness.load_attempt", return_value=snapshot) as loader: + with patch("arkx.event_log.audit_restored_chain", return_value="audited") as auditor: + result = AttemptStore("artifacts").load_audited("attempt-root", "events.jsonl") + self.assertEqual(result, "audited") + loader.assert_called_once_with("attempt-root") + auditor.assert_called_once_with("events.jsonl", snapshot) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_routing.py b/tests/test_routing.py new file mode 100644 index 00000000..f65f1c09 --- /dev/null +++ b/tests/test_routing.py @@ -0,0 +1,254 @@ +import json +import unittest +from dataclasses import replace +from pathlib import Path + +from arkx.characterization import RecommendedPath, Scope, TaskSignals, characterize +from arkx.contracts import ExecutionStatus +from arkx.progress import ProgressSnapshot, ProgressStatus, assess_progress +from arkx.routing import ( + BudgetState, + EscalationAction, + EvidenceSufficiency, + ReasonCode, + RoutingBudget, + RoutingDecisionType, + assess_escalation, + route_characterization, + CapabilityPolicyReason, + assess_capability_policy, +) +from arkx.integration import AdapterIdentity, CapabilityProvenance, DependencyObservation, IntegrationKind, assess_preflight +from arkx.telemetry import TelemetryCollector + + +def characterization(scope: Scope): + if scope is Scope.SIMPLE: + signals = TaskSignals( + candidate_files=("src/a.py",), + dependency_edges=(), + affected_components=("a",), + known_tests=("test_a",), + ambiguity_markers=(), + risk_markers=(), + acceptance_checks=("tests pass",), + ) + elif scope is Scope.LOCALIZED: + signals = TaskSignals( + candidate_files=("src/a.py", "tests/test_a.py"), + dependency_edges=(("a", "contracts"),), + affected_components=("a",), + known_tests=("test_a", "test_contracts"), + ambiguity_markers=(), + risk_markers=(), + acceptance_checks=("tests pass", "round-trip pass"), + ) + elif scope is Scope.REPOSITORY_WIDE: + signals = TaskSignals( + candidate_files=("a.py", "b.py", "c.py", "d.py", "e.py", "f.py"), + dependency_edges=(("a", "b"), ("b", "c"), ("c", "shared"), ("tests", "a")), + affected_components=("a", "b", "c"), + known_tests=("test_a", "test_b", "test_c"), + ambiguity_markers=(), + risk_markers=(), + acceptance_checks=("tests pass",), + state_shared=True, + architectural_change=True, + ) + else: + return characterize(TaskSignals()) + return characterize(signals) + + +def progress(status: ProgressStatus): + base = ProgressSnapshot( + useful_files=("src/a.py",), + passing_tests=(), + explained_failures=(), + diff_distance=3, + acceptance_distance=2, + recent_actions=("run test_a",), + failure_signatures=("failure-a",), + ) + if status is ProgressStatus.PROGRESS_PROVEN: + current = replace(base, passing_tests=("test_a",)) + return assess_progress(base, current) + if status is ProgressStatus.NO_PROGRESS: + return assess_progress(base, base) + return assess_progress(None, ProgressSnapshot()) + + +class RoutingTests(unittest.TestCase): + def test_scopes_route_to_matching_paths(self): + expected = { + Scope.SIMPLE: RecommendedPath.SIMPLE_PATH, + Scope.LOCALIZED: RecommendedPath.LOCALIZED_PATH, + Scope.REPOSITORY_WIDE: RecommendedPath.REPOSITORY_WIDE_PATH, + } + for scope, path in expected.items(): + decision = route_characterization("task", characterization(scope)) + self.assertEqual(decision.decision, RoutingDecisionType.ROUTE) + self.assertEqual(decision.selected_path, path) + + def test_unknown_characterization_requires_qualification(self): + decision = route_characterization("task", characterization(Scope.UNKNOWN)) + self.assertEqual(decision.decision, RoutingDecisionType.REQUIRE_QUALIFICATION) + self.assertEqual(decision.selected_path, RecommendedPath.QUALIFICATION_REQUIRED) + + def test_missing_characterization_requires_qualification(self): + decision = route_characterization("task", None) + self.assertEqual(decision.decision, RoutingDecisionType.REQUIRE_QUALIFICATION) + + def test_inconsistent_characterization_is_rejected(self): + value = characterization(Scope.SIMPLE) + invalid = replace(value, recommended_path=RecommendedPath.LOCALIZED_PATH) + decision = route_characterization("task", invalid) + self.assertEqual(decision.decision, RoutingDecisionType.REJECT_INVALID_INPUT) + + +class EscalationTests(unittest.TestCase): + def test_simple_no_progress_escalates_to_localized(self): + decision = assess_escalation( + RecommendedPath.SIMPLE_PATH, + progress(ProgressStatus.NO_PROGRESS), + EvidenceSufficiency.INSUFFICIENT, + ) + self.assertEqual(decision.action, EscalationAction.ESCALATE) + self.assertEqual(decision.target_path, RecommendedPath.LOCALIZED_PATH) + self.assertEqual(decision.budget_state.path_escalations_used, 1) + + def test_localized_no_progress_escalates_to_repository_wide(self): + decision = assess_escalation( + RecommendedPath.LOCALIZED_PATH, + progress(ProgressStatus.NO_PROGRESS), + EvidenceSufficiency.INSUFFICIENT, + ) + self.assertEqual(decision.action, EscalationAction.ESCALATE) + self.assertEqual(decision.target_path, RecommendedPath.REPOSITORY_WIDE_PATH) + + def test_repository_wide_stagnation_blocks(self): + decision = assess_escalation( + RecommendedPath.REPOSITORY_WIDE_PATH, + progress(ProgressStatus.NO_PROGRESS), + EvidenceSufficiency.INSUFFICIENT, + ) + self.assertEqual(decision.action, EscalationAction.BLOCK) + self.assertIn(ReasonCode.NO_HIGHER_PATH, decision.reason_codes) + + def test_progress_and_sufficient_evidence_stops_without_pass(self): + decision = assess_escalation( + RecommendedPath.SIMPLE_PATH, + progress(ProgressStatus.PROGRESS_PROVEN), + EvidenceSufficiency.SUFFICIENT, + ) + self.assertEqual(decision.action, EscalationAction.STOP_SUFFICIENT_EVIDENCE) + self.assertNotIn("PASS", decision.to_json()) + + def test_progress_with_insufficient_evidence_continues_if_attempt_budget_exists(self): + decision = assess_escalation( + RecommendedPath.SIMPLE_PATH, + progress(ProgressStatus.PROGRESS_PROVEN), + EvidenceSufficiency.INSUFFICIENT, + ) + self.assertEqual(decision.action, EscalationAction.CONTINUE) + + def test_unknown_evidence_cannot_stop(self): + decision = assess_escalation( + RecommendedPath.SIMPLE_PATH, + progress(ProgressStatus.PROGRESS_PROVEN), + EvidenceSufficiency.UNKNOWN, + ) + self.assertEqual(decision.action, EscalationAction.REQUIRE_QUALIFICATION) + + def test_exhausted_budget_blocks(self): + state = BudgetState(RoutingBudget(max_path_escalations=2, max_attempts=3), attempts_used=3, path_escalations_used=2) + decision = assess_escalation( + RecommendedPath.SIMPLE_PATH, + progress(ProgressStatus.NO_PROGRESS), + EvidenceSufficiency.INSUFFICIENT, + budget_state=state, + ) + self.assertEqual(decision.action, EscalationAction.BLOCK) + + def test_invalid_path_transition_blocks(self): + decision = assess_escalation( + RecommendedPath.QUALIFICATION_REQUIRED, + progress(ProgressStatus.PROGRESS_PROVEN), + EvidenceSufficiency.SUFFICIENT, + ) + self.assertEqual(decision.action, EscalationAction.BLOCK) + self.assertIn(ReasonCode.INVALID_PATH_TRANSITION, decision.reason_codes) + + +class DeterminismTests(unittest.TestCase): + def test_same_inputs_have_byte_equivalent_routing_serialization(self): + first = route_characterization("task", characterization(Scope.LOCALIZED)) + second = route_characterization("task", characterization(Scope.LOCALIZED)) + self.assertEqual(first.to_json(), second.to_json()) + + def test_routing_decisions_do_not_change_p0_status(self): + route_characterization("task", characterization(Scope.SIMPLE)) + execution = TelemetryCollector(task_id="task", run_id="run").summarize() + self.assertEqual(execution.status, ExecutionStatus.NOT_EXECUTED) + + def test_synthetic_fixture_covers_seven_scenarios(self): + path = Path(__file__).parents[1] / "experiments" / "routing-escalation-fixture.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(manifest["fixture_type"], "ROUTING_POLICY_FIXTURE") + self.assertEqual(len(manifest["scenarios"]), 7) + scopes = {scope.value: scope for scope in Scope} + statuses = {status.value: status for status in ProgressStatus} + sufficiencies = {value.value: value for value in EvidenceSufficiency} + for scenario in manifest["scenarios"]: + route = route_characterization("fixture", characterization(scopes[scenario["route_scope"]])) + if "expected_route_decision" in scenario: + self.assertEqual(route.decision.value, scenario["expected_route_decision"]) + continue + self.assertEqual(route.selected_path.value, scenario["expected_path"]) + state = BudgetState( + attempts_used=scenario.get("attempts_used", 0), + path_escalations_used=scenario.get("path_escalations_used", 0), + ) + decision = assess_escalation( + route.selected_path, + progress(statuses[scenario["progress"]]), + sufficiencies[scenario["evidence"]], + budget_state=state, + ) + self.assertEqual(decision.action.value, scenario["expected_action"]) + if "expected_target" in scenario: + self.assertEqual(decision.target_path.value, scenario["expected_target"]) + +class CapabilityRoutingTests(unittest.TestCase): + def _preflight(self, provenance=CapabilityProvenance.OBSERVED, capabilities=("patch",)): + return assess_preflight( + AdapterIdentity(IntegrationKind.EXECUTOR, "executor", "1", "cfg"), + (DependencyObservation("runtime", True, "1"),), + capabilities=capabilities, + capability_digest="capability-digest", + capability_provenance=provenance, + ) + + def test_policy_requires_minimum_capability_provenance(self): + decision = assess_capability_policy(self._preflight(), ("patch",), minimum_provenance=CapabilityProvenance.QUALIFIED) + self.assertFalse(decision.allowed) + self.assertEqual(decision.reason, CapabilityPolicyReason.CAPABILITY_PROVENANCE_INSUFFICIENT) + + def test_policy_does_not_route_when_preflight_is_not_ready(self): + preflight = assess_preflight( + AdapterIdentity(IntegrationKind.EXECUTOR, "executor", "1", "cfg"), + (DependencyObservation("runtime", False),), + capabilities=("patch",), capability_digest="capability-digest", + capability_provenance=CapabilityProvenance.OBSERVED, + ) + decision = assess_capability_policy(preflight, ("patch",)) + self.assertFalse(decision.allowed) + self.assertEqual(decision.reason, CapabilityPolicyReason.PREFLIGHT_NOT_READY) + + def test_policy_allows_observed_capability_when_required(self): + decision = assess_capability_policy(self._preflight(), ("patch",), minimum_provenance=CapabilityProvenance.OBSERVED) + self.assertTrue(decision.allowed) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_selection.py b/tests/test_selection.py new file mode 100644 index 00000000..e57390c5 --- /dev/null +++ b/tests/test_selection.py @@ -0,0 +1,75 @@ +import unittest + +from arkx.characterization import RecommendedPath +from arkx.composition import Mechanism +from arkx.executor_qualification import ExecutorIdentity, QualificationStatus +from arkx.selection import ( + ExecutorBinding, + ExecutorRegistry, + SelectionReason, + SelectionStatus, + TreatmentCatalog, + TreatmentDefinition, + select_executor, + select_treatment, +) + + +def binding(name="mini", status=QualificationStatus.QUALIFIABLE, evidence=("evidence://trial",), capabilities=("edit",)): + return ExecutorBinding( + ExecutorIdentity(name, "1.0", "adapter", f"digest-{name}"), + capabilities, + status, + evidence, + ) + + +class SelectionTests(unittest.TestCase): + def test_treatment_is_selected_after_path(self): + treatment = TreatmentDefinition("simple-mini", RecommendedPath.SIMPLE_PATH, ("edit",), (Mechanism.P8_SAFE_EDITOR,)) + result = select_treatment(RecommendedPath.SIMPLE_PATH, TreatmentCatalog((treatment,))) + self.assertEqual(result.status, SelectionStatus.SELECTED) + self.assertEqual(result.treatment.name, "simple-mini") + + def test_ambiguous_treatments_require_qualification(self): + catalog = TreatmentCatalog((TreatmentDefinition("a", RecommendedPath.SIMPLE_PATH), TreatmentDefinition("b", RecommendedPath.SIMPLE_PATH))) + result = select_treatment(RecommendedPath.SIMPLE_PATH, catalog) + self.assertEqual(result.status, SelectionStatus.REQUIRE_QUALIFICATION) + self.assertIn(SelectionReason.AMBIGUOUS_TREATMENT, result.reason_codes) + + def test_missing_treatment_blocks(self): + result = select_treatment(RecommendedPath.LOCALIZED_PATH, TreatmentCatalog(())) + self.assertEqual(result.status, SelectionStatus.BLOCKED) + + def test_executor_requires_capability_and_evidence(self): + treatment = TreatmentDefinition("localized", RecommendedPath.LOCALIZED_PATH, ("safe_edit",)) + result = select_executor(treatment, ExecutorRegistry((binding(capabilities=("edit",)),))) + self.assertEqual(result.status, SelectionStatus.BLOCKED) + self.assertIn(SelectionReason.NO_EXECUTOR_FOR_CAPABILITY, result.reason_codes) + + def test_unqualified_executor_cannot_be_selected(self): + treatment = TreatmentDefinition("simple", RecommendedPath.SIMPLE_PATH, ("edit",)) + result = select_executor(treatment, ExecutorRegistry((binding(status=QualificationStatus.UNKNOWN, evidence=None),))) + self.assertEqual(result.status, SelectionStatus.REQUIRE_QUALIFICATION) + self.assertIn(SelectionReason.QUALIFICATION_EVIDENCE_MISSING, result.reason_codes) + + def test_multiple_qualified_executors_do_not_trigger_implicit_choice(self): + treatment = TreatmentDefinition("simple", RecommendedPath.SIMPLE_PATH, ("edit",)) + result = select_executor(treatment, ExecutorRegistry((binding("a"), binding("b")))) + self.assertEqual(result.status, SelectionStatus.REQUIRE_QUALIFICATION) + self.assertIn(SelectionReason.AMBIGUOUS_EXECUTOR, result.reason_codes) + + def test_exactly_one_qualified_executor_is_selected(self): + treatment = TreatmentDefinition("simple", RecommendedPath.SIMPLE_PATH, ("edit",)) + result = select_executor(treatment, ExecutorRegistry((binding(),))) + self.assertEqual(result.status, SelectionStatus.SELECTED) + self.assertEqual(result.executor.executor.name, "mini") + + def test_serialization_is_deterministic(self): + treatment = TreatmentDefinition("simple", RecommendedPath.SIMPLE_PATH, ("edit",)) + registry = ExecutorRegistry((binding(),)) + self.assertEqual(select_executor(treatment, registry).to_json(), select_executor(treatment, registry).to_json()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_swebench_authority.py b/tests/test_swebench_authority.py new file mode 100644 index 00000000..384f1a49 --- /dev/null +++ b/tests/test_swebench_authority.py @@ -0,0 +1,103 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from arkx.swebench_authority import ( + AUTHORITY_IDENTITY, OfficialEvaluationResult, OfficialEvaluationStatus, + SWEbenchOfficialAuthority, SWEbenchPrediction, SWEbenchRecord, +) + + +def record(): + return SWEbenchRecord( + "owner__repo-1", "owner/repo", "base", "problem", "1.0", "env", + ("fail",), ("pass",), "test patch", "gold patch", "revision", "now", + ) + + +class SWEbenchAuthorityTests(unittest.TestCase): + def test_prediction_serialization_has_no_gold_or_grading_labels(self): + value = SWEbenchPrediction("task", "provider/model", "diff").to_dict() + self.assertEqual(value, {"instance_id": "task", "model_name_or_path": "provider/model", "model_patch": "diff"}) + self.assertNotIn("gold_patch", value) + + def test_executor_payload_excludes_gold_and_test_labels(self): + value = record().executor_payload() + self.assertNotIn("gold_patch", value) + self.assertNotIn("FAIL_TO_PASS", value) + self.assertNotIn("test_patch", value) + + def test_official_outcome_mapping(self): + accepted = OfficialEvaluationResult(OfficialEvaluationStatus.RESOLVED, AUTHORITY_IDENTITY, "r1", "t", "resolved", "report", ()) + rejected = OfficialEvaluationResult(OfficialEvaluationStatus.TESTS_FAILED, AUTHORITY_IDENTITY, "r2", "t", "failed", "report", ()) + infra = OfficialEvaluationResult(OfficialEvaluationStatus.INFRASTRUCTURE_ERROR, AUTHORITY_IDENTITY, "r3", "t", None, None, ()) + self.assertEqual(accepted.acceptance().decision.value, "ACCEPTED") + self.assertEqual(rejected.acceptance().decision.value, "REJECTED") + self.assertEqual(infra.acceptance().decision.value, "INDETERMINATE") + + def test_evaluator_infrastructure_failure_is_not_a_test_failure(self): + infra = OfficialEvaluationResult( + OfficialEvaluationStatus.INFRASTRUCTURE_ERROR, + AUTHORITY_IDENTITY, + "r3", + "t", + None, + "report", + ("log",), + "docker daemon unavailable", + ) + rejected = OfficialEvaluationResult( + OfficialEvaluationStatus.TESTS_FAILED, + AUTHORITY_IDENTITY, + "r4", + "t", + "failed", + "report", + (), + ) + error = infra.error_envelope() + self.assertEqual(error.domain.value, "VERIFICATION") + self.assertEqual(error.code, "EVALUATOR_INFRASTRUCTURE_ERROR") + self.assertEqual(error.retryability.value, "UNKNOWN") + self.assertEqual(error.raw_evidence_refs, ("log", "report")) + self.assertIsNone(rejected.error_envelope()) + + def test_evaluator_maps_to_verification_without_acceptance_or_promotion(self): + result = OfficialEvaluationResult( + OfficialEvaluationStatus.TESTS_FAILED, + AUTHORITY_IDENTITY, + "r5", + "t", + "failed", + "report", + ("log",), + ) + verification = result.verification_result() + self.assertEqual(verification.state.value, "FAIL") + self.assertEqual(verification.authority, AUTHORITY_IDENTITY) + self.assertEqual(verification.commands, ("swebench.harness.run_evaluation",)) + self.assertEqual(verification.evidence, ("log", "report")) + self.assertTrue(verification.reference.startswith("verification://")) + self.assertEqual(result.acceptance().decision.value, "REJECTED") + + def test_authority_persists_prediction_and_raw_artifact_link(self): + with tempfile.TemporaryDirectory() as directory: + def evaluator(payload): + prediction = json.loads(Path(payload["predictions_path"]).read_text(encoding="utf-8")) + self.assertEqual(prediction["instance_id"], "owner__repo-1") + return OfficialEvaluationResult(OfficialEvaluationStatus.RESOLVED, AUTHORITY_IDENTITY, payload["run_id"], "owner__repo-1", "resolved", payload["predictions_path"], ()) + result = SWEbenchOfficialAuthority(directory, evaluator).evaluate(record=record(), model_name_or_path="m", model_patch="diff", run_id="unique-run") + self.assertEqual(result.status, OfficialEvaluationStatus.RESOLVED) + self.assertTrue(Path(result.report_path).exists()) + + def test_same_run_id_is_not_reused_for_a_second_artifact(self): + with tempfile.TemporaryDirectory() as directory: + authority = SWEbenchOfficialAuthority(directory, lambda payload: OfficialEvaluationResult(OfficialEvaluationStatus.RESOLVED, AUTHORITY_IDENTITY, payload["run_id"], "owner__repo-1", "resolved", None, ())) + authority.evaluate(record=record(), model_name_or_path="m", model_patch="one", run_id="r") + blocked = authority.evaluate(record=record(), model_name_or_path="m", model_patch="two", run_id="r") + self.assertEqual(blocked.status, OfficialEvaluationStatus.BLOCKED) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_terminal_event.py b/tests/test_terminal_event.py new file mode 100644 index 00000000..ec2e41d2 --- /dev/null +++ b/tests/test_terminal_event.py @@ -0,0 +1,35 @@ +import unittest +from tempfile import TemporaryDirectory +from pathlib import Path + +from arkx.contracts import EventType +from arkx.event_log import EventLog, replay_events + + +class TerminalEventTests(unittest.TestCase): + def test_terminal_event_is_idempotent_and_replayable(self): + with TemporaryDirectory() as directory: + log = EventLog(Path(directory) / "events.jsonl", run_id="run-1") + self.assertTrue(log.append_terminal(timestamp="2026-01-01T00:00:00+00:00", status="BLOCKED", error_code="SANDBOX_UNAVAILABLE")) + self.assertFalse(log.append_terminal(timestamp="2026-01-01T00:00:01+00:00", status="BLOCKED", error_code="SANDBOX_UNAVAILABLE")) + replay = replay_events(log.read(), run_id="run-1") + self.assertEqual(replay.declared_status, "BLOCKED") + self.assertEqual(replay.error_codes, ("SANDBOX_UNAVAILABLE",)) + + def test_terminal_status_cannot_change(self): + with TemporaryDirectory() as directory: + log = EventLog(Path(directory) / "events.jsonl", run_id="run-1") + log.append_terminal(timestamp="2026-01-01T00:00:00+00:00", status="FAILED") + with self.assertRaises(ValueError): + log.append_terminal(timestamp="2026-01-01T00:00:01+00:00", status="COMPLETED") + + def test_terminal_error_code_cannot_change(self): + with TemporaryDirectory() as directory: + log = EventLog(Path(directory) / "events.jsonl", run_id="run-1") + log.append_terminal(timestamp="2026-01-01T00:00:00+00:00", status="BLOCKED", error_code="A") + with self.assertRaises(ValueError): + log.append_terminal(timestamp="2026-01-01T00:00:01+00:00", status="BLOCKED", error_code="B") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_terminal_replay_validation.py b/tests/test_terminal_replay_validation.py new file mode 100644 index 00000000..782fe27f --- /dev/null +++ b/tests/test_terminal_replay_validation.py @@ -0,0 +1,18 @@ +import unittest + +from arkx.event_log import ReplayState, validate_terminal_replay + + +class TerminalReplayValidationTests(unittest.TestCase): + def test_terminal_status_and_error_are_checked(self): + replay = ReplayState("run-1", 1, declared_status="BLOCKED", error_codes=("SANDBOX_UNAVAILABLE",)) + validate_terminal_replay(replay, expected_status="BLOCKED", expected_error_code="SANDBOX_UNAVAILABLE") + + def test_missing_terminal_is_not_inferred(self): + with self.assertRaises(ValueError): + validate_terminal_replay(ReplayState("run-1", 0), expected_status="COMPLETED") + + def test_wrong_terminal_error_is_rejected(self): + replay = ReplayState("run-1", 1, declared_status="BLOCKED", error_codes=("A",)) + with self.assertRaises(ValueError): + validate_terminal_replay(replay, expected_status="BLOCKED", expected_error_code="B") diff --git a/tests/test_verification.py b/tests/test_verification.py new file mode 100644 index 00000000..4656225c --- /dev/null +++ b/tests/test_verification.py @@ -0,0 +1,118 @@ +import json +import unittest +from pathlib import Path + +from arkx.verification import ( + PatchVerificationInput, + PatchVerificationStatus, + ReasonCode, + TestResult, + TestResultStatus, + verify_patch, +) + + +def valid(**overrides): + values = { + "task_id": "task", + "issue_reproduction_required": True, + "reproduces_issue_before_patch": True, + "patch_applied": True, + "regression_results": (TestResult("regression", TestResultStatus.PASSED, True, "e://test"),), + "reproduces_issue_after_patch": False, + "changed_files": ("src/a.py",), + "expected_scope": ("src/a.py",), + "evidence_refs": ("e://patch",), + } + values.update(overrides) + return PatchVerificationInput(**values) + + +class VerificationTests(unittest.TestCase): + def test_all_gates_produce_verified_not_pass(self): + result = verify_patch(valid()) + self.assertEqual(result.status, PatchVerificationStatus.VERIFIED) + self.assertNotIn("PASS", result.to_json()) + + def test_required_reproduction_absent_is_blocked(self): + result = verify_patch(valid(reproduces_issue_before_patch=None)) + self.assertEqual(result.status, PatchVerificationStatus.BLOCKED) + self.assertIn(ReasonCode.REPRODUCTION_REQUIRED_MISSING, result.reason_codes) + + def test_regression_failure_is_rejected(self): + result = verify_patch(valid(regression_results=(TestResult("regression", TestResultStatus.FAILED, True),))) + self.assertEqual(result.status, PatchVerificationStatus.REJECTED) + + def test_reproduction_before_true_after_true_is_rejected(self): + result = verify_patch(valid(reproduces_issue_after_patch=True)) + self.assertEqual(result.status, PatchVerificationStatus.REJECTED) + self.assertIn(ReasonCode.POST_PATCH_REPRODUCTION_FAILED, result.reason_codes) + + def test_reproduction_before_true_after_false_is_verified(self): + result = verify_patch(valid(reproduces_issue_before_patch=True, reproduces_issue_after_patch=False)) + self.assertEqual(result.status, PatchVerificationStatus.VERIFIED) + self.assertTrue(result.reproduces_issue_before_patch) + self.assertFalse(result.reproduces_issue_after_patch) + + def test_missing_after_reproduction_is_unknown(self): + result = verify_patch(valid(reproduces_issue_after_patch=None)) + self.assertEqual(result.status, PatchVerificationStatus.UNKNOWN) + self.assertIsNone(result.reproduces_issue_after_patch) + + def test_scope_expansion_is_rejected(self): + result = verify_patch(valid(changed_files=("src/a.py", "src/unexpected.py"))) + self.assertEqual(result.status, PatchVerificationStatus.REJECTED) + self.assertTrue(result.scope_changed) + + def test_incomplete_evidence_is_unknown(self): + result = verify_patch(valid(patch_applied=None)) + self.assertEqual(result.status, PatchVerificationStatus.UNKNOWN) + + def test_required_not_executed_is_not_passed(self): + result = verify_patch(valid(regression_results=(TestResult("regression", TestResultStatus.NOT_EXECUTED, True),))) + self.assertEqual(result.status, PatchVerificationStatus.BLOCKED) + self.assertIsNone(result.regression_tests_passed) + + def test_missing_evidence_is_blocked(self): + result = verify_patch(valid(evidence_refs=())) + self.assertEqual(result.status, PatchVerificationStatus.BLOCKED) + self.assertFalse(result.evidence_sufficient) + + def test_fixture_is_classified_as_verification_fixture(self): + path = Path(__file__).parents[1] / "experiments" / "patch-verification-fixture.json" + value = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(value["fixture_type"], "PATCH_VERIFICATION_FIXTURE") + self.assertEqual(len(value["scenarios"]), 6) + + def test_json_uses_unambiguous_reproduction_names(self): + payload = json.loads(verify_patch(valid()).to_json()) + self.assertEqual(payload["reproduces_issue_before_patch"], True) + self.assertEqual(payload["reproduces_issue_after_patch"], False) + self.assertNotIn("reproduction_before", payload) + self.assertNotIn("reproduction_after", payload) + self.assertNotIn("reproduction_passed_after_patch", payload) + self.assertEqual(payload["telemetry"]["reproduces_issue_after_patch"], False) + + def test_test_result_preserves_verifier_execution_identity(self): + result = TestResult( + "regression", + TestResultStatus.PASSED, + True, + "e://test", + ("python", "-m", "pytest", "tests/test_example.py"), + 0, + 1250, + ) + self.assertEqual(result.to_dict()["command"][0], "python") + self.assertEqual(result.to_dict()["exit_code"], 0) + self.assertEqual(result.to_dict()["duration_ms"], 1250) + + def test_test_result_rejects_invalid_execution_metadata(self): + with self.assertRaises(ValueError): + TestResult("regression", TestResultStatus.PASSED, True, command=()) + with self.assertRaises(ValueError): + TestResult("regression", TestResultStatus.PASSED, True, duration_ms=-1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_verifier.py b/tests/test_verifier.py new file mode 100644 index 00000000..2ee4d5c5 --- /dev/null +++ b/tests/test_verifier.py @@ -0,0 +1,113 @@ +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +from arkx.verification import TestResultStatus +from arkx.verifier import CommandVerifier, VerificationEvidenceStore, VerificationErrorKind, run_verification_command +from arkx.outcomes import VerificationState + + +class VerifierTests(unittest.TestCase): + def test_passes_declared_command_and_preserves_observation(self): + with tempfile.TemporaryDirectory() as directory: + observed = run_verification_command( + (sys.executable, "-c", "print('verified')"), + workspace=directory, + test_id="smoke", + evidence_ref="evidence://smoke", + ) + self.assertEqual(observed.result.status, TestResultStatus.PASSED) + self.assertEqual(observed.result.exit_code, 0) + self.assertIn("verified", observed.stdout) + self.assertEqual(observed.result.command[0], sys.executable) + + def test_failure_is_not_success(self): + with tempfile.TemporaryDirectory() as directory: + observed = run_verification_command( + (sys.executable, "-c", "import sys; sys.exit(3)"), + workspace=directory, + test_id="failure", + ) + self.assertEqual(observed.result.status, TestResultStatus.FAILED) + self.assertEqual(observed.result.exit_code, 3) + + def test_timeout_is_explicit(self): + with tempfile.TemporaryDirectory() as directory: + observed = run_verification_command( + (sys.executable, "-c", "import time; time.sleep(1)"), + workspace=directory, + test_id="timeout", + timeout_seconds=0.01, + ) + self.assertEqual(observed.result.status, TestResultStatus.UNKNOWN) + self.assertEqual(observed.error_kind, VerificationErrorKind.TIMEOUT) + + def test_missing_executable_is_explicit(self): + with tempfile.TemporaryDirectory() as directory: + observed = run_verification_command( + ("definitely-not-installed-codepro-verifier",), + workspace=directory, + test_id="missing", + ) + self.assertEqual(observed.result.status, TestResultStatus.UNKNOWN) + self.assertEqual(observed.error_kind, VerificationErrorKind.EXECUTABLE_NOT_FOUND) + + def test_invalid_workspace_and_command_fail_closed(self): + with self.assertRaises(ValueError): + run_verification_command((), workspace=Path.cwd(), test_id="invalid") + with self.assertRaises(ValueError): + run_verification_command((sys.executable,), workspace="missing-workspace", test_id="invalid") + + def test_command_verifier_persists_raw_observation_and_is_idempotent(self): + with tempfile.TemporaryDirectory() as directory: + store = VerificationEvidenceStore(Path(directory) / "evidence") + verifier = CommandVerifier( + authority="local-verifier", + workspace=directory, + command=(sys.executable, "-c", "import time; print('raw-output'); time.sleep(0.02)"), + test_id="smoke", + evidence_store=store, + ) + execution = SimpleNamespace(outcome="COMPLETED", run_id="run-1") + first = verifier.verify(execution) + second = verifier.verify(execution) + self.assertEqual(first.state, VerificationState.PASS) + self.assertEqual(second.state, VerificationState.BLOCKED) + evidence_path = Path(directory) / "evidence" / "run-1" + files = tuple(evidence_path.glob("verification-*.json")) + self.assertEqual(len(files), 1) + self.assertIn("raw-output", files[0].read_text(encoding="utf-8")) + + def test_command_verifier_does_not_accept_failed_command(self): + with tempfile.TemporaryDirectory() as directory: + verifier = CommandVerifier( + authority="local-verifier", + workspace=directory, + command=(sys.executable, "-c", "import sys; sys.exit(2)"), + test_id="failure", + evidence_store=VerificationEvidenceStore(Path(directory) / "evidence"), + ) + result = verifier.verify(SimpleNamespace(outcome="COMPLETED", run_id="run-2")) + self.assertEqual(result.state, VerificationState.FAIL) + + def test_command_verifier_blocks_when_evidence_cannot_be_persisted(self): + with tempfile.TemporaryDirectory() as directory: + class FailingStore: + def persist(self, run_id, observation): + raise OSError("read-only") + + verifier = CommandVerifier( + authority="local-verifier", + workspace=directory, + command=(sys.executable, "-c", "print('verified')"), + test_id="smoke", + evidence_store=FailingStore(), + ) + result = verifier.verify(SimpleNamespace(outcome="COMPLETED", run_id="run-3")) + self.assertEqual(result.state, VerificationState.BLOCKED) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/collect_adapter_preflight.py b/tools/collect_adapter_preflight.py new file mode 100644 index 00000000..f8f01355 --- /dev/null +++ b/tools/collect_adapter_preflight.py @@ -0,0 +1,74 @@ +"""Read-only runtime preflight collector for concrete external boundaries.""" + +from __future__ import annotations + +import importlib.util +import os +from pathlib import Path +import shutil +import subprocess +from typing import Iterable + +from arkx.integration import AdapterIdentity, AdapterPreflight, DependencyObservation, IntegrationKind, assess_preflight + + +def _version(executable: str) -> str | None: + try: + result = subprocess.run([executable, "--version"], capture_output=True, text=True, timeout=10, check=False) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + output = (result.stdout or result.stderr).strip().replace("\x00", " ") + return output or None + + +def _docker_daemon() -> DependencyObservation: + if not shutil.which("docker"): + return DependencyObservation("docker-daemon", False, reason="docker executable missing") + try: + result = subprocess.run(["docker", "info"], capture_output=True, text=True, timeout=15, check=False) + except (OSError, subprocess.SubprocessError) as error: + return DependencyObservation("docker-daemon", None, reason=f"diagnostic error: {type(error).__name__}") + if result.returncode == 0: + return DependencyObservation("docker-daemon", True, _version("docker")) + return DependencyObservation("docker-daemon", False, reason=(result.stderr or result.stdout).strip()[:240]) + + +def collect_preflights() -> tuple[AdapterPreflight, ...]: + minisweagent = importlib.util.find_spec("minisweagent") is not None + docker = shutil.which("docker") + bash = shutil.which("bash") + ollama = shutil.which("ollama") + records: list[AdapterPreflight] = [] + records.append(assess_preflight( + AdapterIdentity(IntegrationKind.PROVIDER, "openrouter", os.getenv("ARKX_MINISWEAGENT_VERSION"), os.getenv("ARKX_OPENROUTER_CONFIG_DIGEST")), + (DependencyObservation("minisweagent", minisweagent, reason=None if minisweagent else "python module unavailable"),), + capabilities=("completion",), + )) + records.append(assess_preflight( + AdapterIdentity(IntegrationKind.SANDBOX, "git-bash", os.getenv("ARKX_GIT_BASH_VERSION"), os.getenv("ARKX_GIT_BASH_CONFIG_DIGEST")), + (DependencyObservation("bash", bool(bash and _version(bash)), _version(bash) if bash else None, None if bash and _version(bash) else "executable unavailable or access denied"),), + capabilities=("command-execution",), + )) + records.append(assess_preflight( + AdapterIdentity(IntegrationKind.EVALUATOR, "docker-swebench", _version("docker") if docker else None, os.getenv("ARKX_DOCKER_CONFIG_DIGEST")), + (_docker_daemon(),), + capabilities=("container-evaluation",), + )) + records.append(assess_preflight( + AdapterIdentity(IntegrationKind.PROVIDER, "ollama", _version("ollama") if ollama else None, os.getenv("ARKX_OLLAMA_CONFIG_DIGEST")), + (DependencyObservation("ollama", ollama is not None, _version(ollama) if ollama else None, None if ollama else "executable unavailable"),), + capabilities=("completion",), + )) + return tuple(records) + + +def main() -> int: + import json + print(json.dumps([record.to_dict() | {"reference": record.reference} for record in collect_preflights()], ensure_ascii=False, sort_keys=True, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/p82_bash_environment.py b/tools/p82_bash_environment.py new file mode 100644 index 00000000..58ec9589 --- /dev/null +++ b/tools/p82_bash_environment.py @@ -0,0 +1,63 @@ +"""Git Bash environment for the P8.2a Windows runner-v3 preflight.""" + +from __future__ import annotations + +import os +import subprocess +from typing import Any + +try: + from minisweagent.environments.local import LocalEnvironment, LocalEnvironmentConfig + MINISWEAGENT_AVAILABLE = True +except ModuleNotFoundError: # dependency boundary; execution remains unavailable + MINISWEAGENT_AVAILABLE = False + + class LocalEnvironmentConfig: # type: ignore[no-redef] + pass + + class LocalEnvironment: # type: ignore[no-redef] + def __init__(self, *args, **kwargs): + raise RuntimeError("minisweagent is required for BashLocalEnvironment execution") + + +class BashLocalEnvironmentConfig(LocalEnvironmentConfig): + bash_executable: str + + +class BashLocalEnvironment(LocalEnvironment): + """Execute mini-SWE-agent bash actions through an explicit Git Bash binary.""" + + def __init__(self, *, config_class: type = BashLocalEnvironmentConfig, **kwargs): + if not MINISWEAGENT_AVAILABLE: + raise RuntimeError("minisweagent is required for BashLocalEnvironment execution") + super().__init__(config_class=config_class, **kwargs) + + def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) -> dict[str, Any]: + command = action.get("command", "") + cwd = cwd or self.config.cwd or os.getcwd() + bash_executable = self.config.bash_executable + try: + result = subprocess.run( + [bash_executable, "-lc", command], + cwd=cwd, + env=os.environ | self.config.env, + text=True, + encoding="utf-8", + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout or self.config.timeout, + check=False, + ) + output = {"output": result.stdout, "returncode": result.returncode, "exception_info": ""} + except Exception as error: + raw_output = getattr(error, "output", None) + raw_output = raw_output.decode("utf-8", errors="replace") if isinstance(raw_output, bytes) else (raw_output or "") + output = { + "output": raw_output, + "returncode": -1, + "exception_info": f"An error occurred while executing the command: {error}", + "extra": {"exception_type": type(error).__name__, "exception": str(error)}, + } + self._check_finished(output) + return output diff --git a/tools/p82_ollama_functional_probe.py b/tools/p82_ollama_functional_probe.py new file mode 100644 index 00000000..5afa97a2 --- /dev/null +++ b/tools/p82_ollama_functional_probe.py @@ -0,0 +1,78 @@ +"""Bounded functional T1 probe for locally served coding models.""" + +from __future__ import annotations + +import argparse +import ast +import json +import time +from typing import Any + +try: + from tools.p82_ollama_probe import probe_model +except ModuleNotFoundError: # direct `python tools/p82_ollama_functional_probe.py` + from p82_ollama_probe import probe_model + + +FUNCTIONAL_PROMPT = ( + "Return only one valid JSON object with keys code and test. " + "code must define validate_value(value): raise ValueError('value is required') " + "when value is None, otherwise return value. test must use pytest.raises " + "to check the None case and assert that validate_value(7) == 7." +) + + +def validate_functional_response(response: str) -> dict[str, Any]: + payload = json.loads(response) + if set(payload) != {"code", "test"}: + raise ValueError("response keys must be exactly code and test") + code = str(payload["code"]) + test = str(payload["test"]) + ast.parse(code) + ast.parse(test) + namespace: dict[str, Any] = {} + exec(compile(code, "", "exec"), namespace) # noqa: S102 - bounded local probe + function = namespace.get("validate_value") + if not callable(function): + raise ValueError("code did not define validate_value") + try: + function(None) + except ValueError as error: + if str(error) != "value is required": + raise ValueError("wrong exception message") from error + else: + raise ValueError("None case did not raise ValueError") + if function(7) != 7: + raise ValueError("non-None case did not preserve value") + if "pytest.raises" not in test: + raise ValueError("test did not use pytest.raises") + return {"code": code, "test": test} + + +def run_functional_probe(model: str, *, timeout_seconds: float = 35.0, num_predict: int = 128) -> dict[str, Any]: + started = time.monotonic() + result = probe_model( + model, + timeout_seconds=timeout_seconds, + prompt=FUNCTIONAL_PROMPT, + num_predict=num_predict, + response_validator=validate_functional_response, + ) + result["prompt_contract"] = FUNCTIONAL_PROMPT + result["functional_status"] = result["status"] + result["elapsed_seconds_total"] = round(time.monotonic() - started, 3) + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("model") + parser.add_argument("--timeout", type=float, default=35.0) + parser.add_argument("--num-predict", type=int, default=128) + args = parser.parse_args() + print(json.dumps(run_functional_probe(args.model, timeout_seconds=args.timeout, num_predict=args.num_predict), sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/p82_ollama_probe.py b/tools/p82_ollama_probe.py new file mode 100644 index 00000000..31131ed0 --- /dev/null +++ b/tools/p82_ollama_probe.py @@ -0,0 +1,77 @@ +"""Bounded, machine-readable Ollama T0 probe for P8.2 candidate qualification.""" + +from __future__ import annotations + +import argparse +import json +import time +from typing import Any, Callable +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +def probe_model( + model: str, + *, + base_url: str = "http://127.0.0.1:11434", + timeout_seconds: float = 35.0, + prompt: str = "Respond with exactly ARKX_T0_OK and nothing else.", + num_predict: int = 32, + response_validator: Callable[[str], dict[str, Any]] | None = None, + requester: Callable[..., Any] = urlopen, +) -> dict[str, Any]: + payload = { + "model": model, + "prompt": prompt, + "stream": False, + "options": {"temperature": 0, "num_predict": num_predict}, + } + request = Request( + f"{base_url.rstrip('/')}/api/generate", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + started = time.monotonic() + result: dict[str, Any] = { + "model": model, + "status": "INDETERMINATE", + "timeout_seconds": timeout_seconds, + "prompt_contract": payload["prompt"], + } + try: + with requester(request, timeout=timeout_seconds) as response: + body = json.loads(response.read().decode("utf-8")) + text = str(body.get("response", "")).strip() + status = "PASS_T0_FORMAT" if body.get("done") and text == "ARKX_T0_OK" else "FORMAT_OR_CONTENT_FAIL" + result.update({"status": status, "response": text, "done": body.get("done"), "eval_count": body.get("eval_count"), "total_duration": body.get("total_duration")}) + if response_validator is not None: + try: + result["validated"] = response_validator(text) + result["status"] = "PASS_FUNCTIONAL" if body.get("done") else "FORMAT_OR_CONTENT_FAIL" + except Exception as error: # noqa: BLE001 - preserve validation boundary + result.update({"status": "FORMAT_OR_CONTENT_FAIL", "validation_error": str(error)}) + except HTTPError as error: + result.update({"status": "RUNTIME_FAILURE", "error_type": "HTTPError", "error": str(error)}) + except URLError as error: + result.update({"status": "RUNTIME_FAILURE", "error_type": "URLError", "error": str(error)}) + except TimeoutError as error: + result.update({"status": "TIMEOUT", "error_type": "TimeoutError", "error": str(error)}) + except Exception as error: # noqa: BLE001 - probe must preserve concrete failure + result.update({"status": "RUNTIME_FAILURE", "error_type": type(error).__name__, "error": str(error)}) + result["elapsed_seconds"] = round(time.monotonic() - started, 3) + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("model") + parser.add_argument("--base-url", default="http://127.0.0.1:11434") + parser.add_argument("--timeout", type=float, default=35.0) + args = parser.parse_args() + print(json.dumps(probe_model(args.model, base_url=args.base_url, timeout_seconds=args.timeout), sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/p82_openrouter_model.py b/tools/p82_openrouter_model.py new file mode 100644 index 00000000..8906a350 --- /dev/null +++ b/tools/p82_openrouter_model.py @@ -0,0 +1,125 @@ +"""Arkx OpenRouter adapter hardening for provider error envelopes.""" + +from __future__ import annotations + +import json +import os +from datetime import datetime, timezone +from pathlib import Path +import time +from typing import Any + +from arkx.integration import provider_error_envelope + +try: + from minisweagent.models.openrouter_model import OpenRouterModel as _OpenRouterModel + MINISWEAGENT_AVAILABLE = True +except ModuleNotFoundError: # dependency boundary; execution remains unavailable + _OpenRouterModel = object + MINISWEAGENT_AVAILABLE = False + + +class ProviderAvailabilityError(RuntimeError): + """A provider returned a retryable availability error envelope.""" + + def __init__(self, error: dict[str, Any]): + self.error = error + code = error.get("code") + message = error.get("message", "") + metadata = error.get("metadata", {}) + self.code = code + self.provider_error_type = metadata.get("error_type") if isinstance(metadata, dict) else None + super().__init__( + f"provider_error code={code} error_type={self.provider_error_type}: {message}" + ) + + def to_error_envelope(self, *, raw_evidence_refs: tuple[str, ...] = ()): + return provider_error_envelope({"error": self.error}, raw_evidence_refs=raw_evidence_refs) + + +class ArkxOpenRouterModel(_OpenRouterModel): + """Keep provider error envelopes out of the success/action parser.""" + + def __init__(self, *args, **kwargs): + if not MINISWEAGENT_AVAILABLE: + raise RuntimeError("minisweagent is required for ArkxOpenRouterModel execution") + super().__init__(*args, **kwargs) + + def _diagnostics_path(self) -> Path | None: + value = os.getenv("ARKX_P82_MODEL_DIAGNOSTICS_PATH") + return Path(value) if value else None + + def _record_diagnostic(self, event: dict[str, Any]) -> None: + path = self._diagnostics_path() + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + payload = {"events": []} + if path.exists(): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + payload = {"events": []} + payload.setdefault("events", []).append(event) + payload["updated_at"] = datetime.now(timezone.utc).isoformat(timespec="milliseconds") + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, ensure_ascii=False, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path) + + def _query(self, messages: list[dict[str, str]], **kwargs): + started = time.monotonic() + event = { + "event": "model_call", + "started_at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), + "model": self.config.model_name, + "call_index": None, + "timeout_origin": "provider_request", + } + path = self._diagnostics_path() + if path and path.exists(): + try: + prior = json.loads(path.read_text(encoding="utf-8")) + event["call_index"] = len(prior.get("events", [])) + 1 + except (OSError, json.JSONDecodeError): + event["call_index"] = 1 + try: + response = super()._query(messages, **kwargs) + except Exception as error: + event.update({ + "finished_at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), + "duration_seconds": round(time.monotonic() - started, 3), + "outcome": "exception", + "exception_type": type(error).__name__, + "exception": str(error), + }) + self._record_diagnostic(event) + raise + event.update({ + "finished_at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), + "duration_seconds": round(time.monotonic() - started, 3), + "outcome": "provider_error_envelope" if isinstance(response, dict) and "error" in response else "response", + }) + if isinstance(response, dict) and isinstance(response.get("error"), dict): + error = response["error"] + event.update({ + "provider_code": error.get("code"), + "provider_error_type": (error.get("metadata") or {}).get("error_type") if isinstance(error.get("metadata"), dict) else None, + "provider_message": error.get("message", ""), + "timeout_origin": "provider_envelope", + }) + self._record_diagnostic(event) + return response + + def _parse_actions(self, response: dict) -> list[dict]: + if isinstance(response, dict) and isinstance(response.get("error"), dict): + self._record_diagnostic({ + "event": "adapter_classification", + "at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), + "outcome": "provider_availability_failure", + "provider_code": response["error"].get("code"), + "provider_error_type": (response["error"].get("metadata") or {}).get("error_type") if isinstance(response["error"].get("metadata"), dict) else None, + }) + raise ProviderAvailabilityError(response["error"]) + if not MINISWEAGENT_AVAILABLE: + raise KeyError("choices") + return super()._parse_actions(response) diff --git a/tools/run_mini_headless.py b/tools/run_mini_headless.py new file mode 100644 index 00000000..574d26d7 --- /dev/null +++ b/tools/run_mini_headless.py @@ -0,0 +1,114 @@ +"""Headless mini-SWE-agent 2.x worker used by the Arkx adapter. + +This file is intentionally tiny and uses the public Python API. It is not +called by unit tests and never evaluates acceptance. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +from pathlib import Path + + +def main() -> int: + payload = json.load(sys.stdin) + diagnostics_path = payload.get("diagnostics_path") + if diagnostics_path: + os.environ["ARKX_P82_MODEL_DIAGNOSTICS_PATH"] = diagnostics_path + os.environ.setdefault("MSWEA_SILENT_STARTUP", "1") + from minisweagent.agents import get_agent + from minisweagent.config import get_config_from_spec + from minisweagent.environments import get_environment + from minisweagent.models import get_model + from minisweagent.utils.serialize import recursive_merge, UNSET + + config = get_config_from_spec("mini.yaml") + config = recursive_merge( + config, + { + "run": {"task": payload["task"] or UNSET}, + "agent": { + "mode": "yolo", + "confirm_exit": False, + "output_path": Path(payload["trajectory"]), + "cost_limit": None if payload["max_cost"] is None else float(payload["max_cost"]), + "wall_time_limit_seconds": None if payload["wall_time_seconds"] is None else int(payload["wall_time_seconds"]), + }, + "model": { + "model_name": payload["model"], + "model_class": payload.get("model_class", UNSET), + **(payload.get("model_configuration") or payload.get("model_kwargs") or {}), + }, + "environment": {"cwd": payload["workspace"]}, + }, + ) + model = get_model(config=config["model"]) + if payload.get("environment") == "git-bash": + from p82_bash_environment import BashLocalEnvironment + + environment_config = config["environment"] | { + "bash_executable": payload["bash_executable"], + } + env = BashLocalEnvironment(**environment_config) + else: + env = get_environment(config["environment"], default_type="local") + retry_limit = int(payload.get("provider_retry_attempts", 0)) + retry_backoff = float(payload.get("provider_retry_backoff_seconds", 0)) + result = None + agent = None + for retry_index in range(retry_limit + 1): + try: + agent = get_agent(model, env, config["agent"], default_type="default") + result = agent.run(payload["task"] or "") + break + except Exception as error: + from p82_openrouter_model import ProviderAvailabilityError + + if not isinstance(error, ProviderAvailabilityError): + raise + if retry_index >= retry_limit: + result = { + "exit_status": "ProviderAvailabilityFailure", + "submission": "", + "failure_category": "PROVIDER_AVAILABILITY_FAILURE", + "provider_error": error.error, + "provider_retry_attempts": retry_index, + } + break + if retry_backoff > 0: + if diagnostics_path: + retry_path = Path(diagnostics_path).with_name("retry-observability.json") + retry_payload = {"events": []} + if retry_path.exists(): + try: + retry_payload = json.loads(retry_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + retry_payload = {"events": []} + retry_payload.setdefault("events", []).append({ + "event": "provider_retry_backoff", + "retry_index": retry_index + 1, + "started_at": time.time(), + "backoff_seconds": retry_backoff, + }) + temporary = retry_path.with_suffix(retry_path.suffix + ".tmp") + temporary.write_text(json.dumps(retry_payload, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(retry_path) + time.sleep(retry_backoff) + model = get_model(config=config["model"]) + env = get_environment(config["environment"], default_type="local") if payload.get("environment") != "git-bash" else BashLocalEnvironment(**environment_config) + output = { + "result": result, + "usage": { + "model_calls": getattr(agent, "n_calls", None), + "cost": getattr(agent, "cost", None), + }, + } + print(json.dumps(output, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/swebench/run_evaluation_lf.py b/tools/swebench/run_evaluation_lf.py new file mode 100644 index 00000000..f82c2518 --- /dev/null +++ b/tools/swebench/run_evaluation_lf.py @@ -0,0 +1,859 @@ +from __future__ import annotations + +import docker +import json +from datetime import datetime, timezone +import os +import platform +import urllib.request +import traceback + +if platform.system() == "Linux": + import resource + +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +from pathlib import Path, PurePosixPath + +from swebench.image_builder.constants import CONTAINER_USER, CONTAINER_WORKDIR +from swebench.harness.constants import ( + APPLY_PATCH_FAIL, + APPLY_PATCH_PASS, + CONTAINER_PATCH_FILE, + LOG_REPORT, + LOG_RUN_METADATA, + LOG_INSTANCE, + LOG_TEST_OUTPUT, + RUN_EVALUATION_LOG_DIR, + START_TEST_OUTPUT, +) +from swebench.harness.docker_utils import ( + cleanup_container, + copy_to_container, + exec_run_with_timeout, +) +import logging +from swebench.harness.grading import get_eval_report +from swebench.harness.reporting import make_run_report +from swebench.harness.modal_eval import ( + run_instances_modal, + validate_modal_credentials, +) +from swebench.types import TestSpec +from swebench.harness.utils import make_test_spec +from swebench.task.repo import asset_path, load_task_repo +from swebench.harness.utils import ( + EvaluationError, + load_swebench_dataset, + get_predictions_from_file, + run_threadpool, + str2bool, +) + +from swebench.logger import setup_logger, close_logger + +GIT_APPLY_CMDS = [ + "git apply --verbose", + "git apply --verbose --3way", + "git apply --verbose --reject", + "patch --batch --forward --fuzz=5 -p1 -i", +] + +DOCKER_CLIENT_TIMEOUT = int(os.environ.get("SWEBENCH_DOCKER_TIMEOUT", "1800")) +DOCKER_CLIENT_POOL_SIZE = int(os.environ.get("SWEBENCH_DOCKER_POOL_SIZE", "128")) + + +def _docker_client() -> docker.DockerClient: + return docker.from_env( + timeout=DOCKER_CLIENT_TIMEOUT, + max_pool_size=DOCKER_CLIENT_POOL_SIZE, + ) + + +def create_container( + test_spec: TestSpec, + client: docker.DockerClient, + run_id: str, + logger: logging.Logger, +): + """ + Creates a container from an instance image for running evaluation. + + Args: + test_spec (TestSpec): Test spec with evaluation details + client (docker.DockerClient): Docker client for creating the container + run_id (str): Run ID identifying process, used for the container name + logger (logging.Logger): Logger to use for logging the creation process + """ + container = None + try: + # Check if the image exists + try: + client.images.get(test_spec.image) + except docker.errors.ImageNotFound: + try: + logger.info("Image not found locally, attempting to pull...") + client.images.pull(test_spec.image) + except docker.errors.ImageNotFound: + raise EvaluationError( + test_spec.instance_id, + f"Image {test_spec.image} not found for {test_spec.instance_id}", + logger, + ) + + logger.info(f"Creating container for {test_spec.instance_id}...") + + container_name = f"sweb.eval.{test_spec.instance_id.lower()}.{run_id}" + # Remove any existing container with this name (handles ghost containers) + try: + old = client.containers.get(container_name) + old.remove(force=True) + logger.info(f"Removed existing container {container_name}") + except docker.errors.NotFound: + pass + except Exception: + pass + try: + container = client.containers.create( + image=test_spec.image, + name=container_name, + user=CONTAINER_USER, + detach=True, + command="tail -f /dev/null", + # Docker's default seccomp profile only permits CLONE_NEWUSER with + # CAP_SYS_ADMIN, which browser sandboxes need (e.g. openlayers karma) + cap_add=["SYS_ADMIN"], + ) + except docker.errors.APIError as e: + if "409" in str(e) or "Conflict" in str(e): + # Ghost container — use a unique suffix + import time + + container_name = f"{container_name}.{int(time.time())}" + logger.info(f"Retrying with unique name: {container_name}") + container = client.containers.create( + image=test_spec.image, + name=container_name, + user=CONTAINER_USER, + detach=True, + command="tail -f /dev/null", + # Docker's default seccomp profile only permits CLONE_NEWUSER with + # CAP_SYS_ADMIN, which browser sandboxes need (e.g. openlayers karma) + cap_add=["SYS_ADMIN"], + ) + else: + raise + logger.info(f"Container for {test_spec.instance_id} created: {container.id}") + return container + except Exception as e: + logger.error(f"Error creating container for {test_spec.instance_id}: {e}") + logger.info(traceback.format_exc()) + cleanup_container(client, container, logger) + raise EvaluationError(test_spec.instance_id, str(e), logger) from e + + +def _resolve_asset_bytes(asset: dict, task_repo: str | None, logger) -> bytes | None: + """Read an asset from the task repo if there is one, else fetch its url.""" + if task_repo is not None: + local = asset_path(task_repo, asset["instance_id"], asset["path"]) + if local.is_file(): + return local.read_bytes() + if not asset.get("url"): + logger.warning(f"No asset for {asset['instance_id']} at {asset['path']}") + return None + try: + with urllib.request.urlopen(asset["url"], timeout=60) as resp: + return resp.read() + except Exception as e: + logger.warning(f"Could not fetch image asset {asset['url']}: {e}") + return None + + +def _stage_image_assets( + container, test_spec, log_dir: Path, logger, task_repo: str | None = None +) -> list[str]: + """Stage a patch's binary assets in the container; return restore commands. + + A text patch cannot carry binary files (e.g. expected.png rendering baselines), + so the dataset lists them in image_assets. They must land in the working tree + *after* the eval script's `rm -f` + `git apply`, so they arrive with the patch + rather than being baked into the image -- test data in the image would be + visible to anything with a shell in it. + """ + declared = test_spec.image_assets or {} + assets = [] + for key in ("test_patch", "patch"): + for entry in declared.get(key) or []: + if entry.get("path"): + assets.append({**entry, "instance_id": test_spec.instance_id}) + if not assets: + return [] + staging = Path(log_dir) / "image_assets" + staging.mkdir(parents=True, exist_ok=True) + container.exec_run("mkdir -p /image_assets", user="root") + restore, from_mirror = [], 0 + for asset in assets: + data = _resolve_asset_bytes(asset, task_repo, logger) + if data is None: + continue + if ( + task_repo is not None + and asset_path(task_repo, asset["instance_id"], asset["path"]).is_file() + ): + from_mirror += 1 + flat = asset["path"].replace("/", "__") + local = staging / flat + local.write_bytes(data) + copy_to_container(container, local, PurePosixPath("/image_assets") / flat) + restore.append( + f"mkdir -p $(dirname {asset['path']}) && cp /image_assets/{flat} {asset['path']}" + ) + if restore: + logger.info( + f"Staged {len(restore)} patch asset(s) for restore after git apply " + f"({from_mirror} from the local mirror, {len(restore) - from_mirror} fetched)" + ) + return restore + + +def _inject_asset_restore(eval_script: str, restore_cmds: list[str]) -> str: + """Insert asset-restore commands just before the test-output start marker.""" + if not restore_cmds: + return eval_script + lines = eval_script.split("\n") + for idx, line in enumerate(lines): + if START_TEST_OUTPUT in line: + return "\n".join(lines[:idx] + restore_cmds + lines[idx:]) + return eval_script + "\n" + "\n".join(restore_cmds) + + +def run_instance( + test_spec: TestSpec, + pred: dict, + client: docker.DockerClient, + run_id: str, + timeout: int | None = None, + rewrite_reports: bool = False, + skip_patch: bool = False, + task_repo: str | None = None, +): + """ + Run a single instance with the given prediction. + + Args: + test_spec (TestSpec): TestSpec instance with pre-built image + pred (dict): Prediction w/ model_name_or_path, model_patch, instance_id + client (docker.DockerClient): Docker client + run_id (str): Run ID + timeout (int): Timeout for running tests + rewrite_reports (bool): True if eval run is just to reformat existing report + skip_patch (bool): True to skip applying model patch (negative test mode) + """ + # Set up logging directory + instance_id = test_spec.instance_id + model_name_or_path = pred.get("model_name_or_path", "None").replace("/", "__") + log_dir = RUN_EVALUATION_LOG_DIR / run_id / model_name_or_path / instance_id + + # Set up report file + report_path = log_dir / LOG_REPORT + if rewrite_reports: + test_output_path = log_dir / LOG_TEST_OUTPUT + if not test_output_path.exists(): + raise ValueError(f"Test output file {test_output_path} does not exist") + report = get_eval_report( + test_spec=test_spec, + prediction=pred, + test_log_path=test_output_path, + include_tests_status=True, + ) + # Write report to report.json + with open(report_path, "w") as f: + f.write(json.dumps(report, indent=4)) + return instance_id, report + if report_path.exists(): + return instance_id, json.loads(report_path.read_text()) + + # Set up logger + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir / LOG_INSTANCE + logger = setup_logger(instance_id, log_file) + + # Run the instance + container = None + try: + # Create container from image + container = create_container(test_spec, client, run_id, logger) + container.start() + logger.info(f"Container for {instance_id} started: {container.id}") + + if not skip_patch: + # Copy model prediction as patch file to container + patch_file = Path(log_dir / "patch.diff") + patch_file.write_text(pred["model_patch"] or "") + logger.info( + f"Intermediate patch for {instance_id} written to {patch_file}, now applying to container..." + ) + copy_to_container( + container, patch_file, PurePosixPath(CONTAINER_PATCH_FILE) + ) + + # Attempt to apply patch to container + applied_patch = False + for attempt, git_apply_cmd in enumerate(GIT_APPLY_CMDS): + if attempt: + # a failed attempt (notably --reject) leaves partial state behind, + # which makes every later command fail; restart from a pristine tree + container.exec_run( + ["/bin/bash", "-c", "git checkout -- . ; git clean -fd"], + workdir=CONTAINER_WORKDIR, + user=CONTAINER_USER, + ) + val = container.exec_run( + f"{git_apply_cmd} {CONTAINER_PATCH_FILE}", + workdir=CONTAINER_WORKDIR, + user=CONTAINER_USER, + ) + if val.exit_code == 0: + logger.info(f"{APPLY_PATCH_PASS}:\n{val.output.decode('utf-8')}") + applied_patch = True + break + else: + logger.info(f"Failed to apply patch to container: {git_apply_cmd}") + if not applied_patch: + # the chain can leave the patch fully applied while each command still exited non-zero + reverse_check = container.exec_run( + f"git apply --check --reverse {CONTAINER_PATCH_FILE}", + workdir=CONTAINER_WORKDIR, + user=CONTAINER_USER, + ) + if reverse_check.exit_code == 0: + logger.info(f"{APPLY_PATCH_PASS}: verified already applied") + applied_patch = True + if not applied_patch: + logger.info(f"{APPLY_PATCH_FAIL}:\n{val.output.decode('utf-8')}") + raise EvaluationError( + instance_id, + f"{APPLY_PATCH_FAIL}:\n{val.output.decode('utf-8')}", + logger, + ) + else: + logger.info(f"Skipping model patch for {instance_id} (--no-patch mode)") + + # Get git diff before running eval script + git_diff_output_before = ( + container.exec_run( + "git -c core.fileMode=false diff", workdir=CONTAINER_WORKDIR + ) + .output.decode("utf-8") + .strip() + ) + logger.info(f"Git diff before:\n{git_diff_output_before}") + + # Materialize multimodal binary assets (e.g. expected.png rendering + # baselines). A text test_patch cannot carry them, so the dataset ships + # them as urls in image_assets; without this the tests run against a + # missing baseline and error out. + restore_cmds = _stage_image_assets( + container, test_spec, log_dir, logger, task_repo + ) + + eval_file = Path(log_dir / "eval.sh") + # The evaluator runs this script inside a Linux container. On Windows, + # Path.write_text() otherwise translates LF to CRLF and bash receives + # carriage returns in every command and argument. + eval_file.write_text( + _inject_asset_restore(test_spec.eval_script, restore_cmds), + newline="\n", + ) + logger.info( + f"Eval script for {instance_id} written to {eval_file}; copying to container..." + ) + copy_to_container(container, eval_file, PurePosixPath("/eval.sh")) + + # Run eval script, write output to logs + test_output, timed_out, total_runtime = exec_run_with_timeout( + container, "/bin/bash /eval.sh", timeout + ) + test_output_path = log_dir / LOG_TEST_OUTPUT + logger.info(f"Test runtime: {total_runtime:_.2f} seconds") + with open(test_output_path, "w") as f: + f.write(test_output) + logger.info(f"Test output for {instance_id} written to {test_output_path}") + if timed_out: + f.write(f"\n\nTimeout error: {timeout} seconds exceeded.") + raise EvaluationError( + instance_id, + f"Test timed out after {timeout} seconds.", + logger, + ) + + # Get git diff after running eval script (ignore permission changes) + git_diff_output_after = ( + container.exec_run( + "git -c core.fileMode=false diff", workdir=CONTAINER_WORKDIR + ) + .output.decode("utf-8") + .strip() + ) + + # Check if git diff changed after running eval script + logger.info(f"Git diff after:\n{git_diff_output_after}") + if git_diff_output_after != git_diff_output_before: + logger.info("Git diff changed after running eval script") + + # Get report from test output + logger.info(f"Grading answer for {instance_id}...") + report = get_eval_report( + test_spec=test_spec, + prediction=pred, + test_log_path=test_output_path, + include_tests_status=True, + ) + logger.info( + f"report: {report}\n" + f"Result for {instance_id}: resolved: {report[instance_id]['resolved']}" + ) + + # Write report to report.json + with open(report_path, "w") as f: + f.write(json.dumps(report, indent=4)) + return instance_id, report + except EvaluationError as e: + error_msg = traceback.format_exc() + logger.info(error_msg) + print(e) + except Exception as e: + error_msg = ( + f"Error in evaluating model for {instance_id}: {e}\n" + f"{traceback.format_exc()}\n" + f"Check ({logger.log_file}) for more information." + ) + logger.error(error_msg) + finally: + # Remove instance container + image, close logger + cleanup_container(client, container, logger) + close_logger(logger) + return + + +def run_instances( + predictions: dict, + instances: list, + max_workers: int, + run_id: str, + timeout: int, + rewrite_reports: bool = False, + skip_patch: bool = False, + task_repo: str | None = None, +): + """ + Run all instances for the given predictions in parallel. + Expects instances to have pre-built images. + + Args: + predictions (dict): Predictions dict generated by the model + instances (list): List of instances with 'image' field + max_workers (int): Maximum number of workers + run_id (str): Run ID + timeout (int): Timeout for running tests + rewrite_reports (bool): True if eval run is just to reformat existing report + """ + client = _docker_client() + test_specs = [make_test_spec(instance) for instance in instances] + + # run instances in parallel + payloads = [] + for test_spec in test_specs: + payloads.append( + ( + test_spec, + predictions[test_spec.instance_id], + client, + run_id, + timeout, + rewrite_reports, + skip_patch, + task_repo, + ) + ) + + # run instances in parallel + print(f"Running {len(instances)} instances...") + run_threadpool(run_instance, payloads, max_workers) + print("All instances run.") + + +def write_run_metadata( + run_id: str, dataset_name: str, split: str, task_repo: str | None +) -> Path: + """Record what this run graded against. + + Re-grading needs the expected tests and the log parser, which live in the + dataset, not in the run's logs. Without this a later `swebench report` has to + be told the dataset again, and gets it wrong silently if told the wrong one. + """ + path = RUN_EVALUATION_LOG_DIR / run_id / LOG_RUN_METADATA + path.parent.mkdir(parents=True, exist_ok=True) + # a re-grade passes no task repo, and overwriting the recorded one with null + # loses the only record of which tests the run was graded against + if task_repo is None and path.is_file(): + task_repo = json.loads(path.read_text()).get("task_repo") + path.write_text( + json.dumps( + { + "dataset": dataset_name, + "split": split, + "task_repo": task_repo, + "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + }, + indent=2, + ) + + "\n" + ) + return path + + +def read_run_metadata(run_id: str) -> dict | None: + """What a previous run graded against, if it recorded it.""" + path = RUN_EVALUATION_LOG_DIR / run_id / LOG_RUN_METADATA + return json.loads(path.read_text()) if path.is_file() else None + + +def load_instances( + dataset_name: str, split: str, instance_ids: list | None, task_repo: str | None +) -> list: + """Instances come from the task repo when one is given, else from the dataset. + + A dataset is already one split; a task repo holds every split at once, so the + split has to be applied here or a run picks up whatever else is in the tree -- + the multimodal repo would evaluate its dev and deprecated tasks alongside test. + + Named ids are honoured from any split, matching `select_tasks`: an instance + being repaired can be run by name while it sits in an unpublished split. + """ + if not task_repo: + return load_swebench_dataset(dataset_name, split, instance_ids) + tasks = load_task_repo(task_repo, instance_ids) + if instance_ids: + return tasks + wanted = [task for task in tasks if task.get("split") == split] + if not wanted: + available = sorted({task["split"] for task in tasks if task.get("split")}) + raise ValueError( + f"{task_repo} has no tasks in split {split!r}. " + f"It has: {' '.join(available) or 'none'}" + ) + return wanted + + +def get_dataset_from_preds( + dataset_name: str, + split: str, + instance_ids: list, + predictions: dict, + run_id: str, + rewrite_reports: bool, + exclude_completed: bool = True, + task_repo: str | None = None, +): + """ + Return only instances that have predictions and are in the dataset. + If instance_ids is provided, only return instances with those IDs. + If exclude_completed is True, only return instances that have not been run yet. + """ + # load dataset + dataset = load_instances(dataset_name, split, None, task_repo) + dataset_ids = {i["instance_id"] for i in dataset} + + if instance_ids: + # check that all instance IDs have predictions + missing_preds = set(instance_ids) - set(predictions.keys()) + if missing_preds: + print( + f"Warning: Missing predictions for {len(missing_preds)} instance IDs." + ) + + # check that all prediction IDs are in the dataset + prediction_ids = set(predictions.keys()) + if prediction_ids - dataset_ids: + raise ValueError( + ( + "Some prediction IDs not found in dataset!" + f"\nMissing IDs:\n{' '.join(prediction_ids - dataset_ids)}" + ) + ) + if instance_ids: + dataset = [i for i in dataset if i["instance_id"] in instance_ids] + + if rewrite_reports: + # we only return instances that have existing test outputs + test_output_ids = set() + for instance in dataset: + if instance["instance_id"] not in predictions: + continue + prediction = predictions[instance["instance_id"]] + test_output_file = ( + RUN_EVALUATION_LOG_DIR + / run_id + / prediction["model_name_or_path"].replace("/", "__") + / prediction["instance_id"] + / "test_output.txt" + ) + if test_output_file.exists(): + test_output_ids.add(instance["instance_id"]) + dataset = [ + i + for i in dataset + if i["instance_id"] in prediction_ids + and i["instance_id"] in test_output_ids + ] + return dataset + + # check which instance IDs have already been run + completed_ids = set() + for instance in dataset: + if instance["instance_id"] not in prediction_ids: + # skip instances without predictions + continue + prediction = predictions[instance["instance_id"]] + report_file = ( + RUN_EVALUATION_LOG_DIR + / run_id + / prediction["model_name_or_path"].replace("/", "__") + / prediction["instance_id"] + / LOG_REPORT + ) + if report_file.exists(): + completed_ids.add(instance["instance_id"]) + + if completed_ids and exclude_completed: + # filter dataset to only instances that have not been run + print(f"{len(completed_ids)} instances already run, skipping...") + dataset = [i for i in dataset if i["instance_id"] not in completed_ids] + + empty_patch_ids = { + k + for k, v in predictions.items() + if v["model_patch"] == "" or v["model_patch"] is None + } + + # filter dataset to only instances with predictions + dataset = [ + i + for i in dataset + if i["instance_id"] in prediction_ids + and i["instance_id"] not in empty_patch_ids + ] + return dataset + + +def _build_before_eval(dataset, dataset_name, split, task_repo, max_workers, client): + """Build this run's images from a task repo instead of trusting the registry. + + Without this a failed build is invisible: the image is pulled instead, so a stale + published image can report a clean pass. Verification is by the image name the + evaluation will actually use, so a naming mismatch cannot pass for a build. + """ + from swebench.image_builder.docker_build import build_instance_images + from swebench.image_builder.image_spec import get_image_specs_from_dataset + from swebench.image_builder.prepare_images import resolve_task_repo + from swebench.task.repo import load_dockerfiles, task_paths + + wanted = {d["instance_id"]: make_test_spec(d).image for d in dataset} + # namespace and tag come from the image the dataset names, so built tags match + sample = next(iter(wanted.values())) + namespace = sample.split("/")[0] if "/" in sample else None + tag = sample.rsplit(":", 1)[1] if ":" in sample.rsplit("/", 1)[-1] else "latest" + + print(f"Building {len(wanted)} image(s) from {task_repo} before evaluating...") + with resolve_task_repo(task_repo) as repo_path: + dockerfiles = load_dockerfiles(repo_path, list(wanted)) + contexts = task_paths(repo_path, list(wanted)) + image_specs = get_image_specs_from_dataset( + dataset, dockerfiles, namespace, tag, contexts + ) + if not image_specs: + print("No Dockerfiles matched these instances; nothing was built.") + else: + build_instance_images( + client=client, + image_specs=image_specs, + force_rebuild=True, + max_workers=max_workers, + ) + + # trust the registry only where a build did not produce the image + missing = [] + for iid, image in wanted.items(): + try: + client.images.get(image) + except docker.errors.ImageNotFound: + missing.append((iid, image)) + if not missing: + print(f"Built {len(wanted)} image(s) locally; none pulled.") + return + print( + f"BUILD FAILED for {len(missing)} instance(s): {', '.join(i for i, _ in sorted(missing))}" + ) + for iid, image in missing: + try: + client.images.pull(image) + print(f" {iid}: build failed; FELL BACK to the published image {image}") + except docker.errors.ImageNotFound: + print(f" {iid}: build failed and no published image exists; it will error") + + +def main( + dataset_name: str, + split: str, + instance_ids: list, + predictions_path: str, + max_workers: int, + open_file_limit: int, + run_id: str, + timeout: int, + rewrite_reports: bool, + modal: bool, + task_repo: str | None = None, +): + """ + Run evaluation harness for the given dataset and predictions. + """ + if dataset_name == "SWE-bench/SWE-bench_Multimodal" and split == "test": + print( + "ℹ️ Running local evaluation for the test split of SWE-bench Multimodal. " + "You may also use sb-cli (https://github.com/swe-bench/sb-cli/) to submit predictions to the hosted evaluation." + ) + + # Modal builds its own images remotely, so a task repo's Dockerfiles would be + # ignored while its tests were used -- the run would report on a tree it never + # built. Refused here, before any work, rather than silently proving the wrong thing. + if modal and task_repo: + raise ValueError( + "--modal cannot build from a task repo: it builds images remotely, so the " + "repo's Dockerfiles would be ignored while its tests were used. Drop " + "--task-repo to run on Modal, or drop --modal to build the repo." + ) + + # set open file limit + assert len(run_id) > 0, "Run ID must be provided" + # load predictions as map of instance_id to prediction + predictions = get_predictions_from_file( + predictions_path, dataset_name, split, task_repo, instance_ids + ) + predictions = {pred["instance_id"]: pred for pred in predictions} + write_run_metadata(run_id, dataset_name, split, task_repo) + + # get dataset from predictions + dataset = get_dataset_from_preds( + dataset_name, + split, + instance_ids, + predictions, + run_id, + rewrite_reports, + task_repo=task_repo, + ) + full_dataset = load_instances(dataset_name, split, instance_ids, task_repo) + + if modal: + # run instances on Modal + if not dataset: + print("No instances to run.") + else: + validate_modal_credentials() + run_instances_modal(predictions, dataset, full_dataset, run_id, timeout) + return + + # run instances locally + if platform.system() == "Linux": + resource.setrlimit(resource.RLIMIT_NOFILE, (open_file_limit, open_file_limit)) + client = _docker_client() + + if not dataset: + print("No instances to run.") + return make_run_report(predictions, full_dataset, run_id, client) + else: + # a re-grade reads existing logs and starts no container, so building the + # images it names would cost hours and change nothing + if task_repo and not rewrite_reports: + _build_before_eval( + dataset, dataset_name, split, task_repo, max_workers, client + ) + # run instances (images assumed to be pre-built) + run_instances( + predictions, + dataset, + max_workers, + run_id, + timeout, + rewrite_reports=rewrite_reports, + task_repo=task_repo, + ) + + # make final report + return make_run_report(predictions, full_dataset, run_id, client) + + +if __name__ == "__main__": + parser = ArgumentParser( + description="Run evaluation harness for the given dataset and predictions.", + formatter_class=ArgumentDefaultsHelpFormatter, + ) + + # Common args + parser.add_argument( + "-d", + "--dataset_name", + default="SWE-bench/SWE-bench_Lite", + type=str, + help="Name of dataset or path to JSON file.", + ) + parser.add_argument( + "-s", "--split", type=str, default="test", help="Split of the dataset" + ) + parser.add_argument( + "-i", + "--instance_ids", + nargs="+", + type=str, + help="Instance IDs to run (space separated)", + ) + parser.add_argument( + "-p", + "--predictions_path", + type=str, + help="Path to predictions file - if 'gold', uses gold predictions", + required=True, + ) + + # Local execution args + parser.add_argument( + "--max_workers", + type=int, + default=4, + help="Maximum number of workers (should be <= 75%% of CPU cores)", + ) + parser.add_argument( + "--open_file_limit", type=int, default=4096, help="Open file limit" + ) + parser.add_argument( + "-t", + "--timeout", + type=int, + default=1_800, + help="Timeout (in seconds) for running tests for each instance", + ) + parser.add_argument( + "-id", "--run_id", type=str, required=True, help="Run ID - identifies the run" + ) + parser.add_argument( + "--rewrite_reports", + type=str2bool, + default=False, + help="Doesn't run new instances, only writes reports for instances with existing test outputs", + ) + # Modal execution args + parser.add_argument("--modal", type=str2bool, default=False, help="Run on Modal") + + args = parser.parse_args() + main(**vars(args))