diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index b6f160c..aab9b2d 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -28,3 +28,30 @@ jobs: run: python3 scripts/run_benchmark.py evaluations/benchmarks/stdlib-smoke.json --cwd . --output /tmp/playbook-smoke-evidence.json - name: Verify fixture artifact assertions run: python3 scripts/run_fixture.py evaluations/benchmarks/copy-task.json --fixture evaluations/benchmarks/fixtures/copy-task --output /tmp/playbook-copy-evidence.json + + compatibility: + name: Python ${{ matrix.python }} on ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu-latest, python: "3.10"} + - {os: ubuntu-latest, python: "3.12"} + - {os: ubuntu-latest, python: "3.13"} + - {os: macos-latest, python: "3.12"} + runs-on: ${{ matrix.os }} + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install package and validation dependencies + run: | + python -m pip install -e . + python -m pip install -r requirements-dev.txt + - name: Validate installed command + run: aicp --version + - name: Run regression tests + run: python -m unittest discover -s tests -v diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md new file mode 100644 index 0000000..7b3438e --- /dev/null +++ b/docs/COMPATIBILITY.md @@ -0,0 +1,17 @@ +# Compatibility and integration testing + +The GitHub Actions [Validate workflow](../.github/workflows/validate.yml) runs the standard validation job and a compatibility matrix for Python 3.10, 3.12, and 3.13 on Linux, plus Python 3.12 on macOS. Each matrix job installs the package, exercises the installed CLI, and runs unit tests. This matrix is a compatibility sample, not a guarantee for every Python version and operating system. + +## Agent integration tests + +Unit tests for `aicp run` mock the agent executable. They do **not** validate real Codex, Claude, or Gemini CLI behavior. Real agent integration testing is intentionally **manual and opt-in**, because it requires vendor installations, authentication, a disposable host, and potentially network access. + +Before attempting a real run: + +1. Provision an isolated, disposable machine with no personal or organization credentials beyond the minimum vendor-specific test account. +2. Review the agent command and the task and checks files; use synthetic data only. +3. Explicitly approve host execution and run `aicp run` as documented in [CLI](CLI.md). +4. Review modified files, command hashes, exit statuses, and test evidence; do not treat the JSON report as scope enforcement or proof of isolation. +5. Delete the disposable environment and revoke temporary credentials. + +The project does not execute live vendor agents in standard CI. Do not add real API keys or private project data to workflow fixtures. diff --git a/docs/README.md b/docs/README.md index 2508837..8d65d49 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,7 @@ Start with [Getting started](GETTING_STARTED.md) for quick setup, the [practical - [Non-destructive adoption](ADOPTION.md) - [Generated tool entrypoints](TOOL_ENTRYPOINTS.md) - [Local verification](VERIFICATION.md) +- [Platform compatibility and agent integration](COMPATIBILITY.md) ## Engineering guides