diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 89ea77d3f8..b661bb2b60 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -502,7 +502,9 @@ jobs: run: | chmod +x scripts/test-integration.sh uv run ./scripts/test-integration.sh - timeout-minutes: 60 + # Linux x64 reached only 78% in 60 minutes in run 39819494530. + # Keep the proven ARM bound; allow x64 setup and full-corpus tail headroom. + timeout-minutes: ${{ matrix.arch == 'x86_64' && 120 || 60 }} - name: Run integration tests (Windows) if: matrix.platform == 'windows' && (github.event_name != 'schedule' || steps.artifact.outputs.available == 'true') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c8197a675..f7ef546f52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -580,6 +580,15 @@ jobs: chmod +x scripts/build-binary.sh uv run ./scripts/build-binary.sh + - name: Test packaged audit workspace identity + timeout-minutes: 5 + env: + APM_BINARY_PATH: ${{ github.workspace }}/dist/apm-linux-x86_64/apm + run: >- + uv run --frozen --extra dev pytest + tests/integration/test_audit_workspace_identity_lifecycle.py + -vv -ra --tb=long --show-capture=all --no-showlocals + # Dogfood the audit-only CI gate we ship and document to users: # - `apm audit --ci` -- the full producer+consumer gate. Runs the # lockfile / content-integrity checks AND the install-replay drift diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 069451f1e5..7e080372ed 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -517,11 +517,20 @@ ledger, and failure/retry contracts when changing those behaviors. 4. Create GitHub Release 5. Publish to PyPI -The macOS Apple Silicon full integration step has a 120-minute limit and keeps -`not live` selection with `-n 4 --dist loadgroup` for HOME-mutating test grouping. -The previous 60-minute limit stopped progressing workers at 77%; the new budget -allows setup and tail headroom without excluding tests. Failures and timeouts -still block release publication. Linux and Intel integration budgets are unchanged. +The Linux x64 and macOS Apple Silicon full integration steps have 120-minute +limits and keep `not live` selection with `-n 4 --dist loadgroup` for HOME-mutating +test grouping. Previous 60-minute limits stopped progressing workers at 78% and +77%, respectively; the budgets allow setup and tail headroom without excluding +tests. Linux ARM keeps 60 minutes, Windows 20, and the focused Intel step 30. +Failures and timeouts still block release publication; a larger budget is not +evidence that a failed assertion is fixed. + +The existing unprivileged `PR Binary Smoke (Linux)` job also runs +`tests/integration/test_audit_workspace_identity_lifecycle.py` against its built +binary with a five-minute limit. The fixture isolates HOME, credentials, cache, +and local Git repositories; captured audit output and full failure tracebacks +remain visible in the job log. This focused witness does not replace the full +release qualification or change required checks. **Daily scheduled release smoke:** - Runs the same promotion validation path once per day. diff --git a/tests/integration/test_audit_workspace_identity_lifecycle.py b/tests/integration/test_audit_workspace_identity_lifecycle.py index 8188b6bc72..1ab4d998b7 100644 --- a/tests/integration/test_audit_workspace_identity_lifecycle.py +++ b/tests/integration/test_audit_workspace_identity_lifecycle.py @@ -126,7 +126,14 @@ def _seed(tmp_path: Path, apm_binary_path: Path, *, dependency_kind: str = "loca timeout=30, check=False, ) - assert auth.returncode == 1 and not auth.stdout + auth_summary = { + "returncode": auth.returncode, + "stdout_length": len(auth.stdout), + "stderr_length": len(auth.stderr), + } + assert auth_summary["returncode"] == 1 and auth_summary["stdout_length"] == 0, ( + auth_summary + ) consumer = packages.create( "consumer-source", dependencies=(dependency,), @@ -168,6 +175,7 @@ def _audit(workspace: _Workspace, root: Path, phase: str) -> tuple[CommandResult cwd=root, env=workspace.env, ) + print(f"{phase}: returncode={result.returncode}\n{result.stdout}\n{result.stderr}") assert_unchanged(before, ArtifactSnapshot.capture(workspace.isolated.root)) payload = json.loads(result.stdout) return result, payload diff --git a/tests/unit/test_platform_contract_workflow.py b/tests/unit/test_platform_contract_workflow.py index b4286ea98f..0a94f50c9e 100644 --- a/tests/unit/test_platform_contract_workflow.py +++ b/tests/unit/test_platform_contract_workflow.py @@ -49,7 +49,10 @@ ("build-and-validate-macos-arm", "Run integration tests"), ) NON_LIVE_UNIX_TIMEOUT_MINUTES = { - ("integration-tests", "Run integration tests (Unix)"): 60, + ( + "integration-tests", + "Run integration tests (Unix)", + ): "${{ matrix.arch == 'x86_64' && 120 || 60 }}", ("build-and-validate-macos-arm", "Run integration tests"): 120, } NON_LIVE_UNIX_PYTEST_ARGS = "-n 4 --dist loadgroup" @@ -133,7 +136,10 @@ def _assert_standalone_integration_timeouts(workflow: dict) -> None: job = workflow_job(workflow, "integration-tests") unix_step = workflow_step(job, "Run integration tests (Unix)") windows_step = workflow_step(job, "Run integration tests (Windows)") - assert unix_step.get("timeout-minutes") == 60 + assert ( + unix_step.get("timeout-minutes") + == NON_LIVE_UNIX_TIMEOUT_MINUTES[("integration-tests", "Run integration tests (Unix)")] + ) assert windows_step.get("timeout-minutes") == 20 @@ -239,14 +245,16 @@ def test_non_live_unix_parallelism_mutation_is_rejected( _assert_non_live_unix_integration_parallelism(workflow) +@pytest.mark.parametrize(("job_id", "step_name"), NON_LIVE_UNIX_INTEGRATION_STEPS) @pytest.mark.parametrize("timeout_minutes", (None, 60)) -def test_arm_missing_or_old_timeout_mutation_is_rejected(timeout_minutes: int | None) -> None: - """ARM must not lose its explicit budget or regress to the exhausted limit.""" +def test_missing_or_old_timeout_mutation_is_rejected( + job_id: str, + step_name: str, + timeout_minutes: int | None, +) -> None: + """Full-corpus nodes cannot lose their budgets or regress to exhausted limits.""" workflow = deepcopy(_workflow()) - step = workflow_step( - workflow_job(workflow, "build-and-validate-macos-arm"), - "Run integration tests", - ) + step = workflow_step(workflow_job(workflow, job_id), step_name) del step["timeout-minutes"] if timeout_minutes is not None: step["timeout-minutes"] = timeout_minutes @@ -266,7 +274,7 @@ def test_non_live_unix_timeout_mutation_is_rejected( """Every non-live Unix release node retains its measured timeout bound.""" workflow = deepcopy(_workflow()) step = workflow_step(workflow_job(workflow, job_id), step_name) - step["timeout-minutes"] = NON_LIVE_UNIX_TIMEOUT_MINUTES[(job_id, step_name)] + 1 + step["timeout-minutes"] = 121 with pytest.raises(AssertionError): _assert_non_live_unix_integration_parallelism(workflow) @@ -396,3 +404,59 @@ def test_macos_echo_replacement_mutation_is_rejected(job_id: str) -> None: with pytest.raises(AssertionError): _assert_macos_startup_steps(workflow) + + +def _assert_linux_audit_witness(workflow: dict) -> None: + """The existing public binary job must execute the complete isolated family.""" + job = workflow_job(workflow, "pr-binary-smoke") + step_name = "Test packaged audit workspace identity" + step = workflow_step(job, step_name) + assert job["runs-on"] == "ubuntu-24.04" + assert job["permissions"] == {"contents": "read"} + assert_unconditional(step, label=step_name) + assert not step.get("continue-on-error", False) + assert step["timeout-minutes"] == 5 + assert effective_env(workflow, job, step) == { + "PYTHON_VERSION": "3.12", + "APM_BINARY_PATH": "${{ github.workspace }}/dist/apm-linux-x86_64/apm", + } + assert workflow_step_index(job, "Build binary") < workflow_step_index(job, step_name) + assert_exact_command( + shell_commands(step), + [ + "uv", + "run", + "--frozen", + "--extra", + "dev", + "pytest", + "tests/integration/test_audit_workspace_identity_lifecycle.py", + "-vv", + "-ra", + "--tb=long", + "--show-capture=all", + "--no-showlocals", + ], + label=step_name, + ) + + +def test_linux_binary_job_runs_isolated_audit_witness() -> None: + """Packaged audit evidence needs no secret-bearing release run.""" + _assert_linux_audit_witness(load_workflow(ROOT / ".github" / "workflows" / "ci.yml")) + + +@pytest.mark.parametrize( + ("key", "value"), + (("if", False), ("continue-on-error", True), ("timeout-minutes", 60), ("run", "echo passed")), +) +def test_linux_audit_witness_bypass_mutation_is_rejected(key: str, value: object) -> None: + """Disabled, unbounded or fabricated audit evidence must fail the contract.""" + workflow = load_workflow(ROOT / ".github" / "workflows" / "ci.yml") + step = workflow_step( + workflow_job(workflow, "pr-binary-smoke"), + "Test packaged audit workspace identity", + ) + step[key] = value + with pytest.raises(AssertionError): + _assert_linux_audit_witness(workflow)