Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/build-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,9 @@ jobs:
run: |
chmod +x scripts/test-integration.sh
uv run ./scripts/test-integration.sh
timeout-minutes: 60
# Linux x64 reached only 78% in 60 minutes in run 39819494530.
# Keep the proven ARM bound; allow x64 setup and full-corpus tail headroom.
timeout-minutes: ${{ matrix.arch == 'x86_64' && 120 || 60 }}

- name: Run integration tests (Windows)
if: matrix.platform == 'windows' && (github.event_name != 'schedule' || steps.artifact.outputs.available == 'true')
Expand Down
9 changes: 9 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -580,6 +580,15 @@ jobs:
chmod +x scripts/build-binary.sh
uv run ./scripts/build-binary.sh

- name: Test packaged audit workspace identity
timeout-minutes: 5
env:
APM_BINARY_PATH: ${{ github.workspace }}/dist/apm-linux-x86_64/apm
run: >-
uv run --frozen --extra dev pytest
tests/integration/test_audit_workspace_identity_lifecycle.py
-vv -ra --tb=long --show-capture=all --no-showlocals

# Dogfood the audit-only CI gate we ship and document to users:
# - `apm audit --ci` -- the full producer+consumer gate. Runs the
# lockfile / content-integrity checks AND the install-replay drift
Expand Down
19 changes: 14 additions & 5 deletions docs/src/content/docs/contributing/integration-testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -517,11 +517,20 @@ ledger, and failure/retry contracts when changing those behaviors.
4. Create GitHub Release
5. Publish to PyPI

The macOS Apple Silicon full integration step has a 120-minute limit and keeps
`not live` selection with `-n 4 --dist loadgroup` for HOME-mutating test grouping.
The previous 60-minute limit stopped progressing workers at 77%; the new budget
allows setup and tail headroom without excluding tests. Failures and timeouts
still block release publication. Linux and Intel integration budgets are unchanged.
The Linux x64 and macOS Apple Silicon full integration steps have 120-minute
limits and keep `not live` selection with `-n 4 --dist loadgroup` for HOME-mutating
test grouping. Previous 60-minute limits stopped progressing workers at 78% and
77%, respectively; the budgets allow setup and tail headroom without excluding
tests. Linux ARM keeps 60 minutes, Windows 20, and the focused Intel step 30.
Failures and timeouts still block release publication; a larger budget is not
evidence that a failed assertion is fixed.

The existing unprivileged `PR Binary Smoke (Linux)` job also runs
`tests/integration/test_audit_workspace_identity_lifecycle.py` against its built
binary with a five-minute limit. The fixture isolates HOME, credentials, cache,
and local Git repositories; captured audit output and full failure tracebacks
remain visible in the job log. This focused witness does not replace the full
release qualification or change required checks.

**Daily scheduled release smoke:**
- Runs the same promotion validation path once per day.
Expand Down
10 changes: 9 additions & 1 deletion tests/integration/test_audit_workspace_identity_lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,14 @@ def _seed(tmp_path: Path, apm_binary_path: Path, *, dependency_kind: str = "loca
timeout=30,
check=False,
)
assert auth.returncode == 1 and not auth.stdout
auth_summary = {
"returncode": auth.returncode,
"stdout_length": len(auth.stdout),
"stderr_length": len(auth.stderr),
}
assert auth_summary["returncode"] == 1 and auth_summary["stdout_length"] == 0, (
auth_summary
)
consumer = packages.create(
"consumer-source",
dependencies=(dependency,),
Expand Down Expand Up @@ -168,6 +175,7 @@ def _audit(workspace: _Workspace, root: Path, phase: str) -> tuple[CommandResult
cwd=root,
env=workspace.env,
)
print(f"{phase}: returncode={result.returncode}\n{result.stdout}\n{result.stderr}")
assert_unchanged(before, ArtifactSnapshot.capture(workspace.isolated.root))
payload = json.loads(result.stdout)
return result, payload
Expand Down
82 changes: 73 additions & 9 deletions tests/unit/test_platform_contract_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,10 @@
("build-and-validate-macos-arm", "Run integration tests"),
)
NON_LIVE_UNIX_TIMEOUT_MINUTES = {
("integration-tests", "Run integration tests (Unix)"): 60,
(
"integration-tests",
"Run integration tests (Unix)",
): "${{ matrix.arch == 'x86_64' && 120 || 60 }}",
("build-and-validate-macos-arm", "Run integration tests"): 120,
}
NON_LIVE_UNIX_PYTEST_ARGS = "-n 4 --dist loadgroup"
Expand Down Expand Up @@ -133,7 +136,10 @@ def _assert_standalone_integration_timeouts(workflow: dict) -> None:
job = workflow_job(workflow, "integration-tests")
unix_step = workflow_step(job, "Run integration tests (Unix)")
windows_step = workflow_step(job, "Run integration tests (Windows)")
assert unix_step.get("timeout-minutes") == 60
assert (
unix_step.get("timeout-minutes")
== NON_LIVE_UNIX_TIMEOUT_MINUTES[("integration-tests", "Run integration tests (Unix)")]
)
assert windows_step.get("timeout-minutes") == 20


Expand Down Expand Up @@ -239,14 +245,16 @@ def test_non_live_unix_parallelism_mutation_is_rejected(
_assert_non_live_unix_integration_parallelism(workflow)


@pytest.mark.parametrize(("job_id", "step_name"), NON_LIVE_UNIX_INTEGRATION_STEPS)
@pytest.mark.parametrize("timeout_minutes", (None, 60))
def test_arm_missing_or_old_timeout_mutation_is_rejected(timeout_minutes: int | None) -> None:
"""ARM must not lose its explicit budget or regress to the exhausted limit."""
def test_missing_or_old_timeout_mutation_is_rejected(
job_id: str,
step_name: str,
timeout_minutes: int | None,
) -> None:
"""Full-corpus nodes cannot lose their budgets or regress to exhausted limits."""
workflow = deepcopy(_workflow())
step = workflow_step(
workflow_job(workflow, "build-and-validate-macos-arm"),
"Run integration tests",
)
step = workflow_step(workflow_job(workflow, job_id), step_name)
del step["timeout-minutes"]
if timeout_minutes is not None:
step["timeout-minutes"] = timeout_minutes
Expand All @@ -266,7 +274,7 @@ def test_non_live_unix_timeout_mutation_is_rejected(
"""Every non-live Unix release node retains its measured timeout bound."""
workflow = deepcopy(_workflow())
step = workflow_step(workflow_job(workflow, job_id), step_name)
step["timeout-minutes"] = NON_LIVE_UNIX_TIMEOUT_MINUTES[(job_id, step_name)] + 1
step["timeout-minutes"] = 121

with pytest.raises(AssertionError):
_assert_non_live_unix_integration_parallelism(workflow)
Expand Down Expand Up @@ -396,3 +404,59 @@ def test_macos_echo_replacement_mutation_is_rejected(job_id: str) -> None:

with pytest.raises(AssertionError):
_assert_macos_startup_steps(workflow)


def _assert_linux_audit_witness(workflow: dict) -> None:
"""The existing public binary job must execute the complete isolated family."""
job = workflow_job(workflow, "pr-binary-smoke")
step_name = "Test packaged audit workspace identity"
step = workflow_step(job, step_name)
assert job["runs-on"] == "ubuntu-24.04"
assert job["permissions"] == {"contents": "read"}
assert_unconditional(step, label=step_name)
assert not step.get("continue-on-error", False)
assert step["timeout-minutes"] == 5
assert effective_env(workflow, job, step) == {
"PYTHON_VERSION": "3.12",
"APM_BINARY_PATH": "${{ github.workspace }}/dist/apm-linux-x86_64/apm",
}
assert workflow_step_index(job, "Build binary") < workflow_step_index(job, step_name)
assert_exact_command(
shell_commands(step),
[
"uv",
"run",
"--frozen",
"--extra",
"dev",
"pytest",
"tests/integration/test_audit_workspace_identity_lifecycle.py",
"-vv",
"-ra",
"--tb=long",
"--show-capture=all",
"--no-showlocals",
],
label=step_name,
)


def test_linux_binary_job_runs_isolated_audit_witness() -> None:
"""Packaged audit evidence needs no secret-bearing release run."""
_assert_linux_audit_witness(load_workflow(ROOT / ".github" / "workflows" / "ci.yml"))


@pytest.mark.parametrize(
("key", "value"),
(("if", False), ("continue-on-error", True), ("timeout-minutes", 60), ("run", "echo passed")),
)
def test_linux_audit_witness_bypass_mutation_is_rejected(key: str, value: object) -> None:
"""Disabled, unbounded or fabricated audit evidence must fail the contract."""
workflow = load_workflow(ROOT / ".github" / "workflows" / "ci.yml")
step = workflow_step(
workflow_job(workflow, "pr-binary-smoke"),
"Test packaged audit workspace identity",
)
step[key] = value
with pytest.raises(AssertionError):
_assert_linux_audit_witness(workflow)
Loading