diff --git a/aicp/__main__.py b/aicp/__main__.py index adab9d3..1019b9a 100644 --- a/aicp/__main__.py +++ b/aicp/__main__.py @@ -148,6 +148,11 @@ def main(argv=None): json.dump(result, handle, indent=2) handle.write("\n") print(f"Verification {'passed' if result['passed'] else 'failed'}: {args.output}") + summary = result["summary"] + print(f"Checks: {summary['passed']}/{summary['total']} passed, {summary['timed_out']} timed out, {summary['launch_errors']} launch errors") + for index, check in enumerate(result["checks"], 1): + if not check["passed"]: + print(f" Check {index}: {check['status']} (exit={check['exit_code']})") return 0 if result["passed"] else 1 elif args.command == "task": validate_config(args.path) diff --git a/aicp/verification.py b/aicp/verification.py index 15420e4..e2aef96 100644 --- a/aicp/verification.py +++ b/aicp/verification.py @@ -45,7 +45,18 @@ def run_checks(project, checks, timeout=120): "stderr_bytes": len(stderr), "passed": status == "completed" and exit_code == 0, }) - return {"schema_version": 1, "passed": all(item["passed"] for item in results), "checks": results} + return { + "schema_version": 1, + "passed": all(item["passed"] for item in results), + "summary": { + "total": len(results), + "passed": sum(item["passed"] for item in results), + "failed": sum(not item["passed"] for item in results), + "timed_out": sum(item["status"] == "timeout" for item in results), + "launch_errors": sum(item["status"] == "launch_error" for item in results), + }, + "checks": results, + } def verify(project, config_path, timeout=120): diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 9699afc..e662b79 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -20,3 +20,7 @@ python3 -m aicp verify --path /path/to/project --checks /path/to/project/checks. ``` Commands execute sequentially as argv arrays without shell interpolation. Every command is attempted, including after a failure; the overall result passes only if all checks exit zero. Timeouts and launch errors fail closed. The report includes exit codes, elapsed time, and output digests, **not raw output**. Review commands before running them: they have the same host privileges as the invoking user and may change files or access secrets. Subprocess timeout is not a process-tree containment mechanism. Avoid passing secrets as command arguments because argv is recorded in the report. The CLI does not automatically discover or invent test commands. + +## Diagnostic summaries + +Verification evidence includes a `summary` object with total, passed, failed, timed-out, and launch-error counts. The CLI prints the counts and failing check indices/statuses without printing captured stdout or stderr. The JSON still records command arguments and hashes, which may reveal sensitive values: only use reviewed commands without secrets in arguments, and keep reports private when appropriate. Timeouts bound the direct subprocess call but are **not a guarantee that all descendant processes are terminated**. This is trusted host execution, not isolation. diff --git a/tests/test_verification_diagnostics.py b/tests/test_verification_diagnostics.py new file mode 100644 index 0000000..3d9ad95 --- /dev/null +++ b/tests/test_verification_diagnostics.py @@ -0,0 +1,35 @@ +"""Verification summaries do not expose captured command output.""" +import sys +import tempfile +from pathlib import Path +import unittest + +from aicp.verification import run_checks + + +class VerificationDiagnosticsTests(unittest.TestCase): + def test_pass_fail_and_timeout_summary(self): + with tempfile.TemporaryDirectory() as tmp: + project = Path(tmp) + checks = [ + [sys.executable, "-c", "print('ok')"], + [sys.executable, "-c", "raise SystemExit(2)"], + [sys.executable, "-c", "import time; time.sleep(3)"], + ] + result = run_checks(project, checks, timeout=1) + self.assertFalse(result["passed"]) + self.assertEqual(result["summary"], { + "total": 3, "passed": 1, "failed": 2, + "timed_out": 1, "launch_errors": 0, + }) + self.assertNotIn("stdout", result["checks"][0]) + self.assertIn("stdout_sha256", result["checks"][0]) + + def test_launch_error_summary(self): + with tempfile.TemporaryDirectory() as tmp: + result = run_checks(Path(tmp), [["/nonexistent-aicp-executable"]]) + self.assertEqual(result["summary"]["launch_errors"], 1) + + +if __name__ == "__main__": + unittest.main()