From 479a498010484c8e31bfaeb7feaedc4cb6ec9ebf Mon Sep 17 00:00:00 2001 From: Ioannis Polyzos Date: Sat, 10 Oct 2026 15:58:58 +0100 Subject: [PATCH 1/4] feat: add machine-readable verification summary counts --- aicp/verification.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/aicp/verification.py b/aicp/verification.py index 15420e4..e2aef96 100644 --- a/aicp/verification.py +++ b/aicp/verification.py @@ -45,7 +45,18 @@ def run_checks(project, checks, timeout=120): "stderr_bytes": len(stderr), "passed": status == "completed" and exit_code == 0, }) - return {"schema_version": 1, "passed": all(item["passed"] for item in results), "checks": results} + return { + "schema_version": 1, + "passed": all(item["passed"] for item in results), + "summary": { + "total": len(results), + "passed": sum(item["passed"] for item in results), + "failed": sum(not item["passed"] for item in results), + "timed_out": sum(item["status"] == "timeout" for item in results), + "launch_errors": sum(item["status"] == "launch_error" for item in results), + }, + "checks": results, + } def verify(project, config_path, timeout=120): From 5ca80afd5f1e55246110cf278c2a1a9ccae3291f Mon Sep 17 00:00:00 2001 From: Ioannis Polyzos Date: Sat, 10 Oct 2026 15:59:07 +0100 Subject: [PATCH 2/4] feat: display safe verification failure summaries --- aicp/__main__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aicp/__main__.py b/aicp/__main__.py index adab9d3..1019b9a 100644 --- a/aicp/__main__.py +++ b/aicp/__main__.py @@ -148,6 +148,11 @@ def main(argv=None): json.dump(result, handle, indent=2) handle.write("\n") print(f"Verification {'passed' if result['passed'] else 'failed'}: {args.output}") + summary = result["summary"] + print(f"Checks: {summary['passed']}/{summary['total']} passed, {summary['timed_out']} timed out, {summary['launch_errors']} launch errors") + for index, check in enumerate(result["checks"], 1): + if not check["passed"]: + print(f" Check {index}: {check['status']} (exit={check['exit_code']})") return 0 if result["passed"] else 1 elif args.command == "task": validate_config(args.path) From 36c925fb17b5814bc320acabe74caa46d4c5aa77 Mon Sep 17 00:00:00 2001 From: Ioannis Polyzos Date: Sat, 10 Oct 2026 15:59:13 +0100 Subject: [PATCH 3/4] test: cover pass, failure, timeout and launch summaries --- tests/test_verification_diagnostics.py | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/test_verification_diagnostics.py diff --git a/tests/test_verification_diagnostics.py b/tests/test_verification_diagnostics.py new file mode 100644 index 0000000..3d9ad95 --- /dev/null +++ b/tests/test_verification_diagnostics.py @@ -0,0 +1,35 @@ +"""Verification summaries do not expose captured command output.""" +import sys +import tempfile +from pathlib import Path +import unittest + +from aicp.verification import run_checks + + +class VerificationDiagnosticsTests(unittest.TestCase): + def test_pass_fail_and_timeout_summary(self): + with tempfile.TemporaryDirectory() as tmp: + project = Path(tmp) + checks = [ + [sys.executable, "-c", "print('ok')"], + [sys.executable, "-c", "raise SystemExit(2)"], + [sys.executable, "-c", "import time; time.sleep(3)"], + ] + result = run_checks(project, checks, timeout=1) + self.assertFalse(result["passed"]) + self.assertEqual(result["summary"], { + "total": 3, "passed": 1, "failed": 2, + "timed_out": 1, "launch_errors": 0, + }) + self.assertNotIn("stdout", result["checks"][0]) + self.assertIn("stdout_sha256", result["checks"][0]) + + def test_launch_error_summary(self): + with tempfile.TemporaryDirectory() as tmp: + result = run_checks(Path(tmp), [["/nonexistent-aicp-executable"]]) + self.assertEqual(result["summary"]["launch_errors"], 1) + + +if __name__ == "__main__": + unittest.main() From f58c7cd29a9a9efbda0c5b2ca0bde7eb4a6d9707 Mon Sep 17 00:00:00 2001 From: Ioannis Polyzos Date: Sat, 10 Oct 2026 15:59:19 +0100 Subject: [PATCH 4/4] docs: describe verification summaries and privacy limits --- docs/VERIFICATION.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 9699afc..e662b79 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -20,3 +20,7 @@ python3 -m aicp verify --path /path/to/project --checks /path/to/project/checks. ``` Commands execute sequentially as argv arrays without shell interpolation. Every command is attempted, including after a failure; the overall result passes only if all checks exit zero. Timeouts and launch errors fail closed. The report includes exit codes, elapsed time, and output digests, **not raw output**. Review commands before running them: they have the same host privileges as the invoking user and may change files or access secrets. Subprocess timeout is not a process-tree containment mechanism. Avoid passing secrets as command arguments because argv is recorded in the report. The CLI does not automatically discover or invent test commands. + +## Diagnostic summaries + +Verification evidence includes a `summary` object with total, passed, failed, timed-out, and launch-error counts. The CLI prints the counts and failing check indices/statuses without printing captured stdout or stderr. The JSON still records command arguments and hashes, which may reveal sensitive values: only use reviewed commands without secrets in arguments, and keep reports private when appropriate. Timeouts bound the direct subprocess call but are **not a guarantee that all descendant processes are terminated**. This is trusted host execution, not isolation.