From e57b7061f30774ca007acb2a866899a7d4c530e2 Mon Sep 17 00:00:00 2001 From: Paradox <159635503+i-am-paradox@users.noreply.github.com> Date: Sat, 3 Oct 2026 04:44:23 +0530 Subject: [PATCH 1/2] fix(core): compute repo compliance via calculate_compliance in audit_org In aggregate_org_results, repository compliance was computed using a local status-count heuristic (fail_count == 0 and warn_count == 0 and pass + na == total), which drifted from the shared calculate_compliance standard used across single-repo audits, harnesses, and attestations. This updates aggregate_org_results to use calculate_compliance from darnit.tools.audit when check results are available, determining compliance at the audited level, and retaining fallback for summary-only data. Fixes #515 Assisted-by: Antigravity:eni-gemini-2.5-pro Signed-off-by: Paradox <159635503+i-am-paradox@users.noreply.github.com> --- packages/darnit/src/darnit/tools/audit_org.py | 46 ++++++++---- tests/darnit/test_audit_org.py | 73 +++++++++++++++++++ 2 files changed, 106 insertions(+), 13 deletions(-) diff --git a/packages/darnit/src/darnit/tools/audit_org.py b/packages/darnit/src/darnit/tools/audit_org.py index b363b883..a76a17a3 100644 --- a/packages/darnit/src/darnit/tools/audit_org.py +++ b/packages/darnit/src/darnit/tools/audit_org.py @@ -273,28 +273,48 @@ def aggregate_org_results( }) continue - pass_count = summary.get("PASS", 0) - fail_count = summary.get("FAIL", 0) - warn_count = summary.get("WARN", 0) - total = summary.get("total", 0) + check_results = result.get("results") or [] + compliance: dict[int, bool] | None = None - # A repo is compliant only if all non-N/A controls pass - na_count = summary.get("N/A", 0) - is_compliant = fail_count == 0 and warn_count == 0 and (pass_count + na_count == total) + if check_results: + from darnit.tools.audit import calculate_compliance + + compliance = calculate_compliance(check_results, level) + is_compliant = compliance.get(level, False) + if not summary: + from darnit.tools.audit import summarize_results + + summary = summarize_results(check_results) + else: + # Fallback if check_results is not provided (e.g. legacy summary-only mocks) + pass_count = summary.get("PASS", 0) + fail_count = summary.get("FAIL", 0) + warn_count = summary.get("WARN", 0) + total = summary.get("total", 0) + na_count = summary.get("N/A", 0) + is_compliant = ( + fail_count == 0 + and warn_count == 0 + and (pass_count + na_count == total) + and total > 0 + ) if is_compliant: org_summary["compliant_repos"] += 1 else: org_summary["non_compliant_repos"] += 1 - org_summary["repos"].append({ + repo_entry: dict[str, Any] = { "repo": repo_name, "status": "COMPLIANT" if is_compliant else "NON_COMPLIANT", - "pass": pass_count, - "fail": fail_count, - "warn": warn_count, - "total": total, - }) + "pass": summary.get("PASS", 0), + "fail": summary.get("FAIL", 0), + "warn": summary.get("WARN", 0), + "total": summary.get("total", 0), + } + if compliance is not None: + repo_entry["compliance"] = compliance + org_summary["repos"].append(repo_entry) return org_summary diff --git a/tests/darnit/test_audit_org.py b/tests/darnit/test_audit_org.py index 93b5106a..a25ce322 100644 --- a/tests/darnit/test_audit_org.py +++ b/tests/darnit/test_audit_org.py @@ -277,6 +277,79 @@ def test_warn_counts_as_non_compliant(self): assert summary["non_compliant_repos"] == 1 assert summary["compliant_repos"] == 0 + def test_results_with_warn_error_and_pending_is_non_compliant(self): + """Results with WARN, ERROR, and PENDING controls are non-compliant.""" + results = [ + { + "repo": "troubled", + "status": "OK", + "results": [ + {"id": "OSPS-AC-01.01", "status": "PASS", "level": 1}, + {"id": "OSPS-BR-01.01", "status": "WARN", "level": 1}, + {"id": "OSPS-QA-01.01", "status": "ERROR", "level": 1}, + {"id": "OSPS-DO-01.01", "status": "PENDING_LLM", "level": 1}, + ], + "summary": { + "PASS": 1, + "FAIL": 0, + "WARN": 1, + "N/A": 0, + "ERROR": 1, + "PENDING_LLM": 1, + "total": 4, + }, + }, + ] + summary = aggregate_org_results("org", results, 1) + assert summary["compliant_repos"] == 0 + assert summary["non_compliant_repos"] == 1 + repo = summary["repos"][0] + assert repo["status"] == "NON_COMPLIANT" + assert repo["compliance"] == {1: False} + + def test_results_with_only_pass_and_na_is_compliant(self): + """Results with only PASS and N/A controls are compliant.""" + results = [ + { + "repo": "clean", + "status": "OK", + "results": [ + {"id": "OSPS-AC-01.01", "status": "PASS", "level": 1}, + {"id": "OSPS-AC-02.01", "status": "PASS", "level": 1}, + {"id": "OSPS-BR-01.01", "status": "N/A", "level": 1}, + ], + "summary": {"PASS": 2, "FAIL": 0, "WARN": 0, "N/A": 1, "total": 3}, + }, + ] + summary = aggregate_org_results("org", results, 1) + assert summary["compliant_repos"] == 1 + assert summary["non_compliant_repos"] == 0 + repo = summary["repos"][0] + assert repo["status"] == "COMPLIANT" + assert repo["compliance"] == {1: True} + + def test_multi_level_compliance_calculation(self): + """Multi-level audit correctly calculates per-level compliance.""" + results = [ + { + "repo": "partially-compliant", + "status": "OK", + "results": [ + {"id": "L1-01", "status": "PASS", "level": 1}, + {"id": "L2-01", "status": "FAIL", "level": 2}, + ], + }, + ] + summary_l1 = aggregate_org_results("org", results, 1) + assert summary_l1["compliant_repos"] == 1 + assert summary_l1["repos"][0]["status"] == "COMPLIANT" + + summary_l2 = aggregate_org_results("org", results, 2) + assert summary_l2["compliant_repos"] == 0 + assert summary_l2["non_compliant_repos"] == 1 + assert summary_l2["repos"][0]["status"] == "NON_COMPLIANT" + assert summary_l2["repos"][0]["compliance"] == {1: True, 2: False} + class TestFormatOrgResults: """Tests for format_org_results_markdown and format_org_results_json.""" From 7a73c29ba9f5cc6fe4b16fe4deab2fa5b8242713 Mon Sep 17 00:00:00 2001 From: Paradox <159635503+i-am-paradox@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:31:57 +0530 Subject: [PATCH 2/2] fix(core): require all levels up to audited level to pass and update test mocks Address review feedback on #537: - Require all levels from 1 up to audited level to pass for repository compliance, matching attestation level_achieved behavior. - Drop legacy summary-only fallback in aggregate_org_results and update test mocks with check results. - Use PENDING instead of PENDING_LLM in test cases. - Add test verifying that level 2 audit requires level 1 to pass. Signed-off-by: Paradox <159635503+i-am-paradox@users.noreply.github.com> --- packages/darnit/src/darnit/tools/audit_org.py | 33 ++------- tests/darnit/test_audit_org.py | 74 +++++++++++++++++-- 2 files changed, 74 insertions(+), 33 deletions(-) diff --git a/packages/darnit/src/darnit/tools/audit_org.py b/packages/darnit/src/darnit/tools/audit_org.py index a76a17a3..e3d54f9b 100644 --- a/packages/darnit/src/darnit/tools/audit_org.py +++ b/packages/darnit/src/darnit/tools/audit_org.py @@ -274,30 +274,14 @@ def aggregate_org_results( continue check_results = result.get("results") or [] - compliance: dict[int, bool] | None = None + from darnit.tools.audit import calculate_compliance, summarize_results - if check_results: - from darnit.tools.audit import calculate_compliance - - compliance = calculate_compliance(check_results, level) - is_compliant = compliance.get(level, False) - if not summary: - from darnit.tools.audit import summarize_results - - summary = summarize_results(check_results) - else: - # Fallback if check_results is not provided (e.g. legacy summary-only mocks) - pass_count = summary.get("PASS", 0) - fail_count = summary.get("FAIL", 0) - warn_count = summary.get("WARN", 0) - total = summary.get("total", 0) - na_count = summary.get("N/A", 0) - is_compliant = ( - fail_count == 0 - and warn_count == 0 - and (pass_count + na_count == total) - and total > 0 - ) + compliance = calculate_compliance(check_results, level) + is_compliant = bool(compliance) and all( + compliance.get(lvl, False) for lvl in range(1, level + 1) + ) + if not summary: + summary = summarize_results(check_results) if is_compliant: org_summary["compliant_repos"] += 1 @@ -311,9 +295,8 @@ def aggregate_org_results( "fail": summary.get("FAIL", 0), "warn": summary.get("WARN", 0), "total": summary.get("total", 0), + "compliance": compliance, } - if compliance is not None: - repo_entry["compliance"] = compliance org_summary["repos"].append(repo_entry) return org_summary diff --git a/tests/darnit/test_audit_org.py b/tests/darnit/test_audit_org.py index a25ce322..4f1eccf7 100644 --- a/tests/darnit/test_audit_org.py +++ b/tests/darnit/test_audit_org.py @@ -206,13 +206,17 @@ def test_all_pass(self): { "repo": "a", "status": "OK", - "results": [], + "results": [ + {"id": f"C-{i}", "status": "PASS", "level": 1} for i in range(5) + ], "summary": {"PASS": 5, "FAIL": 0, "WARN": 0, "N/A": 0, "total": 5}, }, { "repo": "b", "status": "OK", - "results": [], + "results": [ + {"id": f"C-{i}", "status": "PASS", "level": 1} for i in range(5) + ], "summary": {"PASS": 5, "FAIL": 0, "WARN": 0, "N/A": 0, "total": 5}, }, ] @@ -227,13 +231,21 @@ def test_mixed_results(self): { "repo": "good", "status": "OK", - "results": [], + "results": [ + {"id": f"C-{i}", "status": "PASS", "level": 1} for i in range(5) + ], "summary": {"PASS": 5, "FAIL": 0, "WARN": 0, "N/A": 0, "total": 5}, }, { "repo": "bad", "status": "OK", - "results": [], + "results": [ + {"id": "C-1", "status": "PASS", "level": 1}, + {"id": "C-2", "status": "PASS", "level": 1}, + {"id": "C-3", "status": "PASS", "level": 1}, + {"id": "C-4", "status": "FAIL", "level": 1}, + {"id": "C-5", "status": "FAIL", "level": 1}, + ], "summary": {"PASS": 3, "FAIL": 2, "WARN": 0, "N/A": 0, "total": 5}, }, ] @@ -247,7 +259,9 @@ def test_error_repos(self): { "repo": "good", "status": "OK", - "results": [], + "results": [ + {"id": f"C-{i}", "status": "PASS", "level": 1} for i in range(5) + ], "summary": {"PASS": 5, "FAIL": 0, "WARN": 0, "N/A": 0, "total": 5}, }, { @@ -269,7 +283,13 @@ def test_warn_counts_as_non_compliant(self): { "repo": "warned", "status": "OK", - "results": [], + "results": [ + {"id": "C-1", "status": "PASS", "level": 1}, + {"id": "C-2", "status": "PASS", "level": 1}, + {"id": "C-3", "status": "PASS", "level": 1}, + {"id": "C-4", "status": "PASS", "level": 1}, + {"id": "C-5", "status": "WARN", "level": 1}, + ], "summary": {"PASS": 4, "FAIL": 0, "WARN": 1, "N/A": 0, "total": 5}, }, ] @@ -287,7 +307,7 @@ def test_results_with_warn_error_and_pending_is_non_compliant(self): {"id": "OSPS-AC-01.01", "status": "PASS", "level": 1}, {"id": "OSPS-BR-01.01", "status": "WARN", "level": 1}, {"id": "OSPS-QA-01.01", "status": "ERROR", "level": 1}, - {"id": "OSPS-DO-01.01", "status": "PENDING_LLM", "level": 1}, + {"id": "OSPS-DO-01.01", "status": "PENDING", "level": 1}, ], "summary": { "PASS": 1, @@ -295,7 +315,7 @@ def test_results_with_warn_error_and_pending_is_non_compliant(self): "WARN": 1, "N/A": 0, "ERROR": 1, - "PENDING_LLM": 1, + "PENDING": 1, "total": 4, }, }, @@ -350,6 +370,44 @@ def test_multi_level_compliance_calculation(self): assert summary_l2["repos"][0]["status"] == "NON_COMPLIANT" assert summary_l2["repos"][0]["compliance"] == {1: True, 2: False} + def test_level_2_audit_requires_level_1_to_pass(self): + """A repo with L1 FAIL and L2 PASS is non-compliant when audited at level 2.""" + results = [ + { + "repo": "l1-fail-l2-pass", + "status": "OK", + "results": [ + {"id": "L1-01", "status": "FAIL", "level": 1}, + {"id": "L2-01", "status": "PASS", "level": 2}, + ], + }, + ] + summary = aggregate_org_results("org", results, 2) + assert summary["compliant_repos"] == 0 + assert summary["non_compliant_repos"] == 1 + repo = summary["repos"][0] + assert repo["status"] == "NON_COMPLIANT" + assert repo["compliance"] == {1: False, 2: True} + + def test_level_2_audit_passes_when_all_levels_pass(self): + """A repo with L1 PASS and L2 PASS is compliant when audited at level 2.""" + results = [ + { + "repo": "fully-compliant", + "status": "OK", + "results": [ + {"id": "L1-01", "status": "PASS", "level": 1}, + {"id": "L2-01", "status": "PASS", "level": 2}, + ], + }, + ] + summary = aggregate_org_results("org", results, 2) + assert summary["compliant_repos"] == 1 + assert summary["non_compliant_repos"] == 0 + repo = summary["repos"][0] + assert repo["status"] == "COMPLIANT" + assert repo["compliance"] == {1: True, 2: True} + class TestFormatOrgResults: """Tests for format_org_results_markdown and format_org_results_json."""