fix(audit): evaluate only the audited framework's controls (#442)

CHANGELOG.md

@@ -438,6 +438,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- An audit evaluates only its own framework's controls. A long-lived process
+  such as the MCP server used to evaluate every control an earlier audit had
+  registered, so auditing `reproducibility` after `openssf-baseline` returned
+  71 controls instead of 5 (#442).
+
 - The `darnit-hello` and `darnit-example` templates and the documentation
   use the registered `file_exists` step type instead of `file_must_exist`
   (#501).

docs/architecture/framework-design.md

@@ -1397,6 +1397,13 @@ The `packages/darnit/src/darnit/` source tree SHALL NOT contain hardcoded contro
 - **THEN** zero `register_control()` calls SHALL execute as module-level side effects
 - **AND** the global registry SHALL contain zero controls until TOML loading occurs
 
+The global registry is process-wide and keyed by control id, so it holds every control any earlier audit in the process registered. An audit therefore takes its controls from its own framework definition (including controls composed from other frameworks) plus operator custom controls, never from the registry. The registry is used only when no framework definition can be resolved.
+
+#### Scenario: Two frameworks audited in one process (#442)
+- **WHEN** a long-lived process (such as the MCP server) audits framework A and then framework B
+- **THEN** the audit of B SHALL evaluate exactly the controls it evaluates in a fresh process
+- **AND** no control defined only by A SHALL appear in B's results
+
 #### Scenario: Searching framework source for control IDs
 - **WHEN** the `packages/darnit/src/darnit/` source tree is searched for patterns like `OSPS-AC-03.01`
 - **THEN** no hardcoded OSPS control ID patterns SHALL exist in executable code

packages/darnit/src/darnit/tools/audit.py

@@ -834,13 +834,23 @@ def run_sieve_audit(
         # Register controls from TOML framework definition (primary source of truth)
         _register_toml_controls(resolved_fw)
 
-        registry = get_control_registry()
-        all_controls = []
-        for lvl in range(1, level + 1):
-            all_controls.extend(registry.get_specs_by_level(lvl))
+    framework = _load_framework(resolved_fw)
+
+    if controls is None:
+        if framework is not None:
+            # The audit's controls come from its own framework definition, not
+            # from the process-wide registry, which also holds every control an
+            # earlier audit in this process registered (#442).
+            from darnit.config import load_controls_from_framework
+
+            all_controls = [c for c in load_controls_from_framework(framework) if (c.level or 0) <= level]
+        else:
+            registry = get_control_registry()
+            all_controls = []
+            for lvl in range(1, level + 1):
+                all_controls.extend(registry.get_specs_by_level(lvl))
         all_controls = _apply_operator_controls(all_controls, resolved_fw, operator)
 
-    framework = _load_framework(resolved_fw)
     known_control_ids = _known_control_ids(all_controls, framework, operator)
 
     # Filter by level (applies to both provided and loaded controls)

tests/darnit/tools/test_audit_control_scope.py

@@ -0,0 +1,57 @@
+"""An audit evaluates only its own framework's controls (#442).
+
+Controls are registered in a process-wide registry. A long-lived process (the
+MCP server) audits several frameworks in turn, and each audit must see the
+same control set it would see in a fresh process.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from darnit.core.utils import RecordedGhApi, set_gh_api_responder
+from darnit.tools.audit import run_sieve_audit
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def repo(tmp_path: Path) -> Path:
+    path = tmp_path / "repo"
+    path.mkdir()
+    subprocess.run(["git", "init", "-q", str(path)], check=True)
+    return path
+
+
+@pytest.fixture(autouse=True)
+def offline_platform():
+    previous = set_gh_api_responder(RecordedGhApi({}))
+    yield
+    set_gh_api_responder(previous)
+
+
+def _ids(repo: Path, framework: str) -> list[str]:
+    results, _ = run_sieve_audit(
+        owner="o",
+        repo="r",
+        local_path=str(repo),
+        default_branch="main",
+        level=3,
+        stop_on_llm=True,
+        framework_name=framework,
+    )
+    return sorted(r["id"] for r in results)
+
+
+def test_each_audit_sees_only_its_framework(repo: Path) -> None:
+    reproducibility_cold = _ids(repo, "reproducibility")
+    baseline = _ids(repo, "openssf-baseline")
+    reproducibility_warm = _ids(repo, "reproducibility")
+
+    assert reproducibility_cold and all(i.startswith("RE-") for i in reproducibility_cold)
+    assert reproducibility_warm == reproducibility_cold
+    assert not [i for i in baseline if i.startswith("RE-")]
+    assert any(i.startswith("OSPS-") for i in baseline)