From d8a4d9b8b746b66f0417e1012b321ab98c9901e6 Mon Sep 17 00:00:00 2001 From: Michael Lieberman Date: Sun, 4 Oct 2026 20:21:27 -0400 Subject: [PATCH 1/5] docs: specify removal of repository .baseline.toml framework-design 14.4 now says darnit reads nothing from a repository's .baseline.toml (no claims, no extends, no settings) and records one notice pointing at `darnit config migrate`; Appendix C records the removal. The 040 addendum lists what is removed and what stays (config migrate). User and plugin docs move .baseline.toml examples to operator configuration or .project/darnit.yaml, and the root example.baseline.toml is deleted. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Michael Lieberman --- ARCHITECTURE.md | 22 +-- CLAUDE.md | 2 +- README.md | 4 +- TODO.md | 8 +- docs/DECISION_FLOWS.md | 14 +- docs/MIGRATION_GUIDE.md | 12 +- docs/SECURITY_GUIDE.md | 4 +- docs/WORKFLOW.md | 4 +- docs/architecture/audit-pipeline.md | 6 +- docs/architecture/context-collection.md | 2 +- docs/architecture/framework-design.md | 37 +++-- docs/getting-started/framework-development.md | 4 +- docs/packaging-plugins.md | 2 +- docs/plugin-authoring/stores.md | 24 ++-- docs/plugin-discovery-design.md | 39 +---- example.baseline.toml | 136 ------------------ packages/darnit-plugins/README.md | 12 -- packages/darnit-testchecks/README.md | 26 ++-- .../deprecation-completed.md | 65 +++++++++ 19 files changed, 167 insertions(+), 256 deletions(-) delete mode 100644 example.baseline.toml create mode 100644 specs/040-operator-config-trust/deprecation-completed.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 449c925e..1095bc21 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -42,8 +42,8 @@ Here's the end-to-end flow that an AI assistant (or human) goes through when aud Load existing project DETERMINISTIC things actually live context from .project/ │ file exists? API call? (e.g., "my security ▼ docs are at - Merge user overrides PATTERN docs/security.txt, - from .baseline.toml │ regex match? heuristics? not SECURITY.md") + Apply operator config PATTERN docs/security.txt, + (outside the repository) │ regex match? heuristics? not SECURITY.md") ▼ LLM b) Framework needs user │ ask calling AI to judge to confirm values it @@ -423,19 +423,19 @@ Three configuration layers, merged at runtime: │ Defines: controls, passes, templates, context prompts│ │ Owner: implementation author │ ├──────────────────────────────────────────────────────┤ -│ Layer 2: .baseline.toml (user overrides) │ -│ Defines: disabled controls, severity overrides, │ -│ custom tags, plugin trust settings │ -│ Owner: project maintainer │ +│ Layer 2: Operator configuration (outside the repo) │ +│ Defines: pass overrides, custom controls, plugins, │ +│ MCP servers, stores, trust, policy │ +│ Owner: whoever runs darnit │ ├──────────────────────────────────────────────────────┤ -│ Layer 3: .project/project.yaml (project context) │ +│ Layer 3: .project/ (project context and claims) │ │ Defines: maintainers, CI provider, governance model, │ -│ security contacts, release info │ +│ security contacts, not-applicable claims │ │ Owner: project; darnit writes only confirmed values │ └──────────────────────────────────────────────────────┘ ``` -At audit time, the framework merges Layer 1 + Layer 2 into an "effective config", then injects Layer 3 into each `CheckContext.project_context`. +At audit time, the framework merges Layer 1 + Layer 2 into an "effective config", then injects Layer 3 into each `CheckContext.project_context`. Nothing in the audited repository changes Layers 1 and 2; a repository's `.baseline.toml` is not read (an audit reports one notice pointing at `darnit config migrate`). ### Context Collection @@ -457,7 +457,7 @@ AI Assistant ▼ audit_openssf_baseline(level=1) │ - ├─► Load framework TOML + .baseline.toml → EffectiveConfig + ├─► Load framework TOML + operator configuration → EffectiveConfig ├─► Load .project/project.yaml → project_context ├─► Convert controls → ControlSpec + Pass objects ├─► Filter by level (and optionally by tags) @@ -501,7 +501,7 @@ For detailed mermaid diagrams of audit internals, remediation flow, context life | **Config** | `config/loader.py` | Load and parse TOML framework configs | | | `config/framework_schema.py` | Schema for framework TOML validation | | | `config/control_loader.py` | Convert TOML controls → `ControlSpec` objects | -| | `config/merger.py` | Merge framework + user configs → effective config | +| | `config/merger.py` | Merge framework + operator configuration → effective config | | **Context** | `context/dot_project.py` | Load/save `.project/project.yaml` | | | `context/dot_project_mapper.py` | Map TOML context keys to project YAML paths | | | `context/sieve.py` | Context sieve (progressive auto-detection) | diff --git a/CLAUDE.md b/CLAUDE.md index efd78c57..04d0dfa1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -312,7 +312,7 @@ Context values are read only through `resolve_context` (`darnit.config.context_r Project claims live in `.project/darnit.yaml` (`controls.: {status, reason, asserted_by}`). A not-applicable claim, explicit or implied by a `.project/` context value, counts only when its outcome is `honored` (trusted repository and uncontradicted, or operator-confirmed); `pending` counts as non-compliant and `contradicted` has no effect (feature 040). -Tool configuration comes only from operator configuration (`--operator-config PATH`, else `$XDG_CONFIG_HOME/darnit/config.toml` / `~/.config/darnit/config.toml`, else built-in defaults), never from the audited repository. It holds plugins, MCP servers, pass overrides, custom controls, stores, LLM settings, `[trust].repos`, CI trust rules, and policy. `darnit config show|trust|migrate` inspect it, edit the trust list, and move a deprecated `.baseline.toml` (during the deprecation release only per-control `status`/`reason`, read as claims, and `extends` by registered name are honored, with a warning per setting; switch `BASELINE_TOML_DEPRECATION_ACTIVE` in `config/merger.py`). +Tool configuration comes only from operator configuration (`--operator-config PATH`, else `$XDG_CONFIG_HOME/darnit/config.toml` / `~/.config/darnit/config.toml`, else built-in defaults), never from the audited repository. It holds plugins, MCP servers, pass overrides, custom controls, stores, LLM settings, `[trust].repos`, CI trust rules, and policy. `darnit config show|trust|migrate` inspect it, edit the trust list, and move a legacy `.baseline.toml` (claims to `.project/darnit.yaml`, a printed operator configuration fragment for the rest). darnit never reads a repository's `.baseline.toml` otherwise; an audit of a repository that has one reports a single notice pointing at `darnit config migrate`. Remediation plans, then applies only what it planned (feature 043; framework-design.md 4, 15). Handlers get `HandlerContext.mode` (`plan`/`apply`), return `FileChange`s in `evidence["file_changes"]`, and never write; the executor is the single writer, skips files with uncommitted user changes (`user_changes_present`, in the preview too), and records every write in an operator-side run manifest. A handler without `supports_plan=True` is not previewable. Platform settings change only through `platform_setting` and the platform engine (also behind `enable_branch_protection`): it reads first (unreadable means no write), plans the minimal change, never weakens an existing setting, writes nothing when already satisfied or met by a ruleset, uses the default branch, and reads back. `api_call`, `requires_confirmation`, `dry_run_supported`, and `dry_run_command` are removed; an exec remediation never touches the platform. The `[remediation]` policy (`platform`, `high_impact`: `prompt` default, `manual`, `auto`) comes from operator configuration only. Approval is by digest (`approve`), bound to the observed state; `dry_run=False` alone approves nothing, a batch never covers a high-impact change, and `safe = false` or non-previewable items need their own digest under every policy. Git tools take `run_id`, commit only manifest files with a `Darnit-Remediation-Run` trailer, never stash, and refuse unsafe repository states. Outcomes (`fixed`, `changed_not_passing`, `changed_not_verified`, `unchanged`, `needs_approval`, `needs_confirmation`, `manual`, `error`) come from a cache-neutral re-check, and summaries and commit/PR gates read outcomes, never report text. Every entry point previews by default; `darnit run` writes only with `--apply`. diff --git a/README.md b/README.md index a7611604..606a6f55 100644 --- a/README.md +++ b/README.md @@ -456,7 +456,7 @@ Darnit is designed with security in mind. Key security features include: ### Plugin Security -Configure trusted publishers in `.baseline.toml`: +Configure trusted publishers in operator configuration (`~/.config/darnit/config.toml`, or the file named by `--operator-config`): ```toml [plugins] @@ -473,7 +473,7 @@ Default trusted publishers: `kusari-oss`, `kusaridev` - [ ] Use fine-grained GitHub tokens with minimal permissions - [ ] Always use `dry_run=True` first when remediating -- [ ] Review `.baseline.toml` changes in pull requests +- [ ] Review `.project/` changes (not-applicable claims) in pull requests - [ ] Name custom adapter packages with `darnit_` prefix - [ ] Enable plugin verification in production (`allow_unsigned = false`) diff --git a/TODO.md b/TODO.md index 4cd391a1..723334ff 100644 --- a/TODO.md +++ b/TODO.md @@ -6,7 +6,7 @@ This document tracks future enhancements and design work needed. **Status**: Design needed **Priority**: Medium -**Context**: Currently, configuration is per-repository via `.baseline.toml`. We need a more flexible system. +**Context**: Tool configuration is one per-user operator configuration file (feature 040); a repository's `.baseline.toml` is no longer read. Organization-level policy is tracked in #503. We need a more flexible system. ### Problem Statement @@ -20,7 +20,7 @@ Users need the ability to: 1. **Enterprise Central Policy** ```toml - # .baseline.toml + # operator configuration extends = [ "https://config.company.com/darnit/security-policy.toml", "https://config.company.com/darnit/team-backend.toml", @@ -30,7 +30,7 @@ Users need the ability to: 2. **Monorepo Shared Config** ```toml - # packages/my-service/.baseline.toml + # operator configuration for packages/my-service extends = [ "../../.darnit-shared.toml", "openssf-baseline", @@ -41,7 +41,7 @@ Users need the ability to: ``` org-policy.toml (remote server) └── team-policy.toml (remote server) - └── .baseline.toml (local repo) + └── operator configuration (local) ``` ### Design Considerations diff --git a/docs/DECISION_FLOWS.md b/docs/DECISION_FLOWS.md index fa5c65bf..f299fe8f 100644 --- a/docs/DECISION_FLOWS.md +++ b/docs/DECISION_FLOWS.md @@ -745,7 +745,7 @@ Only after the person accepts does the agent fill in the digest; the confirmatio │ ▼ ┌───────────────────────────────────┐ - │ Check .baseline.toml for │ + │ Check framework TOML for │ │ control-specific adapter config │ └───────────────────────────────────┘ │ @@ -1467,16 +1467,16 @@ Use the provided verification script instead. │ Configuration Files │ ├─────────────────────────────────────────────────────────────────────────────┤ │ │ -│ project.toml │ .baseline.toml │ +│ project.toml │ operator configuration (user-level) │ │ ───────────────────────────────────────────────────────────────────────── │ -│ Purpose: Project metadata & │ Purpose: MCP server configuration │ -│ documentation locations │ & adapter settings │ +│ Purpose: Project metadata & │ Purpose: tool settings, never read │ +│ documentation locations │ from the audited repository │ │ │ │ │ Contains: │ Contains: │ │ • schema_version │ • schema_version │ -│ • [project] name, type, controls │ • [settings] defaults, timeouts │ -│ • [security] policy, threat_model │ • [adapters.*] custom adapter configs │ -│ • [governance] contributing, etc. │ • [controls.*] per-control overrides │ +│ • [project] name, type, controls │ • [plugins], [mcp_servers], [stores] │ +│ • [security] policy, threat_model │ • [custom_controls.*] │ +│ • [governance] contributing, etc. │ • [controls.*] pass overrides │ │ • [testing] docs, requirements │ │ │ • [releases] verification │ │ │ • [ci.github] workflows, etc. │ │ diff --git a/docs/MIGRATION_GUIDE.md b/docs/MIGRATION_GUIDE.md index e7b6512a..4095beb6 100644 --- a/docs/MIGRATION_GUIDE.md +++ b/docs/MIGRATION_GUIDE.md @@ -53,9 +53,11 @@ expr = 'output.json.status == "pass"' ## Plugin Security Configuration ### New Configuration -Add to your `.baseline.toml`: +Add to your operator configuration (`~/.config/darnit/config.toml`, or the file named by `--operator-config`): ```toml +schema_version = 1 + [plugins] # Require signed plugins in production allow_unsigned = false @@ -64,12 +66,12 @@ allow_unsigned = false trusted_publishers = [ "https://github.com/my-org", ] - -# Per-plugin configuration -[plugins."my-plugin"] -version = ">=1.0.0" ``` +## Repository `.baseline.toml` + +darnit no longer reads a repository's `.baseline.toml`; an audit of a repository that still has one reports a single notice. Run `darnit config migrate [REPO]` to move its per-control `status`/`reason` claims to `.project/darnit.yaml` and print a proposed operator configuration fragment for its other settings. Review both, add the fragment to your operator configuration, select the framework with `--framework` instead of `extends`, then delete `.baseline.toml`. + ## Context System ### Project Context diff --git a/docs/SECURITY_GUIDE.md b/docs/SECURITY_GUIDE.md index 197ff370..12a44bed 100644 --- a/docs/SECURITY_GUIDE.md +++ b/docs/SECURITY_GUIDE.md @@ -276,9 +276,9 @@ Project data in `.project/` that makes a control not applicable (for example a v An operator can confirm a pending claim through the `confirm_project_data` MCP tool (`confirm_not_applicable`). Confirmations are stored on the operator side, never in the repository, and lapse when they expire or when the claim's reason or evidence changes. An agent must only confirm a claim when the operator explicitly asks it to. -### Deprecated: `.baseline.toml` +### Removed: `.baseline.toml` -`.baseline.toml` is deprecated. In this release darnit reads its per-control `status` and `reason` and treats them exactly like `.project/` claims; it also still honors `extends` naming a registered framework (use `--framework` instead). Every audit warns for each setting in the file, naming where it now belongs. Tool settings in it are not applied. A later release will ignore the file. +darnit no longer reads `.baseline.toml`. Nothing in it has any effect: not per-control `status` and `reason` (even for a trusted repository), not `extends` (use `--framework`), and not tool settings. When the file is present, the audit reports one notice saying it was ignored. Run `darnit config migrate [REPO]` to write its claims to `.project/darnit.yaml` (existing claims are kept unless you pass `--force`) and print a proposed operator configuration fragment for its tool settings. The command never writes operator configuration. Review both, then delete `.baseline.toml`. diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 5bf8a6c3..503db36d 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -47,7 +47,7 @@ What happens inside `audit_openssf_baseline()`. ```mermaid flowchart TD A[audit_openssf_baseline called] --> B[Load framework TOML] - B --> C[Load .baseline.toml user overrides] + B --> C[Load operator configuration] C --> D[Merge configs → EffectiveConfig] D --> E[load_controls_from_effective] E --> F[Convert each control → ControlSpec + Pass objects] @@ -227,7 +227,7 @@ flowchart TD | Term | Meaning | |------|---------| | **TOML config** | Framework control definitions (`openssf-baseline.toml`) | -| **.baseline.toml** | User overrides (disable controls, change severity) | +| **Operator configuration** | Tool settings owned by whoever runs darnit (pass overrides, custom controls, trust), kept outside the audited repository | | **.project/project.yaml** | Project context (maintainers, CI provider, etc.) | | **ControlSpec** | A control with its passes, remediation, and metadata | | **SieveResult** | Outcome of verifying a single control (PASS/FAIL/WARN) | diff --git a/docs/architecture/audit-pipeline.md b/docs/architecture/audit-pipeline.md index f78f6c6e..109c77bb 100644 --- a/docs/architecture/audit-pipeline.md +++ b/docs/architecture/audit-pipeline.md @@ -31,9 +31,9 @@ The `run_sieve_audit()` function SHALL accept a list of pre-loaded `ControlSpec` ### Requirement: Pipeline supports optional features The `run_sieve_audit()` function SHALL support optional parameters for features that not all callers need, with sensible defaults. -#### Scenario: User config exclusions -- **WHEN** `apply_user_config=True` (default) -- **THEN** controls excluded in `.baseline.toml` SHALL be marked as `N/A` in results +#### Scenario: Not-applicable claims +- **WHEN** `evaluate_claims=True` (default) +- **THEN** the repository's not-applicable claims (`.project/darnit.yaml` and applicability-changing `.project/` context values) SHALL be assessed under framework-design 14.3, and only an honored claim SHALL mark its control `N/A` #### Scenario: UnifiedLocator integration - **WHEN** the pipeline runs diff --git a/docs/architecture/context-collection.md b/docs/architecture/context-collection.md index 2ea17bd0..1b7ae591 100644 --- a/docs/architecture/context-collection.md +++ b/docs/architecture/context-collection.md @@ -139,7 +139,7 @@ The framework SHALL persist context a person confirmed, with a confirmation reco #### Scenario: Project file is not written - **WHEN** a context value is persisted -- **THEN** the framework SHALL NOT write it to `.project/project.yaml` or `.baseline.toml` +- **THEN** the framework SHALL NOT write it to `.project/project.yaml` ### Requirement: Context validation The framework SHALL validate context values against defined constraints. diff --git a/docs/architecture/framework-design.md b/docs/architecture/framework-design.md index f11357e9..b3b71922 100644 --- a/docs/architecture/framework-design.md +++ b/docs/architecture/framework-design.md @@ -1,6 +1,6 @@ # Darnit Framework Design Specification -> **Version**: 1.0.0-alpha.11 +> **Version**: 1.0.0-alpha.12 > **Status**: Authoritative > **Last Updated**: 2026-10-04 @@ -163,7 +163,7 @@ steps = ["Verify branch protection in repository settings"] #### Requirement: No Unknown Control Keys - **WHEN** a control declares a key the control schema does not define - **THEN** loading the framework file MUST fail with an error naming the file, the control, and the key (feature 044; step keys are checked as in section 3.0.3) -- **AND** a custom control defined in `.baseline.toml` MUST fail the same way, the error naming `.baseline.toml`; an operator custom control's unknown key fails loading the operator configuration (section 14) +- **AND** an operator custom control's unknown key MUST fail loading the operator configuration (section 14) #### Requirement: SARIF Metadata - **WHEN** SARIF output is generated @@ -1846,9 +1846,8 @@ The darnit framework package SHALL NOT contain code, modules, or string literals #### Requirement: Explicit implementation selection - **WHEN** callers need a compliance implementation - **THEN** they SHALL use `get_implementation(name)` with an explicit name -- **AND** the name SHALL be resolved from `.baseline.toml` `extends` field, - an explicit parameter, or `discover_implementations()` to list available options -- **AND** a repository may name a framework only by registered name (never by file path), and the operator's choice (Section 14) takes precedence +- **AND** the name SHALL come from an explicit parameter (the `--framework` option or a tool's framework argument), or `discover_implementations()` to list available options +- **AND** nothing in the audited repository selects the framework (Section 14.4) --- @@ -1930,7 +1929,19 @@ Design principle: **the audited repository is untrusted input in its entirety, i ### 14.4 Repository-level .baseline.toml -The repository-level `.baseline.toml` is deprecated. During the deprecation release only its per-control status and reason are read, as assertions under 14.3; every other setting is ignored with a warning naming its new home. `darnit config migrate` moves assertions into `.project/` and proposes an operator configuration fragment. +darnit does not read a repository's `.baseline.toml` for anything: no per-control status or reason, no `extends`, no settings, no custom controls, and no other key. Not-applicable claims come only from `.project/` (14.3), the framework is selected only by the `--framework` option or a tool's framework argument, and tool settings come only from operator configuration (14.1). + +- When the file exists in an audited repository, the audit SHALL record exactly one notice, logged at WARNING and listed in the report's `warnings`: the file is ignored, and `darnit config migrate` moves its claims to `.project/darnit.yaml` and prints an operator configuration fragment for its tool settings. Its keys are not listed in `ignored_repository_settings`. +- `darnit config migrate [REPO]` reads the file itself. It writes the per-control status and reason to `.project/darnit.yaml` (keeping an existing claim for the same control unless `--force` is given), prints a proposed operator configuration fragment for the other settings, never writes operator configuration, and never deletes the file. + +#### Scenario: Claim in .baseline.toml for a trusted repository +- **WHEN** a repository the operator trusts has a `.baseline.toml` marking a control `status = "n/a"` with a reason +- **THEN** the control MUST be evaluated as if the file were absent, with no assertion +- **AND** the report MUST carry the single notice + +#### Scenario: extends in .baseline.toml +- **WHEN** `.baseline.toml` sets `extends` to a registered framework and the run names no framework +- **THEN** the audit MUST NOT use the framework the file names ## 15. Remediation Safety @@ -1958,7 +1969,7 @@ high_impact = "prompt" # prompt | manual | auto | `manual` | darnit makes no platform change; the outcome is `manual` with the exact steps for a person | | `auto` | darnit writes without asking, and records the change and its impact in the outcome | -- The policy comes only from operator configuration. A `[remediation]` setting in the audited repository (`.project/`, `.baseline.toml`, or any other file) is ignored. +- The policy comes only from operator configuration. A `[remediation]` setting in the audited repository (`.project/` or any other file) is ignored. - An absent section means both values are `prompt`. Unknown keys or values stop the run (section 14.1). - Every `RemediationRun` records the policy in effect and the operator configuration digest; `darnit config show` prints the resolved section. - Every value still follows the `platform_setting` rules (section 4.5: read first, never weaken, no write when already satisfied, default branch, read back). `auto` removes only the approval step for platform change sets. It never approves an item that requires individual approval because of `safe = false` or because it cannot be previewed (15.3); those need a person's approval under every policy. @@ -2187,7 +2198,7 @@ The `confirm_*` tools are unchanged; approvals are not confirmations. Audit resu ## Appendix C: Removed Requirements -The following requirements have been superseded: by the handler dispatch architecture, and (the last two entries) by the feature 043 remediation design. +The following requirements have been superseded: by the handler dispatch architecture, by the feature 043 remediation design (the `api_call` and `requires_confirmation` entries), and by the operator configuration and trust boundary (the last entry, Section 14). ### Removed: VerificationPassProtocol **Reason**: Replaced by handler dispatch architecture. Pass classes that implemented this protocol (`DeterministicPass`, `PatternPass`, `LLMPass`, `ManualPass`, `ExecPass`) are superseded by handler functions registered in `SieveHandlerRegistry`. @@ -2250,12 +2261,22 @@ The following requirements have been superseded: by the handler dispatch archite **Reason**: Feature 043. They were declared but never enforced (FR-025: a safety, confirmation, or preview property is enforced or absent). **Migration**: Use `safe = false` instead of `requires_confirmation` (section 15.3). Previews come from plan mode (section 4.2); an exec remediation declares `effects = "working_tree"` and `offline = true` instead of a `dry_run_command` (section 4.4). +### Removed: Repository-level .baseline.toml +**Reason**: The audited repository is untrusted input (Section 14). Feature 040 deprecated the file and, for one release, read only its per-control status and reason as claims and its `extends` by registered name; it is now ignored entirely (Section 14.4). +**Migration**: Run `darnit config migrate [REPO]`: claims move to `.project/darnit.yaml`, and the printed fragment goes into operator configuration after review. Select the framework with `--framework`. + +#### Scenario: Repository still has .baseline.toml +- **WHEN** an audited repository contains `.baseline.toml` +- **THEN** no key in it MUST affect the audit +- **AND** the audit MUST report one notice pointing at `darnit config migrate` + --- ## Version History | Version | Date | Changes | |---------|------|---------| +| 1.0.0-alpha.12 | 2026-10-04 | Repository-level `.baseline.toml` is no longer read: one notice points at `darnit config migrate`, framework selection only by `--framework` or a tool argument (Sections 2.3, 10.5, 14.4, 15.1; Appendix C) | | 1.0.0-alpha.11 | 2026-10-04 | Close remaining false-PASS paths (feature 044): an expression that cannot be evaluated or is not boolean makes the step ERROR, expression names per step type with usable `project` values and a repository-aware `file_exists`, load-time expression reference check (Section 3.7); step registration declares `settings` and `expression_names`, plugins cannot replace a registered step type, and unknown control keys, unknown step keys, and unregistered step types fail loading (Sections 2.3, 3.0.3); reproducibility step types conclude only FAIL (Sections 3.0.1, 12); `expr_decides`, an expression that decides on a handler PASS (Sections 3.0.3, 3.7); `gh_api` `evidence_fields`, required for personal records (Section 3.8); `file_must_exist` replaced by the registered `file_exists` (Section 3.2); field tables corrected to what each handler reads (Sections 3.3-3.5) | | 1.0.0-alpha.10 | 2026-10-02 | Remediation safety (feature 043): plan/apply protocol and single writer (Section 4.2), `platform_setting` (4.5), exec `effects`/`offline` and no platform state from exec (4.4), `file_create.project_reference` (4.3), remediation policy, digest-bound approval, outcomes, re-check, run manifest, and version-control rules (Section 15); removed `api_call`, `requires_confirmation`, `dry_run_supported`, `dry_run_command` (Appendix C) | | 1.0.0-alpha.9 | 2026-09-29 | Context value standing, confirmation records, lapse, detection fallbacks, canonical keys, reads never write, targeted project-file writes, confirmation tool contract (Sections 7.4-7.11, feature 042) | diff --git a/docs/getting-started/framework-development.md b/docs/getting-started/framework-development.md index 56ab0587..1c5c2bc3 100644 --- a/docs/getting-started/framework-development.md +++ b/docs/getting-started/framework-development.md @@ -197,8 +197,8 @@ Changes here affect all controls across all implementations — be careful and t Configuration loading lives in `packages/darnit/src/darnit/config/`. The framework loads: 1. Implementation TOML (via `get_framework_config_path()`) -2. Project context (`.project/project.yaml`) -3. Local overrides (`.baseline.toml`) +2. Operator configuration (`config/operator/`; never from the audited repository) +3. Project context and claims (`.project/project.yaml`, `.project/darnit.yaml`) ## Key Files Reference diff --git a/docs/packaging-plugins.md b/docs/packaging-plugins.md index 43c7765d..012baf62 100644 --- a/docs/packaging-plugins.md +++ b/docs/packaging-plugins.md @@ -294,7 +294,7 @@ This works but bypasses the signing chain. Fine for development; not recommended darnit's plugin discovery has an optional verification step backed by Sigstore. By default it allows unsigned plugins (for backward compatibility), but production deployments can flip the switch. -Configure trust in your project's `.baseline.toml`: +Configure trust in operator configuration (`~/.config/darnit/config.toml`, or the file named by `--operator-config`; it is never read from the audited repository): ```toml [plugins] diff --git a/docs/plugin-authoring/stores.md b/docs/plugin-authoring/stores.md index 5f65cdd1..7f84fd9f 100644 --- a/docs/plugin-authoring/stores.md +++ b/docs/plugin-authoring/stores.md @@ -82,7 +82,7 @@ class S3AttestationStore: return None ``` -### Selecting the backend in `.baseline.toml` +### Selecting the backend in operator configuration ```toml [stores.attestation] @@ -177,7 +177,7 @@ is expensive to establish. Feature 034 ships two additional filesystem-backed backends inside darnit-core alongside the in-repo defaults. Both are selectable from -`.baseline.toml` under any `[stores.]` block; both write outside +operator configuration under any `[stores.]` block; both write outside the audited repository. They exist because the in-repo defaults land attestations, reports, and audit-cache under `/.darnit/`, which is often not where an operator wants them (backups, CI artifact @@ -228,17 +228,13 @@ root = "$RUNNER_ARTIFACTS_DIR/darnit-reports" ### Multi-repo templating -Darnit has no org-level or user-level config file. If you want the -same `[stores.]` block active on 30 repos, use one of: - -1. **Env-var interpolation (easiest)**. Keep `root = "$DARNIT_ATT_ROOT"` - in every repo's `.baseline.toml`. Set `DARNIT_ATT_ROOT` once per - machine (shell profile, systemd unit, CI runner env). The 30 repos - share the destination without duplicating the literal path. -2. **CI/CD templating**. Your workflow rewrites `.baseline.toml` before - invoking `darnit audit`. -3. **Cookiecutter / repo-init tool**. One-shot copy the block into - each repo when you first onboard it. +Store selection lives in operator configuration, which is per user +(or per CI runner), not per repository, so one `[stores.]` block +applies to every repository that user audits. To vary the destination +per machine, keep `root = "$DARNIT_ATT_ROOT"` in the block and set +`DARNIT_ATT_ROOT` in the shell profile, systemd unit, or CI runner +environment; to vary it per run, point `--operator-config` at another +file. ### The `.project/` layer (FR-009) @@ -246,7 +242,7 @@ Neither `local-fs` nor `user-local` is registered under `darnit.stores.project`. `.project/project.yaml` is the CNCF `.project/` spec's canonical repo-committable artifact and stays in the repo by design. If you write `[stores.project] backend = "local-fs"` -in `.baseline.toml`, `resolve_stores` raises `StoreNotInstalled` +in operator configuration, `resolve_stores` raises `StoreNotInstalled` before any control runs -- the misconfiguration surfaces at audit start, not in a confusing runtime failure. Redirecting project state outside the repo means governance tooling can no longer find it, so diff --git a/docs/plugin-discovery-design.md b/docs/plugin-discovery-design.md index ceee9333..a7dc172a 100644 --- a/docs/plugin-discovery-design.md +++ b/docs/plugin-discovery-design.md @@ -1,12 +1,13 @@ # Plugin Discovery System Design +> **Note**: This is a design record. Earlier revisions also let a repository-level user configuration (`.baseline.toml`) reference adapters; darnit no longer reads that file. Adapters are referenced from framework TOML, and tool settings live in operator configuration outside the audited repository (`docs/architecture/framework-design.md` section 14). + ## Overview This document describes the design for a unified plugin discovery system that enables: 1. **Framework packages** to define compliance frameworks via TOML + Python adapters 2. **Plugin packages** to provide reusable adapters (checks, remediations) across frameworks -3. **User configs** to reference adapters from any installed package ## Current State Analysis @@ -283,7 +284,7 @@ class AdapterInfo: ## Adapter Resolution Algorithm -When a framework or user config references an adapter, resolution follows this order: +When a framework config references an adapter, resolution follows this order: ``` 1. Explicit module path (type = "python", module = "...") @@ -303,16 +304,15 @@ When a framework or user config references an adapter, resolution follows this o ### Example Resolution ```toml -# User's .baseline.toml +# Framework TOML [controls."OSPS-VM-05.02"] check = { adapter = "kusari" } # Just the name ``` Resolution: -1. Check if `kusari` is defined in `.baseline.toml` `[adapters]` section → No -2. Check if `kusari` is defined in framework's `[adapters]` section → No -3. Check `darnit.check_adapters` entry points → Found! `darnit_plugins.adapters.kusari:KusariAdapter` -4. Load and instantiate `KusariAdapter` +1. Check if `kusari` is defined in framework's `[adapters]` section → No +2. Check `darnit.check_adapters` entry points → Found! `darnit_plugins.adapters.kusari:KusariAdapter` +3. Load and instantiate `KusariAdapter` --- @@ -352,26 +352,6 @@ name = "PreReleaseSCA" check = { adapter = "kusari" } # Uses entry point ``` -### User Config (.baseline.toml) - -```toml -# .baseline.toml -extends = "openssf-baseline" - -# Reference plugin adapter by name -[controls."OSPS-VM-05.02"] -check = { adapter = "kusari" } - -# Or define inline with full path -[adapters.my_scanner] -type = "python" -module = "internal_tools.scanner" -class = "InternalScanner" - -[controls."OSPS-SA-03.01"] -check = { adapter = "my_scanner" } -``` - --- ## Example Plugin Package: darnit-plugins @@ -528,7 +508,6 @@ class KusariCheckAdapter(CheckAdapter): 1. **Existing frameworks** - No changes required, entry points still work 2. **New adapter packages** - Use new `darnit.check_adapters` entry points -3. **User configs** - Can reference adapters by name once plugins installed --- @@ -560,9 +539,6 @@ class TestPluginRegistry: class TestCrossPackageAdapters: def test_framework_uses_plugin_adapter(self): """Framework should be able to use adapter from another package.""" - - def test_user_config_overrides_with_plugin(self): - """User config should override adapter with plugin.""" ``` --- @@ -588,7 +564,6 @@ This design enables a pluggable architecture where: - **Framework authors** can reference adapters from any installed package - **Plugin authors** can provide reusable adapters via entry points -- **Users** can mix and match adapters in their `.baseline.toml` - **Backwards compatibility** is preserved for existing implementations The key addition is the **`PluginRegistry`** that unifies discovery across all plugin types and provides a consistent API for resolution. diff --git a/example.baseline.toml b/example.baseline.toml deleted file mode 100644 index 97c17f1e..00000000 --- a/example.baseline.toml +++ /dev/null @@ -1,136 +0,0 @@ -# Example OpenSSF Baseline User Configuration -# Copy this file to .baseline.toml in your repository root -# -# This file allows you to: -# - Override control settings (mark as N/A, use different adapter) -# - Define custom adapters (Kusari, custom scripts) -# - Configure control groups for batch settings -# - Add custom controls specific to your organization - -version = "1.0" - -# Framework to extend (this is typically "openssf-baseline") -extends = "openssf-baseline" - -# Global settings -[settings] -# Cache check results for faster re-runs -cache_results = true -cache_ttl = 300 # 5 minutes - -# Timeout for adapter operations (seconds) -timeout = 300 - -# Fail the audit if any check produces an error -fail_on_error = false - -# Run independent checks in parallel -parallel_checks = true -max_parallel = 5 - -# ============================================================================= -# Plugin Security Configuration -# ============================================================================= -# Configure trusted publishers for plugin verification. -# By default, kusari-oss and kusaridev are trusted. - -[plugins] -# Global settings for all plugins -# global_allow_unsigned = false # Require signed plugins (production) -# global_trusted_publishers = [ # Additional orgs to trust globally -# "https://github.com/my-company", -# ] - -# Per-plugin configuration examples: - -# Trust default publishers (kusari-oss, kusaridev) - no extra config needed -# [plugins."darnit-baseline"] -# version = ">=1.0.0" - -# Allow a specific plugin to be unsigned (for local development) -# [plugins."my-dev-plugin"] -# version = ">=0.1.0" -# allow_unsigned = true - -# Trust a third-party plugin publisher -# [plugins."third-party-compliance"] -# version = ">=2.0.0" -# trusted_publishers = ["https://github.com/trusted-vendor"] - -# ============================================================================= -# Custom Adapters -# ============================================================================= - -# Example: Kusari adapter for SCA and dependency scanning -[adapters.kusari] -type = "command" -command = "kusari" -output_format = "json" -timeout = 300 - -# Example: Custom script adapter for internal security checks -# [adapters.security-scanner] -# type = "script" -# command = "./scripts/security-check.sh" -# output_format = "json" -# timeout = 120 - -# Example: Python module adapter for custom checks -# [adapters.my-checker] -# type = "python" -# module = "my_company.security.checker" -# class = "SecurityChecker" - -# ============================================================================= -# Control Overrides -# ============================================================================= - -# Use Kusari for pre-release SCA checks -[controls."OSPS-VM-05.02"] -check = { adapter = "kusari" } -# Remediation still uses builtin (provides guidance) -remediation = { adapter = "builtin" } - -# Use Kusari for dependency scanning -[controls."OSPS-VM-05.03"] -check = { adapter = "kusari" } - -# Mark a control as Not Applicable with reason -# Example: Pre-1.0 project without releases -[controls."OSPS-BR-02.01"] -status = "n/a" -reason = "Pre-1.0 project, no releases yet" - -# Example: Disable a control temporarily -# [controls."OSPS-QA-04.01"] -# status = "disabled" -# reason = "Temporarily disabled while migrating to monorepo" - -# ============================================================================= -# Control Groups -# ============================================================================= -# Apply shared configuration to multiple controls at once - -# Group all vulnerability management controls to use Kusari -[control_groups.vulnerability-management] -controls = ["OSPS-VM-05.02", "OSPS-VM-05.03"] -check = { adapter = "kusari" } - -# Example: Mark multiple controls as N/A -# [control_groups.pre-release] -# controls = ["OSPS-BR-02.01", "OSPS-BR-02.02"] -# status = "n/a" -# reason = "No releases for this internal tool" - -# ============================================================================= -# Custom Controls -# ============================================================================= -# Add organization-specific controls beyond the framework - -# Example: Custom internal security review requirement -# [controls."CUSTOM-SEC-01"] -# name = "InternalSecurityReview" -# level = 1 -# domain = "SA" -# description = "Require internal security review sign-off for releases" -# check = { adapter = "security-scanner" } diff --git a/packages/darnit-plugins/README.md b/packages/darnit-plugins/README.md index 2910978b..2312a327 100644 --- a/packages/darnit-plugins/README.md +++ b/packages/darnit-plugins/README.md @@ -44,18 +44,6 @@ description = "Test control using echo adapter" check = { adapter = "echo", config = { status = "PASS" } } ``` -### In User Config (.baseline.toml) - -Override framework adapters with plugins: - -```toml -# .baseline.toml -extends = "openssf-baseline" - -[controls."OSPS-VM-05.02"] -check = { adapter = "kusari" } -``` - ### Programmatic Usage ```python diff --git a/packages/darnit-testchecks/README.md b/packages/darnit-testchecks/README.md index b3e4dccc..73112203 100644 --- a/packages/darnit-testchecks/README.md +++ b/packages/darnit-testchecks/README.md @@ -77,23 +77,23 @@ result = adapter.check( print(f"{result.control_id}: {result.status.value} - {result.message}") ``` -### User Customization +### Selecting the Framework and Claiming Controls Not Applicable -Users can customize checks with `.baseline.toml`: +Select this framework per run: -```toml -version = "1.0" -extends = "testchecks" +```bash +darnit audit --framework testchecks /path/to/repo +``` -# Skip TODO check - we use TODOs in this project -[controls."TEST-QA-01"] -status = "n/a" -reason = "TODOs are acceptable in this project" +A project claims a control is not applicable in `.project/darnit.yaml`. The +claim counts only when the operator trusts the repository and no evidence +contradicts it, or after the operator confirms it: -# Skip print statement check for scripts -[controls."TEST-QA-02"] -status = "n/a" -reason = "Print statements OK in CLI scripts" +```yaml +controls: + TEST-QA-01: + status: n/a + reason: TODOs are acceptable in this project ``` ## Creating Your Own Framework diff --git a/specs/040-operator-config-trust/deprecation-completed.md b/specs/040-operator-config-trust/deprecation-completed.md new file mode 100644 index 00000000..4e95abc7 --- /dev/null +++ b/specs/040-operator-config-trust/deprecation-completed.md @@ -0,0 +1,65 @@ +# Addendum: .baseline.toml deprecation completed + +**Date**: 2026-10-04 +**Completes**: FR-023 (research R8; plan "Before switching `BASELINE_TOML_DEPRECATION_ACTIVE` off") +**Spec**: `docs/architecture/framework-design.md` 14.4 and Appendix C + +## What changed + +darnit no longer reads a repository's `.baseline.toml` for anything. The +deprecation release read its per-control `status`/`reason` as claims and its +`extends` by registered name; both stop. No key in the file affects an audit, +even for a repository the operator trusts. + +Removed with it: + +- The `BASELINE_TOML_DEPRECATION_ACTIVE` switch and the per-setting + deprecation warnings. +- `load_user_config` (with its `trusted` path) and + `load_user_config_with_report`, the untrusted-file restriction, and + `validate_user_config`. +- `.baseline.toml` claims in `darnit.trust.assertions`. +- `darnit.config.user_schema` (`UserConfig`, `UserSettings`, + `ControlOverride`, `ControlGroup`, `CustomControl`, `ControlStatus`, + `create_user_config*`). Operator configuration and `config migrate` used + none of it. +- The user-configuration side of the merge: `merge_configs` and + `merge_control` take no user configuration; `EffectiveControl` loses + `status`, `status_reason`, `from_user`, and `is_applicable()`; + `EffectiveConfig` loses the `.baseline.toml` settings and + `get_excluded_controls()`; `load_effective_config*` and the control loaders + take no repository path. + +## The notice + +When the file is present, every audit records one notice, logged at WARNING +and listed in the report's `warnings`: the file is ignored, and +`darnit config migrate` moves its claims to `.project/darnit.yaml` and +prints an operator configuration fragment for its tool settings. The file's +keys are no longer listed in `ignored_repository_settings`. + +## Unchanged + +- `darnit config migrate [REPO] [--force]` reads the file itself and does + what it did: claims to `.project/darnit.yaml`, a printed (never written) + operator configuration fragment, and the file left in place. +- Claims from `.project/darnit.yaml` and from `.project/` context values + follow FR-013a to FR-020 as before. +- The framework comes from `--framework` or a tool's framework argument, + else the default. `darnit run` gains `--framework` (#507), the remaining + prerequisite in plan.md. + +## Rationale + +The audited repository is untrusted input. One release of warnings gave +existing users the migration path (FR-021, SC-005); keeping a reader for the +file afterwards only kept a second, weaker claims path and a way for +repository content to select the framework. + +## Not done here + +The parity corpus still marks its fixture directories with a +`.baseline.toml` (now only a marker, which the audit ignores). Switching +discovery to `parity.toml` needs a parity-only change, because the parity +guard (`tests/darnit/parity/tier1/test_no_product_changes.py`) forbids +mixing parity-harness and product changes in one pull request. From 6d95865d208c1c6c2541b2fe30b675c012593787 Mon Sep 17 00:00:00 2001 From: Michael Lieberman Date: Sun, 4 Oct 2026 20:44:32 -0400 Subject: [PATCH 2/5] feat(cli): add darnit run --framework; select test frameworks explicitly `darnit run` reads args.framework but had no option for it (#507). Add -f/--framework like `audit` and pass it to the audit state. The CLI and harness test fixtures selected their framework through a `.baseline.toml` `extends`; they now pass `--framework testchecks` / `framework_name="openssf-baseline"` and the fixture files are deleted, along with the .gitignore rules that kept them tracked. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Michael Lieberman --- .gitignore | 7 -- packages/darnit/src/darnit/cli.py | 9 ++- .../cli/fixtures/failing_repo/.baseline.toml | 1 - .../fixtures/malformed_project/.baseline.toml | 1 - .../cli/fixtures/minimal_repo/.baseline.toml | 1 - tests/darnit/cli/test_cmd_run_e2e.py | 66 +++++++++++++++---- tests/darnit/harness/conftest.py | 3 +- .../fixtures/minimal_llm_repo/.baseline.toml | 11 ---- tests/darnit/harness/test_cli.py | 18 ++--- tests/darnit/server/test_harness_loop_mcp.py | 12 ++-- 10 files changed, 77 insertions(+), 52 deletions(-) delete mode 100644 tests/darnit/cli/fixtures/failing_repo/.baseline.toml delete mode 100644 tests/darnit/cli/fixtures/malformed_project/.baseline.toml delete mode 100644 tests/darnit/cli/fixtures/minimal_repo/.baseline.toml delete mode 100644 tests/darnit/harness/fixtures/minimal_llm_repo/.baseline.toml diff --git a/.gitignore b/.gitignore index 0479bf68..3b8bd28b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,16 +29,9 @@ htmlcov/ Thumbs.db # Local configuration (keep examples, ignore actual) -.baseline.toml project.toml !example.*.toml !example.*.yaml -# Test fixtures with tracked .baseline.toml. The top-level rule above -# would otherwise strip them and CI would either deselect the whole -# suite (parity) or auto-pick a different framework than the fixture -# expects (harness). -!tests/darnit/harness/fixtures/**/.baseline.toml -!tests/darnit/parity/fixtures/**/.baseline.toml # Logs *.log diff --git a/packages/darnit/src/darnit/cli.py b/packages/darnit/src/darnit/cli.py index 304e7939..5e8e2aea 100644 --- a/packages/darnit/src/darnit/cli.py +++ b/packages/darnit/src/darnit/cli.py @@ -838,14 +838,13 @@ def cmd_run(args: argparse.Namespace) -> int: print(f" Remediate : {'apply' if args.apply else 'preview (nothing is written; pass --apply to write)'}") print() - # framework_name=None auto-resolves from .baseline.toml inside audit(). owner, repo = owner_repo_from_identity(target) if target else (None, None) state = AuditState( local_path=repo_path, owner=owner, repo=repo, target=target, - framework_name=getattr(args, "framework", None), + framework_name=args.framework, level=getattr(args, "level", 3), ) @@ -1570,6 +1569,10 @@ def create_parser() -> argparse.ArgumentParser: default=".", help="Path to repository (default: current directory)", ) + run_parser.add_argument( + "-f", "--framework", + help="Framework name to use (e.g., openssf-baseline)", + ) run_parser.add_argument( "--feedback", dest="feedback_mode", @@ -1606,7 +1609,7 @@ def create_parser() -> argparse.ArgumentParser: ) harness_parser.add_argument( "--framework", - help="Framework name (e.g., openssf-baseline). Overrides .baseline.toml.", + help="Framework name (e.g., openssf-baseline).", ) harness_parser.add_argument( "--level", diff --git a/tests/darnit/cli/fixtures/failing_repo/.baseline.toml b/tests/darnit/cli/fixtures/failing_repo/.baseline.toml deleted file mode 100644 index df4b10df..00000000 --- a/tests/darnit/cli/fixtures/failing_repo/.baseline.toml +++ /dev/null @@ -1 +0,0 @@ -extends = "testchecks" diff --git a/tests/darnit/cli/fixtures/malformed_project/.baseline.toml b/tests/darnit/cli/fixtures/malformed_project/.baseline.toml deleted file mode 100644 index df4b10df..00000000 --- a/tests/darnit/cli/fixtures/malformed_project/.baseline.toml +++ /dev/null @@ -1 +0,0 @@ -extends = "testchecks" diff --git a/tests/darnit/cli/fixtures/minimal_repo/.baseline.toml b/tests/darnit/cli/fixtures/minimal_repo/.baseline.toml deleted file mode 100644 index df4b10df..00000000 --- a/tests/darnit/cli/fixtures/minimal_repo/.baseline.toml +++ /dev/null @@ -1 +0,0 @@ -extends = "testchecks" diff --git a/tests/darnit/cli/test_cmd_run_e2e.py b/tests/darnit/cli/test_cmd_run_e2e.py index 2df616ca..affbd619 100644 --- a/tests/darnit/cli/test_cmd_run_e2e.py +++ b/tests/darnit/cli/test_cmd_run_e2e.py @@ -61,7 +61,7 @@ def test_golden_exit_code_matches_failed_count( """Pins exit-code contract (spec FR-003(a), acceptance #1/#3; contracts C10/C12 and the top-of-file exit-code rule).""" exit_code, stdout, _stderr = invoke_cmd_run( - [str(minimal_repo_tree), "--feedback", "noninteractive"], + [str(minimal_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) counts = _parse_counts(stdout) @@ -85,7 +85,7 @@ def test_golden_prints_header( ) -> None: """Pins the `Darnit run` header string (contract C1, C2).""" _exit, stdout, _stderr = invoke_cmd_run( - [str(minimal_repo_tree), "--feedback", "noninteractive"], + [str(minimal_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) # C1: header appears exactly once, on its own line. @@ -108,7 +108,7 @@ def test_golden_prints_footer_and_count_lines( ) -> None: """Pins `Run complete.` footer and count-line structure (contracts C3, C4).""" _exit, stdout, _stderr = invoke_cmd_run( - [str(minimal_repo_tree), "--feedback", "noninteractive"], + [str(minimal_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) # C3: `Run complete.` appears exactly once on its own line. @@ -132,7 +132,7 @@ def test_golden_counts_accounting_is_sound( Controls in no printed bucket are N/A, ERROR, or PENDING. """ _exit, stdout, _stderr = invoke_cmd_run( - [str(minimal_repo_tree), "--feedback", "noninteractive"], + [str(minimal_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) counts = _parse_counts(stdout) @@ -151,7 +151,7 @@ def test_golden_no_error_no_traceback_no_pending( """Pins FR-003(d) + contracts C7 (no Error line on success path) and C8 (no Python traceback ever).""" _exit, stdout, _stderr = invoke_cmd_run( - [str(minimal_repo_tree), "--feedback", "noninteractive"], + [str(minimal_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) assert not re.search(r"^Error:", stdout, re.MULTILINE), ( @@ -169,7 +169,7 @@ def test_golden_output_is_ascii( ) -> None: """Pins FR-012 and contract C9: cmd_run stdout is ASCII-only.""" _exit, stdout, _stderr = invoke_cmd_run( - [str(minimal_repo_tree), "--feedback", "noninteractive"], + [str(minimal_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) non_ascii = [(i, ch) for i, ch in enumerate(stdout) if ord(ch) > 127] @@ -188,7 +188,7 @@ def test_golden_failing_fixture_exits_one( so a broken exit-code rule surfaces here. """ exit_code, stdout, _stderr = invoke_cmd_run( - [str(failing_repo_tree), "--feedback", "noninteractive"], + [str(failing_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) counts = _parse_counts(stdout) @@ -222,7 +222,7 @@ def test_deterministic_exit_code_stable_under_stubs( fails with a message identifying the offending call site. """ exit_code, _stdout, _stderr = invoke_cmd_run( - [str(minimal_repo_tree), "--feedback", "noninteractive"], + [str(minimal_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) # Golden fixture design produces zero FAIL, so exit must be 0. @@ -245,7 +245,7 @@ def test_deterministic_subprocess_routes_to_stub( are routed to the canned fake, not to the real system. """ invoke_cmd_run( - [str(minimal_repo_tree), "--feedback", "noninteractive"], + [str(minimal_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) recorded = deterministic_run._recorded_calls @@ -289,7 +289,7 @@ def test_deterministic_no_llm_or_mcp_log_lines( caplog.set_level(logging.WARNING) invoke_cmd_run( - [str(minimal_repo_tree), "--feedback", "noninteractive"], + [str(minimal_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) blocklist = ("llm", "anthropic", "openai", "mcp") @@ -336,7 +336,7 @@ def test_failure_missing_repo_path( """ missing = tmp_path / "does-not-exist" exit_code, stdout, stderr = invoke_cmd_run( - [str(missing), "--feedback", "noninteractive"], + [str(missing), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) combined = stdout + stderr @@ -360,7 +360,7 @@ def test_failure_no_framework_implementation( monkeypatch: pytest.MonkeyPatch, ) -> None: """Pins US3 acceptance #2, FR-005: when the plugin registry cannot - resolve the framework named in .baseline.toml, cmd_run's behavior + resolve the framework named with --framework, cmd_run's behavior is pinned as-observed. Reference contract E3. NOTE: current production behavior swallows the missing-framework @@ -374,7 +374,7 @@ def test_failure_no_framework_implementation( monkeypatch.setattr(discovery, "get_implementation", lambda *_a, **_kw: None) exit_code, stdout, _stderr = invoke_cmd_run( - [str(minimal_repo_tree), "--feedback", "noninteractive"], + [str(minimal_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) # Pin the RULE (exit code follows the Failed count) and the SHAPE @@ -406,7 +406,7 @@ def test_failure_malformed_project_yaml( be updated deliberately in that PR. """ exit_code, stdout, _stderr = invoke_cmd_run( - [str(malformed_project_tree), "--feedback", "noninteractive"], + [str(malformed_project_tree), "--framework", "testchecks", "--feedback", "noninteractive"], capsys, ) counts = _parse_counts(stdout) @@ -541,3 +541,41 @@ def test_apply_writes_the_planned_changes( assert "created CONTRIBUTING.md" in stdout assert exit_code == 1 assert stdout.isascii() + + +class TestFrameworkOption: + """#507: ``darnit run --framework NAME`` selects the framework the audit runs.""" + + def test_framework_option_reaches_the_audit( + self, + minimal_repo_tree: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + ) -> None: + from darnit.agent import graph + + seen: list[str | None] = [] + + def audited(state): + seen.append(state.framework_name) + state.audit_results = [] + state.error = None + return state + + monkeypatch.setattr(graph, "audit", audited) + + invoke_cmd_run([str(minimal_repo_tree), "-f", "testchecks", "--feedback", "noninteractive"], capsys) + + assert seen == ["testchecks"] + + def test_testchecks_controls_are_audited( + self, + minimal_repo_tree: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + _exit, stdout, _stderr = invoke_cmd_run( + [str(minimal_repo_tree), "--framework", "testchecks", "--feedback", "noninteractive"], + capsys, + ) + + assert _parse_counts(stdout)["total"] > 0 diff --git a/tests/darnit/harness/conftest.py b/tests/darnit/harness/conftest.py index 199d2a77..2ea2694d 100644 --- a/tests/darnit/harness/conftest.py +++ b/tests/darnit/harness/conftest.py @@ -42,7 +42,7 @@ def _stub_framework_pending(monkeypatch: pytest.MonkeyPatch) -> None: Feature 027 QuestionResolver tests (and other harness tests) assemble synthesized fake results and count how many end up answered/pending. Once PR #365 wired the framework's own `get_pending_context` into - `_collect_unanswered`, running against a real .baseline.toml fixture + `_collect_unanswered`, running against a real framework fixture starts emitting real per-key questions -- polluting those counts. Tests that want the enumerator active un-stub via `monkeypatch.undo()` or set `run._enumerate_framework_pending = `. @@ -132,6 +132,7 @@ def _factory( ) -> HarnessRun: return HarnessRun( local_path=local_path, + framework_name="openssf-baseline", level=level, answer_resolver=answer_resolver or AnswerResolver(), llm_step=mock_llm_step, diff --git a/tests/darnit/harness/fixtures/minimal_llm_repo/.baseline.toml b/tests/darnit/harness/fixtures/minimal_llm_repo/.baseline.toml deleted file mode 100644 index 43012b51..00000000 --- a/tests/darnit/harness/fixtures/minimal_llm_repo/.baseline.toml +++ /dev/null @@ -1,11 +0,0 @@ -extends = "openssf-baseline" - -# Disable everything except our STAGE1-REF-* reference control so the LLM -# dispatch path is exercised without every OSPS control running (which -# would either need lots of setup or produce lots of noise). The single -# STAGE1-REF-SECURITY-01 control from feature 025 has the exact shape we -# need: suggestive llm_extract + dispositive file_exists. - -[audit_profiles.stage1_only] -description = "Stage 1 reference control only -- for feature 026 harness tests" -tags = { "stage1-ref" = true } diff --git a/tests/darnit/harness/test_cli.py b/tests/darnit/harness/test_cli.py index 537a318d..8ff45ab2 100644 --- a/tests/darnit/harness/test_cli.py +++ b/tests/darnit/harness/test_cli.py @@ -82,7 +82,7 @@ def test_exit_code_audit_failures_when_fail_present( ) -> None: """T037: fixture with FAIL result -> exit 1, stderr names FAIL count.""" exit_code, _stdout, _stderr, records = _invoke_cli( - [str(minimal_llm_repo_tree), "--level", "1"], + [str(minimal_llm_repo_tree), "--framework", "openssf-baseline", "--level", "1"], capsys, caplog, mock_llm=mock_llm_step, @@ -124,7 +124,7 @@ def test_missing_api_key_fails_fast_before_any_control_ran( saved_handlers = list(darnit_logger.handlers) saved_level = darnit_logger.level try: - exit_code = darnit_main(argv=["harness", str(minimal_llm_repo_tree)]) + exit_code = darnit_main(argv=["harness", str(minimal_llm_repo_tree), "--framework", "openssf-baseline"]) finally: darnit_logger.handlers[:] = saved_handlers darnit_logger.setLevel(saved_level) @@ -157,7 +157,7 @@ def test_missing_repo_path_exits_setup_error( capsys: pytest.CaptureFixture[str], caplog: pytest.LogCaptureFixture, ) -> None: - """CLI-1: missing / no .baseline.toml -> exit 2.""" + """CLI-1: missing repository path -> exit 2.""" exit_code, _stdout, _stderr, records = _invoke_cli( [str(tmp_path / "nonexistent")], capsys, @@ -189,7 +189,7 @@ def test_stderr_summary_matches_grep_pattern( """ # Case 1: success/failure path (audit runs). _exit1, _stdout1, _stderr1, records1 = _invoke_cli( - [str(minimal_llm_repo_tree), "--level", "1"], + [str(minimal_llm_repo_tree), "--framework", "openssf-baseline", "--level", "1"], capsys, caplog, mock_llm=mock_llm_step, @@ -198,7 +198,7 @@ def test_stderr_summary_matches_grep_pattern( monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) caplog.clear() _exit2, _stdout2, _stderr2, records2 = _invoke_cli( - [str(minimal_llm_repo_tree)], + [str(minimal_llm_repo_tree), "--framework", "openssf-baseline"], capsys, caplog, ) @@ -243,7 +243,7 @@ def test_stdout_clean_when_output_flag_used( """CLI-16: --output writes to file; stdout is empty.""" output_path = tmp_path / "report.md" _exit, stdout, _stderr, _records = _invoke_cli( - [str(minimal_llm_repo_tree), "--level", "1", "--output", str(output_path)], + [str(minimal_llm_repo_tree), "--framework", "openssf-baseline", "--level", "1", "--output", str(output_path)], capsys, caplog, mock_llm=mock_llm_step, @@ -299,7 +299,7 @@ def test_api_key_never_appears_in_stderr( # Success/failure path _e1, stdout1, _s1, records1 = _invoke_cli( - [str(minimal_llm_repo_tree), "--level", "1"], + [str(minimal_llm_repo_tree), "--framework", "openssf-baseline", "--level", "1"], capsys, caplog, mock_llm=mock_llm_step, @@ -333,7 +333,7 @@ def test_interactive_with_non_tty_stdin_fails_fast( start = time.monotonic() exit_code, _stdout, _stderr, records = _invoke_cli( - [str(minimal_llm_repo_tree), "--interactive", "--level", "1"], + [str(minimal_llm_repo_tree), "--framework", "openssf-baseline", "--interactive", "--level", "1"], capsys, caplog, mock_llm=mock_llm_step, @@ -379,7 +379,7 @@ def _fail_open(path: object, *args: object, **kwargs: object) -> object: monkeypatch.setattr(builtins, "open", _fail_open) exit_code, _stdout, _stderr, records = _invoke_cli( - [str(minimal_llm_repo_tree), "--interactive", "--level", "1"], + [str(minimal_llm_repo_tree), "--framework", "openssf-baseline", "--interactive", "--level", "1"], capsys, caplog, mock_llm=mock_llm_step, diff --git a/tests/darnit/server/test_harness_loop_mcp.py b/tests/darnit/server/test_harness_loop_mcp.py index f53567bb..4fa3ceaf 100644 --- a/tests/darnit/server/test_harness_loop_mcp.py +++ b/tests/darnit/server/test_harness_loop_mcp.py @@ -92,7 +92,9 @@ def test_mcp_walks_loop_to_termination(tmp_path: Path) -> None: from darnit.agent.graph import remediate fixture = _copy_minimal_repo(tmp_path) - state_dict: dict[str, Any] = HarnessState(local_path=str(fixture)).model_dump(mode="json") + state_dict: dict[str, Any] = HarnessState(local_path=str(fixture), framework_name="testchecks").model_dump( + mode="json" + ) for _ in range(20): # safety bound; loop should terminate well before plan_dict = _run(run_next_action_tool(state_dict)) @@ -144,7 +146,7 @@ def test_mcp_equals_direct_equals_cli(tmp_path: Path) -> None: fixture = _copy_minimal_repo(tmp_path) # Path 1: direct-Python (next_action/submit_result loop) - direct_state = HarnessState(local_path=str(fixture)) + direct_state = HarnessState(local_path=str(fixture), framework_name="testchecks") while True: plan = next_action(direct_state) if plan is None: @@ -190,7 +192,7 @@ def test_mcp_equals_direct_equals_cli(tmp_path: Path) -> None: break # Path 2: cmd_run-style CLI loop - cli_state = AuditState(local_path=str(fixture)) + cli_state = AuditState(local_path=str(fixture), framework_name="testchecks") fb = get_feedback_handler("noninteractive") cli_state = audit(cli_state) for _ in range(10): @@ -215,7 +217,9 @@ def test_mcp_equals_direct_equals_cli(tmp_path: Path) -> None: break # Path 3: MCP tools driving through JSON round-trips at every step - mcp_state_dict: dict[str, Any] = HarnessState(local_path=str(fixture)).model_dump(mode="json") + mcp_state_dict: dict[str, Any] = HarnessState(local_path=str(fixture), framework_name="testchecks").model_dump( + mode="json" + ) for _ in range(20): plan_dict = _run(run_next_action_tool(mcp_state_dict)) if plan_dict is None: From b6ede19461415be0bc6b8469990f4303629e1557 Mon Sep 17 00:00:00 2001 From: Michael Lieberman Date: Sun, 4 Oct 2026 20:44:51 -0400 Subject: [PATCH 3/5] feat!: stop reading a repository's .baseline.toml darnit no longer reads .baseline.toml for anything: no per-control status/reason claims (even for a trusted repository), no extends, no settings or custom controls. When the file is present, the audit logs one WARNING and adds the same notice to the report's warnings, pointing at `darnit config migrate`; its keys are no longer listed in ignored_repository_settings. `darnit config migrate` reads the file itself and is unchanged (framework-design 14.4, FR-023). Removed with it: BASELINE_TOML_DEPRECATION_ACTIVE, load_user_config (and its trusted path), load_user_config_with_report, validate_user_config, deep_merge, darnit.config.user_schema (UserConfig, CustomControl, ControlGroup, ...), .baseline.toml claims in trust.assertions, the user side of merge_configs/merge_control, EffectiveControl status/is_applicable, EffectiveConfig settings and get_excluded_controls, the repo_path parameter of load_effective_config* and the control loaders, and tools.audit get_excluded_control_ids, get_adapter_for_control, and load_effective_audit_config. run_checks/run_sieve_audit `apply_user_config` is renamed `evaluate_claims`: it still gates .project/ claim evaluation. Tests: test_baseline_toml_removed.py covers the notice, no claims for a trusted repository, extends ignored, a planted pass not run, and migrate-then-audit. Tests of the removed loader and schema are deleted; merge-precedence tests now use operator configuration. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Michael Lieberman --- .../src/darnit_baseline/openssf-baseline.toml | 2 +- .../remediation/orchestrator.py | 2 +- .../src/darnit_baseline/tools.py | 3 +- .../darnit-csl/src/darnit_csl/mcp_tools.py | 2 +- .../src/darnit_example/tools.py | 2 +- .../src/darnit_plugins/__init__.py | 4 +- .../src/darnit_plugins/adapters/echo.py | 2 +- .../src/darnit_plugins/adapters/kusari.py | 2 +- packages/darnit-testchecks/tests/conftest.py | 17 - .../darnit-testchecks/tests/test_adapter.py | 92 +-- packages/darnit/src/darnit/agent/graph.py | 4 +- packages/darnit/src/darnit/cli.py | 28 +- packages/darnit/src/darnit/config/__init__.py | 31 +- .../src/darnit/config/context_resolve.py | 3 +- .../src/darnit/config/context_storage.py | 8 +- .../src/darnit/config/control_loader.py | 32 +- .../src/darnit/config/framework_schema.py | 8 +- packages/darnit/src/darnit/config/merger.py | 678 +++--------------- .../src/darnit/config/operator/migrate.py | 6 +- .../darnit/src/darnit/config/user_schema.py | 393 ---------- .../darnit/src/darnit/core/audit_cache.py | 2 +- packages/darnit/src/darnit/core/models.py | 2 +- packages/darnit/src/darnit/harness/driver.py | 10 +- .../src/darnit/server/tools/builtin_audit.py | 3 +- .../darnit/src/darnit/storage/backends.py | 4 +- packages/darnit/src/darnit/tools/audit.py | 132 +--- packages/darnit/src/darnit/tools/audit_org.py | 1 - .../darnit/src/darnit/trust/assertions.py | 40 +- scripts/csl_manual_remediate.py | 2 +- scripts/csl_onboard.py | 2 +- .../assertions/test_assertion_reporting.py | 44 +- .../test_context_value_assertions.py | 4 +- .../assertions/test_project_assertions.py | 30 - .../operator/test_baseline_deprecation.py | 226 ------ .../operator/test_baseline_toml_removed.py | 186 +++++ .../config/operator/test_effective_config.py | 45 +- .../test_repository_cannot_configure.py | 12 +- .../config/test_authority_validation_paths.py | 2 +- tests/darnit/config/test_merger.py | 270 +------ .../darnit/config/test_merger_mcp_servers.py | 18 +- tests/darnit/config/test_stores_config.py | 35 +- .../config/test_strict_framework_loading.py | 39 +- .../config/test_untrusted_repo_config.py | 181 ----- tests/darnit/config/test_user_schema.py | 397 ---------- tests/darnit/core/test_audit_cache.py | 2 - .../error_class_baseline/capture_baseline.py | 4 +- tests/darnit/sieve/baseline_capture.py | 2 +- tests/darnit/sieve/test_expr_failure.py | 2 +- .../darnit/sieve/test_handler_architecture.py | 4 +- tests/darnit/stores/test_us2_zero_config.py | 2 +- tests/darnit/test_audit_cache_store_wiring.py | 4 +- tests/darnit/test_audit_mapper_integration.py | 8 - .../test_plugin_handler_registration.py | 4 +- tests/darnit_baseline/corpus/runner.py | 2 +- .../test_baseline_authority.py | 2 +- .../test_handler_dispatch_integration.py | 15 +- tests/darnit_baseline/test_weak_evidence.py | 2 +- tests/darnit_csl/test_csl.py | 2 +- 58 files changed, 473 insertions(+), 2588 deletions(-) delete mode 100644 packages/darnit/src/darnit/config/user_schema.py delete mode 100644 tests/darnit/config/operator/test_baseline_deprecation.py create mode 100644 tests/darnit/config/operator/test_baseline_toml_removed.py delete mode 100644 tests/darnit/config/test_untrusted_repo_config.py delete mode 100644 tests/darnit/config/test_user_schema.py diff --git a/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml b/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml index 383d108b..bfdd1af3 100644 --- a/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml +++ b/packages/darnit-baseline/src/darnit_baseline/openssf-baseline.toml @@ -2,7 +2,7 @@ # Declarative configuration for OSPS v2026.02.19 compliance controls # # This file defines all 65 controls across 3 maturity levels. -# Users can override settings via .baseline.toml in their repository. +# Operators can override passes and add controls in operator configuration. [metadata] name = "openssf-baseline" diff --git a/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py b/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py index e076147e..838d0f38 100644 --- a/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py +++ b/packages/darnit-baseline/src/darnit_baseline/remediation/orchestrator.py @@ -1508,7 +1508,7 @@ def _recheck( wanted = set(control_ids) specs = [ spec - for spec in load_controls_from_effective(merge_configs(framework, None, operator_config.config)) + for spec in load_controls_from_effective(merge_configs(framework, operator_config.config)) if spec.control_id in wanted ] results, _ = audit_tools.run_sieve_audit( diff --git a/packages/darnit-baseline/src/darnit_baseline/tools.py b/packages/darnit-baseline/src/darnit_baseline/tools.py index 0489c7bf..4f68a031 100644 --- a/packages/darnit-baseline/src/darnit_baseline/tools.py +++ b/packages/darnit-baseline/src/darnit_baseline/tools.py @@ -176,7 +176,7 @@ def audit_openssf_baseline( # Load framework config try: - config = load_effective_config_by_name("openssf-baseline", repo_path, operator=operator_config.config) + config = load_effective_config_by_name("openssf-baseline", operator=operator_config.config) except Exception as e: return f"❌ Error loading framework: {e}" @@ -226,7 +226,6 @@ def audit_openssf_baseline( level=level, controls=controls, tags=tags_list, - apply_user_config=True, stop_on_llm=True, framework_name="openssf-baseline", operator_config=operator_config, diff --git a/packages/darnit-csl/src/darnit_csl/mcp_tools.py b/packages/darnit-csl/src/darnit_csl/mcp_tools.py index ccc2131f..b713fc82 100644 --- a/packages/darnit-csl/src/darnit_csl/mcp_tools.py +++ b/packages/darnit-csl/src/darnit_csl/mcp_tools.py @@ -308,7 +308,7 @@ def decided(value: str | None, key: str) -> str | None: results, _ = run_sieve_audit( owner="", repo="", local_path=str(repo), default_branch="main", - apply_user_config=False, framework_name="community-spec", stop_on_llm=False, + evaluate_claims=False, framework_name="community-spec", stop_on_llm=False, ) lines = [f"# CSL remediation for {repo.name}", "", f"Files written: {', '.join(written) if written else 'none'}"] diff --git a/packages/darnit-example/src/darnit_example/tools.py b/packages/darnit-example/src/darnit_example/tools.py index af00d4c0..e66069c1 100644 --- a/packages/darnit-example/src/darnit_example/tools.py +++ b/packages/darnit-example/src/darnit_example/tools.py @@ -23,7 +23,7 @@ def _load_all_controls(repo_path: Path, level: int): from darnit.core.discovery import get_implementation from darnit.sieve.registry import get_control_registry - config = load_effective_config_by_name("example-hygiene", repo_path) + config = load_effective_config_by_name("example-hygiene") toml_controls = load_controls_from_effective(config) impl = get_implementation("example-hygiene") diff --git a/packages/darnit-plugins/src/darnit_plugins/__init__.py b/packages/darnit-plugins/src/darnit_plugins/__init__.py index c4163541..7e40c27a 100644 --- a/packages/darnit-plugins/src/darnit_plugins/__init__.py +++ b/packages/darnit-plugins/src/darnit_plugins/__init__.py @@ -9,9 +9,9 @@ Usage: Adapters are automatically discovered via Python entry points. - Reference them by name in your framework TOML or user config:: + Reference them by name in your framework TOML:: - # In framework.toml or .baseline.toml + # In framework.toml [controls."CTRL-001"] check = { adapter = "kusari" } diff --git a/packages/darnit-plugins/src/darnit_plugins/adapters/echo.py b/packages/darnit-plugins/src/darnit_plugins/adapters/echo.py index 2d05bf29..25f82482 100644 --- a/packages/darnit-plugins/src/darnit_plugins/adapters/echo.py +++ b/packages/darnit-plugins/src/darnit_plugins/adapters/echo.py @@ -35,7 +35,7 @@ Example: Testing a framework's control routing:: - # .baseline.toml + # framework.toml [controls."MY-CTRL-01"] check = { adapter = "echo", config = { status = "PASS" } } diff --git a/packages/darnit-plugins/src/darnit_plugins/adapters/kusari.py b/packages/darnit-plugins/src/darnit_plugins/adapters/kusari.py index 619a363d..bdb7d2ad 100644 --- a/packages/darnit-plugins/src/darnit_plugins/adapters/kusari.py +++ b/packages/darnit-plugins/src/darnit_plugins/adapters/kusari.py @@ -17,7 +17,7 @@ Usage: The adapter can be referenced by name in framework configs:: - # In framework.toml or .baseline.toml + # In framework.toml [controls."OSPS-VM-05.02"] check = { adapter = "kusari" } diff --git a/packages/darnit-testchecks/tests/conftest.py b/packages/darnit-testchecks/tests/conftest.py index 4995bffe..f6e6c640 100644 --- a/packages/darnit-testchecks/tests/conftest.py +++ b/packages/darnit-testchecks/tests/conftest.py @@ -71,20 +71,3 @@ def repo_with_violations(temp_repo: Path) -> Path: ) return temp_repo - - -@pytest.fixture -def user_config_content() -> str: - """Sample user config that skips some controls.""" - return ''' -version = "1.0" -extends = "testchecks" - -[controls."TEST-QA-01"] -status = "n/a" -reason = "TODOs are acceptable" - -[controls."TEST-QA-02"] -status = "n/a" -reason = "Print statements OK in scripts" -''' diff --git a/packages/darnit-testchecks/tests/test_adapter.py b/packages/darnit-testchecks/tests/test_adapter.py index fe63514e..364dff8b 100644 --- a/packages/darnit-testchecks/tests/test_adapter.py +++ b/packages/darnit-testchecks/tests/test_adapter.py @@ -1,4 +1,4 @@ -"""Tests for the TrivialCheckAdapter and user config integration.""" +"""Tests for the TrivialCheckAdapter and the testchecks effective configuration.""" import sys from pathlib import Path @@ -7,7 +7,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent / "darnit" / "src")) sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -from darnit.config.merger import load_effective_config, load_user_config +from darnit.config.merger import load_effective_config from darnit.core.models import CheckStatus from darnit_testchecks import get_framework_path @@ -215,101 +215,37 @@ def test_remediate_unknown_control(self, temp_repo: Path): assert "No remediation available" in result.message -class TestUserConfigIntegration: - """Tests for user config override integration.""" +class TestEffectiveConfig: + """Tests for the framework's effective configuration.""" - def test_load_user_config(self, temp_repo: Path, user_config_content: str): - """Should load user config from .baseline.toml.""" - (temp_repo / ".baseline.toml").write_text(user_config_content) + def test_all_controls_load(self): + """Every control in the framework TOML is in the effective config.""" + effective = load_effective_config(get_framework_path()) - user_config = load_user_config(temp_repo) - - assert user_config is not None - assert user_config.extends == "testchecks" - assert "TEST-QA-01" in user_config.controls - assert "TEST-QA-02" in user_config.controls - - def test_effective_config_excludes_controls(self, temp_repo: Path, user_config_content: str): - """Effective config should exclude n/a controls.""" - (temp_repo / ".baseline.toml").write_text(user_config_content) - - effective = load_effective_config(get_framework_path(), temp_repo) - - excluded = effective.get_excluded_controls() - assert "TEST-QA-01" in excluded - assert "TEST-QA-02" in excluded - assert excluded["TEST-QA-01"] == "TODOs are acceptable" - - def test_effective_config_applicable_controls(self, temp_repo: Path, user_config_content: str): - """Effective config should have correct applicable controls.""" - (temp_repo / ".baseline.toml").write_text(user_config_content) - - effective = load_effective_config(get_framework_path(), temp_repo) - - # 12 total - 2 excluded = 10 applicable - applicable = [c for c in effective.controls.values() if c.is_applicable()] - assert len(applicable) == 10 - - # Excluded controls should not be applicable - assert not effective.controls["TEST-QA-01"].is_applicable() - assert not effective.controls["TEST-QA-02"].is_applicable() - - # Other controls should be applicable - assert effective.controls["TEST-DOC-01"].is_applicable() - assert effective.controls["TEST-SEC-01"].is_applicable() - - def test_effective_config_without_user_config(self, temp_repo: Path): - """Effective config should work without user config.""" - effective = load_effective_config(get_framework_path(), temp_repo) - - # All 12 controls should be applicable - applicable = [c for c in effective.controls.values() if c.is_applicable()] - assert len(applicable) == 12 + assert len(effective.controls) == 12 class TestEndToEnd: """End-to-end tests combining multiple components.""" - def test_full_audit_flow(self, complete_repo: Path, user_config_content: str): - """Should run full audit with user config.""" - # Add user config - (complete_repo / ".baseline.toml").write_text(user_config_content) - - # Load effective config - effective = load_effective_config(get_framework_path(), complete_repo) + def test_full_audit_flow(self, complete_repo: Path): + """Every control passes on a complete repository.""" + effective = load_effective_config(get_framework_path()) - # Get applicable controls - applicable_ids = [ - cid for cid, ctrl in effective.controls.items() - if ctrl.is_applicable() - ] - - # Run checks adapter = TrivialCheckAdapter() results = adapter.check_batch( - control_ids=applicable_ids, + control_ids=list(effective.controls), owner="", repo="test", local_path=str(complete_repo), config={}, ) - # All applicable controls should pass for result in results: assert result.status == CheckStatus.PASS, f"{result.control_id}: {result.message}" - def test_audit_violations_with_exclusions(self, repo_with_violations: Path, user_config_content: str): - """Should pass when violations are excluded.""" - # Add user config that excludes the violated controls - (repo_with_violations / ".baseline.toml").write_text(user_config_content) - - effective = load_effective_config(get_framework_path(), repo_with_violations) - - # These should be excluded - assert not effective.controls["TEST-QA-01"].is_applicable() - assert not effective.controls["TEST-QA-02"].is_applicable() - - # But TEST-SEC-01 is not excluded, should still fail + def test_audit_violations(self, repo_with_violations: Path): + """A hard-coded secret fails TEST-SEC-01.""" adapter = TrivialCheckAdapter() result = adapter.check( control_id="TEST-SEC-01", diff --git a/packages/darnit/src/darnit/agent/graph.py b/packages/darnit/src/darnit/agent/graph.py index 111899b4..a427190b 100644 --- a/packages/darnit/src/darnit/agent/graph.py +++ b/packages/darnit/src/darnit/agent/graph.py @@ -56,8 +56,7 @@ def audit(state: AuditState) -> AuditState: """Run a compliance audit and store results in state. - Uses the framework identified by state.framework_name (or auto-resolved - from .baseline.toml) and injects any context values already confirmed so + Uses the framework identified by state.framework_name and injects any context values already confirmed so that context-dependent controls can be resolved correctly on re-runs. Args: @@ -88,7 +87,6 @@ def audit(state: AuditState) -> AuditState: default_branch=default_branch, level=state.level, stop_on_llm=True, - apply_user_config=True, framework_name=state.framework_name, target=state.target, ) diff --git a/packages/darnit/src/darnit/cli.py b/packages/darnit/src/darnit/cli.py index 5e8e2aea..cfd69f8f 100644 --- a/packages/darnit/src/darnit/cli.py +++ b/packages/darnit/src/darnit/cli.py @@ -286,12 +286,12 @@ def cmd_audit(args: argparse.Namespace) -> int: if args.framework: framework_path = Path(args.framework) if framework_path.exists(): - config = load_effective_config(framework_path, repo_path, operator=operator) + config = load_effective_config(framework_path, operator=operator) else: # Try as framework name - config = load_effective_config_by_name(args.framework, repo_path, operator=operator) + config = load_effective_config_by_name(args.framework, operator=operator) else: - config = load_effective_config_auto(repo_path, operator=operator) + config = load_effective_config_auto(operator=operator) except ValueError as e: logger.error(f"Failed to load framework: {e}") return 1 @@ -334,7 +334,6 @@ def cmd_audit(args: argparse.Namespace) -> int: default_branch=default_branch, level=3, controls=controls, - apply_user_config=True, stop_on_llm=True, # Issue #427: the framework name has to reach the audit driver, not # just the control loader above. Without it the driver cannot @@ -386,11 +385,11 @@ def cmd_plan(args: argparse.Namespace) -> int: if args.framework: framework_path = Path(args.framework) if framework_path.exists(): - config = load_effective_config(framework_path, repo_path if repo_path.exists() else None) + config = load_effective_config(framework_path) else: - config = load_effective_config_by_name(args.framework, repo_path if repo_path.exists() else None) + config = load_effective_config_by_name(args.framework) else: - config = load_effective_config_auto(repo_path) + config = load_effective_config_auto() except (ValueError, FileNotFoundError) as e: logger.error(f"Failed to load framework: {e}") return 1 @@ -446,23 +445,12 @@ def cmd_plan(args: argparse.Namespace) -> int: logger.info(f"Level {level} ({len(shown_controls)} controls):") for cid, ctrl in shown_controls: - if ctrl.is_applicable(): - adapter = ctrl.check_adapter - logger.info(f" • {cid}: {ctrl.name} [adapter: {adapter}]") - else: - logger.info(f" - {cid}: {ctrl.name} [skipped: {ctrl.status_reason}]") + logger.info(f" • {cid}: {ctrl.name} [adapter: {ctrl.check_adapter}]") total_shown += len(shown_controls) if total_filtered > 0: logger.info(f"({total_filtered} controls filtered out)") - # Show excluded controls - excluded = config.get_excluded_controls() - if excluded: - logger.info(f"Excluded ({len(excluded)}):") - for cid, reason in excluded.items(): - logger.info(f" - {cid}: {reason}") - return 0 @@ -538,7 +526,7 @@ def cmd_init(args: argparse.Namespace) -> int: f"Select the framework per run with --framework {framework}.", ] if (repo_path / ".baseline.toml").exists(): - lines.extend(["", "This repository has a deprecated .baseline.toml; run `darnit config migrate` to move it."]) + lines.extend(["", "This repository has a .baseline.toml, which darnit no longer reads; run `darnit config migrate` to move it."]) sys.stdout.write("\n".join(lines) + "\n") return 0 diff --git a/packages/darnit/src/darnit/config/__init__.py b/packages/darnit/src/darnit/config/__init__.py index 9ac89ddd..e8bbdf9d 100644 --- a/packages/darnit/src/darnit/config/__init__.py +++ b/packages/darnit/src/darnit/config/__init__.py @@ -47,7 +47,7 @@ sync_discovered_to_config, ) -# Framework and user configuration schemas +# Framework configuration schema from .framework_schema import ( AdapterType, CheckConfig, @@ -91,19 +91,16 @@ from .merger import ( EffectiveConfig, EffectiveControl, - deep_merge, list_available_frameworks, load_effective_config, load_effective_config_auto, load_effective_config_by_name, load_framework_by_name, load_framework_config, - load_user_config, merge_configs, merge_control, resolve_framework_path, validate_framework_config, - validate_user_config, ) from .resolver import ( resolve_file_for_control, @@ -153,20 +150,6 @@ get_path_from_ref, parse_resource_ref, ) -from .user_schema import ( - ControlGroup, - CustomControl, - UserConfig, - UserSettings, - create_user_config, - create_user_config_with_kusari, -) -from .user_schema import ( - ControlOverride as UserControlOverride, -) -from .user_schema import ( - ControlStatus as UserControlStatus, -) __all__ = [ # Enums @@ -256,23 +239,12 @@ # Context definitions (interactive context collection) "ContextDefinitionConfig", "FrameworkContextConfig", - # User configuration schema - "UserConfig", - "UserSettings", - "UserControlOverride", - "ControlGroup", - "CustomControl", - "UserControlStatus", - "create_user_config", - "create_user_config_with_kusari", # Configuration merger "EffectiveConfig", "EffectiveControl", "merge_configs", "merge_control", - "deep_merge", "load_framework_config", - "load_user_config", "load_effective_config", "load_effective_config_by_name", "load_effective_config_auto", @@ -280,7 +252,6 @@ "resolve_framework_path", "list_available_frameworks", "validate_framework_config", - "validate_user_config", # Control loader "load_controls_from_effective", "load_controls_from_framework", diff --git a/packages/darnit/src/darnit/config/context_resolve.py b/packages/darnit/src/darnit/config/context_resolve.py index aaf6719a..dab406d4 100644 --- a/packages/darnit/src/darnit/config/context_resolve.py +++ b/packages/darnit/src/darnit/config/context_resolve.py @@ -15,7 +15,6 @@ from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field from datetime import UTC, date, datetime, timedelta -from pathlib import Path from typing import TYPE_CHECKING, Any from darnit.config.context_keys import canonical_key, normalize_value, value_digest @@ -286,7 +285,7 @@ def _auto_accept_confidence(local_path: str) -> float: try: from darnit.config.merger import load_effective_config_auto - framework = load_effective_config_auto(Path(local_path))._framework_config + framework = load_effective_config_auto()._framework_config if framework is not None: return framework.context.auto_accept_confidence except Exception as exc: # noqa: BLE001 - a read must not fail on config diff --git a/packages/darnit/src/darnit/config/context_storage.py b/packages/darnit/src/darnit/config/context_storage.py index dccc3617..98b2a427 100644 --- a/packages/darnit/src/darnit/config/context_storage.py +++ b/packages/darnit/src/darnit/config/context_storage.py @@ -60,7 +60,7 @@ def get_context_definitions(local_path: str) -> dict[str, ContextDefinition]: framework configuration (e.g., openssf-baseline.toml [context] section). Args: - local_path: Path to the repository + local_path: Path to the repository (the framework is not read from it) Returns: Dict of context_key -> ContextDefinition @@ -68,7 +68,7 @@ def get_context_definitions(local_path: str) -> dict[str, ContextDefinition]: from darnit.config.merger import load_effective_config_auto try: - effective_config = load_effective_config_auto(local_path) + effective_config = load_effective_config_auto() # Access the underlying framework config framework = effective_config._framework_config if framework is None: @@ -89,7 +89,7 @@ def get_context_definitions_with_detect( (list of HandlerInvocation) from the TOML framework config. Args: - local_path: Path to the repository + local_path: Path to the repository (the framework is not read from it) Returns: Dict of context_key -> (ContextDefinition, detect_pipeline_or_None) @@ -97,7 +97,7 @@ def get_context_definitions_with_detect( from darnit.config.merger import load_effective_config_auto try: - effective_config = load_effective_config_auto(local_path) + effective_config = load_effective_config_auto() framework = effective_config._framework_config if framework is None: return {} diff --git a/packages/darnit/src/darnit/config/control_loader.py b/packages/darnit/src/darnit/config/control_loader.py index 819437aa..d4f90857 100644 --- a/packages/darnit/src/darnit/config/control_loader.py +++ b/packages/darnit/src/darnit/config/control_loader.py @@ -8,7 +8,7 @@ from darnit.config.merger import load_effective_config_by_name # Load and merge configs - config = load_effective_config_by_name("openssf-baseline", Path("/path/to/repo")) + config = load_effective_config_by_name("openssf-baseline", operator=operator) # Convert to executable ControlSpec objects controls = load_controls_from_config(config) @@ -424,7 +424,7 @@ def load_controls_from_effective(config: EffectiveConfig) -> list[ControlSpec]: """Load ControlSpec objects from effective configuration. This is the main entry point for loading controls from merged - framework + user configuration. + framework + operator configuration. Controls a repository claims are not applicable are still loaded: the claim is the repository's assertion, which the audit reports alongside @@ -471,7 +471,7 @@ def load_controls_from_effective(config: EffectiveConfig) -> list[ControlSpec]: def load_controls_from_framework(config: FrameworkConfig) -> list[ControlSpec]: """Load ControlSpec objects directly from framework configuration. - Use this when you want framework controls without user customization. + Use this when you want framework controls without operator overrides. Performs load-time validation and resolution of shared handlers, use_locator, on_pass auto-derivation, and reference validation. @@ -505,40 +505,26 @@ def load_controls_from_framework(config: FrameworkConfig) -> list[ControlSpec]: return controls -def load_controls_from_toml( - framework_path: Path, - repo_path: Path | None = None, -) -> list[ControlSpec]: - """Load controls from TOML files. - - Convenience function that loads framework TOML, optionally merges - with user .baseline.toml, and returns executable controls. +def load_controls_from_toml(framework_path: Path) -> list[ControlSpec]: + """Load executable controls from a framework TOML file. Args: framework_path: Path to framework TOML file - repo_path: Path to repository (for .baseline.toml) Returns: List of executable ControlSpec objects """ from .merger import load_effective_config - config = load_effective_config(framework_path, repo_path) + config = load_effective_config(framework_path) return load_controls_from_effective(config) -def load_controls_by_name( - framework_name: str, - repo_path: Path | None = None, -) -> list[ControlSpec]: - """Load controls by framework name. - - Resolves framework via entry points, merges with user config, - and returns executable controls. +def load_controls_by_name(framework_name: str) -> list[ControlSpec]: + """Load executable controls for a framework resolved via entry points. Args: framework_name: Framework identifier (e.g., "openssf-baseline") - repo_path: Path to repository (for .baseline.toml) Returns: List of executable ControlSpec objects @@ -548,7 +534,7 @@ def load_controls_by_name( """ from .merger import load_effective_config_by_name - config = load_effective_config_by_name(framework_name, repo_path) + config = load_effective_config_by_name(framework_name) return load_controls_from_effective(config) diff --git a/packages/darnit/src/darnit/config/framework_schema.py b/packages/darnit/src/darnit/config/framework_schema.py index 9b032138..5539926e 100644 --- a/packages/darnit/src/darnit/config/framework_schema.py +++ b/packages/darnit/src/darnit/config/framework_schema.py @@ -1197,8 +1197,8 @@ class StoreBlock(BaseModel): configuration keys through to that backend's ``__init__``. String values in the ``model_extra`` bag are passed through :func:`darnit.core.env_subst.substitute_dollar_vars` at load time so - secrets can be sourced from ``os.environ`` rather than committed in - ``.baseline.toml`` (FR-006). + secrets can be sourced from ``os.environ`` rather than written into + configuration files (FR-006). Example TOML:: @@ -1265,7 +1265,7 @@ class McpServerConfig(BaseModel): A control's ``handler = "mcp"`` pass declares ``server = ""``; that name MUST match a key under ``[mcp_servers.]`` in the - effective framework config (or in ``.baseline.toml``, which wins + effective framework config (or in operator configuration, which wins per-name per spec FR-016). Absence of the entry produces ERROR at audit time without spawning anything -- allowlist is the primary trust boundary. @@ -1739,7 +1739,7 @@ class FrameworkConfig(BaseModel): # Per-artifact persistence backend selection (feature 033). Any unset # field means "use the filesystem default" for that artifact class; - # a `.baseline.toml` block for a given kind fully replaces the + # an operator configuration block for a given kind fully replaces the # framework TOML block for that kind (per-kind replacement). stores: StoresConfig = Field(default_factory=StoresConfig) diff --git a/packages/darnit/src/darnit/config/merger.py b/packages/darnit/src/darnit/config/merger.py index e37ea567..2d088876 100644 --- a/packages/darnit/src/darnit/config/merger.py +++ b/packages/darnit/src/darnit/config/merger.py @@ -1,66 +1,24 @@ -"""Config merger for combining framework and user configurations. - -This module provides functionality to merge framework configurations with -user customizations, applying proper override semantics. - -Merge Rules: - 1. Scalar values: User overrides framework - 2. Objects/dicts: Deep merge (user keys override, framework keys preserved) - 3. Arrays/lists: User replaces framework entirely - 4. Special keys: - - status = "n/a" → Marks control as not applicable - - check = {...} → Replaces entire check config - - extends = "..." → Specifies base framework - -Framework Resolution: - The ``extends`` field in user config can reference frameworks by name. - Resolution order: - - 1. Explicit path (if ``extends`` contains "/" or ends in ".toml"); only - honored when the operator trusts the repository's config (see - :func:`load_user_config`) - 2. Entry point lookup via :class:`~darnit.core.registry.PluginRegistry` +"""Config merger for combining framework and operator configurations. - Example:: - - # .baseline.toml - extends = "openssf-baseline" # Resolved via entry points +The framework TOML provides the controls; operator configuration (which +never comes from the audited repository) replaces passes, adds custom +controls, and overrides MCP servers and stores. Nothing in the audited +repository enters the effective configuration; a repository's legacy +``.baseline.toml`` is not read (framework-design 14.4). Example: Loading and merging configurations:: - from darnit.config.merger import ( - load_framework_config, - load_framework_by_name, - load_user_config, - load_effective_config, - ) - - # Load by path - framework = load_framework_config(Path("openssf-baseline.toml")) - - # Load by name (via entry points) - framework = load_framework_by_name("openssf-baseline") + from darnit.config.merger import load_effective_config_by_name - # Load user config and merge - effective = load_effective_config( - framework_path=Path("openssf-baseline.toml"), - repo_path=Path("/path/to/repo"), - ) - - # Or load by framework name - effective = load_effective_config_by_name( - framework_name="openssf-baseline", - repo_path=Path("/path/to/repo"), - ) + effective = load_effective_config_by_name("openssf-baseline", operator=operator) See Also: - :mod:`darnit.core.registry` for plugin discovery - :mod:`darnit.config.framework_schema` for framework config schema - - :mod:`darnit.config.user_schema` for user config schema + - :mod:`darnit.config.operator.schema` for operator configuration """ -import copy from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any @@ -82,11 +40,6 @@ McpServerConfig, StoresConfig, ) -from .user_schema import ( - ControlOverride, - ControlStatus, - UserConfig, -) if TYPE_CHECKING: from .operator.schema import OperatorConfig @@ -100,7 +53,7 @@ @dataclass class EffectiveControl: - """Merged control configuration with framework + user overrides. + """Merged control configuration: framework definition plus operator overrides. Level and domain are optional to support frameworks that don't use maturity levels or domain categorization. Use the tags dict for @@ -116,11 +69,6 @@ class EffectiveControl: # Source tracking from_framework: bool = True - from_user: bool = False - - # Status - status: ControlStatus | None = None - status_reason: str | None = None # Check routing check_adapter: str = "builtin" @@ -150,14 +98,10 @@ class EffectiveControl: inferred_from: str | None = None on_pass: dict[str, Any] | None = None - def is_applicable(self) -> bool: - """Check if control should be evaluated.""" - return self.status not in (ControlStatus.NA, ControlStatus.DISABLED) - @dataclass class EffectiveConfig: - """Merged configuration combining framework and user configs. + """Merged configuration combining framework and operator configuration. This is the runtime configuration used by the audit engine. """ @@ -166,59 +110,38 @@ class EffectiveConfig: framework_version: str spec_version: str | None = None - # Merged adapters (framework + user) + # Framework adapters adapters: dict[str, AdapterConfig] = field(default_factory=dict) # Merged controls controls: dict[str, EffectiveControl] = field(default_factory=dict) - # Settings from user config - cache_results: bool = True - cache_ttl: int = 300 - timeout: int = 300 - - # Merged MCP-server allowlist: framework + .baseline.toml, with - # per-name replacement (spec FR-016). Empty dict is the pre-feature - # default and preserves backward compatibility. + # Merged MCP-server allowlist: framework + operator configuration, with + # per-name replacement (spec FR-016). mcp_servers: dict[str, "McpServerConfig"] = field(default_factory=dict) # Feature 033: merged per-artifact persistence backend selection. - # `.baseline.toml`'s `[stores.]` for a given kind fully + # Operator configuration's `[stores.]` for a given kind fully # replaces the framework TOML block for that kind (per-kind # replacement, disjoint kinds coexist). stores: "StoresConfig | None" = None - # Source configs (for reference) + # Source config (for reference) _framework_config: FrameworkConfig | None = None - _user_config: UserConfig | None = None def get_controls_by_level(self, level: int) -> dict[str, EffectiveControl]: - """Get all applicable controls at a specific level. + """Get all controls at a specific level. Note: Controls without a level (level=None) are not included. """ - return { - cid: ctrl for cid, ctrl in self.controls.items() - if ctrl.level == level and ctrl.is_applicable() - } + return {cid: ctrl for cid, ctrl in self.controls.items() if ctrl.level == level} def get_controls_by_domain(self, domain: str) -> dict[str, EffectiveControl]: - """Get all applicable controls in a specific domain. + """Get all controls in a specific domain. Note: Controls without a domain (domain=None) are not included. """ - return { - cid: ctrl for cid, ctrl in self.controls.items() - if ctrl.domain == domain and ctrl.is_applicable() - } - - def get_excluded_controls(self) -> dict[str, str]: - """Get all non-applicable controls with reasons.""" - return { - cid: ctrl.status_reason or "No reason provided" - for cid, ctrl in self.controls.items() - if not ctrl.is_applicable() - } + return {cid: ctrl for cid, ctrl in self.controls.items() if ctrl.domain == domain} def get_adapter(self, name: str) -> AdapterConfig | None: """Get adapter configuration by name.""" @@ -230,148 +153,75 @@ def get_adapter(self, name: str) -> AdapterConfig | None: # ============================================================================= -def deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: - """Deep merge two dictionaries. - - Rules: - - Scalar values: override replaces base - - Dicts: recursive merge - - Lists: override replaces base entirely - - Args: - base: Base dictionary (from framework) - override: Override dictionary (from user) - - Returns: - Merged dictionary - """ - result = copy.deepcopy(base) - - for key, value in override.items(): - if key in result and isinstance(result[key], dict) and isinstance(value, dict): - # Recursive merge for nested dicts - result[key] = deep_merge(result[key], value) - else: - # Override scalar or list values - result[key] = copy.deepcopy(value) - - return result - - def merge_control( control_id: str, - framework_control: ControlConfig | None, - user_override: ControlOverride | None, + framework_control: ControlConfig, defaults: FrameworkDefaults, ) -> EffectiveControl: - """Merge a single control's framework config with user override. + """Build a control's effective configuration from its definition. Args: control_id: Control identifier - framework_control: Framework definition (may be None for custom controls) - user_override: User override (may be None) + framework_control: The control's definition (framework TOML or an + operator custom control) defaults: Framework defaults Returns: - Merged EffectiveControl + EffectiveControl """ - # Start with framework config or create minimal for custom control - if framework_control: - # Build tags dict - start with explicit tags, then add level/domain if present - tags = dict(framework_control.tags) if framework_control.tags else {} - if framework_control.level is not None: - tags["level"] = framework_control.level - if framework_control.domain is not None: - tags["domain"] = framework_control.domain - if framework_control.security_severity is not None: - tags["security_severity"] = framework_control.security_severity - - effective = EffectiveControl( - control_id=control_id, - name=framework_control.name, - level=framework_control.level, - domain=framework_control.domain, - description=framework_control.description, - from_framework=True, - check_adapter=defaults.check_adapter, - remediation_adapter=defaults.remediation_adapter, - tags=tags, - security_severity=framework_control.security_severity, - docs_url=framework_control.docs_url, - when=framework_control.when, - depends_on=framework_control.depends_on, - inferred_from=framework_control.inferred_from, - on_pass=framework_control.on_pass.model_dump() if framework_control.on_pass else None, - ) - - # Apply framework check config - if framework_control.check: - effective.check_adapter = framework_control.check.adapter - effective.check_handler = framework_control.check.handler - effective.check_config = dict(framework_control.check.config) - - # Apply framework remediation config - if framework_control.remediation: - effective.remediation_config = dict(framework_control.remediation.config) - - # Store passes config for sieve (flat list of handler invocations) - # Resolve use_locator before dumping so handlers get files lists - if framework_control.passes: - from .control_loader import _resolve_handler_invocations - - locator_discover = None - if framework_control.locator and framework_control.locator.discover: - locator_discover = framework_control.locator.discover - - resolved = _resolve_handler_invocations( - framework_control.passes, - {}, # shared_handlers resolved separately - locator_discover, - control_id, - ) - effective.passes_config = [p.model_dump() for p in resolved] + # Build tags dict - start with explicit tags, then add level/domain if present + tags = dict(framework_control.tags) if framework_control.tags else {} + if framework_control.level is not None: + tags["level"] = framework_control.level + if framework_control.domain is not None: + tags["domain"] = framework_control.domain + if framework_control.security_severity is not None: + tags["security_severity"] = framework_control.security_severity + + effective = EffectiveControl( + control_id=control_id, + name=framework_control.name, + level=framework_control.level, + domain=framework_control.domain, + description=framework_control.description, + from_framework=True, + check_adapter=defaults.check_adapter, + remediation_adapter=defaults.remediation_adapter, + tags=tags, + security_severity=framework_control.security_severity, + docs_url=framework_control.docs_url, + when=framework_control.when, + depends_on=framework_control.depends_on, + inferred_from=framework_control.inferred_from, + on_pass=framework_control.on_pass.model_dump() if framework_control.on_pass else None, + ) - else: - # Custom control from user - name and description required, level/domain optional - effective = EffectiveControl( - control_id=control_id, - name=control_id, - description="User-defined control", - from_framework=False, - from_user=True, + # Apply framework check config + if framework_control.check: + effective.check_adapter = framework_control.check.adapter + effective.check_handler = framework_control.check.handler + effective.check_config = dict(framework_control.check.config) + + # Apply framework remediation config + if framework_control.remediation: + effective.remediation_config = dict(framework_control.remediation.config) + + # Store passes config for sieve (flat list of handler invocations) + # Resolve use_locator before dumping so handlers get files lists + if framework_control.passes: + from .control_loader import _resolve_handler_invocations + + locator_discover = None + if framework_control.locator and framework_control.locator.discover: + locator_discover = framework_control.locator.discover + + resolved = _resolve_handler_invocations( + framework_control.passes, + {}, # shared_handlers resolved separately + locator_discover, + control_id, ) - - # Apply user overrides - if user_override: - effective.from_user = True - - # Status override - if user_override.status: - effective.status = user_override.status - effective.status_reason = user_override.reason - - # Check override - if user_override.check: - effective.check_adapter = user_override.check.adapter - if user_override.check.handler: - effective.check_handler = user_override.check.handler - if user_override.check.config: - effective.check_config = deep_merge( - effective.check_config, - user_override.check.config, - ) - - # Remediation override - if user_override.remediation: - if user_override.remediation.config: - effective.remediation_config = deep_merge( - effective.remediation_config, - user_override.remediation.config, - ) - - # Passes override (advanced) — flat list of handler invocations - if user_override.passes: - effective.passes_config = [p.model_dump() for p in user_override.passes] + effective.passes_config = [p.model_dump() for p in resolved] return effective @@ -387,17 +237,15 @@ def ensure_framework_allowed(framework_name: str, operator: "OperatorConfig | No def merge_configs( framework: FrameworkConfig, - user: UserConfig | None = None, operator: "OperatorConfig | None" = None, ) -> EffectiveConfig: - """Merge framework, user, and operator configurations into effective config. + """Merge framework and operator configuration into effective config. Operator configuration is applied last: its pass overrides, custom controls, MCP servers, and stores win over the framework's. Args: framework: Framework configuration - user: User configuration (optional) operator: Operator configuration (optional) Returns: @@ -408,83 +256,39 @@ def merge_configs( """ ensure_framework_allowed(framework.metadata.name, operator) - # Start with framework metadata effective = EffectiveConfig( framework_name=framework.metadata.name, framework_version=framework.metadata.version, spec_version=framework.metadata.spec_version, _framework_config=framework, - _user_config=user, ) - # Merge adapters (framework first, then user overrides) effective.adapters = dict(framework.adapters) - if user: - for name, adapter in user.adapters.items(): - effective.adapters[name] = adapter - - # Merge MCP-server allowlist (spec FR-016). - # Precedence: framework provides the base; each key present in - # `.baseline.toml` REPLACES the framework's block for that name - # entirely (no deep merge within a block; the operator's entry is - # authoritative). Disjoint names coexist. + + # Merge MCP-server allowlist (spec FR-016): each operator entry REPLACES + # the framework's block for that name entirely; disjoint names coexist. effective.mcp_servers = dict(framework.mcp_servers) - if user: - for name, srv in user.mcp_servers.items(): - effective.mcp_servers[name] = srv if operator: effective.mcp_servers.update(operator.mcp_servers) - # Merge persistence backend selection (feature 033). - # Per-kind replacement: `.baseline.toml`'s [stores.] block for - # a given kind fully replaces the framework TOML block for that - # kind. Disjoint kinds coexist. + # Merge persistence backend selection (feature 033): per-kind + # replacement, operator block over framework block. from .framework_schema import StoresConfig as _StoresConfig merged_stores_data: dict[str, Any] = {} for kind in ("project", "attestation", "report", "cache"): fw_block = getattr(framework.stores, kind, None) - user_block = getattr(user.stores, kind, None) if user else None operator_block = getattr(operator.stores, kind, None) if operator else None - block = next((b for b in (operator_block, user_block, fw_block) if b is not None), None) + block = operator_block if operator_block is not None else fw_block if block is not None: merged_stores_data[kind] = block effective.stores = _StoresConfig.model_construct(**merged_stores_data) - # Apply user settings - if user: - effective.cache_results = user.settings.cache_results - effective.cache_ttl = user.settings.cache_ttl - effective.timeout = user.settings.timeout - - # Collect all control IDs, preserving declaration order (issue #428). - # A `set` here randomized iteration order per PYTHONHASHSEED, so the - # same repo audited twice listed its controls differently. `dict` keys - # already carry TOML declaration order, which groups controls by domain - # the way the framework author wrote them -- so dedup through - # `dict.fromkeys` rather than sorting, which would discard that grouping. - # Framework controls first, then any user-only additions. - all_control_ids: list[str] = list( - dict.fromkeys([*framework.controls, *(user.controls if user else [])]) - ) - - # Merge each control - for control_id in all_control_ids: - framework_control = framework.controls.get(control_id) - user_override = user.get_control_override(control_id) if user else None - - # Handle custom controls (user-defined, not in framework) - if not framework_control and user: - user_control = user.controls.get(control_id) - if isinstance(user_control, dict): - # Check if this is a custom control definition - if all(k in user_control for k in ("name", "level", "domain")): - framework_control = _custom_control_config(control_id, user_control) - + # Controls in TOML declaration order (issue #428). + for control_id, framework_control in framework.controls.items(): effective.controls[control_id] = merge_control( control_id=control_id, framework_control=framework_control, - user_override=user_override, defaults=framework.defaults, ) @@ -493,7 +297,6 @@ def merge_configs( effective.controls[control_id] = merge_control( control_id=control_id, framework_control=control, - user_override=None, defaults=framework.defaults, ) effective.controls[control_id].steps_from_operator = True @@ -525,17 +328,6 @@ def _unknown_control_keys_message(source: str, unknown: list[tuple[Any, Any]]) - return f"{source}: {keys} (framework-design 2.3)" -def _custom_control_config(control_id: str, definition: dict[str, Any]) -> ControlConfig: - """A ``.baseline.toml`` custom control, held to the control schema like a framework file's (framework-design 2.3).""" - try: - return ControlConfig(**definition) - except ValidationError as e: - unknown = [(control_id, key) for (key,) in _unknown_keys(e, (), 1)] - if not unknown: - raise - raise ValueError(_unknown_control_keys_message(f"User configuration {USER_CONFIG_FILENAME}", unknown)) from e - - def _parse_framework_only(path: Path) -> FrameworkConfig: """Parse + template-validate a framework TOML — WITHOUT resolving composition. @@ -674,13 +466,10 @@ def load_framework_config(path: Path) -> FrameworkConfig: OPERATOR_CONFIGURATION_HOME = "operator configuration" PROJECT_ASSERTIONS_HOME = ".project/darnit.yaml" -FRAMEWORK_OPTION_HOME = "the --framework option" -USER_CONFIG_FILENAME = ".baseline.toml" - -# True for the .baseline.toml deprecation release (FR-021): per-control -# status/reason are still read as claims and every setting is warned about. -# Set to False in the following minor release to ignore the file (FR-023). -BASELINE_TOML_DEPRECATION_ACTIVE = True +# A repository's legacy configuration file. darnit never reads it for an +# audit; `darnit config migrate` reads it to move its contents +# (framework-design 14.4). +BASELINE_TOML = ".baseline.toml" # Files shaped like operator configuration that darnit never reads from an # audited repository; they are reported so their authors know where the @@ -690,6 +479,7 @@ def load_framework_config(path: Path) -> FrameworkConfig: _PROJECT_TOOL_KEYS = frozenset( {"operator", "plugins", "mcp_servers", "custom_controls", "stores", "llm", "trust", "policy", "adapters", "passes"} ) +_CLAIM_KEYS = frozenset({"status", "reason"}) @dataclass(frozen=True) @@ -701,167 +491,19 @@ class IgnoredSetting: new_home: str -def _new_home(key: str) -> str: - if key.startswith("controls.") and key.rsplit(".", 1)[-1] in ("status", "reason"): - return PROJECT_ASSERTIONS_HOME - return OPERATOR_CONFIGURATION_HOME - - -# Keys a repository's own .baseline.toml may set. -# Everything else can change what darnit executes, which servers, adapters, -# or stores it trusts, or which framework definition it loads. -_UNTRUSTED_TOP_LEVEL_KEYS = frozenset({"version", "extends", "settings", "controls"}) -_UNTRUSTED_CONTROL_KEYS = frozenset({"status", "reason"}) - - -def _restrict_untrusted_user_config(data: dict[str, Any]) -> tuple[dict[str, Any], list[str]]: - """Reduce a repository-supplied config to scope declarations only. - - The audited repository is controlled by whoever can write to it, not by - the operator running darnit. Its .baseline.toml may exclude controls - (status and reason, which are reported), pick a framework by registered - name, and tune settings. It may not supply passes, checks, adapters, - remediation or per-control config, custom controls, control groups, MCP - servers, stores, plugin trust settings, or a framework file by path. - """ - ignored: list[str] = [] - restricted: dict[str, Any] = {} - - for key, value in data.items(): - if key not in _UNTRUSTED_TOP_LEVEL_KEYS: - ignored.append(key) - continue - restricted[key] = value - - extends = restricted.get("extends") - if isinstance(extends, str) and ("/" in extends or "\\" in extends or extends.endswith(".toml")): - ignored.append("extends (path)") - del restricted["extends"] - - controls = restricted.get("controls") - if isinstance(controls, dict): - kept: dict[str, Any] = {} - for control_id, override in controls.items(): - if not isinstance(override, dict): - ignored.append(f"controls.{control_id}") - continue - for field in override: - if field not in _UNTRUSTED_CONTROL_KEYS: - ignored.append(f"controls.{control_id}.{field}") - scope = {k: v for k, v in override.items() if k in _UNTRUSTED_CONTROL_KEYS} - if scope: - kept[control_id] = scope - restricted["controls"] = kept - - return restricted, ignored - - -def load_user_config_with_report(repo_path: Path) -> tuple[UserConfig | None, list[IgnoredSetting]]: - """Load a repository's .baseline.toml as untrusted input and list what was ignored. - - Returns: - The restricted UserConfig (or None when there is no file) and one - IgnoredSetting per key that was not applied. - """ - config_path = Path(repo_path) / USER_CONFIG_FILENAME - if not BASELINE_TOML_DEPRECATION_ACTIVE or not config_path.exists(): - return None, [] - - with open(config_path, "rb") as f: - data = tomllib.load(f) - - restricted, ignored = _restrict_untrusted_user_config(data) - report = [IgnoredSetting(USER_CONFIG_FILENAME, key, _new_home(key)) for key in dict.fromkeys(ignored)] - return UserConfig(**restricted), report - - -def load_user_config(repo_path: Path, *, trusted: bool = False) -> UserConfig | None: - """Load user configuration from repository. - - Searches for .baseline.toml in the repository root. - - The file lives in the audited repository, so by default it is treated as - untrusted input: only per-control ``status``/``reason``, ``version``, - ``settings``, and ``extends`` naming a registered framework are honored, - and anything else is ignored with a warning. Settings that change what - darnit executes or trusts belong in operator configuration, which lives - outside the audited repository. - - Args: - repo_path: Path to repository - trusted: Honor the full file. Only for callers whose trust decision - comes from operator configuration, never from the repository. - - Returns: - Parsed UserConfig or None if not found - """ - config_path = Path(repo_path) / USER_CONFIG_FILENAME - - if not BASELINE_TOML_DEPRECATION_ACTIVE or not config_path.exists(): - return None - - if trusted: - with open(config_path, "rb") as f: - return UserConfig(**tomllib.load(f)) - - user, ignored = load_user_config_with_report(repo_path) - if ignored: - logger.warning( - "Ignoring settings in %s that can change what darnit executes or trusts: %s. " - "A repository's own configuration is untrusted input; settings like these " - "belong in operator configuration outside the audited repository.", - config_path, - ", ".join(sorted(s.key for s in ignored)), - ) - return user - - -def _baseline_toml_settings(data: dict[str, Any]) -> list[str]: - settings: list[str] = [] - for key, value in data.items(): - if key == "version": - continue - if key != "controls" or not isinstance(value, dict): - settings.append(key) - continue - for control_id, override in value.items(): - if not isinstance(override, dict): - settings.append(f"controls.{control_id}") - continue - custom = all(k in override for k in ("name", "level", "domain")) - fields = [f for f in override if not custom or f in _UNTRUSTED_CONTROL_KEYS] - settings.extend(f"controls.{control_id}.{field}" for field in fields) - if custom: - settings.append(f"controls.{control_id}") - return settings - - def baseline_toml_warnings(repo_path: Path) -> list[str]: - """Deprecation warnings for the audited repository's .baseline.toml (FR-021, FR-023). + """The notice for a ``.baseline.toml`` in the audited repository (FR-023). - During the deprecation release, one warning per setting naming where the - setting now belongs; afterwards, a single notice that the file was ignored. + darnit does not read the file; when it exists, the audit reports this one + notice, whatever the file contains. """ - path = Path(repo_path) / USER_CONFIG_FILENAME - if not path.is_file(): + if not (Path(repo_path) / BASELINE_TOML).is_file(): return [] - migrate = "run `darnit config migrate` to move per-control status and reason to " + PROJECT_ASSERTIONS_HOME - if not BASELINE_TOML_DEPRECATION_ACTIVE: - return [ - f"{USER_CONFIG_FILENAME} is no longer read and was ignored; {migrate}. " - f"Tool settings belong in {OPERATOR_CONFIGURATION_HOME}." - ] - try: - data = tomllib.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError): - return [f"{USER_CONFIG_FILENAME} is deprecated and could not be read; {migrate}."] - - warnings = [] - for setting in _baseline_toml_settings(data): - home = FRAMEWORK_OPTION_HOME if setting.startswith("extends") else _new_home(setting) - hint = " (run `darnit config migrate`)" if home == PROJECT_ASSERTIONS_HOME else "" - warnings.append(f"{USER_CONFIG_FILENAME} is deprecated: `{setting}` belongs in {home}{hint}.") - return warnings + return [ + f"{BASELINE_TOML} is no longer read and was ignored; run `darnit config migrate` to move its " + f"claims to {PROJECT_ASSERTIONS_HOME} and get an {OPERATOR_CONFIGURATION_HOME} fragment for " + "its tool settings." + ] def _operator_shaped_settings(repo_path: Path) -> list[IgnoredSetting]: @@ -897,7 +539,7 @@ def _project_extension_settings(repo_path: Path) -> list[IgnoredSetting]: for control_id, override in controls.items(): if isinstance(override, dict): keys.extend( - f"controls.{control_id}.{field}" for field in override if field not in _UNTRUSTED_CONTROL_KEYS + f"controls.{control_id}.{field}" for field in override if field not in _CLAIM_KEYS ) return [IgnoredSetting(_PROJECT_EXTENSION_FILE, key, OPERATOR_CONFIGURATION_HOME) for key in keys] @@ -909,36 +551,24 @@ def find_ignored_repository_settings(repo_path: Path) -> list[IgnoredSetting]: now belongs so the report can show what was ignored (feature 040). """ repo_path = Path(repo_path) - try: - _user, ignored = load_user_config_with_report(repo_path) - except (OSError, tomllib.TOMLDecodeError, ValueError): - ignored = [IgnoredSetting(USER_CONFIG_FILENAME, "*", OPERATOR_CONFIGURATION_HOME)] - return [*ignored, *_operator_shaped_settings(repo_path), *_project_extension_settings(repo_path)] + return [*_operator_shaped_settings(repo_path), *_project_extension_settings(repo_path)] def load_effective_config( framework_path: Path, - repo_path: Path | None = None, *, operator: "OperatorConfig | None" = None, ) -> EffectiveConfig: - """Load and merge framework and user configurations. + """Load a framework TOML and merge operator configuration into it. Args: framework_path: Path to framework TOML file - repo_path: Path to repository (for .baseline.toml) operator: Operator configuration to apply (optional) Returns: Merged EffectiveConfig """ - framework = load_framework_config(framework_path) - - user = None - if repo_path: - user = load_user_config(repo_path) - - return merge_configs(framework, user, operator) + return merge_configs(load_framework_config(framework_path), operator) # ============================================================================= @@ -1039,18 +669,13 @@ def list_available_frameworks() -> list[str]: def load_effective_config_by_name( framework_name: str, - repo_path: Path | None = None, *, operator: "OperatorConfig | None" = None, ) -> EffectiveConfig: - """Load and merge framework (by name) and user configurations. - - This is a convenience function that combines framework name resolution - with config loading and merging. + """Load a framework by name and merge operator configuration into it. Args: framework_name: Framework identifier (e.g., "openssf-baseline") - repo_path: Path to repository (for .baseline.toml) operator: Operator configuration to apply (optional) Returns: @@ -1058,40 +683,24 @@ def load_effective_config_by_name( Raises: ValueError: If framework not found - - Example: - >>> effective = load_effective_config_by_name( - ... "openssf-baseline", - ... Path("/path/to/repo"), - ... ) - >>> print(f"Loaded {len(effective.controls)} controls") """ - framework = load_framework_by_name(framework_name) - - user = None - if repo_path: - user = load_user_config(repo_path) - - return merge_configs(framework, user, operator) + return merge_configs(load_framework_by_name(framework_name), operator) def load_effective_config_auto( - repo_path: Path, framework_path: Path | None = None, framework_name: str | None = None, *, operator: "OperatorConfig | None" = None, ) -> EffectiveConfig: - """Load effective config with automatic framework resolution. + """Load effective config, resolving the framework. - Resolution order: + Resolution order (nothing in the audited repository selects it): 1. Explicit framework_path if provided 2. Explicit framework_name if provided - 3. ``extends`` field from user's .baseline.toml - 4. Default to "openssf-baseline" + 3. Default to "openssf-baseline" Args: - repo_path: Path to repository framework_path: Explicit path to framework TOML (optional) framework_name: Explicit framework name (optional) operator: Operator configuration to apply (optional) @@ -1101,36 +710,12 @@ def load_effective_config_auto( Raises: ValueError: If framework cannot be resolved - - Example: - >>> # Uses framework specified in .baseline.toml - >>> effective = load_effective_config_auto(Path("/path/to/repo")) - - >>> # Override with specific framework - >>> effective = load_effective_config_auto( - ... Path("/path/to/repo"), - ... framework_name="testchecks", - ... ) """ - # Load user config first to check for extends - user = load_user_config(repo_path) - - # Determine framework if framework_path: framework = load_framework_config(framework_path) elif framework_name: framework = load_framework_by_name(framework_name) - elif user and user.extends: - # Resolve from user config's extends field - path = resolve_framework_path(user.extends) - if path is None: - raise ValueError( - f"Framework '{user.extends}' specified in .baseline.toml " - f"not found. Ensure the framework package is installed." - ) - framework = load_framework_config(path) else: - # Default to openssf-baseline try: framework = load_framework_by_name("openssf-baseline") except ValueError: @@ -1139,7 +724,7 @@ def load_effective_config_auto( "Please install darnit-baseline or specify a framework." ) from None - return merge_configs(framework, user, operator) + return merge_configs(framework, operator) # ============================================================================= @@ -1181,48 +766,3 @@ def validate_framework_config(config: FrameworkConfig) -> list[str]: ) return errors - - -def validate_user_config( - user: UserConfig, - framework: FrameworkConfig | None = None, -) -> list[str]: - """Validate user configuration for common issues. - - Args: - user: User configuration to validate - framework: Framework to validate against (optional) - - Returns: - List of validation errors (empty if valid) - """ - errors = [] - - # Check adapter references - for control_id, override in user.controls.items(): - if isinstance(override, dict): - check = override.get("check", {}) - adapter = check.get("adapter") if isinstance(check, dict) else None - elif isinstance(override, ControlOverride) and override.check: - adapter = override.check.adapter - else: - adapter = None - - if adapter and adapter != "builtin": - if adapter not in user.adapters: - # Check framework adapters too - if not framework or adapter not in framework.adapters: - errors.append( - f"Control {control_id} references unknown adapter: {adapter}" - ) - - # Check control groups reference valid controls - if framework: - framework_controls = set(framework.controls.keys()) - for _group_name, group in user.control_groups.items(): - for control_id in group.controls: - if control_id not in framework_controls: - # Could be a custom control, so just warn - pass - - return errors diff --git a/packages/darnit/src/darnit/config/operator/migrate.py b/packages/darnit/src/darnit/config/operator/migrate.py index 1e776e93..6082ec3e 100644 --- a/packages/darnit/src/darnit/config/operator/migrate.py +++ b/packages/darnit/src/darnit/config/operator/migrate.py @@ -18,7 +18,7 @@ from pydantic import ValidationError from darnit.config.loader import _write_yaml_file, get_default_extension -from darnit.config.merger import USER_CONFIG_FILENAME +from darnit.config.merger import BASELINE_TOML from darnit.config.schema import BaselineExtension, ControlOverride PROJECT_FILE = Path(".project") / "darnit.yaml" @@ -177,11 +177,11 @@ def migrate_baseline_toml(repo: Path, *, force: bool = False) -> MigrationResult its schema. Nothing is written in that case. """ repo = Path(repo) - source = repo / USER_CONFIG_FILENAME + source = repo / BASELINE_TOML try: data = tomllib.loads(source.read_text(encoding="utf-8")) except FileNotFoundError as exc: - raise MigrationError(f"no {USER_CONFIG_FILENAME} in {repo}") from exc + raise MigrationError(f"no {BASELINE_TOML} in {repo}") from exc except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc: raise MigrationError(f"cannot read {source}: {exc}") from exc diff --git a/packages/darnit/src/darnit/config/user_schema.py b/packages/darnit/src/darnit/config/user_schema.py deleted file mode 100644 index 90771893..00000000 --- a/packages/darnit/src/darnit/config/user_schema.py +++ /dev/null @@ -1,393 +0,0 @@ -"""Pydantic models for user customization configuration (.baseline.toml). - -This module defines the schema for user-facing configuration files that allow -customization of compliance framework behavior for a specific repository. - -Schema Structure: - - version: Config schema version - - extends: Framework to inherit from - - settings: Global settings (cache, timeout) - - adapters: Custom adapter definitions - - controls: Control overrides and customizations - - control_groups: Batch configuration for multiple controls - -Example: - ```toml - # .baseline.toml - version = "1.0" - extends = "openssf-baseline" - - [settings] - cache_results = true - timeout = 300 - - [adapters.kusari] - type = "command" - command = "kusari" - - [controls."OSPS-VM-05.02"] - check = { adapter = "kusari" } - - [controls."OSPS-BR-02.01"] - status = "n/a" - reason = "Pre-release project" - ``` -""" - -from enum import Enum -from typing import Any - -from pydantic import BaseModel, ConfigDict, Field - -from .framework_schema import ( - AdapterConfig, - CheckConfig, - ControlConfig, - HandlerInvocation, - McpServerConfig, - RemediationConfig, - StoresConfig, -) - -# ============================================================================= -# Enums -# ============================================================================= - - -class ControlStatus(str, Enum): - """Override status values for controls.""" - NA = "n/a" # Not applicable - ENABLED = "enabled" # Explicitly enabled - DISABLED = "disabled" # Explicitly disabled (skip) - - -# ============================================================================= -# User Settings -# ============================================================================= - - -class UserSettings(BaseModel): - """Global settings for the user configuration.""" - # Caching - cache_results: bool = True - cache_ttl: int = 300 # seconds - - # Timeouts - timeout: int = 300 # default timeout for operations - - # Behavior - fail_on_error: bool = False # Fail audit if any check errors - parallel_checks: bool = True # Run independent checks in parallel - max_parallel: int = 5 # Maximum parallel operations - - model_config = ConfigDict(extra="allow") - - -# ============================================================================= -# Control Override -# ============================================================================= - - -class ControlOverride(BaseModel): - """User override for a specific control. - - Can override: - - status: Mark as n/a or disabled - - check: Use different adapter - - remediation: Use different remediation - - passes: Override verification passes - - Any control metadata - - Example: - ```toml - [controls."OSPS-VM-05.02"] - check = { adapter = "kusari" } - - [controls."OSPS-BR-02.01"] - status = "n/a" - reason = "No releases yet" - ``` - """ - # Override status - status: ControlStatus | None = None - reason: str | None = None # Required if status is n/a - - # Override check routing - check: CheckConfig | None = None - - # Override remediation routing - remediation: RemediationConfig | None = None - - # Override verification passes (advanced) — flat list of handler invocations - passes: list[HandlerInvocation] | None = None - - # Additional config passed to adapter - config: dict[str, Any] = Field(default_factory=dict) - - model_config = ConfigDict(extra="allow") - - -# ============================================================================= -# Control Group -# ============================================================================= - - -class ControlGroup(BaseModel): - """Batch configuration for multiple controls. - - Allows applying the same configuration to multiple controls at once. - - Example: - ```toml - [control_groups.vulnerability-management] - controls = ["OSPS-VM-05.02", "OSPS-VM-05.03"] - check = { adapter = "kusari" } - ``` - """ - controls: list[str] # List of control IDs - check: CheckConfig | None = None - remediation: RemediationConfig | None = None - config: dict[str, Any] = Field(default_factory=dict) - - model_config = ConfigDict(extra="allow") - - -# ============================================================================= -# Custom Control (User-Defined) -# ============================================================================= - - -class CustomControl(ControlConfig): - """User-defined custom control. - - Extends ControlConfig to allow users to add their own controls - beyond what the framework defines. - - Example: - ```toml - [controls."CUSTOM-SEC-01"] - name = "InternalSecurityReview" - level = 1 - domain = "SA" - description = "Require internal security review" - check = { adapter = "custom_script" } - ``` - """ - # Inherit everything from ControlConfig - # Additional fields for custom controls: - custom: bool = True # Marker that this is user-defined - - model_config = ConfigDict(extra="allow") - - -# ============================================================================= -# User Configuration -# ============================================================================= - - -class UserConfig(BaseModel): - """User customization configuration loaded from .baseline.toml. - - This is the root model for user configuration files in repository roots. - - Provides: - - Framework inheritance (extends) - - Control overrides and customizations - - Custom adapter definitions - - Custom control definitions - - Batch configuration via control groups - - Example: - ```toml - version = "1.0" - extends = "openssf-baseline" - - [settings] - cache_results = true - timeout = 300 - - [adapters.kusari] - type = "command" - command = "kusari" - output_format = "json" - - [adapters.custom_script] - type = "script" - command = "./scripts/check-compliance.sh" - - [controls."OSPS-VM-05.02"] - check = { adapter = "kusari" } - - [controls."OSPS-VM-05.03"] - check = { adapter = "kusari" } - - [controls."OSPS-BR-02.01"] - status = "n/a" - reason = "Pre-1.0 project, no releases yet" - - [control_groups.vulnerability-management] - controls = ["OSPS-VM-05.02", "OSPS-VM-05.03"] - check = { adapter = "kusari" } - - # Custom control - [controls."CUSTOM-SEC-01"] - name = "InternalSecurityReview" - level = 1 - domain = "SA" - description = "Require internal security review sign-off" - check = { adapter = "custom_script" } - ``` - """ - # Schema version - version: str = "1.0" - - # Framework to inherit from - extends: str | None = None # e.g., "openssf-baseline" - - # Global settings - settings: UserSettings = Field(default_factory=UserSettings) - - # Custom adapter definitions - adapters: dict[str, AdapterConfig] = Field(default_factory=dict) - - # Control overrides and custom controls - # Values can be ControlOverride (for overrides) or CustomControl (for new controls) - controls: dict[str, ControlOverride | CustomControl | dict[str, Any]] = Field( - default_factory=dict - ) - - # Control groups for batch configuration - control_groups: dict[str, ControlGroup] = Field(default_factory=dict) - - # Per-fleet MCP-server allowlist entries. Keys here fully replace the - # framework's ``[mcp_servers.]`` block of the same name at merge - # time (spec FR-016); disjoint names coexist. - mcp_servers: dict[str, McpServerConfig] = Field(default_factory=dict) - - # Per-fleet persistence backend selection (feature 033). Any kind - # set here fully replaces the framework's `[stores.]` block at - # merge time; unset kinds inherit from the framework. - stores: StoresConfig = Field(default_factory=StoresConfig) - - model_config = ConfigDict(extra="allow") - - # ========================================================================= - # Convenience Methods - # ========================================================================= - - def get_control_override(self, control_id: str) -> ControlOverride | None: - """Get override for a specific control, including from groups.""" - # Check direct override first - override = self.controls.get(control_id) - if override: - if isinstance(override, ControlOverride): - return override - elif isinstance(override, dict): - return ControlOverride(**override) - - # Check control groups - for group in self.control_groups.values(): - if control_id in group.controls: - return ControlOverride( - check=group.check, - remediation=group.remediation, - config=group.config, - ) - - return None - - def is_control_applicable(self, control_id: str) -> tuple: - """Check if control is applicable (not marked n/a or disabled). - - Returns: - Tuple of (is_applicable, reason_if_not) - """ - override = self.get_control_override(control_id) - if override and override.status in (ControlStatus.NA, ControlStatus.DISABLED): - return False, override.reason - return True, None - - def get_check_adapter(self, control_id: str) -> str | None: - """Get the adapter name for checking a control, if overridden.""" - override = self.get_control_override(control_id) - if override and override.check: - return override.check.adapter - return None - - def get_custom_controls(self) -> dict[str, CustomControl]: - """Get all user-defined custom controls.""" - custom = {} - for control_id, control in self.controls.items(): - # Custom controls have required fields like name, level, domain - if isinstance(control, dict): - if all(k in control for k in ("name", "level", "domain")): - custom[control_id] = CustomControl(**control) - elif isinstance(control, CustomControl): - custom[control_id] = control - return custom - - def get_adapter_config(self, name: str) -> AdapterConfig | None: - """Get adapter configuration by name.""" - return self.adapters.get(name) - - def get_all_adapter_names(self) -> list[str]: - """Get names of all defined adapters.""" - return list(self.adapters.keys()) - - -# ============================================================================= -# Factory Functions -# ============================================================================= - - -def create_user_config( - extends: str | None = "openssf-baseline", -) -> UserConfig: - """Create a minimal user configuration. - - Args: - extends: Framework to inherit from - - Returns: - Minimal UserConfig instance - """ - return UserConfig( - version="1.0", - extends=extends, - settings=UserSettings(), - ) - - -def create_user_config_with_kusari( - controls: list[str] | None = None, -) -> UserConfig: - """Create a user configuration with Kusari adapter for specified controls. - - Args: - controls: List of control IDs to use Kusari for - (defaults to VM-05 controls) - - Returns: - UserConfig with Kusari adapter configured - """ - if controls is None: - controls = ["OSPS-VM-05.02", "OSPS-VM-05.03"] - - config = UserConfig( - version="1.0", - extends="openssf-baseline", - adapters={ - "kusari": { - "type": "command", - "command": "kusari", - "output_format": "json", - } - }, - control_groups={ - "kusari-controls": ControlGroup( - controls=controls, - check=CheckConfig(adapter="kusari"), - ) - }, - ) - return config diff --git a/packages/darnit/src/darnit/core/audit_cache.py b/packages/darnit/src/darnit/core/audit_cache.py index fe2d3f21..9e587025 100644 --- a/packages/darnit/src/darnit/core/audit_cache.py +++ b/packages/darnit/src/darnit/core/audit_cache.py @@ -4,7 +4,7 @@ when results are fresh. Feature 035 refactored this module from a direct-tempdir implementation into a thin wrapper over feature 033's :class:`~darnit.stores.protocols.AuditCacheStore` Protocol, so an -operator setting ``[stores.cache]`` in ``.baseline.toml`` actually +operator setting ``[stores.cache]`` in operator configuration actually redirects the cache location. Two call forms: diff --git a/packages/darnit/src/darnit/core/models.py b/packages/darnit/src/darnit/core/models.py index 329bd74d..7a4ebfce 100644 --- a/packages/darnit/src/darnit/core/models.py +++ b/packages/darnit/src/darnit/core/models.py @@ -99,7 +99,7 @@ class ExecutionContext: cached_results: dict[str, CheckResult] = field(default_factory=dict) # Feature 031: allowlist of external MCP servers (merged framework + - # `.baseline.toml`, per-name replacement) available to the built-in + # operator configuration, per-name replacement) available to the built-in # ``mcp`` sieve handler. Values are ``McpServerConfig`` instances but # typed as ``Any`` here to avoid a config->core import cycle. Empty # dict is the pre-feature default -- audits that never consult an diff --git a/packages/darnit/src/darnit/harness/driver.py b/packages/darnit/src/darnit/harness/driver.py index 44f4c156..24a29471 100644 --- a/packages/darnit/src/darnit/harness/driver.py +++ b/packages/darnit/src/darnit/harness/driver.py @@ -15,7 +15,6 @@ import os import re from dataclasses import dataclass, field -from pathlib import Path from typing import TYPE_CHECKING, Any from darnit.core.llm_step import ConsultationRequest, LLMJudgment, LLMStep, PydanticAILLMStep @@ -270,7 +269,6 @@ def _initial_audit( default_branch=default_branch, level=self.level, stop_on_llm=True, - apply_user_config=True, framework_name=self.framework_name, operator_config=self.operator_config, target=self.target, @@ -402,14 +400,8 @@ async def _llm_continuation_loop( # Load the effective (composed) config so we can rebuild ControlSpecs # to feed back into verify_with_llm_response after LLM dispatch. - # PR #365 review fix: resolve framework via `load_effective_config_auto` - # so `.baseline.toml`'s `extends` (or --framework) determines the - # framework, matching how run_sieve_audit chose it for the initial - # pass. The previous code called `load_effective_config_by_name` - # with a hardcoded "openssf-baseline" fallback, so a non-baseline - # harness run would silently load the wrong framework. + # Resolve the framework from --framework, as the initial audit did. effective_config = load_effective_config_auto( - Path(self.local_path), framework_name=self.framework_name, operator=self.operator_config.config if self.operator_config else None, ) diff --git a/packages/darnit/src/darnit/server/tools/builtin_audit.py b/packages/darnit/src/darnit/server/tools/builtin_audit.py index 90cab286..61f8bc75 100644 --- a/packages/darnit/src/darnit/server/tools/builtin_audit.py +++ b/packages/darnit/src/darnit/server/tools/builtin_audit.py @@ -80,7 +80,7 @@ async def builtin_audit( # Load effective config (framework TOML merged with operator configuration) try: - config = load_effective_config_by_name(_framework_name, repo_path, operator=operator_config.config) + config = load_effective_config_by_name(_framework_name, operator=operator_config.config) except Exception as e: return f"Error loading framework config '{_framework_name}': {e}" @@ -139,7 +139,6 @@ async def builtin_audit( level=level, controls=all_controls, tags=tags_list, - apply_user_config=True, stop_on_llm=True, framework_name=_framework_name, operator_config=operator_config, diff --git a/packages/darnit/src/darnit/storage/backends.py b/packages/darnit/src/darnit/storage/backends.py index 9cfc1f4b..1ab0fdf4 100644 --- a/packages/darnit/src/darnit/storage/backends.py +++ b/packages/darnit/src/darnit/storage/backends.py @@ -9,7 +9,7 @@ swap in a real database (Archivista, SQL, etc.) without changing the rest of the codebase. -Configuration in .baseline.toml: +Configuration (a ``[storage]`` mapping passed by the caller): [storage] backend = "file" # file | archivista | memory archivista_url = "http://localhost:8082" # only for archivista backend @@ -388,7 +388,7 @@ def get_backend(config: dict[str, Any] | None = None) -> StorageBackend: """Return the configured storage backend. Args: - config: The [storage] section from .baseline.toml, e.g.: + config: A ``[storage]`` mapping, e.g.: {"backend": "archivista", "archivista_url": "http://localhost:8082"} Returns: diff --git a/packages/darnit/src/darnit/tools/audit.py b/packages/darnit/src/darnit/tools/audit.py index cde37fbc..fc54301b 100644 --- a/packages/darnit/src/darnit/tools/audit.py +++ b/packages/darnit/src/darnit/tools/audit.py @@ -149,9 +149,7 @@ def _get_framework_config_path(framework_name: str | None = None) -> Path | None return None -def _load_merged_stores( - local_path: str, framework_name: str | None, operator: "OperatorConfig | None" = None -) -> Any: +def _load_merged_stores(framework_name: str | None, operator: "OperatorConfig | None" = None) -> Any: """Return the merged ``StoresConfig`` for this audit run. Feature 033. Composes the framework TOML's ``[stores]`` block with @@ -160,24 +158,16 @@ def _load_merged_stores( any stores; the caller treats None as "instantiate all filesystem defaults." """ - from darnit.config import ( - load_framework_config, - load_user_config, - merge_configs, - ) + from darnit.config import load_framework_config, merge_configs framework_path = _get_framework_config_path(framework_name) if not framework_path: return operator.stores if operator is not None else None - framework = load_framework_config(framework_path) - user = load_user_config(Path(local_path)) - effective = merge_configs(framework, user, operator) + effective = merge_configs(load_framework_config(framework_path), operator) return getattr(effective, "stores", None) -def _load_merged_mcp_servers( - local_path: str, framework_name: str | None, operator: "OperatorConfig | None" = None -) -> dict[str, Any]: +def _load_merged_mcp_servers(framework_name: str | None, operator: "OperatorConfig | None" = None) -> dict[str, Any]: """Return the merged ``mcp_servers`` allowlist for this audit run. Composes the framework TOML's block with operator configuration via @@ -185,18 +175,12 @@ def _load_merged_mcp_servers( FR-016). Returns an empty dict when neither surface declares any servers. """ - from darnit.config import ( - load_framework_config, - load_user_config, - merge_configs, - ) + from darnit.config import load_framework_config, merge_configs framework_path = _get_framework_config_path(framework_name) if not framework_path: return dict(operator.mcp_servers) if operator is not None else {} - framework = load_framework_config(framework_path) - user = load_user_config(Path(local_path)) - effective = merge_configs(framework, user, operator) + effective = merge_configs(load_framework_config(framework_path), operator) return dict(effective.mcp_servers) @@ -213,7 +197,7 @@ def _apply_operator_controls( from darnit.config import load_framework_config, merge_configs from darnit.config.control_loader import control_from_effective - effective = merge_configs(load_framework_config(framework_path), None, operator) + effective = merge_configs(load_framework_config(framework_path), operator) replaced = {cid for cid, o in operator.controls.items() if o.passes is not None} | set(operator.custom_controls) result = [ control_from_effective(c.control_id, effective.controls[c.control_id]) @@ -271,9 +255,10 @@ def audit_report_metadata( repository tried to supply, each with the place it now belongs, ``unknown_assertions`` lists not-applicable claims about controls ``framework_name`` does not define (they have no effect), and - ``warnings`` (present only when non-empty) carries the ``.baseline.toml`` - deprecation warnings, the errors of a ``.project/`` file that is - present but invalid (not read, and never written; feature 042, FR-019), + ``warnings`` (present only when non-empty) carries the notice that a + ``.baseline.toml`` in the repository was ignored, the errors of a + ``.project/`` file that is present but invalid (not read, and never + written; feature 042, FR-019), and the plugin step type registrations that were refused (feature 044), which are also logged. Only refusals attempted by the audited framework's plugins (including the frameworks it composes) are reported: the registry @@ -571,30 +556,6 @@ def judgment_consultations( } -def load_effective_audit_config(local_path: str, framework_name: str | None = None) -> Any | None: - """Load the effective configuration for auditing. - - This loads the framework config and merges it with any user config - (.baseline.toml) found in the repository. - - Args: - local_path: Path to the repository - framework_name: Explicit framework name. If None, resolved from - .baseline.toml ``extends`` field or defaults to "openssf-baseline". - - Returns: - EffectiveConfig if successful, None otherwise - """ - try: - from darnit.config import load_effective_config_auto - - return load_effective_config_auto(Path(local_path), framework_name=framework_name) - - except Exception as e: - logger.warning(f"Error loading effective config: {e}") - return None - - def framework_metadata(framework_name: str | None) -> dict[str, str]: """Return the framework metadata block for the audit output header (issue #350). @@ -646,39 +607,6 @@ def framework_metadata(framework_name: str | None) -> dict[str, str]: } -def get_excluded_control_ids(local_path: str) -> dict[str, str]: - """Get control IDs that are excluded via user config. - - Args: - local_path: Path to the repository - - Returns: - Dict mapping control_id to exclusion reason - """ - effective = load_effective_audit_config(local_path) - if effective: - return effective.get_excluded_controls() - return {} - - -def get_adapter_for_control(control_id: str, local_path: str) -> str | None: - """Get the adapter name configured for a specific control. - - Args: - control_id: Control identifier - local_path: Path to the repository - - Returns: - Adapter name if configured, None for builtin - """ - effective = load_effective_audit_config(local_path) - if effective: - ctrl = effective.controls.get(control_id) - if ctrl and ctrl.check_adapter != "builtin": - return ctrl.check_adapter - return None - - @dataclass class AuditOptions: """Options for running an audit.""" @@ -743,7 +671,7 @@ def run_checks( default_branch: str, level: int = 3, stop_on_llm: bool = True, - apply_user_config: bool = True, + evaluate_claims: bool = True, framework_name: str | None = None, operator_config: "LoadedOperatorConfig | None" = None, target: str | None = None, @@ -761,9 +689,9 @@ def run_checks( default_branch: Default branch name level: Maximum level to check (1, 2, or 3) stop_on_llm: Return PENDING (llm_judgment) for LLM consultation - apply_user_config: Apply .baseline.toml user config overrides + evaluate_claims: Evaluate the repository's not-applicable claims + (see :func:`run_sieve_audit`). framework_name: Explicit framework name (e.g., "openssf-baseline"). - If None, resolved from .baseline.toml in the repo. operator_config: Operator configuration for this run. If None, resolved from the launch options for ``local_path``. target: Repository identity the operator named, for the trust decision. @@ -779,7 +707,7 @@ def run_checks( local_path, default_branch, level, - apply_user_config=apply_user_config, + evaluate_claims=evaluate_claims, stop_on_llm=stop_on_llm, framework_name=framework_name, operator_config=operator_config, @@ -803,7 +731,7 @@ def run_sieve_audit( *, controls: list | None = None, tags: list[str] | None = None, - apply_user_config: bool = True, + evaluate_claims: bool = True, stop_on_llm: bool = True, framework_name: str | None = None, operator_config: "LoadedOperatorConfig | None" = None, @@ -830,15 +758,15 @@ def run_sieve_audit( controls: Pre-loaded ControlSpec objects. If None, loads from TOML/registry automatically. tags: Tag filters to apply to controls (e.g., ["domain=AC"]). - apply_user_config: Evaluate the repository's not-applicable claims - (``.project/darnit.yaml``, ``.baseline.toml``, and applicability- - changing ``.project/`` context values) and report each with the + evaluate_claims: Evaluate the repository's not-applicable claims + (``.project/darnit.yaml`` and applicability-changing + ``.project/`` context values) and report each with the claimed control's result. When False, claims are ignored and every control is evaluated normally. stop_on_llm: Return PENDING (llm_judgment) for LLM consultation. framework_name: Explicit framework name (e.g., "openssf-baseline"). Required when controls is None and multiple implementations are - installed. If None, resolved from .baseline.toml in the repo. + installed. operator_config: Operator configuration for this run. If None, resolved from the launch options for ``local_path``; an unusable operator configuration raises ``OperatorConfigError``. @@ -866,17 +794,7 @@ def run_sieve_audit( operator_config = resolve_operator_config(local_path) operator = operator_config.config - # Resolve framework name from .baseline.toml if not provided resolved_fw = framework_name - if not resolved_fw: - try: - from darnit.config import load_user_config - - user_cfg = load_user_config(Path(local_path)) - if user_cfg and user_cfg.extends: - resolved_fw = user_cfg.extends - except Exception: - pass if resolved_fw: from darnit.config.merger import ensure_framework_allowed @@ -956,7 +874,7 @@ def run_sieve_audit( # simply see an empty allowlist and any mcp handler pass resolves # ERROR ("unknown MCP server: ...") at dispatch time. try: - execution_context.mcp_servers = _load_merged_mcp_servers(local_path, resolved_fw, operator) + execution_context.mcp_servers = _load_merged_mcp_servers(resolved_fw, operator) except Exception as err: # noqa: BLE001 - config load must not break audit logger.debug("MCP allowlist load failed (non-fatal): %s", err) all_results: list[CheckResult] = [] @@ -965,7 +883,7 @@ def run_sieve_audit( # run. Zero-config produces filesystem defaults (constitution I). from darnit.stores.selection import resolve_stores - stores_config = _load_merged_stores(local_path, resolved_fw, operator) + stores_config = _load_merged_stores(resolved_fw, operator) stores_bundle = resolve_stores(stores_config, repo_path=Path(local_path)) execution_context.stores = stores_bundle @@ -1002,7 +920,7 @@ def run_sieve_audit( # read from the repository that make a control not applicable -- count # only when honored (trusted, reasoned, uncontradicted, or confirmed). assessments: dict[str, Any] = {} - if apply_user_config: + if evaluate_claims: assessments = _assess_assertions( local_path, all_controls, @@ -1833,10 +1751,6 @@ def list_available_checks() -> dict[str, list[dict[str, Any]]]: "format_results_markdown", "list_available_checks", "audit_report_metadata", - # User config integration - "load_effective_audit_config", - "get_excluded_control_ids", - "get_adapter_for_control", # TOML framework support "_register_toml_controls", # Internal but useful for testing ] diff --git a/packages/darnit/src/darnit/tools/audit_org.py b/packages/darnit/src/darnit/tools/audit_org.py index 15e1f970..58fb1e6f 100644 --- a/packages/darnit/src/darnit/tools/audit_org.py +++ b/packages/darnit/src/darnit/tools/audit_org.py @@ -217,7 +217,6 @@ def _audit_single_repo( default_branch=default_branch, level=level, tags=tags, - apply_user_config=True, stop_on_llm=True, framework_name=framework_name, operator_config=operator_config, diff --git a/packages/darnit/src/darnit/trust/assertions.py b/packages/darnit/src/darnit/trust/assertions.py index 11ddb374..ff926d28 100644 --- a/packages/darnit/src/darnit/trust/assertions.py +++ b/packages/darnit/src/darnit/trust/assertions.py @@ -1,10 +1,10 @@ """Project assertions: what the audited repository says about itself (feature 040, research R6). A not-applicable claim is read from ``.project/darnit.yaml`` ``controls`` -and, during the ``.baseline.toml`` deprecation period, from per-control -``status``/``reason`` in ``.baseline.toml``. Project data in ``.project/`` -that makes a control not applicable is a claim too (FR-013a). Claims are -repository content: a claim counts only when its outcome is ``honored`` -- +(a repository's ``.baseline.toml`` is not read, FR-023). Project data in +``.project/`` that makes a control not applicable is a claim too +(FR-013a). Claims are repository content: a claim counts only when its +outcome is ``honored`` -- the repository is trusted, an explicit claim gives a reason (a project data value states its own), and no declared evidence contradicts it -- or when an operator-side confirmation matches it (FR-014 to FR-019). @@ -33,7 +33,6 @@ logger = get_logger("trust.assertions") PROJECT_ASSERTIONS_FILE = ".project/darnit.yaml" -BASELINE_TOML = ".baseline.toml" DEFAULT_ASSERTER = "repository content" _NOT_APPLICABLE = frozenset({ControlStatusValue.NA.value, ControlStatusValue.DISABLED.value}) @@ -113,32 +112,9 @@ def _project_claims(repo: Path) -> list[ProjectAssertion]: return claims -def _baseline_claims(repo: Path) -> list[ProjectAssertion]: - from darnit.config.merger import load_user_config_with_report - - try: - user, _ignored = load_user_config_with_report(repo) - except Exception as exc: # noqa: BLE001 - an unreadable repository file yields no claims - logger.warning("Could not read %s: %s", BASELINE_TOML, exc) - return [] - if user is None: - return [] - - claims = [] - for control_id in user.controls: - override = user.get_control_override(control_id) - status = getattr(override, "status", None) - if status is None: - continue - claim = _claim(control_id, getattr(status, "value", status), override.reason, None, BASELINE_TOML) - if claim: - claims.append(claim) - return claims - - def _claims(repo: Path) -> dict[str, ProjectAssertion]: claims: dict[str, ProjectAssertion] = {} - for claim in [*_project_claims(repo), *_baseline_claims(repo)]: + for claim in _project_claims(repo): claims.setdefault(claim.control_id, claim) return claims @@ -146,10 +122,8 @@ def _claims(repo: Path) -> dict[str, ProjectAssertion]: def collect_assertions(local_path: str | Path, framework_control_ids: Collection[str]) -> list[ProjectAssertion]: """Explicit not-applicable claims the audited repository makes about the framework's controls. - A claim in ``.project/darnit.yaml`` takes precedence over one for the - same control in ``.baseline.toml``. Claims about controls the framework - does not define are logged and ignored; :func:`unknown_assertions` - returns them for reports. + Claims about controls the framework does not define are logged and + ignored; :func:`unknown_assertions` returns them for reports. """ claims = [] for claim in _claims(Path(local_path)).values(): diff --git a/scripts/csl_manual_remediate.py b/scripts/csl_manual_remediate.py index 5c3618e7..27138cfe 100644 --- a/scripts/csl_manual_remediate.py +++ b/scripts/csl_manual_remediate.py @@ -72,7 +72,7 @@ def main(target: str) -> None: print("\n== re-audit ==") results, _ = run_sieve_audit( owner="o", repo="r", local_path=target, - default_branch="main", apply_user_config=False, + default_branch="main", evaluate_claims=False, framework_name="community-spec", stop_on_llm=False, ) for r in sorted(results, key=lambda r: r["id"]): diff --git a/scripts/csl_onboard.py b/scripts/csl_onboard.py index cc615b09..1f6601de 100644 --- a/scripts/csl_onboard.py +++ b/scripts/csl_onboard.py @@ -66,7 +66,7 @@ def audit(path: Path) -> dict[str, str]: controls = load_controls_from_effective(cfg) results, _ = run_sieve_audit( owner="", repo="", local_path=str(path), default_branch="main", - level=3, controls=controls, apply_user_config=False, stop_on_llm=False, + level=3, controls=controls, evaluate_claims=False, stop_on_llm=False, ) return {r["id"]: r["status"] for r in results} diff --git a/tests/darnit/assertions/test_assertion_reporting.py b/tests/darnit/assertions/test_assertion_reporting.py index 4dbb6918..c1637f6a 100644 --- a/tests/darnit/assertions/test_assertion_reporting.py +++ b/tests/darnit/assertions/test_assertion_reporting.py @@ -24,23 +24,18 @@ def _quiet_cli(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("darnit.cli.configure_logging", lambda level: None) -def _repo(tmp_path: Path, *, readme: bool, claim_file: str) -> Path: +def _repo(tmp_path: Path, *, readme: bool) -> Path: path = tmp_path / "repo" path.mkdir() if readme: (path / "README.md").write_text("# repo\n", encoding="utf-8") subprocess.run(["git", "init", "--initial-branch=main", "-q"], cwd=path, check=True) subprocess.run(["git", "remote", "add", "origin", "https://github.com/example/repo.git"], cwd=path, check=True) - if claim_file == "project": - (path / ".project").mkdir() - (path / ".project" / "darnit.yaml").write_text( - f"controls:\n {CONTROL}:\n status: n/a\n reason: docs live elsewhere\n asserted_by: '@alice'\n", - encoding="utf-8", - ) - else: - (path / ".baseline.toml").write_text( - f'[controls."{CONTROL}"]\nstatus = "n/a"\nreason = "docs live elsewhere"\n', encoding="utf-8" - ) + (path / ".project").mkdir() + (path / ".project" / "darnit.yaml").write_text( + f"controls:\n {CONTROL}:\n status: n/a\n reason: docs live elsewhere\n asserted_by: '@alice'\n", + encoding="utf-8", + ) return path @@ -48,14 +43,13 @@ def _result(results: list[dict]) -> dict: return next(r for r in results if r["id"] == CONTROL) -def _expected_assertion(claim_file: str) -> dict: - location = ".project/darnit.yaml" if claim_file == "project" else ".baseline.toml" +def _expected_assertion() -> dict: return { "outcome": "pending", "origin": "explicit_claim", "reason": "docs live elsewhere", - "asserted_by": "@alice" if claim_file == "project" else "repository content", - "location": f"{location}:controls.{CONTROL}", + "asserted_by": "@alice", + "location": f".project/darnit.yaml:controls.{CONTROL}", "confirmation": None, "contradiction": None, } @@ -113,17 +107,15 @@ def _mcp_builtin(repo: Path) -> list[dict]: @pytest.mark.integration -@pytest.mark.parametrize("claim_file", ["project", "baseline"]) @pytest.mark.parametrize("driver", ["sieve", "cli", "mcp_baseline", "mcp_builtin", "harness"]) def test_claim_is_reported_pending_and_control_evaluated( driver: str, - claim_file: str, tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key-not-real") - repo = _repo(tmp_path, readme=False, claim_file=claim_file) + repo = _repo(tmp_path, readme=False) results = { "sieve": lambda: _run_sieve(repo), "cli": lambda: _cli(repo, capsys), @@ -134,7 +126,7 @@ def test_claim_is_reported_pending_and_control_evaluated( result = _result(results) assert result["status"] == "FAIL" - assert result["assertion"] == _expected_assertion(claim_file) + assert result["assertion"] == _expected_assertion() @pytest.mark.integration @@ -142,7 +134,7 @@ def test_evaluated_control_keeps_its_status(tmp_path: Path) -> None: # Feature 041: a README's presence no longer concludes DO-01.01; the # control awaits a model judgment, and the pending claim does not replace # that status. - result = _result(_run_sieve(_repo(tmp_path, readme=True, claim_file="project"))) + result = _result(_run_sieve(_repo(tmp_path, readme=True))) assert result["status"] == "PENDING" assert result["pending"] == {"kind": "llm_judgment"} @@ -151,7 +143,7 @@ def test_evaluated_control_keeps_its_status(tmp_path: Path) -> None: @pytest.mark.integration def test_unclaimed_controls_carry_no_assertion(tmp_path: Path) -> None: - results = _run_sieve(_repo(tmp_path, readme=True, claim_file="project")) + results = _run_sieve(_repo(tmp_path, readme=True)) assert all("assertion" not in r for r in results if r["id"] != CONTROL) @@ -164,7 +156,7 @@ def test_markdown_names_the_claim() -> None: "status": "FAIL", "details": "no README", "level": 1, - "assertion": _expected_assertion("project"), + "assertion": _expected_assertion(), } ] summary = {"PASS": 0, "FAIL": 1, "WARN": 0, "N/A": 0, "ERROR": 0, "PENDING": 0, "total": 1} @@ -187,10 +179,10 @@ def test_markdown_shows_each_outcome_with_its_evidence() -> None: } results = [ {"id": "A", "status": "N/A", "details": "", "level": 1, "assertion": { - **_expected_assertion("project"), "outcome": "honored", "confirmation": confirmation}}, + **_expected_assertion(), "outcome": "honored", "confirmation": confirmation}}, {"id": "B", "status": "FAIL", "details": "", "level": 1, "assertion": { - **_expected_assertion("project"), "outcome": "contradicted", "contradiction": contradiction}}, - {"id": "C", "status": "PASS", "details": "", "level": 1, "assertion": _expected_assertion("project")}, + **_expected_assertion(), "outcome": "contradicted", "contradiction": contradiction}}, + {"id": "C", "status": "PASS", "details": "", "level": 1, "assertion": _expected_assertion()}, ] summary = {"PASS": 1, "FAIL": 1, "WARN": 0, "N/A": 1, "ERROR": 0, "PENDING": 0, "total": 3} @@ -209,7 +201,7 @@ def test_claims_about_unknown_controls_are_reported(tmp_path: Path) -> None: from darnit.config.operator.loader import resolve_operator_config from darnit.tools.audit import _format_audit_metadata_markdown, audit_report_metadata - repo = _repo(tmp_path, readme=True, claim_file="project") + repo = _repo(tmp_path, readme=True) (repo / ".project" / "darnit.yaml").write_text( "controls:\n OSPS-XX-99.99:\n status: n/a\n reason: x\n", encoding="utf-8" ) diff --git a/tests/darnit/assertions/test_context_value_assertions.py b/tests/darnit/assertions/test_context_value_assertions.py index 9f5c423c..4017478e 100644 --- a/tests/darnit/assertions/test_context_value_assertions.py +++ b/tests/darnit/assertions/test_context_value_assertions.py @@ -175,8 +175,8 @@ def test_unconfirmed_stored_value_implies_no_claim(self, tmp_path: Path) -> None assert "assertion" not in results[control_id] assert results[control_id]["status"] != "N/A" - def test_claims_ignored_when_user_config_is_not_applied(self, tmp_path: Path) -> None: - results = _audit(_repo(tmp_path), apply_user_config=False) + def test_claims_ignored_when_claims_are_not_evaluated(self, tmp_path: Path) -> None: + results = _audit(_repo(tmp_path), evaluate_claims=False) for control_id in RELEASE_CONTROLS: assert "assertion" not in results[control_id] diff --git a/tests/darnit/assertions/test_project_assertions.py b/tests/darnit/assertions/test_project_assertions.py index a7b2c049..0245a630 100644 --- a/tests/darnit/assertions/test_project_assertions.py +++ b/tests/darnit/assertions/test_project_assertions.py @@ -101,36 +101,6 @@ def test_no_files_no_claims(self, tmp_path: Path) -> None: assert collect_assertions(tmp_path, FRAMEWORK_IDS) == [] -@pytest.mark.unit -class TestBaselineTomlClaims: - def test_status_and_reason_flow_into_the_same_path(self, tmp_path: Path) -> None: - (tmp_path / ".baseline.toml").write_text( - '[controls."OSPS-VM-02.01"]\nstatus = "n/a"\nreason = "library"\npasses = [{ handler = "manual" }]\n', - encoding="utf-8", - ) - - assert collect_assertions(tmp_path, FRAMEWORK_IDS) == [ - ProjectAssertion( - control_id="OSPS-VM-02.01", - claim="not_applicable", - reason="library", - asserted_by="repository content", - location=".baseline.toml:controls.OSPS-VM-02.01", - origin="explicit_claim", - ) - ] - - def test_project_claim_takes_precedence(self, tmp_path: Path) -> None: - (tmp_path / ".baseline.toml").write_text( - '[controls."OSPS-DO-01.01"]\nstatus = "n/a"\nreason = "old"\n', encoding="utf-8" - ) - _darnit_yaml(tmp_path, "controls:\n OSPS-DO-01.01:\n status: n/a\n reason: new\n") - - (claim,) = collect_assertions(tmp_path, FRAMEWORK_IDS) - assert claim.reason == "new" - assert claim.location == ".project/darnit.yaml:controls.OSPS-DO-01.01" - - @pytest.mark.unit def test_report_block_for_pending_claim() -> None: claim = ProjectAssertion( diff --git a/tests/darnit/config/operator/test_baseline_deprecation.py b/tests/darnit/config/operator/test_baseline_deprecation.py deleted file mode 100644 index 91da1cfd..00000000 --- a/tests/darnit/config/operator/test_baseline_deprecation.py +++ /dev/null @@ -1,226 +0,0 @@ -""".baseline.toml during and after its deprecation release (feature 040, US5, T057). - -During the deprecation release per-control ``status``/``reason`` are read -as not-applicable claims under the same rules as ``.project/`` claims, and -every audit warns once per setting in the file with the setting's new home. -After it, the file is ignored and a single notice says so. -""" - -from __future__ import annotations - -import json -import logging -import subprocess -from pathlib import Path - -import pytest - -from darnit.cli import main as cli_main -from darnit.config import merger -from darnit.config.merger import baseline_toml_warnings, load_user_config, load_user_config_with_report -from darnit.trust.assertions import collect_assertions - -OWN = "github.com/example/repo" -CLAIMED = "OSPS-DO-01.01" -_CI_VARS = ("GITHUB_ACTIONS", "GITLAB_CI", "CI", "JENKINS_URL", "TF_BUILD", "BUILDKITE", "CIRCLECI", "TRAVIS") - -REPRESENTATIVE = f"""version = "1.0" -extends = "openssf-baseline" - -[settings] -timeout = 60 - -[plugins] -allow_unsigned = true - -[adapters.scanner] -type = "command" -command = "scanner" - -[mcp_servers.scanner] -command = ["scanner-mcp", "--stdio"] - -[stores.report] -backend = "filesystem" - -[controls."{CLAIMED}"] -status = "n/a" -reason = "docs live elsewhere" -passes = [{{ handler = "manual" }}] - -[controls."CUSTOM-01"] -name = "Custom" -level = 1 -domain = "DO" -passes = [{{ handler = "manual" }}] -""" - -EXPECTED_HOMES = { - "extends": "--framework", - "settings": "operator configuration", - "plugins": "operator configuration", - "adapters": "operator configuration", - "mcp_servers": "operator configuration", - "stores": "operator configuration", - f"controls.{CLAIMED}.status": ".project/darnit.yaml", - f"controls.{CLAIMED}.reason": ".project/darnit.yaml", - f"controls.{CLAIMED}.passes": "operator configuration", - "controls.CUSTOM-01": "operator configuration", -} - - -@pytest.fixture(autouse=True) -def _outside_ci(monkeypatch: pytest.MonkeyPatch) -> None: - for var in _CI_VARS: - monkeypatch.delenv(var, raising=False) - - -@pytest.fixture -def ended(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(merger, "BASELINE_TOML_DEPRECATION_ACTIVE", False) - - -def _write(repo: Path, text: str = REPRESENTATIVE) -> Path: - repo.mkdir(exist_ok=True) - (repo / ".baseline.toml").write_text(text, encoding="utf-8") - return repo - - -def _git_repo(tmp_path: Path) -> Path: - repo = tmp_path / "repo" - repo.mkdir() - subprocess.run(["git", "init", "--initial-branch=main", "-q"], cwd=repo, check=True) - subprocess.run(["git", "remote", "add", "origin", f"https://{OWN}.git"], cwd=repo, check=True) - return _write(repo) - - -def _setting(warning: str) -> str: - return warning.split("`")[1] - - -def _operator(trusted: bool): - from darnit.config.operator.loader import LoadedOperatorConfig - from darnit.config.operator.schema import OperatorConfig - - config = OperatorConfig.model_validate({"schema_version": 1, "trust": {"repos": [OWN] if trusted else []}}) - return LoadedOperatorConfig(config=config, source="test", digest=None, permission_check="ok", strict=False) - - -def _audit(repo: Path, *, trusted: bool) -> dict: - from darnit.config import load_controls_from_framework, load_framework_config - from darnit.tools.audit import _get_framework_config_path, run_sieve_audit - - framework = load_framework_config(_get_framework_config_path("openssf-baseline")) - controls = [c for c in load_controls_from_framework(framework) if c.control_id == CLAIMED] - results, _ = run_sieve_audit( - owner="example", - repo="repo", - local_path=str(repo), - default_branch="main", - level=1, - controls=controls, - framework_name="openssf-baseline", - operator_config=_operator(trusted), - target=OWN, - ) - return next(r for r in results if r["id"] == CLAIMED) - - -@pytest.mark.unit -class TestDeprecationRelease: - def test_switch_is_on_for_this_release(self) -> None: - assert merger.BASELINE_TOML_DEPRECATION_ACTIVE is True - - def test_one_warning_per_setting_naming_its_new_home(self, tmp_path: Path) -> None: - warnings = baseline_toml_warnings(_write(tmp_path)) - - assert len(warnings) == len(EXPECTED_HOMES) - by_setting = {_setting(w): w for w in warnings} - assert set(by_setting) == set(EXPECTED_HOMES) - for setting, home in EXPECTED_HOMES.items(): - assert home in by_setting[setting], by_setting[setting] - assert "deprecated" in by_setting[setting] - - def test_claim_warnings_point_at_the_migration(self, tmp_path: Path) -> None: - warnings = baseline_toml_warnings(_write(tmp_path)) - - status = next(w for w in warnings if _setting(w) == f"controls.{CLAIMED}.status") - assert "darnit config migrate" in status - - def test_no_file_no_warnings(self, tmp_path: Path) -> None: - assert baseline_toml_warnings(tmp_path) == [] - - def test_unreadable_file_gets_one_warning(self, tmp_path: Path) -> None: - (warning,) = baseline_toml_warnings(_write(tmp_path, "[controls\n")) - assert ".baseline.toml" in warning and "deprecated" in warning - - def test_only_status_and_reason_become_claims(self, tmp_path: Path) -> None: - repo = _write(tmp_path) - - (claim,) = collect_assertions(repo, {CLAIMED, "CUSTOM-01"}) - assert (claim.control_id, claim.reason) == (CLAIMED, "docs live elsewhere") - assert claim.location == f".baseline.toml:controls.{CLAIMED}" - - user = load_user_config(repo) - assert user is not None - assert set(user.controls) == {CLAIMED} - assert user.get_control_override(CLAIMED).passes is None - assert not user.mcp_servers and not user.adapters - - -@pytest.mark.integration -class TestDeprecatedClaimsFollowTheTrustRules: - def test_untrusted_claim_is_pending(self, tmp_path: Path) -> None: - result = _audit(_git_repo(tmp_path), trusted=False) - - assert result["assertion"]["outcome"] == "pending" - assert result["assertion"]["location"] == f".baseline.toml:controls.{CLAIMED}" - - def test_trusted_claim_is_honored(self, tmp_path: Path) -> None: - result = _audit(_git_repo(tmp_path), trusted=True) - - assert result["status"] == "N/A" - assert result["assertion"]["outcome"] == "honored" - - -@pytest.mark.integration -def test_every_audit_reports_the_warnings( - tmp_path: Path, capsys: pytest.CaptureFixture[str], caplog: pytest.LogCaptureFixture, monkeypatch -) -> None: - monkeypatch.setattr("darnit.cli.configure_logging", lambda level: None) - repo = _git_repo(tmp_path) - - with caplog.at_level(logging.WARNING, logger="darnit"): - cli_main(["audit", str(repo), "-f", "openssf-baseline", "--include", CLAIMED, "-o", "json", "--no-fail"]) - output = json.loads(capsys.readouterr().out) - - assert {_setting(w) for w in output["warnings"]} == set(EXPECTED_HOMES) - for setting in EXPECTED_HOMES: - assert f"`{setting}`" in caplog.text - - -@pytest.mark.unit -class TestAfterDeprecation: - def test_file_is_not_read(self, tmp_path: Path, ended: None) -> None: - repo = _write(tmp_path) - - assert load_user_config(repo) is None - assert load_user_config_with_report(repo) == (None, []) - assert collect_assertions(repo, {CLAIMED}) == [] - - def test_single_notice_says_it_was_ignored(self, tmp_path: Path, ended: None) -> None: - (notice,) = baseline_toml_warnings(_write(tmp_path)) - - assert ".baseline.toml" in notice and "ignored" in notice - assert "darnit config migrate" in notice - - def test_no_file_no_notice(self, tmp_path: Path, ended: None) -> None: - assert baseline_toml_warnings(tmp_path) == [] - - -@pytest.mark.integration -def test_claim_has_no_effect_after_deprecation(tmp_path: Path, ended: None) -> None: - result = _audit(_git_repo(tmp_path), trusted=True) - - assert "assertion" not in result or result["assertion"] is None - assert result["status"] != "N/A" diff --git a/tests/darnit/config/operator/test_baseline_toml_removed.py b/tests/darnit/config/operator/test_baseline_toml_removed.py new file mode 100644 index 00000000..4f6038ce --- /dev/null +++ b/tests/darnit/config/operator/test_baseline_toml_removed.py @@ -0,0 +1,186 @@ +""".baseline.toml is not read (feature 040 FR-023; framework-design 14.4). + +No key in a repository's ``.baseline.toml`` affects an audit, even for a +repository the operator trusts. An audit of a repository that has one +records exactly one notice pointing at ``darnit config migrate``, which +still moves the file's claims to ``.project/darnit.yaml``. +""" + +from __future__ import annotations + +import json +import logging +import subprocess +from pathlib import Path + +import pytest + +from darnit.cli import main as cli_main +from darnit.config.merger import baseline_toml_warnings, find_ignored_repository_settings +from darnit.trust.assertions import collect_assertions + +OWN = "github.com/example/repo" +CLAIMED = "OSPS-DO-01.01" +_CI_VARS = ("GITHUB_ACTIONS", "GITLAB_CI", "CI", "JENKINS_URL", "TF_BUILD", "BUILDKITE", "CIRCLECI", "TRAVIS") + +REPRESENTATIVE = f"""version = "1.0" +extends = "testchecks" + +[settings] +timeout = 60 + +[plugins] +allow_unsigned = true + +[mcp_servers.scanner] +command = ["scanner-mcp", "--stdio"] + +[stores.report] +backend = "filesystem" + +[controls."{CLAIMED}"] +status = "n/a" +reason = "docs live elsewhere" + +[controls."CUSTOM-01"] +name = "Custom" +level = 1 +domain = "DO" +passes = [{{ handler = "manual" }}] +""" + + +@pytest.fixture(autouse=True) +def _outside_ci(monkeypatch: pytest.MonkeyPatch) -> None: + for var in _CI_VARS: + monkeypatch.delenv(var, raising=False) + + +def _write(repo: Path, text: str = REPRESENTATIVE) -> Path: + repo.mkdir(exist_ok=True) + (repo / ".baseline.toml").write_text(text, encoding="utf-8") + return repo + + +def _git_repo(tmp_path: Path) -> Path: + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "--initial-branch=main", "-q"], cwd=repo, check=True) + subprocess.run(["git", "remote", "add", "origin", f"https://{OWN}.git"], cwd=repo, check=True) + return _write(repo) + + +def _trusting_operator_file(tmp_path: Path) -> Path: + path = tmp_path / "operator" / "config.toml" + path.parent.mkdir() + path.write_text(f'schema_version = 1\n\n[trust]\nrepos = ["{OWN}"]\n', encoding="utf-8") + path.chmod(0o600) + return path + + +def _cli_audit(repo: Path, capsys: pytest.CaptureFixture[str], *extra: str) -> dict: + exit_code = cli_main(["audit", str(repo), "--include", CLAIMED, "-o", "json", "--no-fail", *extra]) + assert exit_code == 0 + return json.loads(capsys.readouterr().out) + + +def _notices(text: str) -> int: + return text.count(".baseline.toml") + + +@pytest.mark.unit +class TestNotice: + def test_one_notice_pointing_at_migrate(self, tmp_path: Path) -> None: + (notice,) = baseline_toml_warnings(_write(tmp_path)) + + assert ".baseline.toml" in notice and "ignored" in notice + assert "darnit config migrate" in notice + assert ".project/darnit.yaml" in notice + assert "operator configuration" in notice + + def test_unreadable_file_gets_the_same_single_notice(self, tmp_path: Path) -> None: + assert baseline_toml_warnings(_write(tmp_path, "[controls\n")) == baseline_toml_warnings(_write(tmp_path)) + + def test_no_file_no_notice(self, tmp_path: Path) -> None: + assert baseline_toml_warnings(tmp_path) == [] + + def test_keys_are_not_listed_as_ignored_settings(self, tmp_path: Path) -> None: + assert find_ignored_repository_settings(_write(tmp_path)) == [] + + +@pytest.mark.unit +def test_no_claims_are_read(tmp_path: Path) -> None: + assert collect_assertions(_write(tmp_path), {CLAIMED}) == [] + + +@pytest.mark.integration +def test_trusted_repository_claim_has_no_effect_and_one_notice_is_reported( + tmp_path: Path, capsys: pytest.CaptureFixture[str], caplog: pytest.LogCaptureFixture, monkeypatch +) -> None: + monkeypatch.setattr("darnit.cli.configure_logging", lambda level: None) + repo = _git_repo(tmp_path) + operator = _trusting_operator_file(tmp_path) + + with caplog.at_level(logging.WARNING, logger="darnit"): + output = _cli_audit(repo, capsys, "-f", "openssf-baseline", "--repo", OWN, "--operator-config", str(operator)) + + assert output["trust"]["trusted"] is True + (result,) = [r for r in output["results"] if r["id"] == CLAIMED] + assert result["status"] != "N/A" + assert not result.get("assertion") + assert sum(_notices(w) for w in output["warnings"]) == 1 + (logged,) = [r for r in caplog.records if ".baseline.toml" in r.getMessage()] + assert logged.levelno == logging.WARNING + assert _notices(logged.getMessage()) == 1 + assert not any(s["file"] == ".baseline.toml" for s in output["ignored_repository_settings"]) + + +@pytest.mark.integration +def test_extends_does_not_select_the_framework(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """``extends = "testchecks"`` is ignored: the default framework is audited, not testchecks.""" + output = _cli_audit(_git_repo(tmp_path), capsys) + + assert output["framework"] == "openssf-baseline" + (result,) = output["results"] + assert result["id"] == CLAIMED + assert result["level"] == 1 + + +@pytest.mark.integration +def test_repository_supplied_command_is_not_run(tmp_path: Path) -> None: + """A pass in .baseline.toml is never executed, through the MCP audit tool.""" + from darnit_baseline.tools import audit_openssf_baseline + + marker = tmp_path / "marker.txt" + repo = tmp_path / "repo" + repo.mkdir() + (repo / "README.md").write_text("# x\n", encoding="utf-8") + _write( + repo, + f'[controls."{CLAIMED}"]\npasses = [ {{ handler = "exec", command = ["sh", "-c", "echo pwned > {marker}"] }} ]\n', + ) + + audit_openssf_baseline(owner="o", repo="r", local_path=str(repo), output_format="json") + + assert not marker.exists() + + +@pytest.mark.integration +def test_migrate_moves_the_claim_and_the_audit_then_honors_it( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch +) -> None: + monkeypatch.setattr("darnit.cli.configure_logging", lambda level: None) + repo = _git_repo(tmp_path) + operator = _trusting_operator_file(tmp_path) + + assert cli_main(["config", "migrate", str(repo)]) == 0 + out = capsys.readouterr().out + assert "[mcp_servers.scanner]" in out + + output = _cli_audit(repo, capsys, "-f", "openssf-baseline", "--repo", OWN, "--operator-config", str(operator)) + + (result,) = [r for r in output["results"] if r["id"] == CLAIMED] + assert result["status"] == "N/A" + assert result["assertion"]["outcome"] == "honored" + assert result["assertion"]["location"] == f".project/darnit.yaml:controls.{CLAIMED}" + assert sum(_notices(w) for w in output["warnings"]) == 1 diff --git a/tests/darnit/config/operator/test_effective_config.py b/tests/darnit/config/operator/test_effective_config.py index 2629e6fb..8cf7c91c 100644 --- a/tests/darnit/config/operator/test_effective_config.py +++ b/tests/darnit/config/operator/test_effective_config.py @@ -2,8 +2,6 @@ from __future__ import annotations -from pathlib import Path - import pytest from darnit.config.framework_schema import ( @@ -15,11 +13,7 @@ StoreBlock, StoresConfig, ) -from darnit.config.merger import ( - load_user_config, - load_user_config_with_report, - merge_configs, -) +from darnit.config.merger import merge_configs from darnit.config.operator.schema import OperatorConfig @@ -119,40 +113,3 @@ def test_plugins_allowed_refuses_other_frameworks() -> None: effective = merge_configs(_framework(), operator=_operator(plugins={"allowed": ["test-framework"]})) assert effective.framework_name == "test-framework" - - -@pytest.mark.unit -def test_repository_config_cannot_supply_what_operator_config_does(tmp_path: Path) -> None: - (tmp_path / ".baseline.toml").write_text( - '[mcp_servers.shared]\ncommand = ["repo-shared"]\n\n[controls."TEST-02"]\npasses = [{ handler = "manual" }]\n', - encoding="utf-8", - ) - operator = _operator(controls={"TEST-01": {"passes": [{"handler": "file_exists", "files": ["OPERATOR.md"]}]}}) - effective = merge_configs(_framework(), load_user_config(tmp_path), operator=operator) - - assert effective.mcp_servers["shared"].command == ["framework-shared"] - assert effective.controls["TEST-02"].passes_config is None - assert effective.controls["TEST-01"].passes_config[0]["files"] == ["OPERATOR.md"] - - -@pytest.mark.unit -def test_load_user_config_with_report_lists_ignored_keys(tmp_path: Path) -> None: - (tmp_path / ".baseline.toml").write_text( - 'extends = "openssf-baseline"\n\n[stores.report]\nbackend = "filesystem"\n\n' - '[controls."TEST-01"]\nstatus = "n/a"\nreason = "not here"\npasses = [{ handler = "manual" }]\n', - encoding="utf-8", - ) - - user, ignored = load_user_config_with_report(tmp_path) - - assert user is not None - assert user.extends == "openssf-baseline" - assert {(s.file, s.key, s.new_home) for s in ignored} == { - (".baseline.toml", "stores", "operator configuration"), - (".baseline.toml", "controls.TEST-01.passes", "operator configuration"), - } - - -@pytest.mark.unit -def test_load_user_config_with_report_without_file(tmp_path: Path) -> None: - assert load_user_config_with_report(tmp_path) == (None, []) diff --git a/tests/darnit/config/operator/test_repository_cannot_configure.py b/tests/darnit/config/operator/test_repository_cannot_configure.py index 87f29017..16cce5c2 100644 --- a/tests/darnit/config/operator/test_repository_cannot_configure.py +++ b/tests/darnit/config/operator/test_repository_cannot_configure.py @@ -111,13 +111,6 @@ def operator_config(tmp_path: Path) -> Path: EXPECTED_IGNORED = { - (".baseline.toml", "extends (path)"), - (".baseline.toml", "plugins"), - (".baseline.toml", "adapters"), - (".baseline.toml", "mcp_servers"), - (".baseline.toml", "stores"), - (".baseline.toml", f"controls.{CONTROL}.passes"), - (".baseline.toml", "controls.PLANTED-01.passes"), (".darnit/config.toml", "controls"), (".darnit/config.toml", "trust"), ("darnit.toml", "controls"), @@ -135,6 +128,11 @@ def _assert_ignored_reported(output: dict) -> None: ignored = output["ignored_repository_settings"] reported = {(entry["file"], entry["key"]) for entry in ignored} assert EXPECTED_IGNORED <= reported + # .baseline.toml is not read at all: one notice, no per-key entries (FR-023). + assert not any(entry["file"] == ".baseline.toml" for entry in ignored) + assert [w for w in output["warnings"] if ".baseline.toml" in w] == [ + next(w for w in output["warnings"] if "darnit config migrate" in w) + ] for entry in ignored: assert set(entry) == {"file", "key", "new_home"} if (entry["file"], entry["key"]) in EXPECTED_IGNORED: diff --git a/tests/darnit/config/test_authority_validation_paths.py b/tests/darnit/config/test_authority_validation_paths.py index 3a6d34a9..b7dddbc8 100644 --- a/tests/darnit/config/test_authority_validation_paths.py +++ b/tests/darnit/config/test_authority_validation_paths.py @@ -106,7 +106,7 @@ def test_control_from_effective_validates() -> None: def test_load_controls_from_effective_validates(tmp_path: Path) -> None: framework_path = tmp_path / "widening.toml" framework_path.write_text(FRAMEWORK_TOML, encoding="utf-8") - config = load_effective_config(framework_path, None) + config = load_effective_config(framework_path) with pytest.raises(AuthorityViolation) as excinfo: load_controls_from_effective(config) _assert_names_step(excinfo) diff --git a/tests/darnit/config/test_merger.py b/tests/darnit/config/test_merger.py index 0d880914..5b4a0217 100644 --- a/tests/darnit/config/test_merger.py +++ b/tests/darnit/config/test_merger.py @@ -1,6 +1,6 @@ """Tests for config merging functionality. -This module tests the framework + user config merging system. +This module tests merging a framework with operator configuration. """ import pytest @@ -21,19 +21,13 @@ merge_configs, merge_control, ) -from darnit.config.user_schema import ( - ControlOverride, - ControlStatus, - UserConfig, - UserSettings, -) class TestMergeControl: - """Test merging individual control configurations.""" + """Test building a control's effective configuration.""" def test_framework_only(self): - """Test control with no user override.""" + """Test control built from its framework definition.""" framework_control = ControlConfig( name="TestControl", level=1, @@ -43,59 +37,33 @@ def test_framework_only(self): ) defaults = FrameworkDefaults() - result = merge_control("TEST-01", framework_control, None, defaults) + result = merge_control("TEST-01", framework_control, defaults) assert isinstance(result, EffectiveControl) assert result.name == "TestControl" assert result.level == 1 assert result.domain == "AC" - assert result.is_applicable() is True - - def test_user_override_status_na(self): - """Test user marking control as N/A.""" - framework_control = ControlConfig( - name="TestControl", - level=1, - domain="AC", - description="Test description", - ) - defaults = FrameworkDefaults() - - user_override = ControlOverride( - status=ControlStatus.NA, - reason="Pre-1.0 project, no releases yet", - ) - - result = merge_control("TEST-01", framework_control, user_override, defaults) - assert result.is_applicable() is False - assert result.status_reason == "Pre-1.0 project, no releases yet" - - def test_user_override_adapter(self): - """Test user overriding check adapter.""" + def test_framework_check_adapter(self): + """Test the framework's check adapter carries through.""" framework_control = ControlConfig( name="TestControl", level=1, domain="AC", description="Test description", - ) - defaults = FrameworkDefaults(check_adapter="builtin") - - user_override = ControlOverride( check=CheckConfig(adapter="kusari"), ) - result = merge_control("TEST-01", framework_control, user_override, defaults) + result = merge_control("TEST-01", framework_control, FrameworkDefaults(check_adapter="builtin")) assert result.check_adapter == "kusari" - assert result.is_applicable() is True class TestMergeConfigs: - """Test merging complete framework and user configs.""" + """Test merging a framework with operator configuration.""" def test_framework_only(self): - """Test merging when no user config exists.""" + """Test merging with no operator configuration.""" framework = FrameworkConfig( metadata=FrameworkMetadata( name="test", @@ -112,139 +80,12 @@ def test_framework_only(self): }, ) - result = merge_configs(framework, None) + result = merge_configs(framework) assert isinstance(result, EffectiveConfig) assert "TEST-01" in result.controls assert result.controls["TEST-01"].name == "Control1" - def test_user_exclusions(self): - """Test that user exclusions are reflected in effective config.""" - framework = FrameworkConfig( - metadata=FrameworkMetadata( - name="test", - display_name="Test Framework", - version="1.0", - ), - controls={ - "TEST-01": ControlConfig( - name="Control1", - level=1, - domain="AC", - description="Test", - ), - "TEST-02": ControlConfig( - name="Control2", - level=2, - domain="BR", - description="Test 2", - ), - }, - ) - - user = UserConfig( - version="1.0", - extends="test", - controls={ - "TEST-01": ControlOverride( - status=ControlStatus.NA, - reason="Not needed", - ), - }, - ) - - result = merge_configs(framework, user) - - assert result.controls["TEST-01"].is_applicable() is False - assert result.controls["TEST-02"].is_applicable() is True - - def test_get_excluded_controls(self): - """Test getting the list of excluded controls.""" - framework = FrameworkConfig( - metadata=FrameworkMetadata( - name="test", - display_name="Test Framework", - version="1.0", - ), - controls={ - "TEST-01": ControlConfig( - name="Control1", - level=1, - domain="AC", - description="Test", - ), - "TEST-02": ControlConfig( - name="Control2", - level=2, - domain="BR", - description="Test 2", - ), - }, - ) - - user = UserConfig( - version="1.0", - extends="test", - controls={ - "TEST-01": ControlOverride( - status=ControlStatus.NA, - reason="Pre-release project", - ), - }, - ) - - result = merge_configs(framework, user) - excluded = result.get_excluded_controls() - - assert "TEST-01" in excluded - assert excluded["TEST-01"] == "Pre-release project" - assert "TEST-02" not in excluded - - -class TestEffectiveControl: - """Test EffectiveControl behavior.""" - - def test_is_applicable_default(self): - """Test that controls are applicable by default.""" - control = EffectiveControl( - control_id="TEST-01", - name="Test", - level=1, - domain="AC", - description="Test", - status=None, # No status = applicable - ) - - assert control.is_applicable() is True - - def test_is_applicable_na(self): - """Test N/A status makes control not applicable.""" - control = EffectiveControl( - control_id="TEST-01", - name="Test", - level=1, - domain="AC", - description="Test", - status=ControlStatus.NA, - status_reason="Not needed", - ) - - assert control.is_applicable() is False - - def test_is_applicable_disabled(self): - """Test disabled status makes control not applicable.""" - control = EffectiveControl( - control_id="TEST-01", - name="Test", - level=1, - domain="AC", - description="Test", - status=ControlStatus.DISABLED, - status_reason="Temporarily disabled", - ) - - assert control.is_applicable() is False - class TestEffectiveConfig: """Test EffectiveConfig behavior.""" @@ -287,91 +128,6 @@ def test_get_controls_by_level(self): assert len(level2) == 1 assert "L2-01" in level2 - def test_get_controls_by_level_excludes_na(self): - """Test that get_controls_by_level excludes N/A controls.""" - config = EffectiveConfig( - framework_name="test", - framework_version="1.0", - controls={ - "L1-01": EffectiveControl( - control_id="L1-01", - name="L1Active", - level=1, - domain="AC", - description="Active Level 1", - ), - "L1-02": EffectiveControl( - control_id="L1-02", - name="L1NA", - level=1, - domain="AC", - description="N/A Level 1", - status=ControlStatus.NA, - ), - }, - ) - - level1 = config.get_controls_by_level(1) - - assert len(level1) == 1 - assert "L1-01" in level1 - assert "L1-02" not in level1 - - def test_get_applicable_controls(self): - """Test getting only applicable controls via get_controls_by_level.""" - config = EffectiveConfig( - framework_name="test", - framework_version="1.0", - controls={ - "ACTIVE-01": EffectiveControl( - control_id="ACTIVE-01", - name="Active", - level=1, - domain="AC", - description="Active control", - status=None, - ), - "NA-01": EffectiveControl( - control_id="NA-01", - name="NotApplicable", - level=1, - domain="AC", - description="N/A control", - status=ControlStatus.NA, - status_reason="Not needed", - ), - }, - ) - - # get_controls_by_level already filters by is_applicable - applicable = config.get_controls_by_level(1) - - assert len(applicable) == 1 - assert "ACTIVE-01" in applicable - assert "NA-01" not in applicable - - -class TestUserSettings: - """Test user settings behavior.""" - - def test_default_settings(self): - """Test default user settings.""" - settings = UserSettings() - - assert settings.cache_results is True - assert settings.timeout == 300 - - def test_custom_settings(self): - """Test custom user settings.""" - settings = UserSettings( - cache_results=False, - timeout=60, - ) - - assert settings.cache_results is False - assert settings.timeout == 60 - - if __name__ == "__main__": pytest.main([__file__, "-v"]) @@ -404,7 +160,7 @@ def test_merge_control_preserves_gating_metadata(self): framework_control = self._make_framework_control_with_gating() defaults = FrameworkDefaults() - effective = merge_control("OSPS-QA-02.01", framework_control, None, defaults) + effective = merge_control("OSPS-QA-02.01", framework_control, defaults) assert effective.when == {"has_releases": True}, ( "merge_control dropped 'when' — when-gates will be silently ignored" @@ -425,7 +181,7 @@ def test_control_from_effective_preserves_gating_metadata(self): framework_control = self._make_framework_control_with_gating() defaults = FrameworkDefaults() - effective = merge_control("OSPS-QA-02.01", framework_control, None, defaults) + effective = merge_control("OSPS-QA-02.01", framework_control, defaults) spec = control_from_effective("OSPS-QA-02.01", effective) assert "when" in spec.metadata, ( @@ -458,7 +214,7 @@ def test_control_without_gating_metadata_is_unaffected(self): ) defaults = FrameworkDefaults() - effective = merge_control("OSPS-BR-01.01", framework_control, None, defaults) + effective = merge_control("OSPS-BR-01.01", framework_control, defaults) spec = control_from_effective("OSPS-BR-01.01", effective) # Optional fields default to None — must not appear in metadata diff --git a/tests/darnit/config/test_merger_mcp_servers.py b/tests/darnit/config/test_merger_mcp_servers.py index a47a57a1..56d649e3 100644 --- a/tests/darnit/config/test_merger_mcp_servers.py +++ b/tests/darnit/config/test_merger_mcp_servers.py @@ -8,7 +8,7 @@ McpServerConfig, ) from darnit.config.merger import merge_configs -from darnit.config.user_schema import UserConfig +from darnit.config.operator.schema import OperatorConfig def _framework(**servers: McpServerConfig) -> FrameworkConfig: @@ -24,15 +24,15 @@ def _framework(**servers: McpServerConfig) -> FrameworkConfig: # --------------------------------------------------------------------------- -# T026: baseline replaces per-name (no deep merge) +# T026: operator configuration replaces per-name (no deep merge) # --------------------------------------------------------------------------- -def test_mcp_servers_baseline_wins(): +def test_mcp_servers_operator_wins(): fw = _framework(foo=McpServerConfig(command=["fw-cmd"])) - user = UserConfig(mcp_servers={"foo": McpServerConfig(command=["bl-cmd"])}) - eff = merge_configs(fw, user) - assert eff.mcp_servers["foo"].command == ["bl-cmd"] + operator = OperatorConfig(schema_version=1, mcp_servers={"foo": McpServerConfig(command=["op-cmd"])}) + eff = merge_configs(fw, operator) + assert eff.mcp_servers["foo"].command == ["op-cmd"] # --------------------------------------------------------------------------- @@ -42,8 +42,8 @@ def test_mcp_servers_baseline_wins(): def test_mcp_servers_disjoint_names_coexist(): fw = _framework(a=McpServerConfig(command=["fw-a"])) - user = UserConfig(mcp_servers={"b": McpServerConfig(command=["bl-b"])}) - eff = merge_configs(fw, user) + operator = OperatorConfig(schema_version=1, mcp_servers={"b": McpServerConfig(command=["op-b"])}) + eff = merge_configs(fw, operator) assert set(eff.mcp_servers) == {"a", "b"} assert eff.mcp_servers["a"].command == ["fw-a"] - assert eff.mcp_servers["b"].command == ["bl-b"] + assert eff.mcp_servers["b"].command == ["op-b"] diff --git a/tests/darnit/config/test_stores_config.py b/tests/darnit/config/test_stores_config.py index 0a590eb0..a4692a45 100644 --- a/tests/darnit/config/test_stores_config.py +++ b/tests/darnit/config/test_stores_config.py @@ -15,7 +15,11 @@ StoresConfig, ) from darnit.config.merger import merge_configs -from darnit.config.user_schema import UserConfig +from darnit.config.operator.schema import OperatorConfig + + +def _operator(**stores_kwargs) -> OperatorConfig: + return OperatorConfig(schema_version=1, stores=StoresConfig(**stores_kwargs)) def _fw(**stores_kwargs): @@ -73,39 +77,30 @@ def test_unset_var_substitutes_empty(self, monkeypatch): class TestPerKindMerger: - def test_user_replaces_framework_for_same_kind(self): + def test_operator_replaces_framework_for_same_kind(self): fw = _fw(project=StoreBlock(backend="fw-project")) - usr = UserConfig( - stores=StoresConfig(project=StoreBlock(backend="usr-project")) - ) - eff = merge_configs(fw, usr) - assert eff.stores.project.backend == "usr-project" + eff = merge_configs(fw, _operator(project=StoreBlock(backend="op-project"))) + assert eff.stores.project.backend == "op-project" def test_disjoint_kinds_coexist(self): fw = _fw(attestation=StoreBlock(backend="fw-att")) - usr = UserConfig( - stores=StoresConfig(project=StoreBlock(backend="usr-project")) - ) - eff = merge_configs(fw, usr) - assert eff.stores.project.backend == "usr-project" + eff = merge_configs(fw, _operator(project=StoreBlock(backend="op-project"))) + assert eff.stores.project.backend == "op-project" assert eff.stores.attestation.backend == "fw-att" - def test_user_only(self): + def test_operator_only(self): fw = _fw() - usr = UserConfig( - stores=StoresConfig(project=StoreBlock(backend="usr-project")) - ) - eff = merge_configs(fw, usr) - assert eff.stores.project.backend == "usr-project" + eff = merge_configs(fw, _operator(project=StoreBlock(backend="op-project"))) + assert eff.stores.project.backend == "op-project" assert eff.stores.attestation is None def test_framework_only(self): fw = _fw(project=StoreBlock(backend="fw-project")) - eff = merge_configs(fw, None) + eff = merge_configs(fw) assert eff.stores.project.backend == "fw-project" def test_neither_set(self): fw = _fw() - eff = merge_configs(fw, None) + eff = merge_configs(fw) for kind in ("project", "attestation", "report", "cache"): assert getattr(eff.stores, kind) is None diff --git a/tests/darnit/config/test_strict_framework_loading.py b/tests/darnit/config/test_strict_framework_loading.py index 7e9deb47..5d560e30 100644 --- a/tests/darnit/config/test_strict_framework_loading.py +++ b/tests/darnit/config/test_strict_framework_loading.py @@ -92,39 +92,6 @@ def test_unknown_control_key_names_file_control_and_key(self, tmp_path: Path) -> assert fragment in text, (fragment, text) - def test_unknown_key_on_a_baseline_toml_custom_control_names_file_control_and_key(self, tmp_path: Path) -> None: - """A .baseline.toml custom control merged as a raw table is held to 2.3, with its error shape (044 review). - - ``model_construct`` hands ``merge_configs`` the raw table, as it - receives one when neither union member of ``UserConfig.controls`` - validates it. - """ - from pydantic import ValidationError - - from darnit.config.user_schema import UserConfig - - framework = load_framework_config(_toml(tmp_path, "")) - user = UserConfig.model_construct( - controls={ - "CUSTOM-01": { - "name": "Custom", - "description": "A repository's custom control", - "level": 1, - "domain": "CU", - "sevrity": "high", - } - } - ) - - with pytest.raises(ValueError) as excinfo: - merge_configs(framework, user) - - assert not isinstance(excinfo.value, ValidationError) - text = str(excinfo.value) - for fragment in (".baseline.toml", "'CUSTOM-01'", "'sevrity'", "framework-design 2.3"): - assert fragment in text, (fragment, text) - - @pytest.mark.unit class TestStepKeys: """FR-008.""" @@ -233,7 +200,7 @@ def test_operator_control_naming_a_missing_plugin_step_type_is_error(self, tmp_p } ) - specs = {s.control_id: s for s in load_controls_from_effective(merge_configs(framework, None, operator))} + specs = {s.control_id: s for s in load_controls_from_effective(merge_configs(framework, operator))} context = CheckContext(owner="o", repo="r", local_path=str(tmp_path), default_branch="main", control_id="OP-01") result = SieveOrchestrator(stop_on_llm=False).verify(specs["OP-01"], context) @@ -271,7 +238,7 @@ def register(framework_name: str | None) -> bool: } ) - load_controls_from_effective(merge_configs(framework, None, operator)) + load_controls_from_effective(merge_configs(framework, operator)) assert attempts == ["strict-fw"] @@ -283,7 +250,7 @@ def test_operator_pass_override_naming_a_missing_plugin_step_type_is_error(self, {"schema_version": 1, "controls": {"STR-01": {"passes": [{"handler": "absent_plugin_check"}]}}} ) - specs = {s.control_id: s for s in load_controls_from_effective(merge_configs(framework, None, operator))} + specs = {s.control_id: s for s in load_controls_from_effective(merge_configs(framework, operator))} context = CheckContext( owner="o", repo="r", local_path=str(tmp_path), default_branch="main", control_id="STR-01" ) diff --git a/tests/darnit/config/test_untrusted_repo_config.py b/tests/darnit/config/test_untrusted_repo_config.py deleted file mode 100644 index 813de10e..00000000 --- a/tests/darnit/config/test_untrusted_repo_config.py +++ /dev/null @@ -1,181 +0,0 @@ -"""A repository's own .baseline.toml is untrusted input. - -The audited repository is chosen by the operator, but its contents are -controlled by whoever can write to it -- any contributor to an org being -swept by audit_org, or the author of a fork pull request audited in CI. -Nothing it contains may change what darnit executes, which servers or -adapters it trusts, or which framework definition it loads. Only scope -declarations (per-control status and reason) are honored unless the -operator explicitly opts in. -""" - -from __future__ import annotations - -import logging -from pathlib import Path - -import pytest - -from darnit.config import merger -from darnit.config.merger import load_user_config - -EVIL = """ -version = "1.0" -extends = "./evil-framework.toml" - -[plugins] -allow_unsigned = true -trusted_publishers = ["https://github.com/attacker"] - -[adapters.evil] -type = "command" -command = "sh" - -[mcp_servers.scanner] -command = ["sh", "-c", "id"] - -[stores.report] -backend = "filesystem" - -[control_groups.all] -controls = ["OSPS-DO-01.01"] - -[control_groups.all.check] -adapter = "evil" - -[controls."OSPS-DO-01.01"] -passes = [ { handler = "exec", command = ["sh", "-c", "echo pwned"] } ] -config = { anything = true } - -[controls."OSPS-DO-01.01".check] -adapter = "evil" -handler = "run" - -[controls."OSPS-DO-01.01".remediation] -config = { command = "sh" } - -[controls."OSPS-VM-02.01"] -status = "n/a" -reason = "we have no vulnerability process" - -[controls."CUSTOM-01"] -name = "custom" -level = 1 -domain = "XX" -passes = [ { handler = "exec", command = ["true"] } ] -""" - - -def _write(tmp_path: Path, text: str) -> Path: - (tmp_path / ".baseline.toml").write_text(text, encoding="utf-8") - return tmp_path - - -class TestUntrustedByDefault: - @pytest.mark.unit - def test_pass_override_is_ignored(self, tmp_path: Path) -> None: - user = load_user_config(_write(tmp_path, EVIL)) - override = user.get_control_override("OSPS-DO-01.01") - assert override is None or not override.passes - - @pytest.mark.unit - def test_check_remediation_and_config_overrides_are_ignored(self, tmp_path: Path) -> None: - user = load_user_config(_write(tmp_path, EVIL)) - override = user.get_control_override("OSPS-DO-01.01") - assert override is None or (override.check is None and override.remediation is None and not override.config) - - @pytest.mark.unit - def test_custom_controls_are_ignored(self, tmp_path: Path) -> None: - user = load_user_config(_write(tmp_path, EVIL)) - assert "CUSTOM-01" not in user.controls - - @pytest.mark.unit - def test_execution_affecting_sections_are_ignored(self, tmp_path: Path) -> None: - user = load_user_config(_write(tmp_path, EVIL)) - assert user.adapters == {} - assert user.mcp_servers == {} - assert user.control_groups == {} - assert user.stores.model_dump(exclude_none=True) == {} - assert "plugins" not in (user.model_extra or {}) - - @pytest.mark.unit - def test_extends_path_is_ignored(self, tmp_path: Path) -> None: - user = load_user_config(_write(tmp_path, EVIL)) - assert user.extends is None - - @pytest.mark.unit - def test_extends_framework_name_is_kept(self, tmp_path: Path) -> None: - user = load_user_config(_write(tmp_path, 'extends = "openssf-baseline"\n')) - assert user.extends == "openssf-baseline" - - @pytest.mark.unit - def test_status_exclusions_are_kept(self, tmp_path: Path) -> None: - user = load_user_config(_write(tmp_path, EVIL)) - override = user.get_control_override("OSPS-VM-02.01") - assert override is not None - assert override.status is not None and override.status.value == "n/a" - assert override.reason == "we have no vulnerability process" - - @pytest.mark.unit - def test_ignored_keys_are_reported(self, tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: - with caplog.at_level(logging.WARNING): - load_user_config(_write(tmp_path, EVIL)) - text = caplog.text - assert "operator configuration" in text - for key in ("passes", "mcp_servers", "adapters", "extends", "plugins"): - assert key in text - - @pytest.mark.unit - def test_benign_file_logs_nothing(self, tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: - benign = '[controls."OSPS-VM-02.01"]\nstatus = "n/a"\nreason = "library"\n' - with caplog.at_level(logging.WARNING): - load_user_config(_write(tmp_path, benign)) - assert caplog.text == "" - - -class TestOperatorOptIn: - @pytest.mark.unit - def test_trusted_argument_keeps_everything(self, tmp_path: Path) -> None: - user = load_user_config(_write(tmp_path, EVIL), trusted=True) - assert user.get_control_override("OSPS-DO-01.01").passes - assert "scanner" in user.mcp_servers - assert user.extends == "./evil-framework.toml" - - @pytest.mark.unit - @pytest.mark.parametrize("value", ["1", "true", "yes"]) - def test_environment_cannot_opt_in(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, value: str) -> None: - """There is no environment switch; trust comes from operator configuration.""" - monkeypatch.setenv("DARNIT_TRUST_REPO_CONFIG", value) - user = load_user_config(_write(tmp_path, EVIL)) - override = user.get_control_override("OSPS-DO-01.01") - assert override is None or not override.passes - - -class TestEndToEnd: - """The reported attack, through the MCP audit tool: the repository swaps a - control's passes for an exec command. The unrestricted case (restriction - disabled) proves the test reaches the code path that used to execute it.""" - - @pytest.mark.integration - @pytest.mark.parametrize(("restricted", "should_run"), [(True, False), (False, True)]) - def test_repo_supplied_command_is_not_run( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, restricted: bool, should_run: bool - ) -> None: - pytest.importorskip("darnit_baseline") - from darnit_baseline.tools import audit_openssf_baseline - - marker = tmp_path / "marker.txt" - repo = tmp_path / "repo" - repo.mkdir() - (repo / "README.md").write_text("# x\n", encoding="utf-8") - (repo / ".baseline.toml").write_text( - '[controls."OSPS-DO-01.01"]\n' - f'passes = [ {{ handler = "exec", command = ["sh", "-c", "echo pwned > {marker}"] }} ]\n', - encoding="utf-8", - ) - if not restricted: - monkeypatch.setattr(merger, "_restrict_untrusted_user_config", lambda data: (data, [])) - - audit_openssf_baseline(owner="o", repo="r", local_path=str(repo), output_format="json") - - assert marker.exists() is should_run diff --git a/tests/darnit/config/test_user_schema.py b/tests/darnit/config/test_user_schema.py deleted file mode 100644 index 2d963186..00000000 --- a/tests/darnit/config/test_user_schema.py +++ /dev/null @@ -1,397 +0,0 @@ -"""Tests for user_schema module. - -Tests for darnit.config.user_schema - Pydantic models for -user customisation configuration loaded from .baseline.toml. -""" - - -from darnit.config.framework_schema import CheckConfig, CommandAdapterConfig -from darnit.config.user_schema import ( - ControlGroup, - ControlOverride, - ControlStatus, - CustomControl, - UserConfig, - UserSettings, - create_user_config, - create_user_config_with_kusari, -) - -# --------------------------------------------------------------------------- -# UserSettings -# --------------------------------------------------------------------------- - - -class TestUserSettings: - def test_defaults(self): - s = UserSettings() - - assert s.cache_results is True - assert s.cache_ttl == 300 - assert s.timeout == 300 - assert s.fail_on_error is False - assert s.parallel_checks is True - assert s.max_parallel == 5 - - def test_custom_values(self): - s = UserSettings(cache_results=False, timeout=60, max_parallel=10) - - assert s.cache_results is False - assert s.timeout == 60 - assert s.max_parallel == 10 - - def test_extra_fields_allowed(self): - """UserSettings allows extra fields (ConfigDict extra=allow).""" - s = UserSettings(my_custom_key="value") - - assert s.my_custom_key == "value" - - -# --------------------------------------------------------------------------- -# ControlStatus enum -# --------------------------------------------------------------------------- - - -class TestControlStatus: - def test_na_value(self): - assert ControlStatus.NA == "n/a" - - def test_enabled_value(self): - assert ControlStatus.ENABLED == "enabled" - - def test_disabled_value(self): - assert ControlStatus.DISABLED == "disabled" - - -# --------------------------------------------------------------------------- -# ControlOverride -# --------------------------------------------------------------------------- - - -class TestControlOverride: - def test_minimal_override(self): - override = ControlOverride() - - assert override.status is None - assert override.reason is None - assert override.check is None - assert override.passes is None - - def test_na_override(self): - override = ControlOverride(status=ControlStatus.NA, reason="Not needed") - - assert override.status == ControlStatus.NA - assert override.reason == "Not needed" - - def test_disabled_override(self): - override = ControlOverride(status=ControlStatus.DISABLED, reason="Temp disabled") - - assert override.status == ControlStatus.DISABLED - - def test_check_override(self): - override = ControlOverride(check=CheckConfig(adapter="kusari")) - - assert override.check is not None - assert override.check.adapter == "kusari" - - -# --------------------------------------------------------------------------- -# ControlGroup -# --------------------------------------------------------------------------- - - -class TestControlGroup: - def test_basic_group(self): - group = ControlGroup( - controls=["CTRL-01", "CTRL-02"], - check=CheckConfig(adapter="kusari"), - ) - - assert "CTRL-01" in group.controls - assert group.check.adapter == "kusari" - - def test_empty_config_by_default(self): - group = ControlGroup(controls=["CTRL-01"]) - - assert group.config == {} - assert group.check is None - assert group.remediation is None - - -# --------------------------------------------------------------------------- -# UserConfig.get_control_override -# --------------------------------------------------------------------------- - - -class TestGetControlOverride: - def test_returns_none_when_no_overrides(self): - config = UserConfig() - - assert config.get_control_override("CTRL-01") is None - - def test_returns_direct_override(self): - override = ControlOverride(status=ControlStatus.NA, reason="N/A") - config = UserConfig(controls={"CTRL-01": override}) - - result = config.get_control_override("CTRL-01") - - assert result is not None - assert result.status == ControlStatus.NA - - def test_returns_override_from_dict(self): - config = UserConfig(controls={"CTRL-01": {"status": "n/a", "reason": "test"}}) - - result = config.get_control_override("CTRL-01") - - assert result is not None - assert result.status == ControlStatus.NA - - def test_returns_override_from_group(self): - group = ControlGroup( - controls=["CTRL-01"], - check=CheckConfig(adapter="kusari"), - ) - config = UserConfig(control_groups={"my-group": group}) - - result = config.get_control_override("CTRL-01") - - assert result is not None - assert result.check is not None - assert result.check.adapter == "kusari" - - def test_direct_override_takes_priority_over_group(self): - direct = ControlOverride(status=ControlStatus.NA, reason="direct") - group = ControlGroup( - controls=["CTRL-01"], - check=CheckConfig(adapter="kusari"), - ) - config = UserConfig( - controls={"CTRL-01": direct}, - control_groups={"my-group": group}, - ) - - result = config.get_control_override("CTRL-01") - - assert result.status == ControlStatus.NA - - -# --------------------------------------------------------------------------- -# UserConfig.is_control_applicable -# --------------------------------------------------------------------------- - - -class TestIsControlApplicable: - def test_applicable_by_default(self): - config = UserConfig() - - applicable, reason = config.is_control_applicable("CTRL-01") - - assert applicable is True - assert reason is None - - def test_not_applicable_when_na(self): - config = UserConfig( - controls={"CTRL-01": ControlOverride(status=ControlStatus.NA, reason="Pre-1.0")} - ) - - applicable, reason = config.is_control_applicable("CTRL-01") - - assert applicable is False - assert reason == "Pre-1.0" - - def test_not_applicable_when_disabled(self): - config = UserConfig( - controls={"CTRL-01": ControlOverride(status=ControlStatus.DISABLED, reason="skip")} - ) - - applicable, reason = config.is_control_applicable("CTRL-01") - - assert applicable is False - - def test_applicable_when_explicitly_enabled(self): - config = UserConfig( - controls={"CTRL-01": ControlOverride(status=ControlStatus.ENABLED)} - ) - - applicable, reason = config.is_control_applicable("CTRL-01") - - assert applicable is True - - -# --------------------------------------------------------------------------- -# UserConfig.get_check_adapter -# --------------------------------------------------------------------------- - - -class TestGetCheckAdapter: - def test_returns_none_when_no_override(self): - config = UserConfig() - - assert config.get_check_adapter("CTRL-01") is None - - def test_returns_adapter_from_override(self): - config = UserConfig( - controls={"CTRL-01": ControlOverride(check=CheckConfig(adapter="custom"))} - ) - - assert config.get_check_adapter("CTRL-01") == "custom" - - def test_returns_none_when_override_has_no_check(self): - config = UserConfig( - controls={"CTRL-01": ControlOverride(status=ControlStatus.NA)} - ) - - assert config.get_check_adapter("CTRL-01") is None - - -# --------------------------------------------------------------------------- -# UserConfig.get_custom_controls -# --------------------------------------------------------------------------- - - -class TestGetCustomControls: - def test_returns_empty_when_no_custom_controls(self): - config = UserConfig( - controls={"CTRL-01": ControlOverride(status=ControlStatus.NA)} - ) - - assert config.get_custom_controls() == {} - - def test_returns_custom_control_from_dict(self): - config = UserConfig( - controls={ - "CUSTOM-01": { - "name": "MyCustomControl", - "level": 1, - "domain": "SA", - "description": "Custom check", - "custom": True, - } - } - ) - - result = config.get_custom_controls() - - assert "CUSTOM-01" in result - assert result["CUSTOM-01"].name == "MyCustomControl" - - def test_returns_custom_control_instance(self): - custom = CustomControl( - name="MyControl", - level=1, - domain="SA", - description="My custom control", - ) - config = UserConfig(controls={"CUSTOM-01": custom}) - - result = config.get_custom_controls() - - assert "CUSTOM-01" in result - - def test_override_without_custom_fields_not_returned(self): - """ControlOverride without name/level/domain is NOT a custom control.""" - config = UserConfig( - controls={"CTRL-01": {"status": "n/a", "reason": "not applicable"}} - ) - - result = config.get_custom_controls() - - assert "CTRL-01" not in result - - -# --------------------------------------------------------------------------- -# UserConfig.get_adapter_config / get_all_adapter_names -# --------------------------------------------------------------------------- - - -class TestAdapterHelpers: - def test_get_adapter_config_returns_none_for_missing(self): - config = UserConfig() - - assert config.get_adapter_config("nonexistent") is None - - def test_get_adapter_config_returns_config(self): - adapter = CommandAdapterConfig(command="kusari") - config = UserConfig(adapters={"kusari": adapter}) - - result = config.get_adapter_config("kusari") - - assert result is not None - - def test_get_all_adapter_names_empty(self): - config = UserConfig() - - assert config.get_all_adapter_names() == [] - - def test_get_all_adapter_names(self): - config = UserConfig( - adapters={ - "kusari": CommandAdapterConfig(command="kusari"), - "custom": {"type": "script", "command": "./check.sh"}, - } - ) - - names = config.get_all_adapter_names() - - assert set(names) == {"kusari", "custom"} - - -# --------------------------------------------------------------------------- -# Factory functions -# --------------------------------------------------------------------------- - - -class TestCreateUserConfig: - def test_defaults_to_openssf_baseline(self): - config = create_user_config() - - assert config.extends == "openssf-baseline" - assert config.version == "1.0" - - def test_custom_extends(self): - config = create_user_config(extends="my-framework") - - assert config.extends == "my-framework" - - def test_extends_none(self): - config = create_user_config(extends=None) - - assert config.extends is None - - def test_returns_user_config_instance(self): - config = create_user_config() - - assert isinstance(config, UserConfig) - - -class TestCreateUserConfigWithKusari: - def test_has_kusari_adapter(self): - config = create_user_config_with_kusari() - - assert "kusari" in config.adapters - - def test_default_controls_for_kusari(self): - config = create_user_config_with_kusari() - - # Default controls are OSPS-VM-05.02 and OSPS-VM-05.03 in a group - all_group_controls = [] - for group in config.control_groups.values(): - all_group_controls.extend(group.controls) - - assert "OSPS-VM-05.02" in all_group_controls - assert "OSPS-VM-05.03" in all_group_controls - - def test_custom_controls_list(self): - config = create_user_config_with_kusari(controls=["CTRL-01", "CTRL-02"]) - - all_group_controls = [] - for group in config.control_groups.values(): - all_group_controls.extend(group.controls) - - assert "CTRL-01" in all_group_controls - assert "CTRL-02" in all_group_controls - - def test_extends_openssf_baseline(self): - config = create_user_config_with_kusari() - - assert config.extends == "openssf-baseline" diff --git a/tests/darnit/core/test_audit_cache.py b/tests/darnit/core/test_audit_cache.py index f78e5138..d93ae306 100644 --- a/tests/darnit/core/test_audit_cache.py +++ b/tests/darnit/core/test_audit_cache.py @@ -357,8 +357,6 @@ def test_run_sieve_audit_writes_cache(self, temp_git_repo: Path): with ( patch("darnit.tools.audit._get_sieve_components", return_value=sieve_components), patch("darnit.tools.audit._register_toml_controls", return_value=0), - patch("darnit.tools.audit.get_excluded_control_ids", return_value={}), - patch("darnit.config.load_user_config", return_value=None), ): from darnit.tools.audit import run_sieve_audit diff --git a/tests/darnit/fixtures/error_class_baseline/capture_baseline.py b/tests/darnit/fixtures/error_class_baseline/capture_baseline.py index bfa0f831..2d13d376 100644 --- a/tests/darnit/fixtures/error_class_baseline/capture_baseline.py +++ b/tests/darnit/fixtures/error_class_baseline/capture_baseline.py @@ -150,7 +150,7 @@ def main() -> int: run_sieve_audit, ) - config = load_effective_config_by_name("openssf-baseline", repo_path=fixture) + config = load_effective_config_by_name("openssf-baseline") all_controls = load_controls_from_effective(config) controls = filter_controls( all_controls, {}, set(DETERMINISTIC_CONTROL_IDS), None @@ -164,7 +164,7 @@ def main() -> int: default_branch="main", level=1, controls=controls, - apply_user_config=False, + evaluate_claims=False, stop_on_llm=True, ) diff --git a/tests/darnit/sieve/baseline_capture.py b/tests/darnit/sieve/baseline_capture.py index c4d776d0..1209014e 100644 --- a/tests/darnit/sieve/baseline_capture.py +++ b/tests/darnit/sieve/baseline_capture.py @@ -174,7 +174,7 @@ def _capture_one(framework: str, repo_path: str, neutralize: bool = False) -> di default_branch="main", level=3, stop_on_llm=True, - apply_user_config=False, + evaluate_claims=False, framework_name=framework, ) return {_control_id(r): _status(r) for r in results} diff --git a/tests/darnit/sieve/test_expr_failure.py b/tests/darnit/sieve/test_expr_failure.py index 37f0bc7e..2e299761 100644 --- a/tests/darnit/sieve/test_expr_failure.py +++ b/tests/darnit/sieve/test_expr_failure.py @@ -420,7 +420,7 @@ def _audit(self, repo: Path) -> dict[str, Any]: str(repo), "main", controls=[_step_control(handler="exec", command=["true"], expr=self.EXPR)], - apply_user_config=False, + evaluate_claims=False, stop_on_llm=False, framework_name="openssf-baseline", operator_config=operator, diff --git a/tests/darnit/sieve/test_handler_architecture.py b/tests/darnit/sieve/test_handler_architecture.py index 17961c80..d66b91e9 100644 --- a/tests/darnit/sieve/test_handler_architecture.py +++ b/tests/darnit/sieve/test_handler_architecture.py @@ -811,7 +811,7 @@ def test_use_locator_resolved_through_effective_config(self): ) defaults = FrameworkDefaults() - effective = merge_control("T-01", control, None, defaults) + effective = merge_control("T-01", control, defaults) # The passes_config should have files resolved from locator assert effective.passes_config is not None @@ -955,7 +955,7 @@ def test_na_without_when_no_requires_line(self): { "id": "OSPS-BR-01.01", "status": "N/A", - "details": "Excluded via .baseline.toml", + "details": "Not applicable (asserted by repository content)", "level": 1, }, ] diff --git a/tests/darnit/stores/test_us2_zero_config.py b/tests/darnit/stores/test_us2_zero_config.py index ee689409..0eded4ab 100644 --- a/tests/darnit/stores/test_us2_zero_config.py +++ b/tests/darnit/stores/test_us2_zero_config.py @@ -1,7 +1,7 @@ """US2 SC-003 zero-config invariance. Feature 033 T030. When no ``[stores.*]`` block is present in either the -framework TOML or ``.baseline.toml``, ``resolve_stores`` must produce +framework TOML or operator configuration, ``resolve_stores`` must produce the four filesystem defaults and NOT construct any plugin backend. Pre-feature audit paths (system tempdir cache, on-disk .project/, attestations to repo root) remain the ground truth for what darnit does diff --git a/tests/darnit/test_audit_cache_store_wiring.py b/tests/darnit/test_audit_cache_store_wiring.py index da66b503..53c99b37 100644 --- a/tests/darnit/test_audit_cache_store_wiring.py +++ b/tests/darnit/test_audit_cache_store_wiring.py @@ -78,7 +78,7 @@ def _run_audit_with_stores_config(repo: Path, stores_config: StoresConfig | None Patches ``_load_merged_stores`` to return ``stores_config`` (so we bypass the framework-registration step) and ``_register_toml_controls`` - plus ``get_excluded_control_ids`` per the existing test pattern. + per the existing test pattern. """ with ( patch( @@ -86,12 +86,10 @@ def _run_audit_with_stores_config(repo: Path, stores_config: StoresConfig | None return_value=_mock_sieve_components(), ), patch("darnit.tools.audit._register_toml_controls", return_value=0), - patch("darnit.tools.audit.get_excluded_control_ids", return_value={}), patch( "darnit.tools.audit._load_merged_stores", return_value=stores_config, ), - patch("darnit.config.load_user_config", return_value=None), ): from darnit.tools.audit import run_sieve_audit diff --git a/tests/darnit/test_audit_mapper_integration.py b/tests/darnit/test_audit_mapper_integration.py index 440bc698..08ac77ca 100644 --- a/tests/darnit/test_audit_mapper_integration.py +++ b/tests/darnit/test_audit_mapper_integration.py @@ -27,10 +27,8 @@ def mock_sieve_components(self): @patch("darnit.tools.audit._get_sieve_components") @patch("darnit.tools.audit._register_toml_controls") - @patch("darnit.tools.audit.get_excluded_control_ids", return_value={}) def test_mapper_context_injected( self, - mock_excluded, mock_register, mock_get_sieve, mock_sieve_components, @@ -61,10 +59,8 @@ def test_mapper_context_injected( @patch("darnit.tools.audit._get_sieve_components") @patch("darnit.tools.audit._register_toml_controls") - @patch("darnit.tools.audit.get_excluded_control_ids", return_value={}) def test_user_context_overrides_mapper( self, - mock_excluded, mock_register, mock_get_sieve, mock_sieve_components, @@ -117,10 +113,8 @@ def test_user_context_overrides_mapper( @patch("darnit.tools.audit._get_sieve_components") @patch("darnit.tools.audit._register_toml_controls") - @patch("darnit.tools.audit.get_excluded_control_ids", return_value={}) def test_mapper_failure_is_non_fatal( self, - mock_excluded, mock_register, mock_get_sieve, mock_sieve_components, @@ -147,10 +141,8 @@ def test_mapper_failure_is_non_fatal( @patch("darnit.tools.audit._get_sieve_components") @patch("darnit.tools.audit._register_toml_controls") - @patch("darnit.tools.audit.get_excluded_control_ids", return_value={}) def test_mapper_called_without_owner( self, - mock_excluded, mock_register, mock_get_sieve, mock_sieve_components, diff --git a/tests/darnit/test_plugin_handler_registration.py b/tests/darnit/test_plugin_handler_registration.py index 97b0096f..e91f4122 100644 --- a/tests/darnit/test_plugin_handler_registration.py +++ b/tests/darnit/test_plugin_handler_registration.py @@ -152,7 +152,7 @@ def test_example_hygiene_loads_with_a_fresh_registry(self, monkeypatch: pytest.M from darnit.config.merger import load_effective_config_by_name monkeypatch.setattr(handler_registry, "_sieve_handler_registry", None) - config = load_effective_config_by_name("example-hygiene", repo_path=None) + config = load_effective_config_by_name("example-hygiene") assert len(load_controls_from_effective(config)) == len(config.controls) assert get_sieve_handler_registry().get("readme_description") is not None @@ -172,7 +172,7 @@ class TestControlOrderDeterminism: def _merge(self): from darnit.config.merger import load_effective_config_by_name - return load_effective_config_by_name("openssf-baseline", repo_path=None) + return load_effective_config_by_name("openssf-baseline") @pytest.mark.unit def test_control_order_is_stable_across_merges(self) -> None: diff --git a/tests/darnit_baseline/corpus/runner.py b/tests/darnit_baseline/corpus/runner.py index 47015579..8cafc525 100644 --- a/tests/darnit_baseline/corpus/runner.py +++ b/tests/darnit_baseline/corpus/runner.py @@ -593,7 +593,7 @@ def run_fixture(fixture: Fixture, framework: Framework) -> FixtureRun: BRANCH, LEVEL, controls=list(framework.controls), - apply_user_config=False, + evaluate_claims=False, stop_on_llm=True, framework_name=framework.implementation, operator_config=_builtin_operator_config(), diff --git a/tests/darnit_baseline/test_baseline_authority.py b/tests/darnit_baseline/test_baseline_authority.py index 937aef40..80b1351f 100644 --- a/tests/darnit_baseline/test_baseline_authority.py +++ b/tests/darnit_baseline/test_baseline_authority.py @@ -44,7 +44,7 @@ def _steps(control) -> list: def test_every_control_loads_on_the_effective_path(controls: list) -> None: - effective = load_controls_from_effective(load_effective_config_by_name(FRAMEWORK, repo_path=None)) + effective = load_controls_from_effective(load_effective_config_by_name(FRAMEWORK)) assert {c.control_id for c in effective} == {c.control_id for c in controls} diff --git a/tests/darnit_baseline/test_handler_dispatch_integration.py b/tests/darnit_baseline/test_handler_dispatch_integration.py index a210e463..5267c057 100644 --- a/tests/darnit_baseline/test_handler_dispatch_integration.py +++ b/tests/darnit_baseline/test_handler_dispatch_integration.py @@ -437,11 +437,10 @@ class TestUseLocatorEffectivePath: @pytest.mark.unit def test_use_locator_controls_have_files_in_effective_config(self): """All use_locator=true controls must have files populated after effective loading.""" - from pathlib import Path from darnit.config import load_controls_from_effective, load_effective_config_by_name - config = load_effective_config_by_name("openssf-baseline", Path(".")) + config = load_effective_config_by_name("openssf-baseline") controls = load_controls_from_effective(config) # Known controls that use use_locator=true @@ -478,11 +477,10 @@ def test_osps_do_02_01_effective_config_uses_bug_globs(self): pass now carries the full glob list including the bug*.* variants that used to be split between file_exists and pattern. """ - from pathlib import Path from darnit.config import load_controls_from_effective, load_effective_config_by_name - config = load_effective_config_by_name("openssf-baseline", Path(".")) + config = load_effective_config_by_name("openssf-baseline") controls = load_controls_from_effective(config) control = next(ctrl for ctrl in controls if ctrl.control_id == "OSPS-DO-02.01") invocations = control.metadata["handler_invocations"] @@ -520,7 +518,6 @@ def test_osps_do_02_01_finds_bug_template(self, tmp_path, template_filename): waits for a model judgment (PENDING) rather than PASS; this test pins the globbing, which is what the regression was about. """ - from pathlib import Path from darnit.config import load_controls_from_effective, load_effective_config_by_name @@ -542,7 +539,7 @@ def test_osps_do_02_01_finds_bug_template(self, tmp_path, template_filename): """, ) - config = load_effective_config_by_name("openssf-baseline", Path(".")) + config = load_effective_config_by_name("openssf-baseline") controls = load_controls_from_effective(config) control = next(ctrl for ctrl in controls if ctrl.control_id == "OSPS-DO-02.01") @@ -583,7 +580,6 @@ def test_osps_do_02_01_warns_for_feature_only_template(self, tmp_path): INCONCLUSIVE for the no-matching-files case, and pass 3 still returns WARN. See specs/020-definitive-fail-verdict/. """ - from pathlib import Path from darnit.config import load_controls_from_effective, load_effective_config_by_name @@ -602,7 +598,7 @@ def test_osps_do_02_01_warns_for_feature_only_template(self, tmp_path): """, ) - config = load_effective_config_by_name("openssf-baseline", Path(".")) + config = load_effective_config_by_name("openssf-baseline") controls = load_controls_from_effective(config) control = next(ctrl for ctrl in controls if ctrl.control_id == "OSPS-DO-02.01") @@ -628,7 +624,6 @@ def test_osps_do_02_01_matches_contributing_md_with_bug_link(self, tmp_path): is evidence for the control even without a .github/ISSUE_TEMPLATE/bug* file. Feature 041: that match is evidence, not a PASS. """ - from pathlib import Path from darnit.config import load_controls_from_effective, load_effective_config_by_name @@ -638,7 +633,7 @@ def test_osps_do_02_01_matches_contributing_md_with_bug_link(self, tmp_path): "To report a bug, file an issue at https://github.com/x/y/issues\n" ) - config = load_effective_config_by_name("openssf-baseline", Path(".")) + config = load_effective_config_by_name("openssf-baseline") controls = load_controls_from_effective(config) control = next(ctrl for ctrl in controls if ctrl.control_id == "OSPS-DO-02.01") diff --git a/tests/darnit_baseline/test_weak_evidence.py b/tests/darnit_baseline/test_weak_evidence.py index 8da70bae..82b85747 100644 --- a/tests/darnit_baseline/test_weak_evidence.py +++ b/tests/darnit_baseline/test_weak_evidence.py @@ -63,7 +63,7 @@ def _audit(repo: Path) -> dict[str, dict]: default_branch="main", level=3, stop_on_llm=True, - apply_user_config=False, + evaluate_claims=False, framework_name="openssf-baseline", ) return {r["id"]: r for r in results} diff --git a/tests/darnit_csl/test_csl.py b/tests/darnit_csl/test_csl.py index c355ec8a..c2a96ce7 100644 --- a/tests/darnit_csl/test_csl.py +++ b/tests/darnit_csl/test_csl.py @@ -67,7 +67,7 @@ def _status(cid: str, repo: Path, framework_name: str = "community-spec") -> str repo="r", local_path=str(repo), default_branch="main", - apply_user_config=False, + evaluate_claims=False, framework_name=framework_name, stop_on_llm=False, ) From 42261d7e47b4fa92a1e63cdadc477e211c42f6e4 Mon Sep 17 00:00:00 2001 From: Michael Lieberman Date: Sun, 4 Oct 2026 20:45:30 -0400 Subject: [PATCH 4/5] docs(changelog): .baseline.toml is no longer read Replace the unreleased deprecation entries with the BREAKING removal: one notice pointing at `darnit config migrate`, the removed Python APIs and renamed parameter, and `darnit run --framework`. The 040 addendum lists the tools.audit helpers and the evaluate_claims rename too. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Michael Lieberman --- CHANGELOG.md | 51 ++++++++++++++----- .../deprecation-completed.md | 8 ++- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e5588a9..70ff188a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed +- `.baseline.toml` reading and the code that existed only for it: + `load_user_config` (including its `trusted` path), + `load_user_config_with_report`, `validate_user_config`, `deep_merge`, and + `BASELINE_TOML_DEPRECATION_ACTIVE` (`darnit.config.merger`); + `darnit.config.user_schema` (`UserConfig`, `UserSettings`, + `ControlOverride`, `ControlGroup`, `CustomControl`, `ControlStatus`, + `create_user_config`, `create_user_config_with_kusari`) and their + `darnit.config` re-exports (`UserControlOverride`, `UserControlStatus`); + and `load_effective_audit_config`, `get_excluded_control_ids`, and + `get_adapter_for_control` (`darnit.tools.audit`). Custom controls and pass + overrides belong in operator configuration. +- The root `example.baseline.toml`. - Python helpers that read or wrote raw context values: `load_context`, `load_stored_context`, `flatten_user_context`, `get_context_value`, `get_raw_value`, `is_context_confirmed`, `save_context_value`, and @@ -49,6 +61,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 inside the audited repository is refused, and a file writable by others is refused under `--strict-operator-config` (on by default in recognized CI). Reports record its source and digest. +- `darnit run -f/--framework NAME` selects the framework, as `audit` and + `harness` do (#507). - `darnit config show` (resolved operator configuration, digest, permission check, and redacted settings), `darnit config trust add|list|remove` (edits `[trust].repos`), and `darnit config migrate [REPO] [--force]` @@ -260,12 +274,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 network in CI now FAILs RE-02.01. The Nix signal counts only when RE-01.02 passed, and RE-01.02 no longer concludes PASS on its own, so the fetch is no longer outweighed. -- Per-control `status` and `reason` in a repository's `.baseline.toml`, ignored - since 0.1.1, are read during the deprecation release as not-applicable - claims under the trust rules: a claim counts only for a repository the - operator trusts, or after the operator confirms it; otherwise the control - is evaluated and counts as non-compliant. `darnit config migrate` moves - them to `.project/darnit.yaml`. - **BREAKING:** `PENDING_LLM` is removed; `PENDING` with `pending.kind = "llm_judgment"` replaces it in every output, including MCP tool results and JSON reports. @@ -386,13 +394,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 replaces an invalid `.project/project.yaml` with a scaffold: when either `.project/` file is present but invalid, writes are refused with the validation errors, and audit reports list them as warnings. -- `.baseline.toml` is deprecated. In this release darnit reads its - per-control `status`/`reason` as not-applicable claims under the same rules - as `.project/` claims, still honors `extends` naming a registered framework, - ignores its tool settings (see Security above), and warns once per setting - in the file with the setting's new home. A later release will ignore the - file with a notice. `darnit init` no longer creates `.baseline.toml`; it - explains `.project/` claims and operator configuration. +- **BREAKING:** darnit no longer reads a repository's `.baseline.toml`. + Nothing in it has any effect: not per-control `status`/`reason` (even for + a repository the operator trusts), not `extends` (use `--framework`), and + not `version` or `settings`, which 0.1.1 still honored. When the file is + present, an audit logs one WARNING and adds the same notice to the report's + `warnings`, pointing at `darnit config migrate`, which moves its claims to + `.project/darnit.yaml` and prints an operator configuration fragment for + its other settings. Its keys are no longer listed in + `ignored_repository_settings`. `darnit init` no longer creates + `.baseline.toml`; it explains `.project/` claims and operator + configuration. +- **BREAKING:** Python API changes from removing `.baseline.toml`: + `merge_configs(framework, operator=None)` and + `merge_control(control_id, framework_control, defaults)` take no user + configuration; `load_effective_config`, `load_effective_config_by_name`, + `load_controls_from_toml`, and `load_controls_by_name` take no repository + path, and `load_effective_config_auto(framework_path=None, + framework_name=None, *, operator=None)` no longer takes one; + `EffectiveControl` loses `status`, `status_reason`, `from_user`, and + `is_applicable()`, and `EffectiveConfig` loses `cache_results`, + `cache_ttl`, `timeout`, and `get_excluded_controls()`; + `run_checks`/`run_sieve_audit` `apply_user_config` is renamed + `evaluate_claims` (it still turns `.project/` claim evaluation on or off). + No MCP tool or CLI parameter served only `.baseline.toml`. - A not-applicable claim makes a control `N/A` (excluded from the level's denominator) only when it is honored: the repository is trusted, an explicit claim gives a reason, and no declared evidence contradicts it, or diff --git a/specs/040-operator-config-trust/deprecation-completed.md b/specs/040-operator-config-trust/deprecation-completed.md index 4e95abc7..73db9dfa 100644 --- a/specs/040-operator-config-trust/deprecation-completed.md +++ b/specs/040-operator-config-trust/deprecation-completed.md @@ -16,8 +16,8 @@ Removed with it: - The `BASELINE_TOML_DEPRECATION_ACTIVE` switch and the per-setting deprecation warnings. - `load_user_config` (with its `trusted` path) and - `load_user_config_with_report`, the untrusted-file restriction, and - `validate_user_config`. + `load_user_config_with_report`, the untrusted-file restriction, + `validate_user_config`, and `deep_merge`. - `.baseline.toml` claims in `darnit.trust.assertions`. - `darnit.config.user_schema` (`UserConfig`, `UserSettings`, `ControlOverride`, `ControlGroup`, `CustomControl`, `ControlStatus`, @@ -29,6 +29,10 @@ Removed with it: `EffectiveConfig` loses the `.baseline.toml` settings and `get_excluded_controls()`; `load_effective_config*` and the control loaders take no repository path. +- `darnit.tools.audit` `load_effective_audit_config`, + `get_excluded_control_ids`, and `get_adapter_for_control`, which read the + repository's file. `run_checks`/`run_sieve_audit` `apply_user_config` is + renamed `evaluate_claims`; it still gates `.project/` claim evaluation. ## The notice From 5c1107b41d84629908f63591adad31fbd5c8fd70 Mon Sep 17 00:00:00 2001 From: Michael Lieberman Date: Sun, 4 Oct 2026 21:24:28 -0400 Subject: [PATCH 5/5] fix(cli): darnit run rejects an unknown framework and defaults like audit An unknown --framework name loaded nothing and the run reported a clean result over zero controls, and without --framework the run audited no controls at all now that .baseline.toml extends is gone. Resolve the framework the way darnit audit does: an unknown name exits 1, and the default is openssf-baseline. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Michael Lieberman --- CHANGELOG.md | 4 ++- packages/darnit/src/darnit/cli.py | 11 +++++++- tests/darnit/cli/test_cmd_run_e2e.py | 39 ++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70ff188a..60e5fbe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,7 +62,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 refused under `--strict-operator-config` (on by default in recognized CI). Reports record its source and digest. - `darnit run -f/--framework NAME` selects the framework, as `audit` and - `harness` do (#507). + `harness` do (#507). An unknown name exits 1 instead of reporting a clean + run over no controls; without the option, `run` audits `openssf-baseline`, + the same default as `audit`. - `darnit config show` (resolved operator configuration, digest, permission check, and redacted settings), `darnit config trust add|list|remove` (edits `[trust].repos`), and `darnit config migrate [REPO] [--force]` diff --git a/packages/darnit/src/darnit/cli.py b/packages/darnit/src/darnit/cli.py index cfd69f8f..b3756feb 100644 --- a/packages/darnit/src/darnit/cli.py +++ b/packages/darnit/src/darnit/cli.py @@ -798,6 +798,7 @@ def cmd_run(args: argparse.Namespace) -> int: from darnit.agent.feedback import get_feedback_handler from darnit.agent.graph import audit, collect_context, remediate, route from darnit.agent.state import AuditState + from darnit.config.merger import load_effective_config_auto repo_path = str(Path(args.repo_path).resolve()) @@ -812,6 +813,14 @@ def cmd_run(args: argparse.Namespace) -> int: trust = decide_trust(target, operator_config.config, repo_path).report() + try: + framework_name = load_effective_config_auto( + framework_name=args.framework, operator=operator_config.config + ).framework_name + except (ValueError, FileNotFoundError) as e: + logger.error(f"Failed to load framework {args.framework!r}: {e}") + return 1 + # Feedback mode — default to interactive if terminal, noninteractive if not feedback_mode = args.feedback_mode if feedback_mode == "auto": @@ -832,7 +841,7 @@ def cmd_run(args: argparse.Namespace) -> int: owner=owner, repo=repo, target=target, - framework_name=args.framework, + framework_name=framework_name, level=getattr(args, "level", 3), ) diff --git a/tests/darnit/cli/test_cmd_run_e2e.py b/tests/darnit/cli/test_cmd_run_e2e.py index affbd619..32e2ce92 100644 --- a/tests/darnit/cli/test_cmd_run_e2e.py +++ b/tests/darnit/cli/test_cmd_run_e2e.py @@ -579,3 +579,42 @@ def test_testchecks_controls_are_audited( ) assert _parse_counts(stdout)["total"] > 0 + + def test_unknown_framework_is_an_error( + self, + minimal_repo_tree: Path, + capsys: pytest.CaptureFixture[str], + caplog: pytest.LogCaptureFixture, + ) -> None: + """An unknown name exits 1 instead of reporting a clean run over no controls (Principle II).""" + exit_code, stdout, stderr = invoke_cmd_run( + [str(minimal_repo_tree), "--framework", "nonexistent-fw", "--feedback", "noninteractive"], + capsys, + ) + + assert exit_code == 1 + assert "Run complete." not in stdout + assert "nonexistent-fw" in stderr + caplog.text + + def test_default_framework_matches_audit( + self, + minimal_repo_tree: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Without --framework, run audits the default framework, as ``darnit audit`` does.""" + from darnit.agent import graph + + seen: list[str | None] = [] + + def audited(state): + seen.append(state.framework_name) + state.audit_results = [] + state.error = None + return state + + monkeypatch.setattr(graph, "audit", audited) + + invoke_cmd_run([str(minimal_repo_tree), "--feedback", "noninteractive"], capsys) + + assert seen == ["openssf-baseline"]