diff --git a/internal/generate/correctness_assertions_test.go b/internal/generate/correctness_assertions_test.go new file mode 100644 index 0000000..9cdeb2f --- /dev/null +++ b/internal/generate/correctness_assertions_test.go @@ -0,0 +1,200 @@ +package generate + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stablekernel/cascade/internal/config" +) + +// This file holds the generation-CORRECTNESS assertions the census +// (correctness_census_test.go) maps emitted-affecting fields to. Each pins the +// EXACT emitted shape for a field whose output carries a distinct semantic +// contract (least-privilege, secret propagation direction, pinned refs, trigger +// expressions), beyond the validity the actionlint sweeps already guarantee. +// Each is red-first: deliberately breaking its emitter makes it fail. + +// correctnessDir stages the orchestrate/deploy callback stubs guardBaseConfig +// references, so the correctness assertions generate a real workflow set. +func correctnessDir(t *testing.T) string { + t.Helper() + return callbackWorkflowDir(t, "build.yaml", "deploy.yaml") +} + +// TestGenCorrectness_Permissions_LeastPrivilegePerJob pins the least-privilege +// contract for builds[].permissions / deploys[].permissions: a job with a +// configured permissions map emits exactly that block, scoped to that job, and +// a job with NO configured permissions emits NO permissions block (it inherits, +// rather than silently receiving an elevated default). A regression that +// emitted a default block, or leaked one job's scopes onto another, reds here. +func TestGenCorrectness_Permissions_LeastPrivilegePerJob(t *testing.T) { + dir := correctnessDir(t) + cfg := guardBaseConfig() + cfg.Builds[0].Permissions = map[string]string{"contents": "read", "id-token": "write"} + + out, err := NewGenerator(cfg, dir).Generate() + require.NoError(t, err) + + build := pass10JobBlock(t, out, "build-app") + assert.Contains(t, build, "permissions:", "a build with a configured permissions map must emit the block") + assert.Contains(t, build, "contents: read", "configured scopes must be emitted verbatim") + assert.Contains(t, build, "id-token: write") + + deploy := pass10JobBlock(t, out, "deploy-runner") + assert.NotContains(t, deploy, "permissions:", + "a job with no configured permissions must emit no block (least privilege, not a default grant)") + assert.NotContains(t, deploy, "id-token: write", + "one job's scopes must never leak onto another job") +} + +// TestGenCorrectness_SecretsMap_PropagatesSourceToCallee pins the direction of +// a secrets.map entry {calleeInput: sourceSecret}: the emitted secrets: block +// maps the callee input name to ${{ secrets. }}, never the reverse. A +// swapped direction would hand the callee the wrong secret under the right name, +// a valid-but-wrong emission actionlint cannot see. +func TestGenCorrectness_SecretsMap_PropagatesSourceToCallee(t *testing.T) { + dir := correctnessDir(t) + cfg := guardBaseConfig() + cfg.Builds[0].Secrets = &config.SecretsConfig{Map: map[string]string{"GOOD_IN": "GOOD_OUT"}} + + out, err := NewGenerator(cfg, dir).Generate() + require.NoError(t, err) + + build := pass10JobBlock(t, out, "build-app") + assert.Contains(t, build, "GOOD_IN: ${{ secrets.GOOD_OUT }}", + "a secrets.map entry must map the callee input to the source secret expression") + assert.NotContains(t, build, "GOOD_OUT: ${{ secrets.GOOD_IN }}", + "the mapping direction must not be reversed") +} + +// TestGenCorrectness_Concurrency_CancelInProgressHonored pins that a manifest +// cancel_in_progress: false emits cancel-in-progress: false, not a defaulted +// true. The two produce opposite runtime behavior (queued vs cancelled runs), +// so a value dropped to its zero or a hardcoded literal would silently invert +// the operator's intent. +func TestGenCorrectness_Concurrency_CancelInProgressHonored(t *testing.T) { + dir := correctnessDir(t) + + for _, tc := range []struct { + name string + val bool + want string + }{ + {"false", false, "cancel-in-progress: false"}, + {"true", true, "cancel-in-progress: true"}, + } { + tc := tc + t.Run(tc.name, func(t *testing.T) { + cfg := guardBaseConfig() + cfg.Concurrency = &config.ConcurrencyConfig{ + Group: "orchestrate-${{ github.ref }}", + CancelInProgress: boolPtr(tc.val), + } + out, err := NewGenerator(cfg, dir).Generate() + require.NoError(t, err) + assert.Contains(t, out, tc.want, "cancel_in_progress must be emitted as configured, not defaulted") + }) + } +} + +// TestGenCorrectness_ActionPins_UsesCarriesPinnedRef pins that an action_pins +// override splices the pinned ref into the uses: line for that action, so a +// repo that pins actions/checkout to a SHA gets that SHA (with its trailing +// comment), not the generator's built-in default pin. +func TestGenCorrectness_ActionPins_UsesCarriesPinnedRef(t *testing.T) { + dir := correctnessDir(t) + cfg := guardBaseConfig() + cfg.ActionPins = map[string]string{"actions/checkout": "0123abcd # v4"} + + out, err := NewGenerator(cfg, dir).Generate() + require.NoError(t, err) + assert.Contains(t, out, "uses: actions/checkout@0123abcd # v4", + "an action_pins override must splice the pinned ref into the uses: line") +} + +// TestGenCorrectness_DispatchInputs_EmitsTypedInputBlock pins that a +// dispatch_inputs entry emits a workflow_dispatch input keyed by its name with +// the configured type, options, and default, so an operator dispatch form +// exposes the declared choices rather than a bare free-text field. +func TestGenCorrectness_DispatchInputs_EmitsTypedInputBlock(t *testing.T) { + dir := correctnessDir(t) + cfg := guardBaseConfig() + cfg.DispatchInputs = map[string]config.DispatchInput{ + "mode": { + Type: config.DispatchInputTypeChoice, + Options: []string{"fast", "slow"}, + Default: "fast", + }, + } + + out, err := NewGenerator(cfg, dir).Generate() + require.NoError(t, err) + assert.Contains(t, out, "mode:", "the dispatch input must be keyed by its name") + assert.Contains(t, out, "type: choice", "the input type must be emitted") + assert.Contains(t, out, "options:") + assert.Contains(t, out, "- fast") + assert.Contains(t, out, "- slow") + assert.Contains(t, out, "default: 'fast'", "the default must be emitted, single-quote-escaped") +} + +// TestGenCorrectness_ExtraTriggers_EmitsScheduleAndDispatch pins that +// extra_triggers emit the corresponding on: trigger blocks (schedule cron, +// repository_dispatch types, workflow_run workflows+types) into the orchestrate +// workflow, so a manifest that asks for a nightly cron or an upstream-run +// trigger actually gets one. +func TestGenCorrectness_ExtraTriggers_EmitsScheduleAndDispatch(t *testing.T) { + dir := correctnessDir(t) + cfg := guardBaseConfig() + cfg.ExtraTriggers = &config.ExtraTriggers{ + Schedule: []config.ScheduleEntry{{Cron: "0 2 * * 1-5"}}, + RepositoryDispatch: &config.RepositoryDispatchTrigger{Types: []string{"external-update"}}, + WorkflowRun: &config.WorkflowRunTrigger{Workflows: []string{"Upstream CI"}, Types: []string{"completed"}}, + } + + out, err := NewGenerator(cfg, dir).Generate() + require.NoError(t, err) + assert.Contains(t, out, "- cron: '0 2 * * 1-5'", "schedule cron must be emitted, hard single-quoted") + assert.Contains(t, out, "repository_dispatch:") + assert.Contains(t, out, "- external-update", "repository_dispatch types must be emitted") + assert.Contains(t, out, "workflow_run:") + assert.Contains(t, out, "- 'Upstream CI'", "workflow_run workflow names must be emitted, single-quoted") +} + +// TestGenCorrectness_GitCustomUser_SplicesNameAndEmail pins that a custom git +// identity (git.mode: custom) splices the operator's name and email into the +// emitted git config commands, so state/finalize commits are attributed to the +// configured bot identity rather than the runner default. +func TestGenCorrectness_GitCustomUser_SplicesNameAndEmail(t *testing.T) { + dir := correctnessDir(t) + cfg := guardBaseConfig() + cfg.Git = &config.GitConfig{Mode: config.GitModeCustom, UserName: "Release Bot", UserEmail: "bot@example.com"} + + out, err := NewGenerator(cfg, dir).Generate() + require.NoError(t, err) + assert.Contains(t, out, `git config user.name "Release Bot"`, + "a custom git identity must splice the configured user name") + assert.Contains(t, out, `git config user.email "bot@example.com"`, + "a custom git identity must splice the configured user email") +} + +// TestGenCorrectness_EnvironmentURL_EmitsPerEnvCase pins that +// environments[].environment_url is threaded into the native-deployment status +// step as a per-environment shell case, so the GitHub Deployment records the +// configured URL for that environment. The runtime proof that the Deployment +// API receives it is fleet-only; this pins the emitted case shape. +func TestGenCorrectness_EnvironmentURL_EmitsPerEnvCase(t *testing.T) { + dir := correctnessDir(t) + cfg := guardBaseConfig() + cfg.Deployments = &config.DeploymentsConfig{Enabled: boolPtr(true)} + cfg.Environments = []config.EnvironmentEntry{ + {Name: "dev"}, + {Name: "prod", EnvironmentConfig: config.EnvironmentConfig{EnvironmentURL: "https://app.example.com"}}, + } + + out, err := NewPromoteGenerator(cfg, dir).Generate() + require.NoError(t, err) + assert.Contains(t, out, "prod) environment_url='https://app.example.com'", + "the configured environment_url must be threaded as a per-environment shell case") +} diff --git a/internal/generate/correctness_census_map_test.go b/internal/generate/correctness_census_map_test.go new file mode 100644 index 0000000..2b01e91 --- /dev/null +++ b/internal/generate/correctness_census_map_test.go @@ -0,0 +1,184 @@ +package generate + +// correctnessCensus classifies every emitted-affecting manifest field (the +// union emittedAffectingCensus() computes from emittedFieldRegistry and +// emittedAffectingAllowlistMutators) by its generation-correctness coverage. +// TestCorrectnessCensus_EveryEmittedFieldClassified forces every census field +// into exactly one entry here; a new emitted-affecting field reds that guard +// until it is classified. +// +// An entry is one of: +// - assertion: names a test in this package that pins the field's emitted +// semantic shape (the field carries a distinct downstream contract). +// - markerValidityOnly: the only contract is validity + round-trip, already +// pinned by the T1 battery and the actionlint sweep (free scalar/glob/name +// with no distinct downstream shape). +// - markerStructural: the emitted shape is identical to a sibling's and is +// pinned by that sibling's named assertion (note carries the Test name). +// - markerNotEmitted: the supported value produces no emitted splice +// (validated-only enum, reserved block, or the alternative is rejected). +// - markerFollowup: emitted-affecting with a distinct semantic shape that +// warrants a correctness assertion NOT YET written. The markerFollowup set +// is the enumerated remaining-fields backlog for the next tranche. +var correctnessCensus = map[string]correctnessCoverage{ + // -- action folder / pins ------------------------------------------------- + "action_folder": {marker: markerValidityOnly, note: "composite-action folder name; spliced into a filesystem path and uses:, validity + round-trip is the contract"}, + "action_pins.*": {assertion: "TestGenCorrectness_ActionPins_UsesCarriesPinnedRef"}, + "action_pins[key]": {marker: markerStructural, note: "lookup key selecting a pinned action; the spliced ref is pinned by TestGenCorrectness_ActionPins_UsesCarriesPinnedRef"}, + + // -- builds --------------------------------------------------------------- + "builds[].artifacts[].name": {marker: markerValidityOnly, note: "artifact name in emitted shell; validity + round-trip is the contract"}, + "builds[].artifacts[].path": {marker: markerValidityOnly, note: "artifact glob in emitted shell; validity + round-trip is the contract"}, + "builds[].artifact_upload.upload": {marker: markerValidityOnly, note: "passthrough upload glob; validity + round-trip is the contract"}, + "builds[].artifact_upload.downloads[]": {marker: markerValidityOnly, note: "passthrough download job name; validity + round-trip is the contract"}, + "builds[].inputs[key]": {marker: markerValidityOnly, note: "with: input key; validity + round-trip is the contract"}, + "builds[].inputs.*": {assertion: "TestGenerator_UnresolvedStateRefFailsGeneration"}, + "builds[].matrix.dimensions[key]": {marker: markerValidityOnly, note: "strategy.matrix key + ${{ matrix. }} deref; validity + round-trip is the contract"}, + "builds[].matrix.dimensions.*": {marker: markerValidityOnly, note: "matrix value emitted via Go %q into a flow sequence; validity + round-trip is the contract"}, + "builds[].name": {assertion: "TestGenerator_FinalizeOutputKeysUseJobID"}, + "builds[].secrets.map[key]": {assertion: "TestGenCorrectness_SecretsMap_PropagatesSourceToCallee"}, + "builds[].secrets.map.*": {marker: markerStructural, note: "same secrets: block, pinned by TestGenCorrectness_SecretsMap_PropagatesSourceToCallee"}, + "builds[].triggers[]": {marker: markerValidityOnly, note: "paths-filter glob; validity + round-trip is the contract"}, + "builds[].workflow": {marker: markerValidityOnly, note: "reusable-workflow callback path spliced into uses:; validity + round-trip is the contract"}, + "builds[].depends_on[]": {marker: markerFollowup, note: "build-to-build needs: edge gating is not pinned; deploy-side gating is pinned by TestGM5_DependentDeploy_JudgesEffectiveResult"}, + "builds[].optional_depends_on[]": {marker: markerFollowup, note: "optional needs: edge gating (build side) is not pinned"}, + "builds[].env_inputs[key]": {marker: markerValidityOnly, note: "environment reference key; validity + round-trip is the contract"}, + "builds[].env_inputs.*": {marker: markerValidityOnly, note: "env-routed JSON matrix payload; validity + round-trip is the contract"}, + "builds[].on_failure": {marker: markerNotEmitted, note: "abort (the only emittable value) adds no distinct output; continue is rejected at validation, pinned by TestActionlint_FeatureMatrix on_failure_continue_rejected"}, + "builds[].run_policy": {marker: markerFollowup, note: "run_policy alters the job if: gate; the emitted if: expression is not yet pinned"}, + "builds[].permissions[key]": {assertion: "TestGenCorrectness_Permissions_LeastPrivilegePerJob"}, + "builds[].permissions.*": {marker: markerStructural, note: "same permissions: block, pinned by TestGenCorrectness_Permissions_LeastPrivilegePerJob"}, + + // -- changelog / cli ------------------------------------------------------ + "changelog.workflow": {marker: markerValidityOnly, note: "callback path spliced into uses:; validity + round-trip is the contract"}, + "cli_version": {marker: markerValidityOnly, note: "setup-cli@ splice; validity + round-trip is the contract"}, + "cli_version_sha": {marker: markerValidityOnly, note: "40-hex SHA into setup-cli ref; validity + round-trip is the contract"}, + + // -- components ----------------------------------------------------------- + "components[key]": {marker: markerValidityOnly, note: "component name keying the per-component workflow set; validity + round-trip is the contract"}, + "components.*.path": {marker: markerValidityOnly, note: "component subtree path -> paths-filter globs; validity + round-trip is the contract"}, + "components.*.extra_paths[]": {marker: markerValidityOnly, note: "extra paths-filter glob; validity + round-trip is the contract"}, + + // -- concurrency ---------------------------------------------------------- + "concurrency.group": {assertion: "TestGenCorrectness_Concurrency_CancelInProgressHonored"}, + + // -- deploys -------------------------------------------------------------- + "deploys[].artifact_upload.upload": {marker: markerValidityOnly, note: "passthrough upload glob; validity + round-trip is the contract"}, + "deploys[].artifact_upload.downloads[]": {marker: markerValidityOnly, note: "passthrough download job name; validity + round-trip is the contract"}, + "deploys[].inputs[key]": {marker: markerValidityOnly, note: "with: input key; validity + round-trip is the contract"}, + "deploys[].inputs.*": {assertion: "TestPromoteGenerator_UnresolvedEnvStateRefStaysVisible"}, + "deploys[].name": {assertion: "TestGenerator_FinalizeOutputKeysUseJobID"}, + "deploys[].secrets.map[key]": {marker: markerStructural, note: "shared writeSecretsBlock, pinned by TestGenCorrectness_SecretsMap_PropagatesSourceToCallee"}, + "deploys[].secrets.map.*": {marker: markerStructural, note: "shared writeSecretsBlock, pinned by TestGenCorrectness_SecretsMap_PropagatesSourceToCallee"}, + "deploys[].triggers[]": {marker: markerValidityOnly, note: "paths-filter glob; validity + round-trip is the contract"}, + "deploys[].workflow": {marker: markerValidityOnly, note: "callback path spliced into uses:; validity + round-trip is the contract"}, + "deploys[].depends_on[]": {assertion: "TestGM5_DependentDeploy_JudgesEffectiveResult"}, + "deploys[].optional_depends_on[]": {marker: markerFollowup, note: "optional needs: edge gating (deploy side) is not distinctly pinned; required gating is pinned by TestGM5_DependentDeploy_JudgesEffectiveResult"}, + "deploys[].env_inputs[key]": {marker: markerValidityOnly, note: "environment reference key; validity + round-trip is the contract"}, + "deploys[].env_inputs.*": {assertion: "TestPromoteGenerator_UnresolvedEnvStateRefStaysVisible"}, + "deploys[].on_failure": {marker: markerNotEmitted, note: "abort (the only emittable value) adds no distinct output; continue is rejected at validation"}, + "deploys[].run_policy": {marker: markerFollowup, note: "run_policy alters the deploy if: gate; the emitted if: expression is not yet pinned"}, + "deploys[].permissions[key]": {marker: markerStructural, note: "per-job permissions contract (incl. the least-privilege omission) is pinned by TestGenCorrectness_Permissions_LeastPrivilegePerJob"}, + "deploys[].permissions.*": {marker: markerStructural, note: "same permissions: block, pinned by TestGenCorrectness_Permissions_LeastPrivilegePerJob"}, + + // -- dispatch inputs ------------------------------------------------------ + "dispatch_inputs[key]": {assertion: "TestGenCorrectness_DispatchInputs_EmitsTypedInputBlock"}, + "dispatch_inputs.*.default": {marker: markerStructural, note: "same workflow_dispatch input block, pinned by TestGenCorrectness_DispatchInputs_EmitsTypedInputBlock"}, + "dispatch_inputs.*.options[]": {marker: markerStructural, note: "same workflow_dispatch input block, pinned by TestGenCorrectness_DispatchInputs_EmitsTypedInputBlock"}, + "dispatch_inputs.*.description": {marker: markerValidityOnly, note: "free operator help text emitted via Go %q; validity + round-trip is the contract"}, + "dispatch_inputs.*.type": {marker: markerStructural, note: "input type emitted in the workflow_dispatch block, pinned by TestGenCorrectness_DispatchInputs_EmitsTypedInputBlock"}, + + // -- environments --------------------------------------------------------- + "environments[].name": {marker: markerValidityOnly, note: "environment name keying the promote ladder and job IDs; validity + round-trip is the contract"}, + "environments[].environment_url": {assertion: "TestGenCorrectness_EnvironmentURL_EmitsPerEnvCase"}, + "environments[].secrets[]": {marker: markerValidityOnly, note: "secret name in the emitted environment secrets payload; validity + round-trip is the contract"}, + "environments[].variables[]": {marker: markerValidityOnly, note: "variable name in the emitted environment payload; validity + round-trip is the contract"}, + "environments[].role": {marker: markerFollowup, note: "role selects prerelease/release promotion stage; the resulting ladder ordering is not pinned"}, + "environments[].branch_policy": {marker: markerValidityOnly, note: "environment-provisioning API JSON payload; generation tests presence only"}, + "environments[].gha_environment": {marker: markerValidityOnly, note: "environment-provisioning API JSON payload; generation tests presence only"}, + "environments[].branch_patterns[]": {marker: markerValidityOnly, note: "environment-provisioning API JSON payload; generation tests presence only"}, + "environments[].tag_patterns[]": {marker: markerValidityOnly, note: "environment-provisioning API JSON payload; generation tests presence only"}, + "environments[].required_reviewers[]": {marker: markerValidityOnly, note: "environment-provisioning API JSON payload; generation tests presence only"}, + + // -- external ------------------------------------------------------------- + "external[].ref": {marker: markerValidityOnly, note: "git ref spliced into a cross-repo uses:@ref; validity + round-trip is the contract"}, + "external[].repo": {marker: markerValidityOnly, note: "runtime identity comparison; the emitted uses: path comes from the deploy workflow, validity + round-trip is the contract"}, + "external[].deploys[].name": {marker: markerValidityOnly, note: "external deploy job name; validity + round-trip is the contract"}, + "external[].deploys[].workflow": {marker: markerValidityOnly, note: "cross-repo callback path spliced into uses:; validity + round-trip is the contract"}, + "external[].deploys[].on_update.deploy.workflow": {marker: markerValidityOnly, note: "cross-repo callback path spliced into uses:; validity + round-trip is the contract"}, + "external[].deploys[].triggers[]": {marker: markerValidityOnly, note: "paths-filter glob; validity + round-trip is the contract"}, + "external[].deploys[].secrets.map[key]": {marker: markerStructural, note: "shared writeSecretsBlock, pinned by TestGenCorrectness_SecretsMap_PropagatesSourceToCallee"}, + "external[].deploys[].secrets.map.*": {marker: markerStructural, note: "shared writeSecretsBlock, pinned by TestGenCorrectness_SecretsMap_PropagatesSourceToCallee"}, + "external[].deploys[].optional_depends_on[]": {marker: markerFollowup, note: "external optional needs: edge gating is not pinned"}, + "external[].deploys[].permissions[key]": {marker: markerStructural, note: "per-job permissions contract pinned by TestGenCorrectness_Permissions_LeastPrivilegePerJob"}, + "external[].deploys[].permissions.*": {marker: markerStructural, note: "same permissions: block, pinned by TestGenCorrectness_Permissions_LeastPrivilegePerJob"}, + + // -- extra triggers ------------------------------------------------------- + "extra_triggers.schedule[].cron": {assertion: "TestGenCorrectness_ExtraTriggers_EmitsScheduleAndDispatch"}, + "extra_triggers.repository_dispatch.types[]": {marker: markerStructural, note: "same on: trigger set, pinned by TestGenCorrectness_ExtraTriggers_EmitsScheduleAndDispatch"}, + "extra_triggers.workflow_run.types[]": {marker: markerStructural, note: "same on: trigger set, pinned by TestGenCorrectness_ExtraTriggers_EmitsScheduleAndDispatch"}, + "extra_triggers.workflow_run.workflows[]": {marker: markerStructural, note: "same on: trigger set, pinned by TestGenCorrectness_ExtraTriggers_EmitsScheduleAndDispatch"}, + + // -- git ------------------------------------------------------------------ + "git.mode": {marker: markerStructural, note: "mode: custom triggers the git identity splice, pinned by TestGenCorrectness_GitCustomUser_SplicesNameAndEmail"}, + "git.user_name": {assertion: "TestGenCorrectness_GitCustomUser_SplicesNameAndEmail"}, + "git.user_email": {marker: markerStructural, note: "same git config splice, pinned by TestGenCorrectness_GitCustomUser_SplicesNameAndEmail"}, + "git.gpg_key_id": {marker: markerFollowup, note: "GPG key id wired into the signing setup; the emitted signing config is not pinned"}, + "git.gpg_key_secret": {marker: markerFollowup, note: "GPG private-key secret wired into the signing setup; the emitted signing config is not pinned"}, + + // -- manifest addressing -------------------------------------------------- + "manifest_file": {assertion: "TestReconcileGenerator_EmitsManifestFlags"}, + "manifest_key": {marker: markerStructural, note: "emitted --manifest-key flag pinned by TestReconcileGenerator_EmitsManifestFlags"}, + + // -- notify --------------------------------------------------------------- + "notify.repo": {marker: markerValidityOnly, note: "owner/repo splice; validity + round-trip is the contract"}, + "notify.workflow": {marker: markerValidityOnly, note: "single-quoted github-script literal; validity + round-trip is the contract"}, + "notify.deploy_name": {marker: markerValidityOnly, note: "single-quoted github-script literal; validity + round-trip is the contract"}, + "notify.environment": {marker: markerValidityOnly, note: "single-quoted github-script literal; validity + round-trip is the contract"}, + "notify.token": {marker: markerValidityOnly, note: "token expression into an unquoted YAML scalar; validity + round-trip is the contract"}, + + // -- publish / release ---------------------------------------------------- + "publish.workflow": {marker: markerValidityOnly, note: "callback path spliced into uses:; validity + round-trip is the contract"}, + "release_build.workflow": {marker: markerValidityOnly, note: "callback path spliced into uses:; validity + round-trip is the contract"}, + "release_build.tag": {assertion: "TestGenerator_ExternalReleaseDotlessTagErrors"}, + "release_token": {marker: markerValidityOnly, note: "token expression into an unquoted YAML scalar; validity + round-trip is the contract"}, + "state_token": {marker: markerValidityOnly, note: "token expression into an unquoted YAML scalar; validity + round-trip is the contract"}, + "release_token_app.app_id": {marker: markerValidityOnly, note: "secret reference expression; validity + round-trip is the contract"}, + "release_token_app.private_key": {marker: markerValidityOnly, note: "secret reference expression; validity + round-trip is the contract"}, + "state_token_app.app_id": {marker: markerValidityOnly, note: "secret reference expression; validity + round-trip is the contract"}, + "state_token_app.private_key": {marker: markerValidityOnly, note: "secret reference expression; validity + round-trip is the contract"}, + + // -- rollback ------------------------------------------------------------- + "rollback.repository_dispatch.types[]": {assertion: "TestGM4_RollbackDispatch_DryRunTreatsBooleanAndString"}, + + // -- shared paths / tag grammar ------------------------------------------- + "shared_paths[]": {marker: markerValidityOnly, note: "paths-filter glob; validity + round-trip is the contract"}, + "tag_grammar.prefix": {marker: markerValidityOnly, note: "tag prefix in argv; validity + round-trip is the contract"}, + "tag_grammar.prerelease_token": {marker: markerValidityOnly, note: "tag token component; validity + round-trip is the contract"}, + "tag_grammar.prerelease_separator": {marker: markerValidityOnly, note: "tag token component; validity + round-trip is the contract"}, + "tag_grammar.dryrun_token": {marker: markerValidityOnly, note: "tag token component; validity + round-trip is the contract"}, + + // -- top-level triggers / trunk ------------------------------------------- + "triggers[]": {marker: markerValidityOnly, note: "paths-filter glob; validity + round-trip is the contract"}, + "trunk_branch": {assertion: "TestGenerate_EmptyPushBranchList_NeverEmitted"}, + + // -- validate ------------------------------------------------------------- + "validate.workflow": {marker: markerValidityOnly, note: "callback path spliced into uses:; validity + round-trip is the contract"}, + "validate.triggers[]": {marker: markerValidityOnly, note: "paths-filter glob; validity + round-trip is the contract"}, + "validate.inputs[key]": {marker: markerValidityOnly, note: "with: input key; validity + round-trip is the contract"}, + "validate.inputs.*": {marker: markerValidityOnly, note: "with: input value; validity + round-trip is the contract"}, + "validate.env_inputs[key]": {marker: markerValidityOnly, note: "environment reference key; validity + round-trip is the contract"}, + "validate.env_inputs.*": {marker: markerValidityOnly, note: "env-routed JSON matrix payload; validity + round-trip is the contract"}, + "validate.secrets.map[key]": {marker: markerStructural, note: "shared writeSecretsBlock, pinned by TestGenCorrectness_SecretsMap_PropagatesSourceToCallee"}, + "validate.secrets.map.*": {marker: markerStructural, note: "shared writeSecretsBlock, pinned by TestGenCorrectness_SecretsMap_PropagatesSourceToCallee"}, + "validate.on_failure": {marker: markerNotEmitted, note: "abort (the only emittable value) adds no distinct output; continue is rejected at validation"}, + "validate.run_policy": {marker: markerFollowup, note: "run_policy alters the validate if: gate; the emitted if: expression is not yet pinned"}, + "validate.permissions[key]": {marker: markerStructural, note: "per-job permissions contract pinned by TestGenCorrectness_Permissions_LeastPrivilegePerJob"}, + "validate.permissions.*": {marker: markerStructural, note: "same permissions: block, pinned by TestGenCorrectness_Permissions_LeastPrivilegePerJob"}, + + // -- generator-level selectors -------------------------------------------- + "pin_mode": {marker: markerFollowup, note: "pin_mode (tag|sha) changes whether emitted refs are SHAs or tags; not pinned"}, + "release_trigger": {marker: markerFollowup, note: "release_trigger (push|dispatch) changes the emitted release trigger; not pinned"}, + "reconcile.source": {marker: markerFollowup, note: "reconcile source adapter changes the emitted detector; not pinned"}, + "reconcile.commit": {marker: markerFollowup, note: "reconcile commit mode changes the emitted companion commit step; not pinned"}, +} diff --git a/internal/generate/correctness_census_test.go b/internal/generate/correctness_census_test.go new file mode 100644 index 0000000..7c97e53 --- /dev/null +++ b/internal/generate/correctness_census_test.go @@ -0,0 +1,254 @@ +package generate + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// --------------------------------------------------------------------------- +// Generation-CORRECTNESS census. +// +// PR #631's census forces every emitted-affecting manifest field into +// actionlint coverage: the emitted output is VALID (real GitHub accepts it at +// parse). Validity is not correctness. A field can emit a workflow GitHub +// accepts that nonetheless does the WRONG thing: a matrix deploy that drops +// sha, a dry_run guard blind to a boolean, a dependent deploy gated on the +// immutable base result. Those defects (the Pass 10 "silent half") emit +// valid-but-wrong YAML that actionlint, act, and reviews all wave through. +// +// This census closes that gap the same way: it forces every emitted-affecting +// field to declare, in one reviewed table, whether its emitted output has a +// distinct SEMANTIC shape pinned by a named correctness assertion, or whether +// its only contract is validity + round-trip (already covered by the T1 +// battery and the actionlint sweeps). A NEW emitted-affecting field reds +// TestCorrectnessCensus_EveryEmittedFieldClassified until it is classified, so +// "the next feature nobody asserts" cannot escape silently. +// +// The emitted-affecting surface is exactly the union the actionlint sweeps +// already drive: every key of emittedFieldRegistry (fields spliced into an emit +// sink) plus every key of emittedAffectingAllowlistMutators (allowlist fields +// that still change emitted workflow structure). This census reuses that +// surface rather than rebuilding it, so the two stay in lockstep by +// construction. +// +// HONEST LIMIT (stated once, here, and repeated on the runtime-semantic +// assertions): a generation-correctness assertion pins the emitted shape the +// AUTHOR believes correct. It catches a missing key, a wrong-versus-baseline +// expression, and a regression away from the pinned shape. It does NOT prove +// the shape is correct against real GitHub Actions RUNTIME semantics: whether +// `!= 'true'` actually rescues a boolean dry_run, whether a retry shim really +// rescues a failed ladder, whether a `needs:` gate fires as intended. That +// proof is the fleet's job (Part B). Where a field's correctness turns on +// runtime semantics, its assertion says so and defers the runtime proof to the +// fleet. +// --------------------------------------------------------------------------- + +// correctnessMarker classifies an emitted-affecting field that carries no +// distinct correctness assertion. Exactly one marker or one assertion applies +// to each census field. +type correctnessMarker string + +const ( + // markerValidityOnly: the field's only correctness contract is that its + // emitted splice is valid and round-trips. Its value is a free scalar, + // glob, name, or path with no distinct downstream semantic shape, already + // pinned by the T1 hostile/good battery and the actionlint sweep. + markerValidityOnly correctnessMarker = "validity-only" + + // markerStructural: the field's emitted shape is identical to a sibling + // field's and is pinned by that sibling's named assertion (recorded in the + // note). One assertion covers both; duplicating it would add no coverage. + markerStructural correctnessMarker = "structural" + + // markerNotEmitted: the field never reaches emitted output as a splice + // (validated-only enum, reserved block, or rejected combination), so there + // is no emitted shape to pin. Mirrors a notEmittedAllowlist reason. + markerNotEmitted correctnessMarker = "not-emitted" + + // markerFollowup: the field IS emitted-affecting with a distinct semantic + // shape that warrants a correctness assertion not yet written. It is an + // explicit, reviewed placeholder: the census keeps it visible so a + // followup tranche can pin it. The enumerated markerFollowup set is the + // remaining-fields backlog. + markerFollowup correctnessMarker = "followup" +) + +// correctnessCoverage is one census field's classification. Exactly one of +// assertion / marker is set; note is required for a marker and records the +// sibling for markerStructural or the reason for the others. +type correctnessCoverage struct { + assertion string + marker correctnessMarker + note string +} + +// emittedAffectingCensus returns the sorted union of every emitted-affecting +// manifest field path: the registry (direct emit-sink splices) plus the +// emitted-affecting allowlist (fields that change emitted structure without a +// raw splice). This is the exact surface the actionlint sweeps drive, so +// correctness coverage tracks validity coverage field-for-field. +func emittedAffectingCensus() []string { + seen := map[string]bool{} + for p := range emittedFieldRegistry { + seen[p] = true + } + for p := range emittedAffectingAllowlistMutators { + seen[p] = true + } + out := make([]string, 0, len(seen)) + for p := range seen { + out = append(out, p) + } + sort.Strings(out) + return out +} + +// reconcileCorrectnessCensus is the pure forcing core, tested directly with a +// synthetic census so its red behavior is proven without mutating the real +// schema. It returns the census fields with no coverage entry (missing) and the +// coverage entries naming a field no longer in the census (stale). A non-empty +// missing set is the guard's red state: a new emitted-affecting field has +// appeared with no correctness classification. +func reconcileCorrectnessCensus(census []string, coverage map[string]correctnessCoverage) (missing, stale []string) { + present := map[string]bool{} + for _, p := range census { + present[p] = true + if _, ok := coverage[p]; !ok { + missing = append(missing, p) + } + } + for p := range coverage { + if !present[p] { + stale = append(stale, p) + } + } + sort.Strings(missing) + sort.Strings(stale) + return missing, stale +} + +// TestCorrectnessCensus_EveryEmittedFieldClassified is the forcing guard. Every +// emitted-affecting field (the actionlint-sweep surface) must carry exactly one +// correctnessCoverage entry: a named assertion pinning its emitted semantic +// shape, or a reviewed marker recording why it has none. A new field reds +// `missing`; a removed field reds `stale`; so the census cannot rot and a new +// emitted-affecting feature cannot ship with nobody asserting its output. +func TestCorrectnessCensus_EveryEmittedFieldClassified(t *testing.T) { + missing, stale := reconcileCorrectnessCensus(emittedAffectingCensus(), correctnessCensus) + + if len(missing) > 0 { + t.Errorf("emitted-affecting fields with no generation-correctness classification "+ + "(add each to correctnessCensus: name a correctness assertion pinning its emitted shape, "+ + "or a reviewed marker: markerValidityOnly / markerStructural / markerNotEmitted / markerFollowup):\n %s", + strings.Join(missing, "\n ")) + } + if len(stale) > 0 { + t.Errorf("correctnessCensus entries naming a field no longer emitted-affecting (remove them):\n %s", + strings.Join(stale, "\n ")) + } +} + +// TestCorrectnessCensus_ForcesNewField proves the forcing core reds when a +// brand-new emitted-affecting field appears with no classification. This is the +// durable promise: a future field registered into emittedFieldRegistry (or +// given an allowlist mutator) but never classified for correctness reds +// TestCorrectnessCensus_EveryEmittedFieldClassified. The synthetic census keeps +// the proof independent of the real schema. +func TestCorrectnessCensus_ForcesNewField(t *testing.T) { + fake := "brand_new.emitted_affecting_field[]" + missing, stale := reconcileCorrectnessCensus([]string{fake}, correctnessCensus) + require.Contains(t, missing, fake, + "an unclassified emitted-affecting field must red the correctness census") + require.NotEmpty(t, stale, + "the real coverage table must be reported stale against a census that omits its fields, "+ + "proving stale-entry detection is live") +} + +// TestCorrectnessCensus_EntriesWellFormed pins the shape of every coverage +// entry: exactly one of assertion / marker, a note wherever a marker (or a +// structural pointer) needs one, and only recognized markers. A malformed entry +// (both set, neither set, an unknown marker, a markerStructural with no sibling +// note) reds here, so the table stays a real classification rather than a +// grab-bag. +func TestCorrectnessCensus_EntriesWellFormed(t *testing.T) { + known := map[correctnessMarker]bool{ + markerValidityOnly: true, markerStructural: true, + markerNotEmitted: true, markerFollowup: true, + } + for path, c := range correctnessCensus { + hasAssertion := c.assertion != "" + hasMarker := c.marker != "" + if hasAssertion == hasMarker { + t.Errorf("%s: exactly one of assertion / marker must be set (assertion=%q marker=%q)", + path, c.assertion, c.marker) + continue + } + if hasMarker { + if !known[c.marker] { + t.Errorf("%s: unknown marker %q", path, c.marker) + } + if c.note == "" { + t.Errorf("%s: marker %q requires a note recording the reason or sibling", path, c.marker) + } + } + } +} + +// TestCorrectnessCensus_AssertionsExist grounds every named assertion in a real +// test. It scans the package's *_test.go sources for `func (t *testing.T)` +// declarations and fails on any correctnessCensus assertion (or markerStructural +// sibling note) that names a test which does not exist, so a typo or a deleted +// assertion cannot leave a census entry pointing at nothing. +func TestCorrectnessCensus_AssertionsExist(t *testing.T) { + funcs := testFuncNames(t) + + var dangling []string + for path, c := range correctnessCensus { + if c.assertion != "" && !funcs[c.assertion] { + dangling = append(dangling, path+" -> "+c.assertion) + } + if c.marker == markerStructural { + for _, name := range extractTestNames(c.note) { + if !funcs[name] { + dangling = append(dangling, path+" -> "+name+" (structural sibling)") + } + } + } + } + sort.Strings(dangling) + if len(dangling) > 0 { + t.Errorf("correctnessCensus entries naming a test that does not exist in this package:\n %s", + strings.Join(dangling, "\n ")) + } +} + +var testFuncRE = regexp.MustCompile(`func (Test[A-Za-z0-9_]+)\(t \*testing\.T\)`) + +// testFuncNames returns the set of top-level test function names declared in the +// package's _test.go files. +func testFuncNames(t *testing.T) map[string]bool { + t.Helper() + entries, err := filepath.Glob("*_test.go") + require.NoError(t, err) + names := map[string]bool{} + for _, f := range entries { + src, err := os.ReadFile(f) + require.NoError(t, err) + for _, m := range testFuncRE.FindAllStringSubmatch(string(src), -1) { + names[m[1]] = true + } + } + return names +} + +// extractTestNames pulls every Test... identifier out of a structural note so +// the sibling assertion it points at is grounded like a direct assertion. +func extractTestNames(note string) []string { + return regexp.MustCompile(`Test[A-Za-z0-9_]+`).FindAllString(note, -1) +}