From dd8e8786d28383e0d1f46244fff9bb4b2886bafb Mon Sep 17 00:00:00 2001 From: Eric Fitzgerald Date: Wed, 29 Jul 2026 23:05:35 -0400 Subject: [PATCH] feat(dbtool): emit a CATS fixture manifest for run-integrity gating A CATS campaign can DELETE the seeded fixtures its own refData points at. Because CATS walks paths in lexical order, an anchor path is usually fuzzed before everything nested under it, so from that moment on every nested test runs against a 404 -- and those 404s get reported as findings. The refData committed before this change already pointed at a project that no longer existed, so the damage outlives the run that caused it. Anchor decoys (#608) reduce this but nothing measured whether they worked. Write cats-fixtures.json alongside the refData it describes: every seeded id the campaign substitutes, a GET that returns 200 iff it still exists (parent ids resolved inline), the anchor path whose DELETE would consume it, and whether that anchor already has a decoy -- so a fixture that dies anyway is reported as "the decoy is not working" rather than "add a decoy". Fixtures that cannot be checked are named in an `unverifiable` map rather than omitted (credential_id has no GET, addon_id has no spec path). A gate that silently skips part of the fixture set reads as "all fixtures survived" when it really means "the ones I could see survived". The CATS runner consumes this via `cats.fixtures` to gate a run twice: after seeding (refuse to fuzz against absent fixtures) and after the campaign (a fixture the campaign destroyed makes the run invalid, so it never becomes latest.db). Verified: all 18 fixtures resolve against the live API; the manifest URLs were fault-injected to confirm the tests catch a wrong one. Refs #608 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WnrVyySuyJFqcMxB8q3pNc --- cmd/dbtool/reference.go | 143 +++++++++++++++++++++++++- cmd/dbtool/reference_fixtures_test.go | 125 ++++++++++++++++++++++ 2 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 cmd/dbtool/reference_fixtures_test.go diff --git a/cmd/dbtool/reference.go b/cmd/dbtool/reference.go index 11289a06..ab3548b8 100644 --- a/cmd/dbtool/reference.go +++ b/cmd/dbtool/reference.go @@ -375,7 +375,148 @@ all: auditSections, ) - return os.WriteFile(path, []byte(yaml), 0o600) + if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { + return err + } + + // Fixture manifest for the CATS runner's fixture-integrity gates (#608). + // Written alongside the refData it describes so the two can never disagree + // about which ids a campaign depends on. + return writeFixtureManifest( + filepath.Join(filepath.Dir(path), "cats-fixtures.json"), + fixtureIDs{ + threatModelID: tmID, threatID: threatID, diagramID: diagramID, + documentID: documentID, assetID: assetID, noteID: noteID, + feedbackID: feedbackID, repositoryID: repoID, + teamID: teamID, projectID: projectID, + teamNoteID: teamNoteID, projectNoteID: projectNoteID, + groupID: adminGroupID, userID: targetUUID, + surveyID: surveyID, surveyResponseID: responseID, + triageNoteID: triageNoteID, webhookID: webhookID, deliveryID: deliveryID, + }, + ) +} + +// SEM@0: resolved seeded ids the fixture manifest is built from (pure) +type fixtureIDs struct { + threatModelID, threatID, diagramID, documentID, assetID, noteID string + feedbackID, repositoryID string + teamID, projectID, teamNoteID, projectNoteID string + groupID, userID string + surveyID, surveyResponseID, triageNoteID string + webhookID, deliveryID string +} + +// SEM@0: one seeded fixture the campaign depends on, plus how to check it is still alive (pure) +type referenceFixture struct { + // Key is the refData key CATS substitutes this id into. + Key string `json:"key"` + ID string `json:"id"` + // VerifyURL is a server-relative GET that returns 200 iff the fixture + // still exists. Parent ids are already resolved. + VerifyURL string `json:"verify_url"` + // AnchorPath is the CATS path template whose DELETE would consume this + // fixture. Named in the gate's failure message so the fix (add a decoy + // override for that anchor) is obvious from the error alone. + AnchorPath string `json:"anchor_path"` + // Decoyed records whether AnchorPath already has a decoy override, so a + // fixture that dies anyway is reported as "decoy not working" rather than + // "decoy missing". + Decoyed bool `json:"decoyed"` +} + +// SEM@0: fixture manifest file structure listing verifiable and unverifiable seeded fixtures (pure) +type fixtureManifest struct { + Version string `json:"version"` + CreatedAt string `json:"created_at"` + Fixtures []referenceFixture `json:"fixtures"` + // Unverifiable names fixtures the gates cannot check, with the reason. + // Recorded explicitly rather than omitted: a gate that silently skips + // part of the fixture set reads as "all fixtures survived" when it is + // really "the ones I could see survived". + Unverifiable map[string]string `json:"unverifiable"` +} + +// SEM@0: serialize the seeded-fixture integrity manifest consumed by the CATS runner gates (mutates shared state) +func writeFixtureManifest(path string, ids fixtureIDs) error { + tm := ids.threatModelID + // Anchors that writeYAMLReference emits a decoy override for. Keep in step + // with the `decoys` slice above. + decoyed := map[string]bool{ + "/teams/{team_id}": true, + "/projects/{project_id}": true, + "/threat_models/{threat_model_id}": true, + "/admin/groups/{group_id}": true, + "/admin/users/{user_id}": true, + "/intake/survey_responses/{survey_response_id}": true, + } + + candidates := []referenceFixture{ + {"threat_model_id", tm, "/threat_models/" + tm, "/threat_models/{threat_model_id}", false}, + {"threat_id", ids.threatID, "/threat_models/" + tm + "/threats/" + ids.threatID, + "/threat_models/{threat_model_id}/threats/{threat_id}", false}, + {"diagram_id", ids.diagramID, "/threat_models/" + tm + "/diagrams/" + ids.diagramID, + "/threat_models/{threat_model_id}/diagrams/{diagram_id}", false}, + {"document_id", ids.documentID, "/threat_models/" + tm + "/documents/" + ids.documentID, + "/threat_models/{threat_model_id}/documents/{document_id}", false}, + {"asset_id", ids.assetID, "/threat_models/" + tm + "/assets/" + ids.assetID, + "/threat_models/{threat_model_id}/assets/{asset_id}", false}, + {"note_id", ids.noteID, "/threat_models/" + tm + "/notes/" + ids.noteID, + "/threat_models/{threat_model_id}/notes/{note_id}", false}, + {"feedback_id", ids.feedbackID, "/threat_models/" + tm + "/feedback/" + ids.feedbackID, + "/threat_models/{threat_model_id}/feedback/{feedback_id}", false}, + {"repository_id", ids.repositoryID, "/threat_models/" + tm + "/repositories/" + ids.repositoryID, + "/threat_models/{threat_model_id}/repositories/{repository_id}", false}, + {"team_id", ids.teamID, "/teams/" + ids.teamID, "/teams/{team_id}", false}, + {"project_id", ids.projectID, "/projects/" + ids.projectID, "/projects/{project_id}", false}, + {"team_note_id", ids.teamNoteID, "/teams/" + ids.teamID + "/notes/" + ids.teamNoteID, + "/teams/{team_id}/notes/{team_note_id}", false}, + {"project_note_id", ids.projectNoteID, "/projects/" + ids.projectID + "/notes/" + ids.projectNoteID, + "/projects/{project_id}/notes/{project_note_id}", false}, + {"group_id", ids.groupID, "/admin/groups/" + ids.groupID, "/admin/groups/{group_id}", false}, + {"user_id", ids.userID, "/admin/users/" + ids.userID, "/admin/users/{user_id}", false}, + {"survey_id", ids.surveyID, "/admin/surveys/" + ids.surveyID, "/admin/surveys/{survey_id}", false}, + {"survey_response_id", ids.surveyResponseID, "/intake/survey_responses/" + ids.surveyResponseID, + "/intake/survey_responses/{survey_response_id}", false}, + {"triage_note_id", ids.triageNoteID, + "/intake/survey_responses/" + ids.surveyResponseID + "/triage_notes/" + ids.triageNoteID, + "/intake/survey_responses/{survey_response_id}/triage_notes/{triage_note_id}", false}, + {"webhook_id", ids.webhookID, "/admin/webhooks/subscriptions/" + ids.webhookID, + "/admin/webhooks/subscriptions/{webhook_id}", false}, + {"delivery_id", ids.deliveryID, "/admin/webhooks/deliveries/" + ids.deliveryID, + "/admin/webhooks/deliveries/{delivery_id}", false}, + } + + // #nosec G101 -- these are refData key names and prose reasons, not credentials + unverifiable := map[string]string{ + "credential_id": "spec has no GET for /me/client_credentials/{credential_id} (delete-only)", + "addon_id": "no spec path uses {addon_id}", + "key": "metadata key is not an addressable resource", + "entry_id": "audit entries are harvested after seeding, not seeded, and are not deletable fixtures", + } + + fixtures := make([]referenceFixture, 0, len(candidates)) + for _, f := range candidates { + // A fixture that was never seeded has no integrity to check, and + // including it would make the gate fail every run on a false death. + if f.ID == "" || f.ID == nilUUID { + unverifiable[f.Key] = "not present in this seed run" + continue + } + f.Decoyed = decoyed[f.AnchorPath] + fixtures = append(fixtures, f) + } + + data, err := json.MarshalIndent(fixtureManifest{ + Version: "1.0.0", + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Fixtures: fixtures, + Unverifiable: unverifiable, + }, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal fixture manifest: %w", err) + } + return os.WriteFile(path, append(data, '\n'), 0o600) } // Seed refs for the destructive-fuzz decoys. Anchor paths (/teams/{team_id}, diff --git a/cmd/dbtool/reference_fixtures_test.go b/cmd/dbtool/reference_fixtures_test.go new file mode 100644 index 00000000..670bbe4e --- /dev/null +++ b/cmd/dbtool/reference_fixtures_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// readManifest writes a manifest for `ids` into a temp dir and parses it back. +func readManifest(t *testing.T, ids fixtureIDs) fixtureManifest { + t.Helper() + path := filepath.Join(t.TempDir(), "cats-fixtures.json") + if err := writeFixtureManifest(path, ids); err != nil { + t.Fatalf("writeFixtureManifest: %v", err) + } + data, err := os.ReadFile(path) // #nosec G304 -- test-controlled temp path + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var m fixtureManifest + if err := json.Unmarshal(data, &m); err != nil { + t.Fatalf("unmarshal manifest: %v", err) + } + return m +} + +func fullIDs() fixtureIDs { + return fixtureIDs{ + threatModelID: "tm-1", threatID: "th-1", diagramID: "dg-1", + documentID: "doc-1", assetID: "as-1", noteID: "nt-1", + feedbackID: "fb-1", repositoryID: "repo-1", + teamID: "team-1", projectID: "proj-1", + teamNoteID: "tn-1", projectNoteID: "pn-1", + groupID: "grp-1", userID: "usr-1", + surveyID: "sv-1", surveyResponseID: "sr-1", triageNoteID: "1", + webhookID: "wh-1", deliveryID: "dl-1", + } +} + +// A fully-seeded run must produce a verify URL for every fixture, with parent +// ids resolved inline — the runner GETs these verbatim, so an unsubstituted +// "{threat_model_id}" would make the gate report a false fixture death. +func TestFixtureManifestResolvesParentIDs(t *testing.T) { + m := readManifest(t, fullIDs()) + + want := map[string]string{ + "threat_model_id": "/threat_models/tm-1", + "threat_id": "/threat_models/tm-1/threats/th-1", + "diagram_id": "/threat_models/tm-1/diagrams/dg-1", + "asset_id": "/threat_models/tm-1/assets/as-1", + "feedback_id": "/threat_models/tm-1/feedback/fb-1", + "repository_id": "/threat_models/tm-1/repositories/repo-1", + "team_id": "/teams/team-1", + "project_id": "/projects/proj-1", + "team_note_id": "/teams/team-1/notes/tn-1", + "project_note_id": "/projects/proj-1/notes/pn-1", + "group_id": "/admin/groups/grp-1", + "user_id": "/admin/users/usr-1", + "survey_response_id": "/intake/survey_responses/sr-1", + "triage_note_id": "/intake/survey_responses/sr-1/triage_notes/1", + "webhook_id": "/admin/webhooks/subscriptions/wh-1", + "delivery_id": "/admin/webhooks/deliveries/dl-1", + } + + got := make(map[string]string, len(m.Fixtures)) + for _, f := range m.Fixtures { + got[f.Key] = f.VerifyURL + } + for key, url := range want { + if got[key] != url { + t.Errorf("fixture %q verify_url = %q, want %q", key, got[key], url) + } + } +} + +// Every anchor that writeYAMLReference emits a decoy override for must be +// marked decoyed, so a fixture that dies anyway is reported as "the decoy is +// not working" rather than sending the reader off to add one that exists. +func TestFixtureManifestMarksDecoyedAnchors(t *testing.T) { + m := readManifest(t, fullIDs()) + + decoyed := map[string]bool{ + "threat_model_id": true, "team_id": true, "project_id": true, + "group_id": true, "user_id": true, "survey_response_id": true, + // Nested fixtures have no anchor decoy. + "threat_id": false, "note_id": false, "team_note_id": false, + "survey_id": false, "webhook_id": false, + } + for _, f := range m.Fixtures { + want, tracked := decoyed[f.Key] + if tracked && f.Decoyed != want { + t.Errorf("fixture %q decoyed = %v, want %v (anchor %s)", + f.Key, f.Decoyed, want, f.AnchorPath) + } + } +} + +// An id that was never seeded must be reported as unverifiable rather than +// emitted as a fixture: the gate would GET a nil-UUID URL, get a 404, and fail +// every run on a fixture that never existed. +func TestFixtureManifestExcludesUnseededIDs(t *testing.T) { + ids := fullIDs() + ids.webhookID = nilUUID + ids.deliveryID = "" + + m := readManifest(t, ids) + for _, f := range m.Fixtures { + if f.Key == "webhook_id" || f.Key == "delivery_id" { + t.Errorf("unseeded fixture %q must not be verifiable (id %q)", f.Key, f.ID) + } + } + for _, key := range []string{"webhook_id", "delivery_id"} { + if _, ok := m.Unverifiable[key]; !ok { + t.Errorf("unseeded fixture %q must appear in unverifiable", key) + } + } + // Fixtures with no individual GET in the spec are permanently unverifiable + // and must be named, not silently dropped. + for _, key := range []string{"credential_id", "addon_id"} { + if _, ok := m.Unverifiable[key]; !ok { + t.Errorf("%q must be recorded as unverifiable", key) + } + } +}