From 06c1a1edb17bc53289eab76a6c0fa73be63edc63 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 11 Aug 2026 12:39:33 -0700 Subject: [PATCH 1/6] Split the default-terminal pin per signal kind The whole-object pin skipped whenever config.signalDisposition was present at all, so a buffer_overflow terminal that configured only a feedGap key (or vice versa) escaped the empty-invocations requirement and could claim a phantom handler invocation. Replace allOf[1] with two per-signal pins, each keyed on the terminal reason const and the absence of that signal's own disposition key. Proof: a fixture-21 clone with a feedGap-only disposition and a phantom bufferOverflow invocation validates against the old schema (exit 0) and is rejected by the new one (exit 1), mirror-image probe likewise; make event-feed-fixtures-check green over all 22 fixtures. --- conformance/event-feed/schema.json | 75 +++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/conformance/event-feed/schema.json b/conformance/event-feed/schema.json index 0bc8e592ed..11b8924853 100644 --- a/conformance/event-feed/schema.json +++ b/conformance/event-feed/schema.json @@ -62,7 +62,7 @@ } }, { - "description": "A default-terminal semantic-signal fixture (terminal reason buffer_overflow or feed_gap with NO signalDisposition configured) MUST assert ZERO handler invocations — the exact-empty record is what proves the default path was taken rather than a skipped handler.", + "description": "Per-signal default-terminal pin: a buffer_overflow terminal with NO bufferOverflow disposition configured has no overflow handler — regardless of what OTHER signal keys are configured — so the fixture MUST assert zero handler invocations of that kind; with no cross-signal scenario in the suite the whole record must be exactly empty.", "if": { "required": [ "finally" @@ -72,7 +72,73 @@ "not": { "required": [ "signalDisposition" - ] + ], + "properties": { + "signalDisposition": { + "required": [ + "bufferOverflow" + ] + } + } + } + }, + "finally": { + "required": [ + "error" + ], + "properties": { + "error": { + "required": [ + "reason" + ], + "properties": { + "reason": { + "const": "buffer_overflow" + } + } + } + } + } + } + }, + "then": { + "properties": { + "finally": { + "required": [ + "handlerInvocations" + ], + "properties": { + "handlerInvocations": { + "properties": { + "exact": { + "const": [] + } + } + } + } + } + } + } + }, + { + "description": "Per-signal default-terminal pin: a feed_gap terminal with NO feedGap disposition configured has no gap handler — regardless of what OTHER signal keys are configured — so the fixture MUST assert zero handler invocations of that kind; with no cross-signal scenario in the suite the whole record must be exactly empty.", + "if": { + "required": [ + "finally" + ], + "properties": { + "config": { + "not": { + "required": [ + "signalDisposition" + ], + "properties": { + "signalDisposition": { + "required": [ + "feedGap" + ] + } + } } }, "finally": { @@ -86,10 +152,7 @@ ], "properties": { "reason": { - "enum": [ - "buffer_overflow", - "feed_gap" - ] + "const": "feed_gap" } } } From 530c8fcbe9506a805c1b6ebd7871326b58c3bdec Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 11 Aug 2026 13:10:41 -0700 Subject: [PATCH 2/6] Assert zero invocations of the unhandled kind, not an empty record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The const-[] form said more than the contract does: it pinned the WHOLE record empty, so a legitimate cross-signal run — an accepted overflow followed by an unhandled 410 ending feed_gap default-terminal, record [{bufferOverflow, accept}] — would be schema-rejected. The per-signal contract is that an unregistered handler cannot be invoked; say exactly that with a not-contains on the terminal signal's kind. Proof: both phantom probes still rejected (per-kind), the valid cross-signal control accepted by this form and rejected by the const-[] form; make event-feed-fixtures-check green over all 22 fixtures. --- conformance/event-feed/schema.json | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/conformance/event-feed/schema.json b/conformance/event-feed/schema.json index 11b8924853..12d144d931 100644 --- a/conformance/event-feed/schema.json +++ b/conformance/event-feed/schema.json @@ -62,7 +62,7 @@ } }, { - "description": "Per-signal default-terminal pin: a buffer_overflow terminal with NO bufferOverflow disposition configured has no overflow handler — regardless of what OTHER signal keys are configured — so the fixture MUST assert zero handler invocations of that kind; with no cross-signal scenario in the suite the whole record must be exactly empty.", + "description": "Per-signal default-terminal pin: a buffer_overflow terminal with NO bufferOverflow disposition configured (config absent entirely, or present without the key — either way no overflow handler exists) MUST carry a handler-invocation record containing zero invocations of that kind: an unregistered handler cannot be invoked, so a bufferOverflow entry here is unsatisfiable at runtime and invalid at load. Invocations of OTHER kinds whose handlers ARE configured remain legal (a cross-signal run may accept a gap before an unhandled overflow terminates it).", "if": { "required": [ "finally" @@ -111,7 +111,18 @@ "handlerInvocations": { "properties": { "exact": { - "const": [] + "not": { + "contains": { + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "bufferOverflow" + } + } + } + } } } } @@ -121,7 +132,7 @@ } }, { - "description": "Per-signal default-terminal pin: a feed_gap terminal with NO feedGap disposition configured has no gap handler — regardless of what OTHER signal keys are configured — so the fixture MUST assert zero handler invocations of that kind; with no cross-signal scenario in the suite the whole record must be exactly empty.", + "description": "Per-signal default-terminal pin: a feed_gap terminal with NO feedGap disposition configured (config absent entirely, or present without the key — either way no gap handler exists) MUST carry a handler-invocation record containing zero invocations of that kind: an unregistered handler cannot be invoked, so a feedGap entry here is unsatisfiable at runtime and invalid at load. Invocations of OTHER kinds whose handlers ARE configured remain legal (a cross-signal run may accept an overflow before an unhandled gap terminates it).", "if": { "required": [ "finally" @@ -170,7 +181,18 @@ "handlerInvocations": { "properties": { "exact": { - "const": [] + "not": { + "contains": { + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "feedGap" + } + } + } + } } } } From a19a25665e2f33e9dc47fcdb93d8b557fc13298b Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 11 Aug 2026 13:18:03 -0700 Subject: [PATCH 3/6] Gate the per-signal pins with committed invalid fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proof probes lived only in the PR body, so nothing in CI would catch a future schema edit quietly re-widening the default-terminal pins — and this schema gets edited again at PR 4 and PR-T. Commit the two phantom probes under conformance/event-feed/invalid-fixtures/ and teach event-feed-fixtures-check to assert each is REJECTED. Each file is schema-valid except for exactly one violation, so its rejection exercises that pin and nothing else; harnesses never glob this directory. Test the test: the same loop run against the pre-fix schema reports the phantom fixtures as wrongly validated and exits 1. --- Makefile | 10 ++ conformance/event-feed/README.md | 8 ++ .../phantom-gap-invocation.json | 130 ++++++++++++++++++ .../phantom-overflow-invocation.json | 130 ++++++++++++++++++ 4 files changed, 278 insertions(+) create mode 100644 conformance/event-feed/invalid-fixtures/phantom-gap-invocation.json create mode 100644 conformance/event-feed/invalid-fixtures/phantom-overflow-invocation.json diff --git a/Makefile b/Makefile index efbd085d43..813e65d666 100644 --- a/Makefile +++ b/Makefile @@ -559,6 +559,16 @@ event-feed-fixtures-check: --check-metaschema conformance/event-feed/schema.json uvx --from 'check-jsonschema==$(CHECK_JSONSCHEMA_VERSION)' check-jsonschema \ --schemafile conformance/event-feed/schema.json conformance/event-feed/fixtures/*.json + @echo "==> Asserting event-feed invalid fixtures are rejected..." + @for f in conformance/event-feed/invalid-fixtures/*.json; do \ + if uvx --from 'check-jsonschema==$(CHECK_JSONSCHEMA_VERSION)' check-jsonschema \ + --schemafile conformance/event-feed/schema.json "$$f" >/dev/null 2>&1; then \ + echo "ERROR: $$f validated, but this directory holds shapes the schema must reject"; \ + exit 1; \ + else \ + echo "rejected (as required): $$f"; \ + fi; \ + done event-feed-digest-fixtures-check: @echo "==> Validating event-feed srv1 digest vectors..." diff --git a/conformance/event-feed/README.md b/conformance/event-feed/README.md index b5c85465bf..6875daf097 100644 --- a/conformance/event-feed/README.md +++ b/conformance/event-feed/README.md @@ -52,6 +52,14 @@ against the JSON Schema metaschema and every fixture against the schema, with a pinned `check-jsonschema` run through `uvx` (part of `make conformance`, so `make check` gates it). +The same gate then asserts every file in `invalid-fixtures/` is REJECTED by the +schema. Those files are negative regression cases for load-bearing `allOf` pins — +each is schema-valid except for exactly one violation, so its rejection exercises +that pin and nothing else (the current pair pins the per-signal default-terminal +rules: a phantom invocation of a signal kind whose disposition key is absent). +Harnesses must never glob `invalid-fixtures/` — it is a gate input, not a scenario +inventory. + ## Directory is a schema boundary This directory contains exactly one shape: tier-2 scenario scripts. If a second diff --git a/conformance/event-feed/invalid-fixtures/phantom-gap-invocation.json b/conformance/event-feed/invalid-fixtures/phantom-gap-invocation.json new file mode 100644 index 0000000000..3a9caa0888 --- /dev/null +++ b/conformance/event-feed/invalid-fixtures/phantom-gap-invocation.json @@ -0,0 +1,130 @@ +{ + "name": "phantom-gap-invocation", + "description": "MUST BE REJECTED by schema.json (this directory is gated as invalid): a feed_gap default-terminal scenario configuring only a bufferOverflow disposition, claiming a phantom feedGap handler invocation. Exercises the per-signal feedGap pin \u2014 the mirror of phantom-overflow-invocation. Everything else about the file is schema-valid, so rejection isolates the pin.", + "config": { + "liveBufferCapacity": 2, + "signalDisposition": { + "bufferOverflow": "accept" + } + }, + "steps": [ + { + "expectMint": { + "respond": { + "status": 200, + "body": { + "ticket": "{{TICKET:1}}", + "expires_in": 120, + "url": "{{CABLE_URL:1}}" + } + } + } + }, + { + "expectConnect": { + "url": "{{CABLE_URL:1}}" + } + }, + { + "serve": { + "frame": "welcome" + } + }, + { + "expectSubscribe": { + "channel": "EventsChannel" + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 51, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 52, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 53, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "expectSignal": { + "kind": "bufferOverflow", + "droppedIds": [ + 51 + ], + "droppedCount": 1 + } + }, + { + "expectClientClose": {} + }, + { + "expectError": { + "reason": "feed_gap" + } + } + ], + "finally": { + "state": "terminal", + "mintCount": 1, + "connectCount": 1, + "delivered": { + "exact": [] + }, + "checkpoints": { + "exact": [] + }, + "timers": { + "exact": {} + }, + "socket": "closed", + "error": { + "reason": "feed_gap" + }, + "handlerInvocations": { + "exact": [ + { + "kind": "feedGap", + "disposition": "accept" + } + ] + } + } +} \ No newline at end of file diff --git a/conformance/event-feed/invalid-fixtures/phantom-overflow-invocation.json b/conformance/event-feed/invalid-fixtures/phantom-overflow-invocation.json new file mode 100644 index 0000000000..296cabadcb --- /dev/null +++ b/conformance/event-feed/invalid-fixtures/phantom-overflow-invocation.json @@ -0,0 +1,130 @@ +{ + "name": "phantom-overflow-invocation", + "description": "MUST BE REJECTED by schema.json (this directory is gated as invalid): a buffer_overflow default-terminal scenario configuring only a feedGap disposition, claiming a phantom bufferOverflow handler invocation. Exercises the per-signal bufferOverflow pin \u2014 the whole-object form of the pin accepted this shape. Everything else about the file is schema-valid, so rejection isolates the pin.", + "config": { + "liveBufferCapacity": 2, + "signalDisposition": { + "feedGap": "accept" + } + }, + "steps": [ + { + "expectMint": { + "respond": { + "status": 200, + "body": { + "ticket": "{{TICKET:1}}", + "expires_in": 120, + "url": "{{CABLE_URL:1}}" + } + } + } + }, + { + "expectConnect": { + "url": "{{CABLE_URL:1}}" + } + }, + { + "serve": { + "frame": "welcome" + } + }, + { + "expectSubscribe": { + "channel": "EventsChannel" + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 51, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 52, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 53, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "expectSignal": { + "kind": "bufferOverflow", + "droppedIds": [ + 51 + ], + "droppedCount": 1 + } + }, + { + "expectClientClose": {} + }, + { + "expectError": { + "reason": "buffer_overflow" + } + } + ], + "finally": { + "state": "terminal", + "mintCount": 1, + "connectCount": 1, + "delivered": { + "exact": [] + }, + "checkpoints": { + "exact": [] + }, + "timers": { + "exact": {} + }, + "socket": "closed", + "error": { + "reason": "buffer_overflow" + }, + "handlerInvocations": { + "exact": [ + { + "kind": "bufferOverflow", + "disposition": "accept" + } + ] + } + } +} \ No newline at end of file From a68a5647ada97c2041c519ba37df20a3b66f029f Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 11 Aug 2026 13:24:32 -0700 Subject: [PATCH 4/6] Pair each invalid fixture with a minimal-delta positive control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rejected alone doesn't prove the right pin fired: a future unrelated schema constraint could reject the invalid fixture while the pin under test is removed, and the gate would stay green. Each invalid fixture now requires controls/.json — identical except the one violation is removed — which the gate asserts VALIDATES before asserting the invalid file is rejected. A missing or failing control fails the gate. Test the test, both directions: a schema mutated with an unrelated always-reject constraint fails at the control check; the pre-fix schema (pin absent) fails at the reject check. --- Makefile | 12 +- conformance/event-feed/README.md | 15 ++- .../controls/phantom-gap-invocation.json | 125 ++++++++++++++++++ .../controls/phantom-overflow-invocation.json | 125 ++++++++++++++++++ 4 files changed, 270 insertions(+), 7 deletions(-) create mode 100644 conformance/event-feed/invalid-fixtures/controls/phantom-gap-invocation.json create mode 100644 conformance/event-feed/invalid-fixtures/controls/phantom-overflow-invocation.json diff --git a/Makefile b/Makefile index 813e65d666..d2c82c6327 100644 --- a/Makefile +++ b/Makefile @@ -559,14 +559,22 @@ event-feed-fixtures-check: --check-metaschema conformance/event-feed/schema.json uvx --from 'check-jsonschema==$(CHECK_JSONSCHEMA_VERSION)' check-jsonschema \ --schemafile conformance/event-feed/schema.json conformance/event-feed/fixtures/*.json - @echo "==> Asserting event-feed invalid fixtures are rejected..." + @echo "==> Asserting event-feed invalid fixtures are rejected (and their controls accepted)..." @for f in conformance/event-feed/invalid-fixtures/*.json; do \ + c="conformance/event-feed/invalid-fixtures/controls/$$(basename $$f)"; \ + test -f "$$c" || { \ + echo "ERROR: $$f has no positive control at $$c — without it, rejection can't be isolated to the fixture's one violation"; \ + exit 1; }; \ + uvx --from 'check-jsonschema==$(CHECK_JSONSCHEMA_VERSION)' check-jsonschema \ + --schemafile conformance/event-feed/schema.json "$$c" >/dev/null 2>&1 || { \ + echo "ERROR: control $$c failed validation — $$f is now rejected for some reason besides its one violation, so the pin it exercises is unverified"; \ + exit 1; }; \ if uvx --from 'check-jsonschema==$(CHECK_JSONSCHEMA_VERSION)' check-jsonschema \ --schemafile conformance/event-feed/schema.json "$$f" >/dev/null 2>&1; then \ echo "ERROR: $$f validated, but this directory holds shapes the schema must reject"; \ exit 1; \ else \ - echo "rejected (as required): $$f"; \ + echo "rejected, control accepted: $$f"; \ fi; \ done diff --git a/conformance/event-feed/README.md b/conformance/event-feed/README.md index 6875daf097..b963c13144 100644 --- a/conformance/event-feed/README.md +++ b/conformance/event-feed/README.md @@ -54,11 +54,16 @@ pinned `check-jsonschema` run through `uvx` (part of `make conformance`, so The same gate then asserts every file in `invalid-fixtures/` is REJECTED by the schema. Those files are negative regression cases for load-bearing `allOf` pins — -each is schema-valid except for exactly one violation, so its rejection exercises -that pin and nothing else (the current pair pins the per-signal default-terminal -rules: a phantom invocation of a signal kind whose disposition key is absent). -Harnesses must never glob `invalid-fixtures/` — it is a gate input, not a scenario -inventory. +each is schema-valid except for exactly one violation (the current pair pins the +per-signal default-terminal rules: a phantom invocation of a signal kind whose +disposition key is absent). "Rejected" alone can't prove the right pin fired — a +future unrelated constraint could reject the file while the pin under test is +removed — so each invalid fixture requires a paired positive control at +`invalid-fixtures/controls/.json`, identical except the one violation +is removed, which the gate asserts VALIDATES. Rejected-with-control-accepted is +what isolates the rejection to the pin; a missing or failing control fails the +gate. Harnesses must never glob `invalid-fixtures/` (including `controls/`) — it +is a gate input, not a scenario inventory. ## Directory is a schema boundary diff --git a/conformance/event-feed/invalid-fixtures/controls/phantom-gap-invocation.json b/conformance/event-feed/invalid-fixtures/controls/phantom-gap-invocation.json new file mode 100644 index 0000000000..a2ef692a85 --- /dev/null +++ b/conformance/event-feed/invalid-fixtures/controls/phantom-gap-invocation.json @@ -0,0 +1,125 @@ +{ + "name": "control-no-phantom-gap-invocation", + "description": "POSITIVE CONTROL for phantom-gap-invocation.json \u2014 identical except the one violation is removed (the phantom feedGap invocation; the record is empty). MUST VALIDATE: if this file ever fails, the paired invalid fixture's rejection is no longer isolated to its single delta, and the gate reports the pin as unverifiable rather than staying silently green.", + "config": { + "liveBufferCapacity": 2, + "signalDisposition": { + "bufferOverflow": "accept" + } + }, + "steps": [ + { + "expectMint": { + "respond": { + "status": 200, + "body": { + "ticket": "{{TICKET:1}}", + "expires_in": 120, + "url": "{{CABLE_URL:1}}" + } + } + } + }, + { + "expectConnect": { + "url": "{{CABLE_URL:1}}" + } + }, + { + "serve": { + "frame": "welcome" + } + }, + { + "expectSubscribe": { + "channel": "EventsChannel" + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 51, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 52, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 53, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "expectSignal": { + "kind": "bufferOverflow", + "droppedIds": [ + 51 + ], + "droppedCount": 1 + } + }, + { + "expectClientClose": {} + }, + { + "expectError": { + "reason": "feed_gap" + } + } + ], + "finally": { + "state": "terminal", + "mintCount": 1, + "connectCount": 1, + "delivered": { + "exact": [] + }, + "checkpoints": { + "exact": [] + }, + "timers": { + "exact": {} + }, + "socket": "closed", + "error": { + "reason": "feed_gap" + }, + "handlerInvocations": { + "exact": [] + } + } +} \ No newline at end of file diff --git a/conformance/event-feed/invalid-fixtures/controls/phantom-overflow-invocation.json b/conformance/event-feed/invalid-fixtures/controls/phantom-overflow-invocation.json new file mode 100644 index 0000000000..86c9c34f0b --- /dev/null +++ b/conformance/event-feed/invalid-fixtures/controls/phantom-overflow-invocation.json @@ -0,0 +1,125 @@ +{ + "name": "control-no-phantom-overflow-invocation", + "description": "POSITIVE CONTROL for phantom-overflow-invocation.json \u2014 identical except the one violation is removed (the phantom bufferOverflow invocation; the record is empty). MUST VALIDATE: if this file ever fails, the paired invalid fixture's rejection is no longer isolated to its single delta, and the gate reports the pin as unverifiable rather than staying silently green.", + "config": { + "liveBufferCapacity": 2, + "signalDisposition": { + "feedGap": "accept" + } + }, + "steps": [ + { + "expectMint": { + "respond": { + "status": 200, + "body": { + "ticket": "{{TICKET:1}}", + "expires_in": 120, + "url": "{{CABLE_URL:1}}" + } + } + } + }, + { + "expectConnect": { + "url": "{{CABLE_URL:1}}" + } + }, + { + "serve": { + "frame": "welcome" + } + }, + { + "expectSubscribe": { + "channel": "EventsChannel" + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 51, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 52, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 53, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "expectSignal": { + "kind": "bufferOverflow", + "droppedIds": [ + 51 + ], + "droppedCount": 1 + } + }, + { + "expectClientClose": {} + }, + { + "expectError": { + "reason": "buffer_overflow" + } + } + ], + "finally": { + "state": "terminal", + "mintCount": 1, + "connectCount": 1, + "delivered": { + "exact": [] + }, + "checkpoints": { + "exact": [] + }, + "timers": { + "exact": {} + }, + "socket": "closed", + "error": { + "reason": "buffer_overflow" + }, + "handlerInvocations": { + "exact": [] + } + } +} \ No newline at end of file From 20dfe76b4cb49b640a4909bccb8aa01b97bf548b Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 11 Aug 2026 13:34:02 -0700 Subject: [PATCH 5/6] Derive the pin mutants from controls instead of committing invalid files Two review rounds converged on the same root cause: a committed invalid fixture is a drift-prone artifact. Any nonzero validator exit counted as rejection (so a malformed file would false-green the gate), and basename pairing could not enforce the minimal delta between a fixture and its control. Remove the artifact: each pin-probes/*.json declares a control scenario and one mutation, and scripts/check-event-feed-pin-probes.py validates the control (must pass), derives the mutant in-process, and validates it (must fail). The accepted/rejected delta is the declared mutation by construction; a parse failure is a hard error, not a rejection; the control-first ordering means a mutant rejection can only be a schema rejection. Path-typo and value-equals-control probes fail as vacuous. Test the test: pin-less schema fails at the mutant check, always-reject schema fails at the control check, malformed probe / typo path / vacuous mutation each fail with their own error. --- Makefile | 21 +-- conformance/event-feed/README.md | 25 ++-- .../controls/phantom-gap-invocation.json | 125 ---------------- .../controls/phantom-overflow-invocation.json | 125 ---------------- .../phantom-gap-invocation.json | 130 ---------------- .../phantom-overflow-invocation.json | 130 ---------------- .../pin-probes/phantom-gap-invocation.json | 141 ++++++++++++++++++ .../phantom-overflow-invocation.json | 141 ++++++++++++++++++ scripts/check-event-feed-pin-probes.py | 107 +++++++++++++ 9 files changed, 405 insertions(+), 540 deletions(-) delete mode 100644 conformance/event-feed/invalid-fixtures/controls/phantom-gap-invocation.json delete mode 100644 conformance/event-feed/invalid-fixtures/controls/phantom-overflow-invocation.json delete mode 100644 conformance/event-feed/invalid-fixtures/phantom-gap-invocation.json delete mode 100644 conformance/event-feed/invalid-fixtures/phantom-overflow-invocation.json create mode 100644 conformance/event-feed/pin-probes/phantom-gap-invocation.json create mode 100644 conformance/event-feed/pin-probes/phantom-overflow-invocation.json create mode 100644 scripts/check-event-feed-pin-probes.py diff --git a/Makefile b/Makefile index d2c82c6327..bdea2d0e4c 100644 --- a/Makefile +++ b/Makefile @@ -559,24 +559,9 @@ event-feed-fixtures-check: --check-metaschema conformance/event-feed/schema.json uvx --from 'check-jsonschema==$(CHECK_JSONSCHEMA_VERSION)' check-jsonschema \ --schemafile conformance/event-feed/schema.json conformance/event-feed/fixtures/*.json - @echo "==> Asserting event-feed invalid fixtures are rejected (and their controls accepted)..." - @for f in conformance/event-feed/invalid-fixtures/*.json; do \ - c="conformance/event-feed/invalid-fixtures/controls/$$(basename $$f)"; \ - test -f "$$c" || { \ - echo "ERROR: $$f has no positive control at $$c — without it, rejection can't be isolated to the fixture's one violation"; \ - exit 1; }; \ - uvx --from 'check-jsonschema==$(CHECK_JSONSCHEMA_VERSION)' check-jsonschema \ - --schemafile conformance/event-feed/schema.json "$$c" >/dev/null 2>&1 || { \ - echo "ERROR: control $$c failed validation — $$f is now rejected for some reason besides its one violation, so the pin it exercises is unverified"; \ - exit 1; }; \ - if uvx --from 'check-jsonschema==$(CHECK_JSONSCHEMA_VERSION)' check-jsonschema \ - --schemafile conformance/event-feed/schema.json "$$f" >/dev/null 2>&1; then \ - echo "ERROR: $$f validated, but this directory holds shapes the schema must reject"; \ - exit 1; \ - else \ - echo "rejected, control accepted: $$f"; \ - fi; \ - done + @echo "==> Verifying event-feed schema pins via derived mutants..." + python3 scripts/check-event-feed-pin-probes.py \ + conformance/event-feed/schema.json conformance/event-feed/pin-probes '$(CHECK_JSONSCHEMA_VERSION)' event-feed-digest-fixtures-check: @echo "==> Validating event-feed srv1 digest vectors..." diff --git a/conformance/event-feed/README.md b/conformance/event-feed/README.md index b963c13144..8a9fba3afe 100644 --- a/conformance/event-feed/README.md +++ b/conformance/event-feed/README.md @@ -52,18 +52,19 @@ against the JSON Schema metaschema and every fixture against the schema, with a pinned `check-jsonschema` run through `uvx` (part of `make conformance`, so `make check` gates it). -The same gate then asserts every file in `invalid-fixtures/` is REJECTED by the -schema. Those files are negative regression cases for load-bearing `allOf` pins — -each is schema-valid except for exactly one violation (the current pair pins the -per-signal default-terminal rules: a phantom invocation of a signal kind whose -disposition key is absent). "Rejected" alone can't prove the right pin fired — a -future unrelated constraint could reject the file while the pin under test is -removed — so each invalid fixture requires a paired positive control at -`invalid-fixtures/controls/.json`, identical except the one violation -is removed, which the gate asserts VALIDATES. Rejected-with-control-accepted is -what isolates the rejection to the pin; a missing or failing control fails the -gate. Harnesses must never glob `invalid-fixtures/` (including `controls/`) — it -is a gate input, not a scenario inventory. +The same gate then verifies the schema's load-bearing `allOf` pins with the +probes in `pin-probes/` (`scripts/check-event-feed-pin-probes.py`). Each probe +declares a `control` scenario and one `mutation`; the gate requires the control +to VALIDATE, derives the mutant from it in-process, and requires the mutant to be +REJECTED. Deriving (rather than committing an invalid file) is what makes the +isolation claim true by construction: the accepted/rejected delta is exactly the +declared mutation, a control that stops validating fails the gate rather than +masking a wrong-reason rejection, and there is no invalid artifact to go +malformed or drift extra deltas. A mutation whose path is absent from the +control, or whose value equals the control's, fails as vacuous. The current pair +pins the per-signal default-terminal rules: a phantom invocation of a signal kind +whose disposition key is absent. Harnesses must never glob `pin-probes/` — probe +files are gate inputs, not scenario fixtures (and are not scenario-shaped). ## Directory is a schema boundary diff --git a/conformance/event-feed/invalid-fixtures/controls/phantom-gap-invocation.json b/conformance/event-feed/invalid-fixtures/controls/phantom-gap-invocation.json deleted file mode 100644 index a2ef692a85..0000000000 --- a/conformance/event-feed/invalid-fixtures/controls/phantom-gap-invocation.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "name": "control-no-phantom-gap-invocation", - "description": "POSITIVE CONTROL for phantom-gap-invocation.json \u2014 identical except the one violation is removed (the phantom feedGap invocation; the record is empty). MUST VALIDATE: if this file ever fails, the paired invalid fixture's rejection is no longer isolated to its single delta, and the gate reports the pin as unverifiable rather than staying silently green.", - "config": { - "liveBufferCapacity": 2, - "signalDisposition": { - "bufferOverflow": "accept" - } - }, - "steps": [ - { - "expectMint": { - "respond": { - "status": 200, - "body": { - "ticket": "{{TICKET:1}}", - "expires_in": 120, - "url": "{{CABLE_URL:1}}" - } - } - } - }, - { - "expectConnect": { - "url": "{{CABLE_URL:1}}" - } - }, - { - "serve": { - "frame": "welcome" - } - }, - { - "expectSubscribe": { - "channel": "EventsChannel" - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 51, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 52, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 53, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "expectSignal": { - "kind": "bufferOverflow", - "droppedIds": [ - 51 - ], - "droppedCount": 1 - } - }, - { - "expectClientClose": {} - }, - { - "expectError": { - "reason": "feed_gap" - } - } - ], - "finally": { - "state": "terminal", - "mintCount": 1, - "connectCount": 1, - "delivered": { - "exact": [] - }, - "checkpoints": { - "exact": [] - }, - "timers": { - "exact": {} - }, - "socket": "closed", - "error": { - "reason": "feed_gap" - }, - "handlerInvocations": { - "exact": [] - } - } -} \ No newline at end of file diff --git a/conformance/event-feed/invalid-fixtures/controls/phantom-overflow-invocation.json b/conformance/event-feed/invalid-fixtures/controls/phantom-overflow-invocation.json deleted file mode 100644 index 86c9c34f0b..0000000000 --- a/conformance/event-feed/invalid-fixtures/controls/phantom-overflow-invocation.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "name": "control-no-phantom-overflow-invocation", - "description": "POSITIVE CONTROL for phantom-overflow-invocation.json \u2014 identical except the one violation is removed (the phantom bufferOverflow invocation; the record is empty). MUST VALIDATE: if this file ever fails, the paired invalid fixture's rejection is no longer isolated to its single delta, and the gate reports the pin as unverifiable rather than staying silently green.", - "config": { - "liveBufferCapacity": 2, - "signalDisposition": { - "feedGap": "accept" - } - }, - "steps": [ - { - "expectMint": { - "respond": { - "status": 200, - "body": { - "ticket": "{{TICKET:1}}", - "expires_in": 120, - "url": "{{CABLE_URL:1}}" - } - } - } - }, - { - "expectConnect": { - "url": "{{CABLE_URL:1}}" - } - }, - { - "serve": { - "frame": "welcome" - } - }, - { - "expectSubscribe": { - "channel": "EventsChannel" - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 51, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 52, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 53, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "expectSignal": { - "kind": "bufferOverflow", - "droppedIds": [ - 51 - ], - "droppedCount": 1 - } - }, - { - "expectClientClose": {} - }, - { - "expectError": { - "reason": "buffer_overflow" - } - } - ], - "finally": { - "state": "terminal", - "mintCount": 1, - "connectCount": 1, - "delivered": { - "exact": [] - }, - "checkpoints": { - "exact": [] - }, - "timers": { - "exact": {} - }, - "socket": "closed", - "error": { - "reason": "buffer_overflow" - }, - "handlerInvocations": { - "exact": [] - } - } -} \ No newline at end of file diff --git a/conformance/event-feed/invalid-fixtures/phantom-gap-invocation.json b/conformance/event-feed/invalid-fixtures/phantom-gap-invocation.json deleted file mode 100644 index 3a9caa0888..0000000000 --- a/conformance/event-feed/invalid-fixtures/phantom-gap-invocation.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "name": "phantom-gap-invocation", - "description": "MUST BE REJECTED by schema.json (this directory is gated as invalid): a feed_gap default-terminal scenario configuring only a bufferOverflow disposition, claiming a phantom feedGap handler invocation. Exercises the per-signal feedGap pin \u2014 the mirror of phantom-overflow-invocation. Everything else about the file is schema-valid, so rejection isolates the pin.", - "config": { - "liveBufferCapacity": 2, - "signalDisposition": { - "bufferOverflow": "accept" - } - }, - "steps": [ - { - "expectMint": { - "respond": { - "status": 200, - "body": { - "ticket": "{{TICKET:1}}", - "expires_in": 120, - "url": "{{CABLE_URL:1}}" - } - } - } - }, - { - "expectConnect": { - "url": "{{CABLE_URL:1}}" - } - }, - { - "serve": { - "frame": "welcome" - } - }, - { - "expectSubscribe": { - "channel": "EventsChannel" - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 51, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 52, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 53, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "expectSignal": { - "kind": "bufferOverflow", - "droppedIds": [ - 51 - ], - "droppedCount": 1 - } - }, - { - "expectClientClose": {} - }, - { - "expectError": { - "reason": "feed_gap" - } - } - ], - "finally": { - "state": "terminal", - "mintCount": 1, - "connectCount": 1, - "delivered": { - "exact": [] - }, - "checkpoints": { - "exact": [] - }, - "timers": { - "exact": {} - }, - "socket": "closed", - "error": { - "reason": "feed_gap" - }, - "handlerInvocations": { - "exact": [ - { - "kind": "feedGap", - "disposition": "accept" - } - ] - } - } -} \ No newline at end of file diff --git a/conformance/event-feed/invalid-fixtures/phantom-overflow-invocation.json b/conformance/event-feed/invalid-fixtures/phantom-overflow-invocation.json deleted file mode 100644 index 296cabadcb..0000000000 --- a/conformance/event-feed/invalid-fixtures/phantom-overflow-invocation.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "name": "phantom-overflow-invocation", - "description": "MUST BE REJECTED by schema.json (this directory is gated as invalid): a buffer_overflow default-terminal scenario configuring only a feedGap disposition, claiming a phantom bufferOverflow handler invocation. Exercises the per-signal bufferOverflow pin \u2014 the whole-object form of the pin accepted this shape. Everything else about the file is schema-valid, so rejection isolates the pin.", - "config": { - "liveBufferCapacity": 2, - "signalDisposition": { - "feedGap": "accept" - } - }, - "steps": [ - { - "expectMint": { - "respond": { - "status": 200, - "body": { - "ticket": "{{TICKET:1}}", - "expires_in": 120, - "url": "{{CABLE_URL:1}}" - } - } - } - }, - { - "expectConnect": { - "url": "{{CABLE_URL:1}}" - } - }, - { - "serve": { - "frame": "welcome" - } - }, - { - "expectSubscribe": { - "channel": "EventsChannel" - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 51, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 52, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "serve": { - "frame": "message", - "event": { - "id": 53, - "kind": "message", - "event_type": "message.created", - "action": "created", - "created_at": "2026-08-01T12:00:00Z", - "bucket_id": 2, - "creator_id": 3, - "recording_id": 900, - "visible_to_clients": false - } - } - }, - { - "expectSignal": { - "kind": "bufferOverflow", - "droppedIds": [ - 51 - ], - "droppedCount": 1 - } - }, - { - "expectClientClose": {} - }, - { - "expectError": { - "reason": "buffer_overflow" - } - } - ], - "finally": { - "state": "terminal", - "mintCount": 1, - "connectCount": 1, - "delivered": { - "exact": [] - }, - "checkpoints": { - "exact": [] - }, - "timers": { - "exact": {} - }, - "socket": "closed", - "error": { - "reason": "buffer_overflow" - }, - "handlerInvocations": { - "exact": [ - { - "kind": "bufferOverflow", - "disposition": "accept" - } - ] - } - } -} \ No newline at end of file diff --git a/conformance/event-feed/pin-probes/phantom-gap-invocation.json b/conformance/event-feed/pin-probes/phantom-gap-invocation.json new file mode 100644 index 0000000000..98ec0b863c --- /dev/null +++ b/conformance/event-feed/pin-probes/phantom-gap-invocation.json @@ -0,0 +1,141 @@ +{ + "description": "Pin probe for the per-signal feedGap default-terminal rule: an unregistered handler cannot be invoked, so injecting a phantom feedGap invocation into the otherwise-valid control must be rejected. The mutant is DERIVED from the control at gate time, so the delta between accepted and rejected is exactly this mutation \u2014 nothing else can drift.", + "control": { + "name": "pin-probe-gap-invocation-control", + "description": "Control half of the phantom-gap-invocation pin probe: a feed_gap default-terminal scenario whose signalDisposition configures only the OTHER signal's key, with an empty handler-invocation record. Must validate; the gate then applies this probe's declared mutation (a phantom feedGap invocation) and requires the schema to reject the result \u2014 proving the per-signal pin, and only it, fired.", + "config": { + "liveBufferCapacity": 2, + "signalDisposition": { + "bufferOverflow": "accept" + } + }, + "steps": [ + { + "expectMint": { + "respond": { + "status": 200, + "body": { + "ticket": "{{TICKET:1}}", + "expires_in": 120, + "url": "{{CABLE_URL:1}}" + } + } + } + }, + { + "expectConnect": { + "url": "{{CABLE_URL:1}}" + } + }, + { + "serve": { + "frame": "welcome" + } + }, + { + "expectSubscribe": { + "channel": "EventsChannel" + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 51, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 52, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 53, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "expectSignal": { + "kind": "bufferOverflow", + "droppedIds": [ + 51 + ], + "droppedCount": 1 + } + }, + { + "expectClientClose": {} + }, + { + "expectError": { + "reason": "feed_gap" + } + } + ], + "finally": { + "state": "terminal", + "mintCount": 1, + "connectCount": 1, + "delivered": { + "exact": [] + }, + "checkpoints": { + "exact": [] + }, + "timers": { + "exact": {} + }, + "socket": "closed", + "error": { + "reason": "feed_gap" + }, + "handlerInvocations": { + "exact": [] + } + } + }, + "mutation": { + "path": [ + "finally", + "handlerInvocations", + "exact" + ], + "value": [ + { + "kind": "feedGap", + "disposition": "accept" + } + ] + } +} \ No newline at end of file diff --git a/conformance/event-feed/pin-probes/phantom-overflow-invocation.json b/conformance/event-feed/pin-probes/phantom-overflow-invocation.json new file mode 100644 index 0000000000..b4311154fb --- /dev/null +++ b/conformance/event-feed/pin-probes/phantom-overflow-invocation.json @@ -0,0 +1,141 @@ +{ + "description": "Pin probe for the per-signal bufferOverflow default-terminal rule: an unregistered handler cannot be invoked, so injecting a phantom bufferOverflow invocation into the otherwise-valid control must be rejected. The mutant is DERIVED from the control at gate time, so the delta between accepted and rejected is exactly this mutation \u2014 nothing else can drift.", + "control": { + "name": "pin-probe-overflow-invocation-control", + "description": "Control half of the phantom-overflow-invocation pin probe: a buffer_overflow default-terminal scenario whose signalDisposition configures only the OTHER signal's key, with an empty handler-invocation record. Must validate; the gate then applies this probe's declared mutation (a phantom bufferOverflow invocation) and requires the schema to reject the result \u2014 proving the per-signal pin, and only it, fired.", + "config": { + "liveBufferCapacity": 2, + "signalDisposition": { + "feedGap": "accept" + } + }, + "steps": [ + { + "expectMint": { + "respond": { + "status": 200, + "body": { + "ticket": "{{TICKET:1}}", + "expires_in": 120, + "url": "{{CABLE_URL:1}}" + } + } + } + }, + { + "expectConnect": { + "url": "{{CABLE_URL:1}}" + } + }, + { + "serve": { + "frame": "welcome" + } + }, + { + "expectSubscribe": { + "channel": "EventsChannel" + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 51, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 52, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 53, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "expectSignal": { + "kind": "bufferOverflow", + "droppedIds": [ + 51 + ], + "droppedCount": 1 + } + }, + { + "expectClientClose": {} + }, + { + "expectError": { + "reason": "buffer_overflow" + } + } + ], + "finally": { + "state": "terminal", + "mintCount": 1, + "connectCount": 1, + "delivered": { + "exact": [] + }, + "checkpoints": { + "exact": [] + }, + "timers": { + "exact": {} + }, + "socket": "closed", + "error": { + "reason": "buffer_overflow" + }, + "handlerInvocations": { + "exact": [] + } + } + }, + "mutation": { + "path": [ + "finally", + "handlerInvocations", + "exact" + ], + "value": [ + { + "kind": "bufferOverflow", + "disposition": "accept" + } + ] + } +} \ No newline at end of file diff --git a/scripts/check-event-feed-pin-probes.py b/scripts/check-event-feed-pin-probes.py new file mode 100644 index 0000000000..31b4c07c10 --- /dev/null +++ b/scripts/check-event-feed-pin-probes.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Verify the event-feed schema's load-bearing allOf pins via derived mutants. + +Each probe file under the probes directory declares a `control` scenario and one +`mutation` (a path into the control and the value to plant there). The gate: + + 1. validates the control against the schema — it MUST be accepted; + 2. applies the mutation in-process and validates the mutant — it MUST be rejected. + +Because the mutant is derived from a control that just parsed and validated, a +rejection can only be a schema rejection of the mutation's delta: there is no +committed invalid file to go stale, drift extra deltas, or fail as malformed JSON, +and the control-first ordering rules out schema-load failures masquerading as +rejections. A mutation whose path is absent from the control, or whose value +already equals the control's, fails the gate as a vacuous probe. +""" + +import copy +import json +import subprocess +import sys +import tempfile +from pathlib import Path + + +def fail(message): + print(f"ERROR: {message}", file=sys.stderr) + sys.exit(1) + + +def validate(schema, instance_path, checker_version): + return subprocess.run( + [ + "uvx", + "--from", + f"check-jsonschema=={checker_version}", + "check-jsonschema", + "--schemafile", + str(schema), + str(instance_path), + ], + capture_output=True, + text=True, + ) + + +def main(): + if len(sys.argv) != 4: + fail(f"usage: {sys.argv[0]} ") + schema, probes_dir, checker_version = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3] + + probes = sorted(probes_dir.glob("*.json")) + if not probes: + fail(f"no pin probes found in {probes_dir} — the gate would be vacuously green") + + for probe_path in probes: + try: + probe = json.loads(probe_path.read_text()) + except json.JSONDecodeError as e: + fail(f"{probe_path.name}: not valid JSON ({e})") + try: + control, mutation = probe["control"], probe["mutation"] + path, value = mutation["path"], mutation["value"] + except (KeyError, TypeError): + fail(f"{probe_path.name}: a probe must carry 'control' and 'mutation' {{path, value}}") + + node = control + for key in path[:-1]: + if not isinstance(node, dict) or key not in node: + fail(f"{probe_path.name}: mutation path {path} does not exist in the control") + node = node[key] + leaf = path[-1] + if not isinstance(node, dict) or leaf not in node: + fail(f"{probe_path.name}: mutation path {path} does not exist in the control") + if node[leaf] == value: + fail(f"{probe_path.name}: mutation value equals the control's — the probe is vacuous") + + mutant = copy.deepcopy(control) + target = mutant + for key in path[:-1]: + target = target[key] + target[leaf] = value + + with tempfile.TemporaryDirectory() as tmp: + control_path = Path(tmp) / f"{probe_path.stem}.control.json" + mutant_path = Path(tmp) / f"{probe_path.stem}.mutant.json" + control_path.write_text(json.dumps(control)) + mutant_path.write_text(json.dumps(mutant)) + + result = validate(schema, control_path, checker_version) + if result.returncode != 0: + fail( + f"{probe_path.name}: the CONTROL failed validation, so the pin under test " + f"cannot be isolated:\n{result.stdout}{result.stderr}" + ) + result = validate(schema, mutant_path, checker_version) + if result.returncode == 0: + fail( + f"{probe_path.name}: the MUTANT validated — the pin this probe exercises " + f"is missing or has been widened (mutation {path} = {json.dumps(value)})" + ) + + print(f"pin verified: {probe_path.name} (control accepted, mutant rejected)") + + +if __name__ == "__main__": + main() From d94e0e7cc983e7384667def904c9b88fa92724e7 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Tue, 11 Aug 2026 13:41:28 -0700 Subject: [PATCH 6/6] Guard both directions of each pin, and trust only structured rejections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The controls asserted exact: [], so a regression back to the over-strict whole-record-empty pin would have passed both probes — the permissive half of the per-kind rule was unguarded. Each control now records the LEGAL other-kind invocation (its disposition key is configured), which an over-strict pin rejects at the control check; each mutant retains that record and adds the phantom invocation of the unhandled kind. The mutant branch also stops reading any nonzero exit as rejection: check-jsonschema exits nonzero for tool, schema-parse, and reference errors too, and a transient failure on that one invocation would have printed pin-verified. The script now requires the validator's JSON output to show a genuine instance-validation failure (status fail, errors non-empty, no parse errors); anything else fails the gate. Test the test: pin-less schema fails at the mutant check, over-strict const-[] schema fails at the control check, always-reject schema fails at the control check, and the discriminator unit cases (tool-error text, parse_errors, empty errors, success, genuine rejection) all behave. --- .../pin-probes/phantom-gap-invocation.json | 15 +++++-- .../phantom-overflow-invocation.json | 15 +++++-- scripts/check-event-feed-pin-probes.py | 44 ++++++++++++++++--- 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/conformance/event-feed/pin-probes/phantom-gap-invocation.json b/conformance/event-feed/pin-probes/phantom-gap-invocation.json index 98ec0b863c..18f90f3ff3 100644 --- a/conformance/event-feed/pin-probes/phantom-gap-invocation.json +++ b/conformance/event-feed/pin-probes/phantom-gap-invocation.json @@ -1,8 +1,8 @@ { - "description": "Pin probe for the per-signal feedGap default-terminal rule: an unregistered handler cannot be invoked, so injecting a phantom feedGap invocation into the otherwise-valid control must be rejected. The mutant is DERIVED from the control at gate time, so the delta between accepted and rejected is exactly this mutation \u2014 nothing else can drift.", + "description": "Pin probe for the per-signal feedGap default-terminal rule, guarding BOTH directions. The control configures only a bufferOverflow disposition, ends feed_gap default-terminal, and records the legal {bufferOverflow, accept} invocation \u2014 so a regression to the over-strict whole-record-empty form (const []) rejects the control and fails the gate. The mutant retains that record and adds a phantom {feedGap, accept} \u2014 an unregistered handler cannot be invoked \u2014 so a regression that widens or removes the pin accepts the mutant and fails the gate.", "control": { "name": "pin-probe-gap-invocation-control", - "description": "Control half of the phantom-gap-invocation pin probe: a feed_gap default-terminal scenario whose signalDisposition configures only the OTHER signal's key, with an empty handler-invocation record. Must validate; the gate then applies this probe's declared mutation (a phantom feedGap invocation) and requires the schema to reject the result \u2014 proving the per-signal pin, and only it, fired.", + "description": "Control half: feed_gap default-terminal with only a bufferOverflow disposition configured; the record holds exactly the legal bufferOverflow invocation. Must validate \u2014 under an over-strict whole-record-empty pin it would not.", "config": { "liveBufferCapacity": 2, "signalDisposition": { @@ -121,7 +121,12 @@ "reason": "feed_gap" }, "handlerInvocations": { - "exact": [] + "exact": [ + { + "kind": "bufferOverflow", + "disposition": "accept" + } + ] } } }, @@ -132,6 +137,10 @@ "exact" ], "value": [ + { + "kind": "bufferOverflow", + "disposition": "accept" + }, { "kind": "feedGap", "disposition": "accept" diff --git a/conformance/event-feed/pin-probes/phantom-overflow-invocation.json b/conformance/event-feed/pin-probes/phantom-overflow-invocation.json index b4311154fb..8835d887cd 100644 --- a/conformance/event-feed/pin-probes/phantom-overflow-invocation.json +++ b/conformance/event-feed/pin-probes/phantom-overflow-invocation.json @@ -1,8 +1,8 @@ { - "description": "Pin probe for the per-signal bufferOverflow default-terminal rule: an unregistered handler cannot be invoked, so injecting a phantom bufferOverflow invocation into the otherwise-valid control must be rejected. The mutant is DERIVED from the control at gate time, so the delta between accepted and rejected is exactly this mutation \u2014 nothing else can drift.", + "description": "Pin probe for the per-signal bufferOverflow default-terminal rule, guarding BOTH directions. The control configures only a feedGap disposition, ends buffer_overflow default-terminal, and records the legal {feedGap, accept} invocation \u2014 so a regression to the over-strict whole-record-empty form (const []) rejects the control and fails the gate. The mutant retains that record and adds a phantom {bufferOverflow, accept} \u2014 an unregistered handler cannot be invoked \u2014 so a regression that widens or removes the pin accepts the mutant and fails the gate.", "control": { "name": "pin-probe-overflow-invocation-control", - "description": "Control half of the phantom-overflow-invocation pin probe: a buffer_overflow default-terminal scenario whose signalDisposition configures only the OTHER signal's key, with an empty handler-invocation record. Must validate; the gate then applies this probe's declared mutation (a phantom bufferOverflow invocation) and requires the schema to reject the result \u2014 proving the per-signal pin, and only it, fired.", + "description": "Control half: buffer_overflow default-terminal with only a feedGap disposition configured; the record holds exactly the legal feedGap invocation. Must validate \u2014 under an over-strict whole-record-empty pin it would not.", "config": { "liveBufferCapacity": 2, "signalDisposition": { @@ -121,7 +121,12 @@ "reason": "buffer_overflow" }, "handlerInvocations": { - "exact": [] + "exact": [ + { + "kind": "feedGap", + "disposition": "accept" + } + ] } } }, @@ -132,6 +137,10 @@ "exact" ], "value": [ + { + "kind": "feedGap", + "disposition": "accept" + }, { "kind": "bufferOverflow", "disposition": "accept" diff --git a/scripts/check-event-feed-pin-probes.py b/scripts/check-event-feed-pin-probes.py index 31b4c07c10..570e8a3d80 100644 --- a/scripts/check-event-feed-pin-probes.py +++ b/scripts/check-event-feed-pin-probes.py @@ -7,12 +7,14 @@ 1. validates the control against the schema — it MUST be accepted; 2. applies the mutation in-process and validates the mutant — it MUST be rejected. -Because the mutant is derived from a control that just parsed and validated, a -rejection can only be a schema rejection of the mutation's delta: there is no -committed invalid file to go stale, drift extra deltas, or fail as malformed JSON, -and the control-first ordering rules out schema-load failures masquerading as -rejections. A mutation whose path is absent from the control, or whose value -already equals the control's, fails the gate as a vacuous probe. +Because the mutant is derived from a control that just parsed and validated, the +accepted/rejected delta is exactly the mutation: there is no committed invalid +file to go stale, drift extra deltas, or fail as malformed JSON. The mutant's +rejection must additionally prove itself as an instance-validation failure via +the validator's structured JSON output — a tool, schema-parse, or reference +error on that invocation fails the gate instead of counting as the pin firing. +A mutation whose path is absent from the control, or whose value already equals +the control's, fails the gate as a vacuous probe. """ import copy @@ -35,6 +37,8 @@ def validate(schema, instance_path, checker_version): "--from", f"check-jsonschema=={checker_version}", "check-jsonschema", + "--output-format", + "json", "--schemafile", str(schema), str(instance_path), @@ -44,6 +48,27 @@ def validate(schema, instance_path, checker_version): ) +def instance_validation_errors(result): + """The mutant's rejection must be a genuine instance-validation failure. + + check-jsonschema exits nonzero for tool, schema-parse, and reference errors + too, and those must fail the gate rather than count as the pin firing. Under + --output-format json an instance rejection is the one outcome that emits + parseable JSON with status "fail", at least one validation error, and no + parse errors; everything else (non-JSON error text, parse_errors, empty + errors) is an unexpected validator outcome. + """ + if result.returncode == 0: + return None + try: + report = json.loads(result.stdout) + except json.JSONDecodeError: + return None + if report.get("status") != "fail" or report.get("parse_errors") or not report.get("errors"): + return None + return report["errors"] + + def main(): if len(sys.argv) != 4: fail(f"usage: {sys.argv[0]} ") @@ -99,6 +124,13 @@ def main(): f"{probe_path.name}: the MUTANT validated — the pin this probe exercises " f"is missing or has been widened (mutation {path} = {json.dumps(value)})" ) + errors = instance_validation_errors(result) + if errors is None: + fail( + f"{probe_path.name}: the mutant run failed for a reason other than instance " + f"validation (tool, schema, or parse error) — the pin is unverified:\n" + f"{result.stdout}{result.stderr}" + ) print(f"pin verified: {probe_path.name} (control accepted, mutant rejected)")