diff --git a/Makefile b/Makefile index efbd085d43..bdea2d0e4c 100644 --- a/Makefile +++ b/Makefile @@ -559,6 +559,9 @@ event-feed-fixtures-check: --check-metaschema conformance/event-feed/schema.json uvx --from 'check-jsonschema==$(CHECK_JSONSCHEMA_VERSION)' check-jsonschema \ --schemafile conformance/event-feed/schema.json conformance/event-feed/fixtures/*.json + @echo "==> Verifying event-feed schema pins via derived mutants..." + python3 scripts/check-event-feed-pin-probes.py \ + conformance/event-feed/schema.json conformance/event-feed/pin-probes '$(CHECK_JSONSCHEMA_VERSION)' event-feed-digest-fixtures-check: @echo "==> Validating event-feed srv1 digest vectors..." diff --git a/conformance/event-feed/README.md b/conformance/event-feed/README.md index b5c85465bf..8a9fba3afe 100644 --- a/conformance/event-feed/README.md +++ b/conformance/event-feed/README.md @@ -52,6 +52,20 @@ against the JSON Schema metaschema and every fixture against the schema, with a pinned `check-jsonschema` run through `uvx` (part of `make conformance`, so `make check` gates it). +The same gate then verifies the schema's load-bearing `allOf` pins with the +probes in `pin-probes/` (`scripts/check-event-feed-pin-probes.py`). Each probe +declares a `control` scenario and one `mutation`; the gate requires the control +to VALIDATE, derives the mutant from it in-process, and requires the mutant to be +REJECTED. Deriving (rather than committing an invalid file) is what makes the +isolation claim true by construction: the accepted/rejected delta is exactly the +declared mutation, a control that stops validating fails the gate rather than +masking a wrong-reason rejection, and there is no invalid artifact to go +malformed or drift extra deltas. A mutation whose path is absent from the +control, or whose value equals the control's, fails as vacuous. The current pair +pins the per-signal default-terminal rules: a phantom invocation of a signal kind +whose disposition key is absent. Harnesses must never glob `pin-probes/` — probe +files are gate inputs, not scenario fixtures (and are not scenario-shaped). + ## Directory is a schema boundary This directory contains exactly one shape: tier-2 scenario scripts. If a second diff --git a/conformance/event-feed/pin-probes/phantom-gap-invocation.json b/conformance/event-feed/pin-probes/phantom-gap-invocation.json new file mode 100644 index 0000000000..18f90f3ff3 --- /dev/null +++ b/conformance/event-feed/pin-probes/phantom-gap-invocation.json @@ -0,0 +1,150 @@ +{ + "description": "Pin probe for the per-signal feedGap default-terminal rule, guarding BOTH directions. The control configures only a bufferOverflow disposition, ends feed_gap default-terminal, and records the legal {bufferOverflow, accept} invocation \u2014 so a regression to the over-strict whole-record-empty form (const []) rejects the control and fails the gate. The mutant retains that record and adds a phantom {feedGap, accept} \u2014 an unregistered handler cannot be invoked \u2014 so a regression that widens or removes the pin accepts the mutant and fails the gate.", + "control": { + "name": "pin-probe-gap-invocation-control", + "description": "Control half: feed_gap default-terminal with only a bufferOverflow disposition configured; the record holds exactly the legal bufferOverflow invocation. Must validate \u2014 under an over-strict whole-record-empty pin it would not.", + "config": { + "liveBufferCapacity": 2, + "signalDisposition": { + "bufferOverflow": "accept" + } + }, + "steps": [ + { + "expectMint": { + "respond": { + "status": 200, + "body": { + "ticket": "{{TICKET:1}}", + "expires_in": 120, + "url": "{{CABLE_URL:1}}" + } + } + } + }, + { + "expectConnect": { + "url": "{{CABLE_URL:1}}" + } + }, + { + "serve": { + "frame": "welcome" + } + }, + { + "expectSubscribe": { + "channel": "EventsChannel" + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 51, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 52, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 53, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "expectSignal": { + "kind": "bufferOverflow", + "droppedIds": [ + 51 + ], + "droppedCount": 1 + } + }, + { + "expectClientClose": {} + }, + { + "expectError": { + "reason": "feed_gap" + } + } + ], + "finally": { + "state": "terminal", + "mintCount": 1, + "connectCount": 1, + "delivered": { + "exact": [] + }, + "checkpoints": { + "exact": [] + }, + "timers": { + "exact": {} + }, + "socket": "closed", + "error": { + "reason": "feed_gap" + }, + "handlerInvocations": { + "exact": [ + { + "kind": "bufferOverflow", + "disposition": "accept" + } + ] + } + } + }, + "mutation": { + "path": [ + "finally", + "handlerInvocations", + "exact" + ], + "value": [ + { + "kind": "bufferOverflow", + "disposition": "accept" + }, + { + "kind": "feedGap", + "disposition": "accept" + } + ] + } +} \ No newline at end of file diff --git a/conformance/event-feed/pin-probes/phantom-overflow-invocation.json b/conformance/event-feed/pin-probes/phantom-overflow-invocation.json new file mode 100644 index 0000000000..8835d887cd --- /dev/null +++ b/conformance/event-feed/pin-probes/phantom-overflow-invocation.json @@ -0,0 +1,150 @@ +{ + "description": "Pin probe for the per-signal bufferOverflow default-terminal rule, guarding BOTH directions. The control configures only a feedGap disposition, ends buffer_overflow default-terminal, and records the legal {feedGap, accept} invocation \u2014 so a regression to the over-strict whole-record-empty form (const []) rejects the control and fails the gate. The mutant retains that record and adds a phantom {bufferOverflow, accept} \u2014 an unregistered handler cannot be invoked \u2014 so a regression that widens or removes the pin accepts the mutant and fails the gate.", + "control": { + "name": "pin-probe-overflow-invocation-control", + "description": "Control half: buffer_overflow default-terminal with only a feedGap disposition configured; the record holds exactly the legal feedGap invocation. Must validate \u2014 under an over-strict whole-record-empty pin it would not.", + "config": { + "liveBufferCapacity": 2, + "signalDisposition": { + "feedGap": "accept" + } + }, + "steps": [ + { + "expectMint": { + "respond": { + "status": 200, + "body": { + "ticket": "{{TICKET:1}}", + "expires_in": 120, + "url": "{{CABLE_URL:1}}" + } + } + } + }, + { + "expectConnect": { + "url": "{{CABLE_URL:1}}" + } + }, + { + "serve": { + "frame": "welcome" + } + }, + { + "expectSubscribe": { + "channel": "EventsChannel" + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 51, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 52, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "serve": { + "frame": "message", + "event": { + "id": 53, + "kind": "message", + "event_type": "message.created", + "action": "created", + "created_at": "2026-08-01T12:00:00Z", + "bucket_id": 2, + "creator_id": 3, + "recording_id": 900, + "visible_to_clients": false + } + } + }, + { + "expectSignal": { + "kind": "bufferOverflow", + "droppedIds": [ + 51 + ], + "droppedCount": 1 + } + }, + { + "expectClientClose": {} + }, + { + "expectError": { + "reason": "buffer_overflow" + } + } + ], + "finally": { + "state": "terminal", + "mintCount": 1, + "connectCount": 1, + "delivered": { + "exact": [] + }, + "checkpoints": { + "exact": [] + }, + "timers": { + "exact": {} + }, + "socket": "closed", + "error": { + "reason": "buffer_overflow" + }, + "handlerInvocations": { + "exact": [ + { + "kind": "feedGap", + "disposition": "accept" + } + ] + } + } + }, + "mutation": { + "path": [ + "finally", + "handlerInvocations", + "exact" + ], + "value": [ + { + "kind": "feedGap", + "disposition": "accept" + }, + { + "kind": "bufferOverflow", + "disposition": "accept" + } + ] + } +} \ No newline at end of file diff --git a/conformance/event-feed/schema.json b/conformance/event-feed/schema.json index 0bc8e592ed..12d144d931 100644 --- a/conformance/event-feed/schema.json +++ b/conformance/event-feed/schema.json @@ -62,7 +62,7 @@ } }, { - "description": "A default-terminal semantic-signal fixture (terminal reason buffer_overflow or feed_gap with NO signalDisposition configured) MUST assert ZERO handler invocations — the exact-empty record is what proves the default path was taken rather than a skipped handler.", + "description": "Per-signal default-terminal pin: a buffer_overflow terminal with NO bufferOverflow disposition configured (config absent entirely, or present without the key — either way no overflow handler exists) MUST carry a handler-invocation record containing zero invocations of that kind: an unregistered handler cannot be invoked, so a bufferOverflow entry here is unsatisfiable at runtime and invalid at load. Invocations of OTHER kinds whose handlers ARE configured remain legal (a cross-signal run may accept a gap before an unhandled overflow terminates it).", "if": { "required": [ "finally" @@ -72,7 +72,14 @@ "not": { "required": [ "signalDisposition" - ] + ], + "properties": { + "signalDisposition": { + "required": [ + "bufferOverflow" + ] + } + } } }, "finally": { @@ -86,10 +93,7 @@ ], "properties": { "reason": { - "enum": [ - "buffer_overflow", - "feed_gap" - ] + "const": "buffer_overflow" } } } @@ -107,7 +111,88 @@ "handlerInvocations": { "properties": { "exact": { - "const": [] + "not": { + "contains": { + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "bufferOverflow" + } + } + } + } + } + } + } + } + } + } + } + }, + { + "description": "Per-signal default-terminal pin: a feed_gap terminal with NO feedGap disposition configured (config absent entirely, or present without the key — either way no gap handler exists) MUST carry a handler-invocation record containing zero invocations of that kind: an unregistered handler cannot be invoked, so a feedGap entry here is unsatisfiable at runtime and invalid at load. Invocations of OTHER kinds whose handlers ARE configured remain legal (a cross-signal run may accept an overflow before an unhandled gap terminates it).", + "if": { + "required": [ + "finally" + ], + "properties": { + "config": { + "not": { + "required": [ + "signalDisposition" + ], + "properties": { + "signalDisposition": { + "required": [ + "feedGap" + ] + } + } + } + }, + "finally": { + "required": [ + "error" + ], + "properties": { + "error": { + "required": [ + "reason" + ], + "properties": { + "reason": { + "const": "feed_gap" + } + } + } + } + } + } + }, + "then": { + "properties": { + "finally": { + "required": [ + "handlerInvocations" + ], + "properties": { + "handlerInvocations": { + "properties": { + "exact": { + "not": { + "contains": { + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "feedGap" + } + } + } + } } } } diff --git a/scripts/check-event-feed-pin-probes.py b/scripts/check-event-feed-pin-probes.py new file mode 100644 index 0000000000..570e8a3d80 --- /dev/null +++ b/scripts/check-event-feed-pin-probes.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Verify the event-feed schema's load-bearing allOf pins via derived mutants. + +Each probe file under the probes directory declares a `control` scenario and one +`mutation` (a path into the control and the value to plant there). The gate: + + 1. validates the control against the schema — it MUST be accepted; + 2. applies the mutation in-process and validates the mutant — it MUST be rejected. + +Because the mutant is derived from a control that just parsed and validated, the +accepted/rejected delta is exactly the mutation: there is no committed invalid +file to go stale, drift extra deltas, or fail as malformed JSON. The mutant's +rejection must additionally prove itself as an instance-validation failure via +the validator's structured JSON output — a tool, schema-parse, or reference +error on that invocation fails the gate instead of counting as the pin firing. +A mutation whose path is absent from the control, or whose value already equals +the control's, fails the gate as a vacuous probe. +""" + +import copy +import json +import subprocess +import sys +import tempfile +from pathlib import Path + + +def fail(message): + print(f"ERROR: {message}", file=sys.stderr) + sys.exit(1) + + +def validate(schema, instance_path, checker_version): + return subprocess.run( + [ + "uvx", + "--from", + f"check-jsonschema=={checker_version}", + "check-jsonschema", + "--output-format", + "json", + "--schemafile", + str(schema), + str(instance_path), + ], + capture_output=True, + text=True, + ) + + +def instance_validation_errors(result): + """The mutant's rejection must be a genuine instance-validation failure. + + check-jsonschema exits nonzero for tool, schema-parse, and reference errors + too, and those must fail the gate rather than count as the pin firing. Under + --output-format json an instance rejection is the one outcome that emits + parseable JSON with status "fail", at least one validation error, and no + parse errors; everything else (non-JSON error text, parse_errors, empty + errors) is an unexpected validator outcome. + """ + if result.returncode == 0: + return None + try: + report = json.loads(result.stdout) + except json.JSONDecodeError: + return None + if report.get("status") != "fail" or report.get("parse_errors") or not report.get("errors"): + return None + return report["errors"] + + +def main(): + if len(sys.argv) != 4: + fail(f"usage: {sys.argv[0]} ") + schema, probes_dir, checker_version = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3] + + probes = sorted(probes_dir.glob("*.json")) + if not probes: + fail(f"no pin probes found in {probes_dir} — the gate would be vacuously green") + + for probe_path in probes: + try: + probe = json.loads(probe_path.read_text()) + except json.JSONDecodeError as e: + fail(f"{probe_path.name}: not valid JSON ({e})") + try: + control, mutation = probe["control"], probe["mutation"] + path, value = mutation["path"], mutation["value"] + except (KeyError, TypeError): + fail(f"{probe_path.name}: a probe must carry 'control' and 'mutation' {{path, value}}") + + node = control + for key in path[:-1]: + if not isinstance(node, dict) or key not in node: + fail(f"{probe_path.name}: mutation path {path} does not exist in the control") + node = node[key] + leaf = path[-1] + if not isinstance(node, dict) or leaf not in node: + fail(f"{probe_path.name}: mutation path {path} does not exist in the control") + if node[leaf] == value: + fail(f"{probe_path.name}: mutation value equals the control's — the probe is vacuous") + + mutant = copy.deepcopy(control) + target = mutant + for key in path[:-1]: + target = target[key] + target[leaf] = value + + with tempfile.TemporaryDirectory() as tmp: + control_path = Path(tmp) / f"{probe_path.stem}.control.json" + mutant_path = Path(tmp) / f"{probe_path.stem}.mutant.json" + control_path.write_text(json.dumps(control)) + mutant_path.write_text(json.dumps(mutant)) + + result = validate(schema, control_path, checker_version) + if result.returncode != 0: + fail( + f"{probe_path.name}: the CONTROL failed validation, so the pin under test " + f"cannot be isolated:\n{result.stdout}{result.stderr}" + ) + result = validate(schema, mutant_path, checker_version) + if result.returncode == 0: + fail( + f"{probe_path.name}: the MUTANT validated — the pin this probe exercises " + f"is missing or has been widened (mutation {path} = {json.dumps(value)})" + ) + errors = instance_validation_errors(result) + if errors is None: + fail( + f"{probe_path.name}: the mutant run failed for a reason other than instance " + f"validation (tool, schema, or parse error) — the pin is unverified:\n" + f"{result.stdout}{result.stderr}" + ) + + print(f"pin verified: {probe_path.name} (control accepted, mutant rejected)") + + +if __name__ == "__main__": + main()