From b1046203a69f87ad8531ff55b304f07811b0ac4f Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Mon, 7 Sep 2026 23:31:45 -0400 Subject: [PATCH 1/2] test: add fast-check property hardening --- package-lock.json | 41 ++++++++++ package.json | 1 + tests/fast-check-hardening.test.ts | 119 +++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 tests/fast-check-hardening.test.ts diff --git a/package-lock.json b/package-lock.json index 5a2e335..a4f8b88 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@ar-nelson/foldcase": "^1.0.1" }, "devDependencies": { + "fast-check": "^4.9.0", "typescript": "^5.9.3", "vitest": "^3.2.4" } @@ -1105,6 +1106,29 @@ "node": ">=12.0.0" } }, + "node_modules/fast-check": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.9.0.tgz", + "integrity": "sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^8.0.0" + }, + "engines": { + "node": ">=12.17.0" + } + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -1254,6 +1278,23 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/pure-rand": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.2.tgz", + "integrity": "sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, "node_modules/rollup": { "version": "4.60.1", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.1.tgz", diff --git a/package.json b/package.json index ad3795a..90bacfe 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "lint:md": "npx --yes markdownlint-cli2@0.23.0" }, "devDependencies": { + "fast-check": "^4.9.0", "typescript": "^5.9.3", "vitest": "^3.2.4" }, diff --git a/tests/fast-check-hardening.test.ts b/tests/fast-check-hardening.test.ts new file mode 100644 index 0000000..23f68c7 --- /dev/null +++ b/tests/fast-check-hardening.test.ts @@ -0,0 +1,119 @@ +import fc from 'fast-check'; +import { describe, expect, it } from 'vitest'; +import { Engine } from '../src/engine.js'; +import { CanonicalDirective, InvalidDirectiveSyntax, decompose_directive } from '../src/grammar.js'; + +const wordArb = fc.constantFrom('alpha', 'docker', 'focus', 'project', 'rollout', 'staging'); +const itemArb = fc.array(wordArb, { minLength: 1, maxLength: 3 }).map((words) => words.join(' ')); +const premiseValueArb = fc.array(wordArb, { minLength: 1, maxLength: 4 }).map((words) => words.join(' ')); + +const canonicalDirectiveArb = fc.oneof( + premiseValueArb.map((value) => new CanonicalDirective('set_premise', { value })), + premiseValueArb.map((value) => new CanonicalDirective('change_premise', { value })), + itemArb.map((item) => new CanonicalDirective('use_item', { item })), + itemArb.map((item) => new CanonicalDirective('prohibit_item', { item })), + itemArb.map((item) => new CanonicalDirective('remove_policy', { item })), + fc.tuple(itemArb, itemArb).map(([newItem, oldItem]) => new CanonicalDirective('replace_use', { new_item: newItem, old_item: oldItem })), + fc.constant(new CanonicalDirective('clear_premise', {})), + fc.constant(new CanonicalDirective('reset_policies', {})), + fc.constant(new CanonicalDirective('clear_state', {})) +); + +const reachableSequenceArb = fc.array(canonicalDirectiveArb, { maxLength: 8 }); + +function applySequence(engine: Engine, directives: CanonicalDirective[]): void { + for (const directive of directives) engine.apply_directive(directive); +} + +const equivalentPolicyPairArb = fc.record({ + base: fc.constantFrom("don't panic", 'the docker', 'ǰ rollout', 'straße'), + upper: fc.boolean(), + whitespace: fc.boolean(), + apostrophe: fc.boolean() +}).map(({ base, upper, whitespace, apostrophe }) => { + let variant = upper ? base.toUpperCase() : base; + if (whitespace) variant = variant.replaceAll(' ', ' '); + if (apostrophe) variant = variant.replaceAll("'", '’'); + return [base, variant] as const; +}); + +const invalidPayloadArb = fc.constantFrom( + '{', + JSON.stringify(null), + JSON.stringify([]), + JSON.stringify({ premise: null, policies: {}, version: 1 }), + JSON.stringify({ premise: 42, policies: {}, version: 2 }), + JSON.stringify({ premise: null, policies: [], version: 2 }), + JSON.stringify({ premise: null, policies: { docker: 'invalid' }, version: 2 }), + JSON.stringify({ premise: null, policies: { A: 'use', a: 'prohibit' }, version: 2 }), + JSON.stringify({ premise: null, policies: { ' ': 'use' }, version: 2 }), + JSON.stringify({ premise: ' ', policies: {}, version: 2 }), + JSON.stringify({ premise: null, policies: {} }) +); + +describe('fast-check property hardening', () => { + it('classifies arbitrary grammar strings without throwing', () => { + fc.assert(fc.property(fc.string({ maxLength: 160 }), (input) => { + const engine = new Engine(); + const before = engine.export_json(); + let result: unknown; + expect(() => { result = decompose_directive(input); }).not.toThrow(); + expect(result === null || result instanceof CanonicalDirective || result instanceof InvalidDirectiveSyntax).toBe(true); + expect(() => engine.step(input)).not.toThrow(); + if (!(result instanceof CanonicalDirective)) expect(engine.export_json()).toBe(before); + }), { numRuns: 300 }); + }); + + it('round-trips generated canonical directives through public grammar', () => { + fc.assert(fc.property(canonicalDirectiveArb, (directive) => { + const roundTrip = decompose_directive(directive.text); + expect(roundTrip).toBeInstanceOf(CanonicalDirective); + expect(roundTrip).toMatchObject({ kind: directive.kind, operands: directive.operands, text: directive.text }); + }), { numRuns: 200 }); + }); + + it('keeps equivalent policy operands at one identity and idempotent', () => { + fc.assert(fc.property(equivalentPolicyPairArb, ([base, variant]) => { + const engine = new Engine(); + engine.step('use ' + base); + const before = engine.export_json(); + const decision = engine.step('use ' + variant); + expect(decision).toMatchObject({ kind: 'update', changed: false }); + expect(engine.export_json()).toBe(before); + }), { numRuns: 150 }); + }); + + it('preserves state after generated semantic errors', () => { + fc.assert(fc.property(reachableSequenceArb, canonicalDirectiveArb, (prefix, directive) => { + const engine = new Engine(); + applySequence(engine, prefix); + const before = engine.export_json(); + const decision = engine.apply_directive(directive); + if (decision.kind === 'error') expect(engine.export_json()).toBe(before); + }), { numRuns: 250 }); + }); + + it('keeps reachable state export/import at a canonical fixed point', () => { + fc.assert(fc.property(reachableSequenceArb, (directives) => { + const engine = new Engine(); + applySequence(engine, directives); + const payload = engine.export_json(); + const restored = new Engine(); + restored.import_json(payload); + expect(restored.export_json()).toBe(payload); + }), { numRuns: 200 }); + }); + + it('preserves state when generated invalid imports are rejected', () => { + fc.assert(fc.property(reachableSequenceArb, invalidPayloadArb, (directives, payload) => { + const engine = new Engine(); + applySequence(engine, directives); + const before = engine.export_json(); + try { + engine.import_json(payload); + } catch { + expect(engine.export_json()).toBe(before); + } + }), { numRuns: 200 }); + }); +}); From a5a792aa0bc9d1b7e2ec88266627119c6f52770a Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Mon, 7 Sep 2026 23:58:30 -0400 Subject: [PATCH 2/2] test: strengthen fast-check generators --- tests/fast-check-hardening.test.ts | 97 ++++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 25 deletions(-) diff --git a/tests/fast-check-hardening.test.ts b/tests/fast-check-hardening.test.ts index 23f68c7..ee8521e 100644 --- a/tests/fast-check-hardening.test.ts +++ b/tests/fast-check-hardening.test.ts @@ -25,30 +25,80 @@ function applySequence(engine: Engine, directives: CanonicalDirective[]): void { for (const directive of directives) engine.apply_directive(directive); } +const policyAtomPairArb = fc.constantFrom( + ['alpha', 'ALPHA'], + ['docker', 'ＤＯＣＫＥＲ'], + ["don't", 'DON’T'], + ['café', 'cafe\u0301'], + ['straße', 'STRASSE'], + ['ǰ', 'J\u030C'], + ['οδός', 'ΟΔΌΣ'], + ['kelvin', 'KELVIN'] +).map(([base, equivalent]) => ({ base, equivalent })); + const equivalentPolicyPairArb = fc.record({ - base: fc.constantFrom("don't panic", 'the docker', 'ǰ rollout', 'straße'), - upper: fc.boolean(), - whitespace: fc.boolean(), - apostrophe: fc.boolean() -}).map(({ base, upper, whitespace, apostrophe }) => { - let variant = upper ? base.toUpperCase() : base; - if (whitespace) variant = variant.replaceAll(' ', ' '); + atoms: fc.array(policyAtomPairArb, { minLength: 1, maxLength: 5 }), + separators: fc.array(fc.constantFrom(' ', ' ', '\t', '\u00a0'), { minLength: 0, maxLength: 4 }), + apostrophe: fc.boolean(), + caseVariant: fc.boolean() +}).map(({ atoms, separators, apostrophe, caseVariant }) => { + const base = atoms.map(({ base }) => base).join(' '); + let variant = atoms.map(({ equivalent }) => equivalent).join(' '); + if (separators.length > 0) { + variant = atoms.map(({ equivalent }, index) => index === atoms.length - 1 + ? equivalent + : equivalent + separators[index % separators.length]).join(''); + } if (apostrophe) variant = variant.replaceAll("'", '’'); + if (caseVariant) variant = variant.toUpperCase(); return [base, variant] as const; }); -const invalidPayloadArb = fc.constantFrom( - '{', - JSON.stringify(null), - JSON.stringify([]), - JSON.stringify({ premise: null, policies: {}, version: 1 }), - JSON.stringify({ premise: 42, policies: {}, version: 2 }), - JSON.stringify({ premise: null, policies: [], version: 2 }), - JSON.stringify({ premise: null, policies: { docker: 'invalid' }, version: 2 }), - JSON.stringify({ premise: null, policies: { A: 'use', a: 'prohibit' }, version: 2 }), - JSON.stringify({ premise: null, policies: { ' ': 'use' }, version: 2 }), - JSON.stringify({ premise: ' ', policies: {}, version: 2 }), - JSON.stringify({ premise: null, policies: {} }) +type StatePayload = { premise: string | null; policies: Record; version: 2 }; + +const validStatePayloadArb = reachableSequenceArb.map((directives): StatePayload => { + const engine = new Engine(); + applySequence(engine, directives); + return JSON.parse(engine.export_json()) as StatePayload; +}); + +const whitespaceOnlyArb = fc.array(fc.constantFrom(' ', '\t', '\n', '\u00a0'), { minLength: 1, maxLength: 8 }) + .map((parts) => parts.join('')); +const invalidJsonArb = fc.string().map((value) => JSON.stringify(value).slice(0, -1)); +const invalidPremiseValueArb = fc.oneof(fc.integer(), fc.boolean(), fc.array(fc.integer()), fc.dictionary(fc.string(), fc.integer())); +const invalidPolicyValueArb = fc.oneof(fc.constant(null), fc.integer(), fc.boolean(), fc.array(fc.string()), fc.constant('invalid')); + +const invalidPayloadArb = fc.oneof( + invalidJsonArb, + fc.oneof(fc.constant(null), fc.boolean(), fc.integer(), fc.array(fc.string())).map((value) => JSON.stringify(value)), + validStatePayloadArb.chain((state) => fc.constantFrom('premise', 'policies', 'version').map((missing) => { + const invalid = { ...state } as Partial; + delete invalid[missing as keyof StatePayload]; + return JSON.stringify(invalid); + })), + validStatePayloadArb.chain((state) => fc.oneof( + fc.integer({ max: 1 }), + fc.integer({ min: 3 }), + fc.constant('2'), + fc.constant(null) + ).map((version) => JSON.stringify({ ...state, version }))), + validStatePayloadArb.chain((state) => invalidPremiseValueArb.map((premise) => JSON.stringify({ ...state, premise }))), + validStatePayloadArb.chain((state) => fc.oneof( + fc.constant(null), + fc.array(fc.string()), + fc.string() + ).map((policies) => JSON.stringify({ ...state, policies }))), + validStatePayloadArb.chain((state) => invalidPolicyValueArb.map((value) => JSON.stringify({ + ...state, + policies: { item: value } + }))), + whitespaceOnlyArb.chain((premise) => validStatePayloadArb.map((state) => JSON.stringify({ ...state, premise }))), + whitespaceOnlyArb.map((key) => JSON.stringify({ premise: null, policies: { [key]: 'use' }, version: 2 })), + equivalentPolicyPairArb.map(([base, equivalent]) => JSON.stringify({ + premise: null, + policies: { [base]: 'use', [equivalent]: 'prohibit' }, + version: 2 + })) ); describe('fast-check property hardening', () => { @@ -109,11 +159,8 @@ describe('fast-check property hardening', () => { const engine = new Engine(); applySequence(engine, directives); const before = engine.export_json(); - try { - engine.import_json(payload); - } catch { - expect(engine.export_json()).toBe(before); - } + expect(() => engine.import_json(payload)).toThrow(); + expect(engine.export_json()).toBe(before); }), { numRuns: 200 }); - }); +}); });