From c107b3de21dded0f9bd86b58732fa63b4ee43560 Mon Sep 17 00:00:00 2001 From: SandroMaglione Date: Sat, 7 Nov 2026 14:20:01 +0900 Subject: [PATCH] Add MachineTest.check and the effect-machine check command Compose static analysis, schema-derived exploration, law verification, and a determinism law into one call, and expose it to agents as a CLI that prints JSON and fails on any error finding. Copy the agent docs into the core package only while packing, so generated files no longer sit in the workspace where agents read them as instructions. Exclude the guides from dprint, which mangled a code fence. Co-Authored-By: Claude Opus 5.5 --- .changeset/machine-test-check.md | 17 + LLMS.md | 40 +- ai-docs/README.md | 2 +- ai-docs/src/10_modeling/index.md | 40 +- dprint.json | 1 + package.json | 2 +- packages/devtools/README.md | 11 + packages/devtools/src/DevToolsProtocol.ts | 35 ++ packages/devtools/src/ProjectInspector.ts | 16 + packages/devtools/src/bin.ts | 27 +- .../devtools/src/internal/checkCommand.ts | 38 ++ .../devtools/src/internal/evaluationWorker.ts | 112 ++++- .../devtools/src/internal/projectInspector.ts | 44 +- .../devtools/test/MachineRegistry.test.ts | 1 + .../devtools/test/ProjectInspector.test.ts | 46 ++ packages/effect-machine/README.md | 20 +- packages/effect-machine/package.json | 2 + .../src/internal/testing/machine/check.ts | 460 ++++++++++++++++++ .../effect-machine/src/testing/MachineTest.ts | 263 ++++++++++ .../effect-machine/test/testing/Check.test.ts | 358 ++++++++++++++ .../typetest/testing/Check.tst.ts | 31 ++ scripts/ai-docs/copy.mjs | 15 +- scripts/check-architecture.mjs | 2 +- scripts/devtools-pack-check.mjs | 9 + 24 files changed, 1556 insertions(+), 36 deletions(-) create mode 100644 .changeset/machine-test-check.md create mode 100644 packages/devtools/src/internal/checkCommand.ts create mode 100644 packages/effect-machine/src/internal/testing/machine/check.ts create mode 100644 packages/effect-machine/test/testing/Check.test.ts create mode 100644 packages/effect-machine/typetest/testing/Check.tst.ts diff --git a/.changeset/machine-test-check.md b/.changeset/machine-test-check.md new file mode 100644 index 00000000..b5b1a65c --- /dev/null +++ b/.changeset/machine-test-check.md @@ -0,0 +1,17 @@ +--- +"@typeonce/effect-machine": minor +"@typeonce/effect-machine-devtools": minor +--- + +Add `MachineTest.check`, `MachineTest.analyze`, and the `effect-machine check` command, so a machine can be verified with one call and no browser. + +`check` reports states with no path from the initial state, public events with no handler, nondeterministic planning, planner law violations, invariant failures, and declared branches that were never taken. It derives input and events from the machine's schemas, and each finding from a run includes the events that reproduce it. A report is `"exhaustive"` only when nothing limited exploration; otherwise `limits` says why it is `"bounded"`. + +```ts +const program = Effect.gen(function*() { + const report = yield* MachineTest.check(Checkout) + return report.ok // false when any finding is an error +}) +``` + +From a shell, `effect-machine check src/checkout.ts --json` prints one JSON object and exits with status 1 on any error. diff --git a/LLMS.md b/LLMS.md index dfc87972..f33a5aa4 100644 --- a/LLMS.md +++ b/LLMS.md @@ -388,8 +388,7 @@ address as well: ```ts // Register timers: { loadTimeout: "10 seconds" } in make. Loading: { - invoke: ; - ;[ + invoke: [ { src: "loadDocument", input: ({ state }) => state.documentId, @@ -503,6 +502,43 @@ owner input. Every branch the choice can reach, including through nested choices, must stay below the owner; otherwise the definition does not compile and `handle` throws while naming the choice, branch, and escaping target. +### Verify a machine after changing it + +Run this after writing or changing a machine, before writing a specific test: + +```ts +import { MachineTest } from "@typeonce/effect-machine/testing" +import { Effect } from "effect" + +const verifyCounter = Effect.gen(function*() { + const report = yield* MachineTest.check(CounterMachine) + if (!report.ok) yield* Effect.die(MachineTest.formatCheckReport(report)) +}) +``` + +From a shell, `effect-machine check src/counter.ts --json` from +`@typeonce/effect-machine-devtools` does the same for every machine a file exports. + +To fix a finding, keep the failing evidence first. Replay its events to reproduce +it, make the smallest behavior change, then run the check again. Do not silence a +finding by weakening a type or a test. + +| Law | Meaning | Fix | +| ---------------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------- | +| `topology.noPathFromInitial` | No declared transition can enter the state. | Add the missing transition, or remove the state. | +| `events.noHandler` | No state handles a public event. | Handle it, or remove it from the protocol. | +| `planning.deterministic` | The same snapshot and event planned two different results. | Move the clock, randomness, or external read into an event or work. | +| `planning.failed` | A planning callback threw or a value failed its schema. | Replay the events, then fix the callback or the value. | +| `planning.verification` | A planner law failed. This usually means an Effect Machine bug. | Report it with the replay. | +| `invariant.violated` | An invariant you supplied failed. | Replay the events, then fix the behavior or the invariant. | +| `transition.notCovered` | Exploration never took a declared branch. This is a warning. | Cover it with a scenario, or remove it if it cannot happen. | +| `schema.underivable` | Input or events could not be sampled from their schemas. | Pass `input` or `events` explicitly. | + +A report is `"exhaustive"` only when `limits` is empty. Otherwise a pass means +nothing was found in what was explored. Keep sampled checks, bounded exploration, +and proof distinct when reporting a result. Invocations do not run during planning, +so test them on a started machine with `MachineTest.probe`. + ### Test paths and invariants Test the statechart as a graph. Send domain events, inspect reached states, and diff --git a/ai-docs/README.md b/ai-docs/README.md index 668d7265..8273f35e 100644 --- a/ai-docs/README.md +++ b/ai-docs/README.md @@ -1,6 +1,6 @@ # AI docs -`LLMS.md` is generated from `ai-docs/src`. Each build copies it into the core package as `AGENTS.md` and `CLAUDE.md`, next to a copy of `ai-docs`, so an installed package carries documentation matching its version. +`LLMS.md` is generated from `ai-docs/src`. Packing the core package copies it in as `AGENTS.md` and `CLAUDE.md`, next to a copy of `ai-docs`, so an installed package carries documentation matching its version. The copies are removed after packing. ## Add content diff --git a/ai-docs/src/10_modeling/index.md b/ai-docs/src/10_modeling/index.md index 87f09846..5d3a4cc7 100644 --- a/ai-docs/src/10_modeling/index.md +++ b/ai-docs/src/10_modeling/index.md @@ -380,8 +380,7 @@ address as well: ```ts // Register timers: { loadTimeout: "10 seconds" } in make. Loading: { - invoke: ; - ;[ + invoke: [ { src: "loadDocument", input: ({ state }) => state.documentId, @@ -495,6 +494,43 @@ owner input. Every branch the choice can reach, including through nested choices, must stay below the owner; otherwise the definition does not compile and `handle` throws while naming the choice, branch, and escaping target. +### Verify a machine after changing it + +Run this after writing or changing a machine, before writing a specific test: + +```ts +import { MachineTest } from "@typeonce/effect-machine/testing" +import { Effect } from "effect" + +const verifyCounter = Effect.gen(function*() { + const report = yield* MachineTest.check(CounterMachine) + if (!report.ok) yield* Effect.die(MachineTest.formatCheckReport(report)) +}) +``` + +From a shell, `effect-machine check src/counter.ts --json` from +`@typeonce/effect-machine-devtools` does the same for every machine a file exports. + +To fix a finding, keep the failing evidence first. Replay its events to reproduce +it, make the smallest behavior change, then run the check again. Do not silence a +finding by weakening a type or a test. + +| Law | Meaning | Fix | +| ---------------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------- | +| `topology.noPathFromInitial` | No declared transition can enter the state. | Add the missing transition, or remove the state. | +| `events.noHandler` | No state handles a public event. | Handle it, or remove it from the protocol. | +| `planning.deterministic` | The same snapshot and event planned two different results. | Move the clock, randomness, or external read into an event or work. | +| `planning.failed` | A planning callback threw or a value failed its schema. | Replay the events, then fix the callback or the value. | +| `planning.verification` | A planner law failed. This usually means an Effect Machine bug. | Report it with the replay. | +| `invariant.violated` | An invariant you supplied failed. | Replay the events, then fix the behavior or the invariant. | +| `transition.notCovered` | Exploration never took a declared branch. This is a warning. | Cover it with a scenario, or remove it if it cannot happen. | +| `schema.underivable` | Input or events could not be sampled from their schemas. | Pass `input` or `events` explicitly. | + +A report is `"exhaustive"` only when `limits` is empty. Otherwise a pass means +nothing was found in what was explored. Keep sampled checks, bounded exploration, +and proof distinct when reporting a result. Invocations do not run during planning, +so test them on a started machine with `MachineTest.probe`. + ### Test paths and invariants Test the statechart as a graph. Send domain events, inspect reached states, and diff --git a/dprint.json b/dprint.json index a500b54d..19ea844f 100644 --- a/dprint.json +++ b/dprint.json @@ -17,6 +17,7 @@ "**/dist", "**/build", "**/docs", + "ai-docs/src/**/*.md", "**/coverage", "packages/**/CHANGELOG.md", "!scratchpad/**/*", diff --git a/package.json b/package.json index 8bf388b2..41955e16 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "type": "module", "packageManager": "pnpm@10.17.1", "scripts": { - "build": "pnpm --recursive --filter \"./packages/**\" run build && node scripts/ai-docs/copy.mjs", + "build": "pnpm --recursive --filter \"./packages/**\" run build", "test": "node --expose-gc ./node_modules/vitest/vitest.mjs run", "doctest": "vitest run --config vitest.docs.ts", "doctest:types": "node scripts/doc-examples/check.mjs", diff --git a/packages/devtools/README.md b/packages/devtools/README.md index 656290c3..5e790d46 100644 --- a/packages/devtools/README.md +++ b/packages/devtools/README.md @@ -69,6 +69,17 @@ The command writes `index.html`, `machines.json`, `manifest.json`, and content-h Static generation evaluates candidate project modules once in the same isolated worker used by the live visualizer. It fails when no machines are found or any candidate cannot produce a valid document. The command replaces only an empty directory or a directory previously generated by Effect Machine, so an unrelated non-empty destination is never removed. +## Check machines + +Analyze, explore, and verify the project's machines without a browser: + +```sh +pnpm exec effect-machine check +pnpm exec effect-machine check src/checkout.ts --json +``` + +The optional path is a file or glob relative to `--root`. The command runs `MachineTest.check` on every exported machine in the same isolated worker as the visualizer, and exits with status 1 when a report has an error, a module fails to load, or nothing matches. `--json` prints one object: `{ schemaVersion, ok, machines }`. The command cannot supply Effect services, so a machine whose planning callbacks require them reports a planning failure; call `MachineTest.check` from a test with those services instead. + ## Live results The browser reports one of these statuses for every candidate: diff --git a/packages/devtools/src/DevToolsProtocol.ts b/packages/devtools/src/DevToolsProtocol.ts index 4ddc105c..abb43098 100644 --- a/packages/devtools/src/DevToolsProtocol.ts +++ b/packages/devtools/src/DevToolsProtocol.ts @@ -127,6 +127,41 @@ export const MachineResult = Schema.Union([Ready, Partial, Failed]) */ export type MachineResult = Schema.Schema.Type +/** + * The outcome of checking one exported machine. `report` is the JSON form of a + * `MachineTest.CheckReport`. + * + * @category schemas + * @since 0.47.0 + */ +export const Checked = Schema.Struct({ + _tag: Schema.tag("Checked"), + key: Schema.String, + source: MachineDocument.Source, + report: Schema.Json +}) + +/** + * @category models + * @since 0.47.0 + */ +export type Checked = Schema.Schema.Type + +/** + * One result of checking a project: a report, or a module or machine that could + * not be loaded. + * + * @category schemas + * @since 0.47.0 + */ +export const CheckResult = Schema.Union([Checked, Failed]) + +/** + * @category models + * @since 0.47.0 + */ +export type CheckResult = Schema.Schema.Type + /** * A complete live-registry update sent to browser clients. * diff --git a/packages/devtools/src/ProjectInspector.ts b/packages/devtools/src/ProjectInspector.ts index 933b2eea..9343b30a 100644 --- a/packages/devtools/src/ProjectInspector.ts +++ b/packages/devtools/src/ProjectInspector.ts @@ -92,6 +92,9 @@ export class ProjectInspector extends Context.Service Effect.Effect, DiscoveryError | EvaluationError> + readonly check: ( + options: InspectOptions + ) => Effect.Effect, DiscoveryError | EvaluationError> }>()("@typeonce/effect-machine-devtools/ProjectInspector") {} /** @@ -130,6 +133,19 @@ export const inspect = (options: InspectOptions): Effect.Effect< ProjectInspector > => Effect.flatMap(ProjectInspector, (inspector) => inspector.inspect(options)) +/** + * Runs `MachineTest.check` on every machine exported by the project, in the + * isolated evaluator. + * + * @category combinators + * @since 0.47.0 + */ +export const check = (options: InspectOptions): Effect.Effect< + ReadonlyArray, + DiscoveryError | EvaluationError, + ProjectInspector +> => Effect.flatMap(ProjectInspector, (inspector) => inspector.check(options)) + /** * Node-backed inspector layer. Each inspection evaluates candidates inside a * fresh worker so a broken project module cannot corrupt the long-lived host. diff --git a/packages/devtools/src/bin.ts b/packages/devtools/src/bin.ts index 2d42e259..b027c1b5 100644 --- a/packages/devtools/src/bin.ts +++ b/packages/devtools/src/bin.ts @@ -1,13 +1,16 @@ #!/usr/bin/env node import * as NodeRuntime from "@effect/platform-node/NodeRuntime" import * as NodeServices from "@effect/platform-node/NodeServices" +import * as Argument from "effect/cli/Argument" import * as Command from "effect/cli/Command" import * as Flag from "effect/cli/Flag" import * as Console from "effect/Console" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" +import * as Option from "effect/Option" import PackageJson from "../package.json" with { type: "json" } import * as DevServer from "./DevServer.js" +import * as CheckCommand from "./internal/checkCommand.js" import * as StaticSite from "./internal/staticSite.js" import * as MachineRegistry from "./MachineRegistry.js" import * as ProjectInspector from "./ProjectInspector.js" @@ -78,7 +81,29 @@ const build = Command.make("build", { outputDirectory }).pipe( })) ) -const cli = dev.pipe(Command.withSubcommands([build])) +const target = Argument.String("path").pipe( + Argument.withDescription("Machine source file or glob relative to the project root"), + Argument.optional +) + +const json = Flag.Boolean("json").pipe( + Flag.withDescription("Print one JSON object instead of text"), + Flag.withDefault(false) +) + +const check = Command.make("check", { target, json }).pipe( + Command.withDescription("Analyze, explore, and verify the project's machines, and exit non-zero on any error"), + Command.withHandler(Effect.fnUntraced(function*({ json, target }) { + const { include, root } = yield* base + const results = yield* ProjectInspector.check({ root, include: Option.getOrElse(target, () => include) }).pipe( + Effect.provide(ProjectInspector.layer) + ) + yield* Console.log(json ? CheckCommand.renderJson(results) : CheckCommand.renderText(results)) + if (!CheckCommand.isOk(results)) yield* Effect.sync(() => (process.exitCode = 1)) + })) +) + +const cli = dev.pipe(Command.withSubcommands([build, check])) Command.run(cli, { version: PackageJson.version }).pipe( Effect.provide(NodeServices.layer), diff --git a/packages/devtools/src/internal/checkCommand.ts b/packages/devtools/src/internal/checkCommand.ts new file mode 100644 index 00000000..71a35c39 --- /dev/null +++ b/packages/devtools/src/internal/checkCommand.ts @@ -0,0 +1,38 @@ +import type * as DevToolsProtocol from "../DevToolsProtocol.js" + +const reportOf = (result: DevToolsProtocol.Checked): { + readonly ok: boolean + readonly text: string +} => result.report as never + +const failureText = (result: DevToolsProtocol.Failed): string => + [ + `FAIL ${result.key}`, + ...result.diagnostics.map((diagnostic) => ` error ${diagnostic.code}: ${diagnostic.message}`) + ].join("\n") + +/** A project with nothing to check fails too, so a wrong path is not reported as a pass. */ +export const isOk = (results: ReadonlyArray): boolean => + results.length > 0 && results.every((result) => result._tag === "Checked" && reportOf(result).ok) + +export const renderJson = (results: ReadonlyArray): string => + JSON.stringify( + { + schemaVersion: 1, + ok: isOk(results), + machines: results.map((result) => { + if (result._tag !== "Checked") { + return { key: result.key, source: result.source, ok: false, diagnostics: result.diagnostics } + } + const { text: _text, ...report } = result.report as Record + return { key: result.key, source: result.source, ...report } + }) + }, + null, + 2 + ) + +export const renderText = (results: ReadonlyArray): string => + results.length === 0 + ? "No Effect Machine definitions found" + : results.map((result) => result._tag === "Checked" ? reportOf(result).text : failureText(result)).join("\n") diff --git a/packages/devtools/src/internal/evaluationWorker.ts b/packages/devtools/src/internal/evaluationWorker.ts index 59a14b5c..6b15b645 100644 --- a/packages/devtools/src/internal/evaluationWorker.ts +++ b/packages/devtools/src/internal/evaluationWorker.ts @@ -1,5 +1,7 @@ import * as NodeWorkerRunner from "@effect/platform-node/NodeWorkerRunner" import { Machine } from "@typeonce/effect-machine" +import { MachineTest } from "@typeonce/effect-machine/testing" +import * as Cause from "effect/Cause" import * as Effect from "effect/Effect" import * as Schema from "effect/Schema" import * as WorkerRunner from "effect/workers/WorkerRunner" @@ -22,8 +24,19 @@ interface EvaluationResponse { readonly results: ReadonlyArray } -type WorkerRequest = EvaluationRequest -type WorkerResponse = EvaluationResponse +interface CheckRequest { + readonly _tag: "CheckMachines" + readonly root: string + readonly candidates: ReadonlyArray +} + +interface CheckResponse { + readonly _tag: "CheckedMachines" + readonly results: ReadonlyArray +} + +type WorkerRequest = EvaluationRequest | CheckRequest +type WorkerResponse = EvaluationResponse | CheckResponse const isMachine = (value: unknown): value is Machine.Machine.Any => typeof value === "object" && @@ -78,28 +91,37 @@ const failed = ( diagnostics: [diagnostic(candidate.file, code, message)] }) -const evaluateCandidate = ( +const loadMachines = ( server: ViteDevServer, - request: EvaluationRequest, + root: string, candidate: ProjectInspector.Candidate -): Effect.Effect> => +): Effect.Effect, unknown> => Effect.tryPromise({ - try: () => server.ssrLoadModule(pathToFileURL(resolve(request.root, candidate.file)).href), + try: () => server.ssrLoadModule(pathToFileURL(resolve(root, candidate.file)).href), catch: (cause) => cause }).pipe( Effect.flatMap((module) => Effect.try(() => Object.entries(module).filter((entry): entry is [string, Machine.Machine.Any] => isMachine(entry[1])) ) - ), + ) + ) + +const machineExportNotFound = (candidate: ProjectInspector.Candidate): DevToolsProtocol.Failed => + failed( + candidate, + "machine-export-not-found", + "The module contains a .handle(...) call but does not export the resulting machine" + ) + +const evaluateCandidate = ( + server: ViteDevServer, + request: EvaluationRequest, + candidate: ProjectInspector.Candidate +): Effect.Effect> => + loadMachines(server, request.root, candidate).pipe( Effect.flatMap((machines) => { - if (machines.length === 0) { - return Effect.succeed([failed( - candidate, - "machine-export-not-found", - "The module contains a .handle(...) call but does not export the resulting machine" - )]) - } + if (machines.length === 0) return Effect.succeed([machineExportNotFound(candidate)]) const seen = new Set() return Effect.forEach(machines, ([exportName, machine]) => { @@ -138,13 +160,65 @@ const evaluateCandidate = ( Effect.catch((cause) => Effect.succeed([failed(candidate, "module-load-failed", messageOf(cause))])) ) -const handle = (server: ViteDevServer, request: EvaluationRequest): Effect.Effect => - Effect.forEach(request.candidates, (candidate) => evaluateCandidate(server, request, candidate), { - concurrency: 1 - }).pipe( - Effect.map((results) => ({ _tag: "InspectedMachines" as const, results: results.flat() })) +// Reports cross a worker boundary and reach a JSON consumer, so decoded event +// values in replays must become plain JSON here. +const toJson = (value: unknown): Schema.Json => + JSON.parse( + JSON.stringify(value, (_key, item) => { + if (typeof item === "bigint") return item.toString() + if (typeof item === "function" || typeof item === "symbol") return String(item) + return item + }) + ) as Schema.Json + +const checkCandidate = ( + server: ViteDevServer, + request: CheckRequest, + candidate: ProjectInspector.Candidate +): Effect.Effect> => + loadMachines(server, request.root, candidate).pipe( + Effect.flatMap((machines) => { + if (machines.length === 0) return Effect.succeed([machineExportNotFound(candidate)]) + + const seen = new Set() + return Effect.forEach(machines, ([exportName, machine]) => { + if (seen.has(machine)) return Effect.succeed(undefined) + seen.add(machine) + // The CLI cannot supply services. A machine whose planning requires them + // reports that as a planning failure. + return (MachineTest.check as (machine: unknown) => Effect.Effect)(machine).pipe( + Effect.map((report): DevToolsProtocol.CheckResult => ({ + _tag: "Checked", + key: `${candidate.file}#${exportName}`, + source: sourceOf(candidate.file, exportName), + report: toJson({ ...report, text: MachineTest.formatCheckReport(report) }) + })), + Effect.catchCause((cause) => + Effect.succeed(failed(candidate, "machine-check-failed", Cause.pretty(cause), exportName)) + ) + ) + }).pipe( + Effect.map((results) => + results.filter((result): result is DevToolsProtocol.CheckResult => result !== undefined) + ) + ) + }), + Effect.catch((cause) => Effect.succeed([failed(candidate, "module-load-failed", messageOf(cause))])) ) +const handle = (server: ViteDevServer, request: WorkerRequest): Effect.Effect => + request._tag === "CheckMachines" + ? Effect.forEach(request.candidates, (candidate) => checkCandidate(server, request, candidate), { + concurrency: 1 + }).pipe( + Effect.map((results) => ({ _tag: "CheckedMachines" as const, results: results.flat() })) + ) + : Effect.forEach(request.candidates, (candidate) => evaluateCandidate(server, request, candidate), { + concurrency: 1 + }).pipe( + Effect.map((results) => ({ _tag: "InspectedMachines" as const, results: results.flat() })) + ) + export const run = (server: ViteDevServer): Promise => Effect.gen(function*() { const platform = yield* WorkerRunner.WorkerRunnerPlatform diff --git a/packages/devtools/src/internal/projectInspector.ts b/packages/devtools/src/internal/projectInspector.ts index 3e292782..b4e23109 100644 --- a/packages/devtools/src/internal/projectInspector.ts +++ b/packages/devtools/src/internal/projectInspector.ts @@ -39,8 +39,19 @@ interface EvaluationResponse { readonly results: ReadonlyArray } -type WorkerRequest = EvaluationRequest -type WorkerResponse = EvaluationResponse +interface CheckRequest { + readonly _tag: "CheckMachines" + readonly root: string + readonly candidates: ReadonlyArray +} + +interface CheckResponse { + readonly _tag: "CheckedMachines" + readonly results: ReadonlyArray +} + +type WorkerRequest = EvaluationRequest | CheckRequest +type WorkerResponse = EvaluationResponse | CheckResponse const scriptKind = (file: string): ts.ScriptKind => { if (file.endsWith(".tsx")) return ts.ScriptKind.TSX @@ -223,6 +234,32 @@ const evaluate = ( ) } +const check = ( + api: PublicApi, + candidates: ReadonlyArray, + options: ProjectInspector.InspectOptions +): Effect.Effect, ProjectInspector.EvaluationError> => { + if (candidates.length === 0) return Effect.succeed([]) + + return runWorker({ _tag: "CheckMachines", root: options.root, candidates }).pipe( + Effect.flatMap((message) => { + if (message._tag !== "CheckedMachines") { + return Effect.fail(new Error(`Unexpected worker response: ${String(message._tag)}`)) + } + return Effect.forEach( + message.results, + (result) => Schema.decodeUnknownEffect(DevToolsProtocol.CheckResult)(result) + ) + }), + Effect.mapError((cause) => + new api.EvaluationError({ + message: "The isolated machine evaluator failed", + cause + }) + ) + ) +} + const make = (api: PublicApi) => Effect.gen(function*() { const discover = yield* makeDiscovery(api) @@ -231,7 +268,8 @@ const make = (api: PublicApi) => return api.ProjectInspector.of({ discover, evaluate: (candidates, options) => evaluate(api, candidates, options), - inspect + inspect, + check: (options) => Effect.flatMap(discover(options), (candidates) => check(api, candidates, options)) }) }) diff --git a/packages/devtools/test/MachineRegistry.test.ts b/packages/devtools/test/MachineRegistry.test.ts index 34c49ade..16437177 100644 --- a/packages/devtools/test/MachineRegistry.test.ts +++ b/packages/devtools/test/MachineRegistry.test.ts @@ -16,6 +16,7 @@ describe("MachineRegistry", () => { const inspector = Inspector.ProjectInspector.of({ discover: () => Effect.succeed([]), evaluate: () => Effect.succeed([]), + check: () => Effect.succeed([]), inspect: (options) => Effect.gen(function*() { revisions.push(options.revision) diff --git a/packages/devtools/test/ProjectInspector.test.ts b/packages/devtools/test/ProjectInspector.test.ts index c6d8fba8..fcbd4bac 100644 --- a/packages/devtools/test/ProjectInspector.test.ts +++ b/packages/devtools/test/ProjectInspector.test.ts @@ -3,6 +3,7 @@ import * as Effect from "effect/Effect" import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" +import * as CheckCommand from "../src/internal/checkCommand.js" import { parseCandidate } from "../src/internal/projectInspector.js" import * as ProjectInspector from "../src/ProjectInspector.js" @@ -44,6 +45,51 @@ describe("ProjectInspector", () => { } }).pipe(Effect.provide(ProjectInspector.layer))) + it.effect("checks exported machines in the isolated worker and reports problems as data", () => + Effect.acquireUseRelease( + Effect.promise(() => mkdtemp(join(tmpdir(), "effect-machine-check-"))), + (root) => + Effect.gen(function*() { + yield* Effect.promise(() => mkdir(join(root, "src"), { recursive: true })) + yield* Effect.promise(() => + writeFile( + join(root, "src", "orphan.mjs"), + `import { Machine } from "@typeonce/effect-machine" +export const orphan = Machine.make({ + id: "orphan", + root: Machine.state({ states: { Idle: {}, Lost: {} } }), + events: Machine.events({ Go: {} }) +}).handle({ initial: { target: "Idle" }, states: { Idle: { on: { Go: { target: "Idle" } } } } }) +` + ) + ) + const results = yield* ProjectInspector.check({ root, include: "src/*.mjs" }) + const failed = results[0] + // A temporary project cannot resolve the package, so this asserts the failure path. + assert.strictEqual(failed?._tag, "Failed") + + const checked = yield* ProjectInspector.check({ + root: process.cwd(), + include: "packages/devtools/src/internal/browser/example-machine.ts" + }) + assert.strictEqual(checked.length, 1) + const result = checked[0] + assert.strictEqual(result?._tag, "Checked") + if (result?._tag === "Checked") { + const report = result.report as { readonly machine: string; readonly text: string } + assert.strictEqual(report.machine, "inspection-example") + assert.include(report.text, "inspection-example") + assert.isTrue(CheckCommand.renderJson(checked).includes("\"schemaVersion\": 1")) + } + }).pipe(Effect.provide(ProjectInspector.layer)), + (root) => Effect.promise(() => rm(root, { recursive: true, force: true })) + )) + + it("treats a project with nothing to check as a failure", () => { + assert.isFalse(CheckCommand.isOk([])) + assert.strictEqual(CheckCommand.renderText([]), "No Effect Machine definitions found") + }) + it.effect("returns a failed result when one candidate cannot be loaded", () => Effect.gen(function*() { const inspector = yield* ProjectInspector.ProjectInspector diff --git a/packages/effect-machine/README.md b/packages/effect-machine/README.md index 851f43df..aab48300 100644 --- a/packages/effect-machine/README.md +++ b/packages/effect-machine/README.md @@ -607,7 +607,25 @@ encoded snapshot. ## Testing -The testing entrypoint provides complementary layers: +Start with `MachineTest.check`. It reads the declared topology, explores the +machine with events derived from its schemas, verifies every explored transition, +plans each one twice to find nondeterminism, and returns the problems as data: + +```ts +import { MachineTest } from "@typeonce/effect-machine/testing" + +const report = yield* MachineTest.check(Counter) +report.ok // false when any finding is an error +report.exploration // "exhaustive", or "bounded" with the reasons in report.limits +report.findings // each one names its law, and replay events where they apply +``` + +A `"bounded"` pass means nothing was found in what was explored; it is not a +proof. `effect-machine check --json` from `@typeonce/effect-machine-devtools` runs +the same check across a project. `MachineTest.analyze` runs only the static part, +without calling any machine callback. + +The testing entrypoint also provides complementary layers for specific tests: - `MachineTest.run` and `verify` inspect pure planner traces; - `coverage` reports exact transition-definition and conditional-branch hits; diff --git a/packages/effect-machine/package.json b/packages/effect-machine/package.json index 4f14ea2c..960820a3 100644 --- a/packages/effect-machine/package.json +++ b/packages/effect-machine/package.json @@ -69,6 +69,8 @@ "scripts": { "build": "tsc -b tsconfig.build.json", "check": "tsc -b tsconfig.json", + "prepack": "node ../../scripts/ai-docs/copy.mjs", + "postpack": "node ../../scripts/ai-docs/copy.mjs --clean", "test:types": "tstyche" }, "peerDependencies": { diff --git a/packages/effect-machine/src/internal/testing/machine/check.ts b/packages/effect-machine/src/internal/testing/machine/check.ts new file mode 100644 index 00000000..925c483e --- /dev/null +++ b/packages/effect-machine/src/internal/testing/machine/check.ts @@ -0,0 +1,460 @@ +/** + * Static model analysis and the one-call machine check composed from it, + * bounded exploration, coverage, and the planner verification laws. + * + * @since 0.47.0 + */ + +import * as Arbitrary from "effect/Arbitrary" +import * as Cause from "effect/Cause" +import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" +import * as Schema from "effect/Schema" +import * as Machine from "../../../Machine.js" +import type { + AnalysisFinding, + CheckExplorationLimit, + CheckFinding, + CheckOptions, + CheckReplay, + CheckReport, + EventPlan, + ExplorationEdge, + ExplorationNode, + InitialPlan, + RunFailure, + RunServices, + Trace, + TraceStep +} from "../../../testing/MachineTest.js" +import * as Protocol from "../../machine/protocol.js" +import type { EnsureExecutable } from "../../machine/readiness.js" +import { explore } from "./exploration.js" +import { formatValue } from "./format.js" +import { InvariantError } from "./invariant.js" +import { VerificationError, verify } from "./verification.js" + +type AnyMachine = Machine.Machine.Any + +type ReadyMachine = + & M + & EnsureExecutable< + Machine.Machine.States, + Machine.Machine.UnhandledStates, + Machine.Machine.OutputStates + > + +const defaultLimits = { maxDepth: 16, maxStates: 256, maxTransitions: 2_048 } + +const isDescendant = (path: string, ancestor: string): boolean => + ancestor === "" ? path !== "" : path.startsWith(`${ancestor}.`) + +const reachableStatePaths = (machine: AnyMachine): ReadonlySet => { + const nodes = Machine.stateNodes(machine) + const byPath = new Map(nodes.map((node) => [node.path, node])) + const definitions = Machine.transitionDefinitions(machine) + const reached = new Set() + const entered = new Set() + const queue: Array = [] + + const mark = (path: string): void => { + let current = byPath.get(path) + while (current !== undefined) { + if (!reached.has(current.path)) { + reached.add(current.path) + queue.push(current.path) + } + current = current.parent === undefined ? undefined : byPath.get(current.parent) + } + } + // A resolver can construct any explicit descendant of a branch target, and a + // history default any descendant of its owner. Treating those subtrees as + // entered keeps the analysis conservative: it never reports a live state. + const enterSubtree = (path: string): void => { + for (const node of nodes) { + if (node.path === path || isDescendant(node.path, path)) enter(node.path) + } + } + const enter = (path: string): void => { + const node = byPath.get(path) + if (node === undefined) return + mark(path) + if (entered.has(path)) return + entered.add(path) + if (node.type === "parallel") { + for (const child of nodes) { + if (child.parent === path && child.type !== "history" && child.type !== "choice") enter(child.path) + } + } else if (node.type === "compound" && node.initial !== undefined) { + enter(node.initial) + } else if (node.type === "history" && node.parent !== undefined) { + enterSubtree(node.parent) + } + } + + enter(Machine.initialDefinition(machine).target) + for (let index = 0; index < queue.length; index++) { + const source = queue[index]! + for (const definition of definitions) { + if (definition.source !== source) continue + for (const branch of definition.branches) { + if (branch.target === undefined) continue + if (branch.type === "branch") enterSubtree(branch.target) + else enter(branch.target) + } + } + } + return reached +} + +export const analyze = (machine: AnyMachine): ReadonlyArray => { + const findings: Array = [] + + const handled = new Set() + for (const { trigger } of Machine.transitionDefinitions(machine)) { + if (trigger.type === "event") handled.add(trigger.event) + } + for (const event of Reflect.ownKeys(machine.events)) { + if (handled.has(event)) continue + findings.push({ + law: "events.noHandler", + severity: "error", + event: String(event), + message: `Public event "${String(event)}" has no handler in any state. Handle it, or remove it from the protocol.` + }) + } + + const nodes = Machine.stateNodes(machine) + const reached = reachableStatePaths(machine) + const unreachable = nodes.filter((node) => !reached.has(node.path)) + for (const node of unreachable) { + // A descendant of an unreachable state adds no new information. + if (unreachable.some((other) => isDescendant(node.path, other.path))) continue + findings.push({ + law: "topology.noPathFromInitial", + severity: "error", + path: node.path, + descendants: unreachable.filter((other) => isDescendant(other.path, node.path)).length, + message: `State "${node.path}" has no declared path from the initial state. ` + + `Add a transition that targets it, or remove it.` + }) + } + return findings +} + +interface DerivedEvents { + readonly events: ReadonlyArray + readonly enumerated: boolean +} + +const deriveEvents = (machine: AnyMachine, seed: number, perTag: number) => + Effect.gen(function*() { + const selected = new Map>() + let enumerated = true + for (const [index, schema] of Protocol.inputEventSchemas(machine).entries()) { + const samples = yield* Arbitrary.sampleEffect(Arbitrary.schema(schema), { + count: Math.max(64, perTag * 16), + seed: seed + index + }) + for (const sample of samples) { + const event = sample + if (Reflect.ownKeys(event).some((key) => key !== "_tag")) enumerated = false + const values = selected.get(event._tag) ?? new Map() + const key = formatValue(event) + if (!values.has(key) && values.size < perTag) values.set(key, event) + selected.set(event._tag, values) + } + } + // A union may not sample every member. Report that as incomplete rather than guessing. + if (Reflect.ownKeys(machine.events).some((tag) => !selected.has(tag))) enumerated = false + return { + events: [...selected.values()].flatMap((values) => [...values.values()]), + enumerated + } satisfies DerivedEvents + }) + +const causeMessage = (cause: unknown): string => { + if (cause instanceof Error) return cause.message + return formatValue(cause) +} + +const replayOf = (input: unknown, hasInput: boolean, events: ReadonlyArray): CheckReplay => + hasInput ? { input, events } : { events } + +const planEvidence = (plan: { + readonly commands: ReadonlyArray + readonly emittedEvents: ReadonlyArray + readonly done: boolean + readonly output: unknown +}, state: unknown): string => + formatValue({ + state, + commands: plan.commands, + emitted: plan.emittedEvents, + done: plan.done, + output: plan.output + }) + +const statePaths = (step: { readonly afterConfiguration: ReadonlyArray }): string => + step.afterConfiguration.filter((path) => path !== "").join(", ") + +export const check = ( + machine: ReadyMachine, + options: CheckOptions = {} +): Effect.Effect> => + Effect.gen(function*() { + const findings: Array = [...analyze(machine)] + const limits = new Set() + const seed = options.seed ?? 0 + const hasInput = machine.input !== undefined && machine.input !== Schema.Void + + let input: unknown = (options as { readonly input?: unknown }).input + if (hasInput && input === undefined) { + limits.add("sampledInput") + const sampled = yield* Effect.exit(Arbitrary.sampleEffect(Arbitrary.schema(machine.input), { count: 1, seed })) + if (Exit.isFailure(sampled)) { + findings.push({ + law: "schema.underivable", + severity: "error", + message: `Could not derive machine input from its schema: ${ + Cause.pretty(sampled.cause) + }. Pass input explicitly.` + }) + return finish(machine, findings, limits, undefined) + } + input = sampled.value[0] + } + + let events: ReadonlyArray + if (options.events !== undefined) { + limits.add("customEvents") + events = options.events + } else { + const derived = yield* Effect.exit(deriveEvents(machine, seed, options.eventSamples ?? 3)) + if (Exit.isFailure(derived)) { + findings.push({ + law: "schema.underivable", + severity: "error", + message: `Could not derive events from their schemas: ${Cause.pretty(derived.cause)}. Pass events explicitly.` + }) + return finish(machine, findings, limits, undefined) + } + if (!derived.value.enumerated) limits.add("sampledEvents") + events = derived.value.events + } + if (options.stateKey !== undefined) limits.add("customStateKey") + if (Machine.activityDefinitions(machine).length > 0) limits.add("invocations") + + const replay = (replayEvents: ReadonlyArray) => replayOf(input, hasInput, replayEvents) + const stateKey = options.stateKey ?? (({ snapshot }: { readonly snapshot: unknown }) => formatValue(snapshot)) + const explored = yield* Effect.exit( + (explore as any)(machine, { + ...(hasInput ? { input } : {}), + limits: { ...defaultLimits, ...options.limits }, + invariants: options.invariants, + stateKey, + // A completed machine accepts no further events. + events: ({ trace }: { readonly trace: Trace }) => + (trace.steps.at(-1)?.plan ?? trace.initial.plan).done ? [] : events + }) as Effect.Effect | InvariantError, RunServices> + ) + + if (Exit.isFailure(explored)) { + const failure = Cause.findErrorOption(explored.cause) + if (failure._tag === "Some" && failure.value instanceof InvariantError) { + for (const violation of failure.value.violations) { + findings.push({ + law: "invariant.violated", + severity: "error", + invariant: violation.invariant, + replay: replay(failure.value.trace.scenario.events), + message: `Invariant "${violation.invariant}" failed: ${violation.message}` + }) + } + } else if (failure._tag === "Some") { + const runFailure = failure.value as RunFailure + findings.push({ + law: "planning.failed", + severity: "error", + replay: replay(runFailure.scenario.events), + message: `Planning failed: ${causeMessage(runFailure.cause)}` + }) + } else { + if (Cause.hasInterruptsOnly(explored.cause)) return yield* Effect.interrupt + findings.push({ + law: "planning.failed", + severity: "error", + replay: replay([]), + message: `A planning callback threw: ${Cause.pretty(explored.cause)}` + }) + } + return finish(machine, findings, limits, undefined) + } + + const exploration = explored.value + if (exploration.completeness._tag === "Truncated") { + for (const reason of exploration.completeness.reasons) limits.add(reason) + } + + // Planning must be a function of the snapshot and the event. Plan each + // transition again and compare, because a syntax rule cannot see through a + // helper that reads the clock or randomness. + const initialTrace = (exploration.nodes[0] as ExplorationNode).trace + const plannedAgain = yield* Effect.exit( + (hasInput + ? (Machine.planInitial as any)(machine, input) + : (Machine.planInitial as any)(machine)) as Effect.Effect< + InitialPlan, + unknown, + RunServices + > + ) + if ( + Exit.isSuccess(plannedAgain) && + planEvidence(plannedAgain.value, plannedAgain.value.state) !== + planEvidence(initialTrace.initial.plan, initialTrace.initial.plan.state) + ) { + findings.push({ + law: "planning.deterministic", + severity: "error", + source: "initial", + replay: replay([]), + message: "Startup planned differently from the same input. " + + "Do not read the clock, randomness, or external state in planning callbacks." + }) + } + + const seen = new Set() + const graph = exploration.graph + for ( + const edge of graph.edges.values() as Iterable<{ readonly source: number; readonly data: ExplorationEdge }> + ) { + const source = graph.nodes.get(edge.source) as ExplorationNode + const step: TraceStep = edge.data.step + const location = `${statePaths({ afterConfiguration: step.beforeConfiguration })}:${ + String((step.event as { readonly _tag: PropertyKey })._tag) + }` + const events = [...source.trace.scenario.events, step.event] + + const verified = yield* Effect.exit( + verify(machine, { + ...source.trace, + scenario: { ...source.trace.scenario, events }, + steps: [...source.trace.steps, step], + final: step.after, + finalConfiguration: step.afterConfiguration + }) + ) + if (Exit.isFailure(verified)) { + const failure = Cause.findErrorOption(verified.cause) + if (failure._tag === "Some" && failure.value instanceof VerificationError) { + for (const violation of failure.value.violations) { + findings.push({ + law: "planning.verification", + severity: "error", + verificationLaw: violation.law, + replay: replay(events), + message: violation.message + }) + } + } + } + + const again = yield* Effect.exit( + ((Machine.plan as any)(machine, step.before, step.event)) as Effect.Effect< + EventPlan, + unknown, + RunServices + > + ) + if ( + Exit.isSuccess(again) && + planEvidence(again.value, again.value.next) !== planEvidence(step.plan, step.after) && + !seen.has(location) + ) { + seen.add(location) + findings.push({ + law: "planning.deterministic", + severity: "error", + source: location, + replay: replay(events), + message: `Planning "${location}" produced a different result from the same snapshot and event. ` + + `Do not read the clock, randomness, or external state in planning callbacks.` + }) + } + } + + // Exploration does not run invocations, so it cannot take their outcomes or + // enter a state only they lead to. `limits` already reports that gap, and a + // warning for each such transition would only tell the reader to remove live code. + const visited = new Set() + for (const node of exploration.nodes as ReadonlyArray>) { + for (const path of node.configuration as ReadonlyArray) visited.add(path) + } + const invocationsHideStates = limits.has("invocations") + for (const miss of exploration.transitionCoverage.branches.misses) { + if (miss.trigger.type === "invoke") continue + if (invocationsHideStates && !visited.has(miss.source)) continue + findings.push({ + law: "transition.notCovered", + severity: "warning", + source: miss.source, + trigger: miss.trigger, + target: miss.branch.target, + ...(miss.branchKey === undefined ? {} : { branchKey: miss.branchKey }), + message: + `Exploration never took the ${ + miss.trigger.type === "event" ? `"${String(miss.trigger.event)}"` : miss.trigger.type + } ` + + `transition from "${miss.source}"${miss.branch.target === undefined ? "" : ` to "${miss.branch.target}"`}. ` + + `Cover it with a scenario, or remove it if it cannot happen.` + }) + } + + return finish(machine, findings, limits, exploration) + }) + +const finish = ( + machine: AnyMachine, + findings: ReadonlyArray, + limits: ReadonlySet, + exploration: { + readonly stats: { readonly states: number; readonly plannedTransitions: number } + readonly transitionCoverage: { + readonly definitions: { readonly total: number; readonly hit: number } + readonly branches: { readonly total: number; readonly hit: number } + } + } | undefined +): CheckReport => ({ + machine: machine.id ?? "Machine", + ok: findings.every((finding) => finding.severity !== "error"), + exploration: exploration !== undefined && limits.size === 0 ? "exhaustive" : "bounded", + limits: [...limits], + stats: exploration === undefined + ? { states: 0, plannedTransitions: 0, transitions: { total: 0, hit: 0 }, branches: { total: 0, hit: 0 } } + : { + states: exploration.stats.states, + plannedTransitions: exploration.stats.plannedTransitions, + transitions: { + total: exploration.transitionCoverage.definitions.total, + hit: exploration.transitionCoverage.definitions.hit + }, + branches: { + total: exploration.transitionCoverage.branches.total, + hit: exploration.transitionCoverage.branches.hit + } + }, + findings +}) + +const formatFinding = (finding: CheckFinding): string => + ` ${finding.severity === "error" ? "error" : "warn "} ${finding.law}: ${finding.message}` + + ("replay" in finding ? `\n replay: ${formatValue(finding.replay)}` : "") + +export const formatReport = (report: CheckReport): string => + [ + `${report.ok ? "PASS" : "FAIL"} ${report.machine}: ${report.stats.states} states, ` + + `${report.stats.branches.hit}/${report.stats.branches.total} branches, ${report.exploration}` + + (report.limits.length === 0 ? "" : ` (${report.limits.join(", ")})`), + ...report.findings.map(formatFinding) + ].join("\n") diff --git a/packages/effect-machine/src/testing/MachineTest.ts b/packages/effect-machine/src/testing/MachineTest.ts index ad4f6815..58084b1d 100644 --- a/packages/effect-machine/src/testing/MachineTest.ts +++ b/packages/effect-machine/src/testing/MachineTest.ts @@ -10,6 +10,7 @@ import type * as Graph from "effect/Graph" import type * as Schema from "effect/Schema" import type { EnsureExecutable } from "../internal/machine/readiness.js" import type { SchemaArbitraryReport } from "../internal/testing/machine/arbitrary.js" +import * as CheckImpl from "../internal/testing/machine/check.js" import * as ExplorationImpl from "../internal/testing/machine/exploration.js" import type { FiniteModel } from "../internal/testing/machine/finiteModel.js" import * as Format from "../internal/testing/machine/format.js" @@ -2169,6 +2170,268 @@ export const verify: ( options?: VerifyOptions ) => Effect.Effect = internal.verify +/** + * A problem found by reading a machine's declarations, without running any of + * its callbacks. + * + * @category models + * @since 0.47.0 + */ +export type AnalysisFinding = + | { + readonly law: "topology.noPathFromInitial" + readonly severity: "error" + readonly path: string + /** Descendants of `path` that are unreachable for the same reason. */ + readonly descendants: number + readonly message: string + } + | { + readonly law: "events.noHandler" + readonly severity: "error" + readonly event: string + readonly message: string + } + +/** + * Reads a machine's declared topology and reports states with no path from the + * initial state and public events with no handler. + * + * **When to use** + * + * Use when a question has to be answered without running the machine. Use + * {@link check} to also explore the machine and verify its planner laws. + * + * **Details** + * + * The analysis is conservative. A branch or history target can construct any of + * its descendants, so that whole subtree counts as reachable, and a state is + * only reported when no declaration could enter it. + * + * **Example** (Finding a state with no incoming transition) + * + * ```ts import.meta.vitest + * import { Machine } from "@typeonce/effect-machine" + * import { MachineTest } from "@typeonce/effect-machine/testing" + * + * const machine = Machine.make({ + * root: Machine.state({ states: { Idle: {}, Orphan: {} } }), + * events: Machine.events({ Start: {} }) + * }).handle({ + * initial: { target: "Idle" }, + * states: { Idle: { on: { Start: { target: "Idle" } } } } + * }) + * + * MachineTest.analyze(machine).map((finding) => finding.law) // => ["topology.noPathFromInitial"] + * ``` + * + * @category verification + * @since 0.47.0 + */ +export const analyze: (machine: M) => ReadonlyArray = CheckImpl.analyze + +/** + * The events and optional input that reproduce a finding from startup. + * + * @category models + * @since 0.47.0 + */ +export interface CheckReplay { + readonly input?: unknown + readonly events: ReadonlyArray +} + +/** + * Why {@link check} could not prove it explored every behavior. An empty list + * is what makes a report `"exhaustive"`. + * + * @category models + * @since 0.47.0 + */ +export type CheckExplorationLimit = + /** The depth, state, or transition limit stopped exploration early. */ + | "depth" + | "states" + | "transitions" + /** Event payloads were sampled from their schemas. */ + | "sampledEvents" + /** Machine input was sampled from its schema. */ + | "sampledInput" + /** Exploration used caller-supplied events. */ + | "customEvents" + /** A caller-supplied state key may merge distinct snapshots. */ + | "customStateKey" + /** Invocations do not run during planning, so their outcomes were not explored. */ + | "invocations" + +/** + * A problem found by {@link check}. An `"error"` finding fails the report and a + * `"warning"` does not. + * + * @category models + * @since 0.47.0 + */ +export type CheckFinding = + | AnalysisFinding + | { + readonly law: "schema.underivable" + readonly severity: "error" + readonly message: string + } + | { + readonly law: "planning.failed" + readonly severity: "error" + readonly replay: CheckReplay + readonly message: string + } + | { + readonly law: "planning.deterministic" + readonly severity: "error" + /** `"initial"`, or the active state paths and event tag that were planned. */ + readonly source: string + readonly replay: CheckReplay + readonly message: string + } + | { + readonly law: "planning.verification" + readonly severity: "error" + readonly verificationLaw: VerificationLaw + readonly replay: CheckReplay + readonly message: string + } + | { + readonly law: "invariant.violated" + readonly severity: "error" + readonly invariant: string + readonly replay: CheckReplay + readonly message: string + } + | { + readonly law: "transition.notCovered" + readonly severity: "warning" + readonly source: string + readonly trigger: Machine.Machine.TransitionTrigger + readonly target: string | undefined + readonly branchKey?: string + readonly message: string + } + +/** + * The result of {@link check}. + * + * @category models + * @since 0.47.0 + */ +export interface CheckReport { + readonly machine: string + /** Whether the report has no `"error"` finding. */ + readonly ok: boolean + /** + * `"exhaustive"` only when `limits` is empty. A `"bounded"` pass means nothing + * was found within what was explored, not that the machine is correct. + */ + readonly exploration: "exhaustive" | "bounded" + readonly limits: ReadonlyArray + readonly stats: { + readonly states: number + readonly plannedTransitions: number + readonly transitions: { readonly total: number; readonly hit: number } + readonly branches: { readonly total: number; readonly hit: number } + } + readonly findings: ReadonlyArray +} + +/** + * Options for {@link check}. Every option has a default derived from the machine. + * + * @category models + * @since 0.47.0 + */ +export interface CheckOptions { + /** Machine input. Defaults to a value sampled from the input schema. */ + readonly input?: InputValue + /** Events to try in every state. Defaults to samples derived from the event schemas. */ + readonly events?: ReadonlyArray>> + /** Distinct values to sample for each event tag. Defaults to 3. */ + readonly eventSamples?: number + /** Seed for sampled input and events. Defaults to 0, so reports are reproducible. */ + readonly seed?: number + /** Identifies logical states. Defaults to the complete snapshot value. */ + readonly stateKey?: (context: ExplorationStateContext) => ExplorationKey + /** Defaults to `maxDepth: 16`, `maxStates: 256`, and `maxTransitions: 2048`. */ + readonly limits?: ExplorationLimits + /** Application laws checked against every explored state and step. */ + readonly invariants?: ReadonlyArray> +} + +/** + * Checks a machine with one call and reports every problem as data. + * + * **When to use** + * + * Use as the first verification step after writing or changing a machine. Reach + * for {@link explore}, {@link verify}, and {@link coverage} directly when a test + * needs their complete evidence, such as a specific scenario. + * + * **Details** + * + * Runs {@link analyze}, then {@link explore} with events derived from the event + * schemas, then {@link verify} on every explored transition. It also plans each + * transition a second time, so nondeterminism is found even when hidden behind a + * helper. Declared branches that were never taken are reported as warnings. + * + * Findings are returned rather than failed, so the Effect only fails on + * interruption. Each finding that came from running the machine includes the + * events that reproduce it. + * + * **Gotchas** + * + * A passing `"bounded"` report means nothing was found in what was explored. It + * is not a proof. Invocations do not run during planning, so test them with + * {@link probe} on a started machine. + * + * **Example** (Finding nondeterministic planning) + * + * ```ts import.meta.vitest + * import { Machine } from "@typeonce/effect-machine" + * import { MachineTest } from "@typeonce/effect-machine/testing" + * import { Effect, Schema } from "effect" + * + * const nextAmount = () => Math.random() + * const machine = Machine.make({ + * root: Machine.state({ + * states: { Editing: {}, Charging: { fields: { amount: Schema.Number } } } + * }), + * events: Machine.events({ Pay: {} }) + * }).handle({ + * initial: { target: "Editing" }, + * states: { + * Editing: { on: { Pay: { target: "Charging", data: () => ({ amount: nextAmount() }) } } } + * } + * }) + * + * const report = await Effect.runPromise(MachineTest.check(machine)) + * + * report.ok // => false + * report.findings.map((finding) => finding.law) // => ["planning.deterministic"] + * ``` + * + * @category verification + * @since 0.47.0 + */ +export const check: ( + machine: RootReadyMachine, + options?: CheckOptions +) => Effect.Effect> = CheckImpl.check + +/** + * Formats a report as line-oriented text for a terminal or an agent transcript. + * + * @category verification + * @since 0.47.0 + */ +export const formatCheckReport: (report: CheckReport) => string = CheckImpl.formatReport + /** * Formats a trace as deterministic, line-oriented counterexample evidence. * diff --git a/packages/effect-machine/test/testing/Check.test.ts b/packages/effect-machine/test/testing/Check.test.ts new file mode 100644 index 00000000..6ea087be --- /dev/null +++ b/packages/effect-machine/test/testing/Check.test.ts @@ -0,0 +1,358 @@ +import { assert, describe, it } from "@effect/vitest" +import { Effect, Schema } from "effect" +import { Machine } from "../../src/index.js" +import { MachineTest } from "../../src/testing/index.js" + +const laws = (report: { readonly findings: ReadonlyArray<{ readonly law: string }> }) => + report.findings.map((finding) => finding.law) + +const Toggle = Machine.make({ + id: "Toggle", + root: Machine.state({ states: { Off: {}, On: {} } }), + events: Machine.events({ Flip: {} }) +}).handle({ + initial: { target: "Off" }, + states: { + Off: { on: { Flip: { target: "On" } } }, + On: { on: { Flip: { target: "Off" } } } + } +}) + +describe("MachineTest.analyze", () => { + it("reports a state with no path and a public event with no handler", () => { + const machine = Machine.make({ + root: Machine.state({ states: { Idle: {}, Orphan: { states: { Inner: {} } } } }), + events: Machine.events({ Start: {}, Unused: {} }) + }).handle({ + initial: { target: "Idle" }, + states: { + Idle: { on: { Start: { target: "Idle" } } }, + Orphan: { initial: { target: "Orphan.Inner" } } + } + }) + + assert.deepStrictEqual( + MachineTest.analyze(machine).map((finding) => ({ + law: finding.law, + subject: finding.law === "events.noHandler" ? finding.event : finding.path, + descendants: finding.law === "topology.noPathFromInitial" ? finding.descendants : undefined + })), + [ + { law: "events.noHandler", subject: "Unused", descendants: undefined }, + { law: "topology.noPathFromInitial", subject: "Orphan", descendants: 1 } + ] + ) + }) + + it("follows initial children, parallel regions, choices, and invocation outcomes", () => { + const machine = Machine.make({ + root: Machine.state({ + states: { + Start: {}, + Pick: { type: "choice" }, + Left: {}, + Regions: { + type: "parallel", + states: { + a: { states: { A1: {} } }, + b: { states: { B1: {} } } + } + }, + Waiting: {}, + Done: { type: "final" } + } + }), + events: Machine.events({ Go: {}, Next: {} }), + effects: { work: Effect.void }, + branches: { pick: { left: { target: "Left" }, regions: { target: "Regions" } } } + }).handle({ + initial: { target: "Start" }, + states: { + Start: { on: { Go: { target: "Pick" } } }, + Pick: { choice: { branches: "pick", resolve: ({ select }) => select.left() } }, + Left: { on: { Next: { target: "Waiting" } } }, + Regions: { + states: { + a: { initial: { target: "Regions.a.A1" } }, + b: { initial: { target: "Regions.b.B1" } } + } + }, + Waiting: { invoke: { src: "work", onDone: { target: "Done" } } } + } + }) + + assert.deepStrictEqual(MachineTest.analyze(machine), []) + }) + + it("treats a branch target's descendants as reachable, because a resolver can construct them", () => { + const machine = Machine.make({ + root: Machine.state({ states: { Idle: {}, Open: { states: { First: {}, Second: {} } } } }), + events: Machine.events({ Go: {} }), + branches: { go: { open: { target: "Open" } } } + }).handle({ + initial: { target: "Idle" }, + states: { + Idle: { + on: { Go: { branches: "go", resolve: ({ select }) => select.open({ states: { Second: {} } }) } } + }, + Open: { initial: { target: "Open.First" } } + } + }) + + assert.deepStrictEqual(MachineTest.analyze(machine), []) + }) + + it("does not run any callback", () => { + let calls = 0 + const machine = Machine.make({ + root: Machine.state({ states: { A: {}, B: {} } }), + events: Machine.events({ Go: {} }), + branches: { go: { b: { target: "B" } } } + }).handle({ + initial: { target: "A" }, + states: { + A: { + on: { + Go: { + branches: "go", + resolve: ({ select }) => { + calls += 1 + return select.b() + } + } + } + } + } + }) + + MachineTest.analyze(machine) + assert.strictEqual(calls, 0) + }) +}) + +describe("MachineTest.check", () => { + it.effect("passes an exhaustively explored machine", () => + Effect.gen(function*() { + const report = yield* MachineTest.check(Toggle) + + assert.deepStrictEqual(report, { + machine: "Toggle", + ok: true, + exploration: "exhaustive", + limits: [], + stats: { + states: 2, + plannedTransitions: 2, + transitions: { total: 2, hit: 2 }, + branches: { total: 2, hit: 2 } + }, + findings: [] + }) + })) + + it.effect("reports sampled payloads and invocations as limits rather than as proof", () => + Effect.gen(function*() { + const machine = Machine.make({ + root: Machine.state({ states: { Idle: {}, Busy: {} } }), + events: Machine.events({ Add: { by: Schema.Number } }), + effects: { work: Effect.void } + }).handle({ + initial: { target: "Idle" }, + states: { + Idle: { on: { Add: { target: "Busy" } } }, + Busy: { invoke: { src: "work", onDone: { target: "Idle" } } } + } + }) + + const report = yield* MachineTest.check(machine) + + assert.isTrue(report.ok) + assert.strictEqual(report.exploration, "bounded") + assert.deepStrictEqual(report.limits, ["sampledEvents", "invocations"]) + assert.deepStrictEqual(laws(report), []) + })) + + it.effect("does not warn about transitions only an unrun invocation can lead to", () => + Effect.gen(function*() { + const machine = Machine.make({ + root: Machine.state({ states: { Idle: {}, Busy: {}, Failed: {} } }), + events: Machine.events({ Start: {}, Retry: {} }), + effects: { work: Effect.void } + }).handle({ + initial: { target: "Idle" }, + states: { + Idle: { on: { Start: { target: "Busy" } } }, + Busy: { invoke: { src: "work", onDone: { target: "Failed" } } }, + Failed: { on: { Retry: { target: "Busy" } } } + } + }) + + const report = yield* MachineTest.check(machine) + + assert.deepStrictEqual(report.limits, ["invocations"]) + assert.deepStrictEqual(laws(report), []) + })) + + it.effect("finds nondeterminism hidden behind a helper, with the events that reproduce it", () => + Effect.gen(function*() { + const next = () => Math.random() + const Events = Machine.events({ Pay: {} }) + const machine = Machine.make({ + root: Machine.state({ + states: { Editing: {}, Charging: { fields: { amount: Schema.Number } } } + }), + events: Events + }).handle({ + initial: { target: "Editing" }, + states: { + Editing: { on: { Pay: { target: "Charging", data: () => ({ amount: next() }) } } } + } + }) + + const report = yield* MachineTest.check(machine) + const finding = report.findings[0]! + + assert.isFalse(report.ok) + assert.deepStrictEqual(laws(report), ["planning.deterministic"]) + assert.isTrue(finding.law === "planning.deterministic" && finding.replay.events.length === 1) + })) + + it.effect("finds nondeterminism during startup", () => + Effect.gen(function*() { + const machine = Machine.make({ + root: Machine.state({ states: { Idle: { fields: { at: Schema.Number } } } }), + events: Machine.events({}) + }).handle({ + initial: { target: "Idle", data: () => ({ at: Math.random() }) } + }) + + const report = yield* MachineTest.check(machine) + + assert.deepStrictEqual(laws(report), ["planning.deterministic"]) + })) + + it.effect("reports a throwing callback instead of failing", () => + Effect.gen(function*() { + const machine = Machine.make({ + root: Machine.state({ states: { Idle: {}, Done: {} } }), + events: Machine.events({ Go: {} }) + }).handle({ + initial: { target: "Idle" }, + states: { + Idle: { + on: { + Go: { + target: "Done", + guard: () => { + throw new Error("boom") + } + } + } + } + } + }) + + const report = yield* MachineTest.check(machine) + + assert.isFalse(report.ok) + assert.deepStrictEqual(laws(report), ["planning.failed"]) + assert.include(report.findings[0]!.message, "boom") + })) + + it.effect("reports a declared branch that exploration never took as a warning", () => + Effect.gen(function*() { + const machine = Machine.make({ + root: Machine.state({ states: { Idle: {}, Done: {}, Never: {} } }), + events: Machine.events({ Go: {} }), + branches: { go: { done: { target: "Done" }, never: { target: "Never" } } } + }).handle({ + initial: { target: "Idle" }, + states: { Idle: { on: { Go: { branches: "go", resolve: ({ select }) => select.done() } } } } + }) + + const report = yield* MachineTest.check(machine) + + assert.isTrue(report.ok) + assert.deepStrictEqual(laws(report), ["transition.notCovered"]) + })) + + it.effect("reports an application invariant with the events that break it", () => + Effect.gen(function*() { + const Root = Machine.state({ fields: { count: Schema.Number } }) + const Events = Machine.events({ Add: {} }) + const machine = Machine.make({ root: Root, events: Events }).handle({ + root: () => ({ count: 0 }), + on: { Add: { update: "root", data: ({ root }) => ({ count: root.count + 1 }) } } + }) + + const report = yield* MachineTest.check(machine, { + invariants: [ + MachineTest.invariants(machine).state( + "count stays below 2", + ({ snapshot }) => Root.get(snapshot, "root" as never) !== undefined && snapshot.value.count < 2 + ) + ] + }) + + assert.isFalse(report.ok) + const finding = report.findings[0]! + assert.isTrue(finding.law === "invariant.violated" && finding.replay.events.length === 2) + })) + + it.effect("stops at the limits and says so", () => + Effect.gen(function*() { + const machine = Machine.make({ + root: Machine.state({ fields: { count: Schema.Number } }), + events: Machine.events({ Add: {} }) + }).handle({ + root: () => ({ count: 0 }), + on: { Add: { update: "root", data: ({ root }) => ({ count: root.count + 1 }) } } + }) + + const report = yield* MachineTest.check(machine, { limits: { maxDepth: 3 } }) + + assert.isTrue(report.ok) + assert.strictEqual(report.exploration, "bounded") + assert.deepStrictEqual(report.limits, ["depth"]) + })) + + it.effect("samples input from its schema, or uses the supplied input", () => + Effect.gen(function*() { + const machine = Machine.make({ + root: Machine.state({ states: { Idle: { fields: { name: Schema.String } } } }), + input: Schema.String, + events: Machine.events({}) + }).handle({ + initial: { target: "Idle", data: ({ input }) => ({ name: input }) } + }) + + assert.deepStrictEqual((yield* MachineTest.check(machine)).limits, ["sampledInput"]) + assert.deepStrictEqual((yield* MachineTest.check(machine, { input: "a" })).limits, []) + })) + + it.effect("does not send events to a completed machine", () => + Effect.gen(function*() { + const machine = Machine.make({ + root: Machine.state({ states: { Idle: {}, Done: { type: "final" } } }), + events: Machine.events({ Finish: {} }) + }).handle({ + initial: { target: "Idle" }, + states: { + Idle: { on: { Finish: { target: "Done" } } } + } + }) + + const report = yield* MachineTest.check(machine) + + assert.isTrue(report.ok) + assert.strictEqual(report.stats.plannedTransitions, 1) + })) + + it.effect("formats a report as text", () => + Effect.gen(function*() { + assert.strictEqual( + MachineTest.formatCheckReport(yield* MachineTest.check(Toggle)), + "PASS Toggle: 2 states, 2/2 branches, exhaustive" + ) + })) +}) diff --git a/packages/effect-machine/typetest/testing/Check.tst.ts b/packages/effect-machine/typetest/testing/Check.tst.ts new file mode 100644 index 00000000..e35973f3 --- /dev/null +++ b/packages/effect-machine/typetest/testing/Check.tst.ts @@ -0,0 +1,31 @@ +import { Machine } from "@typeonce/effect-machine" +import { MachineTest } from "@typeonce/effect-machine/testing" +import { Effect, Schema } from "effect" +import { describe, expect, it } from "tstyche" + +const Events = Machine.events({ Add: { by: Schema.Number } }) +const machine = Machine.make({ + root: Machine.state({ states: { Idle: { fields: { name: Schema.String } } } }), + input: Schema.String, + events: Events +}).handle({ + initial: { target: "Idle", data: ({ input }) => ({ name: input }) }, + states: { Idle: { on: { Add: { update: "Idle", data: ({ state }) => state } } } } +}) + +describe("MachineTest.check", () => { + it("returns a report that cannot fail", () => { + expect(MachineTest.check(machine)).type.toBe>() + }) + + it("types input and events from the machine", () => { + expect(MachineTest.check(machine, { input: "a", events: [Events.Add({ by: 1 })] })).type.not.toRaiseError() + expect(MachineTest.check).type.not.toBeCallableWith(machine, { input: 1 }) + }) + + it("narrows findings by law", () => { + const finding = {} as MachineTest.CheckFinding + if (finding.law === "topology.noPathFromInitial") expect(finding.path).type.toBe() + if (finding.law === "planning.deterministic") expect(finding.replay).type.toBe() + }) +}) diff --git a/scripts/ai-docs/copy.mjs b/scripts/ai-docs/copy.mjs index 1b6f4ee8..8db94d80 100644 --- a/scripts/ai-docs/copy.mjs +++ b/scripts/ai-docs/copy.mjs @@ -1,11 +1,16 @@ import { copyFileSync, cpSync, rmSync } from "node:fs" import { join, resolve } from "node:path" -// The copies are build outputs: the source of truth stays in `ai-docs` and `LLMS.md`. +// The copies exist only while packing, so agents working in this repository read `ai-docs` and `LLMS.md` directly. const root = resolve(import.meta.dirname, "../..") const packageDirectory = join(root, "packages/effect-machine") -copyFileSync(join(root, "LLMS.md"), join(packageDirectory, "AGENTS.md")) -copyFileSync(join(root, "LLMS.md"), join(packageDirectory, "CLAUDE.md")) -rmSync(join(packageDirectory, "ai-docs"), { recursive: true, force: true }) -cpSync(join(root, "ai-docs"), join(packageDirectory, "ai-docs"), { recursive: true }) +for (const file of ["AGENTS.md", "CLAUDE.md", "ai-docs"]) { + rmSync(join(packageDirectory, file), { recursive: true, force: true }) +} + +if (!process.argv.includes("--clean")) { + copyFileSync(join(root, "LLMS.md"), join(packageDirectory, "AGENTS.md")) + copyFileSync(join(root, "LLMS.md"), join(packageDirectory, "CLAUDE.md")) + cpSync(join(root, "ai-docs"), join(packageDirectory, "ai-docs"), { recursive: true }) +} diff --git a/scripts/check-architecture.mjs b/scripts/check-architecture.mjs index 4137cfae..04688d1f 100644 --- a/scripts/check-architecture.mjs +++ b/scripts/check-architecture.mjs @@ -223,7 +223,7 @@ export const checkArchitecture = ({ const implementationSeams = new Map([ ["src/Machine.ts", new Set(["src/internal/machine/machine.ts"])], ["src/testing/MachineTest.ts", new Set([ - "verification", "runtime", "arbitrary", "finiteModel", "referenceModel", "invariant", + "check", "verification", "runtime", "arbitrary", "finiteModel", "referenceModel", "invariant", "runtimeInvariant", "exploration", "probe", "trace", "format" ].map((name) => `src/internal/testing/machine/${name}.ts`))], ["src/unstable/reactivity/AtomMachine.ts", new Set(["src/internal/machine/atom.ts"])], diff --git a/scripts/devtools-pack-check.mjs b/scripts/devtools-pack-check.mjs index 0fd33d74..5dbacf1e 100644 --- a/scripts/devtools-pack-check.mjs +++ b/scripts/devtools-pack-check.mjs @@ -128,6 +128,15 @@ try { throw new Error("installed CLI help does not document live and static workflows") } + const checked = JSON.parse(run(binary, ["check", "--root", consumer, "--json"], { cwd: consumer }).stdout) + if (checked.ok !== true || checked.machines?.[0]?.machine !== "packed-fixture") { + throw new Error(`installed CLI did not check the machine\n${JSON.stringify(checked)}`) + } + const missing = spawnSync(binary, ["check", "src/missing.ts", "--root", consumer], { cwd: consumer, encoding: "utf8" }) + if (missing.status === 0) { + throw new Error("installed CLI reported success with nothing to check") + } + run(process.execPath, [ "--input-type=module", "--eval",