From 8347babde8c03bbb5ba038754f0d28f763c38ec9 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Wed, 15 Jul 2026 13:35:57 +0200 Subject: [PATCH 01/13] Add Common Crawl HTML API fuzz harness --- .gitignore | 1 + tools/html-api-fuzz/README.md | 563 ++++++ .../codex-triage-orchestrator.php | 935 ++++++++++ tools/html-api-fuzz/commoncrawl-analysis.php | 14 + tools/html-api-fuzz/launcher.php | 359 ++++ tools/html-api-fuzz/lib/CommonCrawlRunner.php | 662 +++++++ tools/html-api-fuzz/lib/Corpus.php | 78 + tools/html-api-fuzz/lib/Generator.php | 1409 ++++++++++++++ tools/html-api-fuzz/lib/HtmlApiBootstrap.php | 52 + tools/html-api-fuzz/lib/Mutator.php | 111 ++ tools/html-api-fuzz/lib/OracleFinding.php | 273 +++ tools/html-api-fuzz/lib/OracleRenderer.php | 324 ++++ tools/html-api-fuzz/lib/Prng.php | 56 + tools/html-api-fuzz/lib/ResultStore.php | 406 ++++ tools/html-api-fuzz/lib/Signature.php | 142 ++ tools/html-api-fuzz/lib/Support.php | 508 +++++ tools/html-api-fuzz/lib/TagInvariants.php | 437 +++++ tools/html-api-fuzz/lib/TreeRenderer.php | 1346 +++++++++++++ tools/html-api-fuzz/lib/Worker.php | 775 ++++++++ tools/html-api-fuzz/lib/autoload.php | 15 + tools/html-api-fuzz/lib/wp-stubs.php | 52 + tools/html-api-fuzz/minimize.php | 494 +++++ tools/html-api-fuzz/oracles/lexbor/README.md | 74 + tools/html-api-fuzz/oracles/lexbor/build.sh | 89 + .../oracles/lexbor/lexbor-tree-oracle.c | 1296 +++++++++++++ tools/html-api-fuzz/replay.php | 148 ++ tools/html-api-fuzz/runner.php | 511 +++++ .../start-continuous-run-tmux.sh | 166 ++ tools/html-api-fuzz/stop.php | 390 ++++ .../tests/codex-triage-orchestrator-smoke.php | 179 ++ .../tests/commoncrawl-analysis-smoke.php | 139 ++ .../fixtures/commoncrawl-timeout-worker.php | 6 + .../tests/generator-policy-smoke.php | 1658 +++++++++++++++++ .../tests/lexbor-oracle-smoke.php | 184 ++ .../tests/result-store-smoke.php | 234 +++ .../tests/runner-retention-smoke.php | 232 +++ tools/html-api-fuzz/tests/stop-smoke.php | 531 ++++++ .../tree-renderer-normalization-smoke.php | 421 +++++ .../tests/watcher-sqlite-smoke.php | 206 ++ tools/html-api-fuzz/watcher.php | 620 ++++++ tools/html-api-fuzz/worker.php | 103 + 41 files changed, 16199 insertions(+) create mode 100644 tools/html-api-fuzz/README.md create mode 100755 tools/html-api-fuzz/codex-triage-orchestrator.php create mode 100644 tools/html-api-fuzz/commoncrawl-analysis.php create mode 100755 tools/html-api-fuzz/launcher.php create mode 100644 tools/html-api-fuzz/lib/CommonCrawlRunner.php create mode 100644 tools/html-api-fuzz/lib/Corpus.php create mode 100644 tools/html-api-fuzz/lib/Generator.php create mode 100644 tools/html-api-fuzz/lib/HtmlApiBootstrap.php create mode 100644 tools/html-api-fuzz/lib/Mutator.php create mode 100644 tools/html-api-fuzz/lib/OracleFinding.php create mode 100644 tools/html-api-fuzz/lib/OracleRenderer.php create mode 100644 tools/html-api-fuzz/lib/Prng.php create mode 100644 tools/html-api-fuzz/lib/ResultStore.php create mode 100644 tools/html-api-fuzz/lib/Signature.php create mode 100644 tools/html-api-fuzz/lib/Support.php create mode 100644 tools/html-api-fuzz/lib/TagInvariants.php create mode 100644 tools/html-api-fuzz/lib/TreeRenderer.php create mode 100644 tools/html-api-fuzz/lib/Worker.php create mode 100644 tools/html-api-fuzz/lib/autoload.php create mode 100644 tools/html-api-fuzz/lib/wp-stubs.php create mode 100755 tools/html-api-fuzz/minimize.php create mode 100644 tools/html-api-fuzz/oracles/lexbor/README.md create mode 100755 tools/html-api-fuzz/oracles/lexbor/build.sh create mode 100644 tools/html-api-fuzz/oracles/lexbor/lexbor-tree-oracle.c create mode 100755 tools/html-api-fuzz/replay.php create mode 100755 tools/html-api-fuzz/runner.php create mode 100755 tools/html-api-fuzz/start-continuous-run-tmux.sh create mode 100644 tools/html-api-fuzz/stop.php create mode 100755 tools/html-api-fuzz/tests/codex-triage-orchestrator-smoke.php create mode 100755 tools/html-api-fuzz/tests/commoncrawl-analysis-smoke.php create mode 100755 tools/html-api-fuzz/tests/fixtures/commoncrawl-timeout-worker.php create mode 100644 tools/html-api-fuzz/tests/generator-policy-smoke.php create mode 100755 tools/html-api-fuzz/tests/lexbor-oracle-smoke.php create mode 100644 tools/html-api-fuzz/tests/result-store-smoke.php create mode 100644 tools/html-api-fuzz/tests/runner-retention-smoke.php create mode 100644 tools/html-api-fuzz/tests/stop-smoke.php create mode 100644 tools/html-api-fuzz/tests/tree-renderer-normalization-smoke.php create mode 100644 tools/html-api-fuzz/tests/watcher-sqlite-smoke.php create mode 100755 tools/html-api-fuzz/watcher.php create mode 100755 tools/html-api-fuzz/worker.php diff --git a/.gitignore b/.gitignore index 15876fa47fee8..dbec517727bbc 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ wp-tests-config.php /packagehash.txt /.gutenberg-hash /artifacts +/tools/html-api-fuzz/oracles/lexbor/build /setup.log /coverage diff --git a/tools/html-api-fuzz/README.md b/tools/html-api-fuzz/README.md new file mode 100644 index 0000000000000..f192a4c7aa1c5 --- /dev/null +++ b/tools/html-api-fuzz/README.md @@ -0,0 +1,563 @@ +# HTML API Fuzzer + +Pure PHP fuzzer for the WordPress HTML API under `src/wp-includes/html-api`. +It compares `WP_HTML_Processor` against a selectable tree-construction oracle +using an html5lib-style textual tree, and separately checks a set of API +invariants described under “Invariants” below. The default oracle is PHP's +`Dom\HTMLDocument`, preserving the historical behavior. + +No browser, Playwright, Node, or `wp-env` is involved. + +## Requirements + +- PHP 8.4+ with ext-dom, for `Dom\HTMLDocument`. +- Run from the repository root. +- Optional source-built Lexbor oracle: `git`, `cmake`, and a C compiler. + +## Common Crawl with cc-analyzer + +`commoncrawl-analysis.php` is an analysis callback for `cc-analyzer.phar`. It +accepts the analyzer's `CcAnalyzer\Analysis\HtmlAnalysisInput` value object, +runs the raw response body through `WP_HTML_Processor` in full-document mode, +compares the resulting tree with Lexbor, and records the Common Crawl +provenance needed to locate or replay the document. + +Build the oracle from the current upstream `master` first: + +```sh +tools/html-api-fuzz/oracles/lexbor/build.sh +``` + +The build resolves the moving ref, uses commit-keyed build/install directories, +refuses a dirty Lexbor checkout, and writes `build/build-manifest.json` with the +requested ref, resolved commit, upstream URL, build time, compiler, CMake +version, and oracle SHA-256. For exact reproduction, rebuild the manifest's +`resolvedCommit`, for example: + +```sh +LEXBOR_COMMIT=de1d07a7765aad37090cc36f7fac3bb59e21467d \ + tools/html-api-fuzz/oracles/lexbor/build.sh +``` + +Then point cc-analyzer's HTML analysis-file argument at the absolute path to: + +```text +tools/html-api-fuzz/commoncrawl-analysis.php +``` + +The PHAR owns crawl selection, ranges, checkpoints, and concurrency. The +callback boundary is the same as other cc-analyzer HTML analyses: the file +returns a `static function (HtmlAnalysisInput $document): void` closure. Use +`php /path/to/cc-analyzer.phar --help` for the binary's crawl-specific command +and argument names. This checkout does not contain `cc-analyzer.phar`, so the +real PHAR CLI boundary is deliberately not claimed as tested; the included +smoke test exercises a value object matching the observed callback contract. + +Each accepted document runs in a separate PHP child with its own memory and +wall-clock limit. The callback writes `input.bin` and an initial replay before +starting that child. A timeout, memory exhaustion, crash, or signal is retained +as a finding without terminating cc-analyzer or losing the triggering bytes. + +The Common Crawl adapter uses Lexbor by default. Its environment is: + +- `CC_ANALYZER_OUTPUT_DIR`: shared output directory for this analyzer run. If + absent, a unique directory is created under `artifacts/html-api-commoncrawl`. +- `HTML_API_CC_RUN_ID`: explicit immutable run identifier. When the analyzer + provides an output directory, the default is a stable hash of that path. +- `HTML_API_FUZZ_LEXBOR_ORACLE`: non-default Lexbor oracle binary path. +- `HTML_API_CC_EXPECT_LEXBOR_COMMIT`: optional exact commit assertion; startup + fails if the executable reports another commit. +- `HTML_API_CC_ORACLE_TIMEOUT_MS`: Lexbor subprocess timeout; default `10000`. +- `HTML_API_CC_PROCESS_TIMEOUT_MS`: whole-document worker timeout; default + `30000`. +- `HTML_API_CC_MEMORY_LIMIT`: PHP memory limit for each worker; default `256M`. +- `HTML_API_CC_MAX_INPUT_BYTES`: largest response body to analyze; default + `2097152`, or `0` for unlimited. +- `HTML_API_CC_MAX_TOKENS`, `HTML_API_CC_MAX_NODES`, + `HTML_API_CC_MAX_DEPTH`, and `HTML_API_CC_MAX_TREE_BYTES`: bounded work and + tree-output ceilings; defaults are `50000`, `50000`, `512`, and `16777216`. +- `HTML_API_CC_CHECKS`: `baseline`, `full`, or `sampled` (default). Baseline + performs the WordPress/Lexbor differential; full also runs API invariants, + mutation differential, and normalize preservation. +- `HTML_API_CC_FULL_SAMPLE_PERCENT`: deterministic full-check share when using + `sampled`; default `1`. +- `HTML_API_CC_REQUIRE_UTF8`: defaults to `1`. Invalid UTF-8 bodies are + summarized as skipped because the WordPress HTML API consumes UTF-8 and a + legacy-encoded response otherwise produces mostly encoding noise. Set it to + `0` for raw-byte/encoding fuzzing. +- `HTML_API_CC_MAX_KEEP_PER_SIGNATURE`: retained examples per distinct + signature; default `3`. +- `HTML_API_CC_RETAIN_ALL`: retain passing inputs too; default `0`. +- `HTML_API_CC_ORACLE=php-dom`: testing/debug escape hatch. Normal Common + Crawl runs should leave this unset so the independent source-built Lexbor + oracle is used. +- `CC_ANALYZER_VERSION`, `CC_ANALYZER_CRAWL`, and `CC_ANALYZER_INVOCATION`: + optional provenance strings recorded verbatim in the immutable run config. + +The output root contains: + +- `configuration.json`: run ID and exact configuration fingerprint plus Git, + PHP, Lexbor build manifest/hash, limits, and analyzer provenance. Reusing the + directory with different settings fails at startup. +- `commoncrawl-summary.ndjson`: one locked, append-only record per document; + safe for concurrent analyzer workers. +- `coverage.json`: aggregate total, covered, failed, and per-status counts. +- `findings/SIGNATURE/DOCUMENT/`: bounded byte-exact exemplars containing + `input.bin`, `result.json`, `replay.json`, both rendered trees, and a + `.complete` marker. Findings are written in `pending/` and atomically renamed + only after every checked write succeeds; retries of the same record/body are + idempotent. Any directory left in `pending/` is recoverable evidence of a + callback interruption. + +Passing inputs are represented only by their summary by default. A finding's +replay embeds the exact input body, so it remains reproducible without another +Common Crawl download: + +```sh +php tools/html-api-fuzz/replay.php \ + --replay artifacts/html-api-commoncrawl/findings/SIGNATURE/DOCUMENT/replay.json +``` + +Transport charset, content type, target URI, WARC record ID, analyzer state +key, and source range are metadata only. The comparison intentionally feeds +the same raw bytes to both parsers; it never silently transcodes one side. + +## Common Commands + +Run one generated seed: + +```sh +php tools/html-api-fuzz/worker.php --seed 1 --output-dir artifacts/html-api-fuzz/seed-1 +``` + +Run one generated seed with a specific terminal payload policy: + +```sh +php tools/html-api-fuzz/worker.php --seed 1 --payload-policy valid-utf8 --output-dir artifacts/html-api-fuzz/seed-1 +``` + +Run a batch in worker subprocesses (seeds are batched into shared worker +processes, 25 per process by default; see `--batch-size`): + +```sh +php tools/html-api-fuzz/runner.php --max-seeds 100 --duration-seconds 60 +``` + +Run a structural UTF-8-biased batch with a post-generation byte cap: + +```sh +php tools/html-api-fuzz/runner.php --max-seeds 100 --payload-policy valid-utf8 --max-input-bytes 4096 +``` + +Build and run against the source-built Lexbor oracle: + +```sh +tools/html-api-fuzz/oracles/lexbor/build.sh +php tools/html-api-fuzz/worker.php --seed 1 --dom-oracle lexbor-source --output-dir artifacts/html-api-fuzz/seed-1-lexbor +php tools/html-api-fuzz/runner.php --max-seeds 100 --dom-oracle lexbor-source --duration-seconds 60 +``` + +Use `--lexbor-oracle-bin PATH` or `HTML_API_FUZZ_LEXBOR_ORACLE` when the +oracle binary is not at +`tools/html-api-fuzz/oracles/lexbor/build/lexbor-tree-oracle`. + +Run indefinitely: + +```sh +php tools/html-api-fuzz/runner.php --duration-seconds 0 --max-seeds 0 +``` + +Run parallel lanes and triage failures after completion: + +```sh +php tools/html-api-fuzz/launcher.php --lanes 4 --max-seeds 1000 --watcher +``` + +For continuous fuzzing, run the launcher with `--duration-seconds 0 --max-seeds 0` +and run `watcher.php` in a second shell against the same output directory. + +Stop an indefinite run gracefully (each lane finishes its current batch, the +watcher performs a final scan, and the codex orchestrator drains its running +jobs): + +```sh +php tools/html-api-fuzz/stop.php --run-dir artifacts/html-api-fuzz/run-... +``` + +Without `--run-dir` the most recently active *unfinished* run under +`artifacts/html-api-fuzz` is targeted. Finished and stale runs are not +preferred; if nothing live is found, the most recent stopped-looking run is +targeted with a warning. The script creates the stop file advertised by the +run state and also `RUN_DIR/STOP` when a run directory is known, so watchers +and orchestrators see the stop request. For a standalone runner with custom +`--stop-file PATH`, both files are written. Relative custom stop files are +resolved with the runner cwd recorded in new runner state; for older state, +pass only `--stop-file PATH` to write a known stop file directly if needed. +With `--run-dir --stop-file PATH`, the explicit path is added to the run-state +and `RUN_DIR/STOP` targets. `touch` works just as well. The launcher and runner refuse to start while a stop file already +exists — remove `STOP` (or the custom stop file) before reusing a run +directory. (A stop requested in the sub-second window between the launcher's +startup check and a lane's own makes that lane refuse rather than stop +gracefully; the run still ends.) +If state cannot be read or old state lacks enough context to locate a relative +custom stop file, the tool still writes `RUN_DIR/STOP` but exits `2` with +`ok: false` and warnings because a standalone custom stop file may be unknown. + +The watcher exits after a final scan once every runner under the run +directory reports a stop reason. A runner whose state has gone silent is +presumed dead after `--stop-stale-seconds` (default 120); per lane that +threshold is floored at twice the lane's advertised batch budget +(`timeout-ms × batch-size`), so long batches are not mistaken for crashes. +The stop tool uses the same default stale threshold when auto-selecting the +latest unfinished run. + +Replay a failure from a retained seed directory, or from the lane's SQLite +store when the seed directory was pruned (see "Artifact Retention"): + +```sh +php tools/html-api-fuzz/replay.php --replay artifacts/html-api-fuzz/run-.../seed-.../primary/replay.json +php tools/html-api-fuzz/replay.php --store artifacts/html-api-fuzz/run-.../lane-00/results.sqlite --seed 12345 +``` + +Minimize a failure while preserving the same signature: + +```sh +php tools/html-api-fuzz/minimize.php --replay artifacts/html-api-fuzz/run-.../seed-.../primary/replay.json +``` + +By default (`--probe-mode auto`) the minimizer evaluates candidates in worker +subprocesses so `--timeout-ms` can kill pathological candidates and each probe +starts with fresh PHP state. Use `--probe-mode in-process` for faster +exact-signature minimization when that isolation is not needed; in-process +probes write only the final minimized artifacts unless +`--keep-candidate-artifacts` is also passed. Use `--probe-mode process` to +force subprocess probes explicitly. + +Watch an existing run directory and minimize new distinct signatures: + +```sh +php tools/html-api-fuzz/watcher.php --run-dir artifacts/html-api-fuzz/run-... --once +``` + +Configured ceilings are reported as `failureClass: "resource-limit"` and remain +in the watcher/minimizer triage path. This bucket includes tag/tree token +ceilings (`tag-token-limit-exceeded`, `mutation-token-limit-exceeded`, +`wordpress-token-limit-exceeded`) and oracle node ceilings +(`node-limit-exceeded`, recorded as `dom-node-limit-exceeded` in historical +signature facts). Process timeouts, PHP fatal errors, and memory failures are +separate failures and are also in scope for triage. + +## Execution Model + +The runner batches consecutive seeds into one worker process +(`worker.php --batch-count N`, default `--batch-size 25` on the runner) so the +WordPress bootstrap and process spawn are paid once per batch rather than once +per seed. Each seed still writes its own `seed-N/primary` artifacts. If a batch +process dies or times out mid-way, seeds left without a `result.json` are +re-run individually in isolation, so a crash on one input cannot take +neighboring seeds' results with it. + +## Input Stages + +Seeds are deterministically split between two input stages: + +- **Generated** (default ~80%): the structural grammar described under + “Generator Profiles”. +- **Corpus-mutated** (default ~20%, `--corpus-mutate-percent N` on + `worker.php`/`runner.php`): a `#data` section from the html5lib-tests + tree-construction corpus (`tests/phpunit/data/html5lib-tests`), passed + through 1–4 deterministic mutations (byte insert/replace, chunk + delete/duplicate, tag-name swap, case toggle, corpus splice). The stage, + corpus file, entry index, and operations are recorded in result metadata, + and the mutated input itself is in the replay manifest, so replays are + standalone. Inputs report `inputSource: "corpus-mutated"` and + `profile: "corpus-mutated"`. + +Both stages derive entirely from the seed, so seed N always produces the same +input for the same fuzzer version and corpus. + +## Artifact Layout + +The runner writes: + +- `results.sqlite`: one row per attempted seed (table `attempts`, WAL mode). + Passing attempts store summary columns only — every attempt is regenerable + from its seed. Failure rows additionally store the summary, result, and + replay JSON documents; the replay embeds the input as base64, so a pruned + failure can be reproduced with `replay.php --store results.sqlite --seed N`. + `signature_hash` and `family_key` are indexed columns for grouping + failures without `json_extract`. `oracle_kind`, `oracle_version`, + `oracle_commit`, and `oracle_binary` record which oracle generated the + summary, including for passing rows whose JSON payloads are pruned. The + watcher tails these stores + incrementally by row id. (`summary.ndjson` files from older runs are still + scanned.) Durability is `synchronous=NORMAL`: an OS crash (not a process + crash) can lose the last moments of a run. +- `events.ndjson`: runner lifecycle events, including batch boundaries. +- `logs/batch-N.log`: output of a batch worker process, kept only when the + batch contained a retained failure or a seed that needed an isolated + re-run — over-cap repeats of a known signature do not accumulate batch + logs. +- `state.json`: aggregate counters, stop reason, and compact Git metadata. + Oracle losses are counted per class: `oracleParseErrors` (inputs the + selected oracle rejects receive no differential coverage), + `oracleUnsupported` (tree shapes the oracle cannot represent), and + `oracleTolerated` + (comparisons that passed only under the documented scalar tolerance). +- `seed-N/primary/input.bin`: raw generated bytes. +- `seed-N/primary/replay.json`: base64 replay manifest, including the commit + hash, tracked-file dirty state, selected oracle, and fragment context needed + to interpret a standalone replay. +- `seed-N/primary/result.json`: full worker result. +- `seed-N/primary/wordpress-tree.txt` and `dom-tree.txt`: rendered trees when available. + +### Artifact Retention + +Seed directories are working space, not the archive. After each seed is +recorded in `results.sqlite`, its `seed-N` directory is deleted unless the +attempt failed *and* the failure's signature has fewer than +`--max-keep-per-signature` (default 5, minimum 1) exemplar directories still +on disk in this lane. The cap counts directories, not rows, so restarting a +runner against the same output directory neither double-counts re-recorded +seeds nor deletes a previously retained exemplar. The first exemplar of every +new signature is always retained, so the watcher's minimization path keeps a +replay file to work from; subsequent repeats of a known signature add a +database row and nothing else. A failure whose replay document is missing +(worker killed before writing it) always keeps its directory — the files are +the only reproduction. Disk growth is therefore proportional to *new +distinct failures*, not to seeds executed. + +The cap applies per lane: the launcher passes the same value to every lane, +so a signature that appears in all lanes keeps up to `N × lanes` directories +across the run. Use `--keep-all-artifacts` (runner or launcher) to keep every +seed directory for debugging. + +The run-level Git metadata intentionally stays compact: full and short commit +hash, current branch when available, commit date, and a dirty flag for +tracked-file changes. The dirty flag is tri-state: `true`, `false`, or `null` +when Git is unavailable or dirty detection fails. Full `git status` or diff +output is not recorded because it is noisy, can expose local edits, and grows +indefinitely in long runs. Launcher and runner processes collect this metadata +once and pass it to workers so long runs do not invoke Git for every seed. +Replayed and minimized manifests keep current checkout metadata at the top level +and preserve discovery provenance in `sourceReplay`. + +The watcher writes triage state under `.triage-watcher` by default, or under +`--state-dir` when provided. Each signature gets a stable directory containing +`failure.json`, minimizer logs, and minimized replay/result artifacts. Failed +minimizations are retried on later scans, up to `--max-minimize-retries` +(default 3) attempts per signature. + +## Modes and Fragment Contexts + +- `fragment-body`: parse as a fragment. The selected oracle uses real fragment + parsing (the `innerHTML` setter on a context element of an empty document), + not a document-wrapping approximation. +- `full-document`: parse as a full HTML document. +- `auto`: weighted choice. + +In fragment mode a context element is selected per seed +(`--fragment-context TAG` on `worker.php` for replays). `` dominates; +the other contexts (`div`, `p`, `td`, `tr`, `table`, `caption`, `colgroup`, +`select`, `option`, `template`, `title`, `textarea`, `script`, `style`, +`svg`, `math`) receive a small probe weight. `WP_HTML_Processor::create_fragment()` +currently supports only ``, so non-body contexts are recorded as +`status: "unsupported"` today; when create_fragment() gains context support +the fuzzer picks up the new coverage with no changes. The selected oracle already +parses every context correctly. + +Unsupported `WP_HTML_Processor` cases are expected by default and are recorded +as successful attempts with `status: "unsupported"`. Use `--fail-unsupported` +when you want unsupported cases to become failures. + +## Invariants + +Each seed checks, in order, stopping at the first failing class: + +1. **Tag Processor invariants** (`tag-invariant-failed`): token loop + termination under the token ceiling; non-null token type/name/tag; + attribute getters and `class_list()` iteration do not throw; + `get_updated_html()` with no queued edits returns the input unchanged; a + simple `set_attribute()` mutation is visible to a re-scan; and + **seek consistency** — a bookmark set at a seed-chosen token, after + scanning to the end and seeking back, must reproduce the identical token + stream (`seek-token-stream-mismatch`). +2. **Differential tree comparison** (`tree-mismatch` / `encoding-mismatch`): + the WordPress tree must equal the selected oracle tree (see “Tree Comparison”). +3. **Breadcrumb consistency** (`breadcrumb-mismatch`): at every tag token, + `get_breadcrumbs()` must agree with the element stack derived from token + order and `expects_closer()`. +4. **Mutation differential** (`mutation-tree-mismatch` / + `mutation-delta-mismatch`), only on a clean baseline: after setting + `data-fuzz="1"` on the first tag, the mutated document must parse + identically in WordPress and the selected oracle, and the WordPress tree must + change by exactly the one attribute line (unless formatting-element + reconstruction clones the attribute, or tree construction legitimately + drops the mutated element, in which case the differential comparison alone + applies). +5. **Normalize tree preservation** (`normalize-tree-changed`), only on a + clean baseline: parsing `normalize()` output must produce the same tree as + the original input, modulo the documented scalar substitutions. This is + stricter than idempotence, which a consistently wrong serializer can pass. +6. **Normalize idempotence** (`normalize-invariant-failed`): + `normalize()` / `serialize()` run twice must be a fixed point, with no + PHP native errors or throwables. Full documents use + `create_full_parser()->serialize()`; non-body fragment contexts use + `create_fragment()->serialize()`. + +### Known invariant oracle follow-ups + +- The simple `set_attribute()` mutation oracle needs to handle inputs that + begin with `
` using the same tag-selection semantics as the mutator. + `next_tag()` skips the raw closing token and mutates the following tag, + while a verifier that scans with `next_token()` can see the spec-special + `BR` element synthesized from `
` first and incorrectly report + `mutation-attribute-missing`. Fix the verifier by selecting the first + mutable tag through `next_tag()` too. + +## Generator Profiles + +The generator uses a structural HTML grammar with weighted profiles: + +- `balanced` +- `full-document` (includes occasional frameset documents, quirks-mode + doctypes, and content after ``) +- `body-fragment` +- `tables` +- `template` +- `select` (option/optgroup nesting, select-ending elements such as `input`, + `textarea` and `button`, nested selects, select-in-table) +- `foreign-content` (MathML/SVG integration points, HTML breakout tags, + `` with and without breakout attributes, `annotation-xml` encoding + variants, CDATA sections in foreign content, case-mangled `foreignObject`) +- `rawtext-rcdata` (script/style/iframe/noembed/noframes/xmp/noscript, + title/textarea, occasional `plaintext`) +- `text-fragment` (standalone terminal payloads, biased toward exact + 0-10 byte inputs plus medium syntax-heavy text unless `stress-long` is + selected explicitly) +- `formatting-adoption` (random formatting elements plus explicit + adoption-agency shapes: misnested closers, block-boundary formatting, + reconstruction across siblings, nested anchors, Noah's Ark overflow, + repeated closers, and four-plus active-formatting reconstruction clusters + covering same tag/no attributes, same tag/distinct attributes, same + tag/matching attributes, mixed interleavings, and table-cell marker + isolation/clearing) +- `attributes-entities` +- `comments-doctype-bogus` +- `deep-nesting` +- `resource-stress` +- `incomplete-malformed` (includes spec-special closers such as `
` and + `

`, stray closers, and ``) + +All profiles can emit duplicate attribute names (first-wins coverage), +auto-closing chains (`li`, `dd`/`dt`, headings, `p`), and named character +references with longest-prefix-match ambiguity (`¬it;`, `©right;`, +`&ngE`, ...). + +Terminal payloads are selected by a separate policy: + +- `valid-utf8`: structural cases with ASCII, Unicode, controls, and + entity references, including NUL byte coverage, but no raw invalid bytes. +- `mostly-valid`: default-biased structural cases with valid UTF-8 Unicode, + controls, NUL bytes, and entity references. +- `ascii-structural`: ASCII-only terminal text and attributes for tokenizer and + tree-construction coverage, including NUL byte coverage. +- `stress-long`: long valid UTF-8 terminal payloads for deliberate + resource-stress runs. +- `auto`: weighted choice. Normal structural profiles favor valid UTF-8 and + mostly-valid payloads; `resource-stress` favors `stress-long`. + +Use `--payload-policy POLICY` on `worker.php`, `runner.php`, or `launcher.php`. +Use `--max-input-bytes N` to apply a post-generation byte cap before the worker +records replay metadata. The cap preserves UTF-8 byte boundaries, but it is not +grammar-aware and may cut through HTML tokens. Replay manifests and minimization +summaries preserve the original payload policy when it was recorded; old or +hand-supplied inputs leave `payloadPolicy` null unless an explicit policy label +is provided. Historical `invalid-byte-heavy` labels are accepted only as replay +metadata for direct inputs and are not selectable for generated runs. +Replayed and minimized manifests keep immediate `inputSource` metadata separate +from `originalGenerator` metadata. + +## Tree Comparison + +The tree renderer follows the html5lib test style used by +`tests/phpunit/tests/html-api/wpHtmlProcessorHtml5lib.php`: + +- attributes sorted by their spec-scrubbed names (so a raw-NUL name on the + WordPress side and its U+FFFD substitution on the DOM side sort identically), + rendered raw +- boolean attributes rendered as `=""` +- namespace-qualified element and attribute names +- template `content` marker +- only the narrow auto-generated `html/head/body` wrapper tolerance + +When the default PHP DOM oracle is selected, template content is rendered +through a self-contained serialization round-trip: PHP hides template child +nodes, so the oracle re-parses the template's `innerHTML` serialization in a +body context and accepts the result only when re-serializing reproduces the +source byte-for-byte. Content that cannot round-trip (table parts, foreign +fragments) is quarantined as `oracle-unsupported`. This check never consults +the WordPress HTML API, which is the system under test. + +Raw bytes are rendered without normalization. The WordPress HTML API +deliberately preserves NUL and CR bytes where spec-following parsers +substitute U+FFFD and normalize newlines during input preprocessing, so the +comparison tolerates a differing line only when that exact substitution +explains the entire difference. The tolerance is additionally gated by line +type: WordPress preserves raw bytes only in attribute values and +tag/attribute names (verified empirically across text, RCDATA, rawtext, +foreign text, CDATA, comment, and doctype contexts, where WordPress applies +the spec substitutions itself), so only tag lines and attribute lines are +eligible. A scalar difference on any other line type is a real divergence +and fails. Tolerated lines are reported per seed +(`comparison.scalarToleratedLines`) and per run (`oracleTolerated`), and the +result is classified `oracle-tolerated` rather than silently passed. Any +difference beyond the substitution fails as usual, and the first-difference +record points at the first *unexplained* line. + +One known PHP DOM oracle bug is tolerated with a runtime probe: PHP's Lexbor parser +fails to treat U+000C FORM FEED as ignorable whitespace in the pre-body +insertion modes. When a full-document comparison fails, the input contains a +form feed, and re-parsing with form feeds substituted by spaces makes the DOM +oracle reproduce the WordPress tree exactly, the case is classified +`oracle-tolerated` with `comparison.formFeedQuirk: true`. The probe disables +the tolerance automatically when PHP fixes the bug. + +Invalid bytes are never normalized away. If WordPress and the selected oracle +surface different byte sequences, the first-difference record includes bounded +line previews, byte lengths, line hashes, the first differing byte offset, and +hex previews, including a diff-window hex preview around the differing byte, so +the mismatch remains inspectable even when JSON display substitutes replacement +characters. Full comparison lines are kept out of `result.json` to avoid large +artifacts from stress inputs. + +### Known classification gaps + +Two known issues affect labeling, not pass/fail correctness: + +- **Dual-axis lines classify as `tree-mismatch`, not `encoding-mismatch`.** + A differing line explained only by *both* an invalid-UTF-8 substitution and + a NUL/CR scalar substitution (e.g. an attribute value containing a raw NUL + *and* a raw `0x82`) matches neither single-axis check: + `linesMatchAfterWordPressUtf8Scrub` fails on the unscrubbed NUL, and the + scalar matcher fails on the invalid byte. Such results report + `tree-mismatch` although encoding is involved. +- **CDATA at SVG/MathML integration points is a real WordPress divergence + the fuzzer will keep reporting.** For + `a\0b` (likewise ``, + ``, HTML-encoded `annotation-xml`), WordPress substitutes + NUL with U+FFFD in CDATA text while the spec routes those characters + through the HTML insertion mode, which drops NUL — WordPress handles + plain text at integration points correctly; only the CDATA path diverges + (a known `@todo` in `WP_HTML_Tag_Processor`'s CDATA handling). The shape + is U+FFFD-versus-removed, which no tolerance covers in either direction, + so these report as genuine `tree-mismatch` findings. This is distinct + from the upstream-fixed PHP-DOM integration-point reparenting family. + +## Minimization + +`minimize.php` reduces in three phases under a shared attempt budget +(`--max-attempts`, default 600): markup-aligned segment deletion, binary +byte-chunk deletion, then per-byte deletion and canonicalization (replacements +never grow the input). Every accepted candidate re-runs the worker and must +reproduce the original signature hash (or any failure with `--any-failure`). diff --git a/tools/html-api-fuzz/codex-triage-orchestrator.php b/tools/html-api-fuzz/codex-triage-orchestrator.php new file mode 100755 index 0000000000000..17a62d6d6ea93 --- /dev/null +++ b/tools/html-api-fuzz/codex-triage-orchestrator.php @@ -0,0 +1,935 @@ +#!/usr/bin/env php + $value ) { + if ( '_' === $name ) { + if ( ! empty( $value ) ) { + throw new InvalidArgumentException( 'Unexpected positional argument: ' . (string) reset( $value ) ); + } + continue; + } + + if ( ! in_array( $name, $allowed, true ) ) { + throw new InvalidArgumentException( 'Unknown option: --' . $name ); + } + if ( in_array( $name, $value_options, true ) && true === $value ) { + throw new InvalidArgumentException( 'Expected --' . $name . ' to have a value.' ); + } + if ( in_array( $name, $bool_options, true ) && true !== $value ) { + throw new InvalidArgumentException( 'Option --' . $name . ' does not accept a value.' ); + } + } +} + +function html_api_fuzz_codex_is_absolute_path( string $path ): bool { + return 1 === preg_match( '#^(?:/|[A-Za-z]:[\\\\/]|\\\\\\\\)#', $path ); +} + +function html_api_fuzz_codex_now_iso_z(): string { + return gmdate( 'Y-m-d\TH:i:s\Z' ); +} + +function html_api_fuzz_codex_normalize_path( string $path ): string { + if ( '' === $path ) { + throw new InvalidArgumentException( 'Expected a non-empty path.' ); + } + + if ( ! html_api_fuzz_codex_is_absolute_path( $path ) ) { + $cwd = getcwd(); + if ( false === $cwd ) { + throw new RuntimeException( 'Could not determine current working directory.' ); + } + $path = $cwd . DIRECTORY_SEPARATOR . $path; + } + + $real = realpath( $path ); + return false === $real ? rtrim( $path, DIRECTORY_SEPARATOR ) : $real; +} + +function html_api_fuzz_codex_require_dir( string $path, string $label ): string { + $normalized = html_api_fuzz_codex_normalize_path( $path ); + if ( ! is_dir( $normalized ) ) { + throw new InvalidArgumentException( "{$label} is not a directory: {$normalized}" ); + } + + return $normalized; +} + +function html_api_fuzz_codex_path_join( string $base, string ...$parts ): string { + $path = rtrim( $base, DIRECTORY_SEPARATOR ); + foreach ( $parts as $part ) { + $path .= DIRECTORY_SEPARATOR . ltrim( $part, DIRECTORY_SEPARATOR ); + } + + return $path; +} + +function html_api_fuzz_codex_signature_dir_name( string $signature_hash ): string { + $dir = preg_replace( '/[^a-zA-Z0-9._-]+/', '_', $signature_hash ); + return null === $dir || '' === $dir ? '_' : $dir; +} + +function html_api_fuzz_codex_valid_signature_hash( string $signature_hash ): bool { + return 1 === preg_match( '/^[A-Za-z0-9._-]+$/', $signature_hash ); +} + +function html_api_fuzz_codex_safe_prompt_line( $value, string $field ): string { + if ( null === $value ) { + return ''; + } + if ( ! is_scalar( $value ) ) { + throw new InvalidArgumentException( "Expected {$field} to be scalar." ); + } + + $text = (string) $value; + if ( 1 === preg_match( '/[\x00-\x1F\x7F]/', $text ) ) { + throw new InvalidArgumentException( "Unsafe control character in {$field}." ); + } + + return $text; +} + +function html_api_fuzz_codex_path_has_control_chars( string $path ): bool { + return 1 === preg_match( '/[\x00-\x1F\x7F]/', $path ); +} + +function html_api_fuzz_codex_path_is_under( string $path, string $root ): bool { + $real_path = realpath( $path ); + $real_root = realpath( $root ); + if ( false === $real_path || false === $real_root ) { + return false; + } + + $real_root = rtrim( $real_root, DIRECTORY_SEPARATOR ); + return $real_path === $real_root || 0 === strpos( $real_path, $real_root . DIRECTORY_SEPARATOR ); +} + +function html_api_fuzz_codex_allowed_artifact_roots( array $args ): array { + $roots = array( + $args['triageDir'], + $args['runDir'] ?? dirname( $args['triageDir'] ), + ); + $allowed = array(); + foreach ( $roots as $root ) { + if ( is_string( $root ) && '' !== $root && is_dir( $root ) && ! html_api_fuzz_codex_path_has_control_chars( $root ) ) { + $real = realpath( $root ); + if ( false !== $real ) { + $allowed[] = $real; + } + } + } + + return array_values( array_unique( $allowed ) ); +} + +function html_api_fuzz_codex_sort_json_value( $value ) { + if ( ! is_array( $value ) ) { + return $value; + } + + if ( array_is_list( $value ) ) { + return array_map( 'html_api_fuzz_codex_sort_json_value', $value ); + } + + foreach ( $value as $key => $item ) { + $value[ $key ] = html_api_fuzz_codex_sort_json_value( $item ); + } + ksort( $value, SORT_STRING ); + + return $value; +} + +function html_api_fuzz_codex_json_encode( $value ): string { + $json = json_encode( + html_api_fuzz_codex_sort_json_value( $value ), + JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_SUBSTITUTE + ); + if ( false === $json ) { + throw new RuntimeException( 'JSON encode failed: ' . json_last_error_msg() ); + } + + return $json; +} + +function html_api_fuzz_codex_read_json( string $path, int $retries = 5 ) { + $last_error = null; + for ( $i = 0; $i < $retries; ++$i ) { + if ( ! file_exists( $path ) ) { + return null; + } + + $text = @file_get_contents( $path ); + if ( false === $text ) { + throw new RuntimeException( "Could not read JSON file: {$path}" ); + } + + $value = json_decode( $text, true ); + if ( JSON_ERROR_NONE === json_last_error() ) { + return $value; + } + + $last_error = json_last_error_msg(); + usleep( 200000 ); + } + + throw new RuntimeException( "Could not parse JSON {$path}: {$last_error}" ); +} + +function html_api_fuzz_codex_exit_code_from_done( $done ): int { + if ( ! is_array( $done ) || ! array_key_exists( 'exitCode', $done ) || ! is_numeric( $done['exitCode'] ) ) { + return 1; + } + + return (int) $done['exitCode']; +} + +function html_api_fuzz_codex_write_json( string $path, $value ): void { + $dir = dirname( $path ); + \HtmlApiFuzz\ensure_dir( $dir ); + $tmp = tempnam( $dir, basename( $path ) . '.tmp.' ); + if ( false === $tmp ) { + throw new RuntimeException( "Could not create temporary JSON file for: {$path}" ); + } + $json = html_api_fuzz_codex_json_encode( $value ) . "\n"; + if ( false === file_put_contents( $tmp, $json ) ) { + @unlink( $tmp ); + throw new RuntimeException( "Could not write temporary JSON file: {$tmp}" ); + } + if ( ! rename( $tmp, $path ) ) { + @unlink( $tmp ); + throw new RuntimeException( "Could not replace JSON file: {$path}" ); + } +} + +function html_api_fuzz_codex_archive_path( string $dir, string $prefix ): string { + $suffix = (string) time(); + $path = html_api_fuzz_codex_path_join( $dir, "{$prefix}.{$suffix}.json" ); + for ( $i = 1; file_exists( $path ); ++$i ) { + $path = html_api_fuzz_codex_path_join( $dir, "{$prefix}.{$suffix}.{$i}.json" ); + } + + return $path; +} + +function html_api_fuzz_codex_process_alive( int $pid ): bool { + if ( $pid < 1 || ! function_exists( 'posix_kill' ) ) { + return false; + } + + if ( @posix_kill( $pid, 0 ) ) { + return true; + } + + if ( function_exists( 'posix_get_last_error' ) && defined( 'POSIX_EPERM' ) ) { + return POSIX_EPERM === posix_get_last_error(); + } + + return false; +} + +function html_api_fuzz_codex_newest_minimize_dir( array $signature ): ?string { + $output_dir = $signature['minimizeOutputDir'] ?? null; + if ( is_string( $output_dir ) && '' !== $output_dir && is_dir( $output_dir ) ) { + return $output_dir; + } + + $result_path = $signature['minimizeResult'] ?? null; + if ( is_string( $result_path ) && '' !== $result_path && is_file( $result_path ) ) { + return dirname( $result_path ); + } + + return null; +} + +function html_api_fuzz_codex_claim_signature( string $diagnostics_dir, string $signature_hash, array $signature, int $stale_after_seconds ): ?string { + $signature_dir = html_api_fuzz_codex_path_join( $diagnostics_dir, html_api_fuzz_codex_signature_dir_name( $signature_hash ) ); + $done_path = html_api_fuzz_codex_path_join( $signature_dir, 'done.json' ); + $claim_path = html_api_fuzz_codex_path_join( $signature_dir, 'claim.json' ); + + if ( is_file( $done_path ) ) { + $done = html_api_fuzz_codex_read_json( $done_path ); + $exit_code = html_api_fuzz_codex_exit_code_from_done( $done ); + if ( 0 === $exit_code ) { + return null; + } + + if ( ! rename( $done_path, html_api_fuzz_codex_archive_path( $signature_dir, 'done.failed' ) ) && is_file( $done_path ) ) { + throw new RuntimeException( "Could not archive failed done file: {$done_path}" ); + } + if ( is_file( $claim_path ) && ! rename( $claim_path, html_api_fuzz_codex_archive_path( $signature_dir, 'claim.failed' ) ) && is_file( $claim_path ) ) { + throw new RuntimeException( "Could not archive failed claim file: {$claim_path}" ); + } + } + + if ( is_file( $claim_path ) ) { + $claim = html_api_fuzz_codex_read_json( $claim_path ) ?: array(); + $pid = (int) ( $claim['pid'] ?? 0 ); + $claimed_at = (float) ( $claim['claimedAtUnix'] ?? 0 ); + $stale = microtime( true ) - $claimed_at > $stale_after_seconds; + if ( ! $stale ) { + if ( $pid > 0 && html_api_fuzz_codex_process_alive( $pid ) ) { + return null; + } + return null; + } + if ( ! rename( $claim_path, html_api_fuzz_codex_archive_path( $signature_dir, 'claim.stale' ) ) && is_file( $claim_path ) ) { + throw new RuntimeException( "Could not archive stale claim file: {$claim_path}" ); + } + } + + \HtmlApiFuzz\ensure_dir( $signature_dir ); + $claim = array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-codex-claim', + 'hash' => $signature_hash, + 'pid' => getmypid(), + 'claimedAt' => html_api_fuzz_codex_now_iso_z(), + 'claimedAtUnix' => microtime( true ), + 'failureClass' => $signature['failureClass'] ?? null, + 'sourceReplayPath' => $signature['replayPath'] ?? null, + 'sourceResultPath' => $signature['resultPath'] ?? null, + ); + + $handle = @fopen( $claim_path, 'xb' ); + if ( false === $handle ) { + if ( is_file( $claim_path ) ) { + return null; + } + throw new RuntimeException( "Could not create claim file: {$claim_path}" ); + } + + $bytes = html_api_fuzz_codex_json_encode( $claim ) . "\n"; + $write = fwrite( $handle, $bytes ); + fclose( $handle ); + if ( false === $write || $write !== strlen( $bytes ) ) { + throw new RuntimeException( "Could not write claim file: {$claim_path}" ); + } + + return $signature_dir; +} + +function html_api_fuzz_codex_existing_paths( array $paths, array $allowed_roots ): array { + $existing = array(); + foreach ( $paths as $path ) { + if ( ! is_string( $path ) || '' === $path || html_api_fuzz_codex_path_has_control_chars( $path ) || ! file_exists( $path ) ) { + continue; + } + + foreach ( $allowed_roots as $root ) { + if ( html_api_fuzz_codex_path_is_under( $path, $root ) ) { + $real = realpath( $path ); + if ( false !== $real ) { + $existing[] = $real; + } + continue 2; + } + } + } + + return array_values( array_unique( $existing ) ); +} + +function html_api_fuzz_codex_prompt_for_signature( array $args, string $signature_hash, array $signature ): string { + if ( ! html_api_fuzz_codex_valid_signature_hash( $signature_hash ) ) { + throw new InvalidArgumentException( 'Unsafe signature hash: ' . $signature_hash ); + } + + $signature_dir_name = html_api_fuzz_codex_signature_dir_name( $signature_hash ); + $signature_triage_dir = html_api_fuzz_codex_path_join( $args['triageDir'], 'signatures', $signature_dir_name ); + $minimize_dir = html_api_fuzz_codex_newest_minimize_dir( $signature ); + $minimized_dir = null === $minimize_dir ? null : html_api_fuzz_codex_path_join( $minimize_dir, 'minimized' ); + $report_path = html_api_fuzz_codex_path_join( $args['diagnosticsDir'], $signature_dir_name, 'report.md' ); + + $paths = array( + html_api_fuzz_codex_path_join( $signature_triage_dir, 'failure.json' ), + is_string( $signature['minimizeResult'] ?? null ) ? $signature['minimizeResult'] : null, + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'result.json' ), + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'replay.json' ), + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'wordpress-tree.txt' ), + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'dom-tree.txt' ), + is_string( $signature['replayPath'] ?? null ) ? $signature['replayPath'] : null, + is_string( $signature['resultPath'] ?? null ) ? $signature['resultPath'] : null, + ); + $existing_paths = html_api_fuzz_codex_existing_paths( $paths, html_api_fuzz_codex_allowed_artifact_roots( $args ) ); + + if ( 'classify' === $args['mode'] ) { + $fix_policy = 'This is classify-only mode. Do not edit files. Do not run write commands. Return the Markdown report as your final response; the Codex CLI will save it. If you believe a fix is needed, describe the smallest fix and the focused verification.'; + } else { + $fix_policy = 'You may implement the smallest justified fix. Preserve the minimized replay and run focused verification.'; + } + + $artifact_lines = implode( "\n", array_map( static fn( string $path ): string => '- ' . $path, $existing_paths ) ); + $classification_lines = implode( "\n", array_map( static fn( string $classification ): string => '- ' . $classification, HTML_API_FUZZ_CODEX_CLASSIFICATIONS ) ); + $failure_class = html_api_fuzz_codex_safe_prompt_line( $signature['failureClass'] ?? '', 'failureClass' ); + $triage_kind = html_api_fuzz_codex_safe_prompt_line( $signature['triageKind'] ?? 'failure', 'triageKind' ); + $oracle_type = html_api_fuzz_codex_safe_prompt_line( $signature['oracleFindingType'] ?? '', 'oracleFindingType' ); + $suspected_owner = html_api_fuzz_codex_safe_prompt_line( $signature['suspectedOwner'] ?? '', 'suspectedOwner' ); + $upstream_issue = html_api_fuzz_codex_safe_prompt_line( $signature['upstreamIssueUrl'] ?? '', 'upstreamIssueUrl' ); + + return << 1, + 'kind' => 'html-api-fuzz-codex-job', + 'hash' => $signature_hash, + 'startedAt' => html_api_fuzz_codex_now_iso_z(), + 'startedAtUnix' => microtime( true ), + 'mode' => $args['mode'], + 'sandbox' => $args['sandbox'], + 'command' => $command, + 'processCommand' => $process_command, + 'promptPath' => $prompt_path, + 'stdoutPath' => $stdout_path, + 'stderrPath' => $stderr_path, + 'reportPath' => $report_path, + 'donePath' => $done_path, + ); + html_api_fuzz_codex_write_json( html_api_fuzz_codex_path_join( $signature_dir, 'job.json' ), $metadata ); + + $descriptor_spec = array( + 0 => array( 'pipe', 'r' ), + 1 => array( 'file', $stdout_path, 'w' ), + 2 => array( 'file', $stderr_path, 'w' ), + ); + $process = proc_open( $process_command, $descriptor_spec, $pipes, $args['repoRoot'] ); + if ( ! is_resource( $process ) ) { + throw new RuntimeException( 'Could not start Codex subprocess.' ); + } + + try { + html_api_fuzz_codex_write_stream( $pipes[0], $prompt ); + fclose( $pipes[0] ); + } catch ( Throwable $error ) { + if ( is_resource( $pipes[0] ) ) { + fclose( $pipes[0] ); + } + proc_terminate( $process ); + proc_close( $process ); + throw $error; + } + $status = proc_get_status( $process ); + + return array( + 'process' => $process, + 'pid' => (int) ( $status['pid'] ?? 0 ), + 'processGroupPid' => $uses_process_group ? (int) ( $status['pid'] ?? 0 ) : null, + 'knownExitCode' => ( ! ( $status['running'] ?? false ) && array_key_exists( 'exitcode', $status ) && -1 !== $status['exitcode'] ) ? (int) $status['exitcode'] : null, + 'signatureDir' => $signature_dir, + 'hash' => $signature_hash, + ); +} + +function html_api_fuzz_codex_poll_exit_code( array &$job ): ?int { + if ( null !== ( $job['knownExitCode'] ?? null ) ) { + $exit_code = (int) $job['knownExitCode']; + proc_close( $job['process'] ); + return $exit_code; + } + + $status = proc_get_status( $job['process'] ); + if ( $status['running'] ?? false ) { + return null; + } + + $exit_code = null; + if ( array_key_exists( 'exitcode', $status ) && -1 !== $status['exitcode'] ) { + $exit_code = (int) $status['exitcode']; + } elseif ( array_key_exists( 'cached_exitcode', $status ) && false !== $status['cached_exitcode'] ) { + $exit_code = (int) $status['cached_exitcode']; + } + + $closed_code = proc_close( $job['process'] ); + if ( null === $exit_code ) { + $exit_code = (int) $closed_code; + } + + return $exit_code; +} + +function html_api_fuzz_codex_mark_done( array $job, int $exit_code, array $extra = array() ): void { + $signature_dir = $job['signatureDir']; + $signature_hash = $job['hash']; + $done = array_merge( + array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-codex-done', + 'hash' => $signature_hash, + 'finishedAt' => html_api_fuzz_codex_now_iso_z(), + 'finishedAtUnix' => microtime( true ), + 'exitCode' => $exit_code, + 'reportPath' => html_api_fuzz_codex_path_join( $signature_dir, 'report.md' ), + 'stdoutPath' => html_api_fuzz_codex_path_join( $signature_dir, 'codex.jsonl' ), + 'stderrPath' => html_api_fuzz_codex_path_join( $signature_dir, 'codex.stderr.log' ), + ), + $extra + ); + html_api_fuzz_codex_write_json( html_api_fuzz_codex_path_join( $signature_dir, 'done.json' ), $done ); +} + +function html_api_fuzz_codex_mark_launch_failed( string $signature_dir, string $signature_hash, Throwable $error ): void { + html_api_fuzz_codex_mark_done( + array( + 'signatureDir' => $signature_dir, + 'hash' => $signature_hash, + ), + 1, + array( + 'launchFailed' => true, + 'errorClass' => get_class( $error ), + 'errorMessage' => $error->getMessage(), + ) + ); +} + +function html_api_fuzz_codex_load_candidate_signatures( array $state ): array { + $signatures = $state['signatures'] ?? array(); + if ( ! is_array( $signatures ) ) { + return array(); + } + + ksort( $signatures, SORT_STRING ); + $candidates = array(); + foreach ( $signatures as $signature_hash => $signature ) { + if ( ! is_array( $signature ) ) { + continue; + } + $signature_hash = (string) $signature_hash; + if ( ! html_api_fuzz_codex_valid_signature_hash( $signature_hash ) ) { + continue; + } + if ( 'minimized' !== ( $signature['status'] ?? null ) ) { + continue; + } + if ( null === html_api_fuzz_codex_newest_minimize_dir( $signature ) ) { + continue; + } + $candidates[] = array( $signature_hash, $signature ); + } + + return $candidates; +} + +function html_api_fuzz_codex_parse_args( array $argv ): array { + if ( in_array( '-h', $argv, true ) ) { + html_api_fuzz_codex_usage(); + exit( 0 ); + } + + $options = \HtmlApiFuzz\parse_cli_options( $argv ); + html_api_fuzz_codex_validate_cli_options( $options ); + if ( \HtmlApiFuzz\option_bool( $options, 'help', false ) ) { + html_api_fuzz_codex_usage(); + exit( 0 ); + } + + $triage_dir = \HtmlApiFuzz\option_string( $options, 'triage-dir' ); + if ( null === $triage_dir ) { + html_api_fuzz_codex_usage(); + exit( 1 ); + } + + $mode = \HtmlApiFuzz\option_string( $options, 'mode', 'classify' ); + if ( ! in_array( $mode, array( 'classify', 'fix' ), true ) ) { + throw new InvalidArgumentException( 'Expected --mode to be classify or fix.' ); + } + + $sandbox = \HtmlApiFuzz\option_string( $options, 'sandbox', 'fix' === $mode ? 'workspace-write' : 'read-only' ); + if ( ! in_array( $sandbox, array( 'read-only', 'workspace-write', 'danger-full-access' ), true ) ) { + throw new InvalidArgumentException( 'Expected --sandbox to be read-only, workspace-write, or danger-full-access.' ); + } + if ( 'fix' === $mode && 'read-only' === $sandbox ) { + throw new InvalidArgumentException( 'Expected --mode fix to use a writable sandbox.' ); + } + + $repo_root = html_api_fuzz_codex_require_dir( \HtmlApiFuzz\option_string( $options, 'repo-root', getcwd() ?: '.' ), 'Repo root' ); + $triage_dir = html_api_fuzz_codex_require_dir( $triage_dir, 'Triage directory' ); + $diagnostics_dir = \HtmlApiFuzz\option_string( $options, 'diagnostics-dir' ); + if ( null === $diagnostics_dir ) { + $diagnostics_dir = html_api_fuzz_codex_path_join( dirname( $triage_dir ), 'diagnostics' ); + } + + $interval_seconds = \HtmlApiFuzz\option_float( $options, 'interval-seconds', 120.0 ); + $max_concurrent = \HtmlApiFuzz\option_int( $options, 'max-concurrent', 1 ); + $max_launch_per_pass = \HtmlApiFuzz\option_int( $options, 'max-launch-per-pass', 1 ); + $stale_after_seconds = \HtmlApiFuzz\option_int( $options, 'stale-after-seconds', 6 * 60 * 60 ); + if ( $interval_seconds < 0 ) { + throw new InvalidArgumentException( 'Expected --interval-seconds to be at least 0.' ); + } + if ( $max_concurrent < 0 ) { + throw new InvalidArgumentException( 'Expected --max-concurrent to be at least 0.' ); + } + if ( $max_launch_per_pass < 0 ) { + throw new InvalidArgumentException( 'Expected --max-launch-per-pass to be at least 0.' ); + } + if ( $stale_after_seconds < 0 ) { + throw new InvalidArgumentException( 'Expected --stale-after-seconds to be at least 0.' ); + } + + return array( + 'triageDir' => $triage_dir, + 'diagnosticsDir' => html_api_fuzz_codex_normalize_path( $diagnostics_dir ), + 'repoRoot' => $repo_root, + 'codexBin' => \HtmlApiFuzz\option_string( $options, 'codex-bin', 'codex' ), + 'model' => \HtmlApiFuzz\option_string( $options, 'model' ), + 'intervalSeconds' => $interval_seconds, + 'maxConcurrent' => $max_concurrent, + 'maxLaunchPerPass' => $max_launch_per_pass, + 'staleAfterSeconds' => $stale_after_seconds, + 'once' => \HtmlApiFuzz\option_bool( $options, 'once', false ), + 'mode' => $mode, + 'sandbox' => $sandbox, + ); +} + +function html_api_fuzz_codex_install_signal_handlers( bool &$stopping ): void { + if ( ! function_exists( 'pcntl_signal' ) ) { + throw new RuntimeException( 'The pcntl extension is required for signal-safe Codex job orchestration.' ); + } + + if ( function_exists( 'pcntl_async_signals' ) ) { + pcntl_async_signals( true ); + } + if ( defined( 'SIGTERM' ) ) { + pcntl_signal( + SIGTERM, + static function () use ( &$stopping ): void { + $stopping = true; + } + ); + } + if ( defined( 'SIGINT' ) ) { + pcntl_signal( + SIGINT, + static function () use ( &$stopping ): void { + $stopping = true; + } + ); + } +} + +function html_api_fuzz_codex_dispatch_signals(): void { + if ( function_exists( 'pcntl_signal_dispatch' ) ) { + pcntl_signal_dispatch(); + } +} + +function html_api_fuzz_codex_sleep( float $seconds, bool &$stopping ): void { + $deadline = microtime( true ) + max( 0.0, $seconds ); + do { + html_api_fuzz_codex_dispatch_signals(); + if ( $stopping ) { + return; + } + + $remaining = $deadline - microtime( true ); + if ( $remaining <= 0 ) { + return; + } + + usleep( (int) min( 250000, max( 1000, round( $remaining * 1000000 ) ) ) ); + } while ( true ); +} + +function html_api_fuzz_codex_stop_job( array &$job ): void { + $status = proc_get_status( $job['process'] ); + if ( $status['running'] ?? false ) { + if ( null !== ( $job['processGroupPid'] ?? null ) && function_exists( 'posix_kill' ) && defined( 'SIGTERM' ) ) { + @posix_kill( -1 * (int) $job['processGroupPid'], SIGTERM ); + } else { + proc_terminate( $job['process'] ); + } + usleep( 200000 ); + $status = proc_get_status( $job['process'] ); + if ( $status['running'] ?? false ) { + if ( null !== ( $job['processGroupPid'] ?? null ) && function_exists( 'posix_kill' ) ) { + @posix_kill( -1 * (int) $job['processGroupPid'], 9 ); + } else { + proc_terminate( $job['process'], 9 ); + } + } + } + proc_close( $job['process'] ); + html_api_fuzz_codex_mark_done( + $job, + 143, + array( + 'interrupted' => true, + ) + ); +} + +function html_api_fuzz_codex_main( array $argv ): int { + $args = html_api_fuzz_codex_parse_args( $argv ); + $state_path = html_api_fuzz_codex_path_join( $args['triageDir'], 'state.json' ); + $running = array(); + $stopping = false; + $scanned = false; + + html_api_fuzz_codex_install_signal_handlers( $stopping ); + \HtmlApiFuzz\ensure_dir( $args['diagnosticsDir'] ); + html_api_fuzz_codex_write_json( + html_api_fuzz_codex_path_join( $args['diagnosticsDir'], 'orchestrator-state.json' ), + array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-codex-orchestrator-state', + 'startedAt' => html_api_fuzz_codex_now_iso_z(), + 'triageDir' => $args['triageDir'], + 'repoRoot' => $args['repoRoot'], + 'mode' => $args['mode'], + 'sandbox' => $args['sandbox'], + ) + ); + + try { + while ( ! $stopping ) { + html_api_fuzz_codex_dispatch_signals(); + foreach ( array_keys( $running ) as $key ) { + $exit_code = html_api_fuzz_codex_poll_exit_code( $running[ $key ] ); + if ( null === $exit_code ) { + continue; + } + + html_api_fuzz_codex_mark_done( $running[ $key ], $exit_code ); + echo 'completed ' . $running[ $key ]['hash'] . ' exit=' . $exit_code . "\n"; + unset( $running[ $key ] ); + } + + /* + * Graceful stop: a STOP file in the run directory stops new + * launches; running jobs finish before the orchestrator exits. + * The run directory comes from the watcher state when available + * (the triage dir may live outside the run dir), matching how + * launches resolve it below. + */ + $stop_run_dir = dirname( $args['triageDir'] ); + $stop_state = html_api_fuzz_codex_read_json( $state_path ); + if ( is_array( $stop_state ) && is_string( $stop_state['runDir'] ?? null ) && ! html_api_fuzz_codex_path_has_control_chars( $stop_state['runDir'] ) ) { + $stop_run_dir = html_api_fuzz_codex_normalize_path( $stop_state['runDir'] ); + } + $stop_file_present = is_file( html_api_fuzz_codex_path_join( $stop_run_dir, 'STOP' ) ); + if ( $stop_file_present && empty( $running ) ) { + echo "stop requested; exiting\n"; + break; + } + + $can_scan = ( ! $args['once'] || ! $scanned ) && ! $stop_file_present; + if ( $can_scan ) { + $available = max( 0, $args['maxConcurrent'] - count( $running ) ); + $launched = 0; + if ( $available > 0 ) { + $state = html_api_fuzz_codex_read_json( $state_path ); + if ( is_array( $state ) ) { + $launch_args = $args; + if ( is_string( $state['runDir'] ?? null ) && ! html_api_fuzz_codex_path_has_control_chars( $state['runDir'] ) ) { + $launch_args['runDir'] = html_api_fuzz_codex_normalize_path( $state['runDir'] ); + } else { + $launch_args['runDir'] = dirname( $args['triageDir'] ); + } + + foreach ( html_api_fuzz_codex_load_candidate_signatures( $state ) as $candidate ) { + if ( $launched >= $available || $launched >= $args['maxLaunchPerPass'] ) { + break; + } + list( $signature_hash, $signature ) = $candidate; + $signature_dir = html_api_fuzz_codex_claim_signature( $args['diagnosticsDir'], $signature_hash, $signature, $args['staleAfterSeconds'] ); + if ( null === $signature_dir ) { + continue; + } + + try { + $job = html_api_fuzz_codex_launch( $launch_args, $signature_hash, $signature, $signature_dir ); + $running[] = $job; + ++$launched; + echo 'launched ' . $signature_hash . ' pid=' . $job['pid'] . "\n"; + } catch ( Throwable $error ) { + html_api_fuzz_codex_mark_launch_failed( $signature_dir, $signature_hash, $error ); + echo 'launch failed ' . $signature_hash . ' error=' . $error->getMessage() . "\n"; + } + } + } + } + $scanned = true; + } + + if ( $args['once'] && empty( $running ) ) { + break; + } + + $sleep_seconds = $args['once'] ? min( 1.0, $args['intervalSeconds'] ) : $args['intervalSeconds']; + html_api_fuzz_codex_sleep( $sleep_seconds, $stopping ); + } + } finally { + foreach ( array_keys( $running ) as $key ) { + html_api_fuzz_codex_stop_job( $running[ $key ] ); + unset( $running[ $key ] ); + } + } + + return 0; +} + +if ( ! defined( 'HTML_API_FUZZ_CODEX_SELF_TESTING' ) || ! HTML_API_FUZZ_CODEX_SELF_TESTING ) { + exit( html_api_fuzz_codex_main( $argv ) ); +} diff --git a/tools/html-api-fuzz/commoncrawl-analysis.php b/tools/html-api-fuzz/commoncrawl-analysis.php new file mode 100644 index 0000000000000..19e91a5d1cc12 --- /dev/null +++ b/tools/html-api-fuzz/commoncrawl-analysis.php @@ -0,0 +1,14 @@ +analyze_document( $document ); +}; diff --git a/tools/html-api-fuzz/launcher.php b/tools/html-api-fuzz/launcher.php new file mode 100755 index 0000000000000..6eadad14e6a53 --- /dev/null +++ b/tools/html-api-fuzz/launcher.php @@ -0,0 +1,359 @@ +#!/usr/bin/env php + array( 'pipe', 'r' ), + 1 => array( 'pipe', 'w' ), + 2 => array( 'pipe', 'w' ), + ); + + $process = proc_open( $command, $spec, $pipes, $cwd ); + if ( ! is_resource( $process ) ) { + throw new RuntimeException( 'Could not start runner lane.' ); + } + + fclose( $pipes[0] ); + stream_set_blocking( $pipes[1], false ); + stream_set_blocking( $pipes[2], false ); + \HtmlApiFuzz\ensure_dir( dirname( $log_path ) ); + file_put_contents( $log_path, '' ); + + return array( + 'process' => $process, + 'pipes' => $pipes, + 'logPath' => $log_path, + 'startedAt' => microtime( true ), + 'stdout' => '', + 'stderr' => '', + ); +} + +function html_api_fuzz_launcher_drain_lane( array &$lane ): void { + $stdout = stream_get_contents( $lane['pipes'][1] ); + $stderr = stream_get_contents( $lane['pipes'][2] ); + if ( '' !== $stdout ) { + $lane['stdout'] .= $stdout; + file_put_contents( $lane['logPath'], $stdout, FILE_APPEND ); + } + if ( '' !== $stderr ) { + $lane['stderr'] .= $stderr; + file_put_contents( $lane['logPath'], $stderr, FILE_APPEND ); + } +} + +function html_api_fuzz_launcher_close_lane( array &$lane ): array { + html_api_fuzz_launcher_drain_lane( $lane ); + fclose( $lane['pipes'][1] ); + fclose( $lane['pipes'][2] ); + $code = proc_close( $lane['process'] ); + + return array( + 'code' => $code, + 'durationMs' => (int) round( ( microtime( true ) - $lane['startedAt'] ) * 1000 ), + 'stdoutTail' => substr( $lane['stdout'], -2000 ), + 'stderrTail' => substr( $lane['stderr'], -2000 ), + ); +} + +$options = \HtmlApiFuzz\parse_cli_options( $argv ); +if ( \HtmlApiFuzz\option_bool( $options, 'help', false ) || \HtmlApiFuzz\option_bool( $options, 'h', false ) ) { + html_api_fuzz_launcher_usage(); + exit( 0 ); +} + +$repo_root = \HtmlApiFuzz\repo_root(); +$output_dir = \HtmlApiFuzz\option_string( $options, 'output-dir', $repo_root . '/artifacts/html-api-fuzz/launch-' . \HtmlApiFuzz\timestamp() ); +$lanes = max( 1, \HtmlApiFuzz\option_int( $options, 'lanes', 2 ) ); +$start_seed = \HtmlApiFuzz\option_int( $options, 'start-seed', 1 ); +$max_seeds = \HtmlApiFuzz\option_int( $options, 'max-seeds', 0 ); +$duration_seconds = \HtmlApiFuzz\option_float( $options, 'duration-seconds', 60.0 ); +$timeout_ms = \HtmlApiFuzz\option_int( $options, 'timeout-ms', 2500 ); +$profile = \HtmlApiFuzz\option_string( $options, 'profile', 'auto' ); +$mode = \HtmlApiFuzz\option_string( $options, 'mode', 'auto' ); +$payload_policy = \HtmlApiFuzz\option_string( $options, 'payload-policy', 'auto' ); +$max_input_bytes = \HtmlApiFuzz\option_int( $options, 'max-input-bytes', 0 ); +$max_tokens = \HtmlApiFuzz\option_int( $options, 'max-tokens', 2000 ); +$max_nodes = \HtmlApiFuzz\option_int( $options, 'max-nodes', 3000 ); +$stop_on_failure = \HtmlApiFuzz\option_bool( $options, 'stop-on-failure', false ); +$fail_unsupported = \HtmlApiFuzz\option_bool( $options, 'fail-unsupported', false ); +$run_watcher = \HtmlApiFuzz\option_bool( $options, 'watcher', false ); +$triage_oracle_findings = \HtmlApiFuzz\option_bool( $options, 'triage-oracle-findings', false ); +$max_keep_per_signature = \HtmlApiFuzz\option_int( $options, 'max-keep-per-signature', 5 ); +$keep_all_artifacts = \HtmlApiFuzz\option_bool( $options, 'keep-all-artifacts', false ); +if ( $max_keep_per_signature < 1 ) { + throw new InvalidArgumentException( 'Expected --max-keep-per-signature to be at least 1.' ); +} +html_api_fuzz_launcher_validate_generator_options( $profile, $mode, $payload_policy ); +html_api_fuzz_launcher_validate_runtime_options( $max_seeds, $duration_seconds, $timeout_ms, $max_input_bytes, $max_tokens, $max_nodes ); + +if ( is_file( $output_dir . '/STOP' ) ) { + // A leftover stop request must not silently turn this launch into a + // 0-seed success; starting again is an explicit operator decision. + fwrite( STDERR, "Stop file already exists: {$output_dir}/STOP\nRemove it to start a run in this directory.\n" ); + exit( 1 ); +} + +\HtmlApiFuzz\ensure_dir( $output_dir ); +$events_path = $output_dir . '/events.ndjson'; +$state_path = $output_dir . '/launcher-state.json'; +$git_metadata = null === \HtmlApiFuzz\option_string( $options, 'git-metadata-base64', null ) + ? \HtmlApiFuzz\git_metadata() + : \HtmlApiFuzz\git_metadata_from_base64( \HtmlApiFuzz\option_string( $options, 'git-metadata-base64' ) ); +$git_metadata_base64 = \HtmlApiFuzz\git_metadata_base64( $git_metadata ); +$oracle_renderer = \HtmlApiFuzz\OracleRenderer::from_options( $options ); +$oracle_metadata = $oracle_renderer->metadata(); +$oracle_worker_args = $oracle_renderer->worker_args(); + +$state = array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-launcher-state', + 'startedAt' => gmdate( 'c' ), + 'updatedAt' => gmdate( 'c' ), + 'outputDir' => $output_dir, + 'lanes' => $lanes, + 'startSeed' => $start_seed, + 'seedStride' => $lanes, + 'profile' => $profile, + 'mode' => $mode, + 'payloadPolicy' => $payload_policy, + 'maxInputBytes' => $max_input_bytes > 0 ? $max_input_bytes : null, + 'git' => $git_metadata, + 'oracle' => $oracle_metadata, + 'finished' => false, + 'laneResults' => array(), +); +\HtmlApiFuzz\write_json_file( $state_path, $state ); +\HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'launcher-start', 'outputDir' => $output_dir, 'lanes' => $lanes, 'git' => $git_metadata, 'oracle' => $oracle_metadata ) ); + +$running = array(); +for ( $i = 0; $i < $lanes; ++$i ) { + $lane_max_seeds = 0; + if ( 0 !== $max_seeds ) { + $lane_max_seeds = intdiv( $max_seeds, $lanes ) + ( $i < ( $max_seeds % $lanes ) ? 1 : 0 ); + if ( 0 === $lane_max_seeds ) { + $lane_dir = $output_dir . '/lane-' . str_pad( (string) $i, 2, '0', STR_PAD_LEFT ); + $state['laneResults'][ $i ] = array( + 'lane' => $i, + 'status' => 'skipped', + 'outputDir' => $lane_dir, + 'reason' => 'no seeds assigned', + ); + continue; + } + } + + $lane_dir = $output_dir . '/lane-' . str_pad( (string) $i, 2, '0', STR_PAD_LEFT ); + $command = array( + PHP_BINARY, + __DIR__ . '/runner.php', + '--output-dir', + $lane_dir, + '--start-seed', + (string) ( $start_seed + $i ), + '--seed-stride', + (string) $lanes, + '--duration-seconds', + (string) $duration_seconds, + '--timeout-ms', + (string) $timeout_ms, + '--profile', + $profile, + '--mode', + $mode, + '--payload-policy', + $payload_policy, + '--max-tokens', + (string) $max_tokens, + '--max-nodes', + (string) $max_nodes, + '--git-metadata-base64', + $git_metadata_base64, + '--max-keep-per-signature', + (string) $max_keep_per_signature, + '--stop-file', + $output_dir . '/STOP', + ); + foreach ( $oracle_worker_args as $arg ) { + $command[] = $arg; + } + + if ( 0 !== $max_seeds ) { + $command[] = '--max-seeds'; + $command[] = (string) $lane_max_seeds; + } + if ( $keep_all_artifacts ) { + $command[] = '--keep-all-artifacts'; + } + if ( $stop_on_failure ) { + $command[] = '--stop-on-failure'; + } + if ( $fail_unsupported ) { + $command[] = '--fail-unsupported'; + } + if ( $max_input_bytes > 0 ) { + $command[] = '--max-input-bytes'; + $command[] = (string) $max_input_bytes; + } + + $running[ $i ] = html_api_fuzz_launcher_start_lane( $command, $repo_root, $lane_dir . '/runner.stdout.log' ); + $state['laneResults'][ $i ] = array( + 'lane' => $i, + 'status' => 'running', + 'command' => \HtmlApiFuzz\command_string( $command ), + 'outputDir' => $lane_dir, + 'logPath' => $lane_dir . '/runner.stdout.log', + ); + \HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'lane-start', 'lane' => $i, 'outputDir' => $lane_dir, 'oracle' => $oracle_metadata ) ); +} +\HtmlApiFuzz\write_json_file( $state_path, $state ); + +while ( $running ) { + foreach ( array_keys( $running ) as $lane_id ) { + html_api_fuzz_launcher_drain_lane( $running[ $lane_id ] ); + $status = proc_get_status( $running[ $lane_id ]['process'] ); + if ( $status['running'] ) { + continue; + } + + $closed = html_api_fuzz_launcher_close_lane( $running[ $lane_id ] ); + unset( $running[ $lane_id ] ); + $lane_dir = $state['laneResults'][ $lane_id ]['outputDir']; + $runner_state = \HtmlApiFuzz\read_json_file( $lane_dir . '/state.json' ); + $state['laneResults'][ $lane_id ] = array_merge( + $state['laneResults'][ $lane_id ], + array( + 'status' => 0 === $closed['code'] ? 'completed' : 'failed', + 'code' => $closed['code'], + 'durationMs' => $closed['durationMs'], + 'runnerState' => $runner_state, + 'stdoutTail' => $closed['stdoutTail'], + 'stderrTail' => $closed['stderrTail'], + ) + ); + $state['updatedAt'] = gmdate( 'c' ); + \HtmlApiFuzz\write_json_file( $state_path, $state ); + \HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'lane-stop', 'lane' => $lane_id, 'code' => $closed['code'] ) ); + } + usleep( 100000 ); +} + +$aggregate = array( + 'successes' => 0, + 'failures' => 0, + 'unsupported' => 0, + 'oracleParseErrors' => 0, + 'oracleUnsupported' => 0, + 'oracleTolerated' => 0, + 'oracleFindings' => 0, +); +foreach ( $state['laneResults'] as $lane ) { + $runner_state = $lane['runnerState'] ?? array(); + foreach ( $aggregate as $name => $count ) { + $aggregate[ $name ] += (int) ( $runner_state[ $name ] ?? 0 ); + } +} + +$state['finished'] = true; +$state['updatedAt'] = gmdate( 'c' ); +$state['aggregate'] = $aggregate; +\HtmlApiFuzz\write_json_file( $state_path, $state ); +\HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'launcher-stop', 'aggregate' => $aggregate ) ); + +$watcher_result = null; +if ( $run_watcher ) { + $triage_dir = $output_dir . '/triage'; + $minimize_timeout_ms = \HtmlApiFuzz\option_int( $options, 'minimize-timeout-ms', 300000 ); + $watcher_timeout_ms = \HtmlApiFuzz\option_int( $options, 'watcher-timeout-ms', max( 600000, $minimize_timeout_ms + 60000 ) ); + $proc = \HtmlApiFuzz\run_php_process( + array_values( + array_filter( + array( + __DIR__ . '/watcher.php', + '--run-dir', + $output_dir, + '--state-dir', + $triage_dir, + '--once', + '--minimize-timeout-ms', + (string) $minimize_timeout_ms, + '--timeout-ms', + (string) $timeout_ms, + array_key_exists( 'max-attempts', $options ) ? '--max-attempts' : null, + array_key_exists( 'max-attempts', $options ) ? (string) \HtmlApiFuzz\option_int( $options, 'max-attempts', 250 ) : null, + array_key_exists( 'max-minimize', $options ) ? '--max-minimize' : null, + array_key_exists( 'max-minimize', $options ) ? (string) \HtmlApiFuzz\option_int( $options, 'max-minimize', 0 ) : null, + array_key_exists( 'probe-mode', $options ) ? '--probe-mode' : null, + array_key_exists( 'probe-mode', $options ) ? \HtmlApiFuzz\option_string( $options, 'probe-mode', 'auto' ) : null, + \HtmlApiFuzz\option_bool( $options, 'keep-candidate-artifacts', false ) ? '--keep-candidate-artifacts' : null, + \HtmlApiFuzz\option_bool( $options, 'no-minimize', false ) ? '--no-minimize' : null, + \HtmlApiFuzz\option_bool( $options, 'any-failure', false ) ? '--any-failure' : null, + $triage_oracle_findings ? '--triage-oracle-findings' : null, + ), + static function ( $value ) { + return null !== $value; + } + ) + ), + $repo_root, + $watcher_timeout_ms, + $triage_dir . '/watcher.log' + ); + $watcher_result = array( + 'code' => $proc['code'], + 'timedOut' => $proc['timedOut'], + 'durationMs' => $proc['durationMs'], + 'logPath' => $proc['logPath'], + ); +} + +echo \HtmlApiFuzz\json_encode_safe( + array( + 'ok' => true, + 'outputDir' => $output_dir, + 'statePath' => $state_path, + 'aggregate' => $aggregate, + 'watcherResult' => $watcher_result, + ) +) . "\n"; diff --git a/tools/html-api-fuzz/lib/CommonCrawlRunner.php b/tools/html-api-fuzz/lib/CommonCrawlRunner.php new file mode 100644 index 0000000000000..b3dc73f76631a --- /dev/null +++ b/tools/html-api-fuzz/lib/CommonCrawlRunner.php @@ -0,0 +1,662 @@ +output_dir = $output_dir; + $this->oracle = $oracle; + $this->limits = $limits; + $this->max_input_bytes = $max_input_bytes; + $this->max_keep_per_signature = $max_keep_per_signature; + $this->process_timeout_ms = $process_timeout_ms; + $this->require_utf8 = $require_utf8; + $this->retain_all = $retain_all; + $this->checks = $checks; + $this->full_sample_percent = $full_sample_percent; + $this->memory_limit = $memory_limit; + $this->worker_script = $worker_script; + $this->git_metadata = git_metadata( 1000, null, false ); + $this->run_id = $run_id; + + ensure_dir( $this->output_dir ); + $this->initialize_configuration(); + } + + public static function from_environment(): self { + $output_dir = getenv( 'CC_ANALYZER_OUTPUT_DIR' ); + if ( ! is_string( $output_dir ) || '' === $output_dir ) { + $run_id = self::environment_string( 'HTML_API_CC_RUN_ID', 'run-' . timestamp() . '-' . getmypid() ); + $output_dir = repo_root() . '/artifacts/html-api-commoncrawl/' . $run_id; + } else { + $run_id = self::environment_string( 'HTML_API_CC_RUN_ID', 'cc-' . substr( hash( 'sha256', $output_dir ), 0, 12 ) ); + } + if ( ! preg_match( '/^[A-Za-z0-9][A-Za-z0-9._-]*$/', $run_id ) ) { + throw new \InvalidArgumentException( 'HTML_API_CC_RUN_ID contains unsafe path characters.' ); + } + + $oracle_kind = self::environment_string( 'HTML_API_CC_ORACLE', OracleRenderer::KIND_LEXBOR_SOURCE ); + $oracle_options = array( + 'dom-oracle' => $oracle_kind, + 'oracle-timeout-ms' => (string) self::environment_int( 'HTML_API_CC_ORACLE_TIMEOUT_MS', self::DEFAULT_ORACLE_TIMEOUT_MS, 1 ), + ); + $oracle_bin = getenv( 'HTML_API_FUZZ_LEXBOR_ORACLE' ); + if ( is_string( $oracle_bin ) && '' !== $oracle_bin ) { + $oracle_options['lexbor-oracle-bin'] = $oracle_bin; + } + + $oracle = OracleRenderer::from_options( $oracle_options ); + $metadata = $oracle->metadata(); + if ( + OracleRenderer::KIND_LEXBOR_SOURCE === $oracle_kind && + ( false === ( $metadata['available'] ?? true ) || isset( $metadata['versionError'] ) ) + ) { + throw new \RuntimeException( + 'Lexbor oracle is unavailable. Run tools/html-api-fuzz/oracles/lexbor/build.sh ' + . 'or set HTML_API_FUZZ_LEXBOR_ORACLE to an executable oracle binary.' + ); + } + $expected_commit = getenv( 'HTML_API_CC_EXPECT_LEXBOR_COMMIT' ); + if ( + is_string( $expected_commit ) && '' !== $expected_commit && + $expected_commit !== ( $metadata['lexborCommit'] ?? null ) + ) { + throw new \RuntimeException( 'Lexbor oracle commit does not match HTML_API_CC_EXPECT_LEXBOR_COMMIT.' ); + } + + $checks = self::environment_string( 'HTML_API_CC_CHECKS', 'sampled' ); + if ( ! in_array( $checks, array( 'baseline', 'full', 'sampled' ), true ) ) { + throw new \InvalidArgumentException( 'HTML_API_CC_CHECKS must be baseline, full, or sampled.' ); + } + $full_sample_percent = self::environment_int( 'HTML_API_CC_FULL_SAMPLE_PERCENT', 1, 0 ); + if ( $full_sample_percent > 100 ) { + throw new \InvalidArgumentException( 'HTML_API_CC_FULL_SAMPLE_PERCENT must be at most 100.' ); + } + $memory_limit = self::environment_string( 'HTML_API_CC_MEMORY_LIMIT', '256M' ); + if ( ! preg_match( '/^[1-9][0-9]*[KMG]?$/i', $memory_limit ) ) { + throw new \InvalidArgumentException( 'HTML_API_CC_MEMORY_LIMIT must be a positive PHP memory limit such as 256M.' ); + } + $worker_script = self::environment_string( 'HTML_API_CC_WORKER_SCRIPT', repo_root() . '/tools/html-api-fuzz/worker.php' ); + if ( ! is_file( $worker_script ) ) { + throw new \RuntimeException( 'HTML_API_CC_WORKER_SCRIPT does not exist.' ); + } + + return new self( + $output_dir, + $oracle, + array( + 'maxTokens' => self::environment_int( 'HTML_API_CC_MAX_TOKENS', self::DEFAULT_MAX_TOKENS, 1 ), + 'maxNodes' => self::environment_int( 'HTML_API_CC_MAX_NODES', self::DEFAULT_MAX_NODES, 1 ), + 'maxDepth' => self::environment_int( 'HTML_API_CC_MAX_DEPTH', self::DEFAULT_MAX_DEPTH, 1 ), + 'maxTreeBytes' => self::environment_int( 'HTML_API_CC_MAX_TREE_BYTES', self::DEFAULT_MAX_TREE_BYTES, 1 ), + ), + self::environment_int( 'HTML_API_CC_MAX_INPUT_BYTES', self::DEFAULT_MAX_INPUT_BYTES, 0 ), + self::environment_int( 'HTML_API_CC_MAX_KEEP_PER_SIGNATURE', self::DEFAULT_KEEP_PER_SIGNATURE, 1 ), + self::environment_int( 'HTML_API_CC_PROCESS_TIMEOUT_MS', self::DEFAULT_PROCESS_TIMEOUT_MS, 1 ), + self::environment_bool( 'HTML_API_CC_REQUIRE_UTF8', true ), + self::environment_bool( 'HTML_API_CC_RETAIN_ALL', false ), + $checks, + $full_sample_percent, + $memory_limit, + $worker_script, + $run_id + ); + } + + /** Analyze one CcAnalyzer\Analysis\HtmlAnalysisInput-compatible object. */ + public function analyze_document( object $document ): array { + $started_at = hrtime( true ); + $metadata = $this->document_metadata( $document ); + $body = $document->body ?? null; + + if ( ! is_string( $body ) ) { + return $this->record_callback_error( $metadata, new \InvalidArgumentException( 'cc-analyzer document body must be a string.' ), $started_at ); + } + + $metadata['byteLength'] = strlen( $body ); + $metadata['inputSha1'] = sha1( $body ); + $metadata['utf8Valid'] = 1 === preg_match( '//u', $body ); + if ( $this->max_input_bytes > 0 && strlen( $body ) > $this->max_input_bytes ) { + return $this->record_skip( $metadata, 'skipped-input-too-large', $started_at ); + } + if ( $this->require_utf8 && ! $metadata['utf8Valid'] ) { + return $this->record_skip( $metadata, 'skipped-invalid-utf8', $started_at ); + } + + $seed = self::seed_for_record( $metadata['recordId'], $metadata['inputSha1'] ); + $checks = $this->checks_for_seed( $seed ); + $staging_dir = $this->create_staging_dir( $metadata ); + try { + $this->persist_initial_input( $staging_dir, $body, $metadata, $seed, $checks ); + $process = run_php_process( $this->worker_args( $staging_dir, $seed, $checks ), repo_root(), $this->process_timeout_ms, $staging_dir . '/worker.log' ); + $result = $this->load_or_synthesize_result( $staging_dir, $process, $body, $seed, $checks ); + $result['profile'] = 'commoncrawl'; + $result['inputSource'] = 'commoncrawl'; + $result['commonCrawl'] = $metadata; + $result['run'] = $this->run_metadata(); + $result['repo'] = $this->git_metadata; + $result['oracleResult'] = $result['dom'] ?? null; // New neutral name; dom remains for replay compatibility. + $result['process'] = self::compact_process( $process ); + $result['durationMs'] = self::elapsed_ms( $started_at ); + if ( false === ( $result['ok'] ?? false ) && ! is_array( $result['signature'] ?? null ) ) { + $signature = Signature::from_result( $result ); + if ( null !== $signature ) { + $result['signature'] = $signature; + } + } + + $has_oracle_finding = is_array( $result['oracleFinding'] ?? null ); + $should_retain = $this->retain_all || false === ( $result['ok'] ?? false ) || $has_oracle_finding; + $artifact_dir = $should_retain ? $this->publish_staging( $staging_dir, $result, $metadata ) : null; + if ( ! $should_retain ) { + remove_dir_recursive( $staging_dir ); + } + + $summary = $this->summary_from_result( $result, $metadata, $artifact_dir ); + $this->append_summary( $summary ); + return $summary; + } catch ( \Throwable $throwable ) { + // Deliberately leave staging in pending/ for manual recovery. + return $this->record_callback_error( $metadata, $throwable, $started_at, $body, $staging_dir ); + } + } + + private function worker_args( string $staging_dir, int $seed, string $checks ): array { + $args = array( + '-d', 'memory_limit=' . $this->memory_limit, + $this->worker_script, + '--input-file', $staging_dir . '/input.bin', + '--mode', Generator::MODE_FULL_DOCUMENT, + '--profile', 'commoncrawl', + '--seed', (string) $seed, + '--output-dir', $staging_dir, + '--max-tokens', (string) $this->limits['maxTokens'], + '--max-nodes', (string) $this->limits['maxNodes'], + '--max-depth', (string) $this->limits['maxDepth'], + '--max-tree-bytes', (string) $this->limits['maxTreeBytes'], + '--checks', $checks, + '--git-metadata-base64', git_metadata_base64( $this->git_metadata ), + ); + return array_merge( $args, $this->oracle->worker_args() ); + } + + private function create_staging_dir( array $metadata ): string { + $document_key = hash( 'sha256', (string) $metadata['recordId'] . "\0" . (string) $metadata['inputSha1'] ); + $random = bin2hex( random_bytes( 6 ) ); + $path = $this->output_dir . '/pending/' . substr( $document_key, 0, 20 ) . '-' . getmypid() . '-' . $random; + ensure_dir( $path ); + return $path; + } + + private function persist_initial_input( string $staging_dir, string $body, array $metadata, int $seed, string $checks ): void { + $input_path = $staging_dir . '/input.bin'; + $written = file_put_contents( $input_path, $body ); + if ( strlen( $body ) !== $written ) { + throw new \RuntimeException( 'Could not persist the complete Common Crawl response body.' ); + } + write_json_file( + $staging_dir . '/replay.json', + array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-replay', + 'createdAt' => gmdate( 'c' ), + 'run' => $this->run_metadata(), + 'repoRoot' => repo_root(), + 'repoCommit' => $this->git_metadata['commit'] ?? null, + 'repoDirty' => $this->git_metadata['dirty'] ?? null, + 'seed' => $seed, + 'profile' => 'commoncrawl', + 'mode' => Generator::MODE_FULL_DOCUMENT, + 'inputSource' => 'commoncrawl', + 'inputSha1' => sha1( $body ), + 'inputLength' => strlen( $body ), + 'limits' => $this->limits, + 'options' => array( 'checks' => $checks ), + 'commonCrawl' => $metadata, + 'status' => 'pending-worker', + ) + ); + } + + private function load_or_synthesize_result( string $staging_dir, array $process, string $body, int $seed, string $checks ): array { + $result = null; + try { + $result = read_json_file( $staging_dir . '/result.json' ); + } catch ( \Throwable $ignored ) { + $result = null; + } + if ( is_array( $result ) ) { + return $result; + } + + if ( $process['timedOut'] ?? false ) { + $failure_class = 'worker-timeout'; + $status = 'timeout'; + } elseif ( false !== stripos( (string) ( $process['stderr'] ?? '' ), 'Allowed memory size' ) ) { + $failure_class = 'worker-oom'; + $status = 'resource-limit'; + } else { + $failure_class = 'worker-crash'; + $status = 'crashed'; + } + + return array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-worker-result', + 'createdAt' => gmdate( 'c' ), + 'ok' => false, + 'status' => $status, + 'failureClass' => $failure_class, + 'failureSnippet' => substr( trim( (string) ( $process['stderr'] ?? '' ) ), -2000 ), + 'seed' => $seed, + 'profile' => 'commoncrawl', + 'mode' => Generator::MODE_FULL_DOCUMENT, + 'inputSource' => 'commoncrawl', + 'inputSha1' => sha1( $body ), + 'inputLength' => strlen( $body ), + 'checks' => $checks, + 'oracle' => $this->oracle->metadata(), + 'comparison' => null, + ); + } + + private function publish_staging( string $staging_dir, array &$result, array $metadata ): ?string { + $signature = $result['signature'] ?? $result['oracleFinding']['signature'] ?? null; + $hash = is_array( $signature ) && is_string( $signature['hash'] ?? null ) + ? $signature['hash'] + : substr( hash( 'sha256', (string) ( $result['failureClass'] ?? $result['status'] ?? 'retained' ) ), 0, 16 ); + $record_key = (string) $metadata['recordId'] . "\0" . (string) $metadata['inputSha1']; + $document_key = substr( hash( 'sha256', $record_key ), 0, 20 ); + $signature_dir = $this->output_dir . '/findings/' . preg_replace( '/[^A-Za-z0-9._-]/', '_', $hash ); + $final_dir = $signature_dir . '/' . $document_key; + $lock = $this->retention_lock(); + + try { + ensure_dir( $signature_dir ); + if ( is_file( $final_dir . '/.complete' ) ) { + remove_dir_recursive( $staging_dir ); + return $final_dir; + } + + $complete_count = 0; + foreach ( scandir( $signature_dir ) ?: array() as $entry ) { + if ( is_file( $signature_dir . '/' . $entry . '/.complete' ) ) { + ++$complete_count; + } + } + if ( ! $this->retain_all && $complete_count >= $this->max_keep_per_signature ) { + remove_dir_recursive( $staging_dir ); + return null; + } + + $result = self::replace_path_prefix( $result, $staging_dir, $final_dir ); + $result['paths'] = array( + 'outputDir' => $final_dir, + 'inputPath' => $final_dir . '/input.bin', + 'replayPath' => $final_dir . '/replay.json', + 'resultPath' => $final_dir . '/result.json', + ); + $replay = read_json_file( $staging_dir . '/replay.json' ); + if ( ! is_array( $replay ) ) { + $replay = array( 'kind' => 'html-api-fuzz-replay' ); + } + $replay = self::replace_path_prefix( $replay, $staging_dir, $final_dir ); + $replay['run'] = $this->run_metadata(); + $replay['repoRoot'] = repo_root(); + $replay['repoCommit'] = $this->git_metadata['commit'] ?? null; + $replay['repoDirty'] = $this->git_metadata['dirty'] ?? null; + $replay['commonCrawl'] = $metadata; + $replay['options']['checks'] = $result['checks'] ?? $this->checks_for_seed( (int) ( $result['seed'] ?? 1 ) ); + $replay['command'] = array( + 'program' => PHP_BINARY, + 'args' => array( 'tools/html-api-fuzz/replay.php', '--replay', $final_dir . '/replay.json' ), + 'cwd' => repo_root(), + ); + $replay['result']['resultPath'] = $final_dir . '/result.json'; + write_json_file( $staging_dir . '/result.json', $result ); + write_json_file( $staging_dir . '/replay.json', $replay ); + $marker = "complete\n"; + if ( strlen( $marker ) !== file_put_contents( $staging_dir . '/.complete', $marker ) ) { + throw new \RuntimeException( 'Could not write finding completion marker.' ); + } + if ( is_dir( $final_dir ) ) { + remove_dir_recursive( $final_dir ); + } + if ( ! rename( $staging_dir, $final_dir ) ) { + throw new \RuntimeException( 'Could not atomically publish Common Crawl finding.' ); + } + return $final_dir; + } finally { + flock( $lock, LOCK_UN ); + fclose( $lock ); + } + } + + private function retention_lock() { + $path = $this->output_dir . '/.retention.lock'; + $file = fopen( $path, 'c+b' ); + if ( false === $file || ! flock( $file, LOCK_EX ) ) { + if ( is_resource( $file ) ) { + fclose( $file ); + } + throw new \RuntimeException( 'Could not lock Common Crawl artifact retention state.' ); + } + return $file; + } + + private static function replace_path_prefix( $value, string $from, string $to ) { + if ( is_string( $value ) ) { + return 0 === strpos( $value, $from ) ? $to . substr( $value, strlen( $from ) ) : $value; + } + if ( is_array( $value ) ) { + foreach ( $value as $key => $item ) { + $value[ $key ] = self::replace_path_prefix( $item, $from, $to ); + } + } + return $value; + } + + private function record_skip( array $metadata, string $status, int $started_at ): array { + $summary = array_merge( $this->base_summary(), array( + 'ok' => true, + 'status' => $status, + 'differentialCovered' => false, + 'durationMs' => self::elapsed_ms( $started_at ), + 'commonCrawl' => $metadata, + 'oracle' => $this->oracle->metadata(), + ) ); + $this->append_summary( $summary ); + return $summary; + } + + private function record_callback_error( array $metadata, \Throwable $throwable, int $started_at, ?string $body = null, ?string $staging_dir = null ): array { + $result = array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-worker-result', + 'createdAt' => gmdate( 'c' ), + 'ok' => false, + 'status' => 'commoncrawl-callback-error', + 'failureClass' => 'commoncrawl-callback-error', + 'failureSnippet' => $throwable->getMessage(), + 'throwable' => get_class( $throwable ), + 'profile' => 'commoncrawl', + 'mode' => Generator::MODE_FULL_DOCUMENT, + 'inputSource' => 'commoncrawl', + 'inputSha1' => $metadata['inputSha1'] ?? ( null === $body ? null : sha1( $body ) ), + 'inputLength' => $metadata['byteLength'] ?? ( null === $body ? null : strlen( $body ) ), + 'oracle' => $this->oracle->metadata(), + 'commonCrawl' => $metadata, + 'run' => $this->run_metadata(), + 'repo' => $this->git_metadata, + 'durationMs' => self::elapsed_ms( $started_at ), + ); + $result['signature'] = Signature::from_result( $result ); + $summary = $this->summary_from_result( $result, $metadata, $staging_dir ); + $summary['pendingRecovery'] = null !== $staging_dir; + $this->append_summary( $summary ); + return $summary; + } + + private function summary_from_result( array $result, array $metadata, ?string $artifact_dir ): array { + return array_merge( $this->base_summary(), array( + 'kind' => false === ( $result['ok'] ?? false ) ? 'commoncrawl-failure' : 'commoncrawl-attempt', + 'createdAt' => $result['createdAt'] ?? gmdate( 'c' ), + 'ok' => (bool) ( $result['ok'] ?? false ), + 'status' => $result['status'] ?? 'unknown', + 'failureClass' => $result['failureClass'] ?? null, + 'differentialCovered' => is_array( $result['comparison'] ?? null ), + 'seed' => $result['seed'] ?? null, + 'checks' => $result['checks'] ?? $this->checks, + 'inputSha1' => $result['inputSha1'] ?? $metadata['inputSha1'] ?? null, + 'inputLength' => $result['inputLength'] ?? $metadata['byteLength'] ?? null, + 'signature' => $result['signature'] ?? null, + 'oracleFinding' => $result['oracleFinding'] ?? null, + 'oracle' => $result['oracle'] ?? $this->oracle->metadata(), + 'artifactsRetained' => null !== $artifact_dir, + 'artifactDir' => $artifact_dir, + 'durationMs' => $result['durationMs'] ?? null, + 'timingsMs' => $result['timingsMs'] ?? null, + 'process' => $result['process'] ?? null, + 'commonCrawl' => $metadata, + ) ); + } + + private function base_summary(): array { + return array( + 'kind' => 'commoncrawl-attempt', + 'createdAt' => gmdate( 'c' ), + 'runId' => $this->run_id, + 'configHash' => $this->configuration_hash, + 'repo' => $this->git_metadata, + 'profile' => 'commoncrawl', + 'mode' => Generator::MODE_FULL_DOCUMENT, + 'inputSource' => 'commoncrawl', + ); + } + + private function append_summary( array $summary ): void { + append_ndjson( $this->output_dir . '/commoncrawl-summary.ndjson', $summary ); + $this->update_coverage( $summary ); + } + + private function update_coverage( array $summary ): void { + $path = $this->output_dir . '/coverage.json'; + $lock = fopen( $this->output_dir . '/.coverage.lock', 'c+b' ); + if ( false === $lock || ! flock( $lock, LOCK_EX ) ) { + throw new \RuntimeException( 'Could not lock Common Crawl coverage counters.' ); + } + try { + $coverage = read_json_file( $path ); + if ( ! is_array( $coverage ) ) { + $coverage = array( + 'kind' => 'html-api-commoncrawl-coverage', + 'runId' => $this->run_id, + 'configHash' => $this->configuration_hash, + 'total' => 0, + 'covered' => 0, + 'failures' => 0, + 'statuses' => array(), + ); + } + ++$coverage['total']; + if ( $summary['differentialCovered'] ?? false ) { + ++$coverage['covered']; + } + if ( false === ( $summary['ok'] ?? false ) ) { + ++$coverage['failures']; + } + $status = (string) ( $summary['status'] ?? 'unknown' ); + $coverage['statuses'][ $status ] = 1 + (int) ( $coverage['statuses'][ $status ] ?? 0 ); + $coverage['coverageRate'] = 0 === $coverage['total'] ? 0 : round( $coverage['covered'] / $coverage['total'], 6 ); + $coverage['updatedAt'] = gmdate( 'c' ); + write_json_file( $path, $coverage ); + } finally { + flock( $lock, LOCK_UN ); + fclose( $lock ); + } + } + + private function initialize_configuration(): void { + $configuration = array( + 'kind' => 'html-api-commoncrawl-configuration', + 'runId' => $this->run_id, + 'repo' => $this->git_metadata, + 'phpVersion' => PHP_VERSION, + 'oracle' => $this->oracle->metadata(), + 'limits' => $this->limits, + 'maxInputBytes' => $this->max_input_bytes, + 'maxKeepPerSignature' => $this->max_keep_per_signature, + 'processTimeoutMs' => $this->process_timeout_ms, + 'memoryLimit' => $this->memory_limit, + 'checks' => $this->checks, + 'fullSamplePercent' => $this->full_sample_percent, + 'requireUtf8' => $this->require_utf8, + 'retainAll' => $this->retain_all, + 'workerScript' => realpath( $this->worker_script ) ?: $this->worker_script, + 'inputSemantics' => 'raw cc-analyzer response body; no charset transcoding', + 'ccAnalyzer' => array( + 'version' => getenv( 'CC_ANALYZER_VERSION' ) ?: null, + 'crawl' => getenv( 'CC_ANALYZER_CRAWL' ) ?: null, + 'invocation' => getenv( 'CC_ANALYZER_INVOCATION' ) ?: null, + ), + ); + $this->configuration_hash = hash( 'sha256', json_encode( self::canonicalize( $configuration ), JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_SUBSTITUTE ) ); + $configuration['configHash'] = $this->configuration_hash; + + $path = $this->output_dir . '/configuration.json'; + $lock = fopen( $this->output_dir . '/.configuration.lock', 'c+b' ); + if ( false === $lock || ! flock( $lock, LOCK_EX ) ) { + throw new \RuntimeException( 'Could not lock Common Crawl configuration.' ); + } + try { + $existing = read_json_file( $path ); + if ( is_array( $existing ) ) { + if ( ! hash_equals( (string) ( $existing['configHash'] ?? '' ), $this->configuration_hash ) ) { + throw new \RuntimeException( 'Common Crawl output directory configuration mismatch; use a new run directory or identical settings.' ); + } + $this->run_id = (string) ( $existing['runId'] ?? $this->run_id ); + return; + } + $configuration['createdAt'] = gmdate( 'c' ); + write_json_file( $path, $configuration ); + } finally { + flock( $lock, LOCK_UN ); + fclose( $lock ); + } + } + + private function run_metadata(): array { + return array( + 'runId' => $this->run_id, + 'configHash' => $this->configuration_hash, + 'outputDir' => $this->output_dir, + ); + } + + private static function canonicalize( $value ) { + if ( ! is_array( $value ) ) { + return $value; + } + if ( array_keys( $value ) !== range( 0, count( $value ) - 1 ) ) { + ksort( $value ); + } + foreach ( $value as $key => $item ) { + $value[ $key ] = self::canonicalize( $item ); + } + return $value; + } + + private function document_metadata( object $document ): array { + return array( + 'recordId' => (string) ( $document->recordId ?? '' ), + 'targetUri' => (string) ( $document->targetUri ?? '' ), + 'responseCode' => is_int( $document->responseCode ?? null ) ? $document->responseCode : null, + 'contentType' => (string) ( $document->contentType ?? '' ), + 'transportCharset' => is_string( $document->transportCharset ?? null ) ? $document->transportCharset : null, + 'inputStateKey' => (string) ( $document->inputStateKey ?? '' ), + 'rangeStart' => is_int( $document->rangeStart ?? null ) ? $document->rangeStart : null, + 'rangeLength' => is_int( $document->rangeLength ?? null ) ? $document->rangeLength : null, + ); + } + + private static function compact_process( array $process ): array { + return array( + 'code' => $process['code'] ?? null, + 'timedOut' => $process['timedOut'] ?? false, + 'durationMs' => $process['durationMs'] ?? null, + 'stderrTail' => substr( (string) ( $process['stderr'] ?? '' ), -2000 ), + 'logPath' => $process['logPath'] ?? null, + ); + } + + private static function seed_for_record( string $record_id, string $input_sha1 ): int { + return 1 + ( hexdec( substr( hash( 'sha256', $record_id . "\0" . $input_sha1 ), 0, 7 ) ) % 2147483646 ); + } + + private function checks_for_seed( int $seed ): string { + if ( 'sampled' !== $this->checks ) { + return $this->checks; + } + $bucket = hexdec( substr( hash( 'sha256', 'commoncrawl-full-checks:' . $seed ), 0, 8 ) ) % 100; + return $bucket < $this->full_sample_percent ? 'full' : 'baseline'; + } + + private static function elapsed_ms( int $started_at ): int { + return (int) round( max( 0, hrtime( true ) - $started_at ) / 1000000 ); + } + + private static function environment_string( string $name, string $default ): string { + $value = getenv( $name ); + return false === $value || '' === $value ? $default : (string) $value; + } + + private static function environment_int( string $name, int $default, int $minimum ): int { + $value = getenv( $name ); + if ( false === $value || '' === $value ) { + return $default; + } + $parsed = filter_var( $value, FILTER_VALIDATE_INT ); + if ( false === $parsed || $parsed < $minimum ) { + throw new \InvalidArgumentException( "{$name} must be an integer greater than or equal to {$minimum}." ); + } + return (int) $parsed; + } + + private static function environment_bool( string $name, bool $default ): bool { + $value = getenv( $name ); + if ( false === $value || '' === $value ) { + return $default; + } + $parsed = filter_var( $value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE ); + if ( null === $parsed ) { + throw new \InvalidArgumentException( "{$name} must be a boolean value." ); + } + return $parsed; + } +} diff --git a/tools/html-api-fuzz/lib/Corpus.php b/tools/html-api-fuzz/lib/Corpus.php new file mode 100644 index 0000000000000..005a91dc8a40b --- /dev/null +++ b/tools/html-api-fuzz/lib/Corpus.php @@ -0,0 +1,78 @@ + basename( $file ), + 'data' => $data, + ); + } + } + + if ( null === $directory ) { + self::$entries = $entries; + } + return $entries; + } + + /** + * Extracts #data sections from html5lib .dat content. A section runs from + * the line after `#data` to the line before the next `#` directive, with + * the trailing newline removed. + */ + private static function parse_dat_data_sections( string $contents ): array { + $sections = array(); + $lines = explode( "\n", $contents ); + $current = null; + foreach ( $lines as $line ) { + if ( '#data' === $line ) { + $current = array(); + continue; + } + if ( null !== $current ) { + if ( '' !== $line && '#' === $line[0] ) { + $sections[] = implode( "\n", $current ); + $current = null; + continue; + } + $current[] = $line; + } + } + if ( null !== $current && array() !== $current ) { + $sections[] = implode( "\n", $current ); + } + + return $sections; + } +} diff --git a/tools/html-api-fuzz/lib/Generator.php b/tools/html-api-fuzz/lib/Generator.php new file mode 100644 index 0000000000000..21eed89952d96 --- /dev/null +++ b/tools/html-api-fuzz/lib/Generator.php @@ -0,0 +1,1409 @@ +`; the others receive a small probe weight so the fuzzer + * exercises the unsupported-context path today and picks up real + * coverage automatically when create_fragment() gains context support. + */ + public static function fragment_contexts(): array { + return array( 'body', 'div', 'p', 'td', 'tr', 'table', 'caption', 'colgroup', 'select', 'option', 'template', 'title', 'textarea', 'script', 'style', 'svg', 'math' ); + } + + public static function generate( int $seed, string $profile = 'auto', string $mode = 'auto', string $payload_policy = 'auto', ?int $max_input_bytes = null ): array { + $rng = new Prng( $seed ); + $requested_profile = $profile; + $requested_mode = $mode; + $requested_payload_policy = $payload_policy; + if ( 'auto' === $profile ) { + $profile = $rng->weighted( + array( + 'balanced' => 22, + 'full-document' => 8, + 'body-fragment' => 8, + 'tables' => 11, + 'template' => 8, + 'select' => 8, + 'foreign-content' => 10, + 'rawtext-rcdata' => 8, + 'text-fragment' => 4, + 'formatting-adoption' => 10, + 'attributes-entities' => 7, + 'comments-doctype-bogus' => 5, + 'deep-nesting' => 2, + 'resource-stress' => 1, + 'incomplete-malformed' => 2, + ) + ); + } elseif ( ! in_array( $profile, self::profiles(), true ) ) { + throw new \InvalidArgumentException( 'Unknown generator profile: ' . $profile ); + } + + if ( 'auto' === $payload_policy ) { + $payload_policy = 'resource-stress' === $profile + ? $rng->weighted( + array( + 'stress-long' => 75, + 'mostly-valid' => 15, + 'valid-utf8' => 10, + ) + ) + : $rng->weighted( + array( + 'valid-utf8' => 52, + 'mostly-valid' => 33, + 'ascii-structural' => 15, + ) + ); + } elseif ( ! in_array( $payload_policy, self::payload_policies(), true ) ) { + throw new \InvalidArgumentException( 'Unknown generator payload policy: ' . $payload_policy ); + } + + if ( 'auto' === $mode ) { + $mode = 'full-document' === $profile ? self::MODE_FULL_DOCUMENT : ( in_array( $profile, array( 'body-fragment', 'text-fragment' ), true ) ? self::MODE_FRAGMENT_BODY : $rng->weighted( array( self::MODE_FRAGMENT_BODY => 70, self::MODE_FULL_DOCUMENT => 30 ) ) ); + } elseif ( ! in_array( $mode, self::modes(), true ) ) { + throw new \InvalidArgumentException( 'Unknown generator mode: ' . $mode ); + } + + $fragment_context = 'body'; + if ( self::MODE_FRAGMENT_BODY === $mode ) { + $weights = array( 'body' => 240 ); + foreach ( self::fragment_contexts() as $context ) { + if ( 'body' !== $context ) { + $weights[ $context ] = 1; + } + } + $fragment_context = $rng->weighted( $weights ); + } + + $generator = new self( $rng, $profile, $payload_policy ); + if ( 'body' !== $fragment_context ) { + $generator->mark_feature( 'fragment-context:' . $fragment_context ); + } + $max_depth = $generator->depth_for_profile(); + $body = 'text-fragment' === $profile ? $generator->text_fragment_input() : $generator->nodes( $max_depth, 'body' ); + + if ( self::MODE_FULL_DOCUMENT === $mode ) { + $html = $generator->full_document( $body ); + } else { + $html = $body; + } + $truncated = false; + if ( null !== $max_input_bytes && $max_input_bytes > 0 && strlen( $html ) > $max_input_bytes ) { + $html = self::trim_to_max_bytes( $html, $max_input_bytes ); + $truncated = true; + $generator->mark_feature( 'generator:truncated' ); + } + + return array( + 'input' => $html, + 'mode' => $mode, + 'profile' => $profile, + 'payloadPolicy' => $payload_policy, + 'fragmentContext' => $fragment_context, + 'parameters' => array( + 'seed' => $seed, + 'requestedProfile' => $requested_profile, + 'requestedMode' => $requested_mode, + 'requestedPayloadPolicy' => $requested_payload_policy, + 'profile' => $profile, + 'mode' => $mode, + 'payloadPolicy' => $payload_policy, + 'fragmentContext' => $fragment_context, + 'maxDepth' => $max_depth, + 'maxInputBytes' => $max_input_bytes, + 'truncated' => $truncated, + 'byteLength' => strlen( $html ), + 'features' => $generator->features(), + ), + ); + } + + private function __construct( Prng $rng, string $profile, string $payload_policy ) { + $this->rng = $rng; + $this->profile = $profile; + $this->payload_policy = $payload_policy; + } + + private static function trim_to_max_bytes( string $html, int $max_input_bytes ): string { + $trimmed = substr( $html, 0, $max_input_bytes ); + while ( '' !== $trimmed && 1 !== preg_match( '//u', $trimmed ) ) { + $trimmed = substr( $trimmed, 0, -1 ); + } + + return $trimmed; + } + + private function mark_feature( string $feature ): void { + $this->features[ $feature ] = true; + } + + private function features(): array { + $features = array_keys( $this->features ); + sort( $features ); + return $features; + } + + private function depth_for_profile(): int { + if ( 'resource-stress' === $this->profile ) { + return $this->rng->int( 10, 20 ); + } + if ( 'deep-nesting' === $this->profile ) { + return $this->rng->int( 8, 18 ); + } + if ( 'balanced' === $this->profile ) { + return $this->rng->int( 3, 5 ); + } + return $this->rng->int( 3, 6 ); + } + + private function full_document( string $body ): string { + $doctype = $this->rng->chance( 70 ) ? $this->doctype() : ''; + $head = $this->rng->chance( 65 ) ? '' . $this->head_nodes() . '' : $this->head_nodes(); + $attrs = $this->attrs(); + $body_at = $this->attrs(); + + if ( $this->rng->chance( 4 ) ) { + $this->mark_feature( 'frameset' ); + $frames = 'attrs() . '>' . $this->terminal_text( true ) . ''; + return $doctype . '' . $head . 'attrs() . '>' . $frames . ( $this->rng->chance( 70 ) ? '' : '' ) . ( $this->rng->chance( 40 ) ? $body : '' ); + } + + if ( $this->rng->chance( 20 ) ) { + return $doctype . $head . $body; + } + + $trailer = ''; + if ( $this->rng->chance( 8 ) ) { + $this->mark_feature( 'content-after-html' ); + $trailer = $this->rng->choice( array( $this->terminal_ascii( 6 ), '', '
tail
', '' ) ); + } + + return $doctype . '' . $head . '' . $body . ( $this->rng->chance( 75 ) ? '' . $trailer : '' ); + } + + private function head_nodes(): string { + $out = ''; + if ( $this->rng->chance( 45 ) ) { + $out .= '' . $this->terminal_text() . ''; + } + if ( $this->rng->chance( 35 ) ) { + $out .= 'attrs() . '>'; + } + if ( $this->rng->chance( 25 ) ) { + $out .= ''; + } + return $out; + } + + private function nodes( int $depth, string $context ): string { + if ( 'deep-nesting' === $this->profile ) { + $count = $this->rng->int( 1, 2 ); + } elseif ( $depth > 4 ) { + $count = $this->rng->int( 1, 3 ); + } elseif ( $depth > 2 ) { + $count = $this->rng->int( 1, 5 ); + } else { + $count = $this->rng->int( 1, 7 ); + } + + $out = ''; + for ( $i = 0; $i < $count; ++$i ) { + $out .= $this->node( $depth, $context ); + if ( strlen( $out ) > 131072 ) { + $this->mark_feature( 'generator:hard-truncated' ); + return self::trim_to_max_bytes( $out, 131072 ); + } + } + return $out; + } + + private function node( int $depth, string $context ): string { + if ( $depth <= 0 ) { + return $this->leaf(); + } + + $weights = array( + 'element' => 33, + 'text' => 16, + 'charref' => 1, + 'comment' => 7, + 'void' => 8, + 'raw' => 5, + 'template' => 5, + 'table' => 5, + 'select' => 2, + 'foreign' => 5, + 'adoption' => 2, + 'reconstruction' => 0, + 'list-chain' => 1, + 'special-closers' => 1, + 'weird-tag' => 0, + 'doctype' => 2, + 'bogus' => 2, + ); + + if ( 'tables' === $this->profile ) { + $weights['table'] = 35; + $weights['element'] = 15; + } elseif ( 'template' === $this->profile ) { + $weights['template'] = 35; + } elseif ( 'select' === $this->profile ) { + $weights['select'] = 35; + $weights['table'] = 8; + $weights['element'] = 15; + } elseif ( 'foreign-content' === $this->profile ) { + $weights['foreign'] = 35; + } elseif ( 'rawtext-rcdata' === $this->profile ) { + $weights['raw'] = 35; + } elseif ( 'comments-doctype-bogus' === $this->profile ) { + $weights['comment'] = 25; + $weights['doctype'] = 12; + $weights['bogus'] = 15; + } elseif ( 'attributes-entities' === $this->profile ) { + $weights['element'] = 50; + $weights['text'] = 22; + $weights['charref'] = 12; + $weights['weird-tag'] = 8; + } elseif ( 'formatting-adoption' === $this->profile ) { + $weights['element'] = 35; + $weights['adoption'] = 20; + $weights['reconstruction'] = 30; + } elseif ( 'incomplete-malformed' === $this->profile ) { + $weights['bogus'] = 25; + $weights['element'] = 28; + $weights['weird-tag'] = 15; + $weights['special-closers'] = 4; + } + + switch ( $this->rng->weighted( $weights ) ) { + case 'text': + $this->mark_feature( 'text' ); + return $this->terminal_text(); + case 'charref': + $this->mark_feature( 'text' ); + return $this->character_reference( 'text' ) . $this->terminal_payload(); + case 'comment': + $this->mark_feature( 'comment' ); + return $this->comment(); + case 'void': + $this->mark_feature( 'void-element' ); + return '<' . $this->rng->choice( $this->void_tags ) . $this->attrs() . ( $this->rng->chance( 25 ) ? '/>' : '>' ); + case 'raw': + return $this->raw_element(); + case 'template': + $this->mark_feature( 'template' ); + return 'attrs() . '>' . $this->nodes( $depth - 1, 'body' ) . ( $this->rng->chance( 80 ) ? '' : '' ); + case 'table': + return $this->table( $depth - 1 ); + case 'select': + return $this->select_stress( $depth - 1 ); + case 'foreign': + return $this->foreign( $depth - 1 ); + case 'adoption': + return $this->adoption_pattern( $depth - 1 ); + case 'reconstruction': + return $this->active_formatting_reconstruction_pattern( $depth - 1 ); + case 'list-chain': + return $this->auto_closing_chain( $depth - 1 ); + case 'special-closers': + return $this->special_closers(); + case 'weird-tag': + return $this->weird_element( $depth - 1 ); + case 'doctype': + $this->mark_feature( 'doctype' ); + return $this->doctype(); + case 'bogus': + $this->mark_feature( 'bogus-markup' ); + return $this->bogus(); + case 'element': + default: + return $this->element( $depth - 1 ); + } + } + + private function element( int $depth ): string { + if ( 'attributes-entities' === $this->profile && $this->rng->chance( 16 ) ) { + return $this->weird_element( $depth ); + } + if ( 'incomplete-malformed' === $this->profile && $this->rng->chance( 12 ) ) { + return $this->weird_element( $depth ); + } + + $tag = $this->tag_name(); + if ( 'formatting-adoption' === $this->profile ) { + $tag = $this->rng->choice( array( 'a', 'b', 'big', 'button', 'em', 'font', 'i', 'nobr', 'p', 'span', 'strong' ) ); + $this->mark_feature( 'formatting-adoption-candidate' ); + } + + $close = $this->rng->chance( 'incomplete-malformed' === $this->profile ? 55 : 85 ); + $end = $close ? 'rng->chance( 88 ) ? $tag : $this->tag_name() ) . '>' : ''; + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . $end; + } + + private function weird_element( int $depth ): string { + $this->mark_feature( 'tag:weird-syntax' ); + $case = $this->rng->weighted( + array( + 'unusual-name' => 36, + 'invalid-name' => 30, + 'boundary-spacing' => 18, + 'malformed-closer' => 16, + ) + ); + + if ( 'invalid-name' === $case ) { + $tag = $this->invalid_tag_name(); + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . ( $this->rng->chance( 40 ) ? '' : '' ); + } + + $tag = 'unusual-name' === $case ? $this->unusual_tag_name() : $this->tag_name(); + if ( 'boundary-spacing' === $case ) { + $this->mark_feature( 'tag:weird-spacing' ); + return '<' . $tag . $this->tag_gap() . $this->attrs() . $this->tag_gap() . ( $this->rng->chance( 25 ) ? '/' . $this->tag_gap() : '' ) . '>' . $this->nodes( $depth, 'body' ) . ( $this->rng->chance( 70 ) ? 'tag_gap() . '>' : '' ); + } + + if ( 'malformed-closer' === $case ) { + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . 'invalid_tag_name() . $this->tag_gap() . '>'; + } + + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . ( $this->rng->chance( 75 ) ? '' : '' ); + } + + /** + * select/option/optgroup nesting stress, including elements that end or + * are disallowed in select (input, textarea, button, nested select) and + * select inside table structures. + */ + private function select_stress( int $depth ): string { + $this->mark_feature( 'select' ); + $out = ''; + $count = $this->rng->int( 1, 4 ); + for ( $i = 0; $i < $count; ++$i ) { + switch ( $this->rng->weighted( array( 'option' => 38, 'optgroup' => 22, 'breaker' => 18, 'nested-select' => 10, 'other' => 12 ) ) ) { + case 'option': + $this->mark_feature( 'select:option' ); + $out .= 'attrs() . '>' . $this->terminal_text() . ( $this->rng->chance( 55 ) ? '' : '' ); + break; + case 'optgroup': + $this->mark_feature( 'select:optgroup' ); + $out .= 'attrs() . '>