diff --git a/.gitignore b/.gitignore index 15876fa47fee8..a0a821d4c15f0 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,8 @@ wp-tests-config.php /packagehash.txt /.gutenberg-hash /artifacts +/tools/html-api-fuzz/oracles/lexbor/build +/tools/html-api-fuzz/oracles/html5ever/build /setup.log /coverage @@ -118,3 +120,6 @@ wp-tests-config.php # Visual regression test diffs tests/visual-regression/specs/__snapshots__ + +# Local Chrome for Testing cache used by the HTML API fuzzer. +/tools/html-api-fuzz/oracles/chrome/.chrome-for-testing diff --git a/tools/html-api-fuzz/README.md b/tools/html-api-fuzz/README.md new file mode 100644 index 0000000000000..704a8576de5b9 --- /dev/null +++ b/tools/html-api-fuzz/README.md @@ -0,0 +1,787 @@ +# HTML API Fuzzer + +Pure PHP fuzzer for the WordPress HTML API under `src/wp-includes/html-api`. +It compares `WP_HTML_Processor` against a selectable tree-construction oracle +using an html5lib-style textual tree, and separately checks a set of API +invariants described under “Invariants” below. The default oracle is PHP's +`Dom\HTMLDocument`, preserving the historical behavior. + +The PHP DOM and source-built adapters need no browser. The optional +`chrome-cdp` adapter uses a pinned Chrome-for-Testing build through a direct, +dependency-free Node/CDP client; it does not use Playwright or `wp-env`. + +## Requirements + +- PHP 8.4+ with ext-dom, for `Dom\HTMLDocument`. +- POSIX and PCNTL PHP extensions for Common Crawl worker process-group + isolation. +- Run from the repository root. +- Optional source-built Lexbor oracle: `git`, `cmake`, and a C compiler. +- Optional source-built html5ever oracle: the pinned Rust toolchain installed by + `tools/html-api-fuzz/oracles/html5ever/install-rust.sh`, or matching Rust and + Cargo 1.88.0 executables. +- Optional Chrome oracle: Node.js plus the exact Chrome-for-Testing archive + installed and verified by `tools/html-api-fuzz/oracles/chrome/install.sh`. + +## Common Crawl with cc-analyzer + +`commoncrawl-analysis.php` is an analysis callback for `cc-analyzer.phar`. It +accepts the analyzer's `CcAnalyzer\Analysis\HtmlAnalysisInput` value object, +runs the raw response body through `WP_HTML_Processor` in full-document mode, +compares the resulting tree with the selected oracle, and records the Common +Crawl provenance needed to locate or replay the document. Lexbor is the +default; `html5ever-source` and `chrome-cdp` are equally supported independent +adapters. + +Build the oracle from the current upstream `master` first: + +```sh +tools/html-api-fuzz/oracles/lexbor/build.sh +``` + +Or build the fully pinned html5ever oracle: + +```sh +tools/html-api-fuzz/oracles/html5ever/install-rust.sh +tools/html-api-fuzz/oracles/html5ever/build.sh +``` + +Or install the pinned Chrome-for-Testing oracle: + +```sh +tools/html-api-fuzz/oracles/chrome/install.sh +``` + +The build resolves the moving ref, uses commit-keyed build/install directories, +refuses a dirty Lexbor checkout, and writes `build/build-manifest.json` with the +requested ref, resolved commit, upstream URL, build time, compiler, CMake +version, and oracle SHA-256. For exact reproduction, rebuild the manifest's +`resolvedCommit`, for example: + +```sh +LEXBOR_COMMIT=de1d07a7765aad37090cc36f7fac3bb59e21467d \ + tools/html-api-fuzz/oracles/lexbor/build.sh +``` + +Then point cc-analyzer's HTML analysis-file argument at the absolute path to: + +```text +tools/html-api-fuzz/commoncrawl-analysis.php +``` + +Fetch a named batch exactly once, with an explicit crawl, then run that cached +batch through all three required oracles with the checked coordinator. The +coordinator never fetches or substitutes documents: + +```sh +PHAR=/absolute/path/to/cc-analyzer.phar +WORKSPACE=/absolute/path/to/cc-workspace +BATCH=wp-html-api-canary-2026-25 + +php "$PHAR" --workspace "$WORKSPACE" batch fetch "$BATCH" \ + --crawl CC-MAIN-2026-25 --limit 20 --duration 5m \ + --cache-max-bytes 134217728 --progress text --output json + +php tools/html-api-fuzz/commoncrawl-batch.php \ + --cc-analyzer "$PHAR" --workspace "$WORKSPACE" --batch "$BATCH" \ + --output-dir /absolute/new/path/shared-corpus-canary \ + --batch-timeout-ms 1800000 --process-timeout-ms 90000 \ + --lexbor-oracle-bin tools/html-api-fuzz/oracles/lexbor/build/lexbor-tree-oracle \ + --html5ever-oracle-bin tools/html-api-fuzz/oracles/html5ever/build/html5ever-tree-oracle \ + --chrome-oracle-script tools/html-api-fuzz/oracles/chrome/chrome-tree-oracle.js \ + --chrome-executable /absolute/path/to/pinned/chrome \ + --node-bin /absolute/path/to/non-symlink/node --retain-all +``` + +Every executable passed to the coordinator must be a direct regular file. In +particular, a version-manager shim such as Volta's `~/.volta/bin/node` may be a +symlink and is rejected. Pass the resolved, non-symlink Node executable itself; +`node -p 'process.execPath'` reports the executable running Node. Verify that +reported path with `test -f`, `test -x`, and `test ! -L` before starting the +write-once coordinator output. + +The output path must not exist. It is claimed once and contains separate +`lexbor-source/`, `html5ever-source/`, and `chrome-cdp/` directories. Each gets +its own immutable configuration, stdout and stderr evidence, append-only +summary, exact sorted `replacement-environment.json`, complete marker, and +file-hash seal. `batch-manifest.json` hashes the +cached bytes and records every ordered record ID, state key, target, input +range, SHA-256, and length. `shared-corpus.json` is published only after each +run reports the exact batch count and the three ordered +`(record ID, SHA-256, length)` vectors match. Missing, duplicate, partial, +reordered, changed-cache, identity-drift, or trust-drift runs fail and retain +their partial evidence without a success marker. + +Coordinator children receive a recorded replacement environment, not the +caller's ambient `HTML_API_*`, `NODE_OPTIONS`, PHP configuration overrides, or +loader-injection variables. It probes PHP and every oracle in that environment +and brackets each trust capture with live cache reads. It rechecks the PHAR, +PHP/ini runtime, complete executed repository code set, Git state, normalized +oracle identities, every raw oracle trust file's path/type/mode/bytes, and +cached bodies immediately before and after every run. Before sealing, it +rereads the on-disk batch +manifest, batch stdout, configurations, summaries, and environments. The +sealed `shared-corpus.payload.json` is written first. After all validation and +sealing, the coordinator publishes `shared-corpus.json`, then the per-run +markers, and finally the root `.complete` as the sole authoritative completion +marker and last fallible operation. Consumers must require that root marker; +an interrupted directory may contain an otherwise valid manifest or seal but +is not complete. A successful directory is write-once; use a new output path +for every canary or full run. + +The PHAR owns crawl selection, ranges, checkpoints, and concurrency. The +callback boundary is the same as other cc-analyzer HTML analyses: the file +returns a `static function (HtmlAnalysisInput $document): void` closure. Use +`php /path/to/cc-analyzer.phar --help` for the binary's crawl-specific command +and argument names. The committed coordinator smoke exercises the same batch +schemas and callback contract with an isolated fake analyzer. Real-PHAR runs +remain a separate external gate and are not claimed by that code-only smoke. + +On 2026-07-16, a bounded real-PHAR canary was also completed against +`CC-MAIN-2026-25`. The PHAR SHA-256 was +`898d17d53674b70cd573f598f31d46d92df4f237d3292a9b6ff8c2e5c0c50cde`. +One fetch produced 5 documents and 247070 bytes; its sealed batch-manifest +SHA-256 was +`a37aa5a1a16d7865e1ad5968db30634125dee1514ddae2e5b6e470eef375638e` +and its corpus fingerprint was +`82e37f25a00daca8782f6f7062d8329f04bae1303c8f3a7e0d26691996fd6fa2`. +Lexbor, html5ever, and Chrome each consumed the same ordered five-document +vector and each reported 3 `passed` and 2 `unsupported` results. One retained +artifact per oracle was then replayed exactly once; all three replays reproduced +`unsupported` for the identical 17129-byte input with SHA-256 +`baa6de008c6530fd7828e9c18e274a9fb60f2fe96e17bf0899c8bb9f16626306`, +without a timeout, oracle mismatch, or process-group cleanup failure. The +ignored local evidence is under +`artifacts/html-api-fuzz/external-gates/20260716-ccb5be5be1-real-phar/`. +This was a five-document canary, not a complete-crawl or full-corpus claim. + +The observed post-gate disk footprint was 180964 KiB for `.cache/lexbor`, +653164 KiB for `.cache/html5ever`, 529780 KiB for the pinned Chrome directory, +and 5512 KiB for the canary evidence. Budget at least 1.4 GiB for those local +build/install caches before allowing for fetched bodies and three runs of +output. A larger run needs a fresh batch name and output path plus deliberately +larger fetch limit, duration, cache ceiling, coordinator timeout, and disk +budget. Use the PHAR's help for its unbounded-fetch semantics, and do not use +`--retain-all` at large scale unless retaining every input three times is +intentional. + +The downloads and generated evidence are intentionally uncommitted: +`.cache/lexbor`, `.cache/html5ever`, both source-oracle `build/` directories, +the Chrome `.chrome-for-testing/` installation, and `artifacts/` are ignored. +Preserve or archive an external-gate workspace separately when its audit trail +is needed; do not force-add it to Git. + +Each accepted document runs in a separate PHP child with its own memory and +wall-clock limit. The callback writes `input.bin` and an initial replay before +starting that child. A timeout, memory exhaustion, crash, or signal is retained +as a finding without terminating cc-analyzer or losing the triggering bytes. +The child is a new process-group leader, so its source-oracle descendants are +terminated with it. Stdout/stderr are continuously drained to `worker.log`; +only bounded tails remain in the long-lived analyzer process. + +Each source-oracle identity probe and render also runs through a dedicated +POSIX/PCNTL supervisor. It creates a private `0700` ownership root, copies the +descriptor-stable executable and supervisor to `0500` snapshots, starts a new +session with a persistent process-group anchor, and does not release the target +until authenticated state has been durably published. Cleanup reauthenticates +the exact process identities, sends group `TERM`, waits 250 ms, then sends one +`KILL` and verifies group/root absence. Owner EOF uses the same path. Uncertain +identity or cleanup state is an infrastructure failure and retains evidence +instead of signaling an unverified PID. Source stdout is capped at exactly +64 MiB and stderr at 1 MiB; either overflow is an infrastructure failure. + +The Chrome adapter instead keeps one authenticated `node ... --serve` process +for the lifetime of one renderer, so a Worker's baseline and mutation renders +reuse the same exact browser. It verifies the checked-in version/checksum +manifests, Chrome executable, oracle script, fragment-context file, actual Node +executable, live Chrome version, and CDP protocol version before accepting any +tree. Request JSON is strict, duplicate keys and trailing frames are rejected, +input is canonical base64, response frames are capped at 24 MiB, stderr at +1 MiB, and decoded trees at 16 MiB. Accepted input is capped at exactly 2 MiB; +2 MiB plus one byte becomes `input-byte-limit-exceeded` without sending a +render frame. Shutdown drains and reaps the exact service and verifies that its +private runtime/profile root disappeared. A transport failure never retries +the current input in PHP; the Node client may perform its one authenticated +recovery only when it observes a dead CDP session. + +The Common Crawl adapter uses Lexbor by default. Its environment is: + +- `CC_ANALYZER_OUTPUT_DIR`: shared output directory for this analyzer run. If + absent, a unique directory is created under `artifacts/html-api-commoncrawl`. +- `HTML_API_CC_RUN_ID`: explicit immutable run identifier. When the analyzer + provides an output directory, the default is a stable hash of that path. +- `HTML_API_CC_ORACLE`: `lexbor-source` (default), `html5ever-source`, + `chrome-cdp`, or the `php-dom` testing/debug escape hatch. +- `HTML_API_FUZZ_LEXBOR_ORACLE`: non-default Lexbor oracle binary path. +- `HTML_API_FUZZ_HTML5EVER_ORACLE`: non-default html5ever oracle binary path. +- `HTML_API_FUZZ_CHROME_ORACLE`: non-default Chrome oracle script path. +- `HTML_API_FUZZ_CHROME_EXECUTABLE`: non-default pinned Chrome executable. +- `HTML_API_FUZZ_NODE_BIN`: Node command or executable path. +- `HTML_API_FUZZ_CHROME_STARTUP_TIMEOUT_MS`: browser startup deadline; default + `35000`. `HTML_API_CC_CHROME_STARTUP_TIMEOUT_MS` is the Common Crawl-specific + override. +- `HTML_API_CC_EXPECT_LEXBOR_COMMIT`: optional exact commit assertion; startup + fails if the selected Lexbor executable reports another commit. Setting this + while another oracle is selected also fails. +- `HTML_API_CC_EXPECT_ORACLE_IDENTITY_SHA256`: optional SHA-256 of the complete + normalized selected-oracle metadata envelope. This generic pin works for + PHP DOM, Lexbor, html5ever, and Chrome. +- `HTML_API_CC_ORACLE_TIMEOUT_MS`: per-render oracle timeout; default `10000`. +- `HTML_API_CC_PROCESS_TIMEOUT_MS`: explicit whole-document worker timeout. + Without it, non-Chrome oracles use `30000`; Chrome budgets startup plus four + render calls plus 10 seconds of cleanup and 5 seconds of PHP headroom. With + the defaults here that is `90000` ms. An explicit shorter value remains an + exact diagnostic override and is recorded in every replay. +- `HTML_API_CC_MEMORY_LIMIT`: PHP memory limit for each worker; default `256M`. +- `HTML_API_CC_MAX_INPUT_BYTES`: largest response body to analyze; default + `2097152`, or `0` for unlimited with non-Chrome oracles. Chrome retains its + authenticated transport ceiling of 2 MiB. +- `HTML_API_CC_MAX_TOKENS`, `HTML_API_CC_MAX_NODES`, + `HTML_API_CC_MAX_DEPTH`, and `HTML_API_CC_MAX_TREE_BYTES`: bounded work and + tree-output ceilings; defaults are `50000`, `50000`, `512`, and `16777216`. +- `HTML_API_CC_CHECKS`: `baseline`, `full`, or `sampled` (default). Baseline + performs the WordPress/selected-oracle differential; full also runs API invariants, + mutation differential, and normalize preservation. +- `HTML_API_CC_FULL_SAMPLE_PERCENT`: deterministic full-check share when using + `sampled`; default `1`. +- `HTML_API_CC_REQUIRE_UTF8`: defaults to `1`. Invalid UTF-8 bodies are + summarized as skipped because the WordPress HTML API consumes UTF-8 and a + legacy-encoded response otherwise produces mostly encoding noise. Set it to + `0` for raw-byte/encoding fuzzing. +- `HTML_API_CC_MAX_KEEP_PER_SIGNATURE`: retained examples per distinct + signature; default `3`. +- `HTML_API_CC_RETAIN_ALL`: retain passing inputs too; default `0`. +- `CC_ANALYZER_VERSION`, `CC_ANALYZER_CRAWL`, and `CC_ANALYZER_INVOCATION`: + optional provenance strings recorded verbatim in the immutable run config. + +The output root contains: + +- `configuration.json`: run ID and exact configuration fingerprint plus Git, + PHP, normalized selected-oracle identity/hash, limits, and analyzer + provenance. Reusing the directory with different settings fails at startup. +- `commoncrawl-summary.ndjson`: one locked, append-only record per document; + safe for concurrent analyzer workers. +- `coverage.json`: aggregate total, covered, failed, and per-status counts. +- `findings/SIGNATURE/DOCUMENT/`: bounded byte-exact exemplars containing + `input.bin`, `result.json`, `replay.json`, both rendered trees, and a + `.complete` marker. Findings are written in `pending/` and atomically renamed + only after every checked write succeeds; retries of the same record/body are + idempotent. Any directory left in `pending/` is recoverable evidence of a + callback interruption. + +Passing inputs are represented only by their summary by default. A finding's +replay embeds the exact input body, so it remains reproducible without another +Common Crawl download. This is also true when the worker died before producing +its own manifest—the callback's pre-worker replay already contains the input, +oracle configuration, limits, and provenance: + +```sh +php tools/html-api-fuzz/replay.php \ + --replay artifacts/html-api-commoncrawl/findings/SIGNATURE/DOCUMENT/replay.json +``` + +A coordinator-retained artifact uses the same replay command and lives below +the selected oracle's sealed run directory. Always write the replay to a new +output directory: + +```sh +php tools/html-api-fuzz/replay.php \ + --replay /absolute/path/shared-corpus/ORACLE/findings/SIGNATURE/DOCUMENT/replay.json \ + --output-dir /absolute/new/path/replay-ORACLE +``` + +Replay uses the finding's recorded Worker script, PHP memory limit, whole-worker +timeout, check pipeline, and oracle timeout by default. It applies the same +process-group isolation and synthesizes the same timeout/OOM/crash result when +the Worker produces none. `--worker-script`, `--memory-limit`, and +`--timeout-ms` are explicit diagnostic overrides. + +Replay and minimization verify the complete recorded oracle identity before +creating or claiming an output directory. Lexbor pins its resolved commit, +manifest fields, self-report, and executable SHA-256. html5ever additionally +pins the checked source/lock/toolchain hashes, direct crate versions/checksums, +build identity, and executable SHA-256. Use `--allow-oracle-mismatch` only for a +deliberate diagnostic comparison; outputs retain `sourceOracle`, +`actualOracle`, and the exact mismatch reasons. A second identity change while +a Worker is running is recorded separately and invalidates any stale signature. +Chrome replay additionally restores and validates the exact script, Chrome and +Node paths, render deadline, startup deadline, and complete durable identity; +runtime PIDs, profiles, and debugging endpoints are deliberately excluded. + +Transport charset, content type, target URI, WARC record ID, analyzer state +key, and source range are metadata only. The comparison intentionally feeds +the same raw bytes to both parsers; it never silently transcodes one side. + +## Common Commands + +Run one generated seed: + +```sh +php tools/html-api-fuzz/worker.php --seed 1 --output-dir artifacts/html-api-fuzz/seed-1 +``` + +Run one generated seed with a specific terminal payload policy: + +```sh +php tools/html-api-fuzz/worker.php --seed 1 --payload-policy valid-utf8 --output-dir artifacts/html-api-fuzz/seed-1 +``` + +Run a batch in worker subprocesses (seeds are batched into shared worker +processes, 25 per process by default; see `--batch-size`): + +```sh +php tools/html-api-fuzz/runner.php --max-seeds 100 --duration-seconds 60 +``` + +Run a structural UTF-8-biased batch with a post-generation byte cap: + +```sh +php tools/html-api-fuzz/runner.php --max-seeds 100 --payload-policy valid-utf8 --max-input-bytes 4096 +``` + +Build and run against the source-built Lexbor oracle: + +```sh +tools/html-api-fuzz/oracles/lexbor/build.sh +php tools/html-api-fuzz/worker.php --seed 1 --dom-oracle lexbor-source --output-dir artifacts/html-api-fuzz/seed-1-lexbor +php tools/html-api-fuzz/runner.php --max-seeds 100 --dom-oracle lexbor-source --duration-seconds 60 +``` + +Use `--lexbor-oracle-bin PATH` or `HTML_API_FUZZ_LEXBOR_ORACLE` when the +oracle binary is not at +`tools/html-api-fuzz/oracles/lexbor/build/lexbor-tree-oracle`. + +Build and run against the pinned html5ever oracle: + +```sh +tools/html-api-fuzz/oracles/html5ever/install-rust.sh +tools/html-api-fuzz/oracles/html5ever/build.sh +php tools/html-api-fuzz/worker.php --seed 1 --dom-oracle html5ever-source --output-dir artifacts/html-api-fuzz/seed-1-html5ever +php tools/html-api-fuzz/runner.php --max-seeds 100 --dom-oracle html5ever-source --timeout-ms 10000 +``` + +Use `--html5ever-oracle-bin PATH` or +`HTML_API_FUZZ_HTML5EVER_ORACLE` for a non-default build location. Source +adapters add process-supervision overhead, so give `runner.php` a whole-worker +`--timeout-ms` budget large enough to cover the configured oracle deadline. + +Install and run against the pinned direct-CDP Chrome oracle: + +```sh +tools/html-api-fuzz/oracles/chrome/install.sh +php tools/html-api-fuzz/worker.php --seed 1 --dom-oracle chrome-cdp --output-dir artifacts/html-api-fuzz/seed-1-chrome +php tools/html-api-fuzz/runner.php --max-seeds 100 --dom-oracle chrome-cdp +``` + +Use `--chrome-oracle-script`, `--chrome-executable`, `--node-bin`, and +`--chrome-startup-timeout-ms` (or their `HTML_API_FUZZ_*` environment +equivalents) for explicit locations/deadlines. When `--timeout-ms` is omitted, +runner, launcher, replay, and minimization use the shared Chrome budget formula: +`startup + (1 baseline or 4 full/sampled renders) × oracle timeout + 15000`. +The ordinary defaults are 52.5 seconds for baseline and 60 seconds for full. + +Run indefinitely: + +```sh +php tools/html-api-fuzz/runner.php --duration-seconds 0 --max-seeds 0 +``` + +Run parallel lanes and triage failures after completion: + +```sh +php tools/html-api-fuzz/launcher.php --lanes 4 --max-seeds 1000 --watcher +``` + +For continuous fuzzing, run the launcher with `--duration-seconds 0 --max-seeds 0` +and run `watcher.php` in a second shell against the same output directory. + +Stop an indefinite run gracefully (each lane finishes its current batch, the +watcher performs a final scan, and the codex orchestrator drains its running +jobs): + +```sh +php tools/html-api-fuzz/stop.php --run-dir artifacts/html-api-fuzz/run-... +``` + +Without `--run-dir` the most recently active *unfinished* run under +`artifacts/html-api-fuzz` is targeted. Finished and stale runs are not +preferred; if nothing live is found, the most recent stopped-looking run is +targeted with a warning. The script creates the stop file advertised by the +run state and also `RUN_DIR/STOP` when a run directory is known, so watchers +and orchestrators see the stop request. For a standalone runner with custom +`--stop-file PATH`, both files are written. Relative custom stop files are +resolved with the runner cwd recorded in new runner state; for older state, +pass only `--stop-file PATH` to write a known stop file directly if needed. +With `--run-dir --stop-file PATH`, the explicit path is added to the run-state +and `RUN_DIR/STOP` targets. `touch` works just as well. The launcher and runner refuse to start while a stop file already +exists — remove `STOP` (or the custom stop file) before reusing a run +directory. (A stop requested in the sub-second window between the launcher's +startup check and a lane's own makes that lane refuse rather than stop +gracefully; the run still ends.) +If state cannot be read or old state lacks enough context to locate a relative +custom stop file, the tool still writes `RUN_DIR/STOP` but exits `2` with +`ok: false` and warnings because a standalone custom stop file may be unknown. + +The watcher exits after a final scan once every runner under the run +directory reports a stop reason. A runner whose state has gone silent is +presumed dead after `--stop-stale-seconds` (default 120); per lane that +threshold is floored at twice the lane's advertised batch budget +(`timeout-ms × batch-size`), so long batches are not mistaken for crashes. +The stop tool uses the same default stale threshold when auto-selecting the +latest unfinished run. + +Replay a failure from a retained seed directory, or from the lane's SQLite +store when the seed directory was pruned (see "Artifact Retention"): + +```sh +php tools/html-api-fuzz/replay.php --replay artifacts/html-api-fuzz/run-.../seed-.../primary/replay.json +php tools/html-api-fuzz/replay.php --store artifacts/html-api-fuzz/run-.../lane-00/results.sqlite --seed 12345 +``` + +Minimize a failure while preserving the same signature: + +```sh +php tools/html-api-fuzz/minimize.php --replay artifacts/html-api-fuzz/run-.../seed-.../primary/replay.json +``` + +By default (`--probe-mode auto`) the minimizer evaluates candidates in worker +subprocesses so `--timeout-ms` can kill pathological candidates and each probe +starts with fresh PHP state. Use `--probe-mode in-process` for faster +exact-signature minimization when that isolation is not needed; in-process +probes write only the final minimized artifacts unless +`--keep-candidate-artifacts` is also passed. Use `--probe-mode process` to +force subprocess probes explicitly. + +Watch an existing run directory and minimize new distinct signatures: + +```sh +php tools/html-api-fuzz/watcher.php --run-dir artifacts/html-api-fuzz/run-... --once +``` + +Configured ceilings are reported as `failureClass: "resource-limit"` and remain +in the watcher/minimizer triage path. This bucket includes tag/tree token +ceilings (`tag-token-limit-exceeded`, `mutation-token-limit-exceeded`, +`wordpress-token-limit-exceeded`) and oracle node ceilings +(`node-limit-exceeded`, recorded as `dom-node-limit-exceeded` in historical +signature facts), plus the Chrome adapter's exact 2 MiB input ceiling. Process +timeouts, PHP fatal errors, and memory failures are +separate failures and are also in scope for triage. + +## Execution Model + +The runner batches consecutive seeds into one worker process +(`worker.php --batch-count N`, default `--batch-size 25` on the runner) so the +WordPress bootstrap and process spawn are paid once per batch rather than once +per seed. Each seed still writes its own `seed-N/primary` artifacts. If a batch +process dies or times out mid-way, seeds left without a `result.json` are +re-run individually in isolation, so a crash on one input cannot take +neighboring seeds' results with it. + +## Input Stages + +Seeds are deterministically split between two input stages: + +- **Generated** (default ~80%): the structural grammar described under + “Generator Profiles”. +- **Corpus-mutated** (default ~20%, `--corpus-mutate-percent N` on + `worker.php`/`runner.php`): a `#data` section from the html5lib-tests + tree-construction corpus (`tests/phpunit/data/html5lib-tests`), passed + through 1–4 deterministic mutations (byte insert/replace, chunk + delete/duplicate, tag-name swap, case toggle, corpus splice). The stage, + corpus file, entry index, and operations are recorded in result metadata, + and the mutated input itself is in the replay manifest, so replays are + standalone. Inputs report `inputSource: "corpus-mutated"` and + `profile: "corpus-mutated"`. + +Both stages derive entirely from the seed, so seed N always produces the same +input for the same fuzzer version and corpus. + +## Artifact Layout + +The runner writes: + +- `results.sqlite`: one row per attempted seed (table `attempts`, WAL mode). + Passing attempts store summary columns only — every attempt is regenerable + from its seed. Failure rows additionally store the summary, result, and + replay JSON documents; the replay embeds the input as base64, so a pruned + failure can be reproduced with `replay.php --store results.sqlite --seed N`. + `signature_hash` and `family_key` are indexed columns for grouping + failures without `json_extract`. `oracle_kind`, `oracle_identity_sha256`, + `oracle_version`, `oracle_commit`, and `oracle_binary` record which oracle + generated the summary, including for passing rows whose JSON payloads are + pruned. `oracle_identity_sha256` groups the complete normalized durable + identity. `oracle_commit` is the Lexbor commit, html5ever build identity, or + Chrome script SHA-256, while `oracle_binary` is the source binary or Chrome + executable SHA-256 rather than a host path. The + watcher tails these stores + incrementally by row id. (`summary.ndjson` files from older runs are still + scanned.) Durability is `synchronous=NORMAL`: an OS crash (not a process + crash) can lose the last moments of a run. +- `events.ndjson`: runner lifecycle events, including batch boundaries. +- `logs/batch-N.log`: output of a batch worker process, kept only when the + batch contained a retained failure or a seed that needed an isolated + re-run — over-cap repeats of a known signature do not accumulate batch + logs. +- `state.json`: aggregate counters, stop reason, and compact Git metadata. + Oracle losses are counted per class: `oracleParseErrors` (inputs the + selected oracle rejects receive no differential coverage), + `oracleUnsupported` (tree shapes the oracle cannot represent), and + `oracleTolerated` + (comparisons that passed only under the documented scalar tolerance). +- `seed-N/primary/input.bin`: raw generated bytes. +- `seed-N/primary/replay.json`: base64 replay manifest, including the commit + hash, tracked-file dirty state, selected oracle, and fragment context needed + to interpret a standalone replay. +- `seed-N/primary/result.json`: full worker result. +- `seed-N/primary/wordpress-tree.txt` and `dom-tree.txt`: rendered trees when available. + +### Artifact Retention + +Seed directories are working space, not the archive. After each seed is +recorded in `results.sqlite`, its `seed-N` directory is deleted unless the +attempt failed *and* the failure's signature has fewer than +`--max-keep-per-signature` (default 5, minimum 1) exemplar directories still +on disk in this lane. The cap counts directories, not rows, so restarting a +runner against the same output directory neither double-counts re-recorded +seeds nor deletes a previously retained exemplar. The first exemplar of every +new signature is always retained, so the watcher's minimization path keeps a +replay file to work from; subsequent repeats of a known signature add a +database row and nothing else. A failure whose replay document is missing +(worker killed before writing it) always keeps its directory — the files are +the only reproduction. Disk growth is therefore proportional to *new +distinct failures*, not to seeds executed. + +The cap applies per lane: the launcher passes the same value to every lane, +so a signature that appears in all lanes keeps up to `N × lanes` directories +across the run. Use `--keep-all-artifacts` (runner or launcher) to keep every +seed directory for debugging. + +The run-level Git metadata intentionally stays compact: full and short commit +hash, current branch when available, commit date, and a dirty flag for +tracked-file changes. The dirty flag is tri-state: `true`, `false`, or `null` +when Git is unavailable or dirty detection fails. Full `git status` or diff +output is not recorded because it is noisy, can expose local edits, and grows +indefinitely in long runs. Launcher and runner processes collect this metadata +once and pass it to workers so long runs do not invoke Git for every seed. +Replayed and minimized manifests keep current checkout metadata at the top level +and preserve discovery provenance in `sourceReplay`. + +The watcher writes triage state under `.triage-watcher` by default, or under +`--state-dir` when provided. Each signature gets a stable directory containing +`failure.json`, minimizer logs, and minimized replay/result artifacts. Failed +minimizations are retried on later scans, up to `--max-minimize-retries` +(default 3) attempts per signature. + +## Modes and Fragment Contexts + +- `fragment-body`: parse as a fragment. The selected oracle uses real fragment + parsing (the `innerHTML` setter on a context element of an empty document), + not a document-wrapping approximation. +- `full-document`: parse as a full HTML document. +- `auto`: weighted choice. + +In fragment mode a context element is selected per seed +(`--fragment-context TAG` on `worker.php` for replays). `` dominates; +the other contexts (`div`, `p`, `td`, `tr`, `table`, `caption`, `colgroup`, +`select`, `option`, `template`, `title`, `textarea`, `script`, `style`, +`svg`, `math`) receive a small probe weight. `WP_HTML_Processor::create_fragment()` +currently supports only ``, so non-body contexts are recorded as +`status: "unsupported"` today; when create_fragment() gains context support +the fuzzer picks up the new coverage with no changes. The selected oracle already +parses every context correctly. + +Unsupported `WP_HTML_Processor` cases are expected by default and are recorded +as successful attempts with `status: "unsupported"`. Use `--fail-unsupported` +when you want unsupported cases to become failures. + +## Invariants + +Each seed checks, in order, stopping at the first failing class: + +1. **Tag Processor invariants** (`tag-invariant-failed`): token loop + termination under the token ceiling; non-null token type/name/tag; + attribute getters and `class_list()` iteration do not throw; + `get_updated_html()` with no queued edits returns the input unchanged; a + simple `set_attribute()` mutation is visible to a re-scan; and + **seek consistency** — a bookmark set at a seed-chosen token, after + scanning to the end and seeking back, must reproduce the identical token + stream (`seek-token-stream-mismatch`). +2. **Differential tree comparison** (`tree-mismatch` / `encoding-mismatch`): + the WordPress tree must equal the selected oracle tree (see “Tree Comparison”). +3. **Breadcrumb consistency** (`breadcrumb-mismatch`): at every tag token, + `get_breadcrumbs()` must agree with the element stack derived from token + order and `expects_closer()`. +4. **Mutation differential** (`mutation-tree-mismatch` / + `mutation-delta-mismatch`), only on a clean baseline: after setting + `data-fuzz="1"` on the first tag, the mutated document must parse + identically in WordPress and the selected oracle, and the WordPress tree must + change by exactly the one attribute line (unless formatting-element + reconstruction clones the attribute, or tree construction legitimately + drops the mutated element, in which case the differential comparison alone + applies). +5. **Normalize tree preservation** (`normalize-tree-changed`), only on a + clean baseline: parsing `normalize()` output must produce the same tree as + the original input, modulo the documented scalar substitutions. This is + stricter than idempotence, which a consistently wrong serializer can pass. +6. **Normalize idempotence** (`normalize-invariant-failed`): + `normalize()` / `serialize()` run twice must be a fixed point, with no + PHP native errors or throwables. Full documents use + `create_full_parser()->serialize()`; non-body fragment contexts use + `create_fragment()->serialize()`. + +### Known invariant oracle follow-ups + +- The simple `set_attribute()` mutation oracle needs to handle inputs that + begin with `
` using the same tag-selection semantics as the mutator. + `next_tag()` skips the raw closing token and mutates the following tag, + while a verifier that scans with `next_token()` can see the spec-special + `BR` element synthesized from `
` first and incorrectly report + `mutation-attribute-missing`. Fix the verifier by selecting the first + mutable tag through `next_tag()` too. + +## Generator Profiles + +The generator uses a structural HTML grammar with weighted profiles: + +- `balanced` +- `full-document` (includes occasional frameset documents, quirks-mode + doctypes, and content after ``) +- `body-fragment` +- `tables` +- `template` +- `select` (option/optgroup nesting, select-ending elements such as `input`, + `textarea` and `button`, nested selects, select-in-table) +- `foreign-content` (MathML/SVG integration points, HTML breakout tags, + `` with and without breakout attributes, `annotation-xml` encoding + variants, CDATA sections in foreign content, case-mangled `foreignObject`) +- `rawtext-rcdata` (script/style/iframe/noembed/noframes/xmp/noscript, + title/textarea, occasional `plaintext`) +- `text-fragment` (standalone terminal payloads, biased toward exact + 0-10 byte inputs plus medium syntax-heavy text unless `stress-long` is + selected explicitly) +- `formatting-adoption` (random formatting elements plus explicit + adoption-agency shapes: misnested closers, block-boundary formatting, + reconstruction across siblings, nested anchors, Noah's Ark overflow, + repeated closers, and four-plus active-formatting reconstruction clusters + covering same tag/no attributes, same tag/distinct attributes, same + tag/matching attributes, mixed interleavings, and table-cell marker + isolation/clearing) +- `attributes-entities` +- `comments-doctype-bogus` +- `deep-nesting` +- `resource-stress` +- `incomplete-malformed` (includes spec-special closers such as `
` and + `

`, stray closers, and ``) + +All profiles can emit duplicate attribute names (first-wins coverage), +auto-closing chains (`li`, `dd`/`dt`, headings, `p`), and named character +references with longest-prefix-match ambiguity (`¬it;`, `©right;`, +`&ngE`, ...). + +Terminal payloads are selected by a separate policy: + +- `valid-utf8`: structural cases with ASCII, Unicode, controls, and + entity references, including NUL byte coverage, but no raw invalid bytes. +- `mostly-valid`: default-biased structural cases with valid UTF-8 Unicode, + controls, NUL bytes, and entity references. +- `ascii-structural`: ASCII-only terminal text and attributes for tokenizer and + tree-construction coverage, including NUL byte coverage. +- `stress-long`: long valid UTF-8 terminal payloads for deliberate + resource-stress runs. +- `auto`: weighted choice. Normal structural profiles favor valid UTF-8 and + mostly-valid payloads; `resource-stress` favors `stress-long`. + +Use `--payload-policy POLICY` on `worker.php`, `runner.php`, or `launcher.php`. +Use `--max-input-bytes N` to apply a post-generation byte cap before the worker +records replay metadata. The cap preserves UTF-8 byte boundaries, but it is not +grammar-aware and may cut through HTML tokens. Replay manifests and minimization +summaries preserve the original payload policy when it was recorded; old or +hand-supplied inputs leave `payloadPolicy` null unless an explicit policy label +is provided. Historical `invalid-byte-heavy` labels are accepted only as replay +metadata for direct inputs and are not selectable for generated runs. +Replayed and minimized manifests keep immediate `inputSource` metadata separate +from `originalGenerator` metadata. + +## Tree Comparison + +The tree renderer follows the html5lib test style used by +`tests/phpunit/tests/html-api/wpHtmlProcessorHtml5lib.php`: + +- attributes sorted by their spec-scrubbed names (so a raw-NUL name on the + WordPress side and its U+FFFD substitution on the DOM side sort identically), + rendered raw +- boolean attributes rendered as `=""` +- namespace-qualified element and attribute names +- template `content` marker +- only the narrow auto-generated `html/head/body` wrapper tolerance + +When the default PHP DOM oracle is selected, template content is rendered +through a self-contained serialization round-trip: PHP hides template child +nodes, so the oracle re-parses the template's `innerHTML` serialization in a +body context and accepts the result only when re-serializing reproduces the +source byte-for-byte. Content that cannot round-trip (table parts, foreign +fragments) is quarantined as `oracle-unsupported`. This check never consults +the WordPress HTML API, which is the system under test. + +Raw bytes are rendered without normalization. The WordPress HTML API +deliberately preserves NUL and CR bytes where spec-following parsers +substitute U+FFFD and normalize newlines during input preprocessing, so the +comparison tolerates a differing line only when that exact substitution +explains the entire difference. The tolerance is additionally gated by line +type: WordPress preserves raw bytes only in attribute values and +tag/attribute names (verified empirically across text, RCDATA, rawtext, +foreign text, CDATA, comment, and doctype contexts, where WordPress applies +the spec substitutions itself), so only tag lines and attribute lines are +eligible. A scalar difference on any other line type is a real divergence +and fails. Tolerated lines are reported per seed +(`comparison.scalarToleratedLines`) and per run (`oracleTolerated`), and the +result is classified `oracle-tolerated` rather than silently passed. Any +difference beyond the substitution fails as usual, and the first-difference +record points at the first *unexplained* line. + +One known PHP DOM oracle bug is tolerated with a runtime probe: PHP's Lexbor parser +fails to treat U+000C FORM FEED as ignorable whitespace in the pre-body +insertion modes. When a full-document comparison fails, the input contains a +form feed, and re-parsing with form feeds substituted by spaces makes the DOM +oracle reproduce the WordPress tree exactly, the case is classified +`oracle-tolerated` with `comparison.formFeedQuirk: true`. The probe disables +the tolerance automatically when PHP fixes the bug. + +Invalid bytes are never normalized away. If WordPress and the selected oracle +surface different byte sequences, the first-difference record includes bounded +line previews, byte lengths, line hashes, the first differing byte offset, and +hex previews, including a diff-window hex preview around the differing byte, so +the mismatch remains inspectable even when JSON display substitutes replacement +characters. Full comparison lines are kept out of `result.json` to avoid large +artifacts from stress inputs. + +### Known classification gaps + +Two known issues affect labeling, not pass/fail correctness: + +- **Dual-axis lines classify as `tree-mismatch`, not `encoding-mismatch`.** + A differing line explained only by *both* an invalid-UTF-8 substitution and + a NUL/CR scalar substitution (e.g. an attribute value containing a raw NUL + *and* a raw `0x82`) matches neither single-axis check: + `linesMatchAfterWordPressUtf8Scrub` fails on the unscrubbed NUL, and the + scalar matcher fails on the invalid byte. Such results report + `tree-mismatch` although encoding is involved. +- **CDATA at SVG/MathML integration points is a real WordPress divergence + the fuzzer will keep reporting.** For + `a\0b` (likewise ``, + ``, HTML-encoded `annotation-xml`), WordPress substitutes + NUL with U+FFFD in CDATA text while the spec routes those characters + through the HTML insertion mode, which drops NUL — WordPress handles + plain text at integration points correctly; only the CDATA path diverges + (a known `@todo` in `WP_HTML_Tag_Processor`'s CDATA handling). The shape + is U+FFFD-versus-removed, which no tolerance covers in either direction, + so these report as genuine `tree-mismatch` findings. This is distinct + from the upstream-fixed PHP-DOM integration-point reparenting family. + +## Minimization + +`minimize.php` reduces in three phases under a shared attempt budget +(`--max-attempts`, default 600): markup-aligned segment deletion, binary +byte-chunk deletion, then per-byte deletion and canonicalization (replacements +never grow the input). Every accepted candidate re-runs the worker and must +reproduce the original signature hash (or any failure with `--any-failure`). diff --git a/tools/html-api-fuzz/codex-triage-orchestrator.php b/tools/html-api-fuzz/codex-triage-orchestrator.php new file mode 100755 index 0000000000000..17a62d6d6ea93 --- /dev/null +++ b/tools/html-api-fuzz/codex-triage-orchestrator.php @@ -0,0 +1,935 @@ +#!/usr/bin/env php + $value ) { + if ( '_' === $name ) { + if ( ! empty( $value ) ) { + throw new InvalidArgumentException( 'Unexpected positional argument: ' . (string) reset( $value ) ); + } + continue; + } + + if ( ! in_array( $name, $allowed, true ) ) { + throw new InvalidArgumentException( 'Unknown option: --' . $name ); + } + if ( in_array( $name, $value_options, true ) && true === $value ) { + throw new InvalidArgumentException( 'Expected --' . $name . ' to have a value.' ); + } + if ( in_array( $name, $bool_options, true ) && true !== $value ) { + throw new InvalidArgumentException( 'Option --' . $name . ' does not accept a value.' ); + } + } +} + +function html_api_fuzz_codex_is_absolute_path( string $path ): bool { + return 1 === preg_match( '#^(?:/|[A-Za-z]:[\\\\/]|\\\\\\\\)#', $path ); +} + +function html_api_fuzz_codex_now_iso_z(): string { + return gmdate( 'Y-m-d\TH:i:s\Z' ); +} + +function html_api_fuzz_codex_normalize_path( string $path ): string { + if ( '' === $path ) { + throw new InvalidArgumentException( 'Expected a non-empty path.' ); + } + + if ( ! html_api_fuzz_codex_is_absolute_path( $path ) ) { + $cwd = getcwd(); + if ( false === $cwd ) { + throw new RuntimeException( 'Could not determine current working directory.' ); + } + $path = $cwd . DIRECTORY_SEPARATOR . $path; + } + + $real = realpath( $path ); + return false === $real ? rtrim( $path, DIRECTORY_SEPARATOR ) : $real; +} + +function html_api_fuzz_codex_require_dir( string $path, string $label ): string { + $normalized = html_api_fuzz_codex_normalize_path( $path ); + if ( ! is_dir( $normalized ) ) { + throw new InvalidArgumentException( "{$label} is not a directory: {$normalized}" ); + } + + return $normalized; +} + +function html_api_fuzz_codex_path_join( string $base, string ...$parts ): string { + $path = rtrim( $base, DIRECTORY_SEPARATOR ); + foreach ( $parts as $part ) { + $path .= DIRECTORY_SEPARATOR . ltrim( $part, DIRECTORY_SEPARATOR ); + } + + return $path; +} + +function html_api_fuzz_codex_signature_dir_name( string $signature_hash ): string { + $dir = preg_replace( '/[^a-zA-Z0-9._-]+/', '_', $signature_hash ); + return null === $dir || '' === $dir ? '_' : $dir; +} + +function html_api_fuzz_codex_valid_signature_hash( string $signature_hash ): bool { + return 1 === preg_match( '/^[A-Za-z0-9._-]+$/', $signature_hash ); +} + +function html_api_fuzz_codex_safe_prompt_line( $value, string $field ): string { + if ( null === $value ) { + return ''; + } + if ( ! is_scalar( $value ) ) { + throw new InvalidArgumentException( "Expected {$field} to be scalar." ); + } + + $text = (string) $value; + if ( 1 === preg_match( '/[\x00-\x1F\x7F]/', $text ) ) { + throw new InvalidArgumentException( "Unsafe control character in {$field}." ); + } + + return $text; +} + +function html_api_fuzz_codex_path_has_control_chars( string $path ): bool { + return 1 === preg_match( '/[\x00-\x1F\x7F]/', $path ); +} + +function html_api_fuzz_codex_path_is_under( string $path, string $root ): bool { + $real_path = realpath( $path ); + $real_root = realpath( $root ); + if ( false === $real_path || false === $real_root ) { + return false; + } + + $real_root = rtrim( $real_root, DIRECTORY_SEPARATOR ); + return $real_path === $real_root || 0 === strpos( $real_path, $real_root . DIRECTORY_SEPARATOR ); +} + +function html_api_fuzz_codex_allowed_artifact_roots( array $args ): array { + $roots = array( + $args['triageDir'], + $args['runDir'] ?? dirname( $args['triageDir'] ), + ); + $allowed = array(); + foreach ( $roots as $root ) { + if ( is_string( $root ) && '' !== $root && is_dir( $root ) && ! html_api_fuzz_codex_path_has_control_chars( $root ) ) { + $real = realpath( $root ); + if ( false !== $real ) { + $allowed[] = $real; + } + } + } + + return array_values( array_unique( $allowed ) ); +} + +function html_api_fuzz_codex_sort_json_value( $value ) { + if ( ! is_array( $value ) ) { + return $value; + } + + if ( array_is_list( $value ) ) { + return array_map( 'html_api_fuzz_codex_sort_json_value', $value ); + } + + foreach ( $value as $key => $item ) { + $value[ $key ] = html_api_fuzz_codex_sort_json_value( $item ); + } + ksort( $value, SORT_STRING ); + + return $value; +} + +function html_api_fuzz_codex_json_encode( $value ): string { + $json = json_encode( + html_api_fuzz_codex_sort_json_value( $value ), + JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_SUBSTITUTE + ); + if ( false === $json ) { + throw new RuntimeException( 'JSON encode failed: ' . json_last_error_msg() ); + } + + return $json; +} + +function html_api_fuzz_codex_read_json( string $path, int $retries = 5 ) { + $last_error = null; + for ( $i = 0; $i < $retries; ++$i ) { + if ( ! file_exists( $path ) ) { + return null; + } + + $text = @file_get_contents( $path ); + if ( false === $text ) { + throw new RuntimeException( "Could not read JSON file: {$path}" ); + } + + $value = json_decode( $text, true ); + if ( JSON_ERROR_NONE === json_last_error() ) { + return $value; + } + + $last_error = json_last_error_msg(); + usleep( 200000 ); + } + + throw new RuntimeException( "Could not parse JSON {$path}: {$last_error}" ); +} + +function html_api_fuzz_codex_exit_code_from_done( $done ): int { + if ( ! is_array( $done ) || ! array_key_exists( 'exitCode', $done ) || ! is_numeric( $done['exitCode'] ) ) { + return 1; + } + + return (int) $done['exitCode']; +} + +function html_api_fuzz_codex_write_json( string $path, $value ): void { + $dir = dirname( $path ); + \HtmlApiFuzz\ensure_dir( $dir ); + $tmp = tempnam( $dir, basename( $path ) . '.tmp.' ); + if ( false === $tmp ) { + throw new RuntimeException( "Could not create temporary JSON file for: {$path}" ); + } + $json = html_api_fuzz_codex_json_encode( $value ) . "\n"; + if ( false === file_put_contents( $tmp, $json ) ) { + @unlink( $tmp ); + throw new RuntimeException( "Could not write temporary JSON file: {$tmp}" ); + } + if ( ! rename( $tmp, $path ) ) { + @unlink( $tmp ); + throw new RuntimeException( "Could not replace JSON file: {$path}" ); + } +} + +function html_api_fuzz_codex_archive_path( string $dir, string $prefix ): string { + $suffix = (string) time(); + $path = html_api_fuzz_codex_path_join( $dir, "{$prefix}.{$suffix}.json" ); + for ( $i = 1; file_exists( $path ); ++$i ) { + $path = html_api_fuzz_codex_path_join( $dir, "{$prefix}.{$suffix}.{$i}.json" ); + } + + return $path; +} + +function html_api_fuzz_codex_process_alive( int $pid ): bool { + if ( $pid < 1 || ! function_exists( 'posix_kill' ) ) { + return false; + } + + if ( @posix_kill( $pid, 0 ) ) { + return true; + } + + if ( function_exists( 'posix_get_last_error' ) && defined( 'POSIX_EPERM' ) ) { + return POSIX_EPERM === posix_get_last_error(); + } + + return false; +} + +function html_api_fuzz_codex_newest_minimize_dir( array $signature ): ?string { + $output_dir = $signature['minimizeOutputDir'] ?? null; + if ( is_string( $output_dir ) && '' !== $output_dir && is_dir( $output_dir ) ) { + return $output_dir; + } + + $result_path = $signature['minimizeResult'] ?? null; + if ( is_string( $result_path ) && '' !== $result_path && is_file( $result_path ) ) { + return dirname( $result_path ); + } + + return null; +} + +function html_api_fuzz_codex_claim_signature( string $diagnostics_dir, string $signature_hash, array $signature, int $stale_after_seconds ): ?string { + $signature_dir = html_api_fuzz_codex_path_join( $diagnostics_dir, html_api_fuzz_codex_signature_dir_name( $signature_hash ) ); + $done_path = html_api_fuzz_codex_path_join( $signature_dir, 'done.json' ); + $claim_path = html_api_fuzz_codex_path_join( $signature_dir, 'claim.json' ); + + if ( is_file( $done_path ) ) { + $done = html_api_fuzz_codex_read_json( $done_path ); + $exit_code = html_api_fuzz_codex_exit_code_from_done( $done ); + if ( 0 === $exit_code ) { + return null; + } + + if ( ! rename( $done_path, html_api_fuzz_codex_archive_path( $signature_dir, 'done.failed' ) ) && is_file( $done_path ) ) { + throw new RuntimeException( "Could not archive failed done file: {$done_path}" ); + } + if ( is_file( $claim_path ) && ! rename( $claim_path, html_api_fuzz_codex_archive_path( $signature_dir, 'claim.failed' ) ) && is_file( $claim_path ) ) { + throw new RuntimeException( "Could not archive failed claim file: {$claim_path}" ); + } + } + + if ( is_file( $claim_path ) ) { + $claim = html_api_fuzz_codex_read_json( $claim_path ) ?: array(); + $pid = (int) ( $claim['pid'] ?? 0 ); + $claimed_at = (float) ( $claim['claimedAtUnix'] ?? 0 ); + $stale = microtime( true ) - $claimed_at > $stale_after_seconds; + if ( ! $stale ) { + if ( $pid > 0 && html_api_fuzz_codex_process_alive( $pid ) ) { + return null; + } + return null; + } + if ( ! rename( $claim_path, html_api_fuzz_codex_archive_path( $signature_dir, 'claim.stale' ) ) && is_file( $claim_path ) ) { + throw new RuntimeException( "Could not archive stale claim file: {$claim_path}" ); + } + } + + \HtmlApiFuzz\ensure_dir( $signature_dir ); + $claim = array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-codex-claim', + 'hash' => $signature_hash, + 'pid' => getmypid(), + 'claimedAt' => html_api_fuzz_codex_now_iso_z(), + 'claimedAtUnix' => microtime( true ), + 'failureClass' => $signature['failureClass'] ?? null, + 'sourceReplayPath' => $signature['replayPath'] ?? null, + 'sourceResultPath' => $signature['resultPath'] ?? null, + ); + + $handle = @fopen( $claim_path, 'xb' ); + if ( false === $handle ) { + if ( is_file( $claim_path ) ) { + return null; + } + throw new RuntimeException( "Could not create claim file: {$claim_path}" ); + } + + $bytes = html_api_fuzz_codex_json_encode( $claim ) . "\n"; + $write = fwrite( $handle, $bytes ); + fclose( $handle ); + if ( false === $write || $write !== strlen( $bytes ) ) { + throw new RuntimeException( "Could not write claim file: {$claim_path}" ); + } + + return $signature_dir; +} + +function html_api_fuzz_codex_existing_paths( array $paths, array $allowed_roots ): array { + $existing = array(); + foreach ( $paths as $path ) { + if ( ! is_string( $path ) || '' === $path || html_api_fuzz_codex_path_has_control_chars( $path ) || ! file_exists( $path ) ) { + continue; + } + + foreach ( $allowed_roots as $root ) { + if ( html_api_fuzz_codex_path_is_under( $path, $root ) ) { + $real = realpath( $path ); + if ( false !== $real ) { + $existing[] = $real; + } + continue 2; + } + } + } + + return array_values( array_unique( $existing ) ); +} + +function html_api_fuzz_codex_prompt_for_signature( array $args, string $signature_hash, array $signature ): string { + if ( ! html_api_fuzz_codex_valid_signature_hash( $signature_hash ) ) { + throw new InvalidArgumentException( 'Unsafe signature hash: ' . $signature_hash ); + } + + $signature_dir_name = html_api_fuzz_codex_signature_dir_name( $signature_hash ); + $signature_triage_dir = html_api_fuzz_codex_path_join( $args['triageDir'], 'signatures', $signature_dir_name ); + $minimize_dir = html_api_fuzz_codex_newest_minimize_dir( $signature ); + $minimized_dir = null === $minimize_dir ? null : html_api_fuzz_codex_path_join( $minimize_dir, 'minimized' ); + $report_path = html_api_fuzz_codex_path_join( $args['diagnosticsDir'], $signature_dir_name, 'report.md' ); + + $paths = array( + html_api_fuzz_codex_path_join( $signature_triage_dir, 'failure.json' ), + is_string( $signature['minimizeResult'] ?? null ) ? $signature['minimizeResult'] : null, + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'result.json' ), + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'replay.json' ), + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'wordpress-tree.txt' ), + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'dom-tree.txt' ), + is_string( $signature['replayPath'] ?? null ) ? $signature['replayPath'] : null, + is_string( $signature['resultPath'] ?? null ) ? $signature['resultPath'] : null, + ); + $existing_paths = html_api_fuzz_codex_existing_paths( $paths, html_api_fuzz_codex_allowed_artifact_roots( $args ) ); + + if ( 'classify' === $args['mode'] ) { + $fix_policy = 'This is classify-only mode. Do not edit files. Do not run write commands. Return the Markdown report as your final response; the Codex CLI will save it. If you believe a fix is needed, describe the smallest fix and the focused verification.'; + } else { + $fix_policy = 'You may implement the smallest justified fix. Preserve the minimized replay and run focused verification.'; + } + + $artifact_lines = implode( "\n", array_map( static fn( string $path ): string => '- ' . $path, $existing_paths ) ); + $classification_lines = implode( "\n", array_map( static fn( string $classification ): string => '- ' . $classification, HTML_API_FUZZ_CODEX_CLASSIFICATIONS ) ); + $failure_class = html_api_fuzz_codex_safe_prompt_line( $signature['failureClass'] ?? '', 'failureClass' ); + $triage_kind = html_api_fuzz_codex_safe_prompt_line( $signature['triageKind'] ?? 'failure', 'triageKind' ); + $oracle_type = html_api_fuzz_codex_safe_prompt_line( $signature['oracleFindingType'] ?? '', 'oracleFindingType' ); + $suspected_owner = html_api_fuzz_codex_safe_prompt_line( $signature['suspectedOwner'] ?? '', 'suspectedOwner' ); + $upstream_issue = html_api_fuzz_codex_safe_prompt_line( $signature['upstreamIssueUrl'] ?? '', 'upstreamIssueUrl' ); + + return << 1, + 'kind' => 'html-api-fuzz-codex-job', + 'hash' => $signature_hash, + 'startedAt' => html_api_fuzz_codex_now_iso_z(), + 'startedAtUnix' => microtime( true ), + 'mode' => $args['mode'], + 'sandbox' => $args['sandbox'], + 'command' => $command, + 'processCommand' => $process_command, + 'promptPath' => $prompt_path, + 'stdoutPath' => $stdout_path, + 'stderrPath' => $stderr_path, + 'reportPath' => $report_path, + 'donePath' => $done_path, + ); + html_api_fuzz_codex_write_json( html_api_fuzz_codex_path_join( $signature_dir, 'job.json' ), $metadata ); + + $descriptor_spec = array( + 0 => array( 'pipe', 'r' ), + 1 => array( 'file', $stdout_path, 'w' ), + 2 => array( 'file', $stderr_path, 'w' ), + ); + $process = proc_open( $process_command, $descriptor_spec, $pipes, $args['repoRoot'] ); + if ( ! is_resource( $process ) ) { + throw new RuntimeException( 'Could not start Codex subprocess.' ); + } + + try { + html_api_fuzz_codex_write_stream( $pipes[0], $prompt ); + fclose( $pipes[0] ); + } catch ( Throwable $error ) { + if ( is_resource( $pipes[0] ) ) { + fclose( $pipes[0] ); + } + proc_terminate( $process ); + proc_close( $process ); + throw $error; + } + $status = proc_get_status( $process ); + + return array( + 'process' => $process, + 'pid' => (int) ( $status['pid'] ?? 0 ), + 'processGroupPid' => $uses_process_group ? (int) ( $status['pid'] ?? 0 ) : null, + 'knownExitCode' => ( ! ( $status['running'] ?? false ) && array_key_exists( 'exitcode', $status ) && -1 !== $status['exitcode'] ) ? (int) $status['exitcode'] : null, + 'signatureDir' => $signature_dir, + 'hash' => $signature_hash, + ); +} + +function html_api_fuzz_codex_poll_exit_code( array &$job ): ?int { + if ( null !== ( $job['knownExitCode'] ?? null ) ) { + $exit_code = (int) $job['knownExitCode']; + proc_close( $job['process'] ); + return $exit_code; + } + + $status = proc_get_status( $job['process'] ); + if ( $status['running'] ?? false ) { + return null; + } + + $exit_code = null; + if ( array_key_exists( 'exitcode', $status ) && -1 !== $status['exitcode'] ) { + $exit_code = (int) $status['exitcode']; + } elseif ( array_key_exists( 'cached_exitcode', $status ) && false !== $status['cached_exitcode'] ) { + $exit_code = (int) $status['cached_exitcode']; + } + + $closed_code = proc_close( $job['process'] ); + if ( null === $exit_code ) { + $exit_code = (int) $closed_code; + } + + return $exit_code; +} + +function html_api_fuzz_codex_mark_done( array $job, int $exit_code, array $extra = array() ): void { + $signature_dir = $job['signatureDir']; + $signature_hash = $job['hash']; + $done = array_merge( + array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-codex-done', + 'hash' => $signature_hash, + 'finishedAt' => html_api_fuzz_codex_now_iso_z(), + 'finishedAtUnix' => microtime( true ), + 'exitCode' => $exit_code, + 'reportPath' => html_api_fuzz_codex_path_join( $signature_dir, 'report.md' ), + 'stdoutPath' => html_api_fuzz_codex_path_join( $signature_dir, 'codex.jsonl' ), + 'stderrPath' => html_api_fuzz_codex_path_join( $signature_dir, 'codex.stderr.log' ), + ), + $extra + ); + html_api_fuzz_codex_write_json( html_api_fuzz_codex_path_join( $signature_dir, 'done.json' ), $done ); +} + +function html_api_fuzz_codex_mark_launch_failed( string $signature_dir, string $signature_hash, Throwable $error ): void { + html_api_fuzz_codex_mark_done( + array( + 'signatureDir' => $signature_dir, + 'hash' => $signature_hash, + ), + 1, + array( + 'launchFailed' => true, + 'errorClass' => get_class( $error ), + 'errorMessage' => $error->getMessage(), + ) + ); +} + +function html_api_fuzz_codex_load_candidate_signatures( array $state ): array { + $signatures = $state['signatures'] ?? array(); + if ( ! is_array( $signatures ) ) { + return array(); + } + + ksort( $signatures, SORT_STRING ); + $candidates = array(); + foreach ( $signatures as $signature_hash => $signature ) { + if ( ! is_array( $signature ) ) { + continue; + } + $signature_hash = (string) $signature_hash; + if ( ! html_api_fuzz_codex_valid_signature_hash( $signature_hash ) ) { + continue; + } + if ( 'minimized' !== ( $signature['status'] ?? null ) ) { + continue; + } + if ( null === html_api_fuzz_codex_newest_minimize_dir( $signature ) ) { + continue; + } + $candidates[] = array( $signature_hash, $signature ); + } + + return $candidates; +} + +function html_api_fuzz_codex_parse_args( array $argv ): array { + if ( in_array( '-h', $argv, true ) ) { + html_api_fuzz_codex_usage(); + exit( 0 ); + } + + $options = \HtmlApiFuzz\parse_cli_options( $argv ); + html_api_fuzz_codex_validate_cli_options( $options ); + if ( \HtmlApiFuzz\option_bool( $options, 'help', false ) ) { + html_api_fuzz_codex_usage(); + exit( 0 ); + } + + $triage_dir = \HtmlApiFuzz\option_string( $options, 'triage-dir' ); + if ( null === $triage_dir ) { + html_api_fuzz_codex_usage(); + exit( 1 ); + } + + $mode = \HtmlApiFuzz\option_string( $options, 'mode', 'classify' ); + if ( ! in_array( $mode, array( 'classify', 'fix' ), true ) ) { + throw new InvalidArgumentException( 'Expected --mode to be classify or fix.' ); + } + + $sandbox = \HtmlApiFuzz\option_string( $options, 'sandbox', 'fix' === $mode ? 'workspace-write' : 'read-only' ); + if ( ! in_array( $sandbox, array( 'read-only', 'workspace-write', 'danger-full-access' ), true ) ) { + throw new InvalidArgumentException( 'Expected --sandbox to be read-only, workspace-write, or danger-full-access.' ); + } + if ( 'fix' === $mode && 'read-only' === $sandbox ) { + throw new InvalidArgumentException( 'Expected --mode fix to use a writable sandbox.' ); + } + + $repo_root = html_api_fuzz_codex_require_dir( \HtmlApiFuzz\option_string( $options, 'repo-root', getcwd() ?: '.' ), 'Repo root' ); + $triage_dir = html_api_fuzz_codex_require_dir( $triage_dir, 'Triage directory' ); + $diagnostics_dir = \HtmlApiFuzz\option_string( $options, 'diagnostics-dir' ); + if ( null === $diagnostics_dir ) { + $diagnostics_dir = html_api_fuzz_codex_path_join( dirname( $triage_dir ), 'diagnostics' ); + } + + $interval_seconds = \HtmlApiFuzz\option_float( $options, 'interval-seconds', 120.0 ); + $max_concurrent = \HtmlApiFuzz\option_int( $options, 'max-concurrent', 1 ); + $max_launch_per_pass = \HtmlApiFuzz\option_int( $options, 'max-launch-per-pass', 1 ); + $stale_after_seconds = \HtmlApiFuzz\option_int( $options, 'stale-after-seconds', 6 * 60 * 60 ); + if ( $interval_seconds < 0 ) { + throw new InvalidArgumentException( 'Expected --interval-seconds to be at least 0.' ); + } + if ( $max_concurrent < 0 ) { + throw new InvalidArgumentException( 'Expected --max-concurrent to be at least 0.' ); + } + if ( $max_launch_per_pass < 0 ) { + throw new InvalidArgumentException( 'Expected --max-launch-per-pass to be at least 0.' ); + } + if ( $stale_after_seconds < 0 ) { + throw new InvalidArgumentException( 'Expected --stale-after-seconds to be at least 0.' ); + } + + return array( + 'triageDir' => $triage_dir, + 'diagnosticsDir' => html_api_fuzz_codex_normalize_path( $diagnostics_dir ), + 'repoRoot' => $repo_root, + 'codexBin' => \HtmlApiFuzz\option_string( $options, 'codex-bin', 'codex' ), + 'model' => \HtmlApiFuzz\option_string( $options, 'model' ), + 'intervalSeconds' => $interval_seconds, + 'maxConcurrent' => $max_concurrent, + 'maxLaunchPerPass' => $max_launch_per_pass, + 'staleAfterSeconds' => $stale_after_seconds, + 'once' => \HtmlApiFuzz\option_bool( $options, 'once', false ), + 'mode' => $mode, + 'sandbox' => $sandbox, + ); +} + +function html_api_fuzz_codex_install_signal_handlers( bool &$stopping ): void { + if ( ! function_exists( 'pcntl_signal' ) ) { + throw new RuntimeException( 'The pcntl extension is required for signal-safe Codex job orchestration.' ); + } + + if ( function_exists( 'pcntl_async_signals' ) ) { + pcntl_async_signals( true ); + } + if ( defined( 'SIGTERM' ) ) { + pcntl_signal( + SIGTERM, + static function () use ( &$stopping ): void { + $stopping = true; + } + ); + } + if ( defined( 'SIGINT' ) ) { + pcntl_signal( + SIGINT, + static function () use ( &$stopping ): void { + $stopping = true; + } + ); + } +} + +function html_api_fuzz_codex_dispatch_signals(): void { + if ( function_exists( 'pcntl_signal_dispatch' ) ) { + pcntl_signal_dispatch(); + } +} + +function html_api_fuzz_codex_sleep( float $seconds, bool &$stopping ): void { + $deadline = microtime( true ) + max( 0.0, $seconds ); + do { + html_api_fuzz_codex_dispatch_signals(); + if ( $stopping ) { + return; + } + + $remaining = $deadline - microtime( true ); + if ( $remaining <= 0 ) { + return; + } + + usleep( (int) min( 250000, max( 1000, round( $remaining * 1000000 ) ) ) ); + } while ( true ); +} + +function html_api_fuzz_codex_stop_job( array &$job ): void { + $status = proc_get_status( $job['process'] ); + if ( $status['running'] ?? false ) { + if ( null !== ( $job['processGroupPid'] ?? null ) && function_exists( 'posix_kill' ) && defined( 'SIGTERM' ) ) { + @posix_kill( -1 * (int) $job['processGroupPid'], SIGTERM ); + } else { + proc_terminate( $job['process'] ); + } + usleep( 200000 ); + $status = proc_get_status( $job['process'] ); + if ( $status['running'] ?? false ) { + if ( null !== ( $job['processGroupPid'] ?? null ) && function_exists( 'posix_kill' ) ) { + @posix_kill( -1 * (int) $job['processGroupPid'], 9 ); + } else { + proc_terminate( $job['process'], 9 ); + } + } + } + proc_close( $job['process'] ); + html_api_fuzz_codex_mark_done( + $job, + 143, + array( + 'interrupted' => true, + ) + ); +} + +function html_api_fuzz_codex_main( array $argv ): int { + $args = html_api_fuzz_codex_parse_args( $argv ); + $state_path = html_api_fuzz_codex_path_join( $args['triageDir'], 'state.json' ); + $running = array(); + $stopping = false; + $scanned = false; + + html_api_fuzz_codex_install_signal_handlers( $stopping ); + \HtmlApiFuzz\ensure_dir( $args['diagnosticsDir'] ); + html_api_fuzz_codex_write_json( + html_api_fuzz_codex_path_join( $args['diagnosticsDir'], 'orchestrator-state.json' ), + array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-codex-orchestrator-state', + 'startedAt' => html_api_fuzz_codex_now_iso_z(), + 'triageDir' => $args['triageDir'], + 'repoRoot' => $args['repoRoot'], + 'mode' => $args['mode'], + 'sandbox' => $args['sandbox'], + ) + ); + + try { + while ( ! $stopping ) { + html_api_fuzz_codex_dispatch_signals(); + foreach ( array_keys( $running ) as $key ) { + $exit_code = html_api_fuzz_codex_poll_exit_code( $running[ $key ] ); + if ( null === $exit_code ) { + continue; + } + + html_api_fuzz_codex_mark_done( $running[ $key ], $exit_code ); + echo 'completed ' . $running[ $key ]['hash'] . ' exit=' . $exit_code . "\n"; + unset( $running[ $key ] ); + } + + /* + * Graceful stop: a STOP file in the run directory stops new + * launches; running jobs finish before the orchestrator exits. + * The run directory comes from the watcher state when available + * (the triage dir may live outside the run dir), matching how + * launches resolve it below. + */ + $stop_run_dir = dirname( $args['triageDir'] ); + $stop_state = html_api_fuzz_codex_read_json( $state_path ); + if ( is_array( $stop_state ) && is_string( $stop_state['runDir'] ?? null ) && ! html_api_fuzz_codex_path_has_control_chars( $stop_state['runDir'] ) ) { + $stop_run_dir = html_api_fuzz_codex_normalize_path( $stop_state['runDir'] ); + } + $stop_file_present = is_file( html_api_fuzz_codex_path_join( $stop_run_dir, 'STOP' ) ); + if ( $stop_file_present && empty( $running ) ) { + echo "stop requested; exiting\n"; + break; + } + + $can_scan = ( ! $args['once'] || ! $scanned ) && ! $stop_file_present; + if ( $can_scan ) { + $available = max( 0, $args['maxConcurrent'] - count( $running ) ); + $launched = 0; + if ( $available > 0 ) { + $state = html_api_fuzz_codex_read_json( $state_path ); + if ( is_array( $state ) ) { + $launch_args = $args; + if ( is_string( $state['runDir'] ?? null ) && ! html_api_fuzz_codex_path_has_control_chars( $state['runDir'] ) ) { + $launch_args['runDir'] = html_api_fuzz_codex_normalize_path( $state['runDir'] ); + } else { + $launch_args['runDir'] = dirname( $args['triageDir'] ); + } + + foreach ( html_api_fuzz_codex_load_candidate_signatures( $state ) as $candidate ) { + if ( $launched >= $available || $launched >= $args['maxLaunchPerPass'] ) { + break; + } + list( $signature_hash, $signature ) = $candidate; + $signature_dir = html_api_fuzz_codex_claim_signature( $args['diagnosticsDir'], $signature_hash, $signature, $args['staleAfterSeconds'] ); + if ( null === $signature_dir ) { + continue; + } + + try { + $job = html_api_fuzz_codex_launch( $launch_args, $signature_hash, $signature, $signature_dir ); + $running[] = $job; + ++$launched; + echo 'launched ' . $signature_hash . ' pid=' . $job['pid'] . "\n"; + } catch ( Throwable $error ) { + html_api_fuzz_codex_mark_launch_failed( $signature_dir, $signature_hash, $error ); + echo 'launch failed ' . $signature_hash . ' error=' . $error->getMessage() . "\n"; + } + } + } + } + $scanned = true; + } + + if ( $args['once'] && empty( $running ) ) { + break; + } + + $sleep_seconds = $args['once'] ? min( 1.0, $args['intervalSeconds'] ) : $args['intervalSeconds']; + html_api_fuzz_codex_sleep( $sleep_seconds, $stopping ); + } + } finally { + foreach ( array_keys( $running ) as $key ) { + html_api_fuzz_codex_stop_job( $running[ $key ] ); + unset( $running[ $key ] ); + } + } + + return 0; +} + +if ( ! defined( 'HTML_API_FUZZ_CODEX_SELF_TESTING' ) || ! HTML_API_FUZZ_CODEX_SELF_TESTING ) { + exit( html_api_fuzz_codex_main( $argv ) ); +} diff --git a/tools/html-api-fuzz/commoncrawl-analysis.php b/tools/html-api-fuzz/commoncrawl-analysis.php new file mode 100644 index 0000000000000..19e91a5d1cc12 --- /dev/null +++ b/tools/html-api-fuzz/commoncrawl-analysis.php @@ -0,0 +1,14 @@ +analyze_document( $document ); +}; diff --git a/tools/html-api-fuzz/commoncrawl-batch.php b/tools/html-api-fuzz/commoncrawl-batch.php new file mode 100755 index 0000000000000..22df7d1254e40 --- /dev/null +++ b/tools/html-api-fuzz/commoncrawl-batch.php @@ -0,0 +1,123 @@ +#!/usr/bin/env php + $value ) { + if ( ! is_string( $name ) || ! is_string( $value ) ) { + unset( $environment[ $name ] ); + } + } + ksort( $environment ); + return $environment; +} + +function html_api_fuzz_batch_git_output( array $arguments ): string { + $result = \HtmlApiFuzz\run_git_command( $arguments, 30000, \HtmlApiFuzz\repo_root() ); + if ( 0 !== ( $result['code'] ?? null ) || true === ( $result['timedOut'] ?? false ) ) { + throw new RuntimeException( 'Sanitized Git probe command failed: git ' . implode( ' ', $arguments ) ); + } + return (string) $result['stdout']; +} + +$options = \HtmlApiFuzz\parse_cli_options( $argv ); + +try { + if ( array_key_exists( 'internal-runtime-probe', $options ) ) { + $scanned = php_ini_scanned_files(); + $scanned_files = array(); + if ( is_string( $scanned ) && '' !== trim( $scanned ) ) { + foreach ( preg_split( '/\s*,\s*/', trim( $scanned ) ) ?: array() as $path ) { + if ( '' !== $path ) { + $scanned_files[] = $path; + } + } + } + $extensions = get_loaded_extensions(); + sort( $extensions ); + echo \HtmlApiFuzz\json_encode_safe( array( + 'phpVersion' => PHP_VERSION, + 'phpBinary' => PHP_BINARY, + 'loadedIni' => php_ini_loaded_file() ?: null, + 'scannedIni' => $scanned_files, + 'extensions' => $extensions, + 'environment' => html_api_fuzz_batch_environment(), + ) ) . "\n"; + exit( 0 ); + } + + if ( array_key_exists( 'internal-git-probe', $options ) ) { + $commit = trim( html_api_fuzz_batch_git_output( array( 'rev-parse', 'HEAD' ) ) ); + $branch = trim( html_api_fuzz_batch_git_output( array( 'branch', '--show-current' ) ) ); + $status = html_api_fuzz_batch_git_output( array( 'status', '--porcelain=v1', '-z', '--untracked-files=all' ) ); + $pathspecs = \HtmlApiFuzz\CommonCrawlBatchCoordinator::executed_code_pathspecs(); + $index = html_api_fuzz_batch_git_output( array_merge( array( 'ls-files', '-s', '-z', '--' ), $pathspecs ) ); + $tracked = array(); + foreach ( explode( "\0", $index ) as $record ) { + if ( '' === $record ) { + continue; + } + if ( 1 !== preg_match( '/^([0-7]{6}) [0-9a-f]{40} [0-3]\t(.+)$/s', $record, $match ) ) { + throw new RuntimeException( 'Git index probe returned an invalid record.' ); + } + $tracked[] = array( 'mode' => $match[1], 'path' => $match[2] ); + } + echo \HtmlApiFuzz\json_encode_safe( array( + 'commit' => $commit, + 'branch' => '' === $branch ? null : $branch, + 'statusBase64' => base64_encode( $status ), + 'tracked' => $tracked, + 'environment' => html_api_fuzz_batch_environment(), + ) ) . "\n"; + exit( 0 ); + } + + if ( array_key_exists( 'internal-oracle-probe', $options ) ) { + $encoded = \HtmlApiFuzz\option_string( $options, 'internal-oracle-probe', null ); + if ( null === $encoded ) { + throw new InvalidArgumentException( 'Internal oracle probe requires an encoded option object.' ); + } + $json = base64_decode( $encoded, true ); + if ( false === $json ) { + throw new InvalidArgumentException( 'Internal oracle probe options are not valid base64.' ); + } + $oracle_options = \HtmlApiFuzz\StrictJsonParser::decode( $json ); + if ( ! is_array( $oracle_options ) ) { + throw new InvalidArgumentException( 'Internal oracle probe options must be an object.' ); + } + $allowed = array( 'dom-oracle', 'lexbor-oracle-bin', 'html5ever-oracle-bin', 'chrome-oracle-script', 'chrome-executable', 'node-bin', 'oracle-timeout-ms', 'chrome-startup-timeout-ms' ); + foreach ( $oracle_options as $name => $value ) { + if ( ! in_array( $name, $allowed, true ) || ! is_string( $value ) || '' === $value ) { + throw new InvalidArgumentException( 'Internal oracle probe contains an unexpected option.' ); + } + } + $renderer = \HtmlApiFuzz\OracleRenderer::from_options( $oracle_options ); + $result = \HtmlApiFuzz\OracleRenderer::with_explicit_close( + $renderer, + static function ( \HtmlApiFuzz\OracleRenderer $renderer ): array { + return array( + 'metadata' => $renderer->metadata(), + 'replayOptions' => $renderer->replay_options(), + 'workerArgs' => $renderer->worker_args(), + 'environment' => html_api_fuzz_batch_environment(), + ); + } + ); + echo \HtmlApiFuzz\json_encode_safe( $result ) . "\n"; + exit( 0 ); + } + + if ( \HtmlApiFuzz\option_bool( $options, 'help', false ) ) { + echo "Usage: php tools/html-api-fuzz/commoncrawl-batch.php --cc-analyzer PATH --workspace DIR --batch NAME --output-dir NEW_DIR --batch-timeout-ms N --chrome-executable PATH --node-bin PATH [--lexbor-oracle-bin PATH] [--html5ever-oracle-bin PATH] [--chrome-oracle-script PATH] [--process-timeout-ms N] [--oracle-timeout-ms N] [--chrome-startup-timeout-ms N] [--checks baseline|full|sampled] [--retain-all] [--require-utf8]\n"; + exit( 0 ); + } + + $result = \HtmlApiFuzz\CommonCrawlBatchCoordinator::run( $options ); + echo \HtmlApiFuzz\json_encode_safe( $result ) . "\n"; + exit( 0 ); +} catch ( Throwable $error ) { + fwrite( STDERR, $error->getMessage() . "\n" ); + exit( 1 ); +} diff --git a/tools/html-api-fuzz/launcher.php b/tools/html-api-fuzz/launcher.php new file mode 100755 index 0000000000000..211be8f9076c8 --- /dev/null +++ b/tools/html-api-fuzz/launcher.php @@ -0,0 +1,375 @@ +#!/usr/bin/env php + array( 'pipe', 'r' ), + 1 => array( 'pipe', 'w' ), + 2 => array( 'pipe', 'w' ), + ); + + $process = proc_open( $command, $spec, $pipes, $cwd ); + if ( ! is_resource( $process ) ) { + throw new RuntimeException( 'Could not start runner lane.' ); + } + + fclose( $pipes[0] ); + stream_set_blocking( $pipes[1], false ); + stream_set_blocking( $pipes[2], false ); + \HtmlApiFuzz\ensure_dir( dirname( $log_path ) ); + file_put_contents( $log_path, '' ); + + return array( + 'process' => $process, + 'pipes' => $pipes, + 'logPath' => $log_path, + 'startedAt' => microtime( true ), + 'stdout' => '', + 'stderr' => '', + ); +} + +function html_api_fuzz_launcher_drain_lane( array &$lane ): void { + $stdout = stream_get_contents( $lane['pipes'][1] ); + $stderr = stream_get_contents( $lane['pipes'][2] ); + if ( '' !== $stdout ) { + $lane['stdout'] .= $stdout; + file_put_contents( $lane['logPath'], $stdout, FILE_APPEND ); + } + if ( '' !== $stderr ) { + $lane['stderr'] .= $stderr; + file_put_contents( $lane['logPath'], $stderr, FILE_APPEND ); + } +} + +function html_api_fuzz_launcher_close_lane( array &$lane ): array { + html_api_fuzz_launcher_drain_lane( $lane ); + fclose( $lane['pipes'][1] ); + fclose( $lane['pipes'][2] ); + $code = proc_close( $lane['process'] ); + + return array( + 'code' => $code, + 'durationMs' => (int) round( ( microtime( true ) - $lane['startedAt'] ) * 1000 ), + 'stdoutTail' => substr( $lane['stdout'], -2000 ), + 'stderrTail' => substr( $lane['stderr'], -2000 ), + ); +} + +$options = \HtmlApiFuzz\parse_cli_options( $argv ); +if ( \HtmlApiFuzz\option_bool( $options, 'help', false ) || \HtmlApiFuzz\option_bool( $options, 'h', false ) ) { + html_api_fuzz_launcher_usage(); + exit( 0 ); +} +if ( array_key_exists( 'timeout-ms', $options ) && true === $options['timeout-ms'] ) { + throw new InvalidArgumentException( 'Expected --timeout-ms to have a value.' ); +} + +$repo_root = \HtmlApiFuzz\repo_root(); +$output_dir = \HtmlApiFuzz\option_string( $options, 'output-dir', $repo_root . '/artifacts/html-api-fuzz/launch-' . \HtmlApiFuzz\timestamp() ); +$lanes = max( 1, \HtmlApiFuzz\option_int( $options, 'lanes', 2 ) ); +$start_seed = \HtmlApiFuzz\option_int( $options, 'start-seed', 1 ); +$max_seeds = \HtmlApiFuzz\option_int( $options, 'max-seeds', 0 ); +$duration_seconds = \HtmlApiFuzz\option_float( $options, 'duration-seconds', 60.0 ); +$timeout_explicit = array_key_exists( 'timeout-ms', $options ); +$timeout_ms = \HtmlApiFuzz\option_int( $options, 'timeout-ms', 2500 ); +$profile = \HtmlApiFuzz\option_string( $options, 'profile', 'auto' ); +$mode = \HtmlApiFuzz\option_string( $options, 'mode', 'auto' ); +$payload_policy = \HtmlApiFuzz\option_string( $options, 'payload-policy', 'auto' ); +$max_input_bytes = \HtmlApiFuzz\option_int( $options, 'max-input-bytes', 0 ); +$max_tokens = \HtmlApiFuzz\option_int( $options, 'max-tokens', 2000 ); +$max_nodes = \HtmlApiFuzz\option_int( $options, 'max-nodes', 3000 ); +$stop_on_failure = \HtmlApiFuzz\option_bool( $options, 'stop-on-failure', false ); +$fail_unsupported = \HtmlApiFuzz\option_bool( $options, 'fail-unsupported', false ); +$run_watcher = \HtmlApiFuzz\option_bool( $options, 'watcher', false ); +$triage_oracle_findings = \HtmlApiFuzz\option_bool( $options, 'triage-oracle-findings', false ); +$max_keep_per_signature = \HtmlApiFuzz\option_int( $options, 'max-keep-per-signature', 5 ); +$keep_all_artifacts = \HtmlApiFuzz\option_bool( $options, 'keep-all-artifacts', false ); +if ( $max_keep_per_signature < 1 ) { + throw new InvalidArgumentException( 'Expected --max-keep-per-signature to be at least 1.' ); +} +html_api_fuzz_launcher_validate_generator_options( $profile, $mode, $payload_policy ); +html_api_fuzz_launcher_validate_runtime_options( $max_seeds, $duration_seconds, $timeout_ms, $max_input_bytes, $max_tokens, $max_nodes ); + +if ( is_file( $output_dir . '/STOP' ) ) { + // A leftover stop request must not silently turn this launch into a + // 0-seed success; starting again is an explicit operator decision. + fwrite( STDERR, "Stop file already exists: {$output_dir}/STOP\nRemove it to start a run in this directory.\n" ); + exit( 1 ); +} + +\HtmlApiFuzz\ensure_dir( $output_dir ); +$events_path = $output_dir . '/events.ndjson'; +$state_path = $output_dir . '/launcher-state.json'; +$git_metadata = null === \HtmlApiFuzz\option_string( $options, 'git-metadata-base64', null ) + ? \HtmlApiFuzz\git_metadata() + : \HtmlApiFuzz\git_metadata_from_base64( \HtmlApiFuzz\option_string( $options, 'git-metadata-base64' ) ); +$git_metadata_base64 = \HtmlApiFuzz\git_metadata_base64( $git_metadata ); +$oracle_renderer = \HtmlApiFuzz\OracleRenderer::from_options( $options ); +$oracle_setup = \HtmlApiFuzz\OracleRenderer::with_explicit_close( + $oracle_renderer, + static function ( \HtmlApiFuzz\OracleRenderer $renderer ) use ( $timeout_explicit, $timeout_ms ): array { + return array( + 'metadata' => $renderer->metadata(), + 'workerArgs' => $renderer->worker_args(), + 'timeoutMs' => $timeout_explicit ? $timeout_ms : $renderer->recommended_process_timeout_ms( 'full', 2500 ), + ); + } +); +$oracle_metadata = $oracle_setup['metadata']; +$oracle_worker_args = $oracle_setup['workerArgs']; +$timeout_ms = $oracle_setup['timeoutMs']; + +$state = array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-launcher-state', + 'startedAt' => gmdate( 'c' ), + 'updatedAt' => gmdate( 'c' ), + 'outputDir' => $output_dir, + 'lanes' => $lanes, + 'startSeed' => $start_seed, + 'seedStride' => $lanes, + 'profile' => $profile, + 'mode' => $mode, + 'payloadPolicy' => $payload_policy, + 'maxInputBytes' => $max_input_bytes > 0 ? $max_input_bytes : null, + 'git' => $git_metadata, + 'oracle' => $oracle_metadata, + 'processTimeoutMs' => $timeout_ms, + 'finished' => false, + 'laneResults' => array(), +); +\HtmlApiFuzz\write_json_file( $state_path, $state ); +\HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'launcher-start', 'outputDir' => $output_dir, 'lanes' => $lanes, 'git' => $git_metadata, 'oracle' => $oracle_metadata ) ); + +$running = array(); +for ( $i = 0; $i < $lanes; ++$i ) { + $lane_max_seeds = 0; + if ( 0 !== $max_seeds ) { + $lane_max_seeds = intdiv( $max_seeds, $lanes ) + ( $i < ( $max_seeds % $lanes ) ? 1 : 0 ); + if ( 0 === $lane_max_seeds ) { + $lane_dir = $output_dir . '/lane-' . str_pad( (string) $i, 2, '0', STR_PAD_LEFT ); + $state['laneResults'][ $i ] = array( + 'lane' => $i, + 'status' => 'skipped', + 'outputDir' => $lane_dir, + 'reason' => 'no seeds assigned', + ); + continue; + } + } + + $lane_dir = $output_dir . '/lane-' . str_pad( (string) $i, 2, '0', STR_PAD_LEFT ); + $command = array( + PHP_BINARY, + __DIR__ . '/runner.php', + '--output-dir', + $lane_dir, + '--start-seed', + (string) ( $start_seed + $i ), + '--seed-stride', + (string) $lanes, + '--duration-seconds', + (string) $duration_seconds, + '--timeout-ms', + (string) $timeout_ms, + '--profile', + $profile, + '--mode', + $mode, + '--payload-policy', + $payload_policy, + '--max-tokens', + (string) $max_tokens, + '--max-nodes', + (string) $max_nodes, + '--git-metadata-base64', + $git_metadata_base64, + '--max-keep-per-signature', + (string) $max_keep_per_signature, + '--stop-file', + $output_dir . '/STOP', + ); + foreach ( $oracle_worker_args as $arg ) { + $command[] = $arg; + } + + if ( 0 !== $max_seeds ) { + $command[] = '--max-seeds'; + $command[] = (string) $lane_max_seeds; + } + if ( $keep_all_artifacts ) { + $command[] = '--keep-all-artifacts'; + } + if ( $stop_on_failure ) { + $command[] = '--stop-on-failure'; + } + if ( $fail_unsupported ) { + $command[] = '--fail-unsupported'; + } + if ( $max_input_bytes > 0 ) { + $command[] = '--max-input-bytes'; + $command[] = (string) $max_input_bytes; + } + + $running[ $i ] = html_api_fuzz_launcher_start_lane( $command, $repo_root, $lane_dir . '/runner.stdout.log' ); + $state['laneResults'][ $i ] = array( + 'lane' => $i, + 'status' => 'running', + 'command' => \HtmlApiFuzz\command_string( $command ), + 'outputDir' => $lane_dir, + 'logPath' => $lane_dir . '/runner.stdout.log', + ); + \HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'lane-start', 'lane' => $i, 'outputDir' => $lane_dir, 'oracle' => $oracle_metadata ) ); +} +\HtmlApiFuzz\write_json_file( $state_path, $state ); + +while ( $running ) { + foreach ( array_keys( $running ) as $lane_id ) { + html_api_fuzz_launcher_drain_lane( $running[ $lane_id ] ); + $status = proc_get_status( $running[ $lane_id ]['process'] ); + if ( $status['running'] ) { + continue; + } + + $closed = html_api_fuzz_launcher_close_lane( $running[ $lane_id ] ); + unset( $running[ $lane_id ] ); + $lane_dir = $state['laneResults'][ $lane_id ]['outputDir']; + $runner_state = \HtmlApiFuzz\read_json_file( $lane_dir . '/state.json' ); + $state['laneResults'][ $lane_id ] = array_merge( + $state['laneResults'][ $lane_id ], + array( + 'status' => 0 === $closed['code'] ? 'completed' : 'failed', + 'code' => $closed['code'], + 'durationMs' => $closed['durationMs'], + 'runnerState' => $runner_state, + 'stdoutTail' => $closed['stdoutTail'], + 'stderrTail' => $closed['stderrTail'], + ) + ); + $state['updatedAt'] = gmdate( 'c' ); + \HtmlApiFuzz\write_json_file( $state_path, $state ); + \HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'lane-stop', 'lane' => $lane_id, 'code' => $closed['code'] ) ); + } + usleep( 100000 ); +} + +$aggregate = array( + 'successes' => 0, + 'failures' => 0, + 'unsupported' => 0, + 'oracleParseErrors' => 0, + 'oracleUnsupported' => 0, + 'oracleTolerated' => 0, + 'oracleFindings' => 0, +); +foreach ( $state['laneResults'] as $lane ) { + $runner_state = $lane['runnerState'] ?? array(); + foreach ( $aggregate as $name => $count ) { + $aggregate[ $name ] += (int) ( $runner_state[ $name ] ?? 0 ); + } +} + +$state['finished'] = true; +$state['updatedAt'] = gmdate( 'c' ); +$state['aggregate'] = $aggregate; +\HtmlApiFuzz\write_json_file( $state_path, $state ); +\HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'launcher-stop', 'aggregate' => $aggregate ) ); + +$watcher_result = null; +if ( $run_watcher ) { + $triage_dir = $output_dir . '/triage'; + $minimize_timeout_ms = \HtmlApiFuzz\option_int( $options, 'minimize-timeout-ms', 300000 ); + $watcher_timeout_ms = \HtmlApiFuzz\option_int( $options, 'watcher-timeout-ms', max( 600000, $minimize_timeout_ms + 60000 ) ); + $proc = \HtmlApiFuzz\run_php_process( + array_values( + array_filter( + array( + __DIR__ . '/watcher.php', + '--run-dir', + $output_dir, + '--state-dir', + $triage_dir, + '--once', + '--minimize-timeout-ms', + (string) $minimize_timeout_ms, + '--timeout-ms', + (string) $timeout_ms, + array_key_exists( 'max-attempts', $options ) ? '--max-attempts' : null, + array_key_exists( 'max-attempts', $options ) ? (string) \HtmlApiFuzz\option_int( $options, 'max-attempts', 250 ) : null, + array_key_exists( 'max-minimize', $options ) ? '--max-minimize' : null, + array_key_exists( 'max-minimize', $options ) ? (string) \HtmlApiFuzz\option_int( $options, 'max-minimize', 0 ) : null, + array_key_exists( 'probe-mode', $options ) ? '--probe-mode' : null, + array_key_exists( 'probe-mode', $options ) ? \HtmlApiFuzz\option_string( $options, 'probe-mode', 'auto' ) : null, + \HtmlApiFuzz\option_bool( $options, 'keep-candidate-artifacts', false ) ? '--keep-candidate-artifacts' : null, + \HtmlApiFuzz\option_bool( $options, 'no-minimize', false ) ? '--no-minimize' : null, + \HtmlApiFuzz\option_bool( $options, 'any-failure', false ) ? '--any-failure' : null, + $triage_oracle_findings ? '--triage-oracle-findings' : null, + ), + static function ( $value ) { + return null !== $value; + } + ) + ), + $repo_root, + $watcher_timeout_ms, + $triage_dir . '/watcher.log' + ); + $watcher_result = array( + 'code' => $proc['code'], + 'timedOut' => $proc['timedOut'], + 'durationMs' => $proc['durationMs'], + 'logPath' => $proc['logPath'], + ); +} + +echo \HtmlApiFuzz\json_encode_safe( + array( + 'ok' => true, + 'outputDir' => $output_dir, + 'statePath' => $state_path, + 'aggregate' => $aggregate, + 'watcherResult' => $watcher_result, + ) +) . "\n"; diff --git a/tools/html-api-fuzz/lib/ChromeOracleRenderer.php b/tools/html-api-fuzz/lib/ChromeOracleRenderer.php new file mode 100644 index 0000000000000..21136c0cb4f7c --- /dev/null +++ b/tools/html-api-fuzz/lib/ChromeOracleRenderer.php @@ -0,0 +1,940 @@ +script = $script; + $this->chrome_executable = $chrome_executable ?? ''; + $this->node_executable = $node_binary; + $this->render_timeout_ms = $render_timeout_ms; + $this->startup_timeout_ms = $startup_timeout_ms; + + self::$instances[] = \WeakReference::create( $this ); + if ( ! self::$shutdown_registered ) { + register_shutdown_function( array( self::class, 'shutdown_all' ) ); + self::$shutdown_registered = true; + } + } + + public static function shutdown_all(): void { + foreach ( self::$instances as $reference ) { + $instance = $reference->get(); + if ( $instance instanceof self ) { + $instance->close_silently(); + } + } + self::$instances = array(); + } + + public function __destruct() { + $this->close_silently(); + } + + public function startup_timeout_ms(): int { + return $this->startup_timeout_ms; + } + + public function render_timeout_ms(): int { + return $this->render_timeout_ms; + } + + public function script(): string { + return $this->script; + } + + public function chrome_executable(): string { + return $this->chrome_executable; + } + + public function node_executable(): string { + return $this->node_executable; + } + + public function recommended_process_timeout_ms( string $checks ): int { + $render_calls = 'baseline' === $checks ? 1 : 4; + if ( $this->render_timeout_ms > intdiv( PHP_INT_MAX, $render_calls ) ) { + throw new \OverflowException( 'Chrome Worker render-call budget exceeds the platform integer range.' ); + } + $parts = array( $this->startup_timeout_ms, $render_calls * $this->render_timeout_ms, self::CLEANUP_TIMEOUT_MS, self::CLEANUP_HEADROOM_MS ); + $total = 0; + foreach ( $parts as $part ) { + if ( $part < 0 || $total > PHP_INT_MAX - $part ) { + throw new \OverflowException( 'Chrome Worker timeout recommendation exceeds the platform integer range.' ); + } + $total += $part; + } + return $total; + } + + public function metadata(): array { + if ( null !== $this->metadata ) { + return $this->metadata; + } + try { + $this->prepare_paths(); + $this->ensure_service(); + $this->metadata = array( + 'schemaVersion' => 1, + 'kind' => OracleRenderer::KIND_CHROME_CDP, + 'available' => true, + 'identity' => $this->verified_identity, + 'error' => null, + ); + } catch ( \Throwable $error ) { + $cleanup_error = $this->stop_service( false ); + $this->metadata = array( + 'schemaVersion' => 1, + 'kind' => OracleRenderer::KIND_CHROME_CDP, + 'available' => false, + 'identity' => null, + 'error' => $error->getMessage() . ( null === $cleanup_error ? '' : '; cleanup failed: ' . $cleanup_error ), + ); + } + return $this->metadata; + } + + public function render( string $html, string $mode, array $limits, string $fragment_context ): array { + $metadata = $this->metadata(); + if ( true !== $metadata['available'] ) { + return $this->infrastructure_result( 'Chrome CDP oracle is unavailable: ' . (string) $metadata['error'] ); + } + if ( strlen( $html ) > self::MAX_INPUT_BYTES ) { + return array( + 'status' => TreeRenderer::STATUS_ERROR, + 'failureClass' => 'input-byte-limit-exceeded', + 'error' => 'Chrome oracle input exceeded the exact 2 MiB byte limit.', + 'nodeCount' => 0, + 'oracle' => $metadata, + ); + } + $max_nodes = self::positive_limit( $limits, 'maxNodes', 3000, 100000 ); + $max_depth = self::positive_limit( $limits, 'maxDepth', 512, 1024 ); + $max_tree_bytes = self::positive_limit( $limits, 'maxTreeBytes', 16777216, 16777216 ); + $started_at = hrtime( true ); + try { + $this->ensure_service(); + $before = $this->local_snapshot(); + $request = array( + 'command' => 'render', + 'htmlBase64' => base64_encode( $html ), + 'mode' => $mode, + 'maxNodes' => $max_nodes, + 'maxDepth' => $max_depth, + 'maxTreeBytes' => $max_tree_bytes, + ); + if ( Generator::MODE_FRAGMENT_BODY === $mode ) { + $request['context'] = $fragment_context; + } + $response = $this->request( + $request, + $this->render_timeout_ms + ); + $after = $this->local_snapshot(); + if ( ! hash_equals( self::canonical_json( $before ), self::canonical_json( $after ) ) ) { + throw new \RuntimeException( 'Chrome oracle trust inputs changed during rendering.' ); + } + $result = $this->validate_render_response( $response, $max_nodes, $max_tree_bytes ); + $result['oracle'] = $metadata; + $result['process'] = array( + 'durationMs' => round( max( 0, hrtime( true ) - $started_at ) / 1000000, 3 ), + 'persistent' => true, + ); + return $result; + } catch ( \Throwable $error ) { + $cleanup_error = $this->stop_service( false ); + return $this->infrastructure_result( + $error->getMessage() . ( null === $cleanup_error ? '' : '; cleanup failed: ' . $cleanup_error ), + round( max( 0, hrtime( true ) - $started_at ) / 1000000, 3 ) + ); + } + } + + public function close(): void { + $error = $this->stop_service( true ); + if ( null !== $error ) { + throw new \RuntimeException( $error ); + } + } + + private function close_silently(): void { + try { + $this->close(); + } catch ( \Throwable $ignored ) { + // Explicit owners surface cleanup failures before completing work. + } + } + + private function ensure_service(): void { + if ( is_resource( $this->process ) ) { + $status = proc_get_status( $this->process ); + if ( $status['running'] ) { + return; + } + $cleanup_error = $this->stop_service( false ); + if ( null !== $cleanup_error ) { + throw new \RuntimeException( 'Dead Chrome service cleanup failed: ' . $cleanup_error ); + } + } + + $before = $this->local_snapshot(); + $command = array( $this->node_executable, $this->script, '--serve', '--engine', 'chrome', '--chrome-executable', $this->chrome_executable ); + $spec = array( + 0 => array( 'pipe', 'r' ), + 1 => array( 'pipe', 'w' ), + 2 => array( 'pipe', 'w' ), + ); + $this->process = @proc_open( $command, $spec, $this->pipes, repo_root(), null, array( 'bypass_shell' => true ) ); + if ( ! is_resource( $this->process ) ) { + $this->process = null; + $this->pipes = array(); + throw new \RuntimeException( 'Could not start the Chrome oracle service.' ); + } + foreach ( $this->pipes as $pipe ) { + stream_set_blocking( $pipe, false ); + } + stream_set_write_buffer( $this->pipes[0], 0 ); + $this->stdout = ''; + $this->stderr_tail = ''; + $this->stderr_bytes = 0; + $this->stderr_overflow = false; + $this->observed_exit_code = null; + + $response = $this->request( array( 'command' => 'version' ), $this->startup_timeout_ms ); + if ( ! self::exact_keys( $response, array( 'id', 'status', 'oracle' ) ) || 'ok' !== $response['status'] ) { + throw new \RuntimeException( 'Chrome service returned an invalid version response.' ); + } + $identity = $this->validate_oracle( $response['oracle'], true ); + if ( null !== $this->verified_identity && ! hash_equals( self::canonical_json( $this->verified_identity ), self::canonical_json( $identity ) ) ) { + throw new \RuntimeException( 'Restarted Chrome service identity differs from its verified identity.' ); + } + $this->verified_identity = $identity; + $after = $this->local_snapshot(); + if ( ! hash_equals( self::canonical_json( $before ), self::canonical_json( $after ) ) ) { + throw new \RuntimeException( 'Chrome oracle trust inputs changed during startup.' ); + } + } + + private function request( array $request, int $timeout_ms, bool $allow_exit_after_response = false ): array { + if ( ! is_resource( $this->process ) || 3 !== count( $this->pipes ) ) { + throw new \RuntimeException( 'Chrome oracle service is not running.' ); + } + $this->drain_available(); + if ( '' !== $this->stdout ) { + throw new \RuntimeException( 'Chrome oracle emitted an unsolicited response frame.' ); + } + if ( $this->stderr_overflow ) { + throw new \RuntimeException( 'Chrome oracle stderr exceeded 1 MiB.' ); + } + + $request['id'] = ++$this->request_id; + $encoded = json_encode( $request, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR ) . "\n"; + if ( strlen( $encoded ) > 4194304 ) { + throw new \RuntimeException( 'Chrome request frame exceeded 4 MiB.' ); + } + $offset = 0; + $deadline = microtime( true ) + ( $timeout_ms / 1000 ); + while ( true ) { + if ( microtime( true ) >= $deadline ) { + throw new \RuntimeException( 'Chrome oracle request timed out.' ); + } + $read = array( $this->pipes[1], $this->pipes[2] ); + $write = $offset < strlen( $encoded ) ? array( $this->pipes[0] ) : array(); + $except = array(); + $remaining_us = max( 1, min( 20000, (int) floor( ( $deadline - microtime( true ) ) * 1000000 ) ) ); + @stream_select( $read, $write, $except, 0, $remaining_us ); + if ( ! empty( $write ) ) { + $written = @fwrite( $this->pipes[0], substr( $encoded, $offset, 65536 ) ); + if ( false === $written ) { + throw new \RuntimeException( 'Could not write the Chrome oracle request.' ); + } + $offset += $written; + } + $this->drain_available(); + if ( $this->stderr_overflow ) { + throw new \RuntimeException( 'Chrome oracle stderr exceeded 1 MiB.' ); + } + $newline = strpos( $this->stdout, "\n" ); + if ( false !== $newline ) { + $line = substr( $this->stdout, 0, $newline ); + $trailing = substr( $this->stdout, $newline + 1 ); + $this->stdout = ''; + if ( '' !== $trailing ) { + throw new \RuntimeException( 'Chrome oracle emitted trailing response bytes.' ); + } + $decoded = StrictJsonParser::decode( $line ); + if ( ! is_array( $decoded ) || ( $decoded['id'] ?? null ) !== $request['id'] ) { + throw new \RuntimeException( 'Chrome oracle response id or schema is invalid.' ); + } + $status = proc_get_status( $this->process ); + $this->remember_exit_code( $status ); + if ( ! $allow_exit_after_response && ! $status['running'] ) { + throw new \RuntimeException( 'Chrome oracle service exited while returning a response.' ); + } + return $decoded; + } + $status = proc_get_status( $this->process ); + $this->remember_exit_code( $status ); + if ( ! $status['running'] ) { + throw new \RuntimeException( 'Chrome oracle service exited before returning a response.' ); + } + } + } + + private function drain_available(): void { + if ( ! isset( $this->pipes[1], $this->pipes[2] ) ) { + return; + } + while ( true ) { + $chunk = @fread( $this->pipes[1], 65536 ); + if ( false === $chunk || '' === $chunk ) { + break; + } + $this->stdout .= $chunk; + if ( strlen( $this->stdout ) > self::MAX_FRAME_BYTES ) { + throw new \RuntimeException( 'Chrome oracle stdout frame exceeded 24 MiB.' ); + } + } + while ( true ) { + $chunk = @fread( $this->pipes[2], 65536 ); + if ( false === $chunk || '' === $chunk ) { + break; + } + $this->stderr_bytes += strlen( $chunk ); + $this->stderr_tail = substr( $this->stderr_tail . $chunk, -self::MAX_STDERR_BYTES ); + if ( $this->stderr_bytes > self::MAX_STDERR_BYTES ) { + $this->stderr_overflow = true; + } + } + } + + private function validate_render_response( array $response, int $max_nodes, int $max_tree_bytes ): array { + $status = $response['status'] ?? null; + if ( ! is_string( $status ) || ! is_array( $response['oracle'] ?? null ) ) { + throw new \RuntimeException( 'Chrome result is missing its status or oracle identity.' ); + } + $identity = $this->validate_oracle( $response['oracle'], 'ok' === $status || 'unsupported' === $status || 'limit' === $status ); + if ( ! hash_equals( self::canonical_json( $this->verified_identity ), self::canonical_json( $identity ) ) ) { + throw new \RuntimeException( 'Chrome render identity differs from its verified identity.' ); + } + if ( 'ok' === $status ) { + if ( ! self::exact_keys( $response, array( 'id', 'status', 'oracle', 'treeBase64', 'treeBytes', 'treeSha256', 'nodeCount' ) ) ) { + throw new \RuntimeException( 'Chrome ok result has an invalid exact schema.' ); + } + $tree = is_string( $response['treeBase64'] ) ? base64_decode( $response['treeBase64'], true ) : false; + if ( + false === $tree || + base64_encode( $tree ) !== $response['treeBase64'] || + ! is_int( $response['treeBytes'] ) || strlen( $tree ) !== $response['treeBytes'] || strlen( $tree ) > $max_tree_bytes || + ! is_string( $response['treeSha256'] ) || ! hash_equals( hash( 'sha256', $tree ), $response['treeSha256'] ) || + ! is_int( $response['nodeCount'] ) || $response['nodeCount'] < 0 || $response['nodeCount'] > $max_nodes || + 1 !== preg_match( '//u', $tree ) + ) { + throw new \RuntimeException( 'Chrome canonical tree bytes are invalid.' ); + } + return array( 'status' => TreeRenderer::STATUS_OK, 'tree' => $tree, 'nodeCount' => $response['nodeCount'] ); + } + if ( 'unsupported' === $status ) { + if ( + ! self::exact_keys( $response, array( 'id', 'status', 'failureClass', 'unsupported', 'oracle' ) ) || + 'invalid-utf8' !== $response['failureClass'] || + ! is_array( $response['unsupported'] ) || + ! self::exact_keys( $response['unsupported'], array( 'reason' ) ) || + 'invalid-utf8' !== $response['unsupported']['reason'] + ) { + throw new \RuntimeException( 'Chrome unsupported result has an invalid exact schema.' ); + } + return array( + 'status' => TreeRenderer::STATUS_UNSUPPORTED, + 'failureClass' => 'invalid-utf8', + 'unsupported' => array( 'message' => 'Chrome rejected invalid UTF-8 input.' ), + 'nodeCount' => 0, + ); + } + if ( 'limit' === $status || ( 'error' === $status && array_key_exists( 'nodeCount', $response ) ) ) { + if ( + ! self::exact_keys( $response, array( 'id', 'status', 'failureClass', 'error', 'nodeCount', 'treeBytes', 'oracle' ) ) || + ! is_string( $response['failureClass'] ) || + ! is_string( $response['error'] ) || + ! is_int( $response['nodeCount'] ) || $response['nodeCount'] < 0 || $response['nodeCount'] > $max_nodes + 1 || + ! is_int( $response['treeBytes'] ) || $response['treeBytes'] < 0 || $response['treeBytes'] > $max_tree_bytes + ) { + throw new \RuntimeException( 'Chrome non-success renderer result has an invalid exact schema.' ); + } + $limits = array( 'node-limit-exceeded', 'depth-limit-exceeded', 'tree-byte-limit-exceeded' ); + if ( 'limit' === $status && ! in_array( $response['failureClass'], $limits, true ) ) { + throw new \RuntimeException( 'Chrome returned an untrusted resource-limit class.' ); + } + if ( 'error' === $status && 'oracle-renderer-error' !== $response['failureClass'] ) { + throw new \RuntimeException( 'Chrome returned an untrusted renderer error class.' ); + } + $result = array( + 'status' => TreeRenderer::STATUS_ERROR, + 'failureClass' => $response['failureClass'], + 'error' => $response['error'], + 'nodeCount' => $response['nodeCount'], + ); + if ( 'error' === $status ) { + $result['infrastructure'] = true; + } + return $result; + } + if ( + 'error' !== $status || + ! self::exact_keys( $response, array( 'id', 'status', 'failureClass', 'error', 'oracle' ) ) || + ! is_string( $response['failureClass'] ) || + ! in_array( $response['failureClass'], array( 'oracle-infrastructure-failure', 'oracle-infrastructure-timeout', 'oracle-evaluation-timeout', 'oracle-renderer-error', 'protocol-error' ), true ) || + ! is_string( $response['error'] ) + ) { + throw new \RuntimeException( 'Chrome infrastructure result has an invalid exact schema.' ); + } + return array( + 'status' => TreeRenderer::STATUS_ERROR, + 'failureClass' => 'oracle-renderer-error', + 'error' => $response['error'], + 'infrastructure' => true, + ); + } + + private function validate_oracle( $oracle, bool $require_available, bool $validate_transport = true ): array { + if ( ! is_array( $oracle ) || ! self::exact_keys( $oracle, array( 'kind', 'engine', 'available', 'identity', 'transport' ) ) ) { + throw new \RuntimeException( 'Chrome oracle metadata envelope has an invalid exact schema.' ); + } + if ( + OracleRenderer::KIND_CHROME_CDP !== $oracle['kind'] || + 'chrome' !== $oracle['engine'] || + ! is_bool( $oracle['available'] ) || + ( $require_available && true !== $oracle['available'] ) || + ! is_array( $oracle['identity'] ) || + ! is_array( $oracle['transport'] ) + ) { + throw new \RuntimeException( 'Chrome oracle metadata envelope values are invalid.' ); + } + $local = $this->local_snapshot(); + $identity = $oracle['identity']; + $identity_keys = array( + 'schemaVersion', 'kind', 'platform', 'pinnedChromeVersion', 'chromeArchiveSha256', + 'expectedChromeExecutableSha256', 'chromeExecutableSha256', 'oracleScriptSha256', + 'fragmentContextsSha256', 'fragmentContexts', 'nodeExecutableSha256', 'nodeVersion', + 'chromeVersion', 'cdpProtocolVersion', + ); + if ( ! self::exact_keys( $identity, $identity_keys ) ) { + throw new \RuntimeException( 'Chrome durable identity has an invalid exact schema.' ); + } + $expected = array( + 'schemaVersion' => 1, + 'kind' => OracleRenderer::KIND_CHROME_CDP, + 'platform' => $local['platform'], + 'pinnedChromeVersion' => $local['pinnedChromeVersion'], + 'chromeArchiveSha256' => $local['chromeArchiveSha256'], + 'expectedChromeExecutableSha256' => $local['expectedChromeExecutableSha256'], + 'chromeExecutableSha256' => $local['chromeExecutableSha256'], + 'oracleScriptSha256' => $local['oracleScriptSha256'], + 'fragmentContextsSha256' => $local['fragmentContextsSha256'], + 'fragmentContexts' => $local['fragmentContexts'], + 'nodeExecutableSha256' => $local['nodeExecutableSha256'], + 'nodeVersion' => $identity['nodeVersion'] ?? null, + 'chromeVersion' => $local['pinnedChromeVersion'], + 'cdpProtocolVersion' => $identity['cdpProtocolVersion'] ?? null, + ); + if ( + ! is_string( $expected['nodeVersion'] ) || 1 !== preg_match( '/^v[0-9]+(?:\.[0-9]+){2}(?:[-+][0-9A-Za-z.-]+)?$/D', $expected['nodeVersion'] ) || + ! is_string( $expected['cdpProtocolVersion'] ) || '' === $expected['cdpProtocolVersion'] || + ! hash_equals( self::canonical_json( $expected ), self::canonical_json( $identity ) ) + ) { + throw new \RuntimeException( 'Chrome durable identity does not match local trust anchors.' ); + } + if ( $validate_transport ) { + $this->validate_transport( $oracle['transport'] ); + } + return $expected; + } + + private function validate_transport( array $transport ): void { + $keys = array( + 'replayExcluded', 'ownerPid', 'chromeExecutablePath', 'oracleScriptPath', 'nodeExecutablePath', + 'runtimeRoot', 'profilePath', 'debugEndpoint', 'supervisorPid', 'browserPid', 'browserInstance', + ); + if ( ! self::exact_keys( $transport, $keys ) ) { + throw new \RuntimeException( 'Chrome transport metadata has an invalid exact schema.' ); + } + $status = is_resource( $this->process ) ? proc_get_status( $this->process ) : array(); + $owner_pid = (int) ( $status['pid'] ?? 0 ); + $base = realpath( sys_get_temp_dir() ); + $root = $transport['runtimeRoot'] ?? null; + $resolved_root = is_string( $root ) ? realpath( $root ) : false; + if ( + true !== $transport['replayExcluded'] || + $owner_pid < 2 || $owner_pid !== $transport['ownerPid'] || + $this->chrome_executable !== ( realpath( $transport['chromeExecutablePath'] ?? '' ) ?: null ) || + $this->script !== ( realpath( $transport['oracleScriptPath'] ?? '' ) ?: null ) || + $this->node_executable !== ( realpath( $transport['nodeExecutablePath'] ?? '' ) ?: null ) || + ! is_string( $base ) || ! is_string( $root ) || ! is_string( $resolved_root ) || 1 !== preg_match( '#^' . preg_quote( $base, '#' ) . '/html-api-fuzz-chrome-' . $owner_pid . '-[0-9a-f]{32}$#D', $resolved_root ) || + $root . '/profile' !== ( $transport['profilePath'] ?? null ) || + ! is_string( $transport['debugEndpoint'] ) || 1 !== preg_match( '#^ws://127\.0\.0\.1:[0-9]+/#', $transport['debugEndpoint'] ) || + ! is_int( $transport['supervisorPid'] ) || $transport['supervisorPid'] < 2 || + ! is_int( $transport['browserPid'] ) || $transport['browserPid'] < 2 || + ! is_int( $transport['browserInstance'] ) || $transport['browserInstance'] < 1 + ) { + throw new \RuntimeException( 'Chrome transport metadata values are invalid.' ); + } + $stat = @lstat( $root ); + if ( false === $stat || ( $stat['mode'] & 0170000 ) !== 0040000 || ( $stat['mode'] & 0777 ) !== 0700 || ( function_exists( 'posix_geteuid' ) && $stat['uid'] !== posix_geteuid() ) ) { + throw new \RuntimeException( 'Chrome runtime root ownership is invalid.' ); + } + foreach ( $this->runtime_roots as $previous ) { + if ( $previous !== $root && false !== @lstat( $previous ) ) { + throw new \RuntimeException( 'A replaced Chrome runtime root still exists.' ); + } + } + $this->runtime_roots[ $root ] = $root; + } + + private function prepare_paths(): void { + $this->script = self::resolve_regular_file( $this->script, false ); + $this->node_executable = self::resolve_node_executable( $this->node_executable ); + $local = $this->local_snapshot( true ); + if ( '' === $this->chrome_executable ) { + $trust_dir = repo_root() . '/tools/html-api-fuzz/oracles/chrome'; + $install_root = getenv( 'HTML_API_FUZZ_CHROME_INSTALL_ROOT' ); + if ( ! is_string( $install_root ) || '' === $install_root ) { + $install_root = $trust_dir . '/.chrome-for-testing'; + } + $archive_dir = str_starts_with( $local['platform'], 'mac-' ) ? 'chrome-' . $local['platform'] : 'chrome-linux64'; + $relative = str_starts_with( $local['platform'], 'mac-' ) + ? 'Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing' + : 'chrome'; + $this->chrome_executable = $install_root . '/' . $local['pinnedChromeVersion'] . '/' . $local['platform'] . '/' . $archive_dir . '/' . $relative; + } + $this->chrome_executable = self::resolve_regular_file( $this->chrome_executable, true ); + $this->local_snapshot(); + } + + private function local_snapshot( bool $without_chrome = false ): array { + $trust_dir = repo_root() . '/tools/html-api-fuzz/oracles/chrome'; + $version_raw = self::read_stable_file( $trust_dir . '/VERSION', 128 ); + if ( 1 !== preg_match( '/^[0-9]+(?:\.[0-9]+){3}\n$/D', $version_raw ) ) { + throw new \RuntimeException( 'Chrome VERSION is not canonical.' ); + } + $version = substr( $version_raw, 0, -1 ); + $platform = self::platform(); + $archive_key = 'chrome-' . $version . '-' . $platform . '.zip'; + $archive_hash = self::manifest_digest( $trust_dir . '/SHA256SUMS', $archive_key ); + $executable_hash = self::manifest_digest( $trust_dir . '/EXECUTABLE_SHA256SUMS', $platform . '.executable' ); + $contexts_raw = self::read_stable_file( repo_root() . '/tools/html-api-fuzz/oracles/fragment-contexts.json', 65536 ); + $contexts = StrictJsonParser::decode( $contexts_raw ); + if ( ! is_array( $contexts ) || $contexts !== Generator::fragment_contexts() ) { + throw new \RuntimeException( 'Chrome fragment contexts differ from the generator contexts.' ); + } + $snapshot = array( + 'platform' => $platform, + 'pinnedChromeVersion' => $version, + 'chromeArchiveSha256' => $archive_hash, + 'expectedChromeExecutableSha256' => $executable_hash, + 'oracleScriptSha256' => self::hash_stable_file( $this->script, false ), + 'fragmentContextsSha256' => hash( 'sha256', $contexts_raw ), + 'fragmentContexts' => $contexts, + 'nodeExecutableSha256' => self::hash_stable_file( $this->node_executable, true ), + 'rawTrustSha256' => array( + 'VERSION' => hash( 'sha256', $version_raw ), + 'SHA256SUMS' => hash( 'sha256', self::read_stable_file( $trust_dir . '/SHA256SUMS', 65536 ) ), + 'EXECUTABLE_SHA256SUMS' => hash( 'sha256', self::read_stable_file( $trust_dir . '/EXECUTABLE_SHA256SUMS', 65536 ) ), + ), + ); + if ( ! $without_chrome ) { + $actual = self::hash_stable_file( $this->chrome_executable, true ); + if ( ! hash_equals( $executable_hash, $actual ) ) { + throw new \RuntimeException( 'Chrome executable does not match its checked-in SHA-256.' ); + } + $snapshot['chromeExecutableSha256'] = $actual; + } + return $snapshot; + } + + private function stop_service( bool $require_acknowledgement ): ?string { + $deadline = microtime( true ) + ( self::CLEANUP_TIMEOUT_MS / 1000 ); + if ( ! is_resource( $this->process ) ) { + $this->process = null; + $this->pipes = array(); + return $this->wait_for_runtime_roots_absent( $deadline ); + } + $errors = array(); + $status = proc_get_status( $this->process ); + $this->remember_exit_code( $status ); + $acknowledged = false; + if ( $status['running'] ) { + try { + $remaining_ms = max( 1, (int) floor( ( $deadline - microtime( true ) ) * 1000 ) ); + $response = $this->request( array( 'command' => 'shutdown' ), min( 2000, $remaining_ms ), true ); + if ( + ! self::exact_keys( $response, array( 'id', 'status', 'oracle', 'shutdown' ) ) || + 'ok' !== $response['status'] || true !== $response['shutdown'] + ) { + throw new \RuntimeException( 'Chrome shutdown acknowledgement is invalid.' ); + } + $this->validate_oracle( $response['oracle'], false, false ); + $acknowledged = true; + } catch ( \Throwable $error ) { + if ( $require_acknowledgement ) { + $errors[] = $error->getMessage(); + } + } + } + if ( $require_acknowledgement && ! $acknowledged && empty( $errors ) ) { + $errors[] = 'Chrome oracle service exited before its shutdown acknowledgement.'; + } + if ( isset( $this->pipes[0] ) && is_resource( $this->pipes[0] ) ) { + fclose( $this->pipes[0] ); + } + $status = $this->wait_for_exit( min( $deadline, microtime( true ) + 5.0 ) ); + if ( $status['running'] ) { + @proc_terminate( $this->process, SIGTERM ); + $status = $this->wait_for_exit( min( $deadline, microtime( true ) + 2.0 ) ); + } + if ( $status['running'] ) { + @proc_terminate( $this->process, SIGKILL ); + $status = $this->wait_for_exit( $deadline ); + } + if ( $status['running'] ) { + $errors[] = 'Exact Chrome oracle child survived SIGKILL deadline.'; + } + try { + $this->drain_available(); + } catch ( \Throwable $error ) { + $errors[] = $error->getMessage(); + } + foreach ( array( 1, 2 ) as $descriptor ) { + if ( isset( $this->pipes[ $descriptor ] ) && is_resource( $this->pipes[ $descriptor ] ) ) { + fclose( $this->pipes[ $descriptor ] ); + } + } + $close_code = @proc_close( $this->process ); + $this->remember_exit_code( $status ); + $observed_code = null !== $this->observed_exit_code ? $this->observed_exit_code : $close_code; + if ( $acknowledged && 0 !== $observed_code ) { + $errors[] = 'Acknowledged Chrome oracle service exited nonzero.'; + } + if ( '' !== $this->stdout ) { + $errors[] = 'Chrome oracle emitted bytes after its shutdown acknowledgement.'; + } + $this->process = null; + $this->pipes = array(); + $this->stdout = ''; + $this->observed_exit_code = null; + if ( $this->stderr_overflow ) { + $errors[] = 'Chrome oracle stderr exceeded 1 MiB.'; + } + $root_error = $this->wait_for_runtime_roots_absent( $deadline ); + if ( null !== $root_error ) { + $errors[] = $root_error; + } + return empty( $errors ) ? null : implode( '; ', array_unique( $errors ) ); + } + + private function wait_for_exit( float $deadline ): array { + $status = proc_get_status( $this->process ); + $this->remember_exit_code( $status ); + while ( $status['running'] && microtime( true ) < $deadline ) { + try { + $this->drain_available(); + } catch ( \Throwable $ignored ) { + $this->stderr_overflow = true; + } + $remaining_us = (int) floor( ( $deadline - microtime( true ) ) * 1000000 ); + if ( $remaining_us > 0 ) { + usleep( min( 10000, $remaining_us ) ); + } + $status = proc_get_status( $this->process ); + $this->remember_exit_code( $status ); + } + return $status; + } + + private function remember_exit_code( array $status ): void { + if ( ! $status['running'] && is_int( $status['exitcode'] ?? null ) && $status['exitcode'] >= 0 ) { + $this->observed_exit_code = $status['exitcode']; + } + } + + private function wait_for_runtime_roots_absent( float $deadline ): ?string { + foreach ( $this->runtime_roots as $root ) { + while ( true ) { + clearstatcache( true, $root ); + if ( false === @lstat( $root ) ) { + continue 2; + } + $remaining_us = (int) floor( ( $deadline - microtime( true ) ) * 1000000 ); + if ( $remaining_us <= 0 ) { + break; + } + usleep( min( 10000, $remaining_us ) ); + } + return 'Authenticated Chrome runtime root survived cleanup: ' . $root; + } + $this->runtime_roots = array(); + return null; + } + + private function infrastructure_result( string $message, ?float $duration_ms = null ): array { + $result = array( + 'status' => TreeRenderer::STATUS_ERROR, + 'failureClass' => 'oracle-renderer-error', + 'error' => $message, + 'infrastructure' => true, + 'oracle' => $this->metadata ?? array( + 'schemaVersion' => 1, + 'kind' => OracleRenderer::KIND_CHROME_CDP, + 'available' => false, + 'identity' => null, + 'error' => $message, + ), + ); + if ( null !== $duration_ms ) { + $result['process'] = array( + 'durationMs' => $duration_ms, + 'persistent' => true, + 'stderrTail' => $this->stderr_tail, + ); + } + return $result; + } + + private static function platform(): string { + $machine = strtolower( php_uname( 'm' ) ); + if ( 'Darwin' === PHP_OS_FAMILY && in_array( $machine, array( 'arm64', 'aarch64' ), true ) ) { + return 'mac-arm64'; + } + if ( 'Darwin' === PHP_OS_FAMILY && in_array( $machine, array( 'x86_64', 'amd64' ), true ) ) { + return 'mac-x64'; + } + if ( 'Linux' === PHP_OS_FAMILY && in_array( $machine, array( 'x86_64', 'amd64' ), true ) ) { + return 'linux64'; + } + throw new \RuntimeException( 'Chrome is unsupported on ' . PHP_OS_FAMILY . '/' . $machine . '.' ); + } + + private static function manifest_digest( string $path, string $key ): string { + $matches = array(); + foreach ( preg_split( '/\r?\n/', self::read_stable_file( $path, 65536 ) ) as $line ) { + $line = trim( $line ); + if ( '' === $line || str_starts_with( $line, '#' ) ) { + continue; + } + $fields = preg_split( '/\s+/', $line ); + if ( 2 === count( $fields ) && $key === $fields[1] ) { + $matches[] = $fields[0]; + } + } + if ( 1 !== count( $matches ) || 1 !== preg_match( '/^[0-9a-f]{64}$/D', $matches[0] ) ) { + throw new \RuntimeException( 'Chrome checksum manifest is missing one exact ' . $key . ' record.' ); + } + return $matches[0]; + } + + private static function resolve_executable( string $command ): string { + if ( str_contains( $command, DIRECTORY_SEPARATOR ) ) { + $path = str_starts_with( $command, DIRECTORY_SEPARATOR ) ? $command : repo_root() . DIRECTORY_SEPARATOR . $command; + return self::resolve_regular_file( $path, true ); + } + foreach ( explode( PATH_SEPARATOR, (string) getenv( 'PATH' ) ) as $directory ) { + if ( '' === $directory ) { + continue; + } + $path = $directory . DIRECTORY_SEPARATOR . $command; + if ( is_file( $path ) && is_executable( $path ) ) { + return self::resolve_regular_file( $path, true ); + } + } + throw new \RuntimeException( 'Could not resolve executable command: ' . $command ); + } + + private static function resolve_node_executable( string $command ): string { + $launch = str_contains( $command, DIRECTORY_SEPARATOR ) + ? self::resolve_executable( $command ) + : $command; + $spec = array( + 0 => array( 'pipe', 'r' ), + 1 => array( 'pipe', 'w' ), + 2 => array( 'pipe', 'w' ), + ); + $process = @proc_open( array( $launch, '-p', 'process.execPath' ), $spec, $pipes, repo_root(), null, array( 'bypass_shell' => true ) ); + if ( ! is_resource( $process ) ) { + throw new \RuntimeException( 'Could not probe the selected Node command.' ); + } + fclose( $pipes[0] ); + stream_set_blocking( $pipes[1], false ); + stream_set_blocking( $pipes[2], false ); + $stdout = ''; + $stderr = ''; + $deadline = microtime( true ) + 5.0; + $status = proc_get_status( $process ); + while ( $status['running'] && microtime( true ) < $deadline ) { + $stdout .= (string) stream_get_contents( $pipes[1] ); + $stderr .= (string) stream_get_contents( $pipes[2] ); + if ( strlen( $stdout ) > 4096 || strlen( $stderr ) > 4096 ) { + break; + } + usleep( 10000 ); + $status = proc_get_status( $process ); + } + if ( $status['running'] ) { + @proc_terminate( $process, SIGTERM ); + usleep( 100000 ); + $status = proc_get_status( $process ); + if ( $status['running'] ) { + @proc_terminate( $process, SIGKILL ); + } + } + $stdout .= (string) stream_get_contents( $pipes[1] ); + $stderr .= (string) stream_get_contents( $pipes[2] ); + fclose( $pipes[1] ); + fclose( $pipes[2] ); + $code = proc_close( $process ); + $path = rtrim( $stdout, "\r\n" ); + if ( 0 !== $code || '' === $path || str_contains( $path, "\n" ) || strlen( $path ) > 4096 || '' !== trim( $stderr ) ) { + throw new \RuntimeException( 'Selected Node command did not return one quiet process.execPath.' ); + } + return self::resolve_regular_file( $path, true ); + } + + private static function resolve_regular_file( string $path, bool $executable ): string { + clearstatcache( true, $path ); + $resolved = realpath( $path ); + $stat = false === $resolved ? false : @lstat( $resolved ); + if ( false === $resolved || false === $stat || ( $stat['mode'] & 0170000 ) !== 0100000 || ( $executable && ! is_executable( $resolved ) ) ) { + throw new \RuntimeException( 'Required Chrome identity path is not a trusted regular file: ' . $path ); + } + return $resolved; + } + + private static function read_stable_file( string $path, int $maximum ): string { + $resolved = self::resolve_regular_file( $path, false ); + $handle = @fopen( $resolved, 'rb' ); + if ( false === $handle ) { + throw new \RuntimeException( 'Could not open Chrome identity file.' ); + } + $before = fstat( $handle ); + $contents = ''; + try { + while ( ! feof( $handle ) ) { + $chunk = fread( $handle, min( 65536, $maximum + 1 - strlen( $contents ) ) ); + if ( false === $chunk ) { + throw new \RuntimeException( 'Could not read Chrome identity file.' ); + } + $contents .= $chunk; + if ( strlen( $contents ) > $maximum ) { + throw new \RuntimeException( 'Chrome identity file exceeded its byte limit.' ); + } + } + $after = fstat( $handle ); + } finally { + fclose( $handle ); + } + clearstatcache( true, $resolved ); + $path_after = @lstat( $resolved ); + self::assert_same_stat( $before, $after, $path_after, 'read' ); + return $contents; + } + + private static function hash_stable_file( string $path, bool $executable ): string { + $resolved = self::resolve_regular_file( $path, $executable ); + $handle = @fopen( $resolved, 'rb' ); + if ( false === $handle ) { + throw new \RuntimeException( 'Could not open Chrome identity file for hashing.' ); + } + $before = fstat( $handle ); + $hash = hash_init( 'sha256' ); + try { + while ( ! feof( $handle ) ) { + $chunk = fread( $handle, 1048576 ); + if ( false === $chunk ) { + throw new \RuntimeException( 'Could not hash Chrome identity file.' ); + } + if ( '' !== $chunk ) { + hash_update( $hash, $chunk ); + } + } + $after = fstat( $handle ); + } finally { + fclose( $handle ); + } + clearstatcache( true, $resolved ); + $path_after = @lstat( $resolved ); + self::assert_same_stat( $before, $after, $path_after, 'hash' ); + return hash_final( $hash ); + } + + private static function assert_same_stat( $before, $after, $path_after, string $operation ): void { + foreach ( array( 'dev', 'ino', 'mode', 'uid', 'size', 'mtime', 'ctime' ) as $field ) { + if ( + ! is_array( $before ) || ! is_array( $after ) || ! is_array( $path_after ) || + ( $before[ $field ] ?? null ) !== ( $after[ $field ] ?? null ) || + ( $after[ $field ] ?? null ) !== ( $path_after[ $field ] ?? null ) + ) { + throw new \RuntimeException( 'Chrome identity file changed during ' . $operation . '.' ); + } + } + } + + private static function positive_limit( array $limits, string $key, int $default, int $maximum ): int { + $value = $limits[ $key ] ?? $default; + if ( ! is_int( $value ) || $value < 1 || $value > $maximum ) { + throw new \InvalidArgumentException( $key . ' must be between 1 and ' . $maximum . ' for Chrome.' ); + } + return $value; + } + + private static function canonical_json( $value ): string { + return json_encode( self::canonicalize( $value ), JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR ); + } + + private static function canonicalize( $value ) { + if ( ! is_array( $value ) ) { + return $value; + } + if ( array_keys( $value ) !== range( 0, count( $value ) - 1 ) ) { + ksort( $value, SORT_STRING ); + } + foreach ( $value as $key => $item ) { + $value[ $key ] = self::canonicalize( $item ); + } + return $value; + } + + private static function exact_keys( array $value, array $keys ): bool { + $actual = array_keys( $value ); + sort( $actual, SORT_STRING ); + sort( $keys, SORT_STRING ); + return $actual === $keys; + } +} diff --git a/tools/html-api-fuzz/lib/CommonCrawlBatchCoordinator.php b/tools/html-api-fuzz/lib/CommonCrawlBatchCoordinator.php new file mode 100644 index 0000000000000..3e30c5366b434 --- /dev/null +++ b/tools/html-api-fuzz/lib/CommonCrawlBatchCoordinator.php @@ -0,0 +1,1319 @@ +phar_path = self::existing_file_option( $options, 'cc-analyzer' ); + $this->workspace = self::existing_directory_option( $options, 'workspace' ); + $this->batch_name = self::required_string_option( $options, 'batch' ); + if ( 1 !== preg_match( '/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/', $this->batch_name ) ) { + throw new \InvalidArgumentException( 'Batch name contains unsafe characters.' ); + } + $this->output_dir = self::new_path_option( $options, 'output-dir' ); + $this->coordinator_id = option_string( $options, 'coordinator-id', 'shared-' . $this->batch_name . '-' . timestamp() ); + if ( null === $this->coordinator_id || 1 !== preg_match( '/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/', $this->coordinator_id ) ) { + throw new \InvalidArgumentException( 'Coordinator ID contains unsafe characters.' ); + } + + $this->batch_timeout_ms = self::required_positive_option( $options, 'batch-timeout-ms' ); + $this->document_timeout_ms = self::positive_option( $options, 'process-timeout-ms', 90000 ); + $this->oracle_timeout_ms = self::positive_option( $options, 'oracle-timeout-ms', 10000 ); + $this->chrome_startup_timeout_ms = self::positive_option( $options, 'chrome-startup-timeout-ms', ChromeOracleRenderer::DEFAULT_STARTUP_TIMEOUT_MS ); + if ( $this->chrome_startup_timeout_ms > PHP_INT_MAX - $this->oracle_timeout_ms || $this->chrome_startup_timeout_ms + $this->oracle_timeout_ms > PHP_INT_MAX - 20000 ) { + throw new \OverflowException( 'Oracle probe timeout exceeds the platform integer range.' ); + } + $probe_budget = $this->chrome_startup_timeout_ms + $this->oracle_timeout_ms + 20000; + $this->probe_timeout_ms = max( 60000, $probe_budget ); + $this->checks = option_string( $options, 'checks', 'baseline' ); + if ( ! in_array( $this->checks, array( 'baseline', 'full', 'sampled' ), true ) ) { + throw new \InvalidArgumentException( '--checks must be baseline, full, or sampled.' ); + } + $this->memory_limit = option_string( $options, 'memory-limit', '256M' ); + if ( null === $this->memory_limit || 1 !== preg_match( '/^[1-9][0-9]*[KMG]?$/i', $this->memory_limit ) ) { + throw new \InvalidArgumentException( '--memory-limit must be a positive PHP limit such as 256M.' ); + } + $this->retain_all = self::strict_bool_option( $options, 'retain-all', false ); + $this->require_utf8 = self::strict_bool_option( $options, 'require-utf8', false ); + $this->limits = array( + 'maxInputBytes' => self::positive_option( $options, 'max-input-bytes', 2097152, 0 ), + 'maxTokens' => self::positive_option( $options, 'max-tokens', 50000 ), + 'maxNodes' => self::positive_option( $options, 'max-nodes', 50000 ), + 'maxDepth' => self::positive_option( $options, 'max-depth', 512 ), + 'maxTreeBytes' => self::positive_option( $options, 'max-tree-bytes', 16777216 ), + ); + + $lexbor = self::resolved_executable( option_string( $options, 'lexbor-oracle-bin', repo_root() . '/tools/html-api-fuzz/oracles/lexbor/build/lexbor-tree-oracle' ), 'Lexbor oracle' ); + $html5ever = self::resolved_executable( option_string( $options, 'html5ever-oracle-bin', repo_root() . '/tools/html-api-fuzz/oracles/html5ever/build/html5ever-tree-oracle' ), 'html5ever oracle' ); + $chrome_script = self::resolved_file( option_string( $options, 'chrome-oracle-script', repo_root() . '/tools/html-api-fuzz/oracles/chrome/chrome-tree-oracle.js' ), 'Chrome oracle script' ); + $chrome_executable = self::resolved_executable( self::required_string_option( $options, 'chrome-executable' ), 'Chrome executable' ); + $node = self::resolved_executable( self::required_string_option( $options, 'node-bin' ), 'Node executable' ); + $this->oracle_options = array( + OracleRenderer::KIND_LEXBOR_SOURCE => array( + 'dom-oracle' => OracleRenderer::KIND_LEXBOR_SOURCE, + 'lexbor-oracle-bin' => $lexbor, + 'oracle-timeout-ms' => (string) $this->oracle_timeout_ms, + ), + OracleRenderer::KIND_HTML5EVER_SOURCE => array( + 'dom-oracle' => OracleRenderer::KIND_HTML5EVER_SOURCE, + 'html5ever-oracle-bin' => $html5ever, + 'oracle-timeout-ms' => (string) $this->oracle_timeout_ms, + ), + OracleRenderer::KIND_CHROME_CDP => array( + 'dom-oracle' => OracleRenderer::KIND_CHROME_CDP, + 'chrome-oracle-script' => $chrome_script, + 'chrome-executable' => $chrome_executable, + 'node-bin' => $node, + 'oracle-timeout-ms' => (string) $this->oracle_timeout_ms, + 'chrome-startup-timeout-ms' => (string) $this->chrome_startup_timeout_ms, + ), + ); + + $this->callback_path = self::resolved_file( dirname( __DIR__ ) . '/commoncrawl-analysis.php', 'Common Crawl callback' ); + $this->worker_path = self::resolved_file( dirname( __DIR__ ) . '/worker.php', 'Worker' ); + $this->coordinator_script = self::resolved_file( dirname( __DIR__ ) . '/commoncrawl-batch.php', 'Coordinator script' ); + $php_binary = self::resolved_executable( PHP_BINARY, 'PHP executable' ); + $path_parts = array_values( array_unique( array( dirname( $php_binary ), dirname( $node ), '/usr/bin', '/bin' ) ) ); + $this->base_environment = array( + 'PATH' => implode( PATH_SEPARATOR, $path_parts ), + 'HOME' => $this->output_dir . '/sandbox-home', + 'TMPDIR' => $this->output_dir . '/sandbox-tmp', + 'LANG' => 'C', + 'LC_ALL' => 'C', + 'TZ' => 'UTC', + // macOS injects this into a child when it is absent; pin it instead. + '__CF_USER_TEXT_ENCODING' => sprintf( '0x%X:0x0:0x0', posix_getuid() ), + ); + } + + public static function run( array $options ): array { + $coordinator = new self( $options ); + $coordinator->claim_output(); + try { + $result = $coordinator->execute(); + $payload = self::strict_json_file( $coordinator->output_dir . '/shared-corpus.payload.json' ); + if ( self::canonical_json( $result ) !== self::canonical_json( $payload ) ) { + throw new \RuntimeException( 'Sealed shared-corpus payload changed before publication.' ); + } + write_json_file_atomic( $coordinator->output_dir . '/coordinator-state.json', array( + 'kind' => 'html-api-commoncrawl-coordinator-state', + 'status' => 'sealed', + 'coordinatorId' => $coordinator->coordinator_id, + 'sharedCorpus' => $coordinator->output_dir . '/shared-corpus.json', + 'completionMarker' => $coordinator->output_dir . '/.complete', + ) ); + write_json_file_atomic( $coordinator->output_dir . '/shared-corpus.json', $result ); + foreach ( self::REQUIRED_KINDS as $kind ) { + write_file_atomic( $coordinator->output_dir . '/' . $kind . '/.complete', "complete\n" ); + } + write_json_file_atomic( $coordinator->output_dir . '/coordinator-state.json', array( + 'kind' => 'html-api-commoncrawl-coordinator-state', + 'status' => 'published', + 'coordinatorId' => $coordinator->coordinator_id, + 'sharedCorpus' => $coordinator->output_dir . '/shared-corpus.json', + 'completionMarker' => $coordinator->output_dir . '/.complete', + ) ); + // This is the sole authoritative completion marker and the final + // fallible operation. No earlier artifact means the root completed. + write_file_atomic( $coordinator->output_dir . '/.complete', "complete\n" ); + return $result; + } catch ( \Throwable $error ) { + $success_paths = array( + $coordinator->output_dir . '/.complete', + $coordinator->output_dir . '/shared-corpus.json', + ); + foreach ( self::REQUIRED_KINDS as $kind ) { + $success_paths[] = $coordinator->output_dir . '/' . $kind . '/.complete'; + } + foreach ( $success_paths as $success_path ) { + if ( is_file( $success_path ) || is_link( $success_path ) ) { + @unlink( $success_path ); + } + } + write_json_file_atomic( $coordinator->output_dir . '/coordinator-state.json', array( + 'kind' => 'html-api-commoncrawl-coordinator-state', + 'status' => 'failed', + 'coordinatorId' => $coordinator->coordinator_id, + 'error' => $error->getMessage(), + 'throwable' => get_class( $error ), + ) ); + throw $error; + } + } + + private function claim_output(): void { + if ( file_exists( $this->output_dir ) || is_link( $this->output_dir ) ) { + throw new \RuntimeException( 'Coordinator output directory already exists; use a fresh path.' ); + } + if ( ! mkdir( $this->output_dir, 0700, false ) ) { + throw new \RuntimeException( 'Could not create coordinator output directory.' ); + } + ensure_dir( $this->base_environment['HOME'] ); + ensure_dir( $this->base_environment['TMPDIR'] ); + write_json_file_atomic( $this->output_dir . '/coordinator-state.json', array( + 'kind' => 'html-api-commoncrawl-coordinator-state', + 'status' => 'running', + 'coordinatorId' => $this->coordinator_id, + 'batchName' => $this->batch_name, + ) ); + } + + private function execute(): array { + $preflight_dir = $this->output_dir . '/preflight'; + ensure_dir( $preflight_dir ); + $entry_phar = self::file_identity( $this->phar_path ); + $info_process = $this->run_analyzer_command( + array( 'batch', 'info', $this->batch_name, '--output', 'json' ), + min( $this->batch_timeout_ms, 60000 ), + $preflight_dir . '/batch-info.stdout', + $preflight_dir . '/batch-info.stderr' + ); + if ( self::canonical_json( $entry_phar ) !== self::canonical_json( self::file_identity( $this->phar_path ) ) ) { + throw new \RuntimeException( 'cc-analyzer PHAR changed during batch info.' ); + } + $info = self::validate_batch_info( self::decode_process_json( $info_process, 'batch info' ), $this->batch_name ); + $cache_path = realpath( $info['cachePath'] ); + if ( false === $cache_path || ! is_file( $cache_path ) ) { + throw new \RuntimeException( 'Batch info cache path is not a readable regular file.' ); + } + $verify_process = $this->run_analyzer_command( + array( 'batch', 'verify', $this->batch_name, '--output', 'json' ), + min( $this->batch_timeout_ms, 60000 ), + $preflight_dir . '/batch-verify.stdout', + $preflight_dir . '/batch-verify.stderr' + ); + if ( self::canonical_json( $entry_phar ) !== self::canonical_json( self::file_identity( $this->phar_path ) ) ) { + throw new \RuntimeException( 'cc-analyzer PHAR changed during batch verify.' ); + } + $verification = self::validate_batch_verification( self::decode_process_json( $verify_process, 'batch verify' ), $this->batch_name, $info['documentCount'] ); + $manifest = self::load_batch_manifest( $cache_path, $info ); + write_json_file_atomic( $this->output_dir . '/batch-manifest.json', $manifest ); + + $invocation = array( PHP_BINARY, $this->phar_path, '--workspace', $this->workspace, 'batch', 'run', $this->batch_name, $this->callback_path, '--output', 'json' ); + $initial_trust = $this->capture_trust_bundle( $this->base_environment, $cache_path, $info, $manifest, 'during initial trust capture' ); + write_json_file_atomic( $this->output_dir . '/preflight.json', array( + 'kind' => 'html-api-commoncrawl-preflight', + 'coordinatorId' => $this->coordinator_id, + 'info' => $info, + 'verification' => $verification, + 'manifestSha256' => $manifest['batchManifestSha256'], + 'corpusFingerprint' => $manifest['corpusFingerprint'], + 'invocation' => $invocation, + 'baseEnvironment' => $this->base_environment, + 'trust' => $initial_trust, + ) ); + + $runs = array(); + foreach ( self::REQUIRED_KINDS as $kind ) { + $run_dir = $this->output_dir . '/' . $kind; + if ( file_exists( $run_dir ) || is_link( $run_dir ) ) { + throw new \RuntimeException( "Oracle output directory already exists: {$kind}" ); + } + ensure_dir( $run_dir ); + $environment = $this->run_environment( $kind, $run_dir, $cache_path, $manifest, $invocation, $initial_trust ); + $environment_evidence = $this->write_environment_evidence( $run_dir, $environment ); + $before = $this->capture_trust_bundle( $environment, $cache_path, $info, $manifest, "immediately before {$kind}" ); + self::assert_same_trust( $initial_trust, $before, "before {$kind}" ); + + $process = $this->run_analyzer_command( + array( 'batch', 'run', $this->batch_name, $this->callback_path, '--output', 'json' ), + $this->batch_timeout_ms, + $run_dir . '/batch-run.stdout', + $run_dir . '/batch-run.stderr', + $environment + ); + $batch_result = self::validate_batch_run_result( + self::decode_process_json( $process, "{$kind} batch run" ), + $manifest['batch'], + $this->callback_path, + $initial_trust['callback']['sha256'] + ); + $after = $this->capture_trust_bundle( $environment, $cache_path, $info, $manifest, "immediately after {$kind}" ); + self::assert_same_trust( $initial_trust, $after, "after {$kind}" ); + $provenance = $this->provenance_from_environment( $environment ); + $summary = self::verify_run_output( $run_dir, $manifest, $environment['HTML_API_CC_RUN_ID'], $kind, $initial_trust['oracles'][ $kind ]['identitySha256'], $provenance, $environment ); + $runs[ $kind ] = array( + 'kind' => $kind, + 'outputDir' => $run_dir, + 'batchResult' => $batch_result, + 'process' => self::compact_process( $process ), + 'summary' => $summary, + 'provenance' => $provenance, + 'environment' => $environment, + 'environmentEvidence' => $environment_evidence, + ); + } + + $final_trust = $this->capture_trust_bundle( $this->base_environment, $cache_path, $info, $manifest, 'during final trust capture' ); + self::assert_same_trust( $initial_trust, $final_trust, 'before final sealing' ); + $disk_manifest = self::strict_json_file( $this->output_dir . '/batch-manifest.json' ); + self::assert_same_manifest( $manifest, $disk_manifest, 'in on-disk batch-manifest.json before sealing' ); + + foreach ( $runs as $kind => &$run ) { + $final_batch_result = self::validate_batch_run_result( + self::strict_json_file( $run['outputDir'] . '/batch-run.stdout' ), + $manifest['batch'], + $this->callback_path, + $initial_trust['callback']['sha256'] + ); + if ( self::canonical_json( $run['batchResult'] ) !== self::canonical_json( $final_batch_result ) ) { + throw new \RuntimeException( "{$kind} batch stdout changed after verification." ); + } + $run['summary'] = self::verify_run_output( + $run['outputDir'], + $manifest, + $run['environment']['HTML_API_CC_RUN_ID'], + $kind, + $initial_trust['oracles'][ $kind ]['identitySha256'], + $run['provenance'], + $run['environment'] + ); + $environment_identity = self::file_identity( $run['environmentEvidence']['path'] ); + if ( ! hash_equals( $run['environmentEvidence']['sha256'], $environment_identity['sha256'] ) ) { + throw new \RuntimeException( "{$kind} replacement environment file changed after publication." ); + } + $seal = self::seal_directory( $run['outputDir'], array( 'run-seal.json', '.complete' ) ); + write_json_file_atomic( $run['outputDir'] . '/run-seal.json', $seal ); + $run['sealSha256'] = self::file_identity( $run['outputDir'] . '/run-seal.json' )['sha256']; + } + unset( $run ); + $vectors = array_map( static fn ( array $run ): array => $run['summary']['vector'], $runs ); + $first_vector = reset( $vectors ); + foreach ( $vectors as $kind => $vector ) { + if ( self::canonical_json( $first_vector ) !== self::canonical_json( $vector ) ) { + throw new \RuntimeException( "Shared corpus vector differs for {$kind}." ); + } + } + self::assert_same_manifest( $manifest, self::load_batch_manifest( $cache_path, $info ), 'in the final pre-seal cache read' ); + + $result = array( + 'schemaVersion' => 1, + 'kind' => 'html-api-commoncrawl-shared-corpus', + 'coordinatorId' => $this->coordinator_id, + 'batchName' => $this->batch_name, + 'crawlId' => $manifest['batch']['crawlId'], + 'documentCount' => $manifest['batch']['documentCount'], + 'batchManifestSha256' => $manifest['batchManifestSha256'], + 'corpusFingerprint' => $manifest['corpusFingerprint'], + 'invocation' => $invocation, + 'trust' => $initial_trust, + 'orderedCorpus' => $first_vector, + 'runs' => $runs, + ); + write_json_file_atomic( $this->output_dir . '/shared-corpus.payload.json', $result ); + $root_seal = self::seal_directory( $this->output_dir, array( 'coordinator-seal.json', 'coordinator-state.json', '.complete' ) ); + write_json_file_atomic( $this->output_dir . '/coordinator-seal.json', $root_seal ); + return $result; + } + + private function run_analyzer_command( array $arguments, int $timeout_ms, string $stdout_path, string $stderr_path, ?array $environment = null ): array { + $process = run_php_process( + array_merge( array( $this->phar_path, '--workspace', $this->workspace ), $arguments ), + repo_root(), + $timeout_ms, + null, + 1048576, + true, + $environment ?? $this->base_environment, + true, + $stdout_path, + $stderr_path + ); + self::assert_process_success( $process, 'cc-analyzer command' ); + return $process; + } + + private function capture_trust_bundle( array $oracle_environment, string $cache_path, array $info, array $expected_manifest, string $when ): array { + $manifest_before = self::load_batch_manifest( $cache_path, $info ); + self::assert_same_manifest( $expected_manifest, $manifest_before, "before {$when}" ); + $files_before = $this->trust_file_identities(); + $runtime_before = $this->runtime_probe( $this->base_environment ); + $code_before = self::repository_code_identity( $this->git_probe( $this->base_environment ) ); + $oracles = array(); + foreach ( self::REQUIRED_KINDS as $kind ) { + $probe = $this->oracle_probe( $kind, $oracle_environment ); + $oracles[ $kind ] = array( + 'metadata' => $probe['metadata'], + 'identitySha256' => OracleRenderer::identity_sha256( $probe['metadata'] ), + 'replayOptions' => $probe['replayOptions'], + 'workerArgs' => $probe['workerArgs'], + ); + } + $runtime_after = $this->runtime_probe( $this->base_environment ); + $code_after = self::repository_code_identity( $this->git_probe( $this->base_environment ) ); + $files_after = $this->trust_file_identities(); + $manifest_after = self::load_batch_manifest( $cache_path, $info ); + self::assert_same_manifest( $expected_manifest, $manifest_after, "after {$when}" ); + if ( self::canonical_json( $files_before ) !== self::canonical_json( $files_after ) || self::canonical_json( $runtime_before ) !== self::canonical_json( $runtime_after ) || self::canonical_json( $code_before ) !== self::canonical_json( $code_after ) ) { + throw new \RuntimeException( "Trust inputs changed {$when}." ); + } + $bundle = array( + 'phpBinary' => $files_after['phpBinary'], + 'phar' => $files_after['phar'], + 'callback' => $files_after['callback'], + 'worker' => $files_after['worker'], + 'oracleTrustFiles' => $files_after['oracleTrustFiles'], + 'runtime' => $runtime_after, + 'repository' => $code_after, + 'oracles' => $oracles, + 'batchManifestSha256' => $manifest_after['batchManifestSha256'], + 'corpusFingerprint' => $manifest_after['corpusFingerprint'], + ); + $bundle['trustBundleSha256'] = hash( 'sha256', self::canonical_json( $bundle ) ); + return $bundle; + } + + private function trust_file_identities(): array { + $lexbor_binary = $this->oracle_options[ OracleRenderer::KIND_LEXBOR_SOURCE ]['lexbor-oracle-bin']; + $html5ever_binary = $this->oracle_options[ OracleRenderer::KIND_HTML5EVER_SOURCE ]['html5ever-oracle-bin']; + $html5ever_root = dirname( dirname( $html5ever_binary ) ); + $chrome_root = repo_root() . '/tools/html-api-fuzz/oracles/chrome'; + $paths = array( + 'lexborBinary' => $lexbor_binary, + 'lexborBuildManifest' => dirname( $lexbor_binary ) . '/build-manifest.json', + 'html5everBinary' => $html5ever_binary, + 'html5everBuildManifest' => dirname( $html5ever_binary ) . '/build-manifest.json', + 'html5everCargoToml' => $html5ever_root . '/Cargo.toml', + 'html5everCargoLock' => $html5ever_root . '/Cargo.lock', + 'html5everRustToolchain' => $html5ever_root . '/rust-toolchain.toml', + 'html5everSource' => $html5ever_root . '/src/main.rs', + 'chromeScript' => $this->oracle_options[ OracleRenderer::KIND_CHROME_CDP ]['chrome-oracle-script'], + 'chromeExecutable' => $this->oracle_options[ OracleRenderer::KIND_CHROME_CDP ]['chrome-executable'], + 'nodeExecutable' => $this->oracle_options[ OracleRenderer::KIND_CHROME_CDP ]['node-bin'], + 'chromeVersion' => $chrome_root . '/VERSION', + 'chromeArchiveChecksums' => $chrome_root . '/SHA256SUMS', + 'chromeExecutableChecksums' => $chrome_root . '/EXECUTABLE_SHA256SUMS', + 'fragmentContexts' => repo_root() . '/tools/html-api-fuzz/oracles/fragment-contexts.json', + ); + $identities = array(); + foreach ( $paths as $name => $path ) { + $identities[ $name ] = self::file_identity( $path ); + } + return array( + 'phpBinary' => self::file_identity( PHP_BINARY ), + 'phar' => self::file_identity( $this->phar_path ), + 'callback' => self::file_identity( $this->callback_path ), + 'worker' => self::file_identity( $this->worker_path ), + 'oracleTrustFiles' => $identities, + ); + } + + private function write_environment_evidence( string $run_dir, array $environment ): array { + ksort( $environment ); + $evidence = array( + 'kind' => 'html-api-commoncrawl-replacement-environment', + 'environment' => $environment, + 'environmentSha256' => hash( 'sha256', self::canonical_json( $environment ) ), + ); + $path = $run_dir . '/replacement-environment.json'; + write_json_file_atomic( $path, $evidence ); + return array( + 'path' => $path, + 'sha256' => self::file_identity( $path )['sha256'], + 'environmentSha256' => $evidence['environmentSha256'], + ); + } + + private function runtime_probe( array $environment ): array { + $process = run_php_process( + array( $this->coordinator_script, '--internal-runtime-probe' ), + repo_root(), + min( $this->probe_timeout_ms, 60000 ), + null, + 1048576, + true, + $environment, + true + ); + $probe = self::decode_process_json( $process, 'sanitized PHP runtime probe' ); + self::assert_exact_keys( $probe, array( 'phpVersion', 'phpBinary', 'loadedIni', 'scannedIni', 'extensions', 'environment' ), 'runtime probe' ); + self::assert_environment_echo( $probe, $environment, 'runtime probe' ); + if ( ! is_string( $probe['phpVersion'] ) || '' === $probe['phpVersion'] || ! is_string( $probe['phpBinary'] ) || '' === $probe['phpBinary'] || ( null !== $probe['loadedIni'] && ( ! is_string( $probe['loadedIni'] ) || '' === $probe['loadedIni'] ) ) || ! is_array( $probe['scannedIni'] ) || ! is_array( $probe['extensions'] ) ) { + throw new \RuntimeException( 'Runtime probe returned invalid types.' ); + } + foreach ( array_merge( $probe['scannedIni'], $probe['extensions'] ) as $item ) { + if ( ! is_string( $item ) || '' === $item ) { + throw new \RuntimeException( 'Runtime probe returned an invalid ini or extension entry.' ); + } + } + if ( realpath( (string) $probe['phpBinary'] ) !== realpath( PHP_BINARY ) ) { + throw new \RuntimeException( 'Runtime probe used a different PHP binary.' ); + } + $ini = array(); + $ini_paths = array_merge( is_string( $probe['loadedIni'] ) && '' !== $probe['loadedIni'] ? array( $probe['loadedIni'] ) : array(), is_array( $probe['scannedIni'] ) ? $probe['scannedIni'] : array() ); + foreach ( $ini_paths as $path ) { + $ini[] = self::file_identity( self::resolved_file( is_string( $path ) ? $path : null, 'PHP ini file' ) ); + } + $identity = array( + 'phpVersion' => $probe['phpVersion'], + 'phpBinary' => realpath( PHP_BINARY ), + 'iniFiles' => $ini, + 'extensions' => $probe['extensions'], + ); + $identity['runtimeSha256'] = hash( 'sha256', self::canonical_json( $identity ) ); + return $identity; + } + + private function git_probe( array $environment ): array { + $process = run_php_process( + array( $this->coordinator_script, '--internal-git-probe' ), + repo_root(), + 60000, + null, + 16777216, + true, + $environment, + true + ); + $probe = self::decode_process_json( $process, 'sanitized Git probe' ); + self::assert_exact_keys( $probe, array( 'commit', 'branch', 'statusBase64', 'tracked', 'environment' ), 'Git probe' ); + self::assert_environment_echo( $probe, $environment, 'Git probe' ); + if ( ! is_string( $probe['commit'] ) || 1 !== preg_match( '/^[0-9a-f]{40}$/', $probe['commit'] ) || ( null !== $probe['branch'] && ! is_string( $probe['branch'] ) ) || ! is_string( $probe['statusBase64'] ) || ! is_array( $probe['tracked'] ) ) { + throw new \RuntimeException( 'Git probe returned invalid repository identity.' ); + } + return $probe; + } + + private function oracle_probe( string $kind, array $environment ): array { + $encoded = base64_encode( self::canonical_json( $this->oracle_options[ $kind ] ) ); + $process = run_php_process( + array( $this->coordinator_script, '--internal-oracle-probe', $encoded ), + repo_root(), + $this->probe_timeout_ms, + null, + 1048576, + true, + $environment, + true + ); + $probe = self::decode_process_json( $process, "{$kind} oracle probe" ); + self::assert_exact_keys( $probe, array( 'metadata', 'replayOptions', 'workerArgs', 'environment' ), "{$kind} oracle probe" ); + self::assert_environment_echo( $probe, $environment, "{$kind} oracle probe" ); + if ( ! is_array( $probe['metadata'] ?? null ) || true !== ( $probe['metadata']['available'] ?? false ) || $kind !== ( $probe['metadata']['kind'] ?? null ) ) { + throw new \RuntimeException( "{$kind} oracle preflight is unavailable or inconsistent." ); + } + if ( ! is_array( $probe['replayOptions'] ?? null ) || ! is_array( $probe['workerArgs'] ?? null ) ) { + throw new \RuntimeException( "{$kind} oracle probe omitted replay or Worker options." ); + } + return $probe; + } + + private function run_environment( string $kind, string $run_dir, string $cache_path, array $manifest, array $invocation, array $trust ): array { + $environment = $this->base_environment; + $environment['CC_ANALYZER_OUTPUT_DIR'] = $run_dir; + $environment['HTML_API_CC_RUN_ID'] = $this->coordinator_id . '-' . $kind; + $environment['HTML_API_CC_ORACLE'] = $kind; + $environment['HTML_API_CC_EXPECT_ORACLE_IDENTITY_SHA256'] = $trust['oracles'][ $kind ]['identitySha256']; + $environment['HTML_API_CC_ORACLE_TIMEOUT_MS'] = (string) $this->oracle_timeout_ms; + $environment['HTML_API_CC_CHROME_STARTUP_TIMEOUT_MS'] = (string) $this->chrome_startup_timeout_ms; + $environment['HTML_API_CC_PROCESS_TIMEOUT_MS'] = (string) $this->document_timeout_ms; + $environment['HTML_API_CC_CHECKS'] = $this->checks; + $environment['HTML_API_CC_FULL_SAMPLE_PERCENT'] = '0'; + $environment['HTML_API_CC_MEMORY_LIMIT'] = $this->memory_limit; + $environment['HTML_API_CC_MAX_INPUT_BYTES'] = (string) $this->limits['maxInputBytes']; + $environment['HTML_API_CC_MAX_TOKENS'] = (string) $this->limits['maxTokens']; + $environment['HTML_API_CC_MAX_NODES'] = (string) $this->limits['maxNodes']; + $environment['HTML_API_CC_MAX_DEPTH'] = (string) $this->limits['maxDepth']; + $environment['HTML_API_CC_MAX_TREE_BYTES'] = (string) $this->limits['maxTreeBytes']; + $environment['HTML_API_CC_MAX_KEEP_PER_SIGNATURE'] = '3'; + $environment['HTML_API_CC_REQUIRE_UTF8'] = $this->require_utf8 ? '1' : '0'; + $environment['HTML_API_CC_RETAIN_ALL'] = $this->retain_all ? '1' : '0'; + $environment['HTML_API_CC_WORKER_SCRIPT'] = $this->worker_path; + $environment['HTML_API_FUZZ_LEXBOR_ORACLE'] = $this->oracle_options[ OracleRenderer::KIND_LEXBOR_SOURCE ]['lexbor-oracle-bin']; + $environment['HTML_API_FUZZ_HTML5EVER_ORACLE'] = $this->oracle_options[ OracleRenderer::KIND_HTML5EVER_SOURCE ]['html5ever-oracle-bin']; + $environment['HTML_API_FUZZ_CHROME_ORACLE'] = $this->oracle_options[ OracleRenderer::KIND_CHROME_CDP ]['chrome-oracle-script']; + $environment['HTML_API_FUZZ_CHROME_EXECUTABLE'] = $this->oracle_options[ OracleRenderer::KIND_CHROME_CDP ]['chrome-executable']; + $environment['HTML_API_FUZZ_NODE_BIN'] = $this->oracle_options[ OracleRenderer::KIND_CHROME_CDP ]['node-bin']; + $environment['HTML_API_CC_COORDINATOR_ID'] = $this->coordinator_id; + $environment['HTML_API_CC_BATCH_NAME'] = $this->batch_name; + $environment['HTML_API_CC_CACHE_PATH'] = $cache_path; + $environment['HTML_API_CC_BATCH_MANIFEST_PATH'] = $this->output_dir . '/batch-manifest.json'; + $environment['HTML_API_CC_BATCH_MANIFEST_SHA256'] = $manifest['batchManifestSha256']; + $environment['HTML_API_CC_CORPUS_FINGERPRINT'] = $manifest['corpusFingerprint']; + $environment['HTML_API_CC_DOCUMENT_COUNT'] = (string) $manifest['batch']['documentCount']; + $environment['HTML_API_CC_CRAWL_ID'] = $manifest['batch']['crawlId']; + $environment['HTML_API_CC_PHAR_PATH'] = $this->phar_path; + $environment['HTML_API_CC_PHAR_SHA256'] = $trust['phar']['sha256']; + $environment['HTML_API_CC_INVOCATION_BASE64'] = base64_encode( self::canonical_json( $invocation ) ); + $environment['HTML_API_CC_CALLBACK_SHA256'] = $trust['callback']['sha256']; + $environment['HTML_API_CC_WORKER_SHA256'] = $trust['worker']['sha256']; + $environment['HTML_API_CC_REPOSITORY_COMMIT'] = $trust['repository']['commit']; + $environment['HTML_API_CC_REPOSITORY_DIRTY'] = $trust['repository']['dirty'] ? '1' : '0'; + $environment['HTML_API_CC_REPOSITORY_CODE_SHA256'] = $trust['repository']['codeSha256']; + $environment['HTML_API_CC_REPOSITORY_STATE_SHA256'] = $trust['repository']['stateSha256']; + $environment['HTML_API_CC_RUNTIME_SHA256'] = $trust['runtime']['runtimeSha256']; + $environment['HTML_API_CC_TRUST_BUNDLE_SHA256'] = $trust['trustBundleSha256']; + ksort( $environment ); + return $environment; + } + + private function provenance_from_environment( array $environment ): array { + $invocation = StrictJsonParser::decode( base64_decode( $environment['HTML_API_CC_INVOCATION_BASE64'], true ) ); + return array( + 'id' => $environment['HTML_API_CC_COORDINATOR_ID'], + 'batchName' => $environment['HTML_API_CC_BATCH_NAME'], + 'cachePath' => $environment['HTML_API_CC_CACHE_PATH'], + 'batchManifestPath' => $environment['HTML_API_CC_BATCH_MANIFEST_PATH'], + 'batchManifestSha256' => $environment['HTML_API_CC_BATCH_MANIFEST_SHA256'], + 'corpusFingerprint' => $environment['HTML_API_CC_CORPUS_FINGERPRINT'], + 'documentCount' => (int) $environment['HTML_API_CC_DOCUMENT_COUNT'], + 'crawlId' => $environment['HTML_API_CC_CRAWL_ID'], + 'pharPath' => $environment['HTML_API_CC_PHAR_PATH'], + 'pharSha256' => $environment['HTML_API_CC_PHAR_SHA256'], + 'invocation' => $invocation, + 'callbackSha256' => $environment['HTML_API_CC_CALLBACK_SHA256'], + 'workerSha256' => $environment['HTML_API_CC_WORKER_SHA256'], + 'repositoryCommit' => $environment['HTML_API_CC_REPOSITORY_COMMIT'], + 'repositoryDirty' => '1' === $environment['HTML_API_CC_REPOSITORY_DIRTY'], + 'repositoryCodeSha256' => $environment['HTML_API_CC_REPOSITORY_CODE_SHA256'], + 'repositoryStateSha256' => $environment['HTML_API_CC_REPOSITORY_STATE_SHA256'], + 'runtimeSha256' => $environment['HTML_API_CC_RUNTIME_SHA256'], + 'trustBundleSha256' => $environment['HTML_API_CC_TRUST_BUNDLE_SHA256'], + ); + } + + public static function load_batch_manifest( string $cache_path, array $info ): array { + $db = new \SQLite3( $cache_path, SQLITE3_OPEN_READONLY ); + $db->busyTimeout( 5000 ); + try { + self::assert_table_columns( $db, 'document_batches', array( 'name', 'source_mode', 'crawl_selection', 'crawl_id', 'url_pattern', 'requested_limit', 'state_path', 'cache_path', 'status', 'created_at', 'updated_at', 'completed_at' ) ); + self::assert_table_columns( $db, 'document_batch_documents', array( 'batch_name', 'sequence', 'input_state_key', 'input_url', 'record_id', 'target_uri', 'response_code', 'byte_length', 'checksum', 'range_start', 'range_length', 'cached_at' ) ); + self::assert_table_columns( $db, 'cached_documents', array( 'input_state_key', 'input_url', 'record_id', 'target_uri', 'response_code', 'content_type', 'transport_charset', 'response_headers_json', 'body', 'byte_length', 'range_start', 'range_length', 'fetched_at', 'last_used_at' ) ); + $statement = $db->prepare( 'SELECT * FROM document_batches WHERE name = :name LIMIT 1' ); + $statement->bindValue( ':name', $info['name'], SQLITE3_TEXT ); + $batch_row = $statement->execute()->fetchArray( SQLITE3_ASSOC ); + if ( ! is_array( $batch_row ) ) { + throw new \RuntimeException( 'Named batch disappeared from the document cache.' ); + } + foreach ( array( 'name', 'source_mode', 'crawl_selection', 'crawl_id', 'url_pattern', 'state_path', 'cache_path', 'status', 'created_at', 'updated_at' ) as $field ) { + if ( ! is_string( $batch_row[ $field ] ?? null ) ) { + throw new \RuntimeException( "Batch database metadata {$field} has an invalid type." ); + } + } + if ( ! is_int( $batch_row['requested_limit'] ?? null ) || $batch_row['requested_limit'] < 1 || ! is_string( $batch_row['completed_at'] ?? null ) || '' === $batch_row['completed_at'] ) { + throw new \RuntimeException( 'Batch database limit or completion metadata is invalid.' ); + } + $batch = array( + 'name' => $batch_row['name'], + 'sourceMode' => $batch_row['source_mode'], + 'crawlSelection' => $batch_row['crawl_selection'], + 'crawlId' => $batch_row['crawl_id'], + 'urlPattern' => $batch_row['url_pattern'], + 'requestedLimit' => $batch_row['requested_limit'], + 'statePath' => $batch_row['state_path'], + 'cachePath' => realpath( $batch_row['cache_path'] ) ?: $batch_row['cache_path'], + 'status' => $batch_row['status'], + 'documentCount' => (int) $info['documentCount'], + 'byteCount' => (int) $info['byteCount'], + 'createdAt' => $batch_row['created_at'], + 'updatedAt' => $batch_row['updated_at'], + 'completedAt' => $batch_row['completed_at'], + ); + foreach ( array( 'name', 'sourceMode', 'crawlSelection', 'crawlId', 'urlPattern', 'requestedLimit', 'statePath', 'status', 'documentCount', 'byteCount', 'createdAt', 'updatedAt', 'completedAt' ) as $key ) { + if ( $batch[ $key ] !== $info[ $key ] ) { + throw new \RuntimeException( "Batch database metadata {$key} does not match batch info." ); + } + } + if ( 'ready' !== $batch['status'] || realpath( $cache_path ) !== realpath( $batch['cachePath'] ) || realpath( $info['cachePath'] ) !== realpath( $batch['cachePath'] ) ) { + throw new \RuntimeException( 'Batch database cache path does not match batch info.' ); + } + + $query = $db->prepare( + 'SELECT d.*, c.input_url AS cache_input_url, c.target_uri AS cache_target_uri, + c.response_code AS cache_response_code, c.content_type, c.transport_charset, + c.body, c.byte_length AS cache_byte_length, c.range_start AS cache_range_start, + c.range_length AS cache_range_length + FROM document_batch_documents d + LEFT JOIN cached_documents c + ON c.input_state_key = d.input_state_key AND c.record_id = d.record_id + WHERE d.batch_name = :name ORDER BY d.sequence ASC' + ); + $query->bindValue( ':name', $info['name'], SQLITE3_TEXT ); + $rows = $query->execute(); + $documents = array(); + $seen_identity = array(); + $seen_vector = array(); + $byte_count = 0; + while ( $row = $rows->fetchArray( SQLITE3_ASSOC ) ) { + $sequence = count( $documents ) + 1; + if ( ! is_int( $row['sequence'] ?? null ) || $sequence !== $row['sequence'] ) { + throw new \RuntimeException( 'Batch document sequence is duplicated or non-contiguous.' ); + } + foreach ( array( 'input_state_key', 'input_url', 'record_id', 'target_uri', 'checksum', 'cached_at' ) as $field ) { + if ( ! is_string( $row[ $field ] ) || '' === $row[ $field ] ) { + throw new \RuntimeException( "Batch manifest field {$field} is empty or invalid." ); + } + } + if ( ! is_string( $row['body'] ?? null ) ) { + throw new \RuntimeException( "Batch cache body is missing at sequence {$sequence}." ); + } + foreach ( array( 'response_code', 'cache_response_code' ) as $field ) { + if ( ! is_int( $row[ $field ] ?? null ) || $row[ $field ] < 100 || $row[ $field ] > 599 ) { + throw new \RuntimeException( "Batch HTTP response metadata is invalid at sequence {$sequence}." ); + } + } + foreach ( array( 'byte_length', 'cache_byte_length' ) as $field ) { + if ( ! is_int( $row[ $field ] ?? null ) || $row[ $field ] < 0 ) { + throw new \RuntimeException( "Batch byte-length metadata is invalid at sequence {$sequence}." ); + } + } + foreach ( array( 'range_start', 'cache_range_start' ) as $field ) { + if ( null !== $row[ $field ] && ( ! is_int( $row[ $field ] ) || $row[ $field ] < 0 ) ) { + throw new \RuntimeException( "Batch range-start metadata is invalid at sequence {$sequence}." ); + } + } + foreach ( array( 'range_length', 'cache_range_length' ) as $field ) { + if ( null !== $row[ $field ] && ( ! is_int( $row[ $field ] ) || $row[ $field ] < 1 ) ) { + throw new \RuntimeException( "Batch range-length metadata is invalid at sequence {$sequence}." ); + } + } + if ( ! is_string( $row['content_type'] ?? null ) || 1 !== preg_match( '/^[!#$%&\'*+.^_`|~0-9A-Za-z-]+\/[!#$%&\'*+.^_`|~0-9A-Za-z-]+(?:[ \t]*;[^\r\n]*)?$/D', $row['content_type'] ) ) { + throw new \RuntimeException( "Batch content-type metadata is invalid at sequence {$sequence}." ); + } + if ( null !== $row['transport_charset'] && ( ! is_string( $row['transport_charset'] ) || '' === trim( $row['transport_charset'] ) || preg_match( '/[\x00-\x20\x7f]/', $row['transport_charset'] ) ) ) { + throw new \RuntimeException( "Batch transport-charset metadata is invalid at sequence {$sequence}." ); + } + $length = strlen( $row['body'] ); + $sha256 = hash( 'sha256', $row['body'] ); + if ( 1 !== preg_match( '/^[0-9a-f]{64}$/', $row['checksum'] ) || ! hash_equals( $row['checksum'], $sha256 ) || $length !== $row['byte_length'] || $length !== $row['cache_byte_length'] ) { + throw new \RuntimeException( "Batch cache body identity mismatch at sequence {$sequence}." ); + } + foreach ( array( 'input_url' => 'cache_input_url', 'target_uri' => 'cache_target_uri', 'response_code' => 'cache_response_code', 'range_start' => 'cache_range_start', 'range_length' => 'cache_range_length' ) as $manifest_field => $cache_field ) { + if ( $row[ $manifest_field ] !== $row[ $cache_field ] ) { + throw new \RuntimeException( "Batch/cache metadata differs for {$manifest_field} at sequence {$sequence}." ); + } + } + $range_start = $row['range_start']; + $range_length = $row['range_length']; + if ( ( null === $range_start ) !== ( null === $range_length ) || ( null !== $range_start && ( $range_start < 0 || $range_length < 1 ) ) ) { + throw new \RuntimeException( "Batch range metadata is invalid at sequence {$sequence}." ); + } + $identity_key = $row['input_state_key'] . "\0" . $row['record_id']; + $vector_key = $row['record_id'] . "\0" . $sha256 . "\0" . $length; + if ( isset( $seen_identity[ $identity_key ] ) || isset( $seen_vector[ $vector_key ] ) ) { + throw new \RuntimeException( "Batch contains a duplicate document at sequence {$sequence}." ); + } + $seen_identity[ $identity_key ] = true; + $seen_vector[ $vector_key ] = true; + $documents[] = array( + 'sequence' => $sequence, + 'inputStateKey' => $row['input_state_key'], + 'inputUrl' => $row['input_url'], + 'recordId' => $row['record_id'], + 'targetUri' => $row['target_uri'], + 'responseCode' => $row['response_code'], + 'contentType' => $row['content_type'], + 'transportCharset' => $row['transport_charset'], + 'inputLength' => $length, + 'inputSha256' => $sha256, + 'rangeStart' => $range_start, + 'rangeLength' => $range_length, + 'cachedAt' => $row['cached_at'], + ); + $byte_count += $length; + } + if ( count( $documents ) !== $batch['documentCount'] || $byte_count !== $batch['byteCount'] || 0 === count( $documents ) ) { + throw new \RuntimeException( 'Batch manifest count or byte total is incomplete.' ); + } + $stable = array( 'batch' => $batch, 'documents' => $documents ); + $vector = array_map( static fn ( array $document ): array => array( 'recordId' => $document['recordId'], 'inputSha256' => $document['inputSha256'], 'inputLength' => $document['inputLength'] ), $documents ); + return array( + 'schemaVersion' => 1, + 'kind' => 'html-api-commoncrawl-batch-manifest', + 'batch' => $batch, + 'documents' => $documents, + 'batchManifestSha256' => hash( 'sha256', self::canonical_json( $stable ) ), + 'corpusFingerprint' => hash( 'sha256', self::canonical_json( $vector ) ), + ); + } finally { + $db->close(); + } + } + + public static function verify_run_output( string $run_dir, array $manifest, string $run_id, string $kind, string $identity_sha256, array $provenance, ?array $expected_environment = null ): array { + $summary_path = $run_dir . '/commoncrawl-summary.ndjson'; + $text = @file_get_contents( $summary_path ); + if ( false === $text || '' === $text || "\n" !== substr( $text, -1 ) ) { + throw new \RuntimeException( "{$kind} summary is missing or not newline-terminated." ); + } + $lines = explode( "\n", substr( $text, 0, -1 ) ); + if ( count( $lines ) !== count( $manifest['documents'] ) ) { + throw new \RuntimeException( "{$kind} summary is partial or contains extra documents." ); + } + $vector = array(); + $statuses = array(); + $config_hash = null; + $seen = array(); + foreach ( $lines as $index => $line ) { + if ( '' === $line ) { + throw new \RuntimeException( "{$kind} summary contains a blank record." ); + } + $record = StrictJsonParser::decode( $line ); + if ( ! is_array( $record ) ) { + throw new \RuntimeException( "{$kind} summary record is not an object." ); + } + $expected = $manifest['documents'][ $index ]; + $common = is_array( $record['commonCrawl'] ?? null ) ? $record['commonCrawl'] : array(); + foreach ( array( 'recordId', 'inputStateKey', 'targetUri', 'rangeStart', 'rangeLength' ) as $field ) { + if ( ( $common[ $field ] ?? null ) !== $expected[ $field ] ) { + throw new \RuntimeException( "{$kind} summary metadata/order differs at sequence " . ( $index + 1 ) . "." ); + } + } + if ( ( $record['inputSha256'] ?? null ) !== $expected['inputSha256'] || ( $record['inputLength'] ?? null ) !== $expected['inputLength'] || ( $common['inputSha256'] ?? null ) !== $expected['inputSha256'] || ( $common['byteLength'] ?? null ) !== $expected['inputLength'] ) { + throw new \RuntimeException( "{$kind} summary input identity differs at sequence " . ( $index + 1 ) . "." ); + } + $key = $expected['recordId'] . "\0" . $expected['inputSha256'] . "\0" . $expected['inputLength']; + if ( isset( $seen[ $key ] ) ) { + throw new \RuntimeException( "{$kind} summary contains a duplicate document." ); + } + $seen[ $key ] = true; + if ( $run_id !== ( $record['runId'] ?? null ) || self::canonical_json( $provenance ) !== self::canonical_json( $record['coordinator'] ?? null ) ) { + throw new \RuntimeException( "{$kind} summary run/coordinator provenance differs." ); + } + if ( ( $record['repo']['commit'] ?? null ) !== $provenance['repositoryCommit'] || ( $record['repo']['dirty'] ?? null ) !== $provenance['repositoryDirty'] ) { + throw new \RuntimeException( "{$kind} summary repository provenance differs." ); + } + if ( null === $config_hash ) { + $config_hash = $record['configHash'] ?? null; + } elseif ( $config_hash !== ( $record['configHash'] ?? null ) ) { + throw new \RuntimeException( "{$kind} summary mixes configurations." ); + } + if ( ! is_array( $record['oracle'] ?? null ) || ! hash_equals( $identity_sha256, OracleRenderer::identity_sha256( $record['oracle'] ) ) ) { + throw new \RuntimeException( "{$kind} summary oracle identity differs." ); + } + if ( in_array( $record['failureClass'] ?? null, array( 'commoncrawl-callback-error', 'worker-input-identity-drift', 'oracle-identity-drift' ), true ) ) { + throw new \RuntimeException( "{$kind} run contains coordinator-critical identity/callback failure." ); + } + $status = (string) ( $record['status'] ?? 'unknown' ); + $statuses[ $status ] = 1 + (int) ( $statuses[ $status ] ?? 0 ); + $vector[] = array( 'recordId' => $expected['recordId'], 'inputSha256' => $expected['inputSha256'], 'inputLength' => $expected['inputLength'] ); + } + $configuration = self::strict_json_file( $run_dir . '/configuration.json' ); + self::assert_exact_keys( $configuration, array( 'kind', 'runId', 'repo', 'phpVersion', 'oracle', 'oracleIdentitySha256', 'limits', 'maxInputBytes', 'maxKeepPerSignature', 'processTimeoutMs', 'memoryLimit', 'checks', 'fullSamplePercent', 'requireUtf8', 'retainAll', 'workerScript', 'inputSemantics', 'ccAnalyzer', 'configHash', 'createdAt' ), "{$kind} configuration" ); + $claimed_config_hash = $configuration['configHash'] ?? null; + $hash_material = $configuration; + unset( $hash_material['configHash'], $hash_material['createdAt'] ); + $computed_config_hash = hash( 'sha256', self::canonical_json( $hash_material ) ); + if ( ! is_string( $claimed_config_hash ) || 1 !== preg_match( '/^[0-9a-f]{64}$/', $claimed_config_hash ) || ! hash_equals( $computed_config_hash, $claimed_config_hash ) || $run_id !== ( $configuration['runId'] ?? null ) || $config_hash !== $claimed_config_hash || self::canonical_json( $provenance ) !== self::canonical_json( $configuration['ccAnalyzer']['coordinator'] ?? null ) ) { + throw new \RuntimeException( "{$kind} configuration does not match its summaries." ); + } + if ( 'html-api-commoncrawl-configuration' !== $configuration['kind'] || ! is_string( $configuration['createdAt'] ) || '' === $configuration['createdAt'] || 'raw cc-analyzer response body; no charset transcoding' !== $configuration['inputSemantics'] || ! is_array( $configuration['repo'] ) || ! is_array( $configuration['ccAnalyzer'] ) ) { + throw new \RuntimeException( "{$kind} configuration identity fields are invalid." ); + } + if ( ( $configuration['repo']['commit'] ?? null ) !== $provenance['repositoryCommit'] || ( $configuration['repo']['dirty'] ?? null ) !== $provenance['repositoryDirty'] ) { + throw new \RuntimeException( "{$kind} configuration repository provenance differs." ); + } + if ( ! is_array( $configuration['oracle'] ?? null ) || ! hash_equals( $identity_sha256, OracleRenderer::identity_sha256( $configuration['oracle'] ) ) || ! is_string( $configuration['oracleIdentitySha256'] ) || ! hash_equals( $identity_sha256, $configuration['oracleIdentitySha256'] ) ) { + throw new \RuntimeException( "{$kind} configuration oracle identity differs." ); + } + if ( null !== $expected_environment ) { + ksort( $expected_environment ); + $environment_evidence = self::strict_json_file( $run_dir . '/replacement-environment.json' ); + self::assert_exact_keys( $environment_evidence, array( 'kind', 'environment', 'environmentSha256' ), "{$kind} replacement environment" ); + $environment_sha256 = hash( 'sha256', self::canonical_json( $expected_environment ) ); + if ( 'html-api-commoncrawl-replacement-environment' !== $environment_evidence['kind'] || self::canonical_json( $expected_environment ) !== self::canonical_json( $environment_evidence['environment'] ?? null ) || ! is_string( $environment_evidence['environmentSha256'] ) || ! hash_equals( $environment_sha256, $environment_evidence['environmentSha256'] ) ) { + throw new \RuntimeException( "{$kind} replacement environment evidence differs." ); + } + $expected_configuration = array( + 'limits' => array( + 'maxTokens' => (int) $expected_environment['HTML_API_CC_MAX_TOKENS'], + 'maxNodes' => (int) $expected_environment['HTML_API_CC_MAX_NODES'], + 'maxDepth' => (int) $expected_environment['HTML_API_CC_MAX_DEPTH'], + 'maxTreeBytes' => (int) $expected_environment['HTML_API_CC_MAX_TREE_BYTES'], + ), + 'maxInputBytes' => (int) $expected_environment['HTML_API_CC_MAX_INPUT_BYTES'], + 'maxKeepPerSignature' => (int) $expected_environment['HTML_API_CC_MAX_KEEP_PER_SIGNATURE'], + 'processTimeoutMs' => (int) $expected_environment['HTML_API_CC_PROCESS_TIMEOUT_MS'], + 'memoryLimit' => $expected_environment['HTML_API_CC_MEMORY_LIMIT'], + 'checks' => $expected_environment['HTML_API_CC_CHECKS'], + 'fullSamplePercent' => (int) $expected_environment['HTML_API_CC_FULL_SAMPLE_PERCENT'], + 'requireUtf8' => '1' === $expected_environment['HTML_API_CC_REQUIRE_UTF8'], + 'retainAll' => '1' === $expected_environment['HTML_API_CC_RETAIN_ALL'], + 'workerScript' => realpath( $expected_environment['HTML_API_CC_WORKER_SCRIPT'] ), + ); + foreach ( $expected_configuration as $field => $expected ) { + if ( $expected !== ( $configuration[ $field ] ?? null ) ) { + throw new \RuntimeException( "{$kind} configuration field {$field} differs from the replacement environment." ); + } + } + if ( PHP_VERSION !== ( $configuration['phpVersion'] ?? null ) ) { + throw new \RuntimeException( "{$kind} configuration PHP version differs." ); + } + } + return array( + 'count' => count( $vector ), + 'configHash' => $config_hash, + 'statuses' => $statuses, + 'vector' => $vector, + 'corpusFingerprint' => hash( 'sha256', self::canonical_json( $vector ) ), + ); + } + + private static function repository_code_identity( array $git ): array { + $status = base64_decode( (string) $git['statusBase64'], true ); + if ( false === $status ) { + throw new \RuntimeException( 'Git status probe is not valid base64.' ); + } + $tracked = array(); + $required = array_fill_keys( array_merge( self::REQUIRED_TOOL_DEPENDENCIES, self::WORDPRESS_DEPENDENCIES ), false ); + foreach ( $git['tracked'] as $entry ) { + if ( ! is_array( $entry ) || ! is_string( $entry['path'] ?? null ) || ! is_string( $entry['mode'] ?? null ) ) { + throw new \RuntimeException( 'Git tracked-file probe is malformed.' ); + } + $path = $entry['path']; + if ( 0 !== strpos( $path, 'tools/html-api-fuzz/' ) && ! array_key_exists( $path, $required ) ) { + throw new \RuntimeException( 'Git probe returned an unexpected executed-code path.' ); + } + $absolute = repo_root() . '/' . $path; + if ( is_link( $absolute ) || ! is_file( $absolute ) ) { + throw new \RuntimeException( "Executed repository code is missing or symlinked: {$path}" ); + } + if ( isset( $required[ $path ] ) ) { + $required[ $path ] = true; + } + $identity = self::file_identity( $absolute ); + $tracked[] = array( + 'path' => $path, + 'mode' => $entry['mode'], + 'actualMode' => $identity['mode'], + 'bytes' => $identity['bytes'], + 'sha256' => $identity['sha256'], + ); + } + foreach ( $required as $path => $present ) { + if ( ! $present ) { + throw new \RuntimeException( "Required executed-code dependency is not tracked: {$path}" ); + } + } + usort( $tracked, static fn ( array $a, array $b ): int => strcmp( $a['path'], $b['path'] ) ); + return array( + 'commit' => $git['commit'], + 'branch' => is_string( $git['branch'] ) ? $git['branch'] : null, + 'dirty' => '' !== $status, + 'stateSha256' => hash( 'sha256', $status ), + 'codeSha256' => hash( 'sha256', self::canonical_json( $tracked ) ), + 'trackedFiles' => $tracked, + ); + } + + private static function validate_batch_info( $value, string $batch_name ): array { + $keys = array( 'name', 'sourceMode', 'crawlSelection', 'crawlId', 'urlPattern', 'requestedLimit', 'statePath', 'cachePath', 'status', 'documentCount', 'byteCount', 'runCount', 'createdAt', 'updatedAt', 'completedAt' ); + self::assert_exact_keys( $value, $keys, 'batch info' ); + foreach ( array( 'name', 'sourceMode', 'crawlSelection', 'crawlId', 'statePath', 'cachePath', 'status', 'createdAt', 'updatedAt' ) as $key ) { + if ( ! is_string( $value[ $key ] ) || '' === $value[ $key ] ) { + throw new \RuntimeException( "Batch info {$key} is invalid." ); + } + } + if ( ! is_string( $value['urlPattern'] ) || ! is_string( $value['completedAt'] ) || '' === $value['completedAt'] ) { + throw new \RuntimeException( 'Batch info URL pattern or completion time is invalid.' ); + } + foreach ( array( 'requestedLimit', 'documentCount' ) as $key ) { + if ( ! is_int( $value[ $key ] ) || $value[ $key ] < 1 ) { + throw new \RuntimeException( "Batch info {$key} is invalid." ); + } + } + foreach ( array( 'byteCount', 'runCount' ) as $key ) { + if ( ! is_int( $value[ $key ] ) || $value[ $key ] < 0 ) { + throw new \RuntimeException( "Batch info {$key} is invalid." ); + } + } + if ( $batch_name !== $value['name'] || 'ready' !== $value['status'] ) { + throw new \RuntimeException( 'Batch is not the requested ready batch.' ); + } + return $value; + } + + private static function validate_batch_verification( $value, string $batch_name, int $count ): array { + self::assert_exact_keys( $value, array( 'batchName', 'documentCount', 'missingCount', 'missingDocuments' ), 'batch verification' ); + if ( ! is_string( $value['batchName'] ) || ! is_int( $value['documentCount'] ) || ! is_int( $value['missingCount'] ) || ! is_array( $value['missingDocuments'] ) || $batch_name !== $value['batchName'] || $count !== $value['documentCount'] || 0 !== $value['missingCount'] || array() !== $value['missingDocuments'] ) { + throw new \RuntimeException( 'Batch verification reports missing or inconsistent cached documents.' ); + } + return $value; + } + + private static function validate_batch_run_result( $value, array $expected_batch, string $callback_path, string $callback_sha256 ): array { + self::assert_exact_keys( $value, array( 'batch', 'run', 'documentsAnalyzed' ), 'batch run result' ); + $batch = self::validate_batch_info( $value['batch'], $expected_batch['name'] ); + foreach ( $expected_batch as $key => $expected ) { + if ( ! array_key_exists( $key, $batch ) ) { + throw new \RuntimeException( "Batch run result omitted batch metadata {$key}." ); + } + if ( 'cachePath' === $key ) { + if ( false === realpath( $expected ) || realpath( $expected ) !== realpath( $batch[ $key ] ) ) { + throw new \RuntimeException( 'Batch run result changed batch metadata cachePath.' ); + } + continue; + } + if ( $expected !== $batch[ $key ] ) { + throw new \RuntimeException( "Batch run result changed batch metadata {$key}." ); + } + } + self::assert_exact_keys( $value['run'], array( 'batchName', 'runId', 'analysisScript', 'analysisScriptHash', 'documentsAnalyzed', 'startedAt', 'completedAt' ), 'batch run record' ); + $run = $value['run']; + $count = $expected_batch['documentCount']; + if ( + ! is_int( $value['documentsAnalyzed'] ) || + ! is_string( $run['batchName'] ) || + ! is_int( $run['runId'] ) || $run['runId'] < 1 || + ! is_string( $run['analysisScript'] ) || + ! is_string( $run['analysisScriptHash'] ) || 1 !== preg_match( '/^[0-9a-f]{64}$/', $run['analysisScriptHash'] ) || + ! is_int( $run['documentsAnalyzed'] ) || + ! is_string( $run['startedAt'] ) || '' === $run['startedAt'] || + ! is_string( $run['completedAt'] ) || '' === $run['completedAt'] || + $count !== $value['documentsAnalyzed'] || + $expected_batch['name'] !== $run['batchName'] || + $callback_path !== $run['analysisScript'] || + $count !== $run['documentsAnalyzed'] || + ! hash_equals( $callback_sha256, $run['analysisScriptHash'] ) + ) { + throw new \RuntimeException( 'Batch run result is partial or inconsistent.' ); + } + return $value; + } + + private static function assert_table_columns( \SQLite3 $db, string $table, array $expected ): void { + $result = $db->query( "PRAGMA table_info({$table})" ); + $columns = array(); + while ( $row = $result->fetchArray( SQLITE3_ASSOC ) ) { + $columns[] = $row['name']; + } + if ( $columns !== $expected ) { + throw new \RuntimeException( "Unsupported cc-analyzer SQLite schema for {$table}." ); + } + } + + private static function assert_process_success( array $process, string $label ): void { + if ( 0 !== ( $process['code'] ?? null ) || true === ( $process['timedOut'] ?? false ) || true === ( $process['stdoutTruncated'] ?? false ) || true === ( $process['stderrTruncated'] ?? false ) || true === ( $process['processGroupCleanupFailed'] ?? false ) ) { + throw new \RuntimeException( "{$label} failed, timed out, truncated output, or leaked a process group." ); + } + } + + private static function decode_process_json( array $process, string $label ) { + self::assert_process_success( $process, $label ); + $stdout = (string) ( $process['stdout'] ?? '' ); + if ( '' === $stdout || "\n" !== substr( $stdout, -1 ) ) { + throw new \RuntimeException( "{$label} did not emit one newline-terminated JSON value." ); + } + return StrictJsonParser::decode( substr( $stdout, 0, -1 ) ); + } + + private static function assert_environment_echo( array &$probe, array $environment, string $label ): void { + $actual = $probe['environment'] ?? null; + $expected = $environment; + ksort( $expected ); + if ( ! is_array( $actual ) || self::canonical_json( $expected ) !== self::canonical_json( $actual ) ) { + throw new \RuntimeException( "{$label} did not receive the exact sanitized replacement environment." ); + } + unset( $probe['environment'] ); + } + + private static function assert_same_trust( array $expected, array $actual, string $when ): void { + if ( self::canonical_json( $expected ) !== self::canonical_json( $actual ) ) { + throw new \RuntimeException( "Global trust bundle changed {$when}." ); + } + } + + private static function assert_same_manifest( array $expected, array $actual, string $when ): void { + if ( self::canonical_json( $expected ) !== self::canonical_json( $actual ) ) { + throw new \RuntimeException( "Cached batch corpus changed {$when}." ); + } + } + + private static function assert_exact_keys( $value, array $expected, string $label ): void { + if ( ! is_array( $value ) ) { + throw new \RuntimeException( "{$label} must be a JSON object." ); + } + $actual = array_keys( $value ); + sort( $actual ); + sort( $expected ); + if ( $actual !== $expected ) { + throw new \RuntimeException( "{$label} has an unexpected JSON schema." ); + } + } + + private static function compact_process( array $process ): array { + return array( + 'command' => $process['command'] ?? null, + 'code' => $process['code'] ?? null, + 'durationMs' => $process['durationMs'] ?? null, + 'stdoutLogPath' => $process['stdoutLogPath'] ?? null, + 'stderrLogPath' => $process['stderrLogPath'] ?? null, + 'processGroupIsolated' => $process['processGroupIsolated'] ?? false, + 'processGroupCleanupFailed' => $process['processGroupCleanupFailed'] ?? false, + ); + } + + private static function strict_json_file( string $path ) { + $text = @file_get_contents( $path ); + if ( false === $text || '' === $text || "\n" !== substr( $text, -1 ) ) { + throw new \RuntimeException( "JSON file is missing or incomplete: {$path}" ); + } + return StrictJsonParser::decode( substr( $text, 0, -1 ) ); + } + + private static function file_identity( string $path ): array { + clearstatcache( true, $path ); + $resolved = realpath( $path ); + if ( false === $resolved || ! is_file( $resolved ) || is_link( $path ) ) { + throw new \RuntimeException( "Trust input is missing, non-regular, or symlinked: {$path}" ); + } + $handle = @fopen( $resolved, 'rb' ); + if ( false === $handle ) { + throw new \RuntimeException( "Trust input is unreadable: {$path}" ); + } + $before = fstat( $handle ); + $context = hash_init( 'sha256' ); + $bytes = hash_update_stream( $context, $handle ); + $sha256 = hash_final( $context ); + $after = fstat( $handle ); + $closed = fclose( $handle ); + clearstatcache( true, $resolved ); + $later = @lstat( $resolved ); + $stable_fields = array( 'dev', 'ino', 'mode', 'size', 'mtime', 'ctime' ); + if ( ! is_array( $before ) || ! is_array( $after ) || ! is_array( $later ) || false === $bytes || ! $closed || $bytes !== $before['size'] || 1 !== preg_match( '/^[0-9a-f]{64}$/', $sha256 ) ) { + throw new \RuntimeException( "Could not read a complete trust identity: {$path}" ); + } + foreach ( $stable_fields as $field ) { + if ( ! array_key_exists( $field, $before ) || $before[ $field ] !== $after[ $field ] || $before[ $field ] !== $later[ $field ] ) { + throw new \RuntimeException( "Trust input changed while hashing: {$path}" ); + } + } + return array( + 'path' => $resolved, + 'mode' => sprintf( '%06o', $before['mode'] & 0177777 ), + 'bytes' => $bytes, + 'sha256' => $sha256, + ); + } + + private static function seal_directory( string $directory, array $excluded ): array { + $rows = array(); + $iterator = new \RecursiveIteratorIterator( new \RecursiveDirectoryIterator( $directory, \FilesystemIterator::SKIP_DOTS ) ); + foreach ( $iterator as $file ) { + $path = $file->getPathname(); + $relative = substr( $path, strlen( $directory ) + 1 ); + if ( in_array( $relative, $excluded, true ) ) { + continue; + } + if ( $file->isLink() || ! $file->isFile() ) { + throw new \RuntimeException( "Cannot seal non-regular evidence: {$relative}" ); + } + $identity = self::file_identity( $path ); + $rows[] = array( 'path' => $relative, 'mode' => $identity['mode'], 'bytes' => $identity['bytes'], 'sha256' => $identity['sha256'] ); + } + usort( $rows, static fn ( array $a, array $b ): int => strcmp( $a['path'], $b['path'] ) ); + return array( + 'kind' => 'html-api-commoncrawl-evidence-seal', + 'files' => $rows, + 'filesSha256' => hash( 'sha256', self::canonical_json( $rows ) ), + ); + } + + private static function canonical_json( $value ): string { + $canonical = self::canonicalize( $value ); + $json = json_encode( $canonical, JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_SUBSTITUTE ); + if ( false === $json ) { + throw new \RuntimeException( 'Could not encode canonical coordinator JSON.' ); + } + return $json; + } + + private static function canonicalize( $value ) { + if ( ! is_array( $value ) ) { + return $value; + } + if ( array_keys( $value ) !== range( 0, count( $value ) - 1 ) ) { + ksort( $value ); + } + foreach ( $value as $key => $item ) { + $value[ $key ] = self::canonicalize( $item ); + } + return $value; + } + + private static function existing_file_option( array $options, string $name ): string { + return self::resolved_file( self::required_string_option( $options, $name ), "--{$name}" ); + } + + private static function existing_directory_option( array $options, string $name ): string { + $value = self::required_string_option( $options, $name ); + $resolved = realpath( $value ); + if ( false === $resolved || ! is_dir( $resolved ) || is_link( $value ) ) { + throw new \InvalidArgumentException( "--{$name} must resolve to a non-symlink directory." ); + } + return $resolved; + } + + private static function new_path_option( array $options, string $name ): string { + $value = self::required_string_option( $options, $name ); + if ( file_exists( $value ) || is_link( $value ) ) { + throw new \InvalidArgumentException( "--{$name} must not already exist." ); + } + $parent = realpath( dirname( $value ) ); + if ( false === $parent || ! is_dir( $parent ) ) { + throw new \InvalidArgumentException( "--{$name} parent directory does not exist." ); + } + return $parent . DIRECTORY_SEPARATOR . basename( $value ); + } + + private static function required_string_option( array $options, string $name ): string { + if ( ! array_key_exists( $name, $options ) || true === $options[ $name ] || ! is_string( $options[ $name ] ) || '' === $options[ $name ] ) { + throw new \InvalidArgumentException( "Expected --{$name} with a non-empty value." ); + } + return $options[ $name ]; + } + + private static function required_positive_option( array $options, string $name ): int { + if ( ! array_key_exists( $name, $options ) || true === $options[ $name ] ) { + throw new \InvalidArgumentException( "Expected --{$name} with a positive integer value." ); + } + return self::positive_option( $options, $name, 0 ); + } + + private static function positive_option( array $options, string $name, int $default, int $minimum = 1 ): int { + $value = option_int( $options, $name, $default ); + if ( $value < $minimum ) { + throw new \InvalidArgumentException( "--{$name} must be at least {$minimum}." ); + } + return $value; + } + + private static function strict_bool_option( array $options, string $name, bool $default ): bool { + if ( ! array_key_exists( $name, $options ) ) { + return $default; + } + if ( true === $options[ $name ] ) { + return true; + } + $value = filter_var( $options[ $name ], FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE ); + if ( null === $value ) { + throw new \InvalidArgumentException( "--{$name} must be a boolean value." ); + } + return $value; + } + + private static function validate_options( array $options ): void { + $value_options = array( + 'cc-analyzer', 'workspace', 'batch', 'output-dir', 'coordinator-id', + 'batch-timeout-ms', 'process-timeout-ms', 'oracle-timeout-ms', 'chrome-startup-timeout-ms', + 'checks', 'memory-limit', 'max-input-bytes', 'max-tokens', 'max-nodes', 'max-depth', 'max-tree-bytes', + 'lexbor-oracle-bin', 'html5ever-oracle-bin', 'chrome-oracle-script', 'chrome-executable', 'node-bin', + ); + $flag_options = array( 'retain-all', 'require-utf8' ); + if ( ! is_array( $options['_'] ?? null ) || array() !== $options['_'] ) { + throw new \InvalidArgumentException( 'Coordinator does not accept positional arguments.' ); + } + foreach ( $options as $name => $value ) { + if ( '_' === $name ) { + continue; + } + if ( ! in_array( $name, $value_options, true ) && ! in_array( $name, $flag_options, true ) ) { + throw new \InvalidArgumentException( "Unknown coordinator option --{$name}." ); + } + if ( in_array( $name, $value_options, true ) && true === $value ) { + throw new \InvalidArgumentException( "Coordinator option --{$name} requires a value." ); + } + } + } + + private static function resolved_file( ?string $path, string $label ): string { + if ( null === $path || '' === $path ) { + throw new \InvalidArgumentException( "{$label} path is required." ); + } + $resolved = realpath( $path ); + if ( false === $resolved || ! is_file( $resolved ) || is_link( $path ) ) { + throw new \InvalidArgumentException( "{$label} must resolve to a non-symlink regular file." ); + } + return $resolved; + } + + private static function resolved_executable( ?string $path, string $label ): string { + $resolved = self::resolved_file( $path, $label ); + if ( ! is_executable( $resolved ) ) { + throw new \InvalidArgumentException( "{$label} is not executable." ); + } + return $resolved; + } +} diff --git a/tools/html-api-fuzz/lib/CommonCrawlRunner.php b/tools/html-api-fuzz/lib/CommonCrawlRunner.php new file mode 100644 index 0000000000000..d60918f0046f3 --- /dev/null +++ b/tools/html-api-fuzz/lib/CommonCrawlRunner.php @@ -0,0 +1,960 @@ +output_dir = $output_dir; + $this->oracle = $oracle; + $this->limits = $limits; + $this->max_input_bytes = $max_input_bytes; + $this->max_keep_per_signature = $max_keep_per_signature; + $this->process_timeout_ms = $process_timeout_ms; + $this->require_utf8 = $require_utf8; + $this->retain_all = $retain_all; + $this->checks = $checks; + $this->full_sample_percent = $full_sample_percent; + $this->memory_limit = $memory_limit; + $this->worker_script = $worker_script; + $this->git_metadata = git_metadata( 1000, null, false ); + $this->run_id = $run_id; + $this->coordinator_provenance = $coordinator_provenance; + + ensure_dir( $this->output_dir ); + $this->initialize_configuration(); + } + + public static function from_environment(): self { + $output_dir = getenv( 'CC_ANALYZER_OUTPUT_DIR' ); + if ( ! is_string( $output_dir ) || '' === $output_dir ) { + $run_id = self::environment_string( 'HTML_API_CC_RUN_ID', 'run-' . timestamp() . '-' . getmypid() ); + $output_dir = repo_root() . '/artifacts/html-api-commoncrawl/' . $run_id; + } else { + $run_id = self::environment_string( 'HTML_API_CC_RUN_ID', 'cc-' . substr( hash( 'sha256', $output_dir ), 0, 12 ) ); + } + if ( ! preg_match( '/^[A-Za-z0-9][A-Za-z0-9._-]*$/', $run_id ) ) { + throw new \InvalidArgumentException( 'HTML_API_CC_RUN_ID contains unsafe path characters.' ); + } + + $oracle_kind = self::environment_string( 'HTML_API_CC_ORACLE', OracleRenderer::KIND_LEXBOR_SOURCE ); + $oracle_options = array( + 'dom-oracle' => $oracle_kind, + 'oracle-timeout-ms' => (string) self::environment_int( 'HTML_API_CC_ORACLE_TIMEOUT_MS', self::DEFAULT_ORACLE_TIMEOUT_MS, 1 ), + ); + $oracle_bin = getenv( 'HTML_API_FUZZ_LEXBOR_ORACLE' ); + if ( is_string( $oracle_bin ) && '' !== $oracle_bin ) { + $oracle_options['lexbor-oracle-bin'] = $oracle_bin; + } + $html5ever_oracle_bin = getenv( 'HTML_API_FUZZ_HTML5EVER_ORACLE' ); + if ( is_string( $html5ever_oracle_bin ) && '' !== $html5ever_oracle_bin ) { + $oracle_options['html5ever-oracle-bin'] = $html5ever_oracle_bin; + } + $common_chrome_startup_timeout = getenv( 'HTML_API_CC_CHROME_STARTUP_TIMEOUT_MS' ); + if ( is_string( $common_chrome_startup_timeout ) && '' !== $common_chrome_startup_timeout ) { + $oracle_options['chrome-startup-timeout-ms'] = (string) self::environment_int( 'HTML_API_CC_CHROME_STARTUP_TIMEOUT_MS', ChromeOracleRenderer::DEFAULT_STARTUP_TIMEOUT_MS, 1 ); + } + + $coordinator_provenance = self::coordinator_provenance_from_environment(); + $oracle = OracleRenderer::from_options( $oracle_options ); + return OracleRenderer::with_explicit_close( + $oracle, + static function ( OracleRenderer $oracle ) use ( $oracle_kind, $output_dir, $run_id, $coordinator_provenance ): self { + $metadata = $oracle->metadata(); + if ( true !== ( $metadata['available'] ?? false ) ) { + throw new \RuntimeException( + "Selected {$oracle_kind} oracle is unavailable: " . (string) ( $metadata['error'] ?? 'unknown identity error' ) + ); + } + $expected_commit = getenv( 'HTML_API_CC_EXPECT_LEXBOR_COMMIT' ); + if ( + is_string( $expected_commit ) && '' !== $expected_commit && + ( + OracleRenderer::KIND_LEXBOR_SOURCE !== $oracle_kind || + $expected_commit !== ( $metadata['identity']['lexborCommit'] ?? null ) + ) + ) { + throw new \RuntimeException( 'Lexbor oracle commit does not match HTML_API_CC_EXPECT_LEXBOR_COMMIT.' ); + } + $expected_identity_sha256 = getenv( 'HTML_API_CC_EXPECT_ORACLE_IDENTITY_SHA256' ); + if ( is_string( $expected_identity_sha256 ) && '' !== $expected_identity_sha256 && 1 !== preg_match( '/^[0-9a-fA-F]{64}$/', $expected_identity_sha256 ) ) { + throw new \InvalidArgumentException( 'HTML_API_CC_EXPECT_ORACLE_IDENTITY_SHA256 must be a SHA-256 hex digest.' ); + } + if ( + is_string( $expected_identity_sha256 ) && '' !== $expected_identity_sha256 && + ! hash_equals( strtolower( $expected_identity_sha256 ), OracleRenderer::identity_sha256( $metadata ) ) + ) { + throw new \RuntimeException( 'Oracle identity does not match HTML_API_CC_EXPECT_ORACLE_IDENTITY_SHA256.' ); + } + + $checks = self::environment_string( 'HTML_API_CC_CHECKS', 'sampled' ); + if ( ! in_array( $checks, array( 'baseline', 'full', 'sampled' ), true ) ) { + throw new \InvalidArgumentException( 'HTML_API_CC_CHECKS must be baseline, full, or sampled.' ); + } + $full_sample_percent = self::environment_int( 'HTML_API_CC_FULL_SAMPLE_PERCENT', 1, 0 ); + if ( $full_sample_percent > 100 ) { + throw new \InvalidArgumentException( 'HTML_API_CC_FULL_SAMPLE_PERCENT must be at most 100.' ); + } + $memory_limit = self::environment_string( 'HTML_API_CC_MEMORY_LIMIT', '256M' ); + if ( ! preg_match( '/^[1-9][0-9]*[KMG]?$/i', $memory_limit ) ) { + throw new \InvalidArgumentException( 'HTML_API_CC_MEMORY_LIMIT must be a positive PHP memory limit such as 256M.' ); + } + $worker_script = self::environment_string( 'HTML_API_CC_WORKER_SCRIPT', repo_root() . '/tools/html-api-fuzz/worker.php' ); + if ( ! is_file( $worker_script ) ) { + throw new \RuntimeException( 'HTML_API_CC_WORKER_SCRIPT does not exist.' ); + } + if ( ! function_exists( 'posix_kill' ) || ! function_exists( 'posix_setsid' ) || ! function_exists( 'pcntl_exec' ) ) { + throw new \RuntimeException( 'Common Crawl worker isolation requires the POSIX and PCNTL PHP extensions.' ); + } + + $process_timeout_environment = getenv( 'HTML_API_CC_PROCESS_TIMEOUT_MS' ); + $process_timeout_ms = is_string( $process_timeout_environment ) && '' !== $process_timeout_environment + ? self::environment_int( 'HTML_API_CC_PROCESS_TIMEOUT_MS', self::DEFAULT_PROCESS_TIMEOUT_MS, 1 ) + : $oracle->recommended_process_timeout_ms( $checks, self::DEFAULT_PROCESS_TIMEOUT_MS ); + + return new self( + $output_dir, + $oracle, + array( + 'maxTokens' => self::environment_int( 'HTML_API_CC_MAX_TOKENS', self::DEFAULT_MAX_TOKENS, 1 ), + 'maxNodes' => self::environment_int( 'HTML_API_CC_MAX_NODES', self::DEFAULT_MAX_NODES, 1 ), + 'maxDepth' => self::environment_int( 'HTML_API_CC_MAX_DEPTH', self::DEFAULT_MAX_DEPTH, 1 ), + 'maxTreeBytes' => self::environment_int( 'HTML_API_CC_MAX_TREE_BYTES', self::DEFAULT_MAX_TREE_BYTES, 1 ), + ), + self::environment_int( 'HTML_API_CC_MAX_INPUT_BYTES', self::DEFAULT_MAX_INPUT_BYTES, 0 ), + self::environment_int( 'HTML_API_CC_MAX_KEEP_PER_SIGNATURE', self::DEFAULT_KEEP_PER_SIGNATURE, 1 ), + $process_timeout_ms, + self::environment_bool( 'HTML_API_CC_REQUIRE_UTF8', true ), + self::environment_bool( 'HTML_API_CC_RETAIN_ALL', false ), + $checks, + $full_sample_percent, + $memory_limit, + $worker_script, + $run_id, + $coordinator_provenance + ); + } + ); + } + + /** Analyze one CcAnalyzer\Analysis\HtmlAnalysisInput-compatible object. */ + public function analyze_document( object $document ): array { + $started_at = hrtime( true ); + $metadata = $this->document_metadata( $document ); + $body = $document->body ?? null; + + if ( ! is_string( $body ) ) { + return $this->record_callback_error( $metadata, new \InvalidArgumentException( 'cc-analyzer document body must be a string.' ), $started_at ); + } + + $metadata['byteLength'] = strlen( $body ); + $metadata['inputSha1'] = sha1( $body ); + $metadata['inputSha256'] = hash( 'sha256', $body ); + $metadata['utf8Valid'] = 1 === preg_match( '//u', $body ); + if ( $this->max_input_bytes > 0 && strlen( $body ) > $this->max_input_bytes ) { + return $this->record_skip( $metadata, 'skipped-input-too-large', $started_at ); + } + if ( $this->require_utf8 && ! $metadata['utf8Valid'] ) { + return $this->record_skip( $metadata, 'skipped-invalid-utf8', $started_at ); + } + + $seed = self::seed_for_record( $metadata['recordId'], $metadata['inputSha1'] ); + $checks = $this->checks_for_seed( $seed ); + $staging_dir = $this->create_staging_dir( $metadata ); + try { + $this->persist_initial_input( $staging_dir, $body, $metadata, $seed, $checks ); + $process = run_php_process( $this->worker_args( $staging_dir, $seed, $checks ), repo_root(), $this->process_timeout_ms, $staging_dir . '/worker.log', 1048576, true ); + $result = $this->load_or_synthesize_result( $staging_dir, $process, $body, $seed, $checks ); + $result = $this->enforce_worker_input_identity( $result, $body ); + $result = $this->enforce_worker_oracle_identity( $result ); + $result['profile'] = 'commoncrawl'; + $result['inputSource'] = 'commoncrawl'; + $result['inputSha256'] = $metadata['inputSha256']; + $result['commonCrawl'] = $metadata; + $result['run'] = $this->run_metadata(); + $result['repo'] = $this->git_metadata; + $result['oracleResult'] = $result['dom'] ?? null; // New neutral name; dom remains for replay compatibility. + $result['process'] = self::compact_process( $process ); + $result['durationMs'] = self::elapsed_ms( $started_at ); + if ( false === ( $result['ok'] ?? false ) && ! is_array( $result['signature'] ?? null ) ) { + $signature = Signature::from_result( $result ); + if ( null !== $signature ) { + $result['signature'] = $signature; + } + } + + $has_oracle_finding = is_array( $result['oracleFinding'] ?? null ); + $should_retain = $this->retain_all || false === ( $result['ok'] ?? false ) || $has_oracle_finding; + $artifact_dir = $should_retain ? $this->publish_staging( $staging_dir, $result, $metadata ) : null; + if ( ! $should_retain ) { + remove_dir_recursive( $staging_dir ); + } + + $summary = $this->summary_from_result( $result, $metadata, $artifact_dir ); + $this->append_summary( $summary ); + return $summary; + } catch ( \Throwable $throwable ) { + // Deliberately leave staging in pending/ for manual recovery. + return $this->record_callback_error( $metadata, $throwable, $started_at, $body, $staging_dir ); + } + } + + private function enforce_worker_oracle_identity( array $result ): array { + $expected = $this->oracle->metadata(); + $actual = $result['oracle'] ?? null; + $mismatches = OracleRenderer::identity_mismatches( $expected, is_array( $actual ) ? $actual : array() ); + if ( empty( $mismatches ) ) { + return $result; + } + $result['ok'] = false; + $result['status'] = 'oracle-identity-drift'; + $result['failureClass'] = 'oracle-identity-drift'; + $result['failureSnippet'] = implode( '; ', $mismatches ); + $result['sourceOracle'] = $expected; + $result['actualOracle'] = $actual; + $result['oracleIdentityMismatches'] = $mismatches; + unset( $result['signature'], $result['oracleFinding'], $result['comparison'] ); + return $result; + } + + private function enforce_worker_input_identity( array $result, string $body ): array { + $expected = array( + 'inputSha1' => sha1( $body ), + 'inputLength' => strlen( $body ), + ); + $actual = array( + 'inputSha1' => $result['inputSha1'] ?? null, + 'inputLength' => $result['inputLength'] ?? null, + ); + if ( $expected === $actual ) { + return $result; + } + + $result['ok'] = false; + $result['status'] = 'worker-input-identity-drift'; + $result['failureClass'] = 'worker-input-identity-drift'; + $result['failureSnippet'] = 'Worker result input identity did not match the parent-persisted Common Crawl body.'; + $result['workerInfrastructure'] = true; + $result['expectedInputIdentity'] = $expected; + $result['actualInputIdentity'] = $actual; + unset( $result['signature'], $result['oracleFinding'], $result['comparison'] ); + return $result; + } + + private function worker_args( string $staging_dir, int $seed, string $checks ): array { + $args = array( + '-d', 'memory_limit=' . $this->memory_limit, + $this->worker_script, + '--input-file', $staging_dir . '/input.bin', + '--mode', Generator::MODE_FULL_DOCUMENT, + '--profile', 'commoncrawl', + '--seed', (string) $seed, + '--output-dir', $staging_dir, + '--max-tokens', (string) $this->limits['maxTokens'], + '--max-nodes', (string) $this->limits['maxNodes'], + '--max-depth', (string) $this->limits['maxDepth'], + '--max-tree-bytes', (string) $this->limits['maxTreeBytes'], + '--checks', $checks, + '--process-timeout-ms', (string) $this->process_timeout_ms, + '--git-metadata-base64', git_metadata_base64( $this->git_metadata ), + ); + return array_merge( $args, $this->oracle->worker_args() ); + } + + private function create_staging_dir( array $metadata ): string { + $document_key = hash( 'sha256', (string) $metadata['recordId'] . "\0" . (string) $metadata['inputSha1'] ); + $random = bin2hex( random_bytes( 6 ) ); + $path = $this->output_dir . '/pending/' . substr( $document_key, 0, 20 ) . '-' . getmypid() . '-' . $random; + ensure_dir( $path ); + return $path; + } + + private function persist_initial_input( string $staging_dir, string $body, array $metadata, int $seed, string $checks ): void { + $input_path = $staging_dir . '/input.bin'; + write_file_atomic( $input_path, $body ); + $oracle_options = $this->oracle->replay_options(); + $replay_options = array_merge( + array( 'failUnsupported' => false, 'checks' => $checks ), + $oracle_options, + array( + 'memoryLimit' => $this->memory_limit, + 'processTimeoutMs' => $this->process_timeout_ms, + 'workerScript' => realpath( $this->worker_script ) ?: $this->worker_script, + ) + ); + write_json_file_atomic( + $staging_dir . '/replay.json', + array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-replay', + 'createdAt' => gmdate( 'c' ), + 'run' => $this->run_metadata(), + 'repoRoot' => repo_root(), + 'repoCommit' => $this->git_metadata['commit'] ?? null, + 'repoDirty' => $this->git_metadata['dirty'] ?? null, + 'phpVersion' => PHP_VERSION, + 'seed' => $seed, + 'profile' => 'commoncrawl', + 'mode' => Generator::MODE_FULL_DOCUMENT, + 'payloadPolicy' => null, + 'fragmentContext' => 'body', + 'generator' => null, + 'inputSource' => 'commoncrawl', + 'inputBase64' => base64_encode( $body ), + 'inputSha1' => sha1( $body ), + 'inputSha256' => hash( 'sha256', $body ), + 'inputLength' => strlen( $body ), + 'inputPreview' => preview_bytes( $body ), + 'limits' => $this->limits, + 'oracle' => $this->oracle->metadata(), + 'options' => $replay_options, + 'commonCrawl' => $metadata, + 'status' => 'pending-worker', + ) + ); + } + + private function load_or_synthesize_result( string $staging_dir, array $process, string $body, int $seed, string $checks ): array { + $result = null; + try { + $result = read_json_file( $staging_dir . '/result.json' ); + } catch ( \Throwable $ignored ) { + $result = null; + } + if ( is_array( $result ) && false === ( $process['processGroupCleanupFailed'] ?? false ) ) { + return $result; + } + + return synthesize_worker_process_failure( + $process, + array( + 'seed' => $seed, + 'profile' => 'commoncrawl', + 'mode' => Generator::MODE_FULL_DOCUMENT, + 'inputSource' => 'commoncrawl', + 'inputSha1' => sha1( $body ), + 'inputSha256' => hash( 'sha256', $body ), + 'inputLength' => strlen( $body ), + 'checks' => $checks, + 'oracle' => $this->oracle->metadata(), + ) + ); + } + + private function publish_staging( string $staging_dir, array &$result, array $metadata ): ?string { + $signature = $result['signature'] ?? $result['oracleFinding']['signature'] ?? null; + $hash = is_array( $signature ) && is_string( $signature['hash'] ?? null ) + ? $signature['hash'] + : substr( hash( 'sha256', (string) ( $result['failureClass'] ?? $result['status'] ?? 'retained' ) ), 0, 16 ); + $record_key = (string) $metadata['recordId'] . "\0" . (string) $metadata['inputSha1']; + $document_key = substr( hash( 'sha256', $record_key ), 0, 20 ); + $signature_dir = $this->output_dir . '/findings/' . preg_replace( '/[^A-Za-z0-9._-]/', '_', $hash ); + $final_dir = $signature_dir . '/' . $document_key; + $lock = $this->retention_lock(); + + try { + ensure_dir( $signature_dir ); + if ( is_file( $final_dir . '/.complete' ) ) { + $result = self::replace_path_prefix( $result, $staging_dir, $final_dir ); + remove_dir_recursive( $staging_dir ); + return $final_dir; + } + + $complete_count = 0; + foreach ( scandir( $signature_dir ) ?: array() as $entry ) { + if ( is_file( $signature_dir . '/' . $entry . '/.complete' ) ) { + ++$complete_count; + } + } + if ( ! $this->retain_all && $complete_count >= $this->max_keep_per_signature ) { + remove_dir_recursive( $staging_dir ); + return null; + } + + $result = self::replace_path_prefix( $result, $staging_dir, $final_dir ); + $result['paths'] = array( + 'outputDir' => $final_dir, + 'inputPath' => $final_dir . '/input.bin', + 'replayPath' => $final_dir . '/replay.json', + 'resultPath' => $final_dir . '/result.json', + ); + $replay = read_json_file( $staging_dir . '/replay.json' ); + if ( ! is_array( $replay ) ) { + $replay = array( 'kind' => 'html-api-fuzz-replay' ); + } + $replay = self::replace_path_prefix( $replay, $staging_dir, $final_dir ); + $replay['run'] = $this->run_metadata(); + $replay['repoRoot'] = repo_root(); + $replay['repoCommit'] = $this->git_metadata['commit'] ?? null; + $replay['repoDirty'] = $this->git_metadata['dirty'] ?? null; + $replay['commonCrawl'] = $metadata; + $replay['inputSha1'] = $metadata['inputSha1'] ?? $replay['inputSha1'] ?? null; + $replay['inputSha256'] = $metadata['inputSha256'] ?? null; + $replay['inputLength'] = $metadata['byteLength'] ?? $replay['inputLength'] ?? null; + $replay['oracle'] = $this->oracle->metadata(); + if ( 'oracle-identity-drift' === ( $result['failureClass'] ?? null ) ) { + $replay['sourceOracle'] = $result['sourceOracle'] ?? $this->oracle->metadata(); + $replay['actualOracle'] = $result['actualOracle'] ?? null; + $replay['oracleIdentityMismatches'] = $result['oracleIdentityMismatches'] ?? array(); + } + $replay['options']['checks'] = $result['checks'] ?? $this->checks_for_seed( (int) ( $result['seed'] ?? 1 ) ); + $replay['options']['memoryLimit'] = $this->memory_limit; + $replay['options']['processTimeoutMs'] = $this->process_timeout_ms; + $replay['options']['workerScript'] = realpath( $this->worker_script ) ?: $this->worker_script; + $replay['signature'] = $result['signature'] ?? null; + $replay['oracleFinding'] = $result['oracleFinding'] ?? null; + $replay['command'] = array( + 'program' => PHP_BINARY, + 'args' => array( 'tools/html-api-fuzz/replay.php', '--replay', $final_dir . '/replay.json' ), + 'cwd' => repo_root(), + ); + $replay['result'] = array( + 'ok' => $result['ok'] ?? false, + 'status' => $result['status'] ?? 'unknown', + 'failureClass' => $result['failureClass'] ?? null, + 'signature' => $result['signature'] ?? null, + 'oracleFinding' => $result['oracleFinding'] ?? null, + 'oracle' => $result['oracle'] ?? $this->oracle->metadata(), + 'resultPath' => $final_dir . '/result.json', + ); + write_json_file_atomic( $staging_dir . '/result.json', $result ); + write_json_file_atomic( $staging_dir . '/replay.json', $replay ); + $marker = "complete\n"; + if ( strlen( $marker ) !== file_put_contents( $staging_dir . '/.complete', $marker ) ) { + throw new \RuntimeException( 'Could not write finding completion marker.' ); + } + if ( is_dir( $final_dir ) ) { + remove_dir_recursive( $final_dir ); + } + if ( ! rename( $staging_dir, $final_dir ) ) { + throw new \RuntimeException( 'Could not atomically publish Common Crawl finding.' ); + } + return $final_dir; + } finally { + flock( $lock, LOCK_UN ); + fclose( $lock ); + } + } + + private function retention_lock() { + $path = $this->output_dir . '/.retention.lock'; + $file = fopen( $path, 'c+b' ); + if ( false === $file || ! flock( $file, LOCK_EX ) ) { + if ( is_resource( $file ) ) { + fclose( $file ); + } + throw new \RuntimeException( 'Could not lock Common Crawl artifact retention state.' ); + } + return $file; + } + + private static function replace_path_prefix( $value, string $from, string $to ) { + if ( is_string( $value ) ) { + return 0 === strpos( $value, $from ) ? $to . substr( $value, strlen( $from ) ) : $value; + } + if ( is_array( $value ) ) { + foreach ( $value as $key => $item ) { + $value[ $key ] = self::replace_path_prefix( $item, $from, $to ); + } + } + return $value; + } + + private function record_skip( array $metadata, string $status, int $started_at ): array { + $summary = array_merge( $this->base_summary(), array( + 'ok' => true, + 'status' => $status, + 'differentialCovered' => false, + 'durationMs' => self::elapsed_ms( $started_at ), + 'commonCrawl' => $metadata, + 'oracle' => $this->oracle->metadata(), + ) ); + $this->append_summary( $summary ); + return $summary; + } + + private function record_callback_error( array $metadata, \Throwable $throwable, int $started_at, ?string $body = null, ?string $staging_dir = null ): array { + $result = array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-worker-result', + 'createdAt' => gmdate( 'c' ), + 'ok' => false, + 'status' => 'commoncrawl-callback-error', + 'failureClass' => 'commoncrawl-callback-error', + 'failureSnippet' => $throwable->getMessage(), + 'throwable' => get_class( $throwable ), + 'profile' => 'commoncrawl', + 'mode' => Generator::MODE_FULL_DOCUMENT, + 'inputSource' => 'commoncrawl', + 'inputSha1' => $metadata['inputSha1'] ?? ( null === $body ? null : sha1( $body ) ), + 'inputSha256' => $metadata['inputSha256'] ?? ( null === $body ? null : hash( 'sha256', $body ) ), + 'inputLength' => $metadata['byteLength'] ?? ( null === $body ? null : strlen( $body ) ), + 'oracle' => $this->oracle->metadata(), + 'commonCrawl' => $metadata, + 'run' => $this->run_metadata(), + 'repo' => $this->git_metadata, + 'durationMs' => self::elapsed_ms( $started_at ), + ); + $result['signature'] = Signature::from_result( $result ); + $summary = $this->summary_from_result( $result, $metadata, $staging_dir ); + $summary['pendingRecovery'] = null !== $staging_dir; + $this->append_summary( $summary ); + return $summary; + } + + private function summary_from_result( array $result, array $metadata, ?string $artifact_dir ): array { + return array_merge( $this->base_summary(), array( + 'kind' => false === ( $result['ok'] ?? false ) ? 'commoncrawl-failure' : 'commoncrawl-attempt', + 'createdAt' => $result['createdAt'] ?? gmdate( 'c' ), + 'ok' => (bool) ( $result['ok'] ?? false ), + 'status' => $result['status'] ?? 'unknown', + 'failureClass' => $result['failureClass'] ?? null, + 'differentialCovered' => is_array( $result['comparison'] ?? null ), + 'seed' => $result['seed'] ?? null, + 'checks' => $result['checks'] ?? $this->checks, + 'inputSha1' => $metadata['inputSha1'] ?? null, + 'inputSha256' => $metadata['inputSha256'] ?? null, + 'inputLength' => $metadata['byteLength'] ?? null, + 'signature' => $result['signature'] ?? null, + 'oracleFinding' => $result['oracleFinding'] ?? null, + 'oracle' => $result['oracle'] ?? $this->oracle->metadata(), + 'sourceOracle' => $result['sourceOracle'] ?? null, + 'actualOracle' => $result['actualOracle'] ?? null, + 'oracleIdentityMismatches' => $result['oracleIdentityMismatches'] ?? array(), + 'artifactsRetained' => null !== $artifact_dir, + 'artifactDir' => $artifact_dir, + 'durationMs' => $result['durationMs'] ?? null, + 'timingsMs' => $result['timingsMs'] ?? null, + 'process' => $result['process'] ?? null, + 'commonCrawl' => $metadata, + ) ); + } + + private function base_summary(): array { + return array( + 'kind' => 'commoncrawl-attempt', + 'createdAt' => gmdate( 'c' ), + 'runId' => $this->run_id, + 'configHash' => $this->configuration_hash, + 'repo' => $this->git_metadata, + 'profile' => 'commoncrawl', + 'mode' => Generator::MODE_FULL_DOCUMENT, + 'inputSource' => 'commoncrawl', + 'coordinator' => $this->coordinator_provenance, + ); + } + + private function append_summary( array $summary ): void { + append_ndjson( $this->output_dir . '/commoncrawl-summary.ndjson', $summary ); + try { + $this->update_coverage( $summary ); + } catch ( \Throwable $ignored ) { + // coverage.json is a rebuildable snapshot; summary.ndjson is the + // durable source of truth and must not be reclassified or duplicated. + } + } + + private function update_coverage( array $summary ): void { + $path = $this->output_dir . '/coverage.json'; + $lock = fopen( $this->output_dir . '/.coverage.lock', 'c+b' ); + if ( false === $lock || ! flock( $lock, LOCK_EX ) ) { + throw new \RuntimeException( 'Could not lock Common Crawl coverage counters.' ); + } + try { + try { + $coverage = read_json_file( $path ); + } catch ( \Throwable $ignored ) { + $coverage = null; + } + $summary_size = @filesize( $this->output_dir . '/commoncrawl-summary.ndjson' ); + if ( + ! is_array( $coverage ) || + ! is_int( $coverage['summaryBytes'] ?? null ) || + false === $summary_size || + $coverage['summaryBytes'] > $summary_size + ) { + $coverage = $this->empty_coverage(); + } + $this->consume_summary_records( $coverage ); + $coverage['coverageRate'] = 0 === $coverage['total'] ? 0 : round( $coverage['covered'] / $coverage['total'], 6 ); + $coverage['updatedAt'] = gmdate( 'c' ); + write_json_file_atomic( $path, $coverage ); + } finally { + flock( $lock, LOCK_UN ); + fclose( $lock ); + } + } + + private function empty_coverage(): array { + return array( + 'kind' => 'html-api-commoncrawl-coverage', + 'runId' => $this->run_id, + 'configHash' => $this->configuration_hash, + 'total' => 0, + 'covered' => 0, + 'failures' => 0, + 'statuses' => array(), + 'summaryBytes' => 0, + ); + } + + private function consume_summary_records( array &$coverage ): void { + $path = $this->output_dir . '/commoncrawl-summary.ndjson'; + $file = fopen( $path, 'rb' ); + if ( false === $file || ! flock( $file, LOCK_SH ) ) { + throw new \RuntimeException( 'Could not read locked Common Crawl summaries for coverage.' ); + } + try { + if ( 0 !== fseek( $file, (int) $coverage['summaryBytes'] ) ) { + throw new \RuntimeException( 'Could not seek Common Crawl coverage cursor.' ); + } + $cursor = (int) $coverage['summaryBytes']; + while ( false !== ( $line = fgets( $file ) ) ) { + if ( '' === $line || "\n" !== substr( $line, -1 ) ) { + break; + } + $record = json_decode( $line, true ); + if ( is_array( $record ) ) { + $this->add_coverage_record( $coverage, $record ); + } + $position = ftell( $file ); + if ( false !== $position ) { + $cursor = $position; + } + } + $coverage['summaryBytes'] = $cursor; + } finally { + flock( $file, LOCK_UN ); + fclose( $file ); + } + } + + private function add_coverage_record( array &$coverage, array $summary ): void { + ++$coverage['total']; + if ( $summary['differentialCovered'] ?? false ) { + ++$coverage['covered']; + } + if ( false === ( $summary['ok'] ?? false ) ) { + ++$coverage['failures']; + } + $status = (string) ( $summary['status'] ?? 'unknown' ); + $coverage['statuses'][ $status ] = 1 + (int) ( $coverage['statuses'][ $status ] ?? 0 ); + } + + private function initialize_configuration(): void { + $configuration = array( + 'kind' => 'html-api-commoncrawl-configuration', + 'runId' => $this->run_id, + 'repo' => $this->git_metadata, + 'phpVersion' => PHP_VERSION, + 'oracle' => $this->oracle->metadata(), + 'oracleIdentitySha256' => OracleRenderer::identity_sha256( $this->oracle->metadata() ), + 'limits' => $this->limits, + 'maxInputBytes' => $this->max_input_bytes, + 'maxKeepPerSignature' => $this->max_keep_per_signature, + 'processTimeoutMs' => $this->process_timeout_ms, + 'memoryLimit' => $this->memory_limit, + 'checks' => $this->checks, + 'fullSamplePercent' => $this->full_sample_percent, + 'requireUtf8' => $this->require_utf8, + 'retainAll' => $this->retain_all, + 'workerScript' => realpath( $this->worker_script ) ?: $this->worker_script, + 'inputSemantics' => 'raw cc-analyzer response body; no charset transcoding', + 'ccAnalyzer' => array( + 'version' => getenv( 'CC_ANALYZER_VERSION' ) ?: null, + 'crawl' => getenv( 'CC_ANALYZER_CRAWL' ) ?: null, + 'invocation' => getenv( 'CC_ANALYZER_INVOCATION' ) ?: null, + 'coordinator' => $this->coordinator_provenance, + ), + ); + $this->configuration_hash = hash( 'sha256', json_encode( self::canonicalize( $configuration ), JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_SUBSTITUTE ) ); + $configuration['configHash'] = $this->configuration_hash; + + $path = $this->output_dir . '/configuration.json'; + $lock = fopen( $this->output_dir . '/.configuration.lock', 'c+b' ); + if ( false === $lock || ! flock( $lock, LOCK_EX ) ) { + throw new \RuntimeException( 'Could not lock Common Crawl configuration.' ); + } + try { + $existing = read_json_file( $path ); + if ( is_array( $existing ) ) { + if ( ! hash_equals( (string) ( $existing['configHash'] ?? '' ), $this->configuration_hash ) ) { + throw new \RuntimeException( 'Common Crawl output directory configuration mismatch; use a new run directory or identical settings.' ); + } + $this->run_id = (string) ( $existing['runId'] ?? $this->run_id ); + return; + } + $configuration['createdAt'] = gmdate( 'c' ); + write_json_file_atomic( $path, $configuration ); + } finally { + flock( $lock, LOCK_UN ); + fclose( $lock ); + } + } + + private function run_metadata(): array { + return array( + 'runId' => $this->run_id, + 'configHash' => $this->configuration_hash, + 'outputDir' => $this->output_dir, + 'coordinator' => $this->coordinator_provenance, + ); + } + + private static function canonicalize( $value ) { + if ( ! is_array( $value ) ) { + return $value; + } + if ( array_keys( $value ) !== range( 0, count( $value ) - 1 ) ) { + ksort( $value ); + } + foreach ( $value as $key => $item ) { + $value[ $key ] = self::canonicalize( $item ); + } + return $value; + } + + private function document_metadata( object $document ): array { + return array( + 'recordId' => (string) ( $document->recordId ?? '' ), + 'targetUri' => (string) ( $document->targetUri ?? '' ), + 'responseCode' => is_int( $document->responseCode ?? null ) ? $document->responseCode : null, + 'contentType' => (string) ( $document->contentType ?? '' ), + 'transportCharset' => is_string( $document->transportCharset ?? null ) ? $document->transportCharset : null, + 'inputStateKey' => (string) ( $document->inputStateKey ?? '' ), + 'rangeStart' => is_int( $document->rangeStart ?? null ) ? $document->rangeStart : null, + 'rangeLength' => is_int( $document->rangeLength ?? null ) ? $document->rangeLength : null, + ); + } + + private static function compact_process( array $process ): array { + return array( + 'code' => $process['code'] ?? null, + 'timedOut' => $process['timedOut'] ?? false, + 'durationMs' => $process['durationMs'] ?? null, + 'stderrTail' => substr( (string) ( $process['stderr'] ?? '' ), -2000 ), + 'stdoutTruncated' => $process['stdoutTruncated'] ?? false, + 'stderrTruncated' => $process['stderrTruncated'] ?? false, + 'logPath' => $process['logPath'] ?? null, + 'processGroupIsolated' => $process['processGroupIsolated'] ?? false, + 'processGroupCleanupFailed' => $process['processGroupCleanupFailed'] ?? false, + ); + } + + private static function seed_for_record( string $record_id, string $input_sha1 ): int { + return 1 + ( hexdec( substr( hash( 'sha256', $record_id . "\0" . $input_sha1 ), 0, 7 ) ) % 2147483646 ); + } + + private function checks_for_seed( int $seed ): string { + if ( 'sampled' !== $this->checks ) { + return $this->checks; + } + $bucket = hexdec( substr( hash( 'sha256', 'commoncrawl-full-checks:' . $seed ), 0, 8 ) ) % 100; + return $bucket < $this->full_sample_percent ? 'full' : 'baseline'; + } + + private static function elapsed_ms( int $started_at ): int { + return (int) round( max( 0, hrtime( true ) - $started_at ) / 1000000 ); + } + + private static function coordinator_provenance_from_environment(): ?array { + $variables = array( + 'id' => 'HTML_API_CC_COORDINATOR_ID', + 'batchName' => 'HTML_API_CC_BATCH_NAME', + 'cachePath' => 'HTML_API_CC_CACHE_PATH', + 'batchManifestPath' => 'HTML_API_CC_BATCH_MANIFEST_PATH', + 'batchManifestSha256' => 'HTML_API_CC_BATCH_MANIFEST_SHA256', + 'corpusFingerprint' => 'HTML_API_CC_CORPUS_FINGERPRINT', + 'documentCount' => 'HTML_API_CC_DOCUMENT_COUNT', + 'crawlId' => 'HTML_API_CC_CRAWL_ID', + 'pharPath' => 'HTML_API_CC_PHAR_PATH', + 'pharSha256' => 'HTML_API_CC_PHAR_SHA256', + 'invocationBase64' => 'HTML_API_CC_INVOCATION_BASE64', + 'callbackSha256' => 'HTML_API_CC_CALLBACK_SHA256', + 'workerSha256' => 'HTML_API_CC_WORKER_SHA256', + 'repositoryCommit' => 'HTML_API_CC_REPOSITORY_COMMIT', + 'repositoryDirty' => 'HTML_API_CC_REPOSITORY_DIRTY', + 'repositoryCodeSha256' => 'HTML_API_CC_REPOSITORY_CODE_SHA256', + 'repositoryStateSha256'=> 'HTML_API_CC_REPOSITORY_STATE_SHA256', + 'runtimeSha256' => 'HTML_API_CC_RUNTIME_SHA256', + 'trustBundleSha256' => 'HTML_API_CC_TRUST_BUNDLE_SHA256', + ); + $values = array(); + $present = 0; + foreach ( $variables as $key => $name ) { + $value = getenv( $name ); + if ( false !== $value && '' !== $value ) { + ++$present; + $values[ $key ] = (string) $value; + } + } + if ( 0 === $present ) { + return null; + } + if ( count( $variables ) !== $present ) { + throw new \InvalidArgumentException( 'Coordinator provenance environment must be supplied as one complete block.' ); + } + + foreach ( array( 'id', 'batchName' ) as $key ) { + if ( 1 !== preg_match( '/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/', $values[ $key ] ) ) { + throw new \InvalidArgumentException( "Coordinator provenance {$key} is invalid." ); + } + } + foreach ( array( 'batchManifestSha256', 'corpusFingerprint', 'pharSha256', 'callbackSha256', 'workerSha256', 'repositoryCodeSha256', 'repositoryStateSha256', 'runtimeSha256', 'trustBundleSha256' ) as $key ) { + if ( 1 !== preg_match( '/^[0-9a-f]{64}$/', $values[ $key ] ) ) { + throw new \InvalidArgumentException( "Coordinator provenance {$key} must be a lowercase SHA-256 digest." ); + } + } + if ( 1 !== preg_match( '/^[0-9a-f]{40}$/', $values['repositoryCommit'] ) ) { + throw new \InvalidArgumentException( 'Coordinator repository commit must be a lowercase Git object ID.' ); + } + if ( ! in_array( $values['repositoryDirty'], array( '0', '1' ), true ) ) { + throw new \InvalidArgumentException( 'Coordinator repository dirty state must be 0 or 1.' ); + } + $document_count = filter_var( $values['documentCount'], FILTER_VALIDATE_INT, array( 'options' => array( 'min_range' => 1 ) ) ); + if ( false === $document_count ) { + throw new \InvalidArgumentException( 'Coordinator document count must be a positive integer.' ); + } + foreach ( array( 'crawlId', 'cachePath', 'batchManifestPath', 'pharPath' ) as $key ) { + if ( '' === trim( $values[ $key ] ) ) { + throw new \InvalidArgumentException( "Coordinator provenance {$key} must be non-empty." ); + } + } + $cache_path = realpath( $values['cachePath'] ); + $manifest_path = realpath( $values['batchManifestPath'] ); + $phar_path = realpath( $values['pharPath'] ); + if ( false === $cache_path || ! is_file( $cache_path ) || false === $manifest_path || ! is_file( $manifest_path ) || false === $phar_path || ! is_file( $phar_path ) ) { + throw new \RuntimeException( 'Coordinator provenance paths must resolve to regular files.' ); + } + if ( ! hash_equals( $values['pharSha256'], hash_file( 'sha256', $phar_path ) ?: '' ) ) { + throw new \RuntimeException( 'Coordinator PHAR hash changed before Common Crawl callback initialization.' ); + } + $callback_path = dirname( __DIR__ ) . '/commoncrawl-analysis.php'; + if ( ! hash_equals( $values['callbackSha256'], hash_file( 'sha256', $callback_path ) ?: '' ) ) { + throw new \RuntimeException( 'Coordinator callback hash does not match commoncrawl-analysis.php.' ); + } + $worker_path = realpath( self::environment_string( 'HTML_API_CC_WORKER_SCRIPT', dirname( __DIR__ ) . '/worker.php' ) ); + if ( false === $worker_path || ! hash_equals( $values['workerSha256'], hash_file( 'sha256', $worker_path ) ?: '' ) ) { + throw new \RuntimeException( 'Coordinator Worker hash does not match the selected Worker script.' ); + } + if ( 'batch' !== getenv( 'CC_ANALYZER_MODE' ) || $values['batchName'] !== getenv( 'CC_ANALYZER_BATCH' ) ) { + throw new \RuntimeException( 'Coordinator provenance requires the matching cc-analyzer batch callback mode.' ); + } + $cc_manifest = realpath( (string) getenv( 'CC_ANALYZER_MANIFEST' ) ); + if ( false === $cc_manifest || $cache_path !== $cc_manifest ) { + throw new \RuntimeException( 'Coordinator cache path does not match CC_ANALYZER_MANIFEST.' ); + } + + $invocation_json = base64_decode( $values['invocationBase64'], true ); + if ( false === $invocation_json ) { + throw new \InvalidArgumentException( 'Coordinator invocation is not valid base64.' ); + } + $invocation = StrictJsonParser::decode( $invocation_json ); + if ( ! is_array( $invocation ) || array_keys( $invocation ) !== range( 0, count( $invocation ) - 1 ) || empty( $invocation ) ) { + throw new \InvalidArgumentException( 'Coordinator invocation must be a non-empty JSON string list.' ); + } + foreach ( $invocation as $argument ) { + if ( ! is_string( $argument ) || '' === $argument || false !== strpos( $argument, "\0" ) ) { + throw new \InvalidArgumentException( 'Coordinator invocation contains an invalid argument.' ); + } + } + + return array( + 'id' => $values['id'], + 'batchName' => $values['batchName'], + 'cachePath' => $cache_path, + 'batchManifestPath' => $manifest_path, + 'batchManifestSha256' => $values['batchManifestSha256'], + 'corpusFingerprint' => $values['corpusFingerprint'], + 'documentCount' => (int) $document_count, + 'crawlId' => $values['crawlId'], + 'pharPath' => $phar_path, + 'pharSha256' => $values['pharSha256'], + 'invocation' => $invocation, + 'callbackSha256' => $values['callbackSha256'], + 'workerSha256' => $values['workerSha256'], + 'repositoryCommit' => $values['repositoryCommit'], + 'repositoryDirty' => '1' === $values['repositoryDirty'], + 'repositoryCodeSha256' => $values['repositoryCodeSha256'], + 'repositoryStateSha256'=> $values['repositoryStateSha256'], + 'runtimeSha256' => $values['runtimeSha256'], + 'trustBundleSha256' => $values['trustBundleSha256'], + ); + } + + private static function environment_string( string $name, string $default ): string { + $value = getenv( $name ); + return false === $value || '' === $value ? $default : (string) $value; + } + + private static function environment_int( string $name, int $default, int $minimum ): int { + $value = getenv( $name ); + if ( false === $value || '' === $value ) { + return $default; + } + $parsed = filter_var( $value, FILTER_VALIDATE_INT ); + if ( false === $parsed || $parsed < $minimum ) { + throw new \InvalidArgumentException( "{$name} must be an integer greater than or equal to {$minimum}." ); + } + return (int) $parsed; + } + + private static function environment_bool( string $name, bool $default ): bool { + $value = getenv( $name ); + if ( false === $value || '' === $value ) { + return $default; + } + $parsed = filter_var( $value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE ); + if ( null === $parsed ) { + throw new \InvalidArgumentException( "{$name} must be a boolean value." ); + } + return $parsed; + } +} diff --git a/tools/html-api-fuzz/lib/Corpus.php b/tools/html-api-fuzz/lib/Corpus.php new file mode 100644 index 0000000000000..005a91dc8a40b --- /dev/null +++ b/tools/html-api-fuzz/lib/Corpus.php @@ -0,0 +1,78 @@ + basename( $file ), + 'data' => $data, + ); + } + } + + if ( null === $directory ) { + self::$entries = $entries; + } + return $entries; + } + + /** + * Extracts #data sections from html5lib .dat content. A section runs from + * the line after `#data` to the line before the next `#` directive, with + * the trailing newline removed. + */ + private static function parse_dat_data_sections( string $contents ): array { + $sections = array(); + $lines = explode( "\n", $contents ); + $current = null; + foreach ( $lines as $line ) { + if ( '#data' === $line ) { + $current = array(); + continue; + } + if ( null !== $current ) { + if ( '' !== $line && '#' === $line[0] ) { + $sections[] = implode( "\n", $current ); + $current = null; + continue; + } + $current[] = $line; + } + } + if ( null !== $current && array() !== $current ) { + $sections[] = implode( "\n", $current ); + } + + return $sections; + } +} diff --git a/tools/html-api-fuzz/lib/Generator.php b/tools/html-api-fuzz/lib/Generator.php new file mode 100644 index 0000000000000..21eed89952d96 --- /dev/null +++ b/tools/html-api-fuzz/lib/Generator.php @@ -0,0 +1,1409 @@ +`; the others receive a small probe weight so the fuzzer + * exercises the unsupported-context path today and picks up real + * coverage automatically when create_fragment() gains context support. + */ + public static function fragment_contexts(): array { + return array( 'body', 'div', 'p', 'td', 'tr', 'table', 'caption', 'colgroup', 'select', 'option', 'template', 'title', 'textarea', 'script', 'style', 'svg', 'math' ); + } + + public static function generate( int $seed, string $profile = 'auto', string $mode = 'auto', string $payload_policy = 'auto', ?int $max_input_bytes = null ): array { + $rng = new Prng( $seed ); + $requested_profile = $profile; + $requested_mode = $mode; + $requested_payload_policy = $payload_policy; + if ( 'auto' === $profile ) { + $profile = $rng->weighted( + array( + 'balanced' => 22, + 'full-document' => 8, + 'body-fragment' => 8, + 'tables' => 11, + 'template' => 8, + 'select' => 8, + 'foreign-content' => 10, + 'rawtext-rcdata' => 8, + 'text-fragment' => 4, + 'formatting-adoption' => 10, + 'attributes-entities' => 7, + 'comments-doctype-bogus' => 5, + 'deep-nesting' => 2, + 'resource-stress' => 1, + 'incomplete-malformed' => 2, + ) + ); + } elseif ( ! in_array( $profile, self::profiles(), true ) ) { + throw new \InvalidArgumentException( 'Unknown generator profile: ' . $profile ); + } + + if ( 'auto' === $payload_policy ) { + $payload_policy = 'resource-stress' === $profile + ? $rng->weighted( + array( + 'stress-long' => 75, + 'mostly-valid' => 15, + 'valid-utf8' => 10, + ) + ) + : $rng->weighted( + array( + 'valid-utf8' => 52, + 'mostly-valid' => 33, + 'ascii-structural' => 15, + ) + ); + } elseif ( ! in_array( $payload_policy, self::payload_policies(), true ) ) { + throw new \InvalidArgumentException( 'Unknown generator payload policy: ' . $payload_policy ); + } + + if ( 'auto' === $mode ) { + $mode = 'full-document' === $profile ? self::MODE_FULL_DOCUMENT : ( in_array( $profile, array( 'body-fragment', 'text-fragment' ), true ) ? self::MODE_FRAGMENT_BODY : $rng->weighted( array( self::MODE_FRAGMENT_BODY => 70, self::MODE_FULL_DOCUMENT => 30 ) ) ); + } elseif ( ! in_array( $mode, self::modes(), true ) ) { + throw new \InvalidArgumentException( 'Unknown generator mode: ' . $mode ); + } + + $fragment_context = 'body'; + if ( self::MODE_FRAGMENT_BODY === $mode ) { + $weights = array( 'body' => 240 ); + foreach ( self::fragment_contexts() as $context ) { + if ( 'body' !== $context ) { + $weights[ $context ] = 1; + } + } + $fragment_context = $rng->weighted( $weights ); + } + + $generator = new self( $rng, $profile, $payload_policy ); + if ( 'body' !== $fragment_context ) { + $generator->mark_feature( 'fragment-context:' . $fragment_context ); + } + $max_depth = $generator->depth_for_profile(); + $body = 'text-fragment' === $profile ? $generator->text_fragment_input() : $generator->nodes( $max_depth, 'body' ); + + if ( self::MODE_FULL_DOCUMENT === $mode ) { + $html = $generator->full_document( $body ); + } else { + $html = $body; + } + $truncated = false; + if ( null !== $max_input_bytes && $max_input_bytes > 0 && strlen( $html ) > $max_input_bytes ) { + $html = self::trim_to_max_bytes( $html, $max_input_bytes ); + $truncated = true; + $generator->mark_feature( 'generator:truncated' ); + } + + return array( + 'input' => $html, + 'mode' => $mode, + 'profile' => $profile, + 'payloadPolicy' => $payload_policy, + 'fragmentContext' => $fragment_context, + 'parameters' => array( + 'seed' => $seed, + 'requestedProfile' => $requested_profile, + 'requestedMode' => $requested_mode, + 'requestedPayloadPolicy' => $requested_payload_policy, + 'profile' => $profile, + 'mode' => $mode, + 'payloadPolicy' => $payload_policy, + 'fragmentContext' => $fragment_context, + 'maxDepth' => $max_depth, + 'maxInputBytes' => $max_input_bytes, + 'truncated' => $truncated, + 'byteLength' => strlen( $html ), + 'features' => $generator->features(), + ), + ); + } + + private function __construct( Prng $rng, string $profile, string $payload_policy ) { + $this->rng = $rng; + $this->profile = $profile; + $this->payload_policy = $payload_policy; + } + + private static function trim_to_max_bytes( string $html, int $max_input_bytes ): string { + $trimmed = substr( $html, 0, $max_input_bytes ); + while ( '' !== $trimmed && 1 !== preg_match( '//u', $trimmed ) ) { + $trimmed = substr( $trimmed, 0, -1 ); + } + + return $trimmed; + } + + private function mark_feature( string $feature ): void { + $this->features[ $feature ] = true; + } + + private function features(): array { + $features = array_keys( $this->features ); + sort( $features ); + return $features; + } + + private function depth_for_profile(): int { + if ( 'resource-stress' === $this->profile ) { + return $this->rng->int( 10, 20 ); + } + if ( 'deep-nesting' === $this->profile ) { + return $this->rng->int( 8, 18 ); + } + if ( 'balanced' === $this->profile ) { + return $this->rng->int( 3, 5 ); + } + return $this->rng->int( 3, 6 ); + } + + private function full_document( string $body ): string { + $doctype = $this->rng->chance( 70 ) ? $this->doctype() : ''; + $head = $this->rng->chance( 65 ) ? '' . $this->head_nodes() . '' : $this->head_nodes(); + $attrs = $this->attrs(); + $body_at = $this->attrs(); + + if ( $this->rng->chance( 4 ) ) { + $this->mark_feature( 'frameset' ); + $frames = 'attrs() . '>' . $this->terminal_text( true ) . ''; + return $doctype . '' . $head . 'attrs() . '>' . $frames . ( $this->rng->chance( 70 ) ? '' : '' ) . ( $this->rng->chance( 40 ) ? $body : '' ); + } + + if ( $this->rng->chance( 20 ) ) { + return $doctype . $head . $body; + } + + $trailer = ''; + if ( $this->rng->chance( 8 ) ) { + $this->mark_feature( 'content-after-html' ); + $trailer = $this->rng->choice( array( $this->terminal_ascii( 6 ), '', '
tail
', '' ) ); + } + + return $doctype . '' . $head . '' . $body . ( $this->rng->chance( 75 ) ? '' . $trailer : '' ); + } + + private function head_nodes(): string { + $out = ''; + if ( $this->rng->chance( 45 ) ) { + $out .= '' . $this->terminal_text() . ''; + } + if ( $this->rng->chance( 35 ) ) { + $out .= 'attrs() . '>'; + } + if ( $this->rng->chance( 25 ) ) { + $out .= ''; + } + return $out; + } + + private function nodes( int $depth, string $context ): string { + if ( 'deep-nesting' === $this->profile ) { + $count = $this->rng->int( 1, 2 ); + } elseif ( $depth > 4 ) { + $count = $this->rng->int( 1, 3 ); + } elseif ( $depth > 2 ) { + $count = $this->rng->int( 1, 5 ); + } else { + $count = $this->rng->int( 1, 7 ); + } + + $out = ''; + for ( $i = 0; $i < $count; ++$i ) { + $out .= $this->node( $depth, $context ); + if ( strlen( $out ) > 131072 ) { + $this->mark_feature( 'generator:hard-truncated' ); + return self::trim_to_max_bytes( $out, 131072 ); + } + } + return $out; + } + + private function node( int $depth, string $context ): string { + if ( $depth <= 0 ) { + return $this->leaf(); + } + + $weights = array( + 'element' => 33, + 'text' => 16, + 'charref' => 1, + 'comment' => 7, + 'void' => 8, + 'raw' => 5, + 'template' => 5, + 'table' => 5, + 'select' => 2, + 'foreign' => 5, + 'adoption' => 2, + 'reconstruction' => 0, + 'list-chain' => 1, + 'special-closers' => 1, + 'weird-tag' => 0, + 'doctype' => 2, + 'bogus' => 2, + ); + + if ( 'tables' === $this->profile ) { + $weights['table'] = 35; + $weights['element'] = 15; + } elseif ( 'template' === $this->profile ) { + $weights['template'] = 35; + } elseif ( 'select' === $this->profile ) { + $weights['select'] = 35; + $weights['table'] = 8; + $weights['element'] = 15; + } elseif ( 'foreign-content' === $this->profile ) { + $weights['foreign'] = 35; + } elseif ( 'rawtext-rcdata' === $this->profile ) { + $weights['raw'] = 35; + } elseif ( 'comments-doctype-bogus' === $this->profile ) { + $weights['comment'] = 25; + $weights['doctype'] = 12; + $weights['bogus'] = 15; + } elseif ( 'attributes-entities' === $this->profile ) { + $weights['element'] = 50; + $weights['text'] = 22; + $weights['charref'] = 12; + $weights['weird-tag'] = 8; + } elseif ( 'formatting-adoption' === $this->profile ) { + $weights['element'] = 35; + $weights['adoption'] = 20; + $weights['reconstruction'] = 30; + } elseif ( 'incomplete-malformed' === $this->profile ) { + $weights['bogus'] = 25; + $weights['element'] = 28; + $weights['weird-tag'] = 15; + $weights['special-closers'] = 4; + } + + switch ( $this->rng->weighted( $weights ) ) { + case 'text': + $this->mark_feature( 'text' ); + return $this->terminal_text(); + case 'charref': + $this->mark_feature( 'text' ); + return $this->character_reference( 'text' ) . $this->terminal_payload(); + case 'comment': + $this->mark_feature( 'comment' ); + return $this->comment(); + case 'void': + $this->mark_feature( 'void-element' ); + return '<' . $this->rng->choice( $this->void_tags ) . $this->attrs() . ( $this->rng->chance( 25 ) ? '/>' : '>' ); + case 'raw': + return $this->raw_element(); + case 'template': + $this->mark_feature( 'template' ); + return 'attrs() . '>' . $this->nodes( $depth - 1, 'body' ) . ( $this->rng->chance( 80 ) ? '' : '' ); + case 'table': + return $this->table( $depth - 1 ); + case 'select': + return $this->select_stress( $depth - 1 ); + case 'foreign': + return $this->foreign( $depth - 1 ); + case 'adoption': + return $this->adoption_pattern( $depth - 1 ); + case 'reconstruction': + return $this->active_formatting_reconstruction_pattern( $depth - 1 ); + case 'list-chain': + return $this->auto_closing_chain( $depth - 1 ); + case 'special-closers': + return $this->special_closers(); + case 'weird-tag': + return $this->weird_element( $depth - 1 ); + case 'doctype': + $this->mark_feature( 'doctype' ); + return $this->doctype(); + case 'bogus': + $this->mark_feature( 'bogus-markup' ); + return $this->bogus(); + case 'element': + default: + return $this->element( $depth - 1 ); + } + } + + private function element( int $depth ): string { + if ( 'attributes-entities' === $this->profile && $this->rng->chance( 16 ) ) { + return $this->weird_element( $depth ); + } + if ( 'incomplete-malformed' === $this->profile && $this->rng->chance( 12 ) ) { + return $this->weird_element( $depth ); + } + + $tag = $this->tag_name(); + if ( 'formatting-adoption' === $this->profile ) { + $tag = $this->rng->choice( array( 'a', 'b', 'big', 'button', 'em', 'font', 'i', 'nobr', 'p', 'span', 'strong' ) ); + $this->mark_feature( 'formatting-adoption-candidate' ); + } + + $close = $this->rng->chance( 'incomplete-malformed' === $this->profile ? 55 : 85 ); + $end = $close ? 'rng->chance( 88 ) ? $tag : $this->tag_name() ) . '>' : ''; + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . $end; + } + + private function weird_element( int $depth ): string { + $this->mark_feature( 'tag:weird-syntax' ); + $case = $this->rng->weighted( + array( + 'unusual-name' => 36, + 'invalid-name' => 30, + 'boundary-spacing' => 18, + 'malformed-closer' => 16, + ) + ); + + if ( 'invalid-name' === $case ) { + $tag = $this->invalid_tag_name(); + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . ( $this->rng->chance( 40 ) ? '' : '' ); + } + + $tag = 'unusual-name' === $case ? $this->unusual_tag_name() : $this->tag_name(); + if ( 'boundary-spacing' === $case ) { + $this->mark_feature( 'tag:weird-spacing' ); + return '<' . $tag . $this->tag_gap() . $this->attrs() . $this->tag_gap() . ( $this->rng->chance( 25 ) ? '/' . $this->tag_gap() : '' ) . '>' . $this->nodes( $depth, 'body' ) . ( $this->rng->chance( 70 ) ? 'tag_gap() . '>' : '' ); + } + + if ( 'malformed-closer' === $case ) { + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . 'invalid_tag_name() . $this->tag_gap() . '>'; + } + + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . ( $this->rng->chance( 75 ) ? '' : '' ); + } + + /** + * select/option/optgroup nesting stress, including elements that end or + * are disallowed in select (input, textarea, button, nested select) and + * select inside table structures. + */ + private function select_stress( int $depth ): string { + $this->mark_feature( 'select' ); + $out = ''; + $count = $this->rng->int( 1, 4 ); + for ( $i = 0; $i < $count; ++$i ) { + switch ( $this->rng->weighted( array( 'option' => 38, 'optgroup' => 22, 'breaker' => 18, 'nested-select' => 10, 'other' => 12 ) ) ) { + case 'option': + $this->mark_feature( 'select:option' ); + $out .= 'attrs() . '>' . $this->terminal_text() . ( $this->rng->chance( 55 ) ? '' : '' ); + break; + case 'optgroup': + $this->mark_feature( 'select:optgroup' ); + $out .= 'attrs() . '>