Benchmark Hyperlight host 0.17.0, JS 0.4.0, Wasm 0.15.0

.github/actions/setup/action.yml

@@ -112,7 +112,7 @@ runs:
       echo "✅ Cargo is available: $(cargo --version)"
 
       # Required tool versions (crates.io)
-      WASM_TOOLS_REQUIRED_VERSION="1.243.0"
+      WASM_TOOLS_MIN_VERSION="1.259.0"
       CARGO_HYPERLIGHT_REQUIRED_VERSION="0.1.14"
       OHA_REQUIRED_VERSION="1.9.0"
 
@@ -127,19 +127,18 @@ runs:
       COMPONENTIZE_QJS_REQUIRED_VERSION="0.3.0"
 
       if [[ "${{ inputs.BENCH_ONLY }}" != "true" ]]; then
-        # Install wasm-tools if not cached or wrong version
         if command -v wasm-tools &> /dev/null; then
           WASM_TOOLS_CURRENT_VERSION=$(wasm-tools --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
           WASM_TOOLS_CURRENT_VERSION=${WASM_TOOLS_CURRENT_VERSION:-"unknown"}
-          if [ "$WASM_TOOLS_CURRENT_VERSION" = "$WASM_TOOLS_REQUIRED_VERSION" ]; then
+          if [[ "$WASM_TOOLS_CURRENT_VERSION" != "unknown" ]] && printf '%s\n' "$WASM_TOOLS_MIN_VERSION" "$WASM_TOOLS_CURRENT_VERSION" | sort -V -C; then
             echo "wasm-tools $WASM_TOOLS_CURRENT_VERSION already available (cached)"
           else
-            echo "wasm-tools version mismatch: found $WASM_TOOLS_CURRENT_VERSION, need $WASM_TOOLS_REQUIRED_VERSION. Reinstalling..."
-            cargo install wasm-tools@$WASM_TOOLS_REQUIRED_VERSION --locked --force
+            echo "wasm-tools $WASM_TOOLS_CURRENT_VERSION does not meet minimum $WASM_TOOLS_MIN_VERSION. Installing..."
+            cargo install wasm-tools@$WASM_TOOLS_MIN_VERSION --locked --force
           fi
         else
-          echo "Installing wasm-tools $WASM_TOOLS_REQUIRED_VERSION..."
-          cargo install wasm-tools@$WASM_TOOLS_REQUIRED_VERSION --locked
+          echo "Installing wasm-tools $WASM_TOOLS_MIN_VERSION..."
+          cargo install wasm-tools@$WASM_TOOLS_MIN_VERSION --locked
         fi
 
         # Install hyperlight-wasm-aot if not cached or wrong version.

.github/workflows/benchmark.yml

@@ -12,7 +12,7 @@ permissions:
 
 concurrency:
   group: benchmark-${{ github.event.pull_request.number }}
-  cancel-in-progress: false
+  cancel-in-progress: true
 
 env:
   CARGO_TERM_COLOR: always
@@ -67,6 +67,7 @@ jobs:
           name: run-identity
           path: artifacts/run.json
           if-no-files-found: error
+          overwrite: true
 
   # Build guest inputs once on KVM for all measurement configurations.
   producer:
@@ -101,6 +102,7 @@ jobs:
       - uses: actions/upload-artifact@v7
         with:
           name: guest-inputs
+          overwrite: true
           path: |
             js/src/wit/handler_wit.wasm
             js/default/handler.wasm
@@ -191,6 +193,7 @@ jobs:
         if: always()
         with:
           name: result-${{ matrix.platform }}-${{ matrix.runtime }}-${{ matrix.strategy }}
+          overwrite: true
           path: results/
           if-no-files-found: error
           retention-days: 14
@@ -216,8 +219,8 @@ jobs:
         with:
           pattern: result-*
           path: results/
-          merge-multiple: true
-      - run: node scripts/merge-results.ts --output run.json results/*/bundle.json
+      - name: Merge measurements
+        run: node scripts/merge-results.ts --output run.json results/*/*/bundle.json
       - uses: actions/upload-artifact@v7
         with:
           name: run-${{ github.run_id }}-${{ github.run_attempt }}

README.md

@@ -1,5 +1,7 @@
 # Hyperlight HTTP Benchmarks
 
+[View the dashboard](https://hyperlight-dev.github.io/hyperlight-bench/).
+
 HTTP benchmarks and a results dashboard for Hyperlight.
 Compare throughput, latency, and memory use across JavaScript and WebAssembly
 runtimes on Linux KVM and MSHV.

docs/README.md

@@ -40,7 +40,7 @@ summaries are not loaded by this interface.
 ### Local End To End
 
 Requires Linux x86_64, Node.js 24+, Rust, just, and read/write access to
-`/dev/kvm` or `/dev/mshv`. Install wasm-tools 1.243.0,
+`/dev/kvm` or `/dev/mshv`. Install wasm-tools 1.259.0 or newer,
 hyperlight-wasm-aot 0.15.0, cargo-hyperlight 0.1.14,
 componentize-qjs-cli 0.3.0 with `--no-default-features`, and oha 1.9.0.
 Use `cargo install <crate>@<version> --locked` for each tool.
@@ -74,6 +74,8 @@ not performance stability. The bundle records the request count and load setting
 JIT-heavy configurations can still take several minutes across the full matrix.
 
 Timed runs accept `--duration` and `--client-timeout` in seconds.
+The default client timeout is unlimited. Requests still in flight are cancelled
+when the load duration ends. `--client-timeout` enables a per-request limit.
 `--concurrency` sets the connection count. For slower local hosts, use:
 
 ```sh
@@ -272,7 +274,13 @@ complete benchmark run for the current candidate.
 
 `Benchmark Publication` runs trusted `main` scripts on hosted runners. Successful
 PR attempts are checked against GitHub workflow and commit metadata. All jobs
-must succeed in one attempt. Use **Re-run all jobs** for publishable retries.
+must have a successful latest result through the selected attempt. Use
+**Re-run failed jobs** to retain successful configurations, or rerun an individual
+job and its dependents. Each retry replaces its configuration's result artifact.
+Collection combines the artifacts from that workflow run.
+A failed retry blocks publication even if an earlier
+attempt succeeded. Source revisions and benchmark definitions must match.
+Use **Re-run all jobs** when required artifacts have expired.
 The executed Benchmark workflow must match the trusted workflow. A workflow
 change can require archival recovery after maintainer review and merge.
 

docs/results.md

@@ -9,6 +9,11 @@ parser when loading bundles.
 Store one immutable JSON bundle per workflow run and attempt. The identity is
 `run.id` plus `run.attempt`. A retry has its own bundle. Commit SHAs may repeat.
 
+`run.attempt` identifies the collection attempt. Partial retries retain earlier
+successful measurements from that workflow run. Source revisions and benchmark
+definitions must match across all shards. Each configuration has a stable artifact
+name. A retry replaces that artifact. Collection rejects duplicate configurations.
+
 | Field | Contents |
 | --- | --- |
 | `schemaVersion` | Storage format version, currently `1` |
@@ -20,7 +25,7 @@ Store one immutable JSON bundle per workflow run and attempt. The identity is
 | `catalog.platforms` | Stable platform IDs and display labels |
 | `catalog.metrics` | Metric IDs, units, direction and measurement method versions |
 | `benchmark` | Workload ID, version, settings, required metrics and expected matrix |
-| `runners` | Runner records scoped to this workflow attempt |
+| `runners` | Runner records for the selected measurements |
 | `measurements` | Results linked to a runner and runtime |
 
 Runtime and metric IDs are extensible strings. Lifecycle IDs are `reload`,

scripts/benchmark-local.ts

@@ -37,7 +37,7 @@ async function main() {
     ...benchmark.settings,
     durationSeconds: values.smoke ? 0 : positive(values.duration, benchmark.settings.durationSeconds, 'Duration'),
     concurrency: values.smoke ? 1 : positive(values.concurrency, benchmark.settings.concurrency, 'Concurrency'),
-    clientTimeoutSeconds: values.smoke ? 120 : positive(values['client-timeout'], benchmark.settings.clientTimeoutSeconds, 'Client timeout'),
+    clientTimeoutSeconds: values.smoke ? 120 : values['client-timeout'] === undefined ? benchmark.settings.clientTimeoutSeconds : positive(values['client-timeout'], 120, 'Client timeout'),
     ...(values.smoke ? { requestCount: 1 } : {}),
   }
   if (selected.some(runtime => runtime.id.startsWith('hyperlight-'))) {

scripts/benchmark.ts

@@ -1,9 +1,9 @@
 import { spawn, type ChildProcess } from 'node:child_process'
-import { createWriteStream, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { createWriteStream, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { resolve } from 'node:path'
 import { createInterface } from 'node:readline'
 import { fileURLToPath } from 'node:url'
-import { parseArgs } from 'node:util'
+import { inspect, parseArgs } from 'node:util'
 import { z } from 'zod'
 import { benchmark, catalog, runnerPools, runtimes, serverFlavor } from '../shared/catalog.ts'
 import { parseRunBundle, strategySchema, type RunBundle } from '../shared/results.ts'
@@ -26,6 +26,11 @@ mkdirSync(directory, { recursive: true })
 const local = values['local-bundle'] ? parseRunBundle(JSON.parse(readFileSync(values['local-bundle'], 'utf8'))) : undefined
 if (local && local.source !== 'local') throw new Error('Local collection requires a local bundle')
 const run = local?.run ?? JSON.parse(readFileSync(values.run, 'utf8')) as RunBundle['run']
+if (!local && process.env.GITHUB_ACTIONS) {
+  if (run.id !== process.env.GITHUB_RUN_ID || run.commit.sha !== process.env.GITHUB_SHA) throw new Error('Run identity differs from the current workflow')
+  run.attempt = z.coerce.number().int().positive().parse(process.env.GITHUB_RUN_ATTEMPT)
+  run.workflow.url = `https://github.com/${run.pullRequest!.repository}/actions/runs/${run.id}/attempts/${run.attempt}`
+}
 const executable = resolve('artifacts/bin', serverFlavor(runtime.id), 'http-bench')
 const oha = 'oha'
 const runner = await runnerMetadata(platform, !!local)
@@ -45,7 +50,7 @@ async function bounded<Value>(promise: Promise<Value>, milliseconds: number, mes
   let timer: ReturnType<typeof setTimeout>
   try {
     return await Promise.race([promise, new Promise<never>((_, reject) => {
-      timer = setTimeout(() => reject(new Error(message)), milliseconds)
+      timer = setTimeout(() => reject(new Error(`${message} after ${milliseconds}ms`)), milliseconds)
     })])
   } finally {
     clearTimeout(timer!)
@@ -55,7 +60,10 @@ async function bounded<Value>(promise: Promise<Value>, milliseconds: number, mes
 function closed(child: ChildProcess) {
   return new Promise<number | null>((resolve, reject) => {
     child.once('error', reject)
-    child.once('close', code => resolve(code))
+    child.once('close', (code, signal) => {
+      if (signal) reject(new Error(`${child.spawnfile} terminated by ${signal}`))
+      else resolve(code)
+    })
   })
 }
 
@@ -69,7 +77,7 @@ try {
     ...z.object({
       durationSeconds: z.number().int().nonnegative(),
       concurrency: z.number().int().positive(),
-      clientTimeoutSeconds: z.number().int().positive(),
+      clientTimeoutSeconds: z.number().int().positive().nullable(),
       requestCount: z.literal(1).optional(),
     }).parse(bundle.benchmark.settings),
   }
@@ -108,37 +116,45 @@ try {
   if (response.status !== 200) throw new Error(`Readiness request returned ${response.status}`)
   z.object({ uri: z.literal('/redirected.html') }).parse(await response.json())
   const limit = settings.requestCount ? ['-n', String(settings.requestCount)] : ['-z', `${settings.durationSeconds}s`]
-  load = spawn(oha, [url, ...limit, '-c', String(settings.concurrency), '-t', `${settings.clientTimeoutSeconds}s`, '--no-tui', '--output-format', 'json', '--output', performancePath], { stdio: ['ignore', 'inherit', 'inherit'] })
+  if (settings.requestCount && settings.clientTimeoutSeconds === null) throw new Error('Request-count runs require a client timeout')
+  const clientTimeout = settings.clientTimeoutSeconds === null ? [] : ['-t', `${settings.clientTimeoutSeconds}s`]
+  load = spawn(oha, [url, ...limit, '-c', String(settings.concurrency), ...clientTimeout, '--no-tui', '--output-format', 'json', '--output', performancePath], { stdio: ['ignore', 'inherit', 'inherit'] })
   loadClosed = closed(load)
   loadClosed.catch(() => {})
-  const exitCode = await bounded(Promise.race([loadClosed, unexpectedExit, aborted]), ((settings.requestCount ? settings.clientTimeoutSeconds : settings.durationSeconds) + 60) * 1000, 'Load generator timed out')
+  const exitCode = await bounded(Promise.race([loadClosed, unexpectedExit, aborted]), ((settings.requestCount ? settings.clientTimeoutSeconds! : settings.durationSeconds) + 60) * 1000, 'Load generator timed out')
   if (exitCode !== 0) throw new Error(`oha exited with ${exitCode}`)
   server.stdin!.end('shutdown\n')
-  if (await bounded(serverClosed, 10000, 'Server shutdown timed out') !== 0) throw new Error('Server shutdown failed')
+  const shutdownCode = await bounded(serverClosed, 10000, 'Server shutdown timed out')
+  if (shutdownCode !== 0) throw new Error(`Server shutdown failed with exit code ${shutdownCode}`)
   const performance = JSON.parse(readFileSync(performancePath, 'utf8'))
-  const memory = JSON.parse(readFileSync(memoryPath, 'utf8'))
-  rawOutputs.push({ tool: 'oha', format: 'json', content: performance }, { tool: 'sysinfo', format: 'json', content: memory })
+  rawOutputs.push({ tool: 'oha', format: 'json', content: performance })
+  const requestDetails = `HTTP statuses: ${JSON.stringify(performance?.statusCodeDistribution)}. Request errors: ${JSON.stringify(performance?.errorDistribution)}. Client timeout: ${settings.clientTimeoutSeconds === null ? 'unlimited' : `${settings.clientTimeoutSeconds}s`}. Concurrency: ${settings.concurrency}.`
   const nonnegative = z.number().finite().nonnegative()
-  const report = z.object({
+  const parsedReport = z.object({
     summary: z.object({ requestsPerSec: nonnegative.positive() }),
     latencyPercentiles: z.object({ p50: nonnegative, p95: nonnegative, 'p99.99': nonnegative }),
     statusCodeDistribution: z.record(z.string(), nonnegative.int()), errorDistribution: z.record(z.string(), nonnegative.int()),
-  }).parse(performance)
+  }).safeParse(performance)
+  if (!parsedReport.success) throw new Error(`Invalid oha report. ${requestDetails}`, { cause: parsedReport.error })
+  const report = parsedReport.data
+  if (!report.statusCodeDistribution['200'] || Object.entries(report.statusCodeDistribution).some(([status, count]) => status !== '200' && count > 0)) throw new Error(`Unexpected HTTP status distribution. ${requestDetails}`)
+  if (Object.entries(report.errorDistribution).some(([error, count]) => count > 0 && error !== 'aborted due to deadline')) throw new Error(`Request errors occurred. ${requestDetails}`)
+  if ((report.errorDistribution['aborted due to deadline'] ?? 0) > settings.concurrency) throw new Error(`Deadline cancellations exceed concurrency. ${requestDetails}`)
+  const memory = JSON.parse(readFileSync(memoryPath, 'utf8'))
+  rawOutputs.push({ tool: 'sysinfo', format: 'json', content: memory })
   const peak = z.array(z.object({ name: z.literal('Peak Resident Memory Usage'), unit: z.literal('bytes'), value: nonnegative.positive() })).length(1).parse(memory)[0]!
-  if (!report.statusCodeDistribution['200'] || Object.entries(report.statusCodeDistribution).some(([status, count]) => status !== '200' && count > 0)) throw new Error('Unexpected HTTP status distribution')
-  if (Object.entries(report.errorDistribution).some(([error, count]) => count > 0 && error !== 'aborted due to deadline')) throw new Error('Request errors occurred')
-  if ((report.errorDistribution['aborted due to deadline'] ?? 0) > settings.concurrency) throw new Error('Unexpected deadline cancellation count')
   bundle.measurements.push({
     id: configuration, runnerId: runner.id, runtimeId: runtime.id, strategy, status: 'success', rawOutputs,
     values: { rps: report.summary.requestsPerSec, memory: peak.value / 1024 ** 2, p50: report.latencyPercentiles.p50 * 1000, p95: report.latencyPercentiles.p95 * 1000, p9999: report.latencyPercentiles['p99.99'] * 1000 },
   })
 } catch (error) {
   console.error(`Benchmark failed: ${configuration}`, error)
+  const reason = error instanceof Error && error.cause === undefined ? String(error) : inspect(error, { depth: 5, colors: false })
   if (process.env.GITHUB_ACTIONS) {
-    const message = String(error).replaceAll('%', '%25').replaceAll('\r', '%0D').replaceAll('\n', '%0A')
+    const message = reason.replaceAll('%', '%25').replaceAll('\r', '%0D').replaceAll('\n', '%0A')
     console.error(`::error::${configuration}: ${message}`)
   }
-  bundle.measurements.push({ id: configuration, runnerId: runner.id, runtimeId: runtime.id, strategy, status: 'failed', reason: String(error), rawOutputs })
+  bundle.measurements.push({ id: configuration, runnerId: runner.id, runtimeId: runtime.id, strategy, status: 'failed', reason, rawOutputs })
   process.exitCode = 1
 } finally {
   if (load && load.exitCode === null && load.signalCode === null) load.kill()
@@ -159,5 +175,18 @@ try {
   log.end()
   process.removeListener('SIGINT', interrupt)
   process.removeListener('SIGTERM', interrupt)
+  for (const [tool, path] of [['oha', performancePath], ['sysinfo', memoryPath]] as const) {
+    if (rawOutputs.some(output => output.tool === tool) || !existsSync(path)) continue
+    try {
+      const content = readFileSync(path, 'utf8')
+      try {
+        rawOutputs.push({ tool, format: 'json', content: JSON.parse(content) })
+      } catch {
+        rawOutputs.push({ tool, format: 'text', content })
+      }
+    } catch (error) {
+      console.error(`Failed to retain ${tool} output at ${path}`, error)
+    }
+  }
   writeFileSync(resolve(directory, 'bundle.json'), `${JSON.stringify(parseRunBundle(bundle), null, 2)}\n`)
 }

scripts/build-harness.ts

@@ -1,5 +1,5 @@
 import { spawnSync } from 'node:child_process'
-import { copyFileSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
+import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
 import { dirname, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
 
@@ -78,6 +78,9 @@ if (['dummy', 'inputs', 'all'].includes(phase)) {
 
 if (['servers', 'all'].includes(phase)) {
   const executable = 'http-bench'
+  if (!existsSync('target/hyperlight-js-runtime/x86_64-hyperlight-none/release/hyperlight-js-runtime')) {
+    run('cargo', ['clean', '-p', 'hyperlight-js', '--release'])
+  }
   for (const serverFlavor of flavor === 'all' ? ['native', 'pulley'] : [flavor]) {
     const features = [serverFlavor === 'pulley' ? 'pulley' : '', phase === 'servers' ? process.argv[4] ?? '' : ''].filter(Boolean).join(',')
     run('cargo', ['clean', '-p', 'hyperlight-wasm', '--release'])