diff --git a/packages/typewriter/PLAN_execute_many.md b/packages/typewriter/PLAN_execute_many.md deleted file mode 100644 index 2c936c9..0000000 --- a/packages/typewriter/PLAN_execute_many.md +++ /dev/null @@ -1,80 +0,0 @@ -# executeMany Plan - -## Goal - -Reduce JS <-> Rust overhead in `packages/typewriter/src/runtime-batch.ts` by batching multiple EVM `execute` calls into one typewriter-evm request. - -## Why this matters - -`acceptBatch` already collects mutations into a list. Today each mutation still pays a separate IPC round trip to typewriter-evm. A batched request should keep the current sequential semantics while removing most of the transport overhead. - -## Proposed shape - -### Request - -Add a new wire method in typewriter-evm: - -```rust -ExecuteMany { id: u64, params: ExecuteManyParams } - -struct ExecuteManyParams { - calls: Vec, -} -``` - -`ExecuteParams` stays unchanged: - -```ts -{ - from: Address; - to: Address; - data: Hex; - value?: Hex; -} -``` - -### Response - -Return results in the same order as the input calls: - -```rust -struct ExecuteManyOk { - results: Vec, -} -``` - -## Runtime-batch usage - -Use `executeMany` first in `acceptBatch`. - -1. Collect the mutations that are about to be accepted. -2. Resolve and encode them into a single `calls` array. -3. Send one `executeMany` request to typewriter-evm. -4. Zip the returned `ExecuteOk[]` back onto the original mutations by index. -5. Preserve existing per-mutation behavior: - - reject individual failures - - assign `journalId` on success - - invalidate storage cache from returned slot writes - - collect new keccak preimages - - persist accepted mutations - - emit the same events - -`watchProgram` can stay on single-call `execute` initially. - -## Rollout order - -1. Add `ExecuteMany` to `packages/typewriter-evm/src/main.rs` and `packages/typewriter-evm/src/types.ts`. -2. Add `executeMany` to `packages/typewriter-evm/src/index.ts`. -3. Add a batched helper in `packages/typewriter/src/runtime-batch.ts`. -4. Switch `acceptBatch` to use the batched helper. -5. Benchmark against the current baseline. - -## Non-goals - -- Changing acceptance semantics. -- Retrying or skipping failures inside typewriter-evm. -- Replacing the sidecar with `napi-rs` or WASM. - -## Measurement target - -The first question is whether batching the IPC call produces a meaningful win before any deeper runtime or transport changes. diff --git a/packages/typewriter/package.json b/packages/typewriter/package.json index eebf66f..a06710e 100644 --- a/packages/typewriter/package.json +++ b/packages/typewriter/package.json @@ -8,6 +8,7 @@ "type": "module", "private": true, "scripts": { + "benchmark": "bun run --filter typewriter-evm build && bun test ./test/harness.benchmark.ts", "test": "bun test --timeout 30000", "typecheck": "tsc --noEmit" }, diff --git a/packages/typewriter/test/counter.benchmark.ts b/packages/typewriter/test/counter.benchmark.ts deleted file mode 100644 index 99fd0b9..0000000 --- a/packages/typewriter/test/counter.benchmark.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { test } from "bun:test"; -import { anvil } from "viem/chains"; -import type { TypewriterConfig } from "../src"; -import Counter from "./contracts/src/Counter.sol"; -import { - SCHEDULER_ACCOUNT, - TEST_DB_URL, - TEST_RPC_URL, - USER_PRIVATE_KEY, -} from "./setup"; -import { - deployCounter, - prepareCounterAdd, - prepareCounterCreateAccount, -} from "./utils"; - -process.env.NODE_ENV = "production"; - -const MUTATION_COUNT = 10_000; - -function counterAddMutation(params: { - readonly address: `0x${string}`; - readonly amount: bigint; - readonly nonce: bigint; -}) { - return prepareCounterAdd({ - privateKey: USER_PRIVATE_KEY, - amount: params.amount, - sequence: params.nonce, - address: params.address, - chainId: anvil.id, - }); -} - -function percentile(values: readonly number[], p: number): number { - const index = Math.min( - values.length - 1, - Math.floor((values.length - 1) * p), - ); - return values[index]!; -} - -function formatMs(durationMs: number): string { - return `${durationMs.toFixed(2)}ms`; -} - -test(`counter accepts ${MUTATION_COUNT.toLocaleString()} mutations`, async () => { - const { createTypewriter } = await import("../src"); - const address = await deployCounter(); - const config = { - address, - account: SCHEDULER_ACCOUNT, - chainId: anvil.id, - rpcUrl: TEST_RPC_URL, - sequencing: { order: "fifo", submitIntervalMs: 3_600_000 }, - database: { url: TEST_DB_URL, maxConnections: 2 }, - blockPollingIntervalMs: 3_600_000, - } as const satisfies TypewriterConfig; - - const typewriter = await createTypewriter(Counter, config); - - await typewriter.execute( - await prepareCounterCreateAccount({ - privateKey: USER_PRIVATE_KEY, - address, - chainId: anvil.id, - }), - ); - - const mutations = await Promise.all( - Array.from({ length: MUTATION_COUNT }, (_, index) => - counterAddMutation({ - address, - amount: 1n, - nonce: BigInt(index), - }), - ), - ); - - const mutationDurationsMs: number[] = []; - const totalStart = performance.now(); - - for (const mutation of mutations) { - const mutationStart = performance.now(); - await typewriter.execute(mutation); - mutationDurationsMs.push(performance.now() - mutationStart); - } - - const totalDurationMs = performance.now() - totalStart; - const sortedDurationsMs = mutationDurationsMs.toSorted((a, b) => a - b); - const averageDurationMs = totalDurationMs / MUTATION_COUNT; - const mutationsPerSecond = MUTATION_COUNT / (totalDurationMs / 1_000); - - console.log(` -Counter benchmark -mutations: ${MUTATION_COUNT.toLocaleString()} -total: ${formatMs(totalDurationMs)} -average: ${formatMs(averageDurationMs)} -p50: ${formatMs(percentile(sortedDurationsMs, 0.5))} -p95: ${formatMs(percentile(sortedDurationsMs, 0.95))} -p99: ${formatMs(percentile(sortedDurationsMs, 0.99))} -throughput: ${mutationsPerSecond.toFixed(2)} mutations/s`); -}, 120_000); diff --git a/packages/typewriter/test/harness.benchmark.ts b/packages/typewriter/test/harness.benchmark.ts index 2f7d380..d5204ca 100644 --- a/packages/typewriter/test/harness.benchmark.ts +++ b/packages/typewriter/test/harness.benchmark.ts @@ -19,66 +19,10 @@ import { process.env.NODE_ENV = "production"; -const MUTATION_COUNT = 10_000; +const LATENCY_MUTATION_COUNT = 1_000; +const THROUGHPUT_MUTATION_COUNT = 10_000; -function harnessCreditMutation(params: { - readonly address: `0x${string}`; - readonly account: `0x${string}`; - readonly amount: bigint; - readonly nonce: bigint; -}) { - return prepareHarnessMutation({ - mutation: "Credit", - params: { amount: params.amount }, - accountID: params.account, - sequence: params.nonce, - keyType: 2, - privateKey: USER_PRIVATE_KEY, - address: params.address, - chainId: anvil.id, - }); -} - -function percentile(values: readonly number[], p: number): number { - const index = Math.min( - values.length - 1, - Math.floor((values.length - 1) * p), - ); - return values[index]!; -} - -function formatMs(durationMs: number): string { - return `${durationMs.toFixed(2)}ms`; -} - -async function withoutConsoleOutput(run: () => Promise): Promise { - const original = { - debug: console.debug, - error: console.error, - info: console.info, - log: console.log, - warn: console.warn, - }; - const noop = () => {}; - - console.debug = noop; - console.error = noop; - console.info = noop; - console.log = noop; - console.warn = noop; - - try { - return await run(); - } finally { - console.debug = original.debug; - console.error = original.error; - console.info = original.info; - console.log = original.log; - console.warn = original.warn; - } -} - -test(`harness batch accepts ${MUTATION_COUNT.toLocaleString()} mutations`, async () => { +test("harness benchmark", async () => { const { createTypewriter } = await import("../src"); const address = await deployHarness(); const config = { @@ -86,21 +30,11 @@ test(`harness batch accepts ${MUTATION_COUNT.toLocaleString()} mutations`, async account: SCHEDULER_ACCOUNT, chainId: anvil.id, rpcUrl: TEST_RPC_URL, - sequencing: { - order: "batch", - batchIntervalMs: 10, - submitIntervalMs: 3_600_000, - batchOrder: ["CreateAccount", "Credit"], - }, - database: { url: TEST_DB_URL, maxConnections: 2 }, - blockPollingIntervalMs: 3_600_000, + database: { url: TEST_DB_URL }, } as const satisfies TypewriterConfig; - const typewriter = await createTypewriter(Harness, config); - const rootPublicKey = secp256k1PublicKey(USER_ACCOUNT.address); const account = nativeAccountID(2, rootPublicKey); - const createAccount = prepareHarnessCreateAccount({ keyType: 2, publicKey: rootPublicKey, @@ -108,47 +42,66 @@ test(`harness batch accepts ${MUTATION_COUNT.toLocaleString()} mutations`, async address, chainId: anvil.id, }); - await withoutConsoleOutput(() => typewriter.execute(createAccount)); - const mutations = await Promise.all( - Array.from({ length: MUTATION_COUNT }, (_, index) => - harnessCreditMutation({ - address, - account, - amount: 1n, - nonce: BigInt(index), - }), + Array.from( + { length: LATENCY_MUTATION_COUNT + THROUGHPUT_MUTATION_COUNT }, + (_, index) => + prepareHarnessMutation({ + mutation: "Credit", + params: { amount: 1n }, + accountID: account, + sequence: BigInt(index), + keyType: 2, + privateKey: USER_PRIVATE_KEY, + address, + chainId: anvil.id, + }), ), ); + const latencyMutations = mutations.slice(0, LATENCY_MUTATION_COUNT); + const throughputMutations = mutations.slice(LATENCY_MUTATION_COUNT); - const mutationDurationsMs = new Array(MUTATION_COUNT); - const totalStart = performance.now(); + // Keep the cost of Debug log formatting, but do not print the lines. + const consoleLog = console.log; + const consoleDebug = console.debug; + console.log = () => {}; + console.debug = () => {}; - await withoutConsoleOutput(async () => { - const promises = mutations.map(async (mutation, index) => { - const mutationStart = performance.now(); - await typewriter.execute(mutation); - mutationDurationsMs[index] = performance.now() - mutationStart; - }); + const latenciesMs: number[] = []; + let throughputDurationMs: number; + try { + const typewriter = await createTypewriter(Harness, config); + await typewriter.execute(createAccount); - await Promise.all(promises); - }); + for (const mutation of latencyMutations) { + const start = performance.now(); + await typewriter.execute(mutation); + latenciesMs.push(performance.now() - start); + } + + const start = performance.now(); + await Promise.all( + throughputMutations.map((mutation) => typewriter.execute(mutation)), + ); + throughputDurationMs = performance.now() - start; + } finally { + console.log = consoleLog; + console.debug = consoleDebug; + } - const totalDurationMs = performance.now() - totalStart; - const sortedDurationsMs = mutationDurationsMs.toSorted((a, b) => a - b); - const averageDurationMs = - mutationDurationsMs.reduce((total, duration) => total + duration, 0) / - MUTATION_COUNT; - const mutationsPerSecond = MUTATION_COUNT / (totalDurationMs / 1_000); + latenciesMs.sort((a, b) => a - b); + const averageMs = + latenciesMs.reduce((total, latency) => total + latency, 0) / + LATENCY_MUTATION_COUNT; + const p50Ms = latenciesMs[Math.floor(LATENCY_MUTATION_COUNT * 0.5)]!; + const p99Ms = latenciesMs[Math.floor(LATENCY_MUTATION_COUNT * 0.99)]!; + const mutationsPerSecond = + THROUGHPUT_MUTATION_COUNT / (throughputDurationMs / 1_000); console.log(` -Harness batch benchmark -mutations: ${MUTATION_COUNT.toLocaleString()} -batch window: 10ms -total: ${formatMs(totalDurationMs)} -average: ${formatMs(averageDurationMs)} -p50: ${formatMs(percentile(sortedDurationsMs, 0.5))} -p95: ${formatMs(percentile(sortedDurationsMs, 0.95))} -p99: ${formatMs(percentile(sortedDurationsMs, 0.99))} -throughput: ${mutationsPerSecond.toFixed(2)} mutations/s`); +Harness benchmark +single user, ${LATENCY_MUTATION_COUNT.toLocaleString()} sequential mutations: + average ${averageMs.toFixed(2)}ms, p50 ${p50Ms.toFixed(2)}ms, p99 ${p99Ms.toFixed(2)}ms +max throughput, ${THROUGHPUT_MUTATION_COUNT.toLocaleString()} concurrent mutations: + ${Math.round(mutationsPerSecond).toLocaleString()} mutations/s`); }, 300_000);