From 3cb48fa5a699709ba5b546b35f007ea0550dfa82 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:28:25 +0000 Subject: [PATCH 1/2] ci(bench): cover every command's startup path The benchmark only timed --version, status and execute --local. Commands that reach the MCP session (whoami, search, git-credential) and kody dev load several times more code than --version, and nothing caught it. Each scenario now states its expected exit code, the login-gated commands run without any config so they stop at the "not logged in" error after loading everything the real call needs, and budgets cover the new scenarios. Co-Authored-By: bot_apk --- scripts/bench-budgets.json | 5 +++++ scripts/bench.mjs | 12 ++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/bench-budgets.json b/scripts/bench-budgets.json index d81c8a8..aec43bf 100644 --- a/scripts/bench-budgets.json +++ b/scripts/bench-budgets.json @@ -1,6 +1,11 @@ { "--version": 60, "status": 60, + "help": 60, + "whoami": 160, + "search": 160, + "git-credential get": 160, + "dev (no app entry)": 220, "execute --local empty": 280, "execute --local kody:@ import": 280, "execute --local runtime call": 280 diff --git a/scripts/bench.mjs b/scripts/bench.mjs index ed9b52f..9d86c75 100644 --- a/scripts/bench.mjs +++ b/scripts/bench.mjs @@ -58,10 +58,18 @@ await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const apiUrl = `http://127.0.0.1:${server.address().port}` const executeFlags = ['execute', '--local', '--api-url', apiUrl, '--token', 'kody_at_bench'] +// Commands that need a login stop at the "not logged in" error, after loading +// everything they would need for the real call. +const noConfig = { XDG_CONFIG_HOME: '/nonexistent', KODY_API_TOKEN: '' } const scenarios = { 'node baseline': { command: process.execPath, args: ['-e', ''] }, '--version': { args: ['--version'] }, - status: { args: ['status'], env: { XDG_CONFIG_HOME: '/nonexistent' } }, + status: { args: ['status'], env: noConfig, exit: 1 }, + help: { args: ['help'] }, + whoami: { args: ['whoami'], env: noConfig, exit: 1 }, + search: { args: ['search', 'kody'], env: noConfig, exit: 1 }, + 'git-credential get': { args: ['git-credential', 'get'], env: noConfig }, + 'dev (no app entry)': { args: ['dev'], env: noConfig, exit: 1 }, 'execute --local empty': { args: [...executeFlags, '--code', 'export default async function main() { return null }'], }, @@ -110,7 +118,7 @@ for (const [name, scenario] of Object.entries(scenarios)) { const started = performance.now() const result = await run(command, args, env) const elapsed = performance.now() - started - if (name !== 'status' && result.status !== 0) { + if (result.status !== (scenario.exit ?? 0)) { throw new Error(`${name} exited ${result.status}: ${result.stderr}`) } samples.push(elapsed) From ee2ca932718d717cd20cdd7dced0f3bd84f434f9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:08:24 +0000 Subject: [PATCH 2/2] ci(bench): raise new scenario budgets for GHA runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub Actions ubuntu-latest Node 24.21 measured whoami/search at ~169ms over baseline, git-credential at ~159ms, and dev at ~228ms — Devin's 160/220 budgets (laptop-calibrated) fail the same way the original execute budgets did in #32. Raise the new rows to 240/300 (~75ms headroom over observed CI p50); leave --version/status/help/execute unchanged. Co-Authored-By: bot_apk Co-authored-by: Kent C. Dodds --- scripts/bench-budgets.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/bench-budgets.json b/scripts/bench-budgets.json index aec43bf..ce22af5 100644 --- a/scripts/bench-budgets.json +++ b/scripts/bench-budgets.json @@ -2,10 +2,10 @@ "--version": 60, "status": 60, "help": 60, - "whoami": 160, - "search": 160, - "git-credential get": 160, - "dev (no app entry)": 220, + "whoami": 240, + "search": 240, + "git-credential get": 240, + "dev (no app entry)": 300, "execute --local empty": 280, "execute --local kody:@ import": 280, "execute --local runtime call": 280