test(subagent): prove cold-start overlap

Author: TheGreatAxios

scripts/measure-subagent-cold-start.ts

@@ -0,0 +1,216 @@
+#!/usr/bin/env bun
+
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { performance } from "node:perf_hooks";
+
+import {
+  createSkillSearchTool,
+  workerSkillSearchDefinition,
+} from "../src/agent/skill-search.js";
+import {
+  createUseSkillTool,
+  workerUseSkillDefinition,
+} from "../src/agent/use-skill.js";
+import { discoverSkills } from "../src/extensions/skills.js";
+import { assembleInferenceBase } from "../src/session/assemble-runtime.js";
+import { createSessionStores } from "../src/session/optimized-context-store.js";
+
+type Mode = "sequential" | "overlap";
+type Measurement = {
+  a: number;
+  b: number;
+  c: number;
+  readiness: number;
+};
+type Sample = {
+  sample: number;
+  sequential: Measurement;
+  overlap: Measurement;
+  savings: number;
+};
+
+const DEFAULT_SAMPLES = 7;
+const repositoryRoot = resolve(import.meta.dir, "..");
+const skillDirs = [join(repositoryRoot, "plugins/corbits-skills")];
+const allowedSkillNames = [
+  "style",
+  "philosophy",
+  "native-runtime",
+  "idiot-proof",
+  "ponytail",
+];
+const attachedSkills = ["style", "philosophy"];
+
+function parseSampleCount(args: readonly string[]): number {
+  const raw = args
+    .find((arg) => arg.startsWith("--samples="))
+    ?.slice("--samples=".length);
+  if (raw === undefined) return DEFAULT_SAMPLES;
+
+  const samples = Number(raw);
+  if (!Number.isSafeInteger(samples) || samples < 1) {
+    throw new Error("--samples must be a positive integer");
+  }
+  return samples;
+}
+
+function elapsedSince(startedAt: number): number {
+  return performance.now() - startedAt;
+}
+
+async function measureLeaf(mode: Mode, workdir: string): Promise<Measurement> {
+  const readinessStartedAt = performance.now();
+
+  let phaseStartedAt = performance.now();
+  await assembleInferenceBase();
+  const a = elapsedSince(phaseStartedAt);
+
+  phaseStartedAt = performance.now();
+  const skills = await discoverSkills(repositoryRoot, skillDirs);
+  createSkillSearchTool({
+    skills,
+    allowedNames: allowedSkillNames,
+    definition: workerSkillSearchDefinition,
+  });
+  createUseSkillTool(
+    repositoryRoot,
+    skillDirs,
+    undefined,
+    allowedSkillNames,
+    workerUseSkillDefinition,
+    attachedSkills,
+  );
+  const b = elapsedSince(phaseStartedAt);
+
+  phaseStartedAt = performance.now();
+  if (mode === "sequential") {
+    await mkdir(workdir, { recursive: true });
+    await createSessionStores(workdir);
+  } else {
+    const storesPromise = createSessionStores(workdir);
+    void storesPromise.catch(() => undefined);
+    await mkdir(workdir, { recursive: true });
+    await storesPromise;
+  }
+  const c = elapsedSince(phaseStartedAt);
+
+  return { a, b, c, readiness: elapsedSince(readinessStartedAt) };
+}
+
+function summary(values: readonly number[]) {
+  const sorted = [...values].sort((left, right) => left - right);
+  const middle = Math.floor(sorted.length / 2);
+  const median =
+    sorted.length % 2 === 0
+      ? ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2
+      : (sorted[middle] ?? 0);
+  return {
+    median,
+    mean: values.reduce((sum, value) => sum + value, 0) / values.length,
+  };
+}
+
+function round(value: number): number {
+  return Number(value.toFixed(3));
+}
+
+function roundedSummary(values: readonly number[]) {
+  const result = summary(values);
+  return { median: round(result.median), mean: round(result.mean) };
+}
+
+async function main(): Promise<void> {
+  const sampleCount = parseSampleCount(process.argv.slice(2));
+  const root = await mkdtemp(join(tmpdir(), "corbits-subagent-cold-start-"));
+  const samples: Sample[] = [];
+
+  try {
+    await measureLeaf("sequential", join(root, "warmup-sequential"));
+    await measureLeaf("overlap", join(root, "warmup-overlap"));
+
+    for (let sample = 1; sample <= sampleCount; sample++) {
+      const modes: readonly Mode[] =
+        sample % 2 === 1
+          ? ["sequential", "overlap"]
+          : ["overlap", "sequential"];
+      const measurements = new Map<Mode, Measurement>();
+
+      for (const mode of modes) {
+        measurements.set(
+          mode,
+          await measureLeaf(mode, join(root, `${sample}-${mode}`)),
+        );
+      }
+
+      const sequential = measurements.get("sequential");
+      const overlap = measurements.get("overlap");
+      if (sequential === undefined || overlap === undefined) {
+        throw new Error("both benchmark modes must complete");
+      }
+      samples.push({
+        sample,
+        sequential,
+        overlap,
+        savings: sequential.readiness - overlap.readiness,
+      });
+    }
+
+    const roundedSamples = samples.map((sample) => ({
+      sample: sample.sample,
+      sequential: Object.fromEntries(
+        Object.entries(sample.sequential).map(([key, value]) => [
+          key,
+          round(value),
+        ]),
+      ),
+      overlap: Object.fromEntries(
+        Object.entries(sample.overlap).map(([key, value]) => [
+          key,
+          round(value),
+        ]),
+      ),
+      savings: round(sample.savings),
+    }));
+    const sequentialReadiness = samples.map(
+      (sample) => sample.sequential.readiness,
+    );
+    const overlapReadiness = samples.map((sample) => sample.overlap.readiness);
+    const savings = samples.map((sample) => sample.savings);
+    const overlapResidual = samples.map((sample) => sample.overlap.c);
+
+    console.log(
+      JSON.stringify(
+        {
+          samplesPerMode: sampleCount,
+          warmupRunsPerMode: 1,
+          units: "milliseconds",
+          boundaries: {
+            a: "assemble inference dependencies",
+            b: "discover skills and construct skill tools",
+            c: "make the workdir and initialize session stores",
+            readiness: "a + b + c, immediately before agent construction",
+            sequential: "await mkdir, then initialize stores",
+            overlap: "start stores, await mkdir, then await stores",
+            residual:
+              "overlap c latency still visible on the leaf-readiness critical path",
+          },
+          samples: roundedSamples,
+          summary: {
+            beforeSequentialReadiness: roundedSummary(sequentialReadiness),
+            afterOverlapReadiness: roundedSummary(overlapReadiness),
+            pairedSavings: roundedSummary(savings),
+            overlapResidual: roundedSummary(overlapResidual),
+          },
+        },
+        null,
+        2,
+      ),
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+}
+
+await main();

src/subagent/run-audit-store.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "bun:test";
-import { mkdtemp, rm, stat } from "node:fs/promises";
+import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { basename, join } from "node:path";
 import type { AuditStore, ContextStore } from "@intx/types/runtime";
@@ -59,19 +59,6 @@ function runParams(cwd: string, id: string) {
   };
 }
 
-async function eventuallyExists(path: string): Promise<boolean> {
-  for (let attempt = 0; attempt < 50; attempt++) {
-    if (
-      await stat(path)
-        .then(() => true)
-        .catch(() => false)
-    )
-      return true;
-    await Bun.sleep(1);
-  }
-  return false;
-}
-
 test("runSubAgent threads the isogit audit store and session id into createAgent", async () => {
   const cwd = await mkdtemp(join(tmpdir(), "corbits-run-audit-"));
   const store = fakeStore();
@@ -125,61 +112,88 @@ test("runSubAgent threads the isogit audit store and session id into createAgent
 test("runSubAgent overlaps store creation with workdir setup", async () => {
   const cwd = await mkdtemp(join(tmpdir(), "corbits-run-store-overlap-"));
   const store = fakeStore();
+  let releaseMkdir: (() => void) | undefined;
+  let signalMkdirStarted: (() => void) | undefined;
+  let signalMkdirFinished: (() => void) | undefined;
+  const pendingMkdir = new Promise<void>((resolve) => {
+    releaseMkdir = resolve;
+  });
+  const mkdirStarted = new Promise<void>((resolve) => {
+    signalMkdirStarted = resolve;
+  });
+  const mkdirFinished = new Promise<void>((resolve) => {
+    signalMkdirFinished = resolve;
+  });
   let resolveStores:
     | ((stores: { storage: ContextStore; audit: AuditStore }) => void)
     | undefined;
-  let signalStoreStarted: (() => void) | undefined;
-  const storeStarted = new Promise<void>((resolve) => {
-    signalStoreStarted = resolve;
-  });
   const pendingStores = new Promise<{
     storage: ContextStore;
     audit: AuditStore;
   }>((resolve) => {
     resolveStores = resolve;
   });
+  let storeStarted = false;
   let agentConstructed = false;
 
   try {
     await withMockedModuleDuring(
-      import.meta.resolve("../session/optimized-context-store.js"),
-      (real: typeof import("../session/optimized-context-store.js")) => ({
+      import.meta.resolve("node:fs/promises"),
+      (real: typeof import("node:fs/promises")) => ({
         ...real,
-        createSessionStores: () => {
-          defined(signalStoreStarted, "store start signal")();
-          return pendingStores;
+        mkdir: () => {
+          defined(signalMkdirStarted, "mkdir start signal")();
+          return pendingMkdir.then(() => {
+            defined(signalMkdirFinished, "mkdir finish signal")();
+            return undefined;
+          });
         },
       }),
       async () => {
         await withMockedModuleDuring(
-          import.meta.resolve("../agent/live-tool-dispatch.js"),
-          (real: typeof import("../agent/live-tool-dispatch.js")) => ({
+          import.meta.resolve("../session/optimized-context-store.js"),
+          (real: typeof import("../session/optimized-context-store.js")) => ({
             ...real,
-            createAgentWithLiveToolDispatch: async () => {
-              agentConstructed = true;
-              return stubAgent() as unknown as Awaited<
-                ReturnType<typeof real.createAgentWithLiveToolDispatch>
-              >;
+            createSessionStores: () => {
+              storeStarted = true;
+              return pendingStores;
             },
           }),
           async () => {
-            const { runSubAgent } = await import("./run.js");
-            const run = runSubAgent(runParams(cwd, "overlap-child"));
-            await storeStarted;
-
-            const workdir = join(cwd, ".ctx", "subagents", "overlap-child");
-            expect(await eventuallyExists(workdir)).toBe(true);
-            expect(agentConstructed).toBe(false);
-
-            defined(
-              resolveStores,
-              "store resolver",
-            )({
-              storage: store,
-              audit: store,
-            });
-            await run;
-            expect(agentConstructed).toBe(true);
+            await withMockedModuleDuring(
+              import.meta.resolve("../agent/live-tool-dispatch.js"),
+              (real: typeof import("../agent/live-tool-dispatch.js")) => ({
+                ...real,
+                createAgentWithLiveToolDispatch: async () => {
+                  agentConstructed = true;
+                  return stubAgent() as unknown as Awaited<
+                    ReturnType<typeof real.createAgentWithLiveToolDispatch>
+                  >;
+                },
+              }),
+              async () => {
+                const { runSubAgent } = await import("./run.js");
+                const run = runSubAgent(runParams(cwd, "overlap-child"));
+                await mkdirStarted;
+
+                expect(storeStarted).toBe(true);
+                expect(agentConstructed).toBe(false);
+
+                defined(releaseMkdir, "mkdir resolver")();
+                await mkdirFinished;
+                expect(agentConstructed).toBe(false);
+
+                defined(
+                  resolveStores,
+                  "store resolver",
+                )({
+                  storage: store,
+                  audit: store,
+                });
+                await run;
+                expect(agentConstructed).toBe(true);
+              },
+            );
           },
         );
       },