Skip to content

Commit 3571dc2

Browse files
committed
Coalesce consecutive tool-failure recovery audits into one count
The audit sink is append-only, so one failed-tool recovery per result looked like distinct recoveries. Hold a burst count until the single pending nudge is applied. Forensics treat a missing count as one.
1 parent 3391db2 commit 3571dc2

4 files changed

Lines changed: 142 additions & 10 deletions

File tree

‎scripts/intervention-forensics.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -151,15 +151,19 @@ for (const file of files) {
151151
bucket = emptyBucket();
152152
buckets.set(key, bucket);
153153
}
154-
bucket.count++;
154+
const occurrences = record.count ?? 1;
155+
bucket.count += occurrences;
155156
const family = record.family ?? record.model ?? "unknown";
156-
bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1);
157+
bucket.byFamily.set(
158+
family,
159+
(bucket.byFamily.get(family) ?? 0) + occurrences,
160+
);
157161
const model = record.model ?? "unknown";
158-
bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1);
162+
bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + occurrences);
159163
if (record.class === "stop" || record.class === "nudge") {
160164
interventionsByModel.set(
161165
model,
162-
(interventionsByModel.get(model) ?? 0) + 1,
166+
(interventionsByModel.get(model) ?? 0) + occurrences,
163167
);
164168
}
165169
if (record.measurement !== undefined) {

‎src/subagent/intervention-log.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,11 @@ export interface InterventionRecord {
9090
};
9191
/** Free-form specifics, kept short (a looped window, a refused fingerprint). */
9292
detail?: string;
93+
/**
94+
* How many consecutive same-(id, state) firings this row stands for.
95+
* Absent means 1 — older records and one-shot interventions omit it.
96+
*/
97+
count?: number;
9398
}
9499

95100
/** Fields every record from one run shares, supplied once at construction. */

‎src/subagent/nudge-director.test.ts‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,77 @@ describe("SubAgentDirector tool failure recovery", () => {
172172
expect(texts?.[0]).toContain("report the blocker");
173173
});
174174

175+
test("N consecutive failed tools in one burst produce one coalesced intervention record", async () => {
176+
const director = new SubAgentDirector("system", [], undefined, 30);
177+
const caps = capabilities();
178+
const records: { id: string; count?: number }[] = [];
179+
director.observeInterventions((event) => {
180+
records.push(
181+
event.count === undefined
182+
? { id: event.id }
183+
: { id: event.id, count: event.count },
184+
);
185+
});
186+
187+
await director.decide(
188+
inferenceDone(["fail-a", "fail-b", "fail-c"]),
189+
state,
190+
caps,
191+
);
192+
await director.decide(toolDone("fail-a", true), state, caps);
193+
await director.decide(toolDone("fail-b", true), state, caps);
194+
expect(records).toEqual([]);
195+
196+
const texts = ephemeralTexts(
197+
inferAction(await director.decide(toolDone("fail-c", true), state, caps)),
198+
);
199+
expect(texts).toHaveLength(1);
200+
expect(texts?.[0]).toContain("A tool call failed");
201+
expect(records).toEqual([{ id: "tool-failure-recovery", count: 3 }]);
202+
});
203+
204+
test("a single failed tool writes one recovery record with count omitted", async () => {
205+
const director = new SubAgentDirector("system", [], undefined, 30);
206+
const caps = capabilities();
207+
const records: { id: string; class?: string; count?: number }[] = [];
208+
director.observeInterventions((event) => {
209+
records.push(
210+
event.count === undefined
211+
? { id: event.id, class: event.class }
212+
: { id: event.id, class: event.class, count: event.count },
213+
);
214+
});
215+
216+
await director.decide(inferenceDone(["failed-call"]), state, caps);
217+
await director.decide(toolDone("failed-call", true), state, caps);
218+
expect(records).toEqual([{ id: "tool-failure-recovery", class: "nudge" }]);
219+
});
220+
221+
test("a later failure burst writes a new coalesced record instead of stacking", async () => {
222+
const director = new SubAgentDirector("system", [], undefined, 30);
223+
const caps = capabilities();
224+
const records: { id: string; count?: number }[] = [];
225+
director.observeInterventions((event) => {
226+
records.push(
227+
event.count === undefined
228+
? { id: event.id }
229+
: { id: event.id, count: event.count },
230+
);
231+
});
232+
233+
await director.decide(inferenceDone(["fail-a", "fail-b"]), state, caps);
234+
await director.decide(toolDone("fail-a", true), state, caps);
235+
await director.decide(toolDone("fail-b", true), state, caps);
236+
expect(records).toEqual([{ id: "tool-failure-recovery", count: 2 }]);
237+
238+
await director.decide(inferenceDone(["fail-c"]), state, caps);
239+
await director.decide(toolDone("fail-c", true), state, caps);
240+
expect(records).toEqual([
241+
{ id: "tool-failure-recovery", count: 2 },
242+
{ id: "tool-failure-recovery" },
243+
]);
244+
});
245+
175246
test("successful tool result has no ephemeral recovery turn", async () => {
176247
const director = new SubAgentDirector("system", [], undefined, 30);
177248
const caps = capabilities();
@@ -221,6 +292,42 @@ describe("SubAgentDirector tool failure recovery", () => {
221292
expect(ephemeralTexts(infer)).toBeUndefined();
222293
});
223294

295+
test("compaction defers the coalesced recovery record until continuation infer", async () => {
296+
const director = new SubAgentDirector(
297+
"system",
298+
[],
299+
() => undefined,
300+
30,
301+
);
302+
const caps = capabilities();
303+
const records: { id: string; count?: number }[] = [];
304+
director.observeInterventions((event) => {
305+
records.push(
306+
event.count === undefined
307+
? { id: event.id }
308+
: { id: event.id, count: event.count },
309+
);
310+
});
311+
312+
await director.decide(
313+
inferenceDone(["fail-a", "fail-b"], 999_999),
314+
longState,
315+
caps,
316+
);
317+
await director.decide(toolDone("fail-a", true), longState, caps);
318+
const compact = actions(
319+
await director.decide(toolDone("fail-b", true), longState, caps),
320+
);
321+
expect(compact.some((action) => action.type === "infer")).toBe(false);
322+
expect(records).toEqual([]);
323+
324+
const resumed = inferAction(
325+
await director.decide(messageReceived(""), longState, caps),
326+
);
327+
expect(ephemeralTexts(resumed)?.[0]).toContain("A tool call failed");
328+
expect(records).toEqual([{ id: "tool-failure-recovery", count: 2 }]);
329+
});
330+
224331
test("retains recovery through compaction and consumes it once on continuation infer", async () => {
225332
let continuations = 0;
226333
const director = new SubAgentDirector(

‎src/subagent/nudge-director.ts‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,12 @@ export class SubAgentDirector extends DefaultDirector {
152152
private lastActivityAt: number;
153153
private consecutiveStalls = 0;
154154
private lastAssistantText = "";
155+
// Consecutive failed tool.done events that armed the single pending
156+
// recovery nudge. The jsonl sink is append-only, so the director holds
157+
// the count until applyPendingNudge actually attaches the nudge, then
158+
// writes one tool-failure-recovery row. Same (id + state) burst → one
159+
// record; a missing count still means 1.
160+
private pendingToolFailureRecoveries = 0;
155161
// Every stop and nudge is recorded with its measured value beside its
156162
// threshold, so a later threshold change can cite data instead of judgment
157163
//. Defaults to a no-op: logging is diagnostic, never required.
@@ -361,13 +367,10 @@ export class SubAgentDirector extends DefaultDirector {
361367
this.lastActivityAt = this.now();
362368
this.consecutiveStalls = 0;
363369
if (event.result.isError === true) {
364-
// Failed-tool recovery guidance.
370+
// Failed-tool recovery guidance. Text stays a single pending slot;
371+
// the audit count is the only thing that accumulates.
365372
this.pendingNudgeText = TOOL_FAILURE_RECOVERY_NUDGE;
366-
this.interventions({
367-
id: "tool-failure-recovery",
368-
class: "nudge",
369-
state: this.interventionState(),
370-
});
373+
this.pendingToolFailureRecoveries += 1;
371374
}
372375
}
373376
const base = await super.decide(event, state, capabilities);
@@ -465,6 +468,19 @@ export class SubAgentDirector extends DefaultDirector {
465468
const text = this.pendingNudgeText;
466469
this.pendingNudgeText = null;
467470
this.lastConsumedNudgeText = text;
471+
if (
472+
text === TOOL_FAILURE_RECOVERY_NUDGE &&
473+
this.pendingToolFailureRecoveries > 0
474+
) {
475+
const count = this.pendingToolFailureRecoveries;
476+
this.pendingToolFailureRecoveries = 0;
477+
this.interventions({
478+
id: "tool-failure-recovery",
479+
class: "nudge",
480+
state: this.interventionState(),
481+
...(count > 1 ? { count } : {}),
482+
});
483+
}
468484
const existing = actions[inferIndex] as Extract<
469485
ReactorAction,
470486
{ type: "infer" }

0 commit comments

Comments
 (0)