diff --git a/packages/docs/src/content/docs/concepts/automations.md b/packages/docs/src/content/docs/concepts/automations.md index 76a95098cb..dfe94e1842 100644 --- a/packages/docs/src/content/docs/concepts/automations.md +++ b/packages/docs/src/content/docs/concepts/automations.md @@ -56,7 +56,7 @@ A Slack channel-message automation runs for each new top-level message in the ch If the plugin is disabled, the automation remains visible but cannot receive events until a compatible plugin is enabled again. Each plugin page lists the resources and events it can publish. -Each event automation can start at most 10 runs in an hour. Set `limits.maxEventAutomationRunsPerHour` in `createApp()` to change this number. The limit applies to that one automation, so other automations on the same trigger keep running. When an automation reaches the limit, Junior blocks it and sends its creator a direct message. Resume the automation after you narrow its trigger. +Each event automation can start at most 10 runs in a rolling minute. Set `limits.eventAutomationRunLimit` in `createApp()` to change the count (`maxRuns`) and the window (`windowMs`), for example 100 runs per hour. The limit applies to that one automation, so other automations on the same trigger keep running. When an automation reaches the limit, Junior blocks it and sends its creator a direct message. Resume the automation after you narrow its trigger. ## Access and Delivery diff --git a/packages/docs/src/content/docs/reference/config-and-env.md b/packages/docs/src/content/docs/reference/config-and-env.md index 4f2971ae30..96c5c43ddd 100644 --- a/packages/docs/src/content/docs/reference/config-and-env.md +++ b/packages/docs/src/content/docs/reference/config-and-env.md @@ -182,9 +182,11 @@ By default, only the root Conversation can start child agents. Set `limits.maxAgentDepth` to let child agents start their own child agents. For example, `1` lets a child agent start child agents, and those cannot start more. -Each event automation can start at most 10 runs in a rolling hour. Set -`limits.maxEventAutomationRunsPerHour` to change this number. The window is -always one hour. When an automation reaches the limit, Junior blocks it, sends +Each event automation can start at most 10 runs in a rolling minute. Set +`limits.eventAutomationRunLimit.maxRuns` to change the count. Set +`limits.eventAutomationRunLimit.windowMs` to change the window, in ms. For +example, `eventAutomationRunLimit: { maxRuns: 100, windowMs: 3_600_000 }` +allows 100 runs per hour. When an automation reaches the limit, Junior blocks it, sends its creator a direct message, and reports a warning issue to Sentry. Each turn can spend at most $100 of model cost and make at most 1,000 tool diff --git a/packages/junior/src/chat/config.ts b/packages/junior/src/chat/config.ts index 4c1b8612b1..ed7e049ccb 100644 --- a/packages/junior/src/chat/config.ts +++ b/packages/junior/src/chat/config.ts @@ -42,12 +42,14 @@ const MAX_TOOL_CALLS_PER_TURN = 1000; */ const MAX_CONSECUTIVE_AUTOMATED_TURNS = 25; /** - * Max runs one Event automation can start in a rolling hour. Each run is a - * full agent turn, and Event automations skip the consecutive automated-turn - * limit. Kept low so a broad trigger, such as every message in a channel, - * blocks quickly instead of starting a turn for each Event. + * Max runs one Event automation can start in one rolling run window. Each run + * is a full agent turn, and Event automations skip the consecutive + * automated-turn limit. Kept low so a broad trigger, such as every message in + * a channel, blocks quickly instead of starting a turn for each Event. */ -const MAX_EVENT_AUTOMATION_RUNS_PER_HOUR = 10; +const MAX_EVENT_AUTOMATION_RUNS = 10; +/** Length of the rolling window for the Event automation run limit. */ +const EVENT_AUTOMATION_RUN_WINDOW_MS = 60 * 1000; const DEFAULT_FUNCTION_MAX_DURATION_SECONDS = 300; const DEFAULT_SLACK_SLASH_COMMAND = "/jr"; const DEFAULT_PROCESSING_REACTION_EMOJI = "eyes"; @@ -108,7 +110,8 @@ export interface BotConfig { maxToolCallsPerTurn: number; maxTurnCostUsd: number; maxConsecutiveAutomatedTurns: number; - maxEventAutomationRunsPerHour: number; + maxEventAutomationRuns: number; + eventAutomationRunWindowMs: number; turnTimeoutMs: number; userName: string; webSearchModelId: string; @@ -520,7 +523,8 @@ function readBotConfig( maxToolCallsPerTurn: MAX_TOOL_CALLS_PER_TURN, maxTurnCostUsd: MAX_TURN_COST_USD, maxConsecutiveAutomatedTurns: MAX_CONSECUTIVE_AUTOMATED_TURNS, - maxEventAutomationRunsPerHour: MAX_EVENT_AUTOMATION_RUNS_PER_HOUR, + maxEventAutomationRuns: MAX_EVENT_AUTOMATION_RUNS, + eventAutomationRunWindowMs: EVENT_AUTOMATION_RUN_WINDOW_MS, turnTimeoutMs: parseAgentTurnTimeoutMs( env.AGENT_TURN_TIMEOUT_MS, maxTurnTimeoutMs, @@ -640,8 +644,13 @@ export interface RuntimeLimits { maxAgentDepth?: number; /** Automated turns in a row before Junior stops answering automation. */ maxConsecutiveAutomatedTurns?: number; - /** Runs one Event automation can start in a rolling hour. */ - maxEventAutomationRunsPerHour?: number; + /** Runs one Event automation can start in a rolling window. */ + eventAutomationRunLimit?: { + /** Runs allowed in one window. The default is 10. */ + maxRuns?: number; + /** Window length in ms. The default, 60000, is one minute. */ + windowMs?: number; + }; /** Run slices one turn may use. */ maxSlicesPerTurn?: number; /** Tool calls one turn may make. */ @@ -738,10 +747,17 @@ export function setRuntimeLimits(limits: RuntimeLimits | undefined): void { limits.maxConsecutiveAutomatedTurns, ); } - if (limits?.maxEventAutomationRunsPerHour !== undefined) { - bot.maxEventAutomationRunsPerHour = requirePositiveLimit( - "maxEventAutomationRunsPerHour", - limits.maxEventAutomationRunsPerHour, + const runLimit = limits?.eventAutomationRunLimit; + if (runLimit?.maxRuns !== undefined) { + bot.maxEventAutomationRuns = requirePositiveLimit( + "eventAutomationRunLimit.maxRuns", + runLimit.maxRuns, + ); + } + if (runLimit?.windowMs !== undefined) { + bot.eventAutomationRunWindowMs = requirePositiveLimit( + "eventAutomationRunLimit.windowMs", + runLimit.windowMs, ); } if (limits?.maxSlicesPerTurn !== undefined) { diff --git a/packages/junior/src/chat/event-automations/README.md b/packages/junior/src/chat/event-automations/README.md index 63fd5e8bf8..14653575b1 100644 --- a/packages/junior/src/chat/event-automations/README.md +++ b/packages/junior/src/chat/event-automations/README.md @@ -58,20 +58,20 @@ Paused rows do not match new events. Resume does not replay events received duri the pause. Provider retries still use the event key to prevent duplicate runs. Deletion keeps the row and execution history. -Each Event automation can start at most -`botConfig.maxEventAutomationRunsPerHour` runs in a rolling hour -(`run-limit.ts`). The default is 10. A host sets it with -`createApp({ limits: { maxEventAutomationRunsPerHour } })`. The one-hour -window is fixed. The limit applies to one Automation, not to its trigger or +Each Event automation can start at most `botConfig.maxEventAutomationRuns` +runs in a rolling `botConfig.eventAutomationRunWindowMs` window +(`run-limit.ts`). The defaults are 10 runs and one minute. A host sets them with +`createApp({ limits: { eventAutomationRunLimit: { maxRuns, windowMs } } })`. +The limit applies to one Automation, not to its trigger or namespace. Another Automation on the same trigger keeps its own count. Event automations do not use the consecutive automated-turn limit, so this limit is what stops a noisy trigger from starting a full agent Turn for every Event. Ingest counts a run before dispatch. Each run appends one entry with its dispatch idempotency key to a short list in the state adapter. Then it counts -the keys from the last hour up to the first entry with its own key. The append +the keys from the last window up to the first entry with its own key. The append is atomic, so the list order decides which runs fit when Events arrive at the same moment. No lock is needed. A provider retry has the same key, so it gets -the same answer and does not count again. When the hour is full, ingest skips the dispatch and blocks +the same answer and does not count again. When the window is full, ingest skips the dispatch and blocks the Automation with a reason, and its creator gets the blocked notice. Each block also captures one Sentry issue event with the fingerprint `event_automation.run_limit.blocked` and the `app.task.id` of the Automation. diff --git a/packages/junior/src/chat/event-automations/ingest.ts b/packages/junior/src/chat/event-automations/ingest.ts index 08df389dce..a9777f69ad 100644 --- a/packages/junior/src/chat/event-automations/ingest.ts +++ b/packages/junior/src/chat/event-automations/ingest.ts @@ -125,8 +125,9 @@ export async function ingestEventAutomations( }, extra: { "app.event.event_type": event.eventType, - "app.event_automation.max_runs_per_hour": - botConfig.maxEventAutomationRunsPerHour, + "app.event_automation.max_runs": botConfig.maxEventAutomationRuns, + "app.event_automation.run_window_ms": + botConfig.eventAutomationRunWindowMs, "app.event_automation.resource_type": task.trigger.resourceType, "app.task.id": task.id, }, diff --git a/packages/junior/src/chat/event-automations/run-limit.ts b/packages/junior/src/chat/event-automations/run-limit.ts index d7fde158d9..7e4db17d92 100644 --- a/packages/junior/src/chat/event-automations/run-limit.ts +++ b/packages/junior/src/chat/event-automations/run-limit.ts @@ -4,12 +4,13 @@ * The cap applies to each Automation, not to a trigger type. A noisy trigger * blocks only the Automation that it fires. Each run costs a full agent Turn, * and Event automations do not use the consecutive automated-turn limit. - * `botConfig.maxEventAutomationRunsPerHour` sets the cap. + * `botConfig.maxEventAutomationRuns` sets the cap, and + * `botConfig.eventAutomationRunWindowMs` sets the rolling window. * * Each run appends one entry with its run key to a short list in the state - * adapter. Then it counts the run keys from the last hour up to the first entry - * with its own key. The append is atomic, so the list order decides which runs - * fit, even when Events arrive at the same moment. The first runs up to the cap + * adapter. Then it counts the run keys from the last window up to the first + * entry with its own key. The append is atomic, so the list order decides + * which runs fit, even when Events arrive at the same moment. The first runs up to the cap * pass, and the rest are refused. A provider retry has the same run key, so it * gets the same answer and does not count again. No lock and no * read-modify-write are needed. @@ -24,9 +25,6 @@ const RUN_LIST_PREFIX = "junior:event_automation_runs:v1"; // stay in the list while other Events of the same burst append after it. const RUN_LIST_BURST_ROOM = 100; -/** Length of the rolling window for the run limit. */ -export const EVENT_AUTOMATION_RUN_WINDOW_MS = 60 * 60 * 1000; - // A reset entry marks a resume or revive. Runs before it do not count. const runEntrySchema = z.union([ z.object({ atMs: z.number().finite(), id: z.string().min(1) }).strict(), @@ -40,7 +38,7 @@ function runListKey(automationId: string): string { } function runListMaxLength(): number { - return botConfig.maxEventAutomationRunsPerHour + RUN_LIST_BURST_ROOM; + return botConfig.maxEventAutomationRuns + RUN_LIST_BURST_ROOM; } async function appendRunEntry( @@ -51,20 +49,38 @@ async function appendRunEntry( await state.connect(); await state.appendToList(runListKey(automationId), entry, { maxLength: runListMaxLength(), - ttlMs: EVENT_AUTOMATION_RUN_WINDOW_MS, + ttlMs: botConfig.eventAutomationRunWindowMs, }); } +// Name the window in the largest whole unit, such as "minute" or "2 hours". +function describeRunWindow(windowMs: number): string { + const units: Array<[string, number]> = [ + ["day", 24 * 60 * 60 * 1000], + ["hour", 60 * 60 * 1000], + ["minute", 60 * 1000], + ["second", 1000], + ]; + for (const [unit, unitMs] of units) { + if (windowMs % unitMs === 0) { + const count = windowMs / unitMs; + return count === 1 ? unit : `${count} ${unit}s`; + } + } + return `${windowMs} ms`; +} + /** Reason stored on an Event automation that the run limit blocked. */ export function eventAutomationRunLimitReason(): string { + const window = describeRunWindow(botConfig.eventAutomationRunWindowMs); return ( - `This automation reached its limit of ${botConfig.maxEventAutomationRunsPerHour} runs per hour. ` + + `This automation reached its limit of ${botConfig.maxEventAutomationRuns} runs per ${window}. ` + "Check that its trigger is not too broad, then resume it." ); } /** - * Count one run of an Event automation, or refuse when its hour is full. + * Count one run of an Event automation, or refuse when its window is full. * * `runKey` is the dispatch idempotency key, so a provider retry of the same * Event does not count again. @@ -80,7 +96,7 @@ export async function admitEventAutomationRun(args: { atMs: args.nowMs, id: args.runKey, }); - const windowStartMs = args.nowMs - EVENT_AUTOMATION_RUN_WINDOW_MS; + const windowStartMs = args.nowMs - botConfig.eventAutomationRunWindowMs; const values = await state.getList(runListKey(args.automationId)); const entries = values.flatMap((value) => { const entry = runEntrySchema.safeParse(value); @@ -92,8 +108,8 @@ export async function admitEventAutomationRun(args: { entries.forEach((entry, index) => { if ("reset" in entry) lastReset = index; }); - // A full list whose oldest entry is from this hour may have dropped runs - // from this hour. Then the hour is full, until a reset entry. + // A full list whose oldest entry is from this window may have dropped runs + // from this window. Then the window is full, until a reset entry. const oldest = entries[0]; const full = lastReset === -1 && @@ -108,12 +124,13 @@ export async function admitEventAutomationRun(args: { runKeys.add(entry.id); } if (entry.id === args.runKey) { - return full || runKeys.size > botConfig.maxEventAutomationRunsPerHour + return full || runKeys.size > botConfig.maxEventAutomationRuns ? "limited" : "allowed"; } } - // More Events than the burst room arrived after this one, so the hour is full. + // More Events than the burst room arrived after this one, so the window is + // full. return "limited"; } diff --git a/packages/junior/tests/component/config/chat-config.test.ts b/packages/junior/tests/component/config/chat-config.test.ts index f35d888304..e47db59bf5 100644 --- a/packages/junior/tests/component/config/chat-config.test.ts +++ b/packages/junior/tests/component/config/chat-config.test.ts @@ -536,9 +536,10 @@ describe("chat config", () => { expect(botConfig.maxConsecutiveAutomatedTurns).toBe(25); }); - it("sets max event automation runs per hour from core config", async () => { + it("sets the event automation run limit from core config", async () => { const { botConfig } = await loadConfig(); - expect(botConfig.maxEventAutomationRunsPerHour).toBe(10); + expect(botConfig.maxEventAutomationRuns).toBe(10); + expect(botConfig.eventAutomationRunWindowMs).toBe(60_000); }); it("uses default AGENT_TURN_TIMEOUT_MS when env var is unset", async () => { diff --git a/packages/junior/tests/integration/event-automation-blocking.test.ts b/packages/junior/tests/integration/event-automation-blocking.test.ts index 818200b593..4d89a8f13f 100644 --- a/packages/junior/tests/integration/event-automation-blocking.test.ts +++ b/packages/junior/tests/integration/event-automation-blocking.test.ts @@ -6,7 +6,6 @@ import { migrateSchema } from "@/chat/conversations/sql/migrations"; import { ingestEventAutomations } from "@/chat/event-automations/ingest"; import { getEventAutomation } from "@/chat/event-automations/store"; import { botConfig } from "@/chat/config"; -import { EVENT_AUTOMATION_RUN_WINDOW_MS } from "@/chat/event-automations/run-limit"; import { disconnectStateAdapter } from "@/chat/state/adapter"; import { createDeleteEventAutomationTool } from "@/chat/tools/delete-event-automation"; import { createUpdateEventAutomationTool } from "@/chat/tools/update-event-automation"; @@ -163,7 +162,7 @@ describe("event automation blocking", () => { queue, teamId, }); - const maxRuns = botConfig.maxEventAutomationRunsPerHour; + const maxRuns = botConfig.maxEventAutomationRuns; const fill = async (prefix: string, count: number, nowMs: number) => { for (let index = 0; index < count; index += 1) { expect(await ingest(`${prefix}-${index}`, nowMs)).toEqual({ @@ -172,10 +171,10 @@ describe("event automation blocking", () => { } }; - const firstHourMs = Date.now(); - await fill("github:burst", maxRuns, firstHourMs); + const firstWindowMs = Date.now(); + await fill("github:burst", maxRuns, firstWindowMs); // A provider retry of a counted Event does not count again. - expect(await ingest("github:burst-0", firstHourMs)).toEqual({ + expect(await ingest("github:burst-0", firstWindowMs)).toEqual({ dispatched: 0, }); expect(await getEventAutomation(db, automation.id)).toMatchObject({ @@ -183,19 +182,22 @@ describe("event automation blocking", () => { }); // Old runs leave the rolling window. - const nextHourMs = firstHourMs + EVENT_AUTOMATION_RUN_WINDOW_MS + 1; - await fill("github:next", maxRuns, nextHourMs); + const nextWindowMs = + firstWindowMs + botConfig.eventAutomationRunWindowMs + 1; + await fill("github:next", maxRuns, nextWindowMs); // A second automation on the same trigger has its own count. const { automation: sibling } = await createTask( "Post a summary.", "sibling-call", ); - expect(await ingest("github:over", nextHourMs)).toEqual({ dispatched: 1 }); + expect(await ingest("github:over", nextWindowMs)).toEqual({ + dispatched: 1, + }); expect(await getEventAutomation(db, automation.id)).toMatchObject({ status: "blocked", statusReason: expect.stringContaining( - `limit of ${maxRuns} runs per hour`, + `limit of ${maxRuns} runs per minute`, ), }); expect(await getEventAutomation(db, sibling.id)).toMatchObject({ @@ -209,7 +211,7 @@ describe("event automation blocking", () => { automationId: automation.id, status: "active", }); - expect(await ingest("github:after-resume", nextHourMs)).toEqual({ + expect(await ingest("github:after-resume", nextWindowMs)).toEqual({ dispatched: 2, }); @@ -219,13 +221,13 @@ describe("event automation blocking", () => { automationId, }); await remove(sibling.id); - await fill("github:refill", maxRuns - 1, nextHourMs); + await fill("github:refill", maxRuns - 1, nextWindowMs); await remove(automation.id); const { automation: revived } = await createTask( "Add the release-train label.", ); expect(revived.id).toBe(automation.id); - expect(await ingest("github:after-revive", nextHourMs)).toEqual({ + expect(await ingest("github:after-revive", nextWindowMs)).toEqual({ dispatched: 1, }); // Filling the window takes about three times the cap in sequential ingests. diff --git a/packages/junior/tests/unit/app-model-config.test.ts b/packages/junior/tests/unit/app-model-config.test.ts index d423a48afa..f49077f6aa 100644 --- a/packages/junior/tests/unit/app-model-config.test.ts +++ b/packages/junior/tests/unit/app-model-config.test.ts @@ -77,7 +77,7 @@ describe("createApp profiles", () => { fastModelId: "anthropic/claude-haiku-4.5", limits: { contextWindowTokens: 2_000, - maxEventAutomationRunsPerHour: 2, + eventAutomationRunLimit: { maxRuns: 2, windowMs: 60 * 60 * 1000 }, maxToolCallsPerTurn: 3, }, slack: { @@ -90,7 +90,8 @@ describe("createApp profiles", () => { contextWindowTokens: 2_000, crossActorMidRunMode: "steer", defaultProfile: "gpt-5", - maxEventAutomationRunsPerHour: 2, + eventAutomationRunWindowMs: 60 * 60 * 1000, + maxEventAutomationRuns: 2, maxToolCallsPerTurn: 3, }); @@ -101,8 +102,8 @@ describe("createApp profiles", () => { crossActorMidRunMode: originalBotConfig.crossActorMidRunMode, defaultProfile: originalBotConfig.defaultProfile, fastModelId: originalBotConfig.fastModelId, - maxEventAutomationRunsPerHour: - originalBotConfig.maxEventAutomationRunsPerHour, + eventAutomationRunWindowMs: originalBotConfig.eventAutomationRunWindowMs, + maxEventAutomationRuns: originalBotConfig.maxEventAutomationRuns, maxToolCallsPerTurn: originalBotConfig.maxToolCallsPerTurn, profiles: originalBotConfig.profiles, });