feat(slack-agent): gate thread follow-ups behind a relevance classifier

apps/slack-agent/.env.local.example

@@ -7,6 +7,10 @@ OPENROUTER_API_KEY=
 # Optional overrides (defaults live in agent/agent.ts):
 # OPENROUTER_MODEL=openai/gpt-5.6-luna   # must stream structured tool calls — see README Notes
 # OPENROUTER_CONTEXT_WINDOW=400000
+# Model for the thread follow-up relevance gate (one tiny RESPOND/PASS call per
+# un-mentioned thread reply — agent/lib/follow-up-relevance.ts). A small, fast
+# model is ideal here; unset, it follows OPENROUTER_MODEL.
+# OPENROUTER_GATE_MODEL=
 
 # ── Slack (self-managed, multi-workspace app — no Vercel Connect) ───────────
 # Signing secret is per-app/static: it HMAC-verifies every inbound webhook.

apps/slack-agent/README.md

@@ -301,6 +301,14 @@ before eve's 200, so anything it fetches is spent out of Slack's ~3s delivery bu
 is parse-only and optimistic; the mention handler confirms it afterwards, against the thread it loads
 for turn context anyway.
 
+Being engaged in a thread is necessary but not sufficient: a confirmed follow-up still passes a
+**relevance gate** — one tiny RESPOND/PASS classifier call (`agent/lib/follow-up-relevance.ts`,
+model via `OPENROUTER_GATE_MODEL`) asking whether the reply is actually directed at the bot. Humans
+talking to each other in a thread the bot answered once no longer trigger a turn for every message;
+the pass is silent (no reaction, no reply), and the next message that _is_ for the bot gets
+answered. Real `@mentions` and DMs skip the gate — an explicit address answers the question itself —
+and the gate fails open, so a classifier outage means an extra answer, never a silent drop.
+
 Changing a request URL does **not** require reinstalling the app; only changing _scopes_ does. (If
 you did edit scopes, the sidebar shows a yellow reinstall banner — follow it, and note that
 reinstalling issues a **new** `SLACK_BOT_TOKEN` that you must copy back into Railway.)

apps/slack-agent/agent/channels/slack.ts

@@ -5,6 +5,7 @@ import { describeActions, truncateTypingStatus } from "#lib/action-status.js"
 import { botUserIdForTeam, rememberBotUserId } from "#lib/bot-identity.js"
 import { loadChannelContext } from "#lib/channel-context.js"
 import { notifyThreadDisengagement } from "#lib/disengage-notice.js"
+import { judgeFollowUpRelevance } from "#lib/follow-up-relevance.js"
 import { resolveBotToken, verifySlackV0Signature, type SlackTokenContext } from "#lib/maple.js"
 import { emitAgentLog } from "#lib/telemetry-log.js"
 import { formatThreadContext, loadThreadMessages } from "#lib/thread-context.js"
@@ -131,8 +132,9 @@ const webhookVerifier: SlackWebhookVerifier = async (request, body) => {
  * follow-up decision lives here rather than in the verifier: this handler
  * already loads the thread, and here it can take as long as it needs. It must
  * not otherwise throw: eve drops the whole mention when this handler does, so
- * both loads degrade to no context instead — the one deliberate throw is the
- * disengagement drop below, which is exactly what that escape hatch is for.
+ * both loads degrade to no context instead — the deliberate throws are the
+ * two follow-up drops below (disengaged / not addressed to the bot), which is
+ * exactly what that escape hatch is for.
  */
 async function dispatchWithConversationContext(
 	ctx: SlackContext,
@@ -202,7 +204,42 @@ async function dispatchWithConversationContext(
 				`Thread follow-up not dispatched (${decision.reason}): the bot is no longer engaged in ${pending.channelId}:${pending.threadTs}.`,
 			)
 		}
-		// Confirmed, so the :eyes: is now a promise we keep.
+		// Engaged is necessary, not sufficient: the bot being part of the thread
+		// says nothing about whether THIS reply is for it. Without this gate every
+		// human reply in an engaged thread — including two people talking to each
+		// other — dispatched a full turn. Runs before the ack and the typing
+		// indicator so a message the bot stays out of gets no reaction at all;
+		// the pass is silent on purpose (the bot is still engaged, and the next
+		// reply that IS for it will be answered). Fails open — see the module.
+		const relevance = await judgeFollowUpRelevance({
+			reply: {
+				text: message.text,
+				markdown: message.markdown,
+				user: message.author?.userId,
+				ts: message.ts,
+				threadTs: message.threadTs,
+				raw: message.raw,
+			},
+			threadMessages,
+			botUserId: pending.botUserId,
+		})
+		if (!relevance.respond) {
+			// Ids only, never text — same rule as the disengagement log above. This
+			// line is the only trace the drop leaves.
+			emitAgentLog("info", "follow_up_not_relevant", {
+				"maple.agent.event": "follow_up_not_relevant",
+				"maple.slack.team_id": pending.teamId,
+				"maple.slack.channel_id": pending.channelId,
+				"maple.slack.thread_ts": pending.threadTs,
+				"maple.slack.message_ts": pending.messageTs,
+			})
+			// Same escape hatch as the disengagement drop: throwing is the only way
+			// to un-dispatch an event eve has already accepted.
+			throw new Error(
+				`Thread follow-up not dispatched (not addressed to the bot): ${pending.channelId}:${pending.threadTs}.`,
+			)
+		}
+		// Confirmed and relevant, so the :eyes: is now a promise we keep.
 		if (pending.ackable) {
 			void acknowledgeMessage({
 				teamId: pending.teamId,

apps/slack-agent/agent/lib/follow-up-relevance.test.ts

@@ -0,0 +1,126 @@
+import { describe, expect, test } from "bun:test"
+import type { SlackThreadMessage } from "eve/channels/slack"
+import {
+	judgeFollowUpRelevance,
+	parseRelevanceVerdict,
+	relevancePrompt,
+	relevanceSystemPrompt,
+	type FollowUpRelevanceInput,
+} from "./follow-up-relevance.js"
+
+const BOT_USER_ID = "U0BOT"
+
+const threadMessage = (overrides: Partial<SlackThreadMessage> = {}): SlackThreadMessage => ({
+	text: "",
+	markdown: "",
+	user: undefined,
+	botId: undefined,
+	ts: "1700000000.000100",
+	threadTs: "1700000000.000100",
+	isMe: false,
+	raw: {},
+	...overrides,
+})
+
+const input = (overrides: Partial<FollowUpRelevanceInput> = {}): FollowUpRelevanceInput => ({
+	reply: {
+		text: "can you check the api service too?",
+		markdown: "can you check the api service too?",
+		user: "U456",
+		ts: "1700000002.000200",
+		threadTs: "1700000000.000100",
+		raw: {},
+	},
+	threadMessages: [
+		threadMessage({ text: "why did this alert fire?", markdown: "why did this alert fire?", user: "U456" }),
+		threadMessage({
+			text: "The error rate on checkout spiked at 14:10.",
+			markdown: "The error rate on checkout spiked at 14:10.",
+			user: BOT_USER_ID,
+			ts: "1700000001.000100",
+		}),
+	],
+	botUserId: BOT_USER_ID,
+	...overrides,
+})
+
+describe("parseRelevanceVerdict", () => {
+	test("plain verdicts, any case", () => {
+		expect(parseRelevanceVerdict("RESPOND")).toBe("respond")
+		expect(parseRelevanceVerdict("pass")).toBe("pass")
+		expect(parseRelevanceVerdict("  Pass.\n")).toBe("pass")
+	})
+
+	test("verdict wrapped in prose still parses", () => {
+		expect(parseRelevanceVerdict("The user is talking to a teammate, so: PASS")).toBe("pass")
+	})
+
+	test("no verdict, or both words, is no answer", () => {
+		expect(parseRelevanceVerdict("")).toBeNull()
+		expect(parseRelevanceVerdict("maybe?")).toBeNull()
+		expect(parseRelevanceVerdict("RESPOND or PASS, hard to say")).toBeNull()
+	})
+
+	test("substrings do not count as verdicts", () => {
+		expect(parseRelevanceVerdict("the password expired")).toBeNull()
+		expect(parseRelevanceVerdict("correspondence")).toBeNull()
+	})
+})
+
+describe("relevancePrompt", () => {
+	test("carries the thread tail and the reply", () => {
+		const prompt = relevancePrompt(input())
+		expect(prompt).toContain("why did this alert fire?")
+		expect(prompt).toContain("can you check the api service too?")
+		expect(prompt).toContain("Newest message:")
+		// The bot's own post is attributed as the agent, other bots are not.
+		expect(prompt).toContain("sender_type: agent")
+	})
+
+	test("says so when the thread was unreadable vs merely empty", () => {
+		expect(relevancePrompt(input({ threadMessages: null }))).toContain("could not be loaded")
+		expect(relevancePrompt(input({ threadMessages: [] }))).toContain("No earlier thread messages")
+	})
+
+	test("system prompt names the bot's user id", () => {
+		expect(relevanceSystemPrompt(BOT_USER_ID)).toContain(`<@${BOT_USER_ID}>`)
+	})
+})
+
+describe("judgeFollowUpRelevance", () => {
+	test("model RESPOND answers the follow-up", async () => {
+		const decision = await judgeFollowUpRelevance(input(), { complete: async () => "RESPOND" })
+		expect(decision).toEqual({ respond: true, reason: "model-respond" })
+	})
+
+	test("model PASS drops it", async () => {
+		const decision = await judgeFollowUpRelevance(input(), { complete: async () => "PASS" })
+		expect(decision).toEqual({ respond: false, reason: "model-pass" })
+	})
+
+	test("a throwing classifier fails open", async () => {
+		const decision = await judgeFollowUpRelevance(input(), {
+			complete: async () => {
+				throw new Error("openrouter down")
+			},
+		})
+		expect(decision).toEqual({ respond: true, reason: "classifier-error" })
+	})
+
+	test("an unparseable verdict fails open", async () => {
+		const decision = await judgeFollowUpRelevance(input(), { complete: async () => "hmm, unclear" })
+		expect(decision).toEqual({ respond: true, reason: "unparseable-verdict" })
+	})
+
+	test("the classifier sees the reply it is judging", async () => {
+		let seen: { system: string; prompt: string } | undefined
+		await judgeFollowUpRelevance(input(), {
+			complete: async ({ system, prompt }) => {
+				seen = { system, prompt }
+				return "PASS"
+			},
+		})
+		expect(seen?.prompt).toContain("can you check the api service too?")
+		expect(seen?.system).toContain("RESPOND or PASS")
+	})
+})