fix(mothership): answer a paused Chat workflow tool still running when its form was submitted before Chat read the pause

Author: waleedlatif1

apps/sim/lib/mothership/request/tools/client.ts

@@ -6,6 +6,7 @@ import {
   type AsyncTerminalCompletionSnapshot,
   getTerminalConfirmationStatus,
   isAsyncTerminalConfirmationStatus,
+  isTerminalAsyncStatus,
 } from '@/lib/mothership/async-runs/lifecycle'
 import { replaceTerminalAsyncToolCallResult } from '@/lib/mothership/async-runs/repository'
 import { MothershipStreamV1ToolOutcome } from '@/lib/mothership/generated/mothership-stream-v1'
@@ -42,6 +43,7 @@ import {
   getHumanPauseResumeUrl,
   getTrustedWorkflowToolExecution,
   getWorkflowExecutionCopilotToolCallId,
+  getWorkflowExecutionLogStatus,
 } from '@/lib/workflows/executor/execution-state'
 import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
@@ -376,21 +378,33 @@ export async function waitForWorkflowToolCompletion({
       completion.data.status === WORKFLOW_TOOL_PAUSED_STATUS
     ) {
       // Re-read rather than trust the confirmation; a run resumed since then reads as finished.
-      const [resumeUrl, boundToolCallId] = await Promise.all([
+      const [resumeUrl, boundToolCallId, logStatus] = await Promise.all([
         getHumanPauseResumeUrl(executionId, workflowId),
         getWorkflowExecutionCopilotToolCallId(executionId, workflowId),
+        getWorkflowExecutionLogStatus(executionId, workflowId),
       ]).catch((error) => {
         logger.warn('Failed to read paused workflow tool execution', {
           toolCallId,
           workflowId,
           executionId,
           error: getErrorMessage(error),
         })
-        return [null, undefined] as const
+        return [null, undefined, undefined] as const
       })
-      if (resumeUrl && boundToolCallId === toolCallId) {
-        toolRegistry?.markIncomplete('client-tool-completion-deferred')
-        return buildPausedWorkflowToolCompletion(workflowId, executionId, resumeUrl)
+      if (boundToolCallId === toolCallId) {
+        if (resumeUrl) {
+          toolRegistry?.markIncomplete('client-tool-completion-deferred')
+          return buildPausedWorkflowToolCompletion(workflowId, executionId, resumeUrl)
+        }
+        // The person answered before this read and the resumed run has no result yet.
+        if (logStatus !== undefined && !isTerminalAsyncStatus(logStatus)) {
+          toolRegistry?.markIncomplete('client-tool-completion-deferred')
+          return structuralWorkflowCompletion(
+            ASYNC_TOOL_CONFIRMATION_STATUS.background,
+            workflowId,
+            executionId
+          )
+        }
       }
     }
 

apps/sim/lib/mothership/request/tools/workflow-client-settlement.integration.ts

@@ -530,6 +530,44 @@ describe.runIf(Boolean(redisUrl))('settled client-claimed workflow tools', () =>
     expect(await toolRow(toolCallId)).toMatchObject({ status: 'completed', claimedBy: null })
   })
 
+  it('answers a paused call still running when the person submitted before Chat read the pause', async () => {
+    const { toolCallId, executionId } = await claimedAndPausedUnreported('human')
+    await reportSettledClientWorkflowTool({ toolCallId, executionId, workflowId })
+    // The resume took the pause point and the resumed run is going.
+    const [pausedRow] = await db
+      .select({ id: pausedExecutions.id, pausePoints: pausedExecutions.pausePoints })
+      .from(pausedExecutions)
+      .where(eq(pausedExecutions.executionId, executionId))
+    await db
+      .update(pausedExecutions)
+      .set({
+        pausePoints: {
+          [APPROVAL_CONTEXT_ID]: {
+            ...(pausedRow.pausePoints as Record<string, Record<string, unknown>>)[
+              APPROVAL_CONTEXT_ID
+            ],
+            resumeStatus: 'resuming',
+          },
+        },
+      })
+      .where(eq(pausedExecutions.id, pausedRow.id))
+    await db
+      .update(workflowExecutionLogs)
+      .set({ status: 'running' })
+      .where(eq(workflowExecutionLogs.executionId, executionId))
+
+    const completion = await waitForWorkflowToolCompletion({
+      toolCallId,
+      workflowId,
+      timeoutMs: WAIT_MS,
+    })
+
+    expect(completion).toMatchObject({
+      status: 'background',
+      data: { status: 'running', workflowId, executionId },
+    })
+  })
+
   it("never hands a call the link to another call's paused run", async () => {
     const other = await claimedAndPausedUnreported('human')
     const { toolCallId } = await claimed()