fix(kb): disable connectors after repeated sync failures

apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts

@@ -222,6 +222,13 @@ export async function PATCH(request: NextRequest, { params }: RouteParams) {
     }
     if (parsed.data.status !== undefined) {
       updates.status = parsed.data.status
+      if (parsed.data.status === 'active') {
+        updates.consecutiveFailures = 0
+        updates.lastSyncError = null
+        if (updates.nextSyncAt === undefined) {
+          updates.nextSyncAt = new Date()
+        }
+      }
     }
 
     await db

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx

@@ -5,6 +5,7 @@ import { createLogger } from '@sim/logger'
 import { format, formatDistanceToNow, isPast } from 'date-fns'
 import {
   AlertCircle,
+  AlertTriangle,
   CheckCircle2,
   ChevronDown,
   Loader2,
@@ -66,6 +67,7 @@ const STATUS_CONFIG = {
   syncing: { label: 'Syncing', variant: 'amber' as const },
   error: { label: 'Error', variant: 'red' as const },
   paused: { label: 'Paused', variant: 'gray' as const },
+  disabled: { label: 'Disabled', variant: 'amber' as const },
 } as const
 
 export function ConnectorsSection({
@@ -159,7 +161,10 @@ export function ConnectorsSection({
           knowledgeBaseId,
           connectorId: connector.id,
           updates: {
-            status: connector.status === 'paused' ? 'active' : 'paused',
+            status:
+              connector.status === 'paused' || connector.status === 'disabled'
+                ? 'active'
+                : 'paused',
           },
         },
         {
@@ -352,7 +357,12 @@ function ConnectorCard({
     <div className='rounded-lg border border-[var(--border-1)]'>
       <div className='flex items-center justify-between px-3 py-2.5'>
         <div className='flex items-center gap-2.5'>
-          {Icon && <Icon className='h-5 w-5 flex-shrink-0' />}
+          <div className='relative flex-shrink-0'>
+            {Icon && <Icon className='h-5 w-5' />}
+            {connector.status === 'disabled' && (
+              <AlertTriangle className='-right-1 -top-1 absolute h-3 w-3 text-amber-500' />
+            )}
+          </div>
           <div className='flex flex-col gap-0.5'>
             <div className='flex items-center gap-2'>
               <span className='flex items-center gap-1.5 font-medium text-[var(--text-primary)] text-small'>
@@ -407,7 +417,12 @@ function ConnectorCard({
                     variant='ghost'
                     className='h-7 w-7 p-0'
                     onClick={onSync}
-                    disabled={connector.status === 'syncing' || isSyncPending || syncCooldown}
+                    disabled={
+                      connector.status === 'syncing' ||
+                      connector.status === 'disabled' ||
+                      isSyncPending ||
+                      syncCooldown
+                    }
                   >
                     <RefreshCw
                       className={cn(
@@ -441,15 +456,17 @@ function ConnectorCard({
                   >
                     {isUpdating ? (
                       <Loader2 className='h-3.5 w-3.5 animate-spin' />
-                    ) : connector.status === 'paused' ? (
+                    ) : connector.status === 'paused' || connector.status === 'disabled' ? (
                       <Play className='h-3.5 w-3.5' />
                     ) : (
                       <Pause className='h-3.5 w-3.5' />
                     )}
                   </Button>
                 </Tooltip.Trigger>
                 <Tooltip.Content>
-                  {connector.status === 'paused' ? 'Resume' : 'Pause'}
+                  {connector.status === 'paused' || connector.status === 'disabled'
+                    ? 'Resume'
+                    : 'Pause'}
                 </Tooltip.Content>
               </Tooltip.Root>
 
@@ -481,7 +498,46 @@ function ConnectorCard({
         </div>
       </div>
 
-      {missingScopes.length > 0 && (
+      {connector.status === 'disabled' && (
+        <div className='border-[var(--border-1)] border-t px-3 py-2'>
+          <div className='flex flex-col gap-1 rounded-sm border border-amber-200 bg-amber-50 px-2 py-1.5 dark:border-amber-900 dark:bg-amber-950'>
+            <div className='flex items-center gap-1.5 font-medium text-amber-800 text-caption dark:text-amber-200'>
+              <AlertTriangle className='h-3 w-3 flex-shrink-0' />
+              Connector disabled after repeated sync failures
+            </div>
+            <p className='text-amber-700 text-micro dark:text-amber-300'>
+              Syncing has been paused due to {connector.consecutiveFailures} consecutive failures.
+              {serviceId
+                ? ' Reconnect your account to resume syncing.'
+                : ' Use the resume button to re-enable syncing.'}
+            </p>
+            {canEdit && serviceId && providerId && (
+              <Button
+                variant='active'
+                onClick={() => {
+                  if (connector.credentialId) {
+                    writeOAuthReturnContext({
+                      origin: 'kb-connectors',
+                      knowledgeBaseId,
+                      displayName: connectorDef?.name ?? connector.connectorType,
+                      providerId: providerId!,
+                      preCount: credentials?.length ?? 0,
+                      workspaceId,
+                      requestedAt: Date.now(),
+                    })
+                  }
+                  setShowOAuthModal(true)
+                }}
+                className='w-full px-2 py-1 font-medium text-caption'
+              >
+                Reconnect
+              </Button>
+            )}
+          </div>
+        </div>
+      )}
+
+      {missingScopes.length > 0 && connector.status !== 'disabled' && (
         <div className='border-[var(--border-1)] border-t px-3 py-2'>
           <div className='flex flex-col gap-1 rounded-sm border bg-[var(--surface-2)] px-2 py-1.5'>
             <div className='flex items-center font-medium text-caption'>

apps/sim/hooks/queries/kb/connectors.ts

@@ -12,7 +12,7 @@ export interface ConnectorData {
   sourceConfig: Record<string, unknown>
   syncMode: string
   syncIntervalMinutes: number
-  status: 'active' | 'paused' | 'syncing' | 'error'
+  status: 'active' | 'paused' | 'syncing' | 'error' | 'disabled'
   lastSyncAt: string | null
   lastSyncError: string | null
   lastSyncDocCount: number | null

apps/sim/lib/knowledge/connectors/sync-engine.ts

@@ -46,6 +46,7 @@ const MAX_PAGES = 500
 const MAX_SAFE_TITLE_LENGTH = 200
 const STALE_PROCESSING_MINUTES = 45
 const RETRY_WINDOW_DAYS = 7
+const MAX_CONSECUTIVE_FAILURES = 10
 
 /** Sanitizes a document title for use in S3 storage keys. */
 function sanitizeStorageTitle(title: string): string {
@@ -230,7 +231,7 @@ async function resolveAccessToken(
   connector: { credentialId: string | null; encryptedApiKey: string | null },
   connectorConfig: { auth: ConnectorAuthConfig },
   userId: string
-): Promise<string | null> {
+): Promise<string> {
   if (connectorConfig.auth.mode === 'apiKey') {
     if (!connector.encryptedApiKey) {
       throw new Error('API key connector is missing encrypted API key')
@@ -243,11 +244,22 @@ async function resolveAccessToken(
     throw new Error('OAuth connector is missing credential ID')
   }
 
-  return refreshAccessTokenIfNeeded(
-    connector.credentialId,
-    userId,
-    `sync-${connector.credentialId}`
-  )
+  const requestId = `sync-${connector.credentialId}`
+  const token = await refreshAccessTokenIfNeeded(connector.credentialId, userId, requestId)
+
+  if (!token) {
+    logger.error(`[${requestId}] refreshAccessTokenIfNeeded returned null`, {
+      credentialId: connector.credentialId,
+      userId,
+      authMode: connectorConfig.auth.mode,
+      authProvider: connectorConfig.auth.provider,
+    })
+    throw new Error(
+      `Failed to obtain access token for credential ${connector.credentialId} (provider: ${connectorConfig.auth.provider})`
+    )
+  }
+
+  return token
 }
 
 /**
@@ -305,12 +317,6 @@ export async function executeSync(
   const userId = kbRows[0].userId
   const sourceConfig = connector.sourceConfig as Record<string, unknown>
 
-  let accessToken = await resolveAccessToken(connector, connectorConfig, userId)
-
-  if (!accessToken) {
-    throw new Error('Failed to obtain access token')
-  }
-
   const lockResult = await db
     .update(knowledgeConnector)
     .set({ status: 'syncing', updatedAt: new Date() })
@@ -341,6 +347,8 @@ export async function executeSync(
   let syncExitedCleanly = false
 
   try {
+    let accessToken = await resolveAccessToken(connector, connectorConfig, userId)
+
     const externalDocs: ExternalDocument[] = []
     let cursor: string | undefined
     let hasMore = true
@@ -357,8 +365,7 @@ export async function executeSync(
 
     for (let pageNum = 0; hasMore && pageNum < MAX_PAGES; pageNum++) {
       if (pageNum > 0 && connectorConfig.auth.mode === 'oauth') {
-        const refreshed = await resolveAccessToken(connector, connectorConfig, userId)
-        if (refreshed) accessToken = refreshed
+        accessToken = await resolveAccessToken(connector, connectorConfig, userId)
       }
 
       const page = await connectorConfig.listDocuments(
@@ -496,8 +503,7 @@ export async function executeSync(
 
       if (deferredOps.length > 0) {
         if (connectorConfig.auth.mode === 'oauth') {
-          const refreshed = await resolveAccessToken(connector, connectorConfig, userId)
-          if (refreshed) accessToken = refreshed
+          accessToken = await resolveAccessToken(connector, connectorConfig, userId)
         }
 
         const hydrated = await Promise.allSettled(
@@ -789,15 +795,25 @@ export async function executeSync(
 
       const now = new Date()
       const failures = (connector.consecutiveFailures ?? 0) + 1
+      const disabled = failures >= MAX_CONSECUTIVE_FAILURES
       const backoffMinutes = Math.min(failures * 30, 1440)
-      const nextSync = new Date(now.getTime() + backoffMinutes * 60 * 1000)
+      const nextSync = disabled ? null : new Date(now.getTime() + backoffMinutes * 60 * 1000)
+
+      if (disabled) {
+        logger.warn('Connector disabled after repeated failures', {
+          connectorId,
+          consecutiveFailures: failures,
+        })
+      }
 
       await db
         .update(knowledgeConnector)
         .set({
-          status: 'error',
+          status: disabled ? 'disabled' : 'error',
           lastSyncAt: now,
-          lastSyncError: errorMessage,
+          lastSyncError: disabled
+            ? 'Connector disabled after repeated sync failures. Please reconnect.'
+            : errorMessage,
           nextSyncAt: nextSync,
           consecutiveFailures: failures,
           updatedAt: now,