fix(kb): disable connectors after repeated sync failures (#4046)

* fix(kb): improve error logging when connector token resolution fails The generic "Failed to obtain access token" error hid the actual root cause. Now logs credentialId, userId, authMode, and provider to help diagnose token refresh failures in trigger.dev. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat(kb): disable connectors after 10 consecutive sync failures Connectors that fail 10 times in a row are set to 'disabled' status, stopping the cron from scheduling further syncs. The UI shows an alert triangle with a reconnect banner. Users can re-enable via the play button or by reconnecting their account, which resets failures. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix(kb): disable sync button for disabled connectors, use amber badge variant Sync button should be disabled when connector is in disabled state to guide users toward reconnecting first. Badge variant changed from red to amber to match the warning banner styling. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix(kb): address PR review comments for disabled connector feature - Use `=== undefined` instead of falsy check for nextSyncAt to preserve explicit null (manual sync only) when syncIntervalMinutes is 0 - Gate Reconnect button on serviceId/providerId so it only renders for OAuth connectors; show appropriate copy for API key connectors Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix(kb): move resolveAccessToken inside try/catch for circuit-breaker coverage Token resolution failures (e.g. revoked OAuth tokens) were thrown before the try/catch block, bypassing consecutiveFailures tracking entirely. Also removes dead `if (refreshed)` guards at mid-sync refresh sites since resolveAccessToken now always returns a string or throws. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix(kb): remove dead interval branch when re-enabling connector When `updates.nextSyncAt === undefined`, syncIntervalMinutes was not in the request, so `parsed.data.syncIntervalMinutes` is always undefined. Simplify to just schedule an immediate sync — the sync engine sets the proper nextSyncAt based on the connector's DB interval after completion. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
simstudioai · waleedlatif1 · Apr 8, 2026 · Apr 8, 2026 · Apr 8, 2026 · Apr 8, 2026
commit 04c905722964c65481d7feb5976a2f336ce0b1d1
diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts
@@ -222,6 +222,13 @@ export async function PATCH(request: NextRequest, { params }: RouteParams) {
     }
     if (parsed.data.status !== undefined) {
       updates.status = parsed.data.status
+      if (parsed.data.status === 'active') {
+        updates.consecutiveFailures = 0
+        updates.lastSyncError = null
+        if (updates.nextSyncAt === undefined) {
+          updates.nextSyncAt = new Date()
+        }
+      }
     }
 
     await db

diff --git a/...rkspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx b/...rkspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx
@@ -5,6 +5,7 @@ import { createLogger } from '@sim/logger'
 import { format, formatDistanceToNow, isPast } from 'date-fns'
 import {
   AlertCircle,
+  AlertTriangle,
   CheckCircle2,
   ChevronDown,
   Loader2,
@@ -66,6 +67,7 @@ const STATUS_CONFIG = {
   syncing: { label: 'Syncing', variant: 'amber' as const },
   error: { label: 'Error', variant: 'red' as const },
   paused: { label: 'Paused', variant: 'gray' as const },
+  disabled: { label: 'Disabled', variant: 'amber' as const },
 } as const
 
 export function ConnectorsSection({
@@ -159,7 +161,10 @@ export function ConnectorsSection({
           knowledgeBaseId,
           connectorId: connector.id,
           updates: {
-            status: connector.status === 'paused' ? 'active' : 'paused',
+            status:
+              connector.status === 'paused' || connector.status === 'disabled'
+                ? 'active'
+                : 'paused',
           },
         },
         {
@@ -352,7 +357,12 @@ function ConnectorCard({
     <div className='rounded-lg border border-[var(--border-1)]'>
       <div className='flex items-center justify-between px-3 py-2.5'>
         <div className='flex items-center gap-2.5'>
-          {Icon && <Icon className='h-5 w-5 flex-shrink-0' />}
+          <div className='relative flex-shrink-0'>
+            {Icon && <Icon className='h-5 w-5' />}
+            {connector.status === 'disabled' && (
+              <AlertTriangle className='-right-1 -top-1 absolute h-3 w-3 text-amber-500' />
+            )}
+          </div>
           <div className='flex flex-col gap-0.5'>
             <div className='flex items-center gap-2'>
               <span className='flex items-center gap-1.5 font-medium text-[var(--text-primary)] text-small'>
@@ -407,7 +417,12 @@ function ConnectorCard({
                     variant='ghost'
                     className='h-7 w-7 p-0'
                     onClick={onSync}
-                    disabled={connector.status === 'syncing' || isSyncPending || syncCooldown}
+                    disabled={
+                      connector.status === 'syncing' ||
+                      connector.status === 'disabled' ||
+                      isSyncPending ||
+                      syncCooldown
+                    }
                   >
                     <RefreshCw
                       className={cn(
@@ -441,15 +456,17 @@ function ConnectorCard({
                   >
                     {isUpdating ? (
                       <Loader2 className='h-3.5 w-3.5 animate-spin' />
-                    ) : connector.status === 'paused' ? (
+                    ) : connector.status === 'paused' || connector.status === 'disabled' ? (
                       <Play className='h-3.5 w-3.5' />
                     ) : (
                       <Pause className='h-3.5 w-3.5' />
                     )}
                   </Button>
                 </Tooltip.Trigger>
                 <Tooltip.Content>
-                  {connector.status === 'paused' ? 'Resume' : 'Pause'}
+                  {connector.status === 'paused' || connector.status === 'disabled'
+                    ? 'Resume'
+                    : 'Pause'}
                 </Tooltip.Content>
               </Tooltip.Root>
 
@@ -481,7 +498,46 @@ function ConnectorCard({
         </div>
       </div>
 
-      {missingScopes.length > 0 && (
+      {connector.status === 'disabled' && (
+        <div className='border-[var(--border-1)] border-t px-3 py-2'>
+          <div className='flex flex-col gap-1 rounded-sm border border-amber-200 bg-amber-50 px-2 py-1.5 dark:border-amber-900 dark:bg-amber-950'>
+            <div className='flex items-center gap-1.5 font-medium text-amber-800 text-caption dark:text-amber-200'>
+              <AlertTriangle className='h-3 w-3 flex-shrink-0' />
+              Connector disabled after repeated sync failures
+            </div>
+            <p className='text-amber-700 text-micro dark:text-amber-300'>
+              Syncing has been paused due to {connector.consecutiveFailures} consecutive failures.
+              {serviceId
+                ? ' Reconnect your account to resume syncing.'
+                : ' Use the resume button to re-enable syncing.'}
+            </p>
+            {canEdit && serviceId && providerId && (
+              <Button
+                variant='active'
+                onClick={() => {
+                  if (connector.credentialId) {
+                    writeOAuthReturnContext({
+                      origin: 'kb-connectors',
+                      knowledgeBaseId,
+                      displayName: connectorDef?.name ?? connector.connectorType,
+                      providerId: providerId!,
+                      preCount: credentials?.length ?? 0,
+                      workspaceId,
+                      requestedAt: Date.now(),
+                    })
+                  }
+                  setShowOAuthModal(true)
+                }}
+                className='w-full px-2 py-1 font-medium text-caption'
+              >
+                Reconnect
+              </Button>
+            )}
+          </div>
+        </div>
+      )}
+
+      {missingScopes.length > 0 && connector.status !== 'disabled' && (
         <div className='border-[var(--border-1)] border-t px-3 py-2'>
           <div className='flex flex-col gap-1 rounded-sm border bg-[var(--surface-2)] px-2 py-1.5'>
             <div className='flex items-center font-medium text-caption'>

diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts
@@ -12,7 +12,7 @@ export interface ConnectorData {
   sourceConfig: Record<string, unknown>
   syncMode: string
   syncIntervalMinutes: number
-  status: 'active' | 'paused' | 'syncing' | 'error'
+  status: 'active' | 'paused' | 'syncing' | 'error' | 'disabled'
   lastSyncAt: string | null
   lastSyncError: string | null
   lastSyncDocCount: number | null

diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts
@@ -46,6 +46,7 @@ const MAX_PAGES = 500
 const MAX_SAFE_TITLE_LENGTH = 200
 const STALE_PROCESSING_MINUTES = 45
 const RETRY_WINDOW_DAYS = 7
+const MAX_CONSECUTIVE_FAILURES = 10
 
 /** Sanitizes a document title for use in S3 storage keys. */
 function sanitizeStorageTitle(title: string): string {
@@ -230,7 +231,7 @@ async function resolveAccessToken(
   connector: { credentialId: string | null; encryptedApiKey: string | null },
   connectorConfig: { auth: ConnectorAuthConfig },
   userId: string
-): Promise<string | null> {
+): Promise<string> {
   if (connectorConfig.auth.mode === 'apiKey') {
     if (!connector.encryptedApiKey) {
       throw new Error('API key connector is missing encrypted API key')
@@ -243,11 +244,22 @@ async function resolveAccessToken(
     throw new Error('OAuth connector is missing credential ID')
   }
 
-  return refreshAccessTokenIfNeeded(
-    connector.credentialId,
-    userId,
-    `sync-${connector.credentialId}`
-  )
+  const requestId = `sync-${connector.credentialId}`
+  const token = await refreshAccessTokenIfNeeded(connector.credentialId, userId, requestId)
+
+  if (!token) {
+    logger.error(`[${requestId}] refreshAccessTokenIfNeeded returned null`, {
+      credentialId: connector.credentialId,
+      userId,
+      authMode: connectorConfig.auth.mode,
+      authProvider: connectorConfig.auth.provider,
+    })
+    throw new Error(
+      `Failed to obtain access token for credential ${connector.credentialId} (provider: ${connectorConfig.auth.provider})`
+    )
+  }
+
+  return token
 }
 
 /**
@@ -305,12 +317,6 @@ export async function executeSync(
   const userId = kbRows[0].userId
   const sourceConfig = connector.sourceConfig as Record<string, unknown>
 
-  let accessToken = await resolveAccessToken(connector, connectorConfig, userId)
-
-  if (!accessToken) {
-    throw new Error('Failed to obtain access token')
-  }
-
   const lockResult = await db
     .update(knowledgeConnector)
     .set({ status: 'syncing', updatedAt: new Date() })
@@ -341,6 +347,8 @@ export async function executeSync(
   let syncExitedCleanly = false
 
   try {
+    let accessToken = await resolveAccessToken(connector, connectorConfig, userId)
+
     const externalDocs: ExternalDocument[] = []
     let cursor: string | undefined
     let hasMore = true
@@ -357,8 +365,7 @@ export async function executeSync(
 
     for (let pageNum = 0; hasMore && pageNum < MAX_PAGES; pageNum++) {
       if (pageNum > 0 && connectorConfig.auth.mode === 'oauth') {
-        const refreshed = await resolveAccessToken(connector, connectorConfig, userId)
-        if (refreshed) accessToken = refreshed
+        accessToken = await resolveAccessToken(connector, connectorConfig, userId)
       }
 
       const page = await connectorConfig.listDocuments(
@@ -496,8 +503,7 @@ export async function executeSync(
 
       if (deferredOps.length > 0) {
         if (connectorConfig.auth.mode === 'oauth') {
-          const refreshed = await resolveAccessToken(connector, connectorConfig, userId)
-          if (refreshed) accessToken = refreshed
+          accessToken = await resolveAccessToken(connector, connectorConfig, userId)
         }
 
         const hydrated = await Promise.allSettled(
@@ -789,15 +795,25 @@ export async function executeSync(
 
       const now = new Date()
       const failures = (connector.consecutiveFailures ?? 0) + 1
+      const disabled = failures >= MAX_CONSECUTIVE_FAILURES
       const backoffMinutes = Math.min(failures * 30, 1440)
-      const nextSync = new Date(now.getTime() + backoffMinutes * 60 * 1000)
+      const nextSync = disabled ? null : new Date(now.getTime() + backoffMinutes * 60 * 1000)
+
+      if (disabled) {
+        logger.warn('Connector disabled after repeated failures', {
+          connectorId,
+          consecutiveFailures: failures,
+        })
+      }
 
       await db
         .update(knowledgeConnector)
         .set({
-          status: 'error',
+          status: disabled ? 'disabled' : 'error',
           lastSyncAt: now,
-          lastSyncError: errorMessage,
+          lastSyncError: disabled
+            ? 'Connector disabled after repeated sync failures. Please reconnect.'
+            : errorMessage,
           nextSyncAt: nextSync,
           consecutiveFailures: failures,
           updatedAt: now,