Skip to content

Commit 12be44d

Browse files
waleedlatif1waleedlatif1
committed
fix(knowledge): fix document processing stuck in processing state
1 parent e9c94fa commit 12be44d

File tree

8 files changed

+105
-95
lines changed

8 files changed

+105
-95
lines changed

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -900,7 +900,11 @@ export function KnowledgeBase({
900900
onClick={() => setShowConnectorsModal(true)}
901901
className='flex shrink-0 cursor-pointer items-center gap-1.5 rounded-md px-2 py-1 text-[var(--text-secondary)] text-caption shadow-[inset_0_0_0_1px_var(--border)] transition-colors hover-hover:bg-[var(--surface-3)]'
902902
>
903-
{ConnectorIcon && <ConnectorIcon className='h-[14px] w-[14px]' />}
903+
{connector.status === 'syncing' ? (
904+
<Loader2 className='h-[14px] w-[14px] animate-spin' />
905+
) : (
906+
ConnectorIcon && <ConnectorIcon className='h-[14px] w-[14px]' />
907+
)}
904908
{def?.name || connector.connectorType}
905909
</button>
906910
)

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx‎

Lines changed: 22 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,10 @@ import {
1919
ModalHeader,
2020
Tooltip,
2121
} from '@/components/emcn'
22-
import { useSession } from '@/lib/auth/auth-client'
23-
import { consumeOAuthReturnContext, writeOAuthReturnContext } from '@/lib/credentials/client-state'
24-
import {
25-
getCanonicalScopesForProvider,
26-
getProviderIdFromServiceId,
27-
type OAuthProvider,
28-
} from '@/lib/oauth'
22+
import { consumeOAuthReturnContext } from '@/lib/credentials/client-state'
23+
import { getProviderIdFromServiceId, type OAuthProvider } from '@/lib/oauth'
2924
import { ConnectorSelectorField } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/components/connector-selector-field'
30-
import { OAuthRequiredModal } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/components/oauth-required-modal'
25+
import { ConnectCredentialModal } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/components/connect-credential-modal'
3126
import { getDependsOnFields } from '@/blocks/utils'
3227
import { CONNECTOR_REGISTRY } from '@/connectors/registry'
3328
import type { ConnectorConfig, ConnectorConfigField } from '@/connectors/types'
@@ -69,7 +64,6 @@ export function AddConnectorModal({ open, onOpenChange, knowledgeBaseId }: AddCo
6964
const [searchTerm, setSearchTerm] = useState('')
7065

7166
const { workspaceId } = useParams<{ workspaceId: string }>()
72-
const { data: session } = useSession()
7367
const { mutate: createConnector, isPending: isCreating } = useCreateConnector()
7468

7569
const connectorConfig = selectedType ? CONNECTOR_REGISTRY[selectedType] : null
@@ -263,51 +257,9 @@ export function AddConnectorModal({ open, onOpenChange, knowledgeBaseId }: AddCo
263257
)
264258
}
265259

266-
const handleConnectNewAccount = useCallback(async () => {
267-
if (!connectorConfig || !connectorProviderId || !workspaceId) return
268-
269-
const userName = session?.user?.name
270-
const integrationName = connectorConfig.name
271-
const displayName = userName ? `${userName}'s ${integrationName}` : integrationName
272-
273-
try {
274-
const res = await fetch('/api/credentials/draft', {
275-
method: 'POST',
276-
headers: { 'Content-Type': 'application/json' },
277-
body: JSON.stringify({
278-
workspaceId,
279-
providerId: connectorProviderId,
280-
displayName,
281-
}),
282-
})
283-
if (!res.ok) {
284-
setError('Failed to prepare credential. Please try again.')
285-
return
286-
}
287-
} catch {
288-
setError('Failed to prepare credential. Please try again.')
289-
return
290-
}
291-
292-
writeOAuthReturnContext({
293-
origin: 'kb-connectors',
294-
knowledgeBaseId,
295-
displayName,
296-
providerId: connectorProviderId,
297-
preCount: credentials.length,
298-
workspaceId,
299-
requestedAt: Date.now(),
300-
})
301-
260+
const handleConnectNewAccount = useCallback(() => {
302261
setShowOAuthModal(true)
303-
}, [
304-
connectorConfig,
305-
connectorProviderId,
306-
workspaceId,
307-
session?.user?.name,
308-
knowledgeBaseId,
309-
credentials.length,
310-
])
262+
}, [])
311263

312264
const filteredEntries = useMemo(() => {
313265
const term = searchTerm.toLowerCase().trim()
@@ -590,20 +542,23 @@ export function AddConnectorModal({ open, onOpenChange, knowledgeBaseId }: AddCo
590542
)}
591543
</ModalContent>
592544
</Modal>
593-
{connectorConfig && connectorConfig.auth.mode === 'oauth' && connectorProviderId && (
594-
<OAuthRequiredModal
595-
isOpen={showOAuthModal}
596-
onClose={() => {
597-
consumeOAuthReturnContext()
598-
setShowOAuthModal(false)
599-
}}
600-
provider={connectorProviderId}
601-
toolName={connectorConfig.name}
602-
requiredScopes={getCanonicalScopesForProvider(connectorProviderId)}
603-
newScopes={[]}
604-
serviceId={connectorConfig.auth.provider}
605-
/>
606-
)}
545+
{showOAuthModal &&
546+
connectorConfig &&
547+
connectorConfig.auth.mode === 'oauth' &&
548+
connectorProviderId && (
549+
<ConnectCredentialModal
550+
isOpen={showOAuthModal}
551+
onClose={() => {
552+
consumeOAuthReturnContext()
553+
setShowOAuthModal(false)
554+
}}
555+
provider={connectorProviderId}
556+
serviceId={connectorConfig.auth.provider}
557+
workspaceId={workspaceId}
558+
knowledgeBaseId={knowledgeBaseId}
559+
credentialCount={credentials.length}
560+
/>
561+
)}
607562
</>
608563
)
609564
}

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import {
3636
} from '@/lib/oauth'
3737
import { getMissingRequiredScopes } from '@/lib/oauth/utils'
3838
import { EditConnectorModal } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal'
39+
import { ConnectCredentialModal } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/components/connect-credential-modal'
3940
import { OAuthRequiredModal } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/components/oauth-required-modal'
4041
import { CONNECTOR_REGISTRY } from '@/connectors/registry'
4142
import type { ConnectorData, SyncLogData } from '@/hooks/queries/kb/connectors'
@@ -333,6 +334,7 @@ function ConnectorCard({
333334
const missingScopes = useMemo(() => {
334335
if (!credentials || !connector.credentialId) return []
335336
const credential = credentials.find((c) => c.id === connector.credentialId)
337+
if (!credential) return []
336338
return getMissingRequiredScopes(credential, requiredScopes)
337339
}, [credentials, connector.credentialId, requiredScopes])
338340

@@ -484,15 +486,17 @@ function ConnectorCard({
484486
<Button
485487
variant='active'
486488
onClick={() => {
487-
writeOAuthReturnContext({
488-
origin: 'kb-connectors',
489-
knowledgeBaseId,
490-
displayName: connectorDef?.name ?? connector.connectorType,
491-
providerId: providerId!,
492-
preCount: credentials?.length ?? 0,
493-
workspaceId,
494-
requestedAt: Date.now(),
495-
})
489+
if (connector.credentialId) {
490+
writeOAuthReturnContext({
491+
origin: 'kb-connectors',
492+
knowledgeBaseId,
493+
displayName: connectorDef?.name ?? connector.connectorType,
494+
providerId: providerId!,
495+
preCount: credentials?.length ?? 0,
496+
workspaceId,
497+
requestedAt: Date.now(),
498+
})
499+
}
496500
setShowOAuthModal(true)
497501
}}
498502
className='w-full px-2 py-1 font-medium text-caption'
@@ -510,7 +514,22 @@ function ConnectorCard({
510514
</div>
511515
)}
512516

513-
{showOAuthModal && serviceId && providerId && (
517+
{showOAuthModal && serviceId && providerId && !connector.credentialId && (
518+
<ConnectCredentialModal
519+
isOpen={showOAuthModal}
520+
onClose={() => {
521+
consumeOAuthReturnContext()
522+
setShowOAuthModal(false)
523+
}}
524+
provider={providerId as OAuthProvider}
525+
serviceId={serviceId}
526+
workspaceId={workspaceId}
527+
knowledgeBaseId={knowledgeBaseId}
528+
credentialCount={credentials?.length ?? 0}
529+
/>
530+
)}
531+
532+
{showOAuthModal && serviceId && providerId && connector.credentialId && (
514533
<OAuthRequiredModal
515534
isOpen={showOAuthModal}
516535
onClose={() => {

‎apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/components/connect-credential-modal.tsx‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
ModalHeader,
1515
} from '@/components/emcn'
1616
import { client } from '@/lib/auth/auth-client'
17+
import type { OAuthReturnContext } from '@/lib/credentials/client-state'
1718
import { writeOAuthReturnContext } from '@/lib/credentials/client-state'
1819
import {
1920
getCanonicalScopesForProvider,
@@ -33,7 +34,8 @@ export interface ConnectCredentialModalProps {
3334
provider: OAuthProvider
3435
serviceId: string
3536
workspaceId: string
36-
workflowId: string
37+
workflowId?: string
38+
knowledgeBaseId?: string
3739
/** Number of existing credentials for this provider — used to detect a successful new connection. */
3840
credentialCount: number
3941
}
@@ -45,6 +47,7 @@ export function ConnectCredentialModal({
4547
serviceId,
4648
workspaceId,
4749
workflowId,
50+
knowledgeBaseId,
4851
credentialCount,
4952
}: ConnectCredentialModalProps) {
5053
const [displayName, setDisplayName] = useState('')
@@ -97,15 +100,19 @@ export function ConnectCredentialModal({
97100
try {
98101
await createDraft.mutateAsync({ workspaceId, providerId, displayName: trimmedName })
99102

100-
writeOAuthReturnContext({
101-
origin: 'workflow',
102-
workflowId,
103+
const baseContext = {
103104
displayName: trimmedName,
104105
providerId,
105106
preCount: credentialCount,
106107
workspaceId,
107108
requestedAt: Date.now(),
108-
})
109+
}
110+
111+
const returnContext: OAuthReturnContext = knowledgeBaseId
112+
? { ...baseContext, origin: 'kb-connectors' as const, knowledgeBaseId }
113+
: { ...baseContext, origin: 'workflow' as const, workflowId: workflowId ?? '' }
114+
115+
writeOAuthReturnContext(returnContext)
109116

110117
if (providerId === 'trello') {
111118
window.location.href = '/api/auth/trello/authorize'

‎apps/sim/lib/knowledge/connectors/sync-engine.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { db } from '@sim/db'
22
import {
33
document,
4+
embedding,
45
knowledgeBase,
56
knowledgeConnector,
67
knowledgeConnectorSyncLog,
@@ -658,6 +659,23 @@ export async function executeSync(
658659
if (stuckDocs.length > 0) {
659660
logger.info(`Retrying ${stuckDocs.length} stuck documents`, { connectorId })
660661
try {
662+
const stuckDocIds = stuckDocs.map((doc) => doc.id)
663+
664+
await db.delete(embedding).where(inArray(embedding.documentId, stuckDocIds))
665+
666+
await db
667+
.update(document)
668+
.set({
669+
processingStatus: 'pending',
670+
processingStartedAt: null,
671+
processingCompletedAt: null,
672+
processingError: null,
673+
chunkCount: 0,
674+
tokenCount: 0,
675+
characterCount: 0,
676+
})
677+
.where(inArray(document.id, stuckDocIds))
678+
661679
await processDocumentsWithQueue(
662680
stuckDocs.map((doc) => ({
663681
documentId: doc.id,

‎apps/sim/lib/knowledge/documents/service.ts‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -156,17 +156,12 @@ export async function dispatchDocumentProcessingJob(payload: DocumentJobData): P
156156
return
157157
}
158158

159-
void processDocumentAsync(
159+
await processDocumentAsync(
160160
payload.knowledgeBaseId,
161161
payload.documentId,
162162
payload.docData,
163163
payload.processingOptions
164-
).catch((error) => {
165-
logger.error(`[${payload.requestId}] Direct document processing failed`, {
166-
documentId: payload.documentId,
167-
error: error instanceof Error ? error.message : String(error),
168-
})
169-
})
164+
)
170165
}
171166

172167
export interface DocumentTagData {
@@ -434,6 +429,7 @@ export async function processDocumentAsync(
434429
.set({
435430
processingStatus: 'processing',
436431
processingStartedAt: new Date(),
432+
processingCompletedAt: null,
437433
processingError: null,
438434
})
439435
.where(
@@ -624,8 +620,9 @@ export async function processDocumentAsync(
624620
logger.info(`[${documentId}] Successfully processed document in ${processingTime}ms`)
625621
} catch (error) {
626622
const processingTime = Date.now() - startTime
623+
const errorMessage = error instanceof Error ? error.message : 'Unknown error'
627624
logger.error(`[${documentId}] Failed to process document after ${processingTime}ms:`, {
628-
error: error instanceof Error ? error.message : 'Unknown error',
625+
error: errorMessage,
629626
stack: error instanceof Error ? error.stack : undefined,
630627
filename: docData.filename,
631628
fileUrl: docData.fileUrl,
@@ -636,10 +633,12 @@ export async function processDocumentAsync(
636633
.update(document)
637634
.set({
638635
processingStatus: 'failed',
639-
processingError: error instanceof Error ? error.message : 'Unknown error',
636+
processingError: errorMessage,
640637
processingCompletedAt: new Date(),
641638
})
642639
.where(eq(document.id, documentId))
640+
641+
throw error
643642
}
644643
}
645644

@@ -1527,7 +1526,7 @@ export async function markDocumentAsFailedTimeout(
15271526
.update(document)
15281527
.set({
15291528
processingStatus: 'failed',
1530-
processingError: 'Processing timed out - background process may have been terminated',
1529+
processingError: 'Processing timed out. Please retry or re-sync the connector.',
15311530
processingCompletedAt: now,
15321531
})
15331532
.where(eq(document.id, documentId))

‎apps/sim/lib/knowledge/embeddings.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ async function getEmbeddingConfig(
101101
}
102102
}
103103

104+
const EMBEDDING_REQUEST_TIMEOUT_MS = 60_000
105+
104106
async function callEmbeddingAPI(inputs: string[], config: EmbeddingConfig): Promise<number[][]> {
105107
return retryWithExponentialBackoff(
106108
async () => {
@@ -119,11 +121,15 @@ async function callEmbeddingAPI(inputs: string[], config: EmbeddingConfig): Prom
119121
...(useDimensions && { dimensions: EMBEDDING_DIMENSIONS }),
120122
}
121123

124+
const controller = new AbortController()
125+
const timeout = setTimeout(() => controller.abort(), EMBEDDING_REQUEST_TIMEOUT_MS)
126+
122127
const response = await fetch(config.apiUrl, {
123128
method: 'POST',
124129
headers: config.headers,
125130
body: JSON.stringify(requestBody),
126-
})
131+
signal: controller.signal,
132+
}).finally(() => clearTimeout(timeout))
127133

128134
if (!response.ok) {
129135
const errorText = await response.text()

‎apps/sim/lib/oauth/oauth.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1504,7 +1504,9 @@ export async function refreshOAuthToken(
15041504
refreshToken: newRefreshToken || refreshToken, // Return new refresh token if available
15051505
}
15061506
} catch (error) {
1507-
logger.error('Error refreshing token:', { error })
1507+
logger.error('Error refreshing token:', {
1508+
error: error instanceof Error ? error.message : String(error),
1509+
})
15081510
return null
15091511
}
15101512
}

0 commit comments

Comments
 (0)