Skip to content

Commit 068fa59

Browse files
waleedlatif1waleedlatif1
committed
fix(knowledge): fix document processing stuck in processing state
1 parent e9c94fa commit 068fa59

File tree

10 files changed

+139
-99
lines changed

10 files changed

+139
-99
lines changed

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -900,7 +900,11 @@ export function KnowledgeBase({
900900
onClick={() => setShowConnectorsModal(true)}
901901
className='flex shrink-0 cursor-pointer items-center gap-1.5 rounded-md px-2 py-1 text-[var(--text-secondary)] text-caption shadow-[inset_0_0_0_1px_var(--border)] transition-colors hover-hover:bg-[var(--surface-3)]'
902902
>
903-
{ConnectorIcon && <ConnectorIcon className='h-[14px] w-[14px]' />}
903+
{connector.status === 'syncing' ? (
904+
<Loader2 className='h-[14px] w-[14px] animate-spin' />
905+
) : (
906+
ConnectorIcon && <ConnectorIcon className='h-[14px] w-[14px]' />
907+
)}
904908
{def?.name || connector.connectorType}
905909
</button>
906910
)

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx‎

Lines changed: 22 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,10 @@ import {
1919
ModalHeader,
2020
Tooltip,
2121
} from '@/components/emcn'
22-
import { useSession } from '@/lib/auth/auth-client'
23-
import { consumeOAuthReturnContext, writeOAuthReturnContext } from '@/lib/credentials/client-state'
24-
import {
25-
getCanonicalScopesForProvider,
26-
getProviderIdFromServiceId,
27-
type OAuthProvider,
28-
} from '@/lib/oauth'
22+
import { consumeOAuthReturnContext } from '@/lib/credentials/client-state'
23+
import { getProviderIdFromServiceId, type OAuthProvider } from '@/lib/oauth'
2924
import { ConnectorSelectorField } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/components/connector-selector-field'
30-
import { OAuthRequiredModal } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/components/oauth-required-modal'
25+
import { ConnectCredentialModal } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/components/connect-credential-modal'
3126
import { getDependsOnFields } from '@/blocks/utils'
3227
import { CONNECTOR_REGISTRY } from '@/connectors/registry'
3328
import type { ConnectorConfig, ConnectorConfigField } from '@/connectors/types'
@@ -69,7 +64,6 @@ export function AddConnectorModal({ open, onOpenChange, knowledgeBaseId }: AddCo
6964
const [searchTerm, setSearchTerm] = useState('')
7065

7166
const { workspaceId } = useParams<{ workspaceId: string }>()
72-
const { data: session } = useSession()
7367
const { mutate: createConnector, isPending: isCreating } = useCreateConnector()
7468

7569
const connectorConfig = selectedType ? CONNECTOR_REGISTRY[selectedType] : null
@@ -263,51 +257,9 @@ export function AddConnectorModal({ open, onOpenChange, knowledgeBaseId }: AddCo
263257
)
264258
}
265259

266-
const handleConnectNewAccount = useCallback(async () => {
267-
if (!connectorConfig || !connectorProviderId || !workspaceId) return
268-
269-
const userName = session?.user?.name
270-
const integrationName = connectorConfig.name
271-
const displayName = userName ? `${userName}'s ${integrationName}` : integrationName
272-
273-
try {
274-
const res = await fetch('/api/credentials/draft', {
275-
method: 'POST',
276-
headers: { 'Content-Type': 'application/json' },
277-
body: JSON.stringify({
278-
workspaceId,
279-
providerId: connectorProviderId,
280-
displayName,
281-
}),
282-
})
283-
if (!res.ok) {
284-
setError('Failed to prepare credential. Please try again.')
285-
return
286-
}
287-
} catch {
288-
setError('Failed to prepare credential. Please try again.')
289-
return
290-
}
291-
292-
writeOAuthReturnContext({
293-
origin: 'kb-connectors',
294-
knowledgeBaseId,
295-
displayName,
296-
providerId: connectorProviderId,
297-
preCount: credentials.length,
298-
workspaceId,
299-
requestedAt: Date.now(),
300-
})
301-
260+
const handleConnectNewAccount = useCallback(() => {
302261
setShowOAuthModal(true)
303-
}, [
304-
connectorConfig,
305-
connectorProviderId,
306-
workspaceId,
307-
session?.user?.name,
308-
knowledgeBaseId,
309-
credentials.length,
310-
])
262+
}, [])
311263

312264
const filteredEntries = useMemo(() => {
313265
const term = searchTerm.toLowerCase().trim()
@@ -590,20 +542,23 @@ export function AddConnectorModal({ open, onOpenChange, knowledgeBaseId }: AddCo
590542
)}
591543
</ModalContent>
592544
</Modal>
593-
{connectorConfig && connectorConfig.auth.mode === 'oauth' && connectorProviderId && (
594-
<OAuthRequiredModal
595-
isOpen={showOAuthModal}
596-
onClose={() => {
597-
consumeOAuthReturnContext()
598-
setShowOAuthModal(false)
599-
}}
600-
provider={connectorProviderId}
601-
toolName={connectorConfig.name}
602-
requiredScopes={getCanonicalScopesForProvider(connectorProviderId)}
603-
newScopes={[]}
604-
serviceId={connectorConfig.auth.provider}
605-
/>
606-
)}
545+
{showOAuthModal &&
546+
connectorConfig &&
547+
connectorConfig.auth.mode === 'oauth' &&
548+
connectorProviderId && (
549+
<ConnectCredentialModal
550+
isOpen={showOAuthModal}
551+
onClose={() => {
552+
consumeOAuthReturnContext()
553+
setShowOAuthModal(false)
554+
}}
555+
provider={connectorProviderId}
556+
serviceId={connectorConfig.auth.provider}
557+
workspaceId={workspaceId}
558+
knowledgeBaseId={knowledgeBaseId}
559+
credentialCount={credentials.length}
560+
/>
561+
)}
607562
</>
608563
)
609564
}

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import {
3636
} from '@/lib/oauth'
3737
import { getMissingRequiredScopes } from '@/lib/oauth/utils'
3838
import { EditConnectorModal } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal'
39+
import { ConnectCredentialModal } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/components/connect-credential-modal'
3940
import { OAuthRequiredModal } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/components/oauth-required-modal'
4041
import { CONNECTOR_REGISTRY } from '@/connectors/registry'
4142
import type { ConnectorData, SyncLogData } from '@/hooks/queries/kb/connectors'
@@ -333,6 +334,7 @@ function ConnectorCard({
333334
const missingScopes = useMemo(() => {
334335
if (!credentials || !connector.credentialId) return []
335336
const credential = credentials.find((c) => c.id === connector.credentialId)
337+
if (!credential) return []
336338
return getMissingRequiredScopes(credential, requiredScopes)
337339
}, [credentials, connector.credentialId, requiredScopes])
338340

@@ -484,15 +486,17 @@ function ConnectorCard({
484486
<Button
485487
variant='active'
486488
onClick={() => {
487-
writeOAuthReturnContext({
488-
origin: 'kb-connectors',
489-
knowledgeBaseId,
490-
displayName: connectorDef?.name ?? connector.connectorType,
491-
providerId: providerId!,
492-
preCount: credentials?.length ?? 0,
493-
workspaceId,
494-
requestedAt: Date.now(),
495-
})
489+
if (connector.credentialId) {
490+
writeOAuthReturnContext({
491+
origin: 'kb-connectors',
492+
knowledgeBaseId,
493+
displayName: connectorDef?.name ?? connector.connectorType,
494+
providerId: providerId!,
495+
preCount: credentials?.length ?? 0,
496+
workspaceId,
497+
requestedAt: Date.now(),
498+
})
499+
}
496500
setShowOAuthModal(true)
497501
}}
498502
className='w-full px-2 py-1 font-medium text-caption'
@@ -510,7 +514,22 @@ function ConnectorCard({
510514
</div>
511515
)}
512516

513-
{showOAuthModal && serviceId && providerId && (
517+
{showOAuthModal && serviceId && providerId && !connector.credentialId && (
518+
<ConnectCredentialModal
519+
isOpen={showOAuthModal}
520+
onClose={() => {
521+
consumeOAuthReturnContext()
522+
setShowOAuthModal(false)
523+
}}
524+
provider={providerId as OAuthProvider}
525+
serviceId={serviceId}
526+
workspaceId={workspaceId}
527+
knowledgeBaseId={knowledgeBaseId}
528+
credentialCount={credentials?.length ?? 0}
529+
/>
530+
)}
531+
532+
{showOAuthModal && serviceId && providerId && connector.credentialId && (
514533
<OAuthRequiredModal
515534
isOpen={showOAuthModal}
516535
onClose={() => {

‎apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/components/connect-credential-modal.tsx‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
ModalHeader,
1515
} from '@/components/emcn'
1616
import { client } from '@/lib/auth/auth-client'
17+
import type { OAuthReturnContext } from '@/lib/credentials/client-state'
1718
import { writeOAuthReturnContext } from '@/lib/credentials/client-state'
1819
import {
1920
getCanonicalScopesForProvider,
@@ -33,7 +34,8 @@ export interface ConnectCredentialModalProps {
3334
provider: OAuthProvider
3435
serviceId: string
3536
workspaceId: string
36-
workflowId: string
37+
workflowId?: string
38+
knowledgeBaseId?: string
3739
/** Number of existing credentials for this provider — used to detect a successful new connection. */
3840
credentialCount: number
3941
}
@@ -45,6 +47,7 @@ export function ConnectCredentialModal({
4547
serviceId,
4648
workspaceId,
4749
workflowId,
50+
knowledgeBaseId,
4851
credentialCount,
4952
}: ConnectCredentialModalProps) {
5053
const [displayName, setDisplayName] = useState('')
@@ -97,15 +100,19 @@ export function ConnectCredentialModal({
97100
try {
98101
await createDraft.mutateAsync({ workspaceId, providerId, displayName: trimmedName })
99102

100-
writeOAuthReturnContext({
101-
origin: 'workflow',
102-
workflowId,
103+
const baseContext = {
103104
displayName: trimmedName,
104105
providerId,
105106
preCount: credentialCount,
106107
workspaceId,
107108
requestedAt: Date.now(),
108-
})
109+
}
110+
111+
const returnContext: OAuthReturnContext = knowledgeBaseId
112+
? { ...baseContext, origin: 'kb-connectors' as const, knowledgeBaseId }
113+
: { ...baseContext, origin: 'workflow' as const, workflowId: workflowId ?? '' }
114+
115+
writeOAuthReturnContext(returnContext)
109116

110117
if (providerId === 'trello') {
111118
window.location.href = '/api/auth/trello/authorize'

‎apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/credential-selector/credential-selector.tsx‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { Button, Combobox } from '@/components/emcn/components'
77
import { getSubscriptionAccessState } from '@/lib/billing/client'
88
import { getEnv, isTruthy } from '@/lib/core/config/env'
99
import { getPollingProviderFromOAuth } from '@/lib/credential-sets/providers'
10+
import { consumeOAuthReturnContext, writeOAuthReturnContext } from '@/lib/credentials/client-state'
1011
import {
1112
getCanonicalScopesForProvider,
1213
getProviderIdFromServiceId,
@@ -357,7 +358,18 @@ export function CredentialSelector({
357358
</div>
358359
<Button
359360
variant='active'
360-
onClick={() => setShowOAuthModal(true)}
361+
onClick={() => {
362+
writeOAuthReturnContext({
363+
origin: 'workflow',
364+
workflowId: activeWorkflowId || '',
365+
displayName: selectedCredential?.name ?? getProviderName(provider),
366+
providerId: effectiveProviderId,
367+
preCount: credentials.length,
368+
workspaceId,
369+
requestedAt: Date.now(),
370+
})
371+
setShowOAuthModal(true)
372+
}}
361373
className='w-full px-2 py-1 font-medium text-caption'
362374
>
363375
Update access
@@ -380,7 +392,10 @@ export function CredentialSelector({
380392
{showOAuthModal && (
381393
<OAuthRequiredModal
382394
isOpen={showOAuthModal}
383-
onClose={() => setShowOAuthModal(false)}
395+
onClose={() => {
396+
consumeOAuthReturnContext()
397+
setShowOAuthModal(false)
398+
}}
384399
provider={provider}
385400
toolName={getProviderName(provider)}
386401
requiredScopes={getCanonicalScopesForProvider(effectiveProviderId)}

‎apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tool-input/components/tools/credential-selector.tsx‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { createElement, useCallback, useMemo, useRef, useState } from 'react'
44
import { ExternalLink } from 'lucide-react'
55
import { useParams } from 'next/navigation'
66
import { Button, Combobox } from '@/components/emcn/components'
7+
import { consumeOAuthReturnContext, writeOAuthReturnContext } from '@/lib/credentials/client-state'
78
import {
89
getCanonicalScopesForProvider,
910
getProviderIdFromServiceId,
@@ -222,7 +223,18 @@ export function ToolCredentialSelector({
222223
</div>
223224
<Button
224225
variant='active'
225-
onClick={() => setShowOAuthModal(true)}
226+
onClick={() => {
227+
writeOAuthReturnContext({
228+
origin: 'workflow',
229+
workflowId: effectiveWorkflowId || '',
230+
displayName: selectedCredential?.name ?? getProviderName(provider),
231+
providerId: effectiveProviderId,
232+
preCount: credentials.length,
233+
workspaceId,
234+
requestedAt: Date.now(),
235+
})
236+
setShowOAuthModal(true)
237+
}}
226238
className='w-full px-2 py-1 font-medium text-caption'
227239
>
228240
Update access
@@ -245,7 +257,10 @@ export function ToolCredentialSelector({
245257
{showOAuthModal && (
246258
<OAuthRequiredModal
247259
isOpen={showOAuthModal}
248-
onClose={() => setShowOAuthModal(false)}
260+
onClose={() => {
261+
consumeOAuthReturnContext()
262+
setShowOAuthModal(false)
263+
}}
249264
provider={provider}
250265
toolName={getProviderName(provider)}
251266
requiredScopes={getCanonicalScopesForProvider(effectiveProviderId)}

‎apps/sim/lib/knowledge/connectors/sync-engine.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { db } from '@sim/db'
22
import {
33
document,
4+
embedding,
45
knowledgeBase,
56
knowledgeConnector,
67
knowledgeConnectorSyncLog,
@@ -658,6 +659,23 @@ export async function executeSync(
658659
if (stuckDocs.length > 0) {
659660
logger.info(`Retrying ${stuckDocs.length} stuck documents`, { connectorId })
660661
try {
662+
const stuckDocIds = stuckDocs.map((doc) => doc.id)
663+
664+
await db.delete(embedding).where(inArray(embedding.documentId, stuckDocIds))
665+
666+
await db
667+
.update(document)
668+
.set({
669+
processingStatus: 'pending',
670+
processingStartedAt: null,
671+
processingCompletedAt: null,
672+
processingError: null,
673+
chunkCount: 0,
674+
tokenCount: 0,
675+
characterCount: 0,
676+
})
677+
.where(inArray(document.id, stuckDocIds))
678+
661679
await processDocumentsWithQueue(
662680
stuckDocs.map((doc) => ({
663681
documentId: doc.id,

0 commit comments

Comments
 (0)