diff --git a/apps/sim/app/api/v1/knowledge/route.ts b/apps/sim/app/api/v1/knowledge/route.ts index cacb36ed482..9593cc8466e 100644 --- a/apps/sim/app/api/v1/knowledge/route.ts +++ b/apps/sim/app/api/v1/knowledge/route.ts @@ -10,7 +10,7 @@ import { } from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { performCreateKnowledgeBase } from '@/lib/knowledge/orchestration' -import { getKnowledgeBases } from '@/lib/knowledge/service' +import { listWorkspaceAndLegacyKnowledgeBases } from '@/lib/knowledge/service' import { formatKnowledgeBase, handleError } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, @@ -43,7 +43,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId) if (accessError) return accessError - const knowledgeBases = await getKnowledgeBases(userId, workspaceId) + /** Read only after `validateWorkspaceAccess` authorized this caller; same list the + * internal surface serves, from the same place. */ + const knowledgeBases = await listWorkspaceAndLegacyKnowledgeBases(userId, workspaceId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx index faa99423d44..e5fa53f5108 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx @@ -14,8 +14,8 @@ import { ChipModalHeader, handleKeyboardActivation, Label, - Trash, } from '@sim/emcn' +import { Trash } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { formatDate } from '@sim/utils/formatting' import { @@ -378,11 +378,7 @@ export function DocumentTagsModal({ return ( - handleClose(false)}> -
- Document Tags -
-
+ handleClose(false)}>Document Tags diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx index 974a0896848..5c9e0e4d99d 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx @@ -1,8 +1,17 @@ 'use client' import { useCallback, useEffect, useEffectEvent, useMemo, useRef, useState } from 'react' -import { Badge, ChipCombobox, ChipConfirmModal, Plus, Trash } from '@sim/emcn' -import { ChevronDown, ChevronUp, Database, FileText, Pencil, TagIcon } from '@sim/emcn/icons' +import { Badge, ChipCombobox, ChipConfirmModal, chipContentLabelClass, cn } from '@sim/emcn' +import { + ChevronDown, + ChevronUp, + Database, + FileText, + Pencil, + Plus, + TagIcon, + Trash, +} from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { truncate } from '@sim/utils/string' import { useParams, useRouter } from 'next/navigation' @@ -204,7 +213,6 @@ export function Document({ chunks: initialChunks, currentPage: initialPage, totalPages: initialTotalPages, - goToPage: initialGoToPage, error: initialError, updateChunk: initialUpdateChunk, } = useDocumentChunks( @@ -292,26 +300,22 @@ export function Document({ const totalPagesRef = useRef(totalPages) totalPagesRef.current = totalPages - const goToPage = useCallback( - async (page: number) => { - await setDocumentParams({ page }) - - if (showingSearch) { - return - } - return initialGoToPage(page) - }, - [showingSearch, initialGoToPage, setDocumentParams] - ) + const goToPage = useCallback((page: number) => setDocumentParams({ page }), [setDocumentParams]) const updateChunk = showingSearch ? (_id: string, _updates: Record) => {} : initialUpdateChunk const [chunkToDelete, setChunkToDelete] = useState(null) - const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false) const [showDeleteDocumentDialog, setShowDeleteDocumentDialog] = useState(false) - const [contextMenuChunk, setContextMenuChunk] = useState(null) + const [contextMenuChunkId, setContextMenuChunkId] = useState(null) + /** + * The id, not the row: the chunk list polls while a document processes, and a menu that + * captured the row on open would keep offering "Enable" for a chunk already enabled. + */ + const contextMenuChunk = contextMenuChunkId + ? (displayChunks.find((chunk) => chunk.id === contextMenuChunkId) ?? null) + : null const { mutate: updateChunkMutation } = useUpdateChunk() const { mutate: deleteDocumentMutation, isPending: isDeletingDocument } = useDeleteDocument() @@ -351,15 +355,10 @@ export function Document({ const isInEditorView = selectedChunkId !== null || isCreatingNewChunk - const selectedChunk = useMemo( - () => (selectedChunkId ? (displayChunks.find((c) => c.id === selectedChunkId) ?? null) : null), - [selectedChunkId, displayChunks] - ) - - const currentChunkIndex = useMemo( - () => (selectedChunk ? displayChunks.findIndex((c) => c.id === selectedChunk.id) : -1), - [selectedChunk, displayChunks] - ) + const currentChunkIndex = selectedChunkId + ? displayChunks.findIndex((chunk) => chunk.id === selectedChunkId) + : -1 + const selectedChunk = currentChunkIndex >= 0 ? displayChunks[currentChunkIndex] : null const canNavigatePrev = currentChunkIndex > 0 || currentPage > 1 const canNavigateNext = currentChunkIndex < displayChunks.length - 1 || currentPage < totalPages @@ -402,14 +401,14 @@ export function Document({ } }, [isDirty, isCreatingNewChunk]) - const handleUnsavedChangesOpenChange = useCallback((open: boolean) => { + const handleUnsavedChangesOpenChange = (open: boolean) => { if (!open) { setShowUnsavedChangesAlert(false) setPendingAction(null) } - }, []) + } - const handleDiscardChanges = useCallback(() => { + const handleDiscardChanges = () => { setShowUnsavedChangesAlert(false) const action = pendingAction setPendingAction(null) @@ -419,7 +418,7 @@ export function Document({ } else { closeEditor() } - }, [pendingAction, closeEditor]) + } const handleSaveEvent = useEffectEvent(handleSave) @@ -646,7 +645,6 @@ export function Document({ if (found) { setSelectedChunkId(chunkId) } else if (!navigatedToNewPage && totalPagesRef.current > totalPages) { - // A new page was created — navigate to it navigatedToNewPage = true retries = 0 void goToPage(totalPagesRef.current) @@ -681,10 +679,8 @@ export function Document({ } : undefined - const enabledDisplayLabel = useMemo(() => { - if (enabledFilter.length === 0) return 'All' - return enabledFilter[0] === 'enabled' ? 'Enabled' : 'Disabled' - }, [enabledFilter]) + const enabledDisplayLabel = + enabledFilter.length === 0 ? 'All' : enabledFilter[0] === 'enabled' ? 'Enabled' : 'Disabled' const filterContent = useMemo( () => ( @@ -724,7 +720,7 @@ export function Document({ )} ), - [enabledFilter, enabledDisplayLabel, setEnabledFilter] + [enabledFilter, setEnabledFilter] ) const filterTags: FilterTag[] = useMemo( @@ -746,31 +742,22 @@ export function Document({ [setSelectedChunkId] ) - const handleToggleEnabled = useCallback( - (chunkId: string) => { - const chunk = displayChunks.find((c) => c.id === chunkId) - if (!chunk) return + const handleToggleEnabled = (chunkId: string) => { + const chunk = displayChunks.find((c) => c.id === chunkId) + if (!chunk) return - const newEnabled = !chunk.enabled - updateChunk(chunkId, { enabled: newEnabled }) - updateChunkMutation( - { knowledgeBaseId, documentId, chunkId, enabled: newEnabled }, - { onError: () => updateChunk(chunkId, { enabled: chunk.enabled }) } - ) - }, - [displayChunks, knowledgeBaseId, documentId, updateChunk] - ) + const newEnabled = !chunk.enabled + updateChunk(chunkId, { enabled: newEnabled }) + updateChunkMutation( + { knowledgeBaseId, documentId, chunkId, enabled: newEnabled }, + { onError: () => updateChunk(chunkId, { enabled: chunk.enabled }) } + ) + } - const handleDeleteChunk = useCallback( - (chunkId: string) => { - const chunk = displayChunks.find((c) => c.id === chunkId) - if (chunk) { - setChunkToDelete(chunk) - setIsDeleteModalOpen(true) - } - }, - [displayChunks] - ) + const handleDeleteChunk = (chunkId: string) => { + const chunk = displayChunks.find((c) => c.id === chunkId) + if (chunk) setChunkToDelete(chunk) + } const handleCloseDeleteModal = () => { if (chunkToDelete) { @@ -780,7 +767,6 @@ export function Document({ return newSet }) } - setIsDeleteModalOpen(false) setChunkToDelete(null) } @@ -863,17 +849,14 @@ export function Document({ performBulkChunkOperation('delete', chunksToDelete) } - const [enabledCount, disabledCount] = useMemo(() => { - let enabled = 0 - let disabled = 0 - for (const chunk of displayChunks) { - if (selectedChunks.has(chunk.id)) { - if (chunk.enabled) enabled++ - else disabled++ - } + let enabledCount = 0 + let disabledCount = 0 + for (const chunk of displayChunks) { + if (selectedChunks.has(chunk.id)) { + if (chunk.enabled) enabledCount++ + else disabledCount++ } - return [enabled, disabled] - }, [displayChunks, selectedChunks]) + } const isAllSelected = displayChunks.length > 0 && selectedChunks.size === displayChunks.length @@ -890,7 +873,7 @@ export function Document({ } } - setContextMenuChunk(chunk) + setContextMenuChunkId(chunk.id) baseHandleContextMenu(e) }, [ @@ -902,18 +885,15 @@ export function Document({ ] ) - const handleEmptyContextMenu = useCallback( - (e: React.MouseEvent) => { - setContextMenuChunk(null) - baseHandleContextMenu(e) - }, - [baseHandleContextMenu] - ) + const handleEmptyContextMenu = (e: React.MouseEvent) => { + setContextMenuChunkId(null) + baseHandleContextMenu(e) + } - const handleContextMenuClose = useCallback(() => { + const handleContextMenuClose = () => { closeContextMenu() - setContextMenuChunk(null) - }, [closeContextMenu]) + setContextMenuChunkId(null) + } const selectableConfig: SelectableConfig | undefined = isCompleted ? { @@ -955,7 +935,17 @@ export function Document({ [activeSort, onSortColumn, onClearSort, goToPage] ) + const hasDocumentData = documentData !== null + const processingStatus = documentData?.processingStatus + const chunkRows: ResourceRow[] = useMemo(() => { + /** + * No document yet is "not known", not "not ready". Falling through to the status row + * flashed `Document not ready` on every open, for the frame between mount and the + * document query resolving — a claim about a document nothing had read yet. + */ + if (!hasDocumentData) return [] + if (!isCompleted) { return [ { @@ -966,12 +956,10 @@ export function Document({
- {documentData?.processingStatus === 'pending' && - 'Document processing pending...'} - {documentData?.processingStatus === 'processing' && - 'Document processing in progress...'} - {documentData?.processingStatus === 'failed' && 'Document processing failed'} - {!documentData?.processingStatus && 'Document not ready'} + {processingStatus === 'pending' && 'Document processing pending...'} + {processingStatus === 'processing' && 'Document processing in progress...'} + {processingStatus === 'failed' && 'Document processing failed'} + {!processingStatus && 'Document not ready'}
), @@ -992,16 +980,14 @@ export function Document({ cells: { content: { content: ( - + ), }, index: { content: ( - - {chunk.chunkIndex} - + {chunk.chunkIndex} ), }, tokens: { @@ -1017,7 +1003,7 @@ export function Document({ }, } }) - }, [isCompleted, documentData?.processingStatus, displayChunks, searchQuery]) + }, [isCompleted, hasDocumentData, processingStatus, displayChunks, searchQuery]) const saveLabel = saveStatus === 'saving' @@ -1232,7 +1218,7 @@ export function Document({ chunk={chunkToDelete} knowledgeBaseId={knowledgeBaseId} documentId={documentId} - isOpen={isDeleteModalOpen} + isOpen={chunkToDelete !== null} onClose={handleCloseDeleteModal} /> diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/loading.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/loading.tsx index ed67b33a791..63369f7a3d1 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/loading.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/loading.tsx @@ -1,7 +1,6 @@ 'use client' -import { Plus } from '@sim/emcn' -import { Database, FileText } from '@sim/emcn/icons' +import { Database, FileText, Plus } from '@sim/emcn/icons' import { noop } from '@sim/utils/helpers' import { type BreadcrumbItem, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx index b64b4642abb..a01bb376e57 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx @@ -20,12 +20,20 @@ import { cn, FloatingTooltip, isTextClipped, - Loader, Tooltip, - Trash, useFloatingTooltip, } from '@sim/emcn' -import { CircleAlert, Database, DatabaseX, Pencil, Plus, TagIcon, X } from '@sim/emcn/icons' +import { + CircleAlert, + Database, + DatabaseX, + Loader, + Pencil, + Plus, + TagIcon, + Trash, + X, +} from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' @@ -77,19 +85,13 @@ import { documentFiltersParsers, documentFiltersUrlKeys, kbDocumentSortParams, - pageParam, - pageUrlKeys, } from '@/app/workspace/[workspaceId]/knowledge/[id]/search-params' import { getDocumentIcon } from '@/app/workspace/[workspaceId]/knowledge/components' import { useRegisterGlobalCommands } from '@/app/workspace/[workspaceId]/providers/global-commands-provider' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' -import { - useKnowledgeBase, - useKnowledgeBaseDocuments, - useKnowledgeBasesList, -} from '@/hooks/kb/use-knowledge' +import { useKnowledgeBase, useKnowledgeBaseDocuments } from '@/hooks/kb/use-knowledge' import { type TagDefinition, useKnowledgeBaseTagDefinitions, @@ -280,14 +282,12 @@ export function KnowledgeBase({ }, [id, passedKnowledgeBaseName, posthog]) useOAuthReturnForKBConnectors(id) - const { removeKnowledgeBase } = useKnowledgeBasesList(workspaceId, { enabled: false }) const userPermissions = useUserPermissionsContext() const { mutate: updateDocumentMutation, mutateAsync: updateDocumentAsync } = useUpdateDocument() const { mutate: deleteDocumentMutation } = useDeleteDocument() - const { mutate: deleteKnowledgeBaseMutation, isPending: isDeleting } = - useDeleteKnowledgeBase(workspaceId) - const { mutateAsync: updateKnowledgeBaseMutation } = useUpdateKnowledgeBase(workspaceId) + const { mutate: deleteKnowledgeBaseMutation, isPending: isDeleting } = useDeleteKnowledgeBase() + const { mutateAsync: updateKnowledgeBaseMutation } = useUpdateKnowledgeBase() const kbRename = useInlineRename({ onSave: (kbId, name) => @@ -336,14 +336,13 @@ export function KnowledgeBase({ const [documentToDelete, setDocumentToDelete] = useState(null) const [showBulkDeleteModal, setShowBulkDeleteModal] = useState(false) const [showConnectorsModal, setShowConnectorsModal] = useState(false) - const [currentPage, setCurrentPage] = useQueryState(pageParam.key, { - ...pageParam.parser, - ...pageUrlKeys, - }) + const [{ q: searchQuery, enabled: enabledFilter, page: currentPage }, setDocumentFilters] = + useQueryStates(documentFiltersParsers, documentFiltersUrlKeys) - const [{ q: searchQuery, enabled: enabledFilter }, setDocumentFilters] = useQueryStates( - documentFiltersParsers, - documentFiltersUrlKeys + /** Page 1 is the group's default, so it strips from the URL rather than lingering as `?page=1`. */ + const setCurrentPage = useCallback( + (page: number) => void setDocumentFilters({ page }), + [setDocumentFilters] ) /** @@ -352,8 +351,7 @@ export function KnowledgeBase({ * doesn't refetch on every keystroke. Changing the search resets pagination. */ const handleSearchChange = useDebouncedSearchSetter((value, options) => { - setDocumentFilters({ q: value }, options) - setCurrentPage(1) + void setDocumentFilters({ q: value, page: 1 }, options) }) const debouncedSearchQuery = useDebounce(searchQuery, SEARCH_DEBOUNCE_MS) /** Raw URL value drives the input; matching/highlighting always sees it trimmed. */ @@ -369,13 +367,12 @@ export function KnowledgeBase({ const setEnabledFilter = useCallback( (value: 'all' | 'enabled' | 'disabled') => { - setDocumentFilters({ enabled: value }) - setCurrentPage(1) + void setDocumentFilters({ enabled: value, page: 1 }) }, - [setDocumentFilters, setCurrentPage] + [setDocumentFilters] ) - const [contextMenuDocument, setContextMenuDocument] = useState(null) + const [contextMenuDocumentId, setContextMenuDocumentId] = useState(null) const [showRenameModal, setShowRenameModal] = useState(false) const [documentToRename, setDocumentToRename] = useState(null) const [showDocumentTagsModal, setShowDocumentTagsModal] = useState(false) @@ -440,6 +437,15 @@ export function KnowledgeBase({ const { tagDefinitions } = useKnowledgeBaseTagDefinitions(id) + /** + * The id, not the row: the document list polls every few seconds while anything is + * processing, so a menu holding the row it opened on would offer actions against a status + * that has since moved on. + */ + const contextMenuDocument = contextMenuDocumentId + ? (documents.find((doc) => doc.id === contextMenuDocumentId) ?? null) + : null + const prevHadSyncingRef = useRef(false) useEffect(() => { if (prevHadSyncingRef.current && !hasSyncingConnectors) { @@ -699,7 +705,6 @@ export function KnowledgeBase({ { knowledgeBaseId: id }, { onSuccess: () => { - removeKnowledgeBase(id) router.push(`/workspace/${workspaceId}/knowledge`) }, } @@ -885,24 +890,21 @@ export function KnowledgeBase({ setSelectedDocuments(new Set([doc.id])) } - setContextMenuDocument(doc) + setContextMenuDocumentId(doc.id) baseHandleContextMenu(e) }, [documents, selectedDocuments, baseHandleContextMenu] ) - const handleEmptyContextMenu = useCallback( - (e: React.MouseEvent) => { - setContextMenuDocument(null) - baseHandleContextMenu(e) - }, - [baseHandleContextMenu] - ) + const handleEmptyContextMenu = (e: React.MouseEvent) => { + setContextMenuDocumentId(null) + baseHandleContextMenu(e) + } - const handleContextMenuClose = useCallback(() => { + const handleContextMenuClose = () => { closeContextMenu() - setContextMenuDocument(null) - }, [closeContextMenu]) + setContextMenuDocumentId(null) + } const breadcrumbs: BreadcrumbItem[] = useMemo( () => diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/action-bar/action-bar.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/action-bar/action-bar.tsx index e6817dce63c..ced7d0eb350 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/action-bar/action-bar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/action-bar/action-bar.tsx @@ -1,8 +1,14 @@ -import { Button, cn, Tooltip, Trash } from '@sim/emcn' -import { Ban, Circle } from '@sim/emcn/icons' +import { Button, chipFilledFillTokens, cn, Tooltip } from '@sim/emcn' +import { Ban, Circle, Trash } from '@sim/emcn/icons' import { domAnimation, LazyMotion, m } from 'framer-motion' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' +/** One source of truth for the button chrome, so the three actions read as one control strip. */ +const ACTION_BUTTON_CLASS = cn( + chipFilledFillTokens, + 'hover-hover:!text-[var(--text-inverse)] size-[28px] rounded-lg p-0 text-[var(--text-secondary)] hover-hover:bg-[var(--brand-secondary)]' +) + interface ActionBarProps { selectedCount: number onEnable?: () => void @@ -51,8 +57,10 @@ export function ActionBar({ animate={{ opacity: 1, y: 0 }} exit={{ opacity: 0, y: 10 }} transition={{ duration: 0.2 }} - className={cn('-translate-x-1/2 fixed bottom-6 z-50 transform', className)} - style={{ left: '50%' }} + className={cn( + '-translate-x-1/2 fixed bottom-6 left-1/2 z-[var(--z-dropdown)] transform', + className + )} >
@@ -63,7 +71,7 @@ export function ActionBar({ @@ -75,7 +83,7 @@ export function ActionBar({ @@ -89,9 +97,10 @@ export function ActionBar({ @@ -105,9 +114,10 @@ export function ActionBar({ @@ -121,9 +131,10 @@ export function ActionBar({ diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx index ddeb723a94c..90862127cb8 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx @@ -10,9 +10,8 @@ import { ChipModalFooter, ChipModalHeader, cn, - Loader, } from '@sim/emcn' -import { RefreshCw, X } from '@sim/emcn/icons' +import { Loader, RefreshCw, X } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { useParams } from 'next/navigation' import { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/base-tags-modal/base-tags-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/base-tags-modal/base-tags-modal.tsx index 8bf94aec13a..d82a9bd6937 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/base-tags-modal/base-tags-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/base-tags-modal/base-tags-modal.tsx @@ -13,8 +13,8 @@ import { ChipModalHeader, type ComboboxOption, handleKeyboardActivation, - Trash, } from '@sim/emcn' +import { Trash } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import type { TagUsageData } from '@/lib/api/contracts/knowledge' import { @@ -393,7 +393,6 @@ export function BaseTagsModal({ open, onOpenChange, knowledgeBaseId }: BaseTagsM /> - {/* Delete Tag Confirmation Dialog */} { @@ -432,7 +431,6 @@ export function BaseTagsModal({ open, onOpenChange, knowledgeBaseId }: BaseTagsM )} - {/* View Documents Dialog */} >> + +function addToSet(setter: IdSetSetter, id: string) { + setter((prev) => new Set(prev).add(id)) +} + +function removeFromSet(setter: IdSetSetter, id: string) { + setter((prev) => { + const next = new Set(prev) + next.delete(id) + return next + }) +} + const STATUS_CONFIG = { active: { label: 'Active', variant: 'green' as const }, syncing: { label: 'Syncing', variant: 'amber' as const }, @@ -86,27 +103,15 @@ export function ConnectorsSection({ const [deleteTarget, setDeleteTarget] = useState(null) const [deleteDocuments, setDeleteDocuments] = useState(false) - const closeDeleteModal = useCallback(() => { + const closeDeleteModal = () => { setDeleteTarget(null) setDeleteDocuments(false) - }, []) + } const [editingConnector, setEditingConnector] = useState(null) const [error, setError] = useState(null) const [syncingIds, setSyncingIds] = useState>(() => new Set()) const [updatingIds, setUpdatingIds] = useState>(() => new Set()) - const addToSet = useCallback((setter: typeof setSyncingIds, id: string) => { - setter((prev) => new Set(prev).add(id)) - }, []) - - const removeFromSet = useCallback((setter: typeof setSyncingIds, id: string) => { - setter((prev) => { - const next = new Set(prev) - next.delete(id) - return next - }) - }, []) - const syncTriggeredAt = useRef>({}) const cooldownTimersRef = useRef> | null>(null) cooldownTimersRef.current ??= new Set() @@ -120,69 +125,61 @@ export function ConnectorsSection({ } }, []) - const isSyncOnCooldown = useCallback((connectorId: string) => { + const isSyncOnCooldown = (connectorId: string) => { const triggeredAt = syncTriggeredAt.current[connectorId] if (!triggeredAt) return false return Date.now() - triggeredAt < SYNC_COOLDOWN_MS - }, []) + } + + const handleSync = (connectorId: string, rehydrate = false) => { + if (isSyncOnCooldown(connectorId)) return - const handleSync = useCallback( - (connectorId: string, rehydrate = false) => { - if (isSyncOnCooldown(connectorId)) return - - syncTriggeredAt.current[connectorId] = Date.now() - addToSet(setSyncingIds, connectorId) - - triggerSync( - { knowledgeBaseId, connectorId, rehydrate }, - { - onSuccess: () => { - setError(null) - const timer = setTimeout(() => { - cooldownTimersRef.current?.delete(timer) - forceUpdate((n) => n + 1) - }, SYNC_COOLDOWN_MS) - cooldownTimersRef.current?.add(timer) - }, - onError: (err) => { - logger.error('Sync trigger failed', { error: err.message }) - setError(err.message) - delete syncTriggeredAt.current[connectorId] + syncTriggeredAt.current[connectorId] = Date.now() + addToSet(setSyncingIds, connectorId) + + triggerSync( + { knowledgeBaseId, connectorId, rehydrate }, + { + onSuccess: () => { + setError(null) + const timer = setTimeout(() => { + cooldownTimersRef.current?.delete(timer) forceUpdate((n) => n + 1) - }, - onSettled: () => removeFromSet(setSyncingIds, connectorId), - } - ) - }, - [knowledgeBaseId, triggerSync, isSyncOnCooldown, addToSet, removeFromSet] - ) + }, SYNC_COOLDOWN_MS) + cooldownTimersRef.current?.add(timer) + }, + onError: (err) => { + logger.error('Sync trigger failed', { error: err.message }) + setError(err.message) + delete syncTriggeredAt.current[connectorId] + forceUpdate((n) => n + 1) + }, + onSettled: () => removeFromSet(setSyncingIds, connectorId), + } + ) + } - const handleTogglePause = useCallback( - (connector: ConnectorData) => { - addToSet(setUpdatingIds, connector.id) - updateConnector( - { - knowledgeBaseId, - connectorId: connector.id, - updates: { - status: - connector.status === 'paused' || connector.status === 'disabled' - ? 'active' - : 'paused', - }, + const handleTogglePause = (connector: ConnectorData) => { + addToSet(setUpdatingIds, connector.id) + updateConnector( + { + knowledgeBaseId, + connectorId: connector.id, + updates: { + status: + connector.status === 'paused' || connector.status === 'disabled' ? 'active' : 'paused', }, - { - onSettled: () => removeFromSet(setUpdatingIds, connector.id), - onSuccess: () => setError(null), - onError: (err) => { - logger.error('Toggle pause failed', { error: err.message }) - setError(err.message) - }, - } - ) - }, - [knowledgeBaseId, updateConnector, addToSet, removeFromSet] - ) + }, + { + onSettled: () => removeFromSet(setUpdatingIds, connector.id), + onSuccess: () => setError(null), + onError: (err) => { + logger.error('Toggle pause failed', { error: err.message }) + setError(err.message) + }, + } + ) + } const handleDeleteConnector = () => { if (!deleteTarget) return @@ -315,10 +312,10 @@ function ConnectorCard({ const serviceId = connectorDef?.auth.mode === 'oauth' ? connectorDef.auth.provider : undefined const providerId = serviceId ? getProviderIdFromServiceId(serviceId) : undefined - const requiredScopes = useMemo( - () => (connectorDef?.auth.mode === 'oauth' ? (connectorDef.auth.requiredScopes ?? []) : []), - [connectorDef] - ) + const requiredScopes = + connectorDef?.auth.mode === 'oauth' + ? (connectorDef.auth.requiredScopes ?? EMPTY_REQUIRED_SCOPES) + : EMPTY_REQUIRED_SCOPES const { data: credentials, refetch: refetchCredentials } = useOAuthCredentials(providerId, { workspaceId, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/loading.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/loading.tsx index 5e31ae87167..640117df44b 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/loading.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/loading.tsx @@ -1,7 +1,6 @@ 'use client' -import { Plus } from '@sim/emcn' -import { Database } from '@sim/emcn/icons' +import { Database, Plus } from '@sim/emcn/icons' import { noop } from '@sim/utils/helpers' import { type BreadcrumbItem, @@ -29,7 +28,7 @@ const ACTIONS: ChromeActionSpec[] = [ const BREADCRUMBS: BreadcrumbItem[] = [ { label: KNOWLEDGE_HEADER.rootLabel, icon: Database, onClick: noop }, - { label: '…', icon: Database, terminal: true }, + { label: '…', terminal: true }, ] export default function KnowledgeBaseLoading() { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/search-params.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/search-params.ts index c7f1ae8f27e..1e3436da727 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/search-params.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/search-params.ts @@ -16,22 +16,6 @@ export const addConnectorParam = { parser: parseAsString, } as const -/** - * `page` is the 1-based document-list pagination index for this knowledge base. - * Distinct from the single-document subview's `page` (a different route). The - * default page (1) clears from the URL. - */ -export const pageParam = { - key: 'page', - parser: parseAsInteger.withDefault(1), -} as const - -/** Pagination view-state: clean URLs, no back-stack churn. */ -export const pageUrlKeys = { - history: 'replace', - clearOnDefault: true, -} as const - /** Document `enabled` filter buckets, matching the status filter dropdown. */ const ENABLED_FILTERS = ['all', 'enabled', 'disabled'] as const @@ -56,12 +40,16 @@ export const kbDocumentSortParams = createSortParams(KB_SORT_COLUMNS, { }) /** - * Grouped filter/search URL state for the document list. + * Grouped filter/search/pagination URL state for the document list. * * - `q` is the document name search. The input is controlled directly by the * instant nuqs value; only its URL write is debounced via * `useDebouncedSearchSetter` — never written on every keystroke. * - `enabled` filters by processing/enabled status (`all` clears from the URL). + * - `page` is the 1-based pagination index, grouped here so a search or filter + * change resets it in the SAME write. Resetting it from a second hook escapes + * the search's debounce and writes the URL on every keystroke. Distinct from + * the single-document subview's `page`, which is a different route. * * `tagFilterEntries` is intentionally NOT represented here: it is an array of * rich filter-rule objects (slot, field type, operator, value, value-to per @@ -71,6 +59,7 @@ export const kbDocumentSortParams = createSortParams(KB_SORT_COLUMNS, { export const documentFiltersParsers = { q: parseAsString.withDefault(''), enabled: parseAsStringLiteral(ENABLED_FILTERS).withDefault('all'), + page: parseAsInteger.withDefault(1), } as const /** Filter/search/sort view-state: clean URLs, no back-stack churn. */ diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx index 9722c695371..858aeab0dd5 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx @@ -16,10 +16,9 @@ import { ChipTextarea, type ComboboxOption, cn, - Loader, toast, } from '@sim/emcn' -import { X } from '@sim/emcn/icons' +import { Loader, X } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { useParams } from 'next/navigation' @@ -175,8 +174,8 @@ export const CreateBaseModal = memo(function CreateBaseModal({ const params = useParams() const workspaceId = params.workspaceId as string - const createKnowledgeBaseMutation = useCreateKnowledgeBase(workspaceId) - const deleteKnowledgeBaseMutation = useDeleteKnowledgeBase(workspaceId) + const createKnowledgeBaseMutation = useCreateKnowledgeBase() + const deleteKnowledgeBaseMutation = useDeleteKnowledgeBase() const [submitStatus, setSubmitStatus] = useState(null) const [files, setFiles] = useState([]) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/components/edit-knowledge-base-modal/edit-knowledge-base-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/components/edit-knowledge-base-modal/edit-knowledge-base-modal.tsx index 464a64f4f91..f52d30c48a0 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/components/edit-knowledge-base-modal/edit-knowledge-base-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/components/edit-knowledge-base-modal/edit-knowledge-base-modal.tsx @@ -137,30 +137,24 @@ export const EditKnowledgeBaseModal = memo(function EditKnowledgeBaseModal({
-

Max Size

+

Max Size

{chunkingConfig.maxSize.toLocaleString()} - - tokens - + tokens

-

Min Size

+

Min Size

{chunkingConfig.minSize.toLocaleString()} - - chars - + chars

-

Overlap

+

Overlap

{chunkingConfig.overlap.toLocaleString()} - - tokens - + tokens

diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.test.tsx index c14cc716e7c..3fc840e5322 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.test.tsx @@ -85,4 +85,30 @@ describe('useKnowledgeUpload admission', () => { unmount() }) + + /** + * A partial batch failure still created every document that DID upload, so the caches have + * to reconcile on the throwing path too — otherwise the list renders without rows the + * server already has. + */ + it('reconciles the caches when part of a batch fails', async () => { + const onError = vi.fn() + const { result, unmount } = renderKnowledgeUploadHook(onError) + mockUploadKnowledgeDocumentSession + .mockResolvedValueOnce({ id: 'doc-1', filename: 'ok.bin' }) + .mockRejectedValueOnce(new Error('network died')) + + await act(async () => { + await expect( + result().uploadFiles([sizedFile('ok.bin', 10), sizedFile('bad.bin', 10)], 'kb-1') + ).rejects.toMatchObject({ code: 'PARTIAL_UPLOAD_FAILURE' }) + }) + + expect(mockInvalidateQueries).toHaveBeenCalledWith({ + queryKey: ['knowledge', 'detail', 'kb-1'], + }) + expect(mockInvalidateQueries).toHaveBeenCalledWith({ queryKey: ['knowledge', 'list'] }) + + unmount() + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts index 5923e9e03f6..90ae6736e20 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts @@ -119,6 +119,14 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) { }) } + /** Reconciles both caches an upload moves: the base's documents and the list's `docCount`. */ + const invalidateKnowledgeCaches = async (knowledgeBaseId: string) => { + await Promise.all([ + queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId) }), + queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }), + ]) + } + const uploadFilesInBatches = async ( files: File[], knowledgeBaseId: string, @@ -209,16 +217,21 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) { setUploadProgress((prev) => ({ ...prev, stage: 'processing' })) logger.info(`Successfully started processing ${uploadedDocuments.length} documents`) - await Promise.all([ - queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId) }), - /** The knowledge-base list rows carry `docCount`, so an upload changes them too. */ - queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }), - ]) + await invalidateKnowledgeCaches(knowledgeBaseId) return uploadedDocuments } catch (err) { logger.error('Error uploading documents:', err) + /** + * A partial batch failure still created every document that did upload, so the caches + * must reconcile on this path too — otherwise the list is missing rows that exist until + * its staleTime expires. Admission failures create nothing and need no refetch. + */ + if (err instanceof KnowledgeUploadError && err.code === 'PARTIAL_UPLOAD_FAILURE') { + void invalidateKnowledgeCaches(knowledgeBaseId) + } + const error: UploadError = err instanceof KnowledgeUploadError ? { message: err.message, code: err.code, details: err.details, timestamp: Date.now() } diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx index 0cb354d4c45..207f8b9599e 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx @@ -2,8 +2,8 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import type { ChipDropdownOption } from '@sim/emcn' -import { Button, ChipConfirmModal, ChipDropdown, Plus, Tooltip, toast } from '@sim/emcn' -import { Database, FolderPlus, Pencil, Trash } from '@sim/emcn/icons' +import { Button, ChipConfirmModal, ChipDropdown, Tooltip, toast } from '@sim/emcn' +import { Database, FolderPlus, Pencil, Plus, Trash } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { useParams, useRouter } from 'next/navigation' @@ -220,8 +220,8 @@ export function Knowledge() { const canEditRef = useRef(canEdit) canEditRef.current = canEdit - const { mutateAsync: updateKnowledgeBaseMutation } = useUpdateKnowledgeBase(workspaceId) - const deleteKnowledgeBase = useDeleteKnowledgeBase(workspaceId) + const { mutateAsync: updateKnowledgeBaseMutation } = useUpdateKnowledgeBase() + const deleteKnowledgeBase = useDeleteKnowledgeBase() const bulkMoveKnowledgeBases = useBulkMoveKnowledgeBases(workspaceId) const bulkDeleteKnowledgeBases = useBulkDeleteKnowledgeBases(workspaceId) @@ -694,14 +694,12 @@ export function Knowledge() { [selectedRowIds] ) - const bulkDeleteLabel = useMemo(() => { - const count = selectedKnowledgeBaseIds.length + selectedFolderIds.length - const firstName = - selectedKnowledgeBaseIds.length > 0 - ? knowledgeBasesRef.current.find((kb) => kb.id === selectedKnowledgeBaseIds[0])?.name - : foldersRef.current.find((folder) => folder.id === selectedFolderIds[0])?.name - return selectionLabel(count, firstName) - }, [selectedKnowledgeBaseIds, selectedFolderIds]) + const bulkDeleteCount = selectedKnowledgeBaseIds.length + selectedFolderIds.length + const bulkDeleteFirstName = + selectedKnowledgeBaseIds.length > 0 + ? knowledgeBases.find((kb) => kb.id === selectedKnowledgeBaseIds[0])?.name + : folders.find((folder) => folder.id === selectedFolderIds[0])?.name + const bulkDeleteLabel = selectionLabel(bulkDeleteCount, bulkDeleteFirstName) const handleRowClick = useCallback( (rowId: string) => { @@ -1422,8 +1420,8 @@ export function Knowledge() { onOpenChange={(open) => { if (!open) setFolderPendingDelete(null) }} - srTitle='Delete folder' - title='Delete folder' + srTitle='Delete Folder' + title='Delete Folder' text={[ 'Are you sure you want to delete ', { text: folderPendingDelete?.name ?? 'this folder', bold: true }, @@ -1440,8 +1438,8 @@ export function Knowledge() { { + const fetchTagDefinitions = async () => { if (!knowledgeBaseId) return await queryClient.invalidateQueries({ queryKey: knowledgeKeys.tagDefinitions(knowledgeBaseId), }) - }, [queryClient, knowledgeBaseId]) + } - const tagDefinitions = useMemo(() => (query.data ?? []) as TagDefinition[], [query.data]) + const tagDefinitions = (query.data ?? EMPTY_TAG_DEFINITIONS) as TagDefinition[] return { tagDefinitions, isLoading: query.isLoading, - error: query.error instanceof Error ? query.error.message : null, + error: query.error ? getErrorMessage(query.error) : null, fetchTagDefinitions, } } diff --git a/apps/sim/hooks/kb/use-knowledge.ts b/apps/sim/hooks/kb/use-knowledge.ts index 5c1f18142c8..cb20889e09f 100644 --- a/apps/sim/hooks/kb/use-knowledge.ts +++ b/apps/sim/hooks/kb/use-knowledge.ts @@ -1,7 +1,8 @@ -import { useCallback, useMemo } from 'react' +import { useCallback } from 'react' +import { getErrorMessage } from '@sim/utils/errors' import { useQueryClient } from '@tanstack/react-query' import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types' -import type { ChunkData, DocumentData, KnowledgeBaseData } from '@/lib/knowledge/types' +import type { ChunkData, DocumentData } from '@/lib/knowledge/types' import { type DocumentTagFilter, type KnowledgeChunksResponse, @@ -36,7 +37,7 @@ export function useKnowledgeBase(id: string) { knowledgeBase: query.data ?? null, isLoading: query.isLoading, isFetching: query.isFetching, - error: query.error instanceof Error ? query.error.message : null, + error: query.error ? getErrorMessage(query.error) : null, refresh, } } @@ -52,7 +53,7 @@ export function useDocument(knowledgeBaseId: string, documentId: string) { document: query.data ?? null, isLoading: query.isLoading, isFetching: query.isFetching, - error: query.error instanceof Error ? query.error.message : null, + error: query.error ? getErrorMessage(query.error) : null, } } @@ -93,15 +94,12 @@ export function useKnowledgeBaseDocuments( tagFilters, }) - const refetchIntervalFn = useMemo(() => { - if (typeof options?.refetchInterval === 'function') { - const userFn = options.refetchInterval - return (query: { state: { data?: KnowledgeDocumentsResponse } }) => { - return userFn(query.state.data) - } - } - return options?.refetchInterval - }, [options?.refetchInterval]) + const userRefetchInterval = options?.refetchInterval + const refetchIntervalFn = + typeof userRefetchInterval === 'function' + ? (query: { state: { data?: KnowledgeDocumentsResponse } }) => + userRefetchInterval(query.state.data) + : userRefetchInterval const query = useKnowledgeDocumentsQuery( { @@ -128,12 +126,8 @@ export function useKnowledgeBaseDocuments( hasMore: false, } - const hasProcessingDocs = useMemo( - () => - documents.some( - (doc) => doc.processingStatus === 'pending' || doc.processingStatus === 'processing' - ), - [documents] + const hasProcessingDocs = documents.some( + (doc) => doc.processingStatus === 'pending' || doc.processingStatus === 'processing' ) const refreshDocuments = useCallback(async () => { @@ -142,23 +136,20 @@ export function useKnowledgeBaseDocuments( }) }, [queryClient, knowledgeBaseId, paramsKey]) - const updateDocument = useCallback( - (documentId: string, updates: Partial) => { - queryClient.setQueryData( - knowledgeKeys.documents(knowledgeBaseId, paramsKey), - (previous) => { - if (!previous) return previous - return { - ...previous, - documents: previous.documents.map((doc) => - doc.id === documentId ? { ...doc, ...updates } : doc - ), - } + const updateDocument = (documentId: string, updates: Partial) => { + queryClient.setQueryData( + knowledgeKeys.documents(knowledgeBaseId, paramsKey), + (previous) => { + if (!previous) return previous + return { + ...previous, + documents: previous.documents.map((doc) => + doc.id === documentId ? { ...doc, ...updates } : doc + ), } - ) - }, - [knowledgeBaseId, paramsKey, queryClient] - ) + } + ) + } return { documents, @@ -166,7 +157,7 @@ export function useKnowledgeBaseDocuments( isLoading: query.isLoading, isFetching: query.isFetching, isPlaceholderData: query.isPlaceholderData, - error: query.error instanceof Error ? query.error.message : null, + error: query.error ? getErrorMessage(query.error) : null, hasProcessingDocuments: hasProcessingDocs, refreshDocuments, updateDocument, @@ -177,51 +168,15 @@ export function useKnowledgeBaseDocuments( * Hook to fetch and manage knowledge bases list * Uses React Query as single source of truth */ -export function useKnowledgeBasesList( - workspaceId?: string, - options?: { - enabled?: boolean - } -) { - const queryClient = useQueryClient() - const query = useKnowledgeBasesQuery(workspaceId, { enabled: options?.enabled ?? true }) - - const removeKnowledgeBase = useCallback( - (knowledgeBaseId: string) => { - queryClient.setQueryData( - knowledgeKeys.list(workspaceId), - (previous) => previous?.filter((kb) => kb.id !== knowledgeBaseId) ?? [] - ) - }, - [queryClient, workspaceId] - ) - - const updateKnowledgeBase = useCallback( - (id: string, updates: Partial) => { - queryClient.setQueryData( - knowledgeKeys.list(workspaceId), - (previous) => previous?.map((kb) => (kb.id === id ? { ...kb, ...updates } : kb)) ?? [] - ) - queryClient.setQueryData(knowledgeKeys.detail(id), (previous) => - previous ? { ...previous, ...updates } : previous - ) - }, - [queryClient, workspaceId] - ) - - const refreshList = useCallback(async () => { - await queryClient.invalidateQueries({ queryKey: knowledgeKeys.list(workspaceId) }) - }, [queryClient, workspaceId]) +export function useKnowledgeBasesList(workspaceId?: string) { + const query = useKnowledgeBasesQuery(workspaceId) return { knowledgeBases: query.data ?? [], isLoading: query.isLoading, isFetching: query.isFetching, isPlaceholderData: query.isPlaceholderData, - error: query.error instanceof Error ? query.error.message : null, - refreshList, - removeKnowledgeBase, - updateKnowledgeBase, + error: query.error ? getErrorMessage(query.error) : null, } } @@ -270,29 +225,6 @@ export function useDocumentChunks( const hasNextPage = currentPage < totalPages const hasPrevPage = currentPage > 1 - const goToPage = useCallback( - (newPage: number): boolean => { - return newPage >= 1 && newPage <= totalPages - }, - [totalPages] - ) - - const refreshChunks = useCallback(async () => { - const paramsKey = serializeChunkParams({ - knowledgeBaseId, - documentId, - limit: DEFAULT_PAGE_SIZE, - offset, - search: search || undefined, - enabledFilter, - sortBy, - sortOrder, - }) - await queryClient.invalidateQueries({ - queryKey: knowledgeKeys.chunks(knowledgeBaseId, documentId, paramsKey), - }) - }, [knowledgeBaseId, documentId, offset, search, enabledFilter, sortBy, sortOrder, queryClient]) - const updateChunk = useCallback( (chunkId: string, updates: Partial) => { const paramsKey = serializeChunkParams({ @@ -325,13 +257,11 @@ export function useDocumentChunks( chunks, isLoading: chunkQuery.isLoading, isFetching: chunkQuery.isFetching, - error: chunkQuery.error instanceof Error ? chunkQuery.error.message : null, + error: chunkQuery.error ? getErrorMessage(chunkQuery.error) : null, currentPage, totalPages, hasNextPage, hasPrevPage, - goToPage, - refreshChunks, updateChunk, } } diff --git a/apps/sim/hooks/kb/use-tag-definitions.ts b/apps/sim/hooks/kb/use-tag-definitions.ts index 01bb9c18f74..ac751aa45c2 100644 --- a/apps/sim/hooks/kb/use-tag-definitions.ts +++ b/apps/sim/hooks/kb/use-tag-definitions.ts @@ -1,8 +1,13 @@ 'use client' -import { useCallback, useMemo } from 'react' +import { useCallback } from 'react' +import { getErrorMessage } from '@sim/utils/errors' import { useQueryClient } from '@tanstack/react-query' import type { AllTagSlot } from '@/lib/knowledge/constants' +import { + EMPTY_TAG_DEFINITIONS, + type TagDefinition, +} from '@/hooks/kb/use-knowledge-base-tag-definitions' import { type DocumentTagDefinitionInput, useDeleteDocumentTagDefinitions, @@ -11,14 +16,12 @@ import { } from '@/hooks/queries/kb/knowledge' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' -export interface TagDefinition { - id: string - tagSlot: AllTagSlot - displayName: string - fieldType: string - createdAt: string - updatedAt: string -} +/** + * Re-exported so both tag hooks name ONE type: consumers already import `TagDefinition` from + * one of these files and `TagDefinitionInput` from the other, and two structurally identical + * declarations would let them drift with nothing to catch it. + */ +export type { TagDefinition } export interface TagDefinitionInput { tagSlot: AllTagSlot @@ -40,7 +43,7 @@ export function useTagDefinitions( const { mutateAsync: saveTagDefinitionsMutation } = useSaveDocumentTagDefinitions() const { mutateAsync: deleteTagDefinitionsMutation } = useDeleteDocumentTagDefinitions() - const tagDefinitions = useMemo(() => (query.data ?? []) as TagDefinition[], [query.data]) + const tagDefinitions = (query.data ?? EMPTY_TAG_DEFINITIONS) as TagDefinition[] const fetchTagDefinitions = useCallback(async () => { if (!knowledgeBaseId || !documentId) return @@ -49,55 +52,23 @@ export function useTagDefinitions( }) }, [queryClient, knowledgeBaseId, documentId]) - const saveTagDefinitions = useCallback( - async (definitions: TagDefinitionInput[]) => { - if (!knowledgeBaseId || !documentId) { - throw new Error('Knowledge base ID and document ID are required') - } - - return saveTagDefinitionsMutation({ - knowledgeBaseId, - documentId, - definitions: definitions as DocumentTagDefinitionInput[], - }) - }, - [knowledgeBaseId, documentId, saveTagDefinitionsMutation] - ) - - const deleteTagDefinitions = useCallback(async () => { + const saveTagDefinitions = async (definitions: TagDefinitionInput[]) => { if (!knowledgeBaseId || !documentId) { throw new Error('Knowledge base ID and document ID are required') } - return deleteTagDefinitionsMutation({ + return saveTagDefinitionsMutation({ knowledgeBaseId, documentId, + definitions: definitions as DocumentTagDefinitionInput[], }) - }, [knowledgeBaseId, documentId, deleteTagDefinitionsMutation]) - - const getTagLabel = useCallback( - (tagSlot: string): string => { - const definition = tagDefinitions.find((def) => def.tagSlot === tagSlot) - return definition?.displayName || tagSlot - }, - [tagDefinitions] - ) - - const getTagDefinition = useCallback( - (tagSlot: string): TagDefinition | undefined => { - return tagDefinitions.find((def) => def.tagSlot === tagSlot) - }, - [tagDefinitions] - ) + } return { tagDefinitions, isLoading: query.isLoading, - error: query.error instanceof Error ? query.error.message : null, + error: query.error ? getErrorMessage(query.error) : null, fetchTagDefinitions, saveTagDefinitions, - deleteTagDefinitions, - getTagLabel, - getTagDefinition, } } diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index fe4d6d21004..a00aad93346 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -1,4 +1,3 @@ -import { createLogger } from '@sim/logger' import { keepPreviousData, useInfiniteQuery, @@ -24,8 +23,6 @@ import { import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE } from '@/lib/knowledge/constants' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' -const logger = createLogger('KnowledgeConnectorQueries') - export type { ConnectorData, ConnectorDetailData, SyncLogData } export const CONNECTOR_LIST_STALE_TIME = 30 * 1000 @@ -231,9 +228,13 @@ export function useTriggerSync() { return useMutation({ mutationFn: triggerSync, + /** + * The sync itself runs async — the connector list's own syncing poll surfaces its + * progress. Only the connector rows have anything to say yet. + */ onSettled: (_data, _error, { knowledgeBaseId }) => { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), + queryKey: connectorKeys.all(knowledgeBaseId), }) }, }) @@ -244,8 +245,8 @@ export const connectorDocumentKeys = { [...connectorKeys.detail(knowledgeBaseId, connectorId), 'documents'] as const, lists: (knowledgeBaseId?: string, connectorId?: string) => [...connectorDocumentKeys.all(knowledgeBaseId, connectorId), 'list'] as const, - list: (knowledgeBaseId?: string, connectorId?: string) => - connectorDocumentKeys.lists(knowledgeBaseId, connectorId), + list: (knowledgeBaseId?: string, connectorId?: string, includeExcluded = false) => + [...connectorDocumentKeys.lists(knowledgeBaseId, connectorId), includeExcluded] as const, } async function fetchConnectorDocuments( @@ -275,7 +276,7 @@ export function useConnectorDocuments( ) { const includeExcluded = options?.includeExcluded ?? false return useInfiniteQuery({ - queryKey: [...connectorDocumentKeys.list(knowledgeBaseId, connectorId), includeExcluded], + queryKey: connectorDocumentKeys.list(knowledgeBaseId, connectorId, includeExcluded), queryFn: ({ signal, pageParam }) => fetchConnectorDocuments( knowledgeBaseId as string, @@ -323,7 +324,7 @@ export function useExcludeConnectorDocument() { mutationFn: excludeConnectorDocuments, onSettled: (_data, _error, { knowledgeBaseId, connectorId }) => { queryClient.invalidateQueries({ - queryKey: connectorDocumentKeys.list(knowledgeBaseId, connectorId), + queryKey: connectorDocumentKeys.lists(knowledgeBaseId, connectorId), }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), @@ -352,7 +353,7 @@ export function useRestoreConnectorDocument() { mutationFn: restoreConnectorDocuments, onSettled: (_data, _error, { knowledgeBaseId, connectorId }) => { queryClient.invalidateQueries({ - queryKey: connectorDocumentKeys.list(knowledgeBaseId, connectorId), + queryKey: connectorDocumentKeys.lists(knowledgeBaseId, connectorId), }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), diff --git a/apps/sim/hooks/queries/kb/knowledge.test.ts b/apps/sim/hooks/queries/kb/knowledge.test.ts index 4c01ae0fcf5..1fcaaaca386 100644 --- a/apps/sim/hooks/queries/kb/knowledge.test.ts +++ b/apps/sim/hooks/queries/kb/knowledge.test.ts @@ -26,7 +26,12 @@ vi.mock('@/lib/api/client/request', () => ({ requestJson: mocks.requestJson, })) -import { useBulkDocumentOperation, useDeleteDocument } from '@/hooks/queries/kb/knowledge' +import { + useBulkDocumentOperation, + useDeleteDocument, + useUpdateDocument, + useUpdateDocumentTags, +} from '@/hooks/queries/kb/knowledge' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' interface CapturedMutation { @@ -65,6 +70,27 @@ describe('knowledge document mutations', () => { expect(mocks.invalidateQueries).toHaveBeenCalledWith({ queryKey: knowledgeKeys.lists() }) }) + /** + * `documents` (the list pages) and `document` (one row) are siblings under `detail`, so + * invalidating the row's key alone leaves the list rendering the filename, status, tags, and + * counts the write just changed. + */ + it.each([ + ['a document update', () => useUpdateDocument()], + ['a document tag update', () => useUpdateDocumentTags()], + ])('refreshes the document list pages after %s', (_label, build) => { + const mutation = captureMutation(build) + + mutation.onSettled(undefined, undefined, { knowledgeBaseId: 'kb-1', documentId: 'doc-1' }) + + expect(mocks.invalidateQueries).toHaveBeenCalledWith({ + queryKey: knowledgeKeys.documentLists('kb-1'), + }) + expect(mocks.invalidateQueries).toHaveBeenCalledWith({ + queryKey: knowledgeKeys.document('kb-1', 'doc-1'), + }) + }) + it('leaves the knowledge-base lists alone on a bulk enable', () => { const mutation = captureMutation(() => useBulkDocumentOperation()) diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index 5902e0d7be1..4c064e479c9 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -327,6 +327,8 @@ async function fetchAllDocumentChunks( const limit = 100 while (hasMore) { + /** Throw rather than break: a short list returned here would cache as the complete one. */ + signal?.throwIfAborted() const response = await fetchKnowledgeChunks( { knowledgeBaseId, @@ -403,10 +405,11 @@ export function useUpdateChunk() { mutationFn: updateChunk, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), + queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ queryClient.invalidateQueries({ - queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), }) }, }) @@ -435,10 +438,15 @@ export function useDeleteChunk() { mutationFn: deleteChunk, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), + queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ queryClient.invalidateQueries({ - queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), + }) + queryClient.invalidateQueries({ + queryKey: knowledgeKeys.detail(knowledgeBaseId), + exact: true, }) }, }) @@ -472,10 +480,15 @@ export function useCreateChunk() { mutationFn: createChunk, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), + queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ queryClient.invalidateQueries({ - queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), + }) + queryClient.invalidateQueries({ + queryKey: knowledgeKeys.detail(knowledgeBaseId), + exact: true, }) }, }) @@ -512,10 +525,11 @@ export function useUpdateDocument() { mutationFn: updateDocument, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), + queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ queryClient.invalidateQueries({ - queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), }) }, }) @@ -619,7 +633,7 @@ async function createKnowledgeBase(params: CreateKnowledgeBaseParams): Promise { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), + queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ queryClient.invalidateQueries({ - queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), + }) + queryClient.invalidateQueries({ + queryKey: knowledgeKeys.detail(knowledgeBaseId), + exact: true, }) }, }) @@ -785,10 +807,11 @@ export function useUpdateDocumentTags() { mutationFn: updateDocumentTags, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), + queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ queryClient.invalidateQueries({ - queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), }) }, }) diff --git a/apps/sim/hooks/queries/utils/knowledge-keys.ts b/apps/sim/hooks/queries/utils/knowledge-keys.ts index 12edbd76717..743ffc81a81 100644 --- a/apps/sim/hooks/queries/utils/knowledge-keys.ts +++ b/apps/sim/hooks/queries/utils/knowledge-keys.ts @@ -15,6 +15,13 @@ export type KnowledgeQueryScope = KnowledgeScope /** Shared with the server prefetch so a hydrated list and a client fetch never disagree. */ export const KNOWLEDGE_BASE_LIST_STALE_TIME = 60 * 1000 +/** + * `document`, `documents`, `chunks`, `tagDefinitions`, and `tagUsage` all sit UNDER + * `detail(kb)`, so invalidating `detail` non-exactly refetches all of them at once. A mutation + * scoped to one document instead invalidates the two keys that actually render it — its own + * `document` key and the `documentLists` prefix, which are siblings — and, when the base's own + * totals move, `detail` with `exact: true`. + */ export const knowledgeKeys = { all: ['knowledge'] as const, lists: () => [...knowledgeKeys.all, 'list'] as const, @@ -27,8 +34,15 @@ export const knowledgeKeys = { [...knowledgeKeys.detail(knowledgeBaseId), 'tagDefinitions'] as const, tagUsage: (knowledgeBaseId: string) => [...knowledgeKeys.detail(knowledgeBaseId), 'tagUsage'] as const, + /** + * Prefix over every cached page of a base's document list. `documents` and `document` are + * SIBLINGS, not parent and child — a write to one document does not reach the lists that + * render its filename, status, tags, and counts unless this key is invalidated too. + */ + documentLists: (knowledgeBaseId: string) => + [...knowledgeKeys.detail(knowledgeBaseId), 'documents'] as const, documents: (knowledgeBaseId: string, paramsKey: string) => - [...knowledgeKeys.detail(knowledgeBaseId), 'documents', paramsKey] as const, + [...knowledgeKeys.documentLists(knowledgeBaseId), paramsKey] as const, document: (knowledgeBaseId: string, documentId: string) => [...knowledgeKeys.detail(knowledgeBaseId), 'document', documentId] as const, documentTagDefinitions: (knowledgeBaseId: string, documentId: string) => diff --git a/apps/sim/lib/api/list-convention.test.ts b/apps/sim/lib/api/list-convention.test.ts index 80a8257f2a8..77453b972c1 100644 --- a/apps/sim/lib/api/list-convention.test.ts +++ b/apps/sim/lib/api/list-convention.test.ts @@ -56,7 +56,7 @@ vi.mock('@/lib/workflows/skills/builtin-skills', () => ({ import { listVisibleWorkspaceCredentials } from '@/lib/credentials/queries' import { listFoldersForWorkspace } from '@/lib/folders/queries' import { getDocuments } from '@/lib/knowledge/documents/service' -import { getKnowledgeBases } from '@/lib/knowledge/service' +import { getWorkspaceKnowledgeBases } from '@/lib/knowledge/service' import { listWorkspaceMcpServers } from '@/lib/mcp/queries' import { listTables } from '@/lib/table/service' import { listWorkspaceCustomTools } from '@/lib/workflows/custom-tools/operations' @@ -123,7 +123,7 @@ const CASES: ListCase[] = [ column: schemaMock.knowledgeBase.name, table: schemaMock.knowledgeBase, run: ({ search, sortBy, sortOrder }) => - getKnowledgeBases('user-1', WS, 'active', { search, sortBy: sortBy as never, sortOrder }), + getWorkspaceKnowledgeBases(WS, 'active', { search, sortBy: sortBy as never, sortOrder }), sort: { sortBy: 'name', sortOrder: 'asc', diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts index 678f5205498..c92716aa480 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts @@ -15,7 +15,8 @@ const mocks = vi.hoisted(() => ({ updateRecord: vi.fn(), deleteRecord: vi.fn(), listRecords: vi.fn(), - listInternalRecords: vi.fn(), + listLegacyPersonalRecords: vi.fn(), + listVisibleRecords: vi.fn(), getRecord: vi.fn(), getRestorableRecord: vi.fn(), performUpdate: vi.fn(), @@ -81,7 +82,8 @@ vi.mock('@/lib/knowledge/service', () => ({ updateKnowledgeBase: mocks.updateRecord, deleteKnowledgeBase: mocks.deleteRecord, getKnowledgeBaseById: mocks.getRecord, - getKnowledgeBases: mocks.listInternalRecords, + getLegacyPersonalKnowledgeBases: mocks.listLegacyPersonalRecords, + listWorkspaceAndLegacyKnowledgeBases: mocks.listVisibleRecords, getWorkspaceKnowledgeBases: mocks.listRecords, })) @@ -151,7 +153,8 @@ describe('knowledge base application use cases', () => { mocks.loadFolderIndex.mockResolvedValue({ pathById: new Map(), idByPath: new Map() }) mocks.createRecord.mockResolvedValue(knowledgeBase) mocks.listRecords.mockResolvedValue({ data: [], nextCursorKeys: null }) - mocks.listInternalRecords.mockResolvedValue([knowledgeBase]) + mocks.listLegacyPersonalRecords.mockResolvedValue([knowledgeBase]) + mocks.listVisibleRecords.mockResolvedValue([knowledgeBase]) mocks.getRecord.mockResolvedValue(knowledgeBase) mocks.getRestorableRecord.mockResolvedValue(knowledgeBase) mocks.performUpdate.mockResolvedValue({ @@ -190,7 +193,7 @@ describe('knowledge base application use cases', () => { expect(mocks.resolveWorkspace).not.toHaveBeenCalled() expect(mocks.resolvePermission).not.toHaveBeenCalled() - expect(mocks.listInternalRecords).toHaveBeenCalledWith('user-1', undefined, 'all') + expect(mocks.listLegacyPersonalRecords).toHaveBeenCalledWith('user-1', 'all') }) it('authorizes a canonical workspace before listing its internal knowledge bases', async () => { @@ -207,7 +210,27 @@ describe('knowledge base application use cases', () => { undefined, { forUpdate: undefined } ) - expect(mocks.listInternalRecords).toHaveBeenCalledWith('user-1', 'workspace-1', 'archived') + expect(mocks.listVisibleRecords).toHaveBeenCalledWith('user-1', 'workspace-1', 'archived') + expect(mocks.listLegacyPersonalRecords).not.toHaveBeenCalled() + }) + + /** + * Workspace `admin` can come from an organization role alone, with no workspace permission + * row behind it. Re-deriving list access from that row would hide every knowledge base from + * a caller the create path already authorizes — a base they make and then cannot see. + */ + it('lists a workspace for an authorized caller who holds no workspace permission row', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + mocks.listVisibleRecords.mockResolvedValueOnce([knowledgeBase]) + + await expect( + listInternalKnowledgeBases.execute({ + principal: { kind: 'session', userId: 'org-admin-1', sessionId: 'session-1' }, + input: { workspaceId: 'workspace-1', scope: 'active' }, + }) + ).resolves.toEqual({ knowledgeBases: [knowledgeBase] }) + + expect(mocks.listVisibleRecords).toHaveBeenCalledWith('org-admin-1', 'workspace-1', 'active') }) it('loads the active knowledge catalog and tag metadata only after workspace authorization', async () => { @@ -284,7 +307,7 @@ describe('knowledge base application use cases', () => { }) ).rejects.toMatchObject({ code: 'forbidden' }) - expect(mocks.listInternalRecords).not.toHaveBeenCalled() + expect(mocks.listLegacyPersonalRecords).not.toHaveBeenCalled() }) it('rejects non-session principals before resolving internal list input', async () => { @@ -296,7 +319,7 @@ describe('knowledge base application use cases', () => { ).rejects.toMatchObject({ code: 'forbidden' }) expect(mocks.resolveWorkspace).not.toHaveBeenCalled() - expect(mocks.listInternalRecords).not.toHaveBeenCalled() + expect(mocks.listLegacyPersonalRecords).not.toHaveBeenCalled() }) it('rejects an insufficient role before the protected mutation', async () => { diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.ts b/apps/sim/lib/knowledge/application/knowledge-bases.ts index 392e0ada0dd..8884d926d0b 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.ts @@ -56,9 +56,10 @@ import { createAuthorizedKnowledgeBase, deleteKnowledgeBase, getKnowledgeBaseById, - getKnowledgeBases, + getLegacyPersonalKnowledgeBases, getWorkspaceKnowledgeBases, type KnowledgeBaseScope, + listWorkspaceAndLegacyKnowledgeBases, updateKnowledgeBase, } from '@/lib/knowledge/service' import type { ChunkingConfig, KnowledgeBaseWithCounts } from '@/lib/knowledge/types' @@ -438,12 +439,19 @@ export const listInternalKnowledgeBases = { if (principal.kind !== 'session') { throw new PrincipalKindAuthorizationError(principal.kind, knowledgeSessionOperations.list.id) } - if (input.workspaceId !== undefined) { - const context = await resolveKnowledgeWorkspaceContext({ workspaceId: input.workspaceId }) - await authorizeWorkspaceOperation(principal, knowledgeOperations.list, context) + if (input.workspaceId === undefined) { + return { + knowledgeBases: await getLegacyPersonalKnowledgeBases(principal.userId, input.scope), + } } + const context = await resolveKnowledgeWorkspaceContext({ workspaceId: input.workspaceId }) + await authorizeWorkspaceOperation(principal, knowledgeOperations.list, context) return { - knowledgeBases: await getKnowledgeBases(principal.userId, input.workspaceId, input.scope), + knowledgeBases: await listWorkspaceAndLegacyKnowledgeBases( + principal.userId, + context.workspaceId, + input.scope + ), } }, } satisfies OperationUseCase< diff --git a/apps/sim/lib/knowledge/chunks/service.ts b/apps/sim/lib/knowledge/chunks/service.ts index c9bb89776a2..6c6f5155770 100644 --- a/apps/sim/lib/knowledge/chunks/service.ts +++ b/apps/sim/lib/knowledge/chunks/service.ts @@ -206,7 +206,6 @@ export async function createChunk( embeddingModel: kbEmbeddingModel, startOffset: 0, // Manual chunks don't have document offsets endOffset: chunkData.content.length, - // Inherit text tags from parent document tag1: docTags.tag1 as string | null, tag2: docTags.tag2 as string | null, tag3: docTags.tag3 as string | null, @@ -214,16 +213,13 @@ export async function createChunk( tag5: docTags.tag5 as string | null, tag6: docTags.tag6 as string | null, tag7: docTags.tag7 as string | null, - // Inherit number tags from parent document (5 slots) number1: docTags.number1 as number | null, number2: docTags.number2 as number | null, number3: docTags.number3 as number | null, number4: docTags.number4 as number | null, number5: docTags.number5 as number | null, - // Inherit date tags from parent document (2 slots) date1: docTags.date1 as Date | null, date2: docTags.date2 as Date | null, - // Inherit boolean tags from parent document (3 slots) boolean1: docTags.boolean1 as boolean | null, boolean2: docTags.boolean2 as boolean | null, boolean3: docTags.boolean3 as boolean | null, @@ -242,7 +238,6 @@ export async function createChunk( ) } - // Update document statistics await tx .update(document) .set({ @@ -315,12 +310,10 @@ export async function batchChunkOperation( const totalTokensToRemove = chunksToDelete.reduce((sum, chunk) => sum + chunk.tokenCount, 0) const totalCharsToRemove = chunksToDelete.reduce((sum, chunk) => sum + chunk.contentLength, 0) - // Delete chunks const deleteResult = await tx .delete(embedding) .where(and(eq(embedding.documentId, documentId), inArray(embedding.id, chunkIds))) - // Update document statistics await tx .update(document) .set({ @@ -333,7 +326,6 @@ export async function batchChunkOperation( successCount = chunksToDelete.length }) } else { - // Handle enable/disable operations const enabled = operation === 'enable' await db @@ -529,7 +521,6 @@ export async function updateChunk( }) .where(eq(embedding.id, chunkId)) - // Fetch the updated chunk const updatedChunk = await db .select({ id: embedding.id, @@ -588,10 +579,8 @@ export async function deleteChunk( const chunk = chunkToDelete[0] - // Delete the chunk await tx.delete(embedding).where(eq(embedding.id, chunkId)) - // Update document statistics await tx .update(document) .set({ diff --git a/apps/sim/lib/knowledge/constants.ts b/apps/sim/lib/knowledge/constants.ts index 7b75bc6ae17..3e6310eba32 100644 --- a/apps/sim/lib/knowledge/constants.ts +++ b/apps/sim/lib/knowledge/constants.ts @@ -4,6 +4,13 @@ import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' export const KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH = 10_000 /** Hard bound for full-workspace knowledge-base list projections. */ export const MAX_KNOWLEDGE_BASES_PER_WORKSPACE = 10_000 + +/** + * Cap on one caller's legacy workspace-less knowledge bases. Separate from the per-workspace + * cap because it bounds a per-user set governed by no workspace rule — the two limits should + * be free to move independently. + */ +export const MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES = 10_000 /** Hard bound for path-indexed knowledge folder trees and recursive cascades. */ export const MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE = MAX_FOLDERS_PER_WORKSPACE diff --git a/apps/sim/lib/knowledge/documents/types.ts b/apps/sim/lib/knowledge/documents/types.ts index 5ef9c8c71d0..65e165011b0 100644 --- a/apps/sim/lib/knowledge/documents/types.ts +++ b/apps/sim/lib/knowledge/documents/types.ts @@ -1,4 +1,3 @@ -// Document sorting options export type DocumentSortField = | 'filename' | 'fileSize' diff --git a/apps/sim/lib/knowledge/documents/utils.ts b/apps/sim/lib/knowledge/documents/utils.ts index 89f30c7db99..b180e9b4bee 100644 --- a/apps/sim/lib/knowledge/documents/utils.ts +++ b/apps/sim/lib/knowledge/documents/utils.ts @@ -47,7 +47,6 @@ function isRetryableErrorType(error: unknown): error is RetryableError { export function isRetryableError(error: unknown): boolean { if (!isRetryableErrorType(error)) return false - // Check for rate limiting status codes if ( hasStatus(error) && (error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504) @@ -55,7 +54,6 @@ export function isRetryableError(error: unknown): boolean { return true } - // Check for network-level errors (DNS, connection, timeout) const errorMessage = toError(error).message const lowerMessage = errorMessage.toLowerCase() @@ -77,7 +75,6 @@ export function isRetryableError(error: unknown): boolean { return true } - // Check for rate limiting in error messages const rateLimitKeywords = [ 'rate limit', 'rate_limit', @@ -124,13 +121,11 @@ export async function retryWithExponentialBackoff( lastError = toError(error) logger.warn(`Operation failed on attempt ${attempt + 1}`, { error }) - // If this is the last attempt, throw the error if (attempt === maxRetries) { logger.error(`Operation failed after ${maxRetries + 1} attempts`, { error }) throw lastError } - // Check if error is retryable if (!retryCondition(error as RetryableError)) { logger.warn('Error is not retryable, throwing immediately', { error }) throw lastError @@ -187,7 +182,6 @@ export async function fetchWithRetry( return retryWithExponentialBackoff(async () => { const response = await fetch(url, options) - // If response is not ok and status indicates rate limiting, throw an error if (!response.ok && isRetryableError({ status: response.status })) { const errorText = await response.text() const error: HTTPError = new Error( diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index a28a2e78536..9c2867ea857 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -95,7 +95,6 @@ export interface SearchParams { distanceThreshold?: number } -// Use shared embedding utility export { generateSearchEmbedding } from '@/lib/knowledge/embeddings' /** All valid tag slot keys */ @@ -173,7 +172,6 @@ function buildFilterCondition(filter: StructuredFilter, embeddingTable: any) { const column = embeddingTable[tagSlot] if (!column) return null - // Handle text operators if (fieldType === 'text') { const coerced = coerceTagFilterValue(value, 'text') if (!coerced.ok) return null @@ -197,7 +195,6 @@ function buildFilterCondition(filter: StructuredFilter, embeddingTable: any) { } } - // Handle number operators if (fieldType === 'number') { const coerced = coerceTagFilterValue(value, 'number') if (!coerced.ok) return null @@ -228,7 +225,7 @@ function buildFilterCondition(filter: StructuredFilter, embeddingTable: any) { } } - // Handle date operators - expects YYYY-MM-DD format from frontend + // Date values arrive as YYYY-MM-DD strings from the frontend. if (fieldType === 'date') { const coerced = coerceTagFilterValue(value, 'date') if (!coerced.ok) return null @@ -262,7 +259,6 @@ function buildFilterCondition(filter: StructuredFilter, embeddingTable: any) { } } - // Handle boolean operators if (fieldType === 'boolean') { const coerced = coerceTagFilterValue(value, 'boolean') if (!coerced.ok) return null @@ -277,7 +273,6 @@ function buildFilterCondition(filter: StructuredFilter, embeddingTable: any) { } } - // Fallback to equality return sql`${column} = ${value}` } @@ -438,7 +433,6 @@ export async function handleTagOnlySearch(params: SearchParams): Promise { @@ -496,7 +490,6 @@ export async function handleVectorOnlySearch(params: SearchParams): Promise`${embedding.embedding} <=> ${queryVector}::vector`.as('distance') if (strategy.useParallel) { - // Parallel approach for many KBs const parallelLimit = Math.ceil(topK / knowledgeBaseIds.length) + 5 const queryPromises = knowledgeBaseIds.map(async (kbId) => { @@ -734,14 +727,12 @@ export async function handleTagAndVectorSearch(params: SearchParams): Promise r.id), queryVector, diff --git a/apps/sim/lib/knowledge/service.test.ts b/apps/sim/lib/knowledge/service.test.ts index 654e91d1d3f..584633d9c1a 100644 --- a/apps/sim/lib/knowledge/service.test.ts +++ b/apps/sim/lib/knowledge/service.test.ts @@ -41,9 +41,10 @@ vi.mock('@/lib/billing/core/usage', () => ({ import { MAX_KNOWLEDGE_BASES_PER_WORKSPACE } from '@/lib/knowledge/constants' import { - getKnowledgeBases, + getLegacyPersonalKnowledgeBases, getWorkspaceKnowledgeBases, KnowledgeBasePermissionError, + listWorkspaceAndLegacyKnowledgeBases, updateKnowledgeBase, } from '@/lib/knowledge/service' @@ -68,54 +69,95 @@ describe('getWorkspaceKnowledgeBases — bounded reads', () => { }) /** - * The listing query authorizes on current workspace membership, never on stale creator - * identity: a user removed from a workspace must stop seeing knowledge bases they created - * there. The creator fallback exists only for legacy knowledge bases with no `workspaceId`. + * Legacy knowledge bases predate workspaces and carry no `workspaceId`, so their creator is + * the only possible authority. Workspace-owned rows are read by `getWorkspaceKnowledgeBases` + * after an application use case authorized the workspace — this query must never widen to + * them, and must never re-derive workspace access from a `permissions` row, which would + * contradict an authorization that already passed. */ -describe('getKnowledgeBases — creator fallback is scoped to legacy non-workspace KBs', () => { +describe('getLegacyPersonalKnowledgeBases', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() }) - /** Every disjunct that grants on `knowledgeBase.userId`, from the last select chain's WHERE. */ - const capturedCreatorBranches = (): unknown[] => { + it('reads only the caller’s workspace-less rows', async () => { + await getLegacyPersonalKnowledgeBases('user-a', 'all') + const [condition] = dbChainMockFns.where.mock.calls.at(-1) ?? [] - const orNode = flattenMockConditions(condition).find((node) => node.type === 'or') - expect(orNode, 'WHERE clause has no or(...) branch').toBeDefined() - return (orNode?.conditions as unknown[]).filter((disjunct) => + expect( hasMockCondition( - disjunct, + condition, (node) => node.type === 'eq' && node.left === schemaMock.knowledgeBase.userId && node.right === 'user-a' ) - ) - } - - /** The creator fallback must be the sole grant for legacy KBs and never reach workspace KBs. */ - const expectCreatorBranchIsLegacyOnly = () => { - const branches = capturedCreatorBranches() - expect(branches).toHaveLength(1) + ).toBe(true) expect( hasMockCondition( - branches[0], + condition, (node) => node.type === 'isNull' && node.column === schemaMock.knowledgeBase.workspaceId ) ).toBe(true) - } + expect(flattenMockConditions(condition).some((node) => node.type === 'or')).toBe(false) + }) + + it('never joins the permissions table', async () => { + await getLegacyPersonalKnowledgeBases('user-a') + + const joinedTables = dbChainMockFns.leftJoin.mock.calls.map(([table]) => table) + expect(joinedTables).toContain(schemaMock.document) + expect(joinedTables).not.toContain(schemaMock.permissions) + }) + + it('fails before projecting connector data for an oversized set', async () => { + dbChainMockFns.limit.mockResolvedValueOnce( + Array.from({ length: MAX_KNOWLEDGE_BASES_PER_WORKSPACE + 1 }, (_, index) => ({ + id: `kb-${index}`, + })) + ) - it('requires workspaceId IS NULL on the creator branch when no workspace filter is given', async () => { - await getKnowledgeBases('user-a', undefined, 'all') + await expect(getLegacyPersonalKnowledgeBases('user-a')).rejects.toThrow( + `Legacy personal knowledge base list exceeds the ${MAX_KNOWLEDGE_BASES_PER_WORKSPACE} row limit` + ) + }) +}) - expectCreatorBranchIsLegacyOnly() +/** + * The workspace list and the legacy personal list are separate reads answering to separate + * authorities, but they render as ONE list — so the merge has to order them together and + * project connectors once over the result, not once per source. + */ +describe('listWorkspaceAndLegacyKnowledgeBases', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() }) - it('keeps the same guard on the workspace-filtered branch', async () => { - await getKnowledgeBases('user-a', 'ws-1', 'active') + it('orders both sources as one list and projects connectors once', async () => { + const workspaceRow = { + id: 'kb-workspace', + chunkingConfig: {}, + docCount: 0, + createdAt: new Date('2026-02-01T00:00:00Z'), + } + const legacyRow = { + id: 'kb-legacy', + chunkingConfig: {}, + docCount: 0, + createdAt: new Date('2025-01-01T00:00:00Z'), + } + dbChainMockFns.limit + .mockResolvedValueOnce([workspaceRow]) + .mockResolvedValueOnce([legacyRow]) + .mockResolvedValueOnce([]) + + const result = await listWorkspaceAndLegacyKnowledgeBases('user-a', 'ws-1') - expectCreatorBranchIsLegacyOnly() + expect(result.map((kb) => kb.id)).toEqual(['kb-legacy', 'kb-workspace']) + /** Two row reads and ONE connector projection — three chains, never four. */ + expect(dbChainMockFns.limit).toHaveBeenCalledTimes(3) }) }) diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index d3b9ffa5e37..2af47ea2525 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -1,16 +1,10 @@ import { db } from '@sim/db' -import { - document, - knowledgeBase, - knowledgeConnector, - permissions, - workspace, - workspaceFiles, -} from '@sim/db/schema' +import { document, knowledgeBase, knowledgeConnector, workspaceFiles } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getPostgresErrorCode } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { and, count, eq, exists, inArray, isNotNull, isNull, ne, or, sql } from 'drizzle-orm' +import type { SQL } from 'drizzle-orm' +import { and, count, eq, exists, inArray, isNotNull, isNull, ne, sql } from 'drizzle-orm' import type { V2KnowledgeBaseSortBy } from '@/lib/api/contracts/v2/knowledge' import type { CursorKey, KeysetKey, ListSortOrder } from '@/lib/api/list-query' import { @@ -37,6 +31,7 @@ import { findActiveFolder, resolveRestoredFolderId } from '@/lib/folders/queries import { MAX_KNOWLEDGE_BASES_PER_WORKSPACE, MAX_KNOWLEDGE_CONNECTOR_TYPE_ROWS_PER_LIST, + MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES, } from '@/lib/knowledge/constants' import type { ChunkingConfig, @@ -170,6 +165,63 @@ export interface GetKnowledgeBasesOptions { cursorKeys?: CursorKey[] } +/** `active` hides soft-deleted rows, `archived` shows only them, `all` filters neither. */ +function knowledgeBaseScopeCondition(scope: KnowledgeBaseScope) { + if (scope === 'all') return undefined + return scope === 'archived' + ? sql`${knowledgeBase.deletedAt} IS NOT NULL` + : isNull(knowledgeBase.deletedAt) +} + +/** + * The one projection every knowledge-base list renders: the base's own columns plus its live + * document count. Both list queries read through here so a column added to one list can never + * be missing from the other — they are concatenated into a single rendered list. + */ +async function readKnowledgeBaseRows( + where: SQL | undefined, + orderBy: SQL[], + limit: number +): Promise>> { + const rows = await db + .select({ + id: knowledgeBase.id, + userId: knowledgeBase.userId, + name: knowledgeBase.name, + description: knowledgeBase.description, + tokenCount: sql`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number), + embeddingModel: knowledgeBase.embeddingModel, + embeddingDimension: knowledgeBase.embeddingDimension, + chunkingConfig: knowledgeBase.chunkingConfig, + createdAt: knowledgeBase.createdAt, + updatedAt: knowledgeBase.updatedAt, + deletedAt: knowledgeBase.deletedAt, + workspaceId: knowledgeBase.workspaceId, + folderId: knowledgeBase.folderId, + docCount: count(document.id), + }) + .from(knowledgeBase) + .leftJoin( + document, + and( + eq(document.knowledgeBaseId, knowledgeBase.id), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) + ) + .where(where) + .groupBy(knowledgeBase.id) + .orderBy(...orderBy) + .limit(limit) + + return rows.map((kb) => ({ + ...kb, + chunkingConfig: kb.chunkingConfig as ChunkingConfig, + docCount: Number(kb.docCount), + })) +} + async function attachConnectorTypes( knowledgeBases: Array> ): Promise { @@ -215,11 +267,14 @@ async function attachConnectorTypes( * authorization. Unlike the legacy user-oriented query, this never widens the * scope to workspace-less rows and never depends on a human permission join. */ -export async function getWorkspaceKnowledgeBases( +async function readWorkspaceKnowledgeBaseRows( workspaceId: string, - scope: KnowledgeBaseScope = 'active', + scope: KnowledgeBaseScope, options?: GetKnowledgeBasesOptions -): Promise<{ data: KnowledgeBaseWithCounts[]; nextCursorKeys: CursorKey[] | null }> { +): Promise<{ + data: Array> + nextCursorKeys: CursorKey[] | null +}> { const { folderId, search, @@ -229,63 +284,28 @@ export async function getWorkspaceKnowledgeBases( cursorKeys, } = options ?? {} const keys = KNOWLEDGE_BASE_SORTS[sortBy] - const resumeAfter = resumeKeyset(keys, cursorKeys, sortOrder) /** * An unpaged read still reads one row past the cap so an oversized workspace * is a hard failure rather than a silently truncated list. */ const readLimit = (limit ?? MAX_KNOWLEDGE_BASES_PER_WORKSPACE) + 1 - const scopeCondition = - scope === 'all' - ? undefined - : scope === 'archived' - ? sql`${knowledgeBase.deletedAt} IS NOT NULL` - : isNull(knowledgeBase.deletedAt) - const rows = await db - .select({ - id: knowledgeBase.id, - userId: knowledgeBase.userId, - name: knowledgeBase.name, - description: knowledgeBase.description, - tokenCount: sql`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number), - embeddingModel: knowledgeBase.embeddingModel, - embeddingDimension: knowledgeBase.embeddingDimension, - chunkingConfig: knowledgeBase.chunkingConfig, - createdAt: knowledgeBase.createdAt, - updatedAt: knowledgeBase.updatedAt, - deletedAt: knowledgeBase.deletedAt, - workspaceId: knowledgeBase.workspaceId, - folderId: knowledgeBase.folderId, - docCount: count(document.id), - }) - .from(knowledgeBase) - .leftJoin( - document, - and( - eq(document.knowledgeBaseId, knowledgeBase.id), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .where( - and( - eq(knowledgeBase.workspaceId, workspaceId), - scopeCondition, - folderId === undefined - ? undefined - : folderId === null - ? isNull(knowledgeBase.folderId) - : eq(knowledgeBase.folderId, folderId), - searchFilter(knowledgeBase.name, search), - resumeAfter - ) - ) - .groupBy(knowledgeBase.id) - .orderBy(...listOrderBy(keysetColumns(keys), sortOrder)) - .limit(readLimit) + const rows = await readKnowledgeBaseRows( + and( + eq(knowledgeBase.workspaceId, workspaceId), + knowledgeBaseScopeCondition(scope), + folderId === undefined + ? undefined + : folderId === null + ? isNull(knowledgeBase.folderId) + : eq(knowledgeBase.folderId, folderId), + searchFilter(knowledgeBase.name, search), + resumeKeyset(keys, cursorKeys, sortOrder) + ), + listOrderBy(keysetColumns(keys), sortOrder), + readLimit + ) if (limit === undefined && rows.length > MAX_KNOWLEDGE_BASES_PER_WORKSPACE) { throw new Error( @@ -293,146 +313,92 @@ export async function getWorkspaceKnowledgeBases( ) } - const page = keysetPage(keys, rows, limit) + return keysetPage(keys, rows, limit) +} +export async function getWorkspaceKnowledgeBases( + workspaceId: string, + scope: KnowledgeBaseScope = 'active', + options?: GetKnowledgeBasesOptions +): Promise<{ data: KnowledgeBaseWithCounts[]; nextCursorKeys: CursorKey[] | null }> { + const page = await readWorkspaceKnowledgeBaseRows(workspaceId, scope, options) return { - data: await attachConnectorTypes( - page.data.map((kb) => ({ - ...kb, - chunkingConfig: kb.chunkingConfig as ChunkingConfig, - docCount: Number(kb.docCount), - })) - ), + data: await attachConnectorTypes(page.data), nextCursorKeys: page.nextCursorKeys, } } /** - * Get knowledge bases that a user can access. + * Lists the caller's legacy personal knowledge bases — the ones that predate workspaces and + * carry no `workspaceId`, where the creator is the only possible authority. Workspace-owned + * rows are read by {@link getWorkspaceKnowledgeBases} after an application use case has + * authorized the workspace; nothing here re-derives that access. * - * Filter and sort are applied in the query, so a search costs one narrowed scan - * rather than materializing every knowledge base the caller can reach. + * @deprecated Nothing creates workspace-less knowledge bases any more, so this population only + * shrinks. Backfill the remaining rows onto a workspace and this function, its branch in + * {@link listWorkspaceAndLegacyKnowledgeBases}, and the concept itself can go. */ -export async function getKnowledgeBases( +async function readLegacyPersonalKnowledgeBaseRows( userId: string, - workspaceId?: string | null, - scope: KnowledgeBaseScope = 'active', - options?: GetKnowledgeBasesOptions -): Promise { - const { folderId, search, sortBy = 'createdAt', sortOrder = 'asc' } = options ?? {} - const scopeCondition = - scope === 'all' - ? undefined - : scope === 'archived' - ? sql`${knowledgeBase.deletedAt} IS NOT NULL` - : isNull(knowledgeBase.deletedAt) - - /** - * Legacy knowledge bases predate workspaces and have no `workspaceId`, so the creator is - * their only possible authority. Anything with a `workspaceId` must clear - * `currentWorkspaceMembership` instead — creator identity goes stale the moment a member - * is removed from the workspace. - */ - const legacyOwnedKnowledgeBase = and( - eq(knowledgeBase.userId, userId), - isNull(knowledgeBase.workspaceId) - ) - const currentWorkspaceMembership = and( - isNotNull(permissions.userId), - isNull(workspace.archivedAt) + scope: KnowledgeBaseScope +): Promise>> { + const rows = await readKnowledgeBaseRows( + and( + knowledgeBaseScopeCondition(scope), + eq(knowledgeBase.userId, userId), + isNull(knowledgeBase.workspaceId) + ), + listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS.createdAt), 'asc'), + MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES + 1 ) - const knowledgeBasesWithCounts = await db - .select({ - id: knowledgeBase.id, - userId: knowledgeBase.userId, - name: knowledgeBase.name, - description: knowledgeBase.description, - tokenCount: sql`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number), - embeddingModel: knowledgeBase.embeddingModel, - embeddingDimension: knowledgeBase.embeddingDimension, - chunkingConfig: knowledgeBase.chunkingConfig, - createdAt: knowledgeBase.createdAt, - updatedAt: knowledgeBase.updatedAt, - deletedAt: knowledgeBase.deletedAt, - workspaceId: knowledgeBase.workspaceId, - folderId: knowledgeBase.folderId, - docCount: count(document.id), - }) - .from(knowledgeBase) - .leftJoin( - document, - and( - eq(document.knowledgeBaseId, knowledgeBase.id), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .leftJoin( - permissions, - and( - eq(permissions.entityType, 'workspace'), - eq(permissions.entityId, knowledgeBase.workspaceId), - eq(permissions.userId, userId) - ) - ) - .leftJoin(workspace, eq(knowledgeBase.workspaceId, workspace.id)) - .where( - and( - scopeCondition, - folderId === undefined - ? undefined - : folderId === null - ? isNull(knowledgeBase.folderId) - : eq(knowledgeBase.folderId, folderId), - searchFilter(knowledgeBase.name, search), - or( - and( - workspaceId ? eq(knowledgeBase.workspaceId, workspaceId) : undefined, - currentWorkspaceMembership - ), - legacyOwnedKnowledgeBase - ) - ) + /** One row past the cap, so an oversized set fails loudly instead of truncating in silence. */ + if (rows.length > MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES) { + throw new Error( + `Legacy personal knowledge base list exceeds the ${MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES} row limit` ) - .groupBy(knowledgeBase.id) - .orderBy(...listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS[sortBy]), sortOrder)) - - const kbIds = knowledgeBasesWithCounts.map((kb) => kb.id) + } - const connectorRows = - kbIds.length > 0 - ? await db - .select({ - knowledgeBaseId: knowledgeConnector.knowledgeBaseId, - connectorType: knowledgeConnector.connectorType, - }) - .from(knowledgeConnector) - .where( - and( - inArray(knowledgeConnector.knowledgeBaseId, kbIds), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - : [] + return rows +} - const connectorTypesByKb = new Map() - for (const row of connectorRows) { - const types = connectorTypesByKb.get(row.knowledgeBaseId) ?? [] - if (!types.includes(row.connectorType)) { - types.push(row.connectorType) - } - connectorTypesByKb.set(row.knowledgeBaseId, types) - } +export async function getLegacyPersonalKnowledgeBases( + userId: string, + scope: KnowledgeBaseScope = 'active' +): Promise { + return attachConnectorTypes(await readLegacyPersonalKnowledgeBaseRows(userId, scope)) +} - return knowledgeBasesWithCounts.map((kb) => ({ - ...kb, - chunkingConfig: kb.chunkingConfig as ChunkingConfig, - docCount: Number(kb.docCount), - connectorTypes: connectorTypesByKb.get(kb.id) ?? [], - })) +/** + * Every knowledge base a caller can see under one workspace, as one ordered list. + * + * Two reads, because the list answers to two authorities. The workspace's own rows are read + * once the caller has been authorized FOR that workspace — re-deriving that access from a + * `permissions` row would contradict the authorization that just passed, since workspace + * `admin` can come from an organization role with no such row behind it. Legacy workspace-less + * bases answer only to their creator and belong under no workspace at all, so they ride along + * here; otherwise they are reachable from nowhere. + * + * Callers authorize first. Nothing here decides access. + */ +export async function listWorkspaceAndLegacyKnowledgeBases( + userId: string, + workspaceId: string, + scope: KnowledgeBaseScope = 'active' +): Promise { + const [workspaceRows, legacyPersonalRows] = await Promise.all([ + readWorkspaceKnowledgeBaseRows(workspaceId, scope).then((page) => page.data), + readLegacyPersonalKnowledgeBaseRows(userId, scope), + ]) + + /** One connector projection over the merged set, rather than one per source. */ + return attachConnectorTypes( + legacyPersonalRows.length === 0 + ? workspaceRows + : [...workspaceRows, ...legacyPersonalRows].sort( + (a, b) => a.createdAt.getTime() - b.createdAt.getTime() + ) + ) } /** diff --git a/apps/sim/lib/knowledge/tags/service.ts b/apps/sim/lib/knowledge/tags/service.ts index bee395eace1..5fe313989a3 100644 --- a/apps/sim/lib/knowledge/tags/service.ts +++ b/apps/sim/lib/knowledge/tags/service.ts @@ -261,34 +261,28 @@ export async function createOrUpdateTagDefinitionsBulk( const updated: DocumentTagDefinition[] = [] const errors: string[] = [] - // Get existing definitions to check for conflicts and determine operations const existingDefinitions = await getDocumentTagDefinitions(knowledgeBaseId) const existingBySlot = new Map(existingDefinitions.map((def) => [def.tagSlot, def])) const existingByDisplayName = new Map(existingDefinitions.map((def) => [def.displayName, def])) - // Process each definition for (const defData of definitions) { try { const { tagSlot, displayName, fieldType, originalDisplayName } = defData - // Validate field type if (!SUPPORTED_FIELD_TYPES.includes(fieldType as (typeof SUPPORTED_FIELD_TYPES)[number])) { errors.push(`Invalid field type: ${fieldType}`) continue } - // Check if this is an update (has originalDisplayName) or create const isUpdate = !!originalDisplayName if (isUpdate) { - // Update existing definition const existingDef = existingByDisplayName.get(originalDisplayName!) if (!existingDef) { errors.push(`Tag definition with display name "${originalDisplayName}" not found`) continue } - // Check if new display name conflicts with another definition if (displayName !== originalDisplayName && existingByDisplayName.has(displayName)) { errors.push(`Display name "${displayName}" already exists`) continue @@ -314,10 +308,8 @@ export async function createOrUpdateTagDefinitionsBulk( updatedAt: now, }) } else { - // Create new definition let finalTagSlot = tagSlot - // If no slot provided or slot is taken, find next available if (!finalTagSlot || existingBySlot.has(finalTagSlot)) { const nextSlot = await getNextAvailableSlot(knowledgeBaseId, fieldType, existingBySlot) if (!nextSlot) { @@ -327,13 +319,11 @@ export async function createOrUpdateTagDefinitionsBulk( finalTagSlot = nextSlot } - // Check slot conflicts if (existingBySlot.has(finalTagSlot)) { errors.push(`Tag slot "${finalTagSlot}" is already in use`) continue } - // Check display name conflicts if (existingByDisplayName.has(displayName)) { errors.push(`Display name "${displayName}" already exists`) continue @@ -660,7 +650,6 @@ export async function getTagUsage( const tagSlot = def.tagSlot validateTagSlot(tagSlot) - // Build WHERE conditions based on field type // Text columns need both IS NOT NULL and != '' checks // Numeric/date/boolean columns only need IS NOT NULL const fieldType = getFieldTypeForSlot(tagSlot) @@ -674,7 +663,6 @@ export async function getTagUsage( isNotNull(sql`${sql.raw(tagSlot)}`), ] - // Only add empty string check for text columns if (isTextColumn) { whereConditions.push(sql`${sql.raw(tagSlot)} != ''`) } diff --git a/apps/sim/lib/knowledge/tags/utils.ts b/apps/sim/lib/knowledge/tags/utils.ts index 58d000a3664..1d00da28889 100644 --- a/apps/sim/lib/knowledge/tags/utils.ts +++ b/apps/sim/lib/knowledge/tags/utils.ts @@ -136,7 +136,6 @@ export function parseNumberValue(value: string): number | null { export function parseDateValue(value: string): Date | null { const stringValue = String(value).trim() - // Must be YYYY-MM-DD format if (!/^\d{4}-\d{2}-\d{2}$/.test(stringValue)) { return null }