From ee1bbc82d84a20ecc655df7a7152a4dc65958c5d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 16 Aug 2026 21:34:00 -0700 Subject: [PATCH 1/5] fix(knowledge): list a workspace's bases on the same authority that creates them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /api/knowledge authorizes the session against the canonical workspace and then re-derives access inside the row query from a `permissions` join. Those two no longer agree: workspace `admin` can come from an organization role alone, with no workspace permission row behind it. Such a caller passes authorization, creates a knowledge base, and then sees an empty list forever — the row is filtered out by the join. Tables and files carry no equivalent join, which is why only knowledge is affected. Read the workspace's own rows through `getWorkspaceKnowledgeBases` once the operation is authorized. The caller-scoped query stays only on the path with no workspace to authorize against. --- .../application/knowledge-bases.test.ts | 22 ++++++++++++++++++- .../knowledge/application/knowledge-bases.ts | 17 +++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts index 678f5205498..a64e6938e10 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts @@ -207,7 +207,27 @@ describe('knowledge base application use cases', () => { undefined, { forUpdate: undefined } ) - expect(mocks.listInternalRecords).toHaveBeenCalledWith('user-1', 'workspace-1', 'archived') + expect(mocks.listRecords).toHaveBeenCalledWith('workspace-1', 'archived') + expect(mocks.listInternalRecords).not.toHaveBeenCalled() + }) + + /** + * Workspace `admin` can come from an organization role alone, with no workspace permission + * row behind it. Re-deriving list access from that row would hide every knowledge base from + * a caller the create path already authorizes — a base they make and then cannot see. + */ + it('lists a workspace for an authorized caller who holds no workspace permission row', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + mocks.listRecords.mockResolvedValueOnce({ data: [knowledgeBase], nextCursorKeys: null }) + + await expect( + listInternalKnowledgeBases.execute({ + principal: { kind: 'session', userId: 'org-admin-1', sessionId: 'session-1' }, + input: { workspaceId: 'workspace-1', scope: 'active' }, + }) + ).resolves.toEqual({ knowledgeBases: [knowledgeBase] }) + + expect(mocks.listRecords).toHaveBeenCalledWith('workspace-1', 'active') }) it('loads the active knowledge catalog and tag metadata only after workspace authorization', async () => { diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.ts b/apps/sim/lib/knowledge/application/knowledge-bases.ts index 392e0ada0dd..ba7720cd06b 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.ts @@ -441,9 +441,24 @@ export const listInternalKnowledgeBases = { if (input.workspaceId !== undefined) { const context = await resolveKnowledgeWorkspaceContext({ workspaceId: input.workspaceId }) await authorizeWorkspaceOperation(principal, knowledgeOperations.list, context) + /** + * Read the workspace's own rows once the operation is authorized. The legacy + * user-oriented query re-derives access from a `permissions` row join, which no longer + * matches what `authorizeWorkspaceOperation` accepts: an organization admin holds + * workspace `admin` through their org membership alone. That caller could create a + * knowledge base and then never see it listed. + */ + return { + knowledgeBases: (await getWorkspaceKnowledgeBases(context.workspaceId, input.scope)).data, + } } + /** + * No workspace named, so there is no canonical workspace to authorize against and the + * caller-scoped query stays the only answer: every workspace the session holds a + * permission row in, plus their legacy workspace-less bases. + */ return { - knowledgeBases: await getKnowledgeBases(principal.userId, input.workspaceId, input.scope), + knowledgeBases: await getKnowledgeBases(principal.userId, undefined, input.scope), } }, } satisfies OperationUseCase< From 8e96ce6f101da50c6503493744376790efbe452c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 16 Aug 2026 21:41:48 -0700 Subject: [PATCH 2/5] refactor(knowledge): stop re-deriving workspace access inside the list query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module resolves workspace authority one way everywhere — an explicit permission row OR an organization admin role — except in the listing query, which joined `permissions` and required a row. Both list surfaces authorized the caller and then contradicted that authorization: an org admin could create a knowledge base through /api/knowledge or /api/v1/knowledge and never see it listed. Tables and files carry no such join. Replace the caller-scoped query with `getLegacyPersonalKnowledgeBases`, which answers only for workspace-less bases whose creator IS their only authority, and have both surfaces read the workspace's own rows through `getWorkspaceKnowledgeBases` after authorizing. The legacy rows keep riding along so they stay reachable. The permissions join now appears nowhere in the module, and the duplicated connector projection collapses onto the shared helper that enforces the row cap. --- apps/sim/app/api/v1/knowledge/route.ts | 19 ++- .../application/knowledge-bases.test.ts | 36 ++++-- .../knowledge/application/knowledge-bases.ts | 35 +++--- apps/sim/lib/knowledge/service.test.ts | 56 +++++---- apps/sim/lib/knowledge/service.ts | 119 ++++-------------- 5 files changed, 120 insertions(+), 145 deletions(-) diff --git a/apps/sim/app/api/v1/knowledge/route.ts b/apps/sim/app/api/v1/knowledge/route.ts index cacb36ed482..132be1b0a8a 100644 --- a/apps/sim/app/api/v1/knowledge/route.ts +++ b/apps/sim/app/api/v1/knowledge/route.ts @@ -10,7 +10,10 @@ import { } from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { performCreateKnowledgeBase } from '@/lib/knowledge/orchestration' -import { getKnowledgeBases } from '@/lib/knowledge/service' +import { + getLegacyPersonalKnowledgeBases, + getWorkspaceKnowledgeBases, +} from '@/lib/knowledge/service' import { formatKnowledgeBase, handleError } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, @@ -43,7 +46,19 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId) if (accessError) return accessError - const knowledgeBases = await getKnowledgeBases(userId, workspaceId) + /** + * The workspace's own rows, read after `validateWorkspaceAccess` has authorized this + * caller rather than re-deriving that access inside the row query, plus the caller's + * legacy workspace-less bases, which belong to no workspace and answer only to their + * creator. This is the shape the internal list serves too. + */ + const [workspaceBases, legacyPersonalBases] = await Promise.all([ + getWorkspaceKnowledgeBases(workspaceId), + getLegacyPersonalKnowledgeBases(userId), + ]) + const knowledgeBases = [...workspaceBases.data, ...legacyPersonalBases].sort( + (a, b) => a.createdAt.getTime() - b.createdAt.getTime() + ) return NextResponse.json({ success: true, diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts index a64e6938e10..b66fed82129 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts @@ -15,7 +15,7 @@ const mocks = vi.hoisted(() => ({ updateRecord: vi.fn(), deleteRecord: vi.fn(), listRecords: vi.fn(), - listInternalRecords: vi.fn(), + listLegacyPersonalRecords: vi.fn(), getRecord: vi.fn(), getRestorableRecord: vi.fn(), performUpdate: vi.fn(), @@ -81,7 +81,7 @@ vi.mock('@/lib/knowledge/service', () => ({ updateKnowledgeBase: mocks.updateRecord, deleteKnowledgeBase: mocks.deleteRecord, getKnowledgeBaseById: mocks.getRecord, - getKnowledgeBases: mocks.listInternalRecords, + getLegacyPersonalKnowledgeBases: mocks.listLegacyPersonalRecords, getWorkspaceKnowledgeBases: mocks.listRecords, })) @@ -151,7 +151,7 @@ describe('knowledge base application use cases', () => { mocks.loadFolderIndex.mockResolvedValue({ pathById: new Map(), idByPath: new Map() }) mocks.createRecord.mockResolvedValue(knowledgeBase) mocks.listRecords.mockResolvedValue({ data: [], nextCursorKeys: null }) - mocks.listInternalRecords.mockResolvedValue([knowledgeBase]) + mocks.listLegacyPersonalRecords.mockResolvedValue([knowledgeBase]) mocks.getRecord.mockResolvedValue(knowledgeBase) mocks.getRestorableRecord.mockResolvedValue(knowledgeBase) mocks.performUpdate.mockResolvedValue({ @@ -190,7 +190,7 @@ describe('knowledge base application use cases', () => { expect(mocks.resolveWorkspace).not.toHaveBeenCalled() expect(mocks.resolvePermission).not.toHaveBeenCalled() - expect(mocks.listInternalRecords).toHaveBeenCalledWith('user-1', undefined, 'all') + expect(mocks.listLegacyPersonalRecords).toHaveBeenCalledWith('user-1', 'all') }) it('authorizes a canonical workspace before listing its internal knowledge bases', async () => { @@ -208,7 +208,6 @@ describe('knowledge base application use cases', () => { { forUpdate: undefined } ) expect(mocks.listRecords).toHaveBeenCalledWith('workspace-1', 'archived') - expect(mocks.listInternalRecords).not.toHaveBeenCalled() }) /** @@ -219,6 +218,7 @@ describe('knowledge base application use cases', () => { it('lists a workspace for an authorized caller who holds no workspace permission row', async () => { mocks.resolvePermission.mockResolvedValue('admin') mocks.listRecords.mockResolvedValueOnce({ data: [knowledgeBase], nextCursorKeys: null }) + mocks.listLegacyPersonalRecords.mockResolvedValueOnce([]) await expect( listInternalKnowledgeBases.execute({ @@ -230,6 +230,28 @@ describe('knowledge base application use cases', () => { expect(mocks.listRecords).toHaveBeenCalledWith('workspace-1', 'active') }) + /** + * Legacy workspace-less bases belong to no workspace, so a workspace list is the only + * place the UI can reach them. They ride along beside the workspace's own rows. + */ + it('includes the caller’s legacy personal bases beside the workspace’s own rows', async () => { + const legacyBase = { + ...knowledgeBase, + id: 'legacy-1', + workspaceId: null, + createdAt: new Date('2025-01-01T00:00:00Z'), + } + mocks.listRecords.mockResolvedValueOnce({ data: [knowledgeBase], nextCursorKeys: null }) + mocks.listLegacyPersonalRecords.mockResolvedValueOnce([legacyBase]) + + await expect( + listInternalKnowledgeBases.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { workspaceId: 'workspace-1', scope: 'active' }, + }) + ).resolves.toEqual({ knowledgeBases: [legacyBase, knowledgeBase] }) + }) + it('loads the active knowledge catalog and tag metadata only after workspace authorization', async () => { mocks.listRecords.mockResolvedValueOnce({ data: [knowledgeBase], nextCursorKeys: null }) dbChainMockFns.orderBy.mockResolvedValueOnce([ @@ -304,7 +326,7 @@ describe('knowledge base application use cases', () => { }) ).rejects.toMatchObject({ code: 'forbidden' }) - expect(mocks.listInternalRecords).not.toHaveBeenCalled() + expect(mocks.listLegacyPersonalRecords).not.toHaveBeenCalled() }) it('rejects non-session principals before resolving internal list input', async () => { @@ -316,7 +338,7 @@ describe('knowledge base application use cases', () => { ).rejects.toMatchObject({ code: 'forbidden' }) expect(mocks.resolveWorkspace).not.toHaveBeenCalled() - expect(mocks.listInternalRecords).not.toHaveBeenCalled() + expect(mocks.listLegacyPersonalRecords).not.toHaveBeenCalled() }) it('rejects an insufficient role before the protected mutation', async () => { diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.ts b/apps/sim/lib/knowledge/application/knowledge-bases.ts index ba7720cd06b..8ef704f34c0 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.ts @@ -56,7 +56,7 @@ import { createAuthorizedKnowledgeBase, deleteKnowledgeBase, getKnowledgeBaseById, - getKnowledgeBases, + getLegacyPersonalKnowledgeBases, getWorkspaceKnowledgeBases, type KnowledgeBaseScope, updateKnowledgeBase, @@ -438,27 +438,30 @@ export const listInternalKnowledgeBases = { if (principal.kind !== 'session') { throw new PrincipalKindAuthorizationError(principal.kind, knowledgeSessionOperations.list.id) } - if (input.workspaceId !== undefined) { - const context = await resolveKnowledgeWorkspaceContext({ workspaceId: input.workspaceId }) - await authorizeWorkspaceOperation(principal, knowledgeOperations.list, context) - /** - * Read the workspace's own rows once the operation is authorized. The legacy - * user-oriented query re-derives access from a `permissions` row join, which no longer - * matches what `authorizeWorkspaceOperation` accepts: an organization admin holds - * workspace `admin` through their org membership alone. That caller could create a - * knowledge base and then never see it listed. - */ + if (input.workspaceId === undefined) { return { - knowledgeBases: (await getWorkspaceKnowledgeBases(context.workspaceId, input.scope)).data, + knowledgeBases: await getLegacyPersonalKnowledgeBases(principal.userId, input.scope), } } + const context = await resolveKnowledgeWorkspaceContext({ workspaceId: input.workspaceId }) + await authorizeWorkspaceOperation(principal, knowledgeOperations.list, context) /** - * No workspace named, so there is no canonical workspace to authorize against and the - * caller-scoped query stays the only answer: every workspace the session holds a - * permission row in, plus their legacy workspace-less bases. + * Two reads, because this list answers for two authorities. The workspace's own rows are + * read once the operation is authorized — deriving list access a second time from a + * `permissions` row would contradict the authorization that just passed, since workspace + * `admin` can come from an organization role with no such row behind it, and that caller + * could create a knowledge base and then never see it listed. Legacy workspace-less bases + * answer only to their creator and have no workspace to be listed under, so they ride + * along here as they always have — otherwise they are reachable from nowhere in the UI. */ + const [workspaceBases, legacyPersonalBases] = await Promise.all([ + getWorkspaceKnowledgeBases(context.workspaceId, input.scope), + getLegacyPersonalKnowledgeBases(principal.userId, input.scope), + ]) return { - knowledgeBases: await getKnowledgeBases(principal.userId, undefined, input.scope), + knowledgeBases: [...workspaceBases.data, ...legacyPersonalBases].sort( + (a, b) => a.createdAt.getTime() - b.createdAt.getTime() + ), } }, } satisfies OperationUseCase< diff --git a/apps/sim/lib/knowledge/service.test.ts b/apps/sim/lib/knowledge/service.test.ts index 654e91d1d3f..f7fd7a155fc 100644 --- a/apps/sim/lib/knowledge/service.test.ts +++ b/apps/sim/lib/knowledge/service.test.ts @@ -41,7 +41,7 @@ vi.mock('@/lib/billing/core/usage', () => ({ import { MAX_KNOWLEDGE_BASES_PER_WORKSPACE } from '@/lib/knowledge/constants' import { - getKnowledgeBases, + getLegacyPersonalKnowledgeBases, getWorkspaceKnowledgeBases, KnowledgeBasePermissionError, updateKnowledgeBase, @@ -68,54 +68,58 @@ describe('getWorkspaceKnowledgeBases — bounded reads', () => { }) /** - * The listing query authorizes on current workspace membership, never on stale creator - * identity: a user removed from a workspace must stop seeing knowledge bases they created - * there. The creator fallback exists only for legacy knowledge bases with no `workspaceId`. + * Legacy knowledge bases predate workspaces and carry no `workspaceId`, so their creator is + * the only possible authority. Workspace-owned rows are read by `getWorkspaceKnowledgeBases` + * after an application use case authorized the workspace — this query must never widen to + * them, and must never re-derive workspace access from a `permissions` row, which would + * contradict an authorization that already passed. */ -describe('getKnowledgeBases — creator fallback is scoped to legacy non-workspace KBs', () => { +describe('getLegacyPersonalKnowledgeBases', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() }) - /** Every disjunct that grants on `knowledgeBase.userId`, from the last select chain's WHERE. */ - const capturedCreatorBranches = (): unknown[] => { + it('reads only the caller’s workspace-less rows', async () => { + await getLegacyPersonalKnowledgeBases('user-a', 'all') + const [condition] = dbChainMockFns.where.mock.calls.at(-1) ?? [] - const orNode = flattenMockConditions(condition).find((node) => node.type === 'or') - expect(orNode, 'WHERE clause has no or(...) branch').toBeDefined() - return (orNode?.conditions as unknown[]).filter((disjunct) => + expect( hasMockCondition( - disjunct, + condition, (node) => node.type === 'eq' && node.left === schemaMock.knowledgeBase.userId && node.right === 'user-a' ) - ) - } - - /** The creator fallback must be the sole grant for legacy KBs and never reach workspace KBs. */ - const expectCreatorBranchIsLegacyOnly = () => { - const branches = capturedCreatorBranches() - expect(branches).toHaveLength(1) + ).toBe(true) expect( hasMockCondition( - branches[0], + condition, (node) => node.type === 'isNull' && node.column === schemaMock.knowledgeBase.workspaceId ) ).toBe(true) - } + expect(flattenMockConditions(condition).some((node) => node.type === 'or')).toBe(false) + }) - it('requires workspaceId IS NULL on the creator branch when no workspace filter is given', async () => { - await getKnowledgeBases('user-a', undefined, 'all') + it('never joins the permissions table', async () => { + await getLegacyPersonalKnowledgeBases('user-a') - expectCreatorBranchIsLegacyOnly() + const joinedTables = dbChainMockFns.leftJoin.mock.calls.map(([table]) => table) + expect(joinedTables).toContain(schemaMock.document) + expect(joinedTables).not.toContain(schemaMock.permissions) }) - it('keeps the same guard on the workspace-filtered branch', async () => { - await getKnowledgeBases('user-a', 'ws-1', 'active') + it('fails before projecting connector data for an oversized set', async () => { + dbChainMockFns.limit.mockResolvedValueOnce( + Array.from({ length: MAX_KNOWLEDGE_BASES_PER_WORKSPACE + 1 }, (_, index) => ({ + id: `kb-${index}`, + })) + ) - expectCreatorBranchIsLegacyOnly() + await expect(getLegacyPersonalKnowledgeBases('user-a')).rejects.toThrow( + `Legacy personal knowledge base list exceeds the ${MAX_KNOWLEDGE_BASES_PER_WORKSPACE} row limit` + ) }) }) diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index d3b9ffa5e37..87110003fa5 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -1,16 +1,9 @@ import { db } from '@sim/db' -import { - document, - knowledgeBase, - knowledgeConnector, - permissions, - workspace, - workspaceFiles, -} from '@sim/db/schema' +import { document, knowledgeBase, knowledgeConnector, workspaceFiles } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getPostgresErrorCode } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { and, count, eq, exists, inArray, isNotNull, isNull, ne, or, sql } from 'drizzle-orm' +import { and, count, eq, exists, inArray, isNotNull, isNull, ne, sql } from 'drizzle-orm' import type { V2KnowledgeBaseSortBy } from '@/lib/api/contracts/v2/knowledge' import type { CursorKey, KeysetKey, ListSortOrder } from '@/lib/api/list-query' import { @@ -308,18 +301,16 @@ export async function getWorkspaceKnowledgeBases( } /** - * Get knowledge bases that a user can access. - * - * Filter and sort are applied in the query, so a search costs one narrowed scan - * rather than materializing every knowledge base the caller can reach. + * Lists the caller's legacy personal knowledge bases — the ones that predate + * workspaces and carry no `workspaceId`, where the creator is the only possible + * authority. Workspace-owned rows are read by + * {@link getWorkspaceKnowledgeBases} after an application use case has + * authorized the workspace; nothing here re-derives that access. */ -export async function getKnowledgeBases( +export async function getLegacyPersonalKnowledgeBases( userId: string, - workspaceId?: string | null, - scope: KnowledgeBaseScope = 'active', - options?: GetKnowledgeBasesOptions + scope: KnowledgeBaseScope = 'active' ): Promise { - const { folderId, search, sortBy = 'createdAt', sortOrder = 'asc' } = options ?? {} const scopeCondition = scope === 'all' ? undefined @@ -327,22 +318,7 @@ export async function getKnowledgeBases( ? sql`${knowledgeBase.deletedAt} IS NOT NULL` : isNull(knowledgeBase.deletedAt) - /** - * Legacy knowledge bases predate workspaces and have no `workspaceId`, so the creator is - * their only possible authority. Anything with a `workspaceId` must clear - * `currentWorkspaceMembership` instead — creator identity goes stale the moment a member - * is removed from the workspace. - */ - const legacyOwnedKnowledgeBase = and( - eq(knowledgeBase.userId, userId), - isNull(knowledgeBase.workspaceId) - ) - const currentWorkspaceMembership = and( - isNotNull(permissions.userId), - isNull(workspace.archivedAt) - ) - - const knowledgeBasesWithCounts = await db + const rows = await db .select({ id: knowledgeBase.id, userId: knowledgeBase.userId, @@ -369,70 +345,25 @@ export async function getKnowledgeBases( isNull(document.deletedAt) ) ) - .leftJoin( - permissions, - and( - eq(permissions.entityType, 'workspace'), - eq(permissions.entityId, knowledgeBase.workspaceId), - eq(permissions.userId, userId) - ) - ) - .leftJoin(workspace, eq(knowledgeBase.workspaceId, workspace.id)) - .where( - and( - scopeCondition, - folderId === undefined - ? undefined - : folderId === null - ? isNull(knowledgeBase.folderId) - : eq(knowledgeBase.folderId, folderId), - searchFilter(knowledgeBase.name, search), - or( - and( - workspaceId ? eq(knowledgeBase.workspaceId, workspaceId) : undefined, - currentWorkspaceMembership - ), - legacyOwnedKnowledgeBase - ) - ) - ) + .where(and(scopeCondition, eq(knowledgeBase.userId, userId), isNull(knowledgeBase.workspaceId))) .groupBy(knowledgeBase.id) - .orderBy(...listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS[sortBy]), sortOrder)) - - const kbIds = knowledgeBasesWithCounts.map((kb) => kb.id) + .orderBy(...listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS.createdAt), 'asc')) + .limit(MAX_KNOWLEDGE_BASES_PER_WORKSPACE + 1) - const connectorRows = - kbIds.length > 0 - ? await db - .select({ - knowledgeBaseId: knowledgeConnector.knowledgeBaseId, - connectorType: knowledgeConnector.connectorType, - }) - .from(knowledgeConnector) - .where( - and( - inArray(knowledgeConnector.knowledgeBaseId, kbIds), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - : [] - - const connectorTypesByKb = new Map() - for (const row of connectorRows) { - const types = connectorTypesByKb.get(row.knowledgeBaseId) ?? [] - if (!types.includes(row.connectorType)) { - types.push(row.connectorType) - } - connectorTypesByKb.set(row.knowledgeBaseId, types) + /** One row past the cap, so an oversized set fails loudly instead of truncating in silence. */ + if (rows.length > MAX_KNOWLEDGE_BASES_PER_WORKSPACE) { + throw new Error( + `Legacy personal knowledge base list exceeds the ${MAX_KNOWLEDGE_BASES_PER_WORKSPACE} row limit` + ) } - return knowledgeBasesWithCounts.map((kb) => ({ - ...kb, - chunkingConfig: kb.chunkingConfig as ChunkingConfig, - docCount: Number(kb.docCount), - connectorTypes: connectorTypesByKb.get(kb.id) ?? [], - })) + return attachConnectorTypes( + rows.map((kb) => ({ + ...kb, + chunkingConfig: kb.chunkingConfig as ChunkingConfig, + docCount: Number(kb.docCount), + })) + ) } /** From 9e2dbd44774cbddd0e70795e81e61c7cbc80b880 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 16 Aug 2026 21:52:32 -0700 Subject: [PATCH 3/5] refactor(knowledge): clean up the module's client layer and fix two state bugs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanup pass over the knowledge module — effects, state, memo, callback, React Query, url-state, emcn, and comments — keeping the fixes that change behavior for the better and leaving the ones that would change how the UI feels. Bugs found and fixed: - Opening a document flashed "Document not ready" for a frame. The chunk-row builder rendered the loading state as a status claim: with no document loaded yet it fell through to the branch that reports a missing processing status. - A partial upload failure skipped every cache invalidation, because the throw jumped past them, so the list stayed missing rows the server had already created. Admission failures create nothing and still skip the refetch. - The document and chunk context menus captured the row they opened on, so the Enable/Disable label went stale under the list's own polling. They hold an id and resolve against live data now. - The action bar's "Select all"/"Clear" links were painted with `--brand-primary`, which is defined nowhere: the links fell back to `currentColor` and were indistinguishable from the text beside them. Consistency and weight: - Mutations no longer invalidate `detail` non-exactly for writes that touch one document: that key is the parent of every documents page, chunk page, tag definition, and connector row cached for the base. - Dead hook surface removed (five exports with no consumer, a query instantiated only to reach a cache helper, a `goToPage` that only range-checked), unused parameters dropped, `getErrorMessage` replacing hand-rolled instanceof checks. - `page` joins the document list's param group, so a search resets pagination in the same debounced write instead of writing the URL on every keystroke. - Icons import from `@sim/emcn/icons`, the action bar composes `chipFilledFillTokens` instead of restating it three times, chunk cells use the canonical content-label chrome, and the icon-only buttons have accessible names. --- .../document-tags-modal.tsx | 8 +- .../knowledge/[id]/[documentId]/document.tsx | 161 +++++++++--------- .../knowledge/[id]/[documentId]/loading.tsx | 3 +- .../[workspaceId]/knowledge/[id]/base.tsx | 80 ++++----- .../[id]/components/action-bar/action-bar.tsx | 29 +++- .../add-documents-modal.tsx | 3 +- .../base-tags-modal/base-tags-modal.tsx | 4 +- .../connector-selector-field.tsx | 3 +- .../connectors-section/connectors-section.tsx | 149 ++++++++-------- .../[workspaceId]/knowledge/[id]/loading.tsx | 5 +- .../knowledge/[id]/search-params.ts | 23 +-- .../create-base-modal/create-base-modal.tsx | 7 +- .../edit-knowledge-base-modal.tsx | 18 +- .../hooks/use-knowledge-upload.test.tsx | 26 +++ .../knowledge/hooks/use-knowledge-upload.ts | 23 ++- .../[workspaceId]/knowledge/knowledge.tsx | 26 ++- .../[workspaceId]/knowledge/loading.tsx | 3 +- .../kb/use-knowledge-base-tag-definitions.ts | 13 +- apps/sim/hooks/kb/use-knowledge.ts | 123 ++++--------- apps/sim/hooks/kb/use-tag-definitions.ts | 50 ++---- apps/sim/hooks/queries/kb/connectors.ts | 19 ++- apps/sim/hooks/queries/kb/knowledge.ts | 61 +++++-- apps/sim/lib/knowledge/chunks/service.ts | 11 -- apps/sim/lib/knowledge/documents/types.ts | 1 - apps/sim/lib/knowledge/documents/utils.ts | 6 - apps/sim/lib/knowledge/search/queries.ts | 11 +- apps/sim/lib/knowledge/tags/service.ts | 12 -- apps/sim/lib/knowledge/tags/utils.ts | 1 - 28 files changed, 394 insertions(+), 485 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx index faa99423d44..e5fa53f5108 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx @@ -14,8 +14,8 @@ import { ChipModalHeader, handleKeyboardActivation, Label, - Trash, } from '@sim/emcn' +import { Trash } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { formatDate } from '@sim/utils/formatting' import { @@ -378,11 +378,7 @@ export function DocumentTagsModal({ return ( - handleClose(false)}> -
- Document Tags -
-
+ handleClose(false)}>Document Tags diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx index 974a0896848..0b99f0710bd 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx @@ -1,8 +1,17 @@ 'use client' import { useCallback, useEffect, useEffectEvent, useMemo, useRef, useState } from 'react' -import { Badge, ChipCombobox, ChipConfirmModal, Plus, Trash } from '@sim/emcn' -import { ChevronDown, ChevronUp, Database, FileText, Pencil, TagIcon } from '@sim/emcn/icons' +import { Badge, ChipCombobox, ChipConfirmModal, chipContentLabelClass, cn } from '@sim/emcn' +import { + ChevronDown, + ChevronUp, + Database, + FileText, + Pencil, + Plus, + TagIcon, + Trash, +} from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { truncate } from '@sim/utils/string' import { useParams, useRouter } from 'next/navigation' @@ -204,7 +213,6 @@ export function Document({ chunks: initialChunks, currentPage: initialPage, totalPages: initialTotalPages, - goToPage: initialGoToPage, error: initialError, updateChunk: initialUpdateChunk, } = useDocumentChunks( @@ -295,13 +303,8 @@ export function Document({ const goToPage = useCallback( async (page: number) => { await setDocumentParams({ page }) - - if (showingSearch) { - return - } - return initialGoToPage(page) }, - [showingSearch, initialGoToPage, setDocumentParams] + [setDocumentParams] ) const updateChunk = showingSearch @@ -309,9 +312,15 @@ export function Document({ : initialUpdateChunk const [chunkToDelete, setChunkToDelete] = useState(null) - const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false) const [showDeleteDocumentDialog, setShowDeleteDocumentDialog] = useState(false) - const [contextMenuChunk, setContextMenuChunk] = useState(null) + const [contextMenuChunkId, setContextMenuChunkId] = useState(null) + /** + * The id, not the row: the chunk list polls while a document processes, and a menu that + * captured the row on open would keep offering "Enable" for a chunk already enabled. + */ + const contextMenuChunk = contextMenuChunkId + ? (displayChunks.find((chunk) => chunk.id === contextMenuChunkId) ?? null) + : null const { mutate: updateChunkMutation } = useUpdateChunk() const { mutate: deleteDocumentMutation, isPending: isDeletingDocument } = useDeleteDocument() @@ -351,15 +360,13 @@ export function Document({ const isInEditorView = selectedChunkId !== null || isCreatingNewChunk - const selectedChunk = useMemo( - () => (selectedChunkId ? (displayChunks.find((c) => c.id === selectedChunkId) ?? null) : null), - [selectedChunkId, displayChunks] - ) + const selectedChunk = selectedChunkId + ? (displayChunks.find((c) => c.id === selectedChunkId) ?? null) + : null - const currentChunkIndex = useMemo( - () => (selectedChunk ? displayChunks.findIndex((c) => c.id === selectedChunk.id) : -1), - [selectedChunk, displayChunks] - ) + const currentChunkIndex = selectedChunk + ? displayChunks.findIndex((c) => c.id === selectedChunk.id) + : -1 const canNavigatePrev = currentChunkIndex > 0 || currentPage > 1 const canNavigateNext = currentChunkIndex < displayChunks.length - 1 || currentPage < totalPages @@ -402,14 +409,14 @@ export function Document({ } }, [isDirty, isCreatingNewChunk]) - const handleUnsavedChangesOpenChange = useCallback((open: boolean) => { + const handleUnsavedChangesOpenChange = (open: boolean) => { if (!open) { setShowUnsavedChangesAlert(false) setPendingAction(null) } - }, []) + } - const handleDiscardChanges = useCallback(() => { + const handleDiscardChanges = () => { setShowUnsavedChangesAlert(false) const action = pendingAction setPendingAction(null) @@ -419,7 +426,7 @@ export function Document({ } else { closeEditor() } - }, [pendingAction, closeEditor]) + } const handleSaveEvent = useEffectEvent(handleSave) @@ -646,7 +653,6 @@ export function Document({ if (found) { setSelectedChunkId(chunkId) } else if (!navigatedToNewPage && totalPagesRef.current > totalPages) { - // A new page was created — navigate to it navigatedToNewPage = true retries = 0 void goToPage(totalPagesRef.current) @@ -681,10 +687,8 @@ export function Document({ } : undefined - const enabledDisplayLabel = useMemo(() => { - if (enabledFilter.length === 0) return 'All' - return enabledFilter[0] === 'enabled' ? 'Enabled' : 'Disabled' - }, [enabledFilter]) + const enabledDisplayLabel = + enabledFilter.length === 0 ? 'All' : enabledFilter[0] === 'enabled' ? 'Enabled' : 'Disabled' const filterContent = useMemo( () => ( @@ -724,7 +728,7 @@ export function Document({ )} ), - [enabledFilter, enabledDisplayLabel, setEnabledFilter] + [enabledFilter, setEnabledFilter] ) const filterTags: FilterTag[] = useMemo( @@ -746,31 +750,22 @@ export function Document({ [setSelectedChunkId] ) - const handleToggleEnabled = useCallback( - (chunkId: string) => { - const chunk = displayChunks.find((c) => c.id === chunkId) - if (!chunk) return + const handleToggleEnabled = (chunkId: string) => { + const chunk = displayChunks.find((c) => c.id === chunkId) + if (!chunk) return - const newEnabled = !chunk.enabled - updateChunk(chunkId, { enabled: newEnabled }) - updateChunkMutation( - { knowledgeBaseId, documentId, chunkId, enabled: newEnabled }, - { onError: () => updateChunk(chunkId, { enabled: chunk.enabled }) } - ) - }, - [displayChunks, knowledgeBaseId, documentId, updateChunk] - ) + const newEnabled = !chunk.enabled + updateChunk(chunkId, { enabled: newEnabled }) + updateChunkMutation( + { knowledgeBaseId, documentId, chunkId, enabled: newEnabled }, + { onError: () => updateChunk(chunkId, { enabled: chunk.enabled }) } + ) + } - const handleDeleteChunk = useCallback( - (chunkId: string) => { - const chunk = displayChunks.find((c) => c.id === chunkId) - if (chunk) { - setChunkToDelete(chunk) - setIsDeleteModalOpen(true) - } - }, - [displayChunks] - ) + const handleDeleteChunk = (chunkId: string) => { + const chunk = displayChunks.find((c) => c.id === chunkId) + if (chunk) setChunkToDelete(chunk) + } const handleCloseDeleteModal = () => { if (chunkToDelete) { @@ -780,7 +775,6 @@ export function Document({ return newSet }) } - setIsDeleteModalOpen(false) setChunkToDelete(null) } @@ -863,17 +857,14 @@ export function Document({ performBulkChunkOperation('delete', chunksToDelete) } - const [enabledCount, disabledCount] = useMemo(() => { - let enabled = 0 - let disabled = 0 - for (const chunk of displayChunks) { - if (selectedChunks.has(chunk.id)) { - if (chunk.enabled) enabled++ - else disabled++ - } + let enabledCount = 0 + let disabledCount = 0 + for (const chunk of displayChunks) { + if (selectedChunks.has(chunk.id)) { + if (chunk.enabled) enabledCount++ + else disabledCount++ } - return [enabled, disabled] - }, [displayChunks, selectedChunks]) + } const isAllSelected = displayChunks.length > 0 && selectedChunks.size === displayChunks.length @@ -890,7 +881,7 @@ export function Document({ } } - setContextMenuChunk(chunk) + setContextMenuChunkId(chunk.id) baseHandleContextMenu(e) }, [ @@ -902,18 +893,15 @@ export function Document({ ] ) - const handleEmptyContextMenu = useCallback( - (e: React.MouseEvent) => { - setContextMenuChunk(null) - baseHandleContextMenu(e) - }, - [baseHandleContextMenu] - ) + const handleEmptyContextMenu = (e: React.MouseEvent) => { + setContextMenuChunkId(null) + baseHandleContextMenu(e) + } - const handleContextMenuClose = useCallback(() => { + const handleContextMenuClose = () => { closeContextMenu() - setContextMenuChunk(null) - }, [closeContextMenu]) + setContextMenuChunkId(null) + } const selectableConfig: SelectableConfig | undefined = isCompleted ? { @@ -956,6 +944,13 @@ export function Document({ ) const chunkRows: ResourceRow[] = useMemo(() => { + /** + * No document yet is "not known", not "not ready". Falling through to the status row + * flashed `Document not ready` on every open, for the frame between mount and the + * document query resolving — a claim about a document nothing had read yet. + */ + if (!documentData) return [] + if (!isCompleted) { return [ { @@ -966,12 +961,12 @@ export function Document({
- {documentData?.processingStatus === 'pending' && + {documentData.processingStatus === 'pending' && 'Document processing pending...'} - {documentData?.processingStatus === 'processing' && + {documentData.processingStatus === 'processing' && 'Document processing in progress...'} - {documentData?.processingStatus === 'failed' && 'Document processing failed'} - {!documentData?.processingStatus && 'Document not ready'} + {documentData.processingStatus === 'failed' && 'Document processing failed'} + {!documentData.processingStatus && 'Document not ready'}
), @@ -992,16 +987,14 @@ export function Document({ cells: { content: { content: ( - + ), }, index: { content: ( - - {chunk.chunkIndex} - + {chunk.chunkIndex} ), }, tokens: { @@ -1017,7 +1010,7 @@ export function Document({ }, } }) - }, [isCompleted, documentData?.processingStatus, displayChunks, searchQuery]) + }, [isCompleted, documentData, displayChunks, searchQuery]) const saveLabel = saveStatus === 'saving' @@ -1232,7 +1225,7 @@ export function Document({ chunk={chunkToDelete} knowledgeBaseId={knowledgeBaseId} documentId={documentId} - isOpen={isDeleteModalOpen} + isOpen={chunkToDelete !== null} onClose={handleCloseDeleteModal} /> diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/loading.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/loading.tsx index ed67b33a791..63369f7a3d1 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/loading.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/loading.tsx @@ -1,7 +1,6 @@ 'use client' -import { Plus } from '@sim/emcn' -import { Database, FileText } from '@sim/emcn/icons' +import { Database, FileText, Plus } from '@sim/emcn/icons' import { noop } from '@sim/utils/helpers' import { type BreadcrumbItem, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx index b64b4642abb..a01bb376e57 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx @@ -20,12 +20,20 @@ import { cn, FloatingTooltip, isTextClipped, - Loader, Tooltip, - Trash, useFloatingTooltip, } from '@sim/emcn' -import { CircleAlert, Database, DatabaseX, Pencil, Plus, TagIcon, X } from '@sim/emcn/icons' +import { + CircleAlert, + Database, + DatabaseX, + Loader, + Pencil, + Plus, + TagIcon, + Trash, + X, +} from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' @@ -77,19 +85,13 @@ import { documentFiltersParsers, documentFiltersUrlKeys, kbDocumentSortParams, - pageParam, - pageUrlKeys, } from '@/app/workspace/[workspaceId]/knowledge/[id]/search-params' import { getDocumentIcon } from '@/app/workspace/[workspaceId]/knowledge/components' import { useRegisterGlobalCommands } from '@/app/workspace/[workspaceId]/providers/global-commands-provider' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' -import { - useKnowledgeBase, - useKnowledgeBaseDocuments, - useKnowledgeBasesList, -} from '@/hooks/kb/use-knowledge' +import { useKnowledgeBase, useKnowledgeBaseDocuments } from '@/hooks/kb/use-knowledge' import { type TagDefinition, useKnowledgeBaseTagDefinitions, @@ -280,14 +282,12 @@ export function KnowledgeBase({ }, [id, passedKnowledgeBaseName, posthog]) useOAuthReturnForKBConnectors(id) - const { removeKnowledgeBase } = useKnowledgeBasesList(workspaceId, { enabled: false }) const userPermissions = useUserPermissionsContext() const { mutate: updateDocumentMutation, mutateAsync: updateDocumentAsync } = useUpdateDocument() const { mutate: deleteDocumentMutation } = useDeleteDocument() - const { mutate: deleteKnowledgeBaseMutation, isPending: isDeleting } = - useDeleteKnowledgeBase(workspaceId) - const { mutateAsync: updateKnowledgeBaseMutation } = useUpdateKnowledgeBase(workspaceId) + const { mutate: deleteKnowledgeBaseMutation, isPending: isDeleting } = useDeleteKnowledgeBase() + const { mutateAsync: updateKnowledgeBaseMutation } = useUpdateKnowledgeBase() const kbRename = useInlineRename({ onSave: (kbId, name) => @@ -336,14 +336,13 @@ export function KnowledgeBase({ const [documentToDelete, setDocumentToDelete] = useState(null) const [showBulkDeleteModal, setShowBulkDeleteModal] = useState(false) const [showConnectorsModal, setShowConnectorsModal] = useState(false) - const [currentPage, setCurrentPage] = useQueryState(pageParam.key, { - ...pageParam.parser, - ...pageUrlKeys, - }) + const [{ q: searchQuery, enabled: enabledFilter, page: currentPage }, setDocumentFilters] = + useQueryStates(documentFiltersParsers, documentFiltersUrlKeys) - const [{ q: searchQuery, enabled: enabledFilter }, setDocumentFilters] = useQueryStates( - documentFiltersParsers, - documentFiltersUrlKeys + /** Page 1 is the group's default, so it strips from the URL rather than lingering as `?page=1`. */ + const setCurrentPage = useCallback( + (page: number) => void setDocumentFilters({ page }), + [setDocumentFilters] ) /** @@ -352,8 +351,7 @@ export function KnowledgeBase({ * doesn't refetch on every keystroke. Changing the search resets pagination. */ const handleSearchChange = useDebouncedSearchSetter((value, options) => { - setDocumentFilters({ q: value }, options) - setCurrentPage(1) + void setDocumentFilters({ q: value, page: 1 }, options) }) const debouncedSearchQuery = useDebounce(searchQuery, SEARCH_DEBOUNCE_MS) /** Raw URL value drives the input; matching/highlighting always sees it trimmed. */ @@ -369,13 +367,12 @@ export function KnowledgeBase({ const setEnabledFilter = useCallback( (value: 'all' | 'enabled' | 'disabled') => { - setDocumentFilters({ enabled: value }) - setCurrentPage(1) + void setDocumentFilters({ enabled: value, page: 1 }) }, - [setDocumentFilters, setCurrentPage] + [setDocumentFilters] ) - const [contextMenuDocument, setContextMenuDocument] = useState(null) + const [contextMenuDocumentId, setContextMenuDocumentId] = useState(null) const [showRenameModal, setShowRenameModal] = useState(false) const [documentToRename, setDocumentToRename] = useState(null) const [showDocumentTagsModal, setShowDocumentTagsModal] = useState(false) @@ -440,6 +437,15 @@ export function KnowledgeBase({ const { tagDefinitions } = useKnowledgeBaseTagDefinitions(id) + /** + * The id, not the row: the document list polls every few seconds while anything is + * processing, so a menu holding the row it opened on would offer actions against a status + * that has since moved on. + */ + const contextMenuDocument = contextMenuDocumentId + ? (documents.find((doc) => doc.id === contextMenuDocumentId) ?? null) + : null + const prevHadSyncingRef = useRef(false) useEffect(() => { if (prevHadSyncingRef.current && !hasSyncingConnectors) { @@ -699,7 +705,6 @@ export function KnowledgeBase({ { knowledgeBaseId: id }, { onSuccess: () => { - removeKnowledgeBase(id) router.push(`/workspace/${workspaceId}/knowledge`) }, } @@ -885,24 +890,21 @@ export function KnowledgeBase({ setSelectedDocuments(new Set([doc.id])) } - setContextMenuDocument(doc) + setContextMenuDocumentId(doc.id) baseHandleContextMenu(e) }, [documents, selectedDocuments, baseHandleContextMenu] ) - const handleEmptyContextMenu = useCallback( - (e: React.MouseEvent) => { - setContextMenuDocument(null) - baseHandleContextMenu(e) - }, - [baseHandleContextMenu] - ) + const handleEmptyContextMenu = (e: React.MouseEvent) => { + setContextMenuDocumentId(null) + baseHandleContextMenu(e) + } - const handleContextMenuClose = useCallback(() => { + const handleContextMenuClose = () => { closeContextMenu() - setContextMenuDocument(null) - }, [closeContextMenu]) + setContextMenuDocumentId(null) + } const breadcrumbs: BreadcrumbItem[] = useMemo( () => diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/action-bar/action-bar.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/action-bar/action-bar.tsx index e6817dce63c..ced7d0eb350 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/action-bar/action-bar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/action-bar/action-bar.tsx @@ -1,8 +1,14 @@ -import { Button, cn, Tooltip, Trash } from '@sim/emcn' -import { Ban, Circle } from '@sim/emcn/icons' +import { Button, chipFilledFillTokens, cn, Tooltip } from '@sim/emcn' +import { Ban, Circle, Trash } from '@sim/emcn/icons' import { domAnimation, LazyMotion, m } from 'framer-motion' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' +/** One source of truth for the button chrome, so the three actions read as one control strip. */ +const ACTION_BUTTON_CLASS = cn( + chipFilledFillTokens, + 'hover-hover:!text-[var(--text-inverse)] size-[28px] rounded-lg p-0 text-[var(--text-secondary)] hover-hover:bg-[var(--brand-secondary)]' +) + interface ActionBarProps { selectedCount: number onEnable?: () => void @@ -51,8 +57,10 @@ export function ActionBar({ animate={{ opacity: 1, y: 0 }} exit={{ opacity: 0, y: 10 }} transition={{ duration: 0.2 }} - className={cn('-translate-x-1/2 fixed bottom-6 z-50 transform', className)} - style={{ left: '50%' }} + className={cn( + '-translate-x-1/2 fixed bottom-6 left-1/2 z-[var(--z-dropdown)] transform', + className + )} >
@@ -63,7 +71,7 @@ export function ActionBar({ @@ -75,7 +83,7 @@ export function ActionBar({ @@ -89,9 +97,10 @@ export function ActionBar({ @@ -105,9 +114,10 @@ export function ActionBar({ @@ -121,9 +131,10 @@ export function ActionBar({ diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx index ddeb723a94c..90862127cb8 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx @@ -10,9 +10,8 @@ import { ChipModalFooter, ChipModalHeader, cn, - Loader, } from '@sim/emcn' -import { RefreshCw, X } from '@sim/emcn/icons' +import { Loader, RefreshCw, X } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { useParams } from 'next/navigation' import { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/base-tags-modal/base-tags-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/base-tags-modal/base-tags-modal.tsx index 8bf94aec13a..d82a9bd6937 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/base-tags-modal/base-tags-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/base-tags-modal/base-tags-modal.tsx @@ -13,8 +13,8 @@ import { ChipModalHeader, type ComboboxOption, handleKeyboardActivation, - Trash, } from '@sim/emcn' +import { Trash } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import type { TagUsageData } from '@/lib/api/contracts/knowledge' import { @@ -393,7 +393,6 @@ export function BaseTagsModal({ open, onOpenChange, knowledgeBaseId }: BaseTagsM /> - {/* Delete Tag Confirmation Dialog */} { @@ -432,7 +431,6 @@ export function BaseTagsModal({ open, onOpenChange, knowledgeBaseId }: BaseTagsM )} - {/* View Documents Dialog */} >> + +function addToSet(setter: IdSetSetter, id: string) { + setter((prev) => new Set(prev).add(id)) +} + +function removeFromSet(setter: IdSetSetter, id: string) { + setter((prev) => { + const next = new Set(prev) + next.delete(id) + return next + }) +} + const STATUS_CONFIG = { active: { label: 'Active', variant: 'green' as const }, syncing: { label: 'Syncing', variant: 'amber' as const }, @@ -86,27 +103,15 @@ export function ConnectorsSection({ const [deleteTarget, setDeleteTarget] = useState(null) const [deleteDocuments, setDeleteDocuments] = useState(false) - const closeDeleteModal = useCallback(() => { + const closeDeleteModal = () => { setDeleteTarget(null) setDeleteDocuments(false) - }, []) + } const [editingConnector, setEditingConnector] = useState(null) const [error, setError] = useState(null) const [syncingIds, setSyncingIds] = useState>(() => new Set()) const [updatingIds, setUpdatingIds] = useState>(() => new Set()) - const addToSet = useCallback((setter: typeof setSyncingIds, id: string) => { - setter((prev) => new Set(prev).add(id)) - }, []) - - const removeFromSet = useCallback((setter: typeof setSyncingIds, id: string) => { - setter((prev) => { - const next = new Set(prev) - next.delete(id) - return next - }) - }, []) - const syncTriggeredAt = useRef>({}) const cooldownTimersRef = useRef> | null>(null) cooldownTimersRef.current ??= new Set() @@ -120,69 +125,61 @@ export function ConnectorsSection({ } }, []) - const isSyncOnCooldown = useCallback((connectorId: string) => { + const isSyncOnCooldown = (connectorId: string) => { const triggeredAt = syncTriggeredAt.current[connectorId] if (!triggeredAt) return false return Date.now() - triggeredAt < SYNC_COOLDOWN_MS - }, []) + } + + const handleSync = (connectorId: string, rehydrate = false) => { + if (isSyncOnCooldown(connectorId)) return - const handleSync = useCallback( - (connectorId: string, rehydrate = false) => { - if (isSyncOnCooldown(connectorId)) return - - syncTriggeredAt.current[connectorId] = Date.now() - addToSet(setSyncingIds, connectorId) - - triggerSync( - { knowledgeBaseId, connectorId, rehydrate }, - { - onSuccess: () => { - setError(null) - const timer = setTimeout(() => { - cooldownTimersRef.current?.delete(timer) - forceUpdate((n) => n + 1) - }, SYNC_COOLDOWN_MS) - cooldownTimersRef.current?.add(timer) - }, - onError: (err) => { - logger.error('Sync trigger failed', { error: err.message }) - setError(err.message) - delete syncTriggeredAt.current[connectorId] + syncTriggeredAt.current[connectorId] = Date.now() + addToSet(setSyncingIds, connectorId) + + triggerSync( + { knowledgeBaseId, connectorId, rehydrate }, + { + onSuccess: () => { + setError(null) + const timer = setTimeout(() => { + cooldownTimersRef.current?.delete(timer) forceUpdate((n) => n + 1) - }, - onSettled: () => removeFromSet(setSyncingIds, connectorId), - } - ) - }, - [knowledgeBaseId, triggerSync, isSyncOnCooldown, addToSet, removeFromSet] - ) + }, SYNC_COOLDOWN_MS) + cooldownTimersRef.current?.add(timer) + }, + onError: (err) => { + logger.error('Sync trigger failed', { error: err.message }) + setError(err.message) + delete syncTriggeredAt.current[connectorId] + forceUpdate((n) => n + 1) + }, + onSettled: () => removeFromSet(setSyncingIds, connectorId), + } + ) + } - const handleTogglePause = useCallback( - (connector: ConnectorData) => { - addToSet(setUpdatingIds, connector.id) - updateConnector( - { - knowledgeBaseId, - connectorId: connector.id, - updates: { - status: - connector.status === 'paused' || connector.status === 'disabled' - ? 'active' - : 'paused', - }, + const handleTogglePause = (connector: ConnectorData) => { + addToSet(setUpdatingIds, connector.id) + updateConnector( + { + knowledgeBaseId, + connectorId: connector.id, + updates: { + status: + connector.status === 'paused' || connector.status === 'disabled' ? 'active' : 'paused', }, - { - onSettled: () => removeFromSet(setUpdatingIds, connector.id), - onSuccess: () => setError(null), - onError: (err) => { - logger.error('Toggle pause failed', { error: err.message }) - setError(err.message) - }, - } - ) - }, - [knowledgeBaseId, updateConnector, addToSet, removeFromSet] - ) + }, + { + onSettled: () => removeFromSet(setUpdatingIds, connector.id), + onSuccess: () => setError(null), + onError: (err) => { + logger.error('Toggle pause failed', { error: err.message }) + setError(err.message) + }, + } + ) + } const handleDeleteConnector = () => { if (!deleteTarget) return @@ -315,10 +312,10 @@ function ConnectorCard({ const serviceId = connectorDef?.auth.mode === 'oauth' ? connectorDef.auth.provider : undefined const providerId = serviceId ? getProviderIdFromServiceId(serviceId) : undefined - const requiredScopes = useMemo( - () => (connectorDef?.auth.mode === 'oauth' ? (connectorDef.auth.requiredScopes ?? []) : []), - [connectorDef] - ) + const requiredScopes = + connectorDef?.auth.mode === 'oauth' + ? (connectorDef.auth.requiredScopes ?? EMPTY_REQUIRED_SCOPES) + : EMPTY_REQUIRED_SCOPES const { data: credentials, refetch: refetchCredentials } = useOAuthCredentials(providerId, { workspaceId, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/loading.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/loading.tsx index 5e31ae87167..640117df44b 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/loading.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/loading.tsx @@ -1,7 +1,6 @@ 'use client' -import { Plus } from '@sim/emcn' -import { Database } from '@sim/emcn/icons' +import { Database, Plus } from '@sim/emcn/icons' import { noop } from '@sim/utils/helpers' import { type BreadcrumbItem, @@ -29,7 +28,7 @@ const ACTIONS: ChromeActionSpec[] = [ const BREADCRUMBS: BreadcrumbItem[] = [ { label: KNOWLEDGE_HEADER.rootLabel, icon: Database, onClick: noop }, - { label: '…', icon: Database, terminal: true }, + { label: '…', terminal: true }, ] export default function KnowledgeBaseLoading() { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/search-params.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/search-params.ts index c7f1ae8f27e..1e3436da727 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/search-params.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/search-params.ts @@ -16,22 +16,6 @@ export const addConnectorParam = { parser: parseAsString, } as const -/** - * `page` is the 1-based document-list pagination index for this knowledge base. - * Distinct from the single-document subview's `page` (a different route). The - * default page (1) clears from the URL. - */ -export const pageParam = { - key: 'page', - parser: parseAsInteger.withDefault(1), -} as const - -/** Pagination view-state: clean URLs, no back-stack churn. */ -export const pageUrlKeys = { - history: 'replace', - clearOnDefault: true, -} as const - /** Document `enabled` filter buckets, matching the status filter dropdown. */ const ENABLED_FILTERS = ['all', 'enabled', 'disabled'] as const @@ -56,12 +40,16 @@ export const kbDocumentSortParams = createSortParams(KB_SORT_COLUMNS, { }) /** - * Grouped filter/search URL state for the document list. + * Grouped filter/search/pagination URL state for the document list. * * - `q` is the document name search. The input is controlled directly by the * instant nuqs value; only its URL write is debounced via * `useDebouncedSearchSetter` — never written on every keystroke. * - `enabled` filters by processing/enabled status (`all` clears from the URL). + * - `page` is the 1-based pagination index, grouped here so a search or filter + * change resets it in the SAME write. Resetting it from a second hook escapes + * the search's debounce and writes the URL on every keystroke. Distinct from + * the single-document subview's `page`, which is a different route. * * `tagFilterEntries` is intentionally NOT represented here: it is an array of * rich filter-rule objects (slot, field type, operator, value, value-to per @@ -71,6 +59,7 @@ export const kbDocumentSortParams = createSortParams(KB_SORT_COLUMNS, { export const documentFiltersParsers = { q: parseAsString.withDefault(''), enabled: parseAsStringLiteral(ENABLED_FILTERS).withDefault('all'), + page: parseAsInteger.withDefault(1), } as const /** Filter/search/sort view-state: clean URLs, no back-stack churn. */ diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx index 9722c695371..858aeab0dd5 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx @@ -16,10 +16,9 @@ import { ChipTextarea, type ComboboxOption, cn, - Loader, toast, } from '@sim/emcn' -import { X } from '@sim/emcn/icons' +import { Loader, X } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { useParams } from 'next/navigation' @@ -175,8 +174,8 @@ export const CreateBaseModal = memo(function CreateBaseModal({ const params = useParams() const workspaceId = params.workspaceId as string - const createKnowledgeBaseMutation = useCreateKnowledgeBase(workspaceId) - const deleteKnowledgeBaseMutation = useDeleteKnowledgeBase(workspaceId) + const createKnowledgeBaseMutation = useCreateKnowledgeBase() + const deleteKnowledgeBaseMutation = useDeleteKnowledgeBase() const [submitStatus, setSubmitStatus] = useState(null) const [files, setFiles] = useState([]) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/components/edit-knowledge-base-modal/edit-knowledge-base-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/components/edit-knowledge-base-modal/edit-knowledge-base-modal.tsx index 464a64f4f91..f52d30c48a0 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/components/edit-knowledge-base-modal/edit-knowledge-base-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/components/edit-knowledge-base-modal/edit-knowledge-base-modal.tsx @@ -137,30 +137,24 @@ export const EditKnowledgeBaseModal = memo(function EditKnowledgeBaseModal({
-

Max Size

+

Max Size

{chunkingConfig.maxSize.toLocaleString()} - - tokens - + tokens

-

Min Size

+

Min Size

{chunkingConfig.minSize.toLocaleString()} - - chars - + chars

-

Overlap

+

Overlap

{chunkingConfig.overlap.toLocaleString()} - - tokens - + tokens

diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.test.tsx index c14cc716e7c..3fc840e5322 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.test.tsx @@ -85,4 +85,30 @@ describe('useKnowledgeUpload admission', () => { unmount() }) + + /** + * A partial batch failure still created every document that DID upload, so the caches have + * to reconcile on the throwing path too — otherwise the list renders without rows the + * server already has. + */ + it('reconciles the caches when part of a batch fails', async () => { + const onError = vi.fn() + const { result, unmount } = renderKnowledgeUploadHook(onError) + mockUploadKnowledgeDocumentSession + .mockResolvedValueOnce({ id: 'doc-1', filename: 'ok.bin' }) + .mockRejectedValueOnce(new Error('network died')) + + await act(async () => { + await expect( + result().uploadFiles([sizedFile('ok.bin', 10), sizedFile('bad.bin', 10)], 'kb-1') + ).rejects.toMatchObject({ code: 'PARTIAL_UPLOAD_FAILURE' }) + }) + + expect(mockInvalidateQueries).toHaveBeenCalledWith({ + queryKey: ['knowledge', 'detail', 'kb-1'], + }) + expect(mockInvalidateQueries).toHaveBeenCalledWith({ queryKey: ['knowledge', 'list'] }) + + unmount() + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts index 5923e9e03f6..81f884453be 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts @@ -119,6 +119,14 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) { }) } + /** Reconciles both caches an upload moves: the base's documents and the list's `docCount`. */ + const invalidateKnowledgeCaches = async (knowledgeBaseId: string) => { + await Promise.all([ + queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId) }), + queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }), + ]) + } + const uploadFilesInBatches = async ( files: File[], knowledgeBaseId: string, @@ -209,16 +217,21 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) { setUploadProgress((prev) => ({ ...prev, stage: 'processing' })) logger.info(`Successfully started processing ${uploadedDocuments.length} documents`) - await Promise.all([ - queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId) }), - /** The knowledge-base list rows carry `docCount`, so an upload changes them too. */ - queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }), - ]) + await invalidateKnowledgeCaches(knowledgeBaseId) return uploadedDocuments } catch (err) { logger.error('Error uploading documents:', err) + /** + * A partial batch failure still created every document that did upload, so the caches + * must reconcile on this path too — otherwise the list is missing rows that exist until + * its staleTime expires. Admission failures create nothing and need no refetch. + */ + if (err instanceof KnowledgeUploadError && err.code === 'PARTIAL_UPLOAD_FAILURE') { + await invalidateKnowledgeCaches(knowledgeBaseId) + } + const error: UploadError = err instanceof KnowledgeUploadError ? { message: err.message, code: err.code, details: err.details, timestamp: Date.now() } diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx index 0cb354d4c45..11cb67f02c7 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx @@ -2,8 +2,8 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import type { ChipDropdownOption } from '@sim/emcn' -import { Button, ChipConfirmModal, ChipDropdown, Plus, Tooltip, toast } from '@sim/emcn' -import { Database, FolderPlus, Pencil, Trash } from '@sim/emcn/icons' +import { Button, ChipConfirmModal, ChipDropdown, Tooltip, toast } from '@sim/emcn' +import { Database, FolderPlus, Pencil, Plus, Trash } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { useParams, useRouter } from 'next/navigation' @@ -220,8 +220,8 @@ export function Knowledge() { const canEditRef = useRef(canEdit) canEditRef.current = canEdit - const { mutateAsync: updateKnowledgeBaseMutation } = useUpdateKnowledgeBase(workspaceId) - const deleteKnowledgeBase = useDeleteKnowledgeBase(workspaceId) + const { mutateAsync: updateKnowledgeBaseMutation } = useUpdateKnowledgeBase() + const deleteKnowledgeBase = useDeleteKnowledgeBase() const bulkMoveKnowledgeBases = useBulkMoveKnowledgeBases(workspaceId) const bulkDeleteKnowledgeBases = useBulkDeleteKnowledgeBases(workspaceId) @@ -694,14 +694,12 @@ export function Knowledge() { [selectedRowIds] ) - const bulkDeleteLabel = useMemo(() => { - const count = selectedKnowledgeBaseIds.length + selectedFolderIds.length - const firstName = - selectedKnowledgeBaseIds.length > 0 - ? knowledgeBasesRef.current.find((kb) => kb.id === selectedKnowledgeBaseIds[0])?.name - : foldersRef.current.find((folder) => folder.id === selectedFolderIds[0])?.name - return selectionLabel(count, firstName) - }, [selectedKnowledgeBaseIds, selectedFolderIds]) + const bulkDeleteCount = selectedKnowledgeBaseIds.length + selectedFolderIds.length + const bulkDeleteFirstName = + selectedKnowledgeBaseIds.length > 0 + ? knowledgeBases.find((kb) => kb.id === selectedKnowledgeBaseIds[0])?.name + : folders.find((folder) => folder.id === selectedFolderIds[0])?.name + const bulkDeleteLabel = selectionLabel(bulkDeleteCount, bulkDeleteFirstName) const handleRowClick = useCallback( (rowId: string) => { @@ -1423,7 +1421,7 @@ export function Knowledge() { if (!open) setFolderPendingDelete(null) }} srTitle='Delete folder' - title='Delete folder' + title='Delete Folder' text={[ 'Are you sure you want to delete ', { text: folderPendingDelete?.name ?? 'this folder', bold: true }, @@ -1441,7 +1439,7 @@ export function Knowledge() { open={isBulkDeleteModalOpen} onOpenChange={setIsBulkDeleteModalOpen} srTitle='Delete selected' - title='Delete selected' + title='Delete Selected' text={[ 'Are you sure you want to delete ', { text: bulkDeleteLabel, bold: true }, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/loading.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/loading.tsx index c2de47d0f0b..66921bb0f43 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/loading.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/loading.tsx @@ -1,7 +1,6 @@ 'use client' -import { Plus } from '@sim/emcn' -import { Database, FolderPlus } from '@sim/emcn/icons' +import { Database, FolderPlus, Plus } from '@sim/emcn/icons' import { type ChromeActionSpec, ResourceChromeFallback, diff --git a/apps/sim/hooks/kb/use-knowledge-base-tag-definitions.ts b/apps/sim/hooks/kb/use-knowledge-base-tag-definitions.ts index 84b62f55ad5..cec2279d000 100644 --- a/apps/sim/hooks/kb/use-knowledge-base-tag-definitions.ts +++ b/apps/sim/hooks/kb/use-knowledge-base-tag-definitions.ts @@ -1,6 +1,6 @@ 'use client' -import { useCallback, useMemo } from 'react' +import { getErrorMessage } from '@sim/utils/errors' import { useQueryClient } from '@tanstack/react-query' import type { AllTagSlot } from '@/lib/knowledge/constants' import { useTagDefinitionsQuery } from '@/hooks/queries/kb/knowledge' @@ -19,23 +19,26 @@ export interface TagDefinition { * Hook for fetching KB-scoped tag definitions (for filtering/selection) * Uses React Query as single source of truth */ +/** Stable empty fallback, so a pending query does not hand consumers a new array each render. */ +const EMPTY_TAG_DEFINITIONS: TagDefinition[] = [] + export function useKnowledgeBaseTagDefinitions(knowledgeBaseId: string | null) { const queryClient = useQueryClient() const query = useTagDefinitionsQuery(knowledgeBaseId) - const fetchTagDefinitions = useCallback(async () => { + const fetchTagDefinitions = async () => { if (!knowledgeBaseId) return await queryClient.invalidateQueries({ queryKey: knowledgeKeys.tagDefinitions(knowledgeBaseId), }) - }, [queryClient, knowledgeBaseId]) + } - const tagDefinitions = useMemo(() => (query.data ?? []) as TagDefinition[], [query.data]) + const tagDefinitions = (query.data ?? EMPTY_TAG_DEFINITIONS) as TagDefinition[] return { tagDefinitions, isLoading: query.isLoading, - error: query.error instanceof Error ? query.error.message : null, + error: query.error ? getErrorMessage(query.error) : null, fetchTagDefinitions, } } diff --git a/apps/sim/hooks/kb/use-knowledge.ts b/apps/sim/hooks/kb/use-knowledge.ts index 5c1f18142c8..12c2184ea13 100644 --- a/apps/sim/hooks/kb/use-knowledge.ts +++ b/apps/sim/hooks/kb/use-knowledge.ts @@ -1,7 +1,8 @@ -import { useCallback, useMemo } from 'react' +import { useCallback } from 'react' +import { getErrorMessage } from '@sim/utils/errors' import { useQueryClient } from '@tanstack/react-query' import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types' -import type { ChunkData, DocumentData, KnowledgeBaseData } from '@/lib/knowledge/types' +import type { ChunkData, DocumentData } from '@/lib/knowledge/types' import { type DocumentTagFilter, type KnowledgeChunksResponse, @@ -36,7 +37,7 @@ export function useKnowledgeBase(id: string) { knowledgeBase: query.data ?? null, isLoading: query.isLoading, isFetching: query.isFetching, - error: query.error instanceof Error ? query.error.message : null, + error: query.error ? getErrorMessage(query.error) : null, refresh, } } @@ -52,7 +53,7 @@ export function useDocument(knowledgeBaseId: string, documentId: string) { document: query.data ?? null, isLoading: query.isLoading, isFetching: query.isFetching, - error: query.error instanceof Error ? query.error.message : null, + error: query.error ? getErrorMessage(query.error) : null, } } @@ -93,15 +94,12 @@ export function useKnowledgeBaseDocuments( tagFilters, }) - const refetchIntervalFn = useMemo(() => { - if (typeof options?.refetchInterval === 'function') { - const userFn = options.refetchInterval - return (query: { state: { data?: KnowledgeDocumentsResponse } }) => { - return userFn(query.state.data) - } - } - return options?.refetchInterval - }, [options?.refetchInterval]) + const userRefetchInterval = options?.refetchInterval + const refetchIntervalFn = + typeof userRefetchInterval === 'function' + ? (query: { state: { data?: KnowledgeDocumentsResponse } }) => + userRefetchInterval(query.state.data) + : userRefetchInterval const query = useKnowledgeDocumentsQuery( { @@ -128,12 +126,8 @@ export function useKnowledgeBaseDocuments( hasMore: false, } - const hasProcessingDocs = useMemo( - () => - documents.some( - (doc) => doc.processingStatus === 'pending' || doc.processingStatus === 'processing' - ), - [documents] + const hasProcessingDocs = documents.some( + (doc) => doc.processingStatus === 'pending' || doc.processingStatus === 'processing' ) const refreshDocuments = useCallback(async () => { @@ -142,23 +136,20 @@ export function useKnowledgeBaseDocuments( }) }, [queryClient, knowledgeBaseId, paramsKey]) - const updateDocument = useCallback( - (documentId: string, updates: Partial) => { - queryClient.setQueryData( - knowledgeKeys.documents(knowledgeBaseId, paramsKey), - (previous) => { - if (!previous) return previous - return { - ...previous, - documents: previous.documents.map((doc) => - doc.id === documentId ? { ...doc, ...updates } : doc - ), - } + const updateDocument = (documentId: string, updates: Partial) => { + queryClient.setQueryData( + knowledgeKeys.documents(knowledgeBaseId, paramsKey), + (previous) => { + if (!previous) return previous + return { + ...previous, + documents: previous.documents.map((doc) => + doc.id === documentId ? { ...doc, ...updates } : doc + ), } - ) - }, - [knowledgeBaseId, paramsKey, queryClient] - ) + } + ) + } return { documents, @@ -166,7 +157,7 @@ export function useKnowledgeBaseDocuments( isLoading: query.isLoading, isFetching: query.isFetching, isPlaceholderData: query.isPlaceholderData, - error: query.error instanceof Error ? query.error.message : null, + error: query.error ? getErrorMessage(query.error) : null, hasProcessingDocuments: hasProcessingDocs, refreshDocuments, updateDocument, @@ -183,45 +174,14 @@ export function useKnowledgeBasesList( enabled?: boolean } ) { - const queryClient = useQueryClient() const query = useKnowledgeBasesQuery(workspaceId, { enabled: options?.enabled ?? true }) - const removeKnowledgeBase = useCallback( - (knowledgeBaseId: string) => { - queryClient.setQueryData( - knowledgeKeys.list(workspaceId), - (previous) => previous?.filter((kb) => kb.id !== knowledgeBaseId) ?? [] - ) - }, - [queryClient, workspaceId] - ) - - const updateKnowledgeBase = useCallback( - (id: string, updates: Partial) => { - queryClient.setQueryData( - knowledgeKeys.list(workspaceId), - (previous) => previous?.map((kb) => (kb.id === id ? { ...kb, ...updates } : kb)) ?? [] - ) - queryClient.setQueryData(knowledgeKeys.detail(id), (previous) => - previous ? { ...previous, ...updates } : previous - ) - }, - [queryClient, workspaceId] - ) - - const refreshList = useCallback(async () => { - await queryClient.invalidateQueries({ queryKey: knowledgeKeys.list(workspaceId) }) - }, [queryClient, workspaceId]) - return { knowledgeBases: query.data ?? [], isLoading: query.isLoading, isFetching: query.isFetching, isPlaceholderData: query.isPlaceholderData, - error: query.error instanceof Error ? query.error.message : null, - refreshList, - removeKnowledgeBase, - updateKnowledgeBase, + error: query.error ? getErrorMessage(query.error) : null, } } @@ -270,29 +230,6 @@ export function useDocumentChunks( const hasNextPage = currentPage < totalPages const hasPrevPage = currentPage > 1 - const goToPage = useCallback( - (newPage: number): boolean => { - return newPage >= 1 && newPage <= totalPages - }, - [totalPages] - ) - - const refreshChunks = useCallback(async () => { - const paramsKey = serializeChunkParams({ - knowledgeBaseId, - documentId, - limit: DEFAULT_PAGE_SIZE, - offset, - search: search || undefined, - enabledFilter, - sortBy, - sortOrder, - }) - await queryClient.invalidateQueries({ - queryKey: knowledgeKeys.chunks(knowledgeBaseId, documentId, paramsKey), - }) - }, [knowledgeBaseId, documentId, offset, search, enabledFilter, sortBy, sortOrder, queryClient]) - const updateChunk = useCallback( (chunkId: string, updates: Partial) => { const paramsKey = serializeChunkParams({ @@ -325,13 +262,11 @@ export function useDocumentChunks( chunks, isLoading: chunkQuery.isLoading, isFetching: chunkQuery.isFetching, - error: chunkQuery.error instanceof Error ? chunkQuery.error.message : null, + error: chunkQuery.error ? getErrorMessage(chunkQuery.error) : null, currentPage, totalPages, hasNextPage, hasPrevPage, - goToPage, - refreshChunks, updateChunk, } } diff --git a/apps/sim/hooks/kb/use-tag-definitions.ts b/apps/sim/hooks/kb/use-tag-definitions.ts index 01bb9c18f74..e53b875f603 100644 --- a/apps/sim/hooks/kb/use-tag-definitions.ts +++ b/apps/sim/hooks/kb/use-tag-definitions.ts @@ -1,6 +1,7 @@ 'use client' -import { useCallback, useMemo } from 'react' +import { useCallback } from 'react' +import { getErrorMessage } from '@sim/utils/errors' import { useQueryClient } from '@tanstack/react-query' import type { AllTagSlot } from '@/lib/knowledge/constants' import { @@ -20,6 +21,9 @@ export interface TagDefinition { updatedAt: string } +/** Stable empty fallback, so a pending query does not hand consumers a new array each render. */ +const EMPTY_TAG_DEFINITIONS: TagDefinition[] = [] + export interface TagDefinitionInput { tagSlot: AllTagSlot displayName: string @@ -40,7 +44,7 @@ export function useTagDefinitions( const { mutateAsync: saveTagDefinitionsMutation } = useSaveDocumentTagDefinitions() const { mutateAsync: deleteTagDefinitionsMutation } = useDeleteDocumentTagDefinitions() - const tagDefinitions = useMemo(() => (query.data ?? []) as TagDefinition[], [query.data]) + const tagDefinitions = (query.data ?? EMPTY_TAG_DEFINITIONS) as TagDefinition[] const fetchTagDefinitions = useCallback(async () => { if (!knowledgeBaseId || !documentId) return @@ -49,55 +53,23 @@ export function useTagDefinitions( }) }, [queryClient, knowledgeBaseId, documentId]) - const saveTagDefinitions = useCallback( - async (definitions: TagDefinitionInput[]) => { - if (!knowledgeBaseId || !documentId) { - throw new Error('Knowledge base ID and document ID are required') - } - - return saveTagDefinitionsMutation({ - knowledgeBaseId, - documentId, - definitions: definitions as DocumentTagDefinitionInput[], - }) - }, - [knowledgeBaseId, documentId, saveTagDefinitionsMutation] - ) - - const deleteTagDefinitions = useCallback(async () => { + const saveTagDefinitions = async (definitions: TagDefinitionInput[]) => { if (!knowledgeBaseId || !documentId) { throw new Error('Knowledge base ID and document ID are required') } - return deleteTagDefinitionsMutation({ + return saveTagDefinitionsMutation({ knowledgeBaseId, documentId, + definitions: definitions as DocumentTagDefinitionInput[], }) - }, [knowledgeBaseId, documentId, deleteTagDefinitionsMutation]) - - const getTagLabel = useCallback( - (tagSlot: string): string => { - const definition = tagDefinitions.find((def) => def.tagSlot === tagSlot) - return definition?.displayName || tagSlot - }, - [tagDefinitions] - ) - - const getTagDefinition = useCallback( - (tagSlot: string): TagDefinition | undefined => { - return tagDefinitions.find((def) => def.tagSlot === tagSlot) - }, - [tagDefinitions] - ) + } return { tagDefinitions, isLoading: query.isLoading, - error: query.error instanceof Error ? query.error.message : null, + error: query.error ? getErrorMessage(query.error) : null, fetchTagDefinitions, saveTagDefinitions, - deleteTagDefinitions, - getTagLabel, - getTagDefinition, } } diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index fe4d6d21004..a00aad93346 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -1,4 +1,3 @@ -import { createLogger } from '@sim/logger' import { keepPreviousData, useInfiniteQuery, @@ -24,8 +23,6 @@ import { import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE } from '@/lib/knowledge/constants' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' -const logger = createLogger('KnowledgeConnectorQueries') - export type { ConnectorData, ConnectorDetailData, SyncLogData } export const CONNECTOR_LIST_STALE_TIME = 30 * 1000 @@ -231,9 +228,13 @@ export function useTriggerSync() { return useMutation({ mutationFn: triggerSync, + /** + * The sync itself runs async — the connector list's own syncing poll surfaces its + * progress. Only the connector rows have anything to say yet. + */ onSettled: (_data, _error, { knowledgeBaseId }) => { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), + queryKey: connectorKeys.all(knowledgeBaseId), }) }, }) @@ -244,8 +245,8 @@ export const connectorDocumentKeys = { [...connectorKeys.detail(knowledgeBaseId, connectorId), 'documents'] as const, lists: (knowledgeBaseId?: string, connectorId?: string) => [...connectorDocumentKeys.all(knowledgeBaseId, connectorId), 'list'] as const, - list: (knowledgeBaseId?: string, connectorId?: string) => - connectorDocumentKeys.lists(knowledgeBaseId, connectorId), + list: (knowledgeBaseId?: string, connectorId?: string, includeExcluded = false) => + [...connectorDocumentKeys.lists(knowledgeBaseId, connectorId), includeExcluded] as const, } async function fetchConnectorDocuments( @@ -275,7 +276,7 @@ export function useConnectorDocuments( ) { const includeExcluded = options?.includeExcluded ?? false return useInfiniteQuery({ - queryKey: [...connectorDocumentKeys.list(knowledgeBaseId, connectorId), includeExcluded], + queryKey: connectorDocumentKeys.list(knowledgeBaseId, connectorId, includeExcluded), queryFn: ({ signal, pageParam }) => fetchConnectorDocuments( knowledgeBaseId as string, @@ -323,7 +324,7 @@ export function useExcludeConnectorDocument() { mutationFn: excludeConnectorDocuments, onSettled: (_data, _error, { knowledgeBaseId, connectorId }) => { queryClient.invalidateQueries({ - queryKey: connectorDocumentKeys.list(knowledgeBaseId, connectorId), + queryKey: connectorDocumentKeys.lists(knowledgeBaseId, connectorId), }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), @@ -352,7 +353,7 @@ export function useRestoreConnectorDocument() { mutationFn: restoreConnectorDocuments, onSettled: (_data, _error, { knowledgeBaseId, connectorId }) => { queryClient.invalidateQueries({ - queryKey: connectorDocumentKeys.list(knowledgeBaseId, connectorId), + queryKey: connectorDocumentKeys.lists(knowledgeBaseId, connectorId), }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index 5902e0d7be1..515ee4a47b6 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -327,6 +327,7 @@ async function fetchAllDocumentChunks( const limit = 100 while (hasMore) { + if (signal?.aborted) break const response = await fetchKnowledgeChunks( { knowledgeBaseId, @@ -402,9 +403,11 @@ export function useUpdateChunk() { return useMutation({ mutationFn: updateChunk, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { - queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), - }) + /** + * `document` is a prefix of `detail`, so the wider key added only a refetch of every + * other document page, chunk page, tag definition, and connector row cached for this + * base. Nothing this mutation writes is rendered from those. + */ queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) @@ -435,10 +438,15 @@ export function useDeleteChunk() { mutationFn: deleteChunk, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), + queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** + * `exact`, so the base's own `chunkCount`/`tokenCount` refresh without dragging every + * sibling document and chunk page cached beneath `detail` along with them. + */ queryClient.invalidateQueries({ - queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), + queryKey: knowledgeKeys.detail(knowledgeBaseId), + exact: true, }) }, }) @@ -472,10 +480,15 @@ export function useCreateChunk() { mutationFn: createChunk, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), + queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** + * `exact`, so the base's own `chunkCount`/`tokenCount` refresh without dragging every + * sibling document and chunk page cached beneath `detail` along with them. + */ queryClient.invalidateQueries({ - queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), + queryKey: knowledgeKeys.detail(knowledgeBaseId), + exact: true, }) }, }) @@ -511,9 +524,11 @@ export function useUpdateDocument() { return useMutation({ mutationFn: updateDocument, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { - queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), - }) + /** + * `document` is a prefix of `detail`, so the wider key added only a refetch of every + * other document page, chunk page, tag definition, and connector row cached for this + * base. Nothing this mutation writes is rendered from those. + */ queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) @@ -619,7 +634,7 @@ async function createKnowledgeBase(params: CreateKnowledgeBaseParams): Promise { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), + queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** + * `exact`, so the base's own `chunkCount`/`tokenCount` refresh without dragging every + * sibling document and chunk page cached beneath `detail` along with them. + */ queryClient.invalidateQueries({ - queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), + queryKey: knowledgeKeys.detail(knowledgeBaseId), + exact: true, }) }, }) @@ -784,9 +807,11 @@ export function useUpdateDocumentTags() { return useMutation({ mutationFn: updateDocumentTags, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { - queryClient.invalidateQueries({ - queryKey: knowledgeKeys.detail(knowledgeBaseId), - }) + /** + * `document` is a prefix of `detail`, so the wider key added only a refetch of every + * other document page, chunk page, tag definition, and connector row cached for this + * base. Nothing this mutation writes is rendered from those. + */ queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) diff --git a/apps/sim/lib/knowledge/chunks/service.ts b/apps/sim/lib/knowledge/chunks/service.ts index c9bb89776a2..6c6f5155770 100644 --- a/apps/sim/lib/knowledge/chunks/service.ts +++ b/apps/sim/lib/knowledge/chunks/service.ts @@ -206,7 +206,6 @@ export async function createChunk( embeddingModel: kbEmbeddingModel, startOffset: 0, // Manual chunks don't have document offsets endOffset: chunkData.content.length, - // Inherit text tags from parent document tag1: docTags.tag1 as string | null, tag2: docTags.tag2 as string | null, tag3: docTags.tag3 as string | null, @@ -214,16 +213,13 @@ export async function createChunk( tag5: docTags.tag5 as string | null, tag6: docTags.tag6 as string | null, tag7: docTags.tag7 as string | null, - // Inherit number tags from parent document (5 slots) number1: docTags.number1 as number | null, number2: docTags.number2 as number | null, number3: docTags.number3 as number | null, number4: docTags.number4 as number | null, number5: docTags.number5 as number | null, - // Inherit date tags from parent document (2 slots) date1: docTags.date1 as Date | null, date2: docTags.date2 as Date | null, - // Inherit boolean tags from parent document (3 slots) boolean1: docTags.boolean1 as boolean | null, boolean2: docTags.boolean2 as boolean | null, boolean3: docTags.boolean3 as boolean | null, @@ -242,7 +238,6 @@ export async function createChunk( ) } - // Update document statistics await tx .update(document) .set({ @@ -315,12 +310,10 @@ export async function batchChunkOperation( const totalTokensToRemove = chunksToDelete.reduce((sum, chunk) => sum + chunk.tokenCount, 0) const totalCharsToRemove = chunksToDelete.reduce((sum, chunk) => sum + chunk.contentLength, 0) - // Delete chunks const deleteResult = await tx .delete(embedding) .where(and(eq(embedding.documentId, documentId), inArray(embedding.id, chunkIds))) - // Update document statistics await tx .update(document) .set({ @@ -333,7 +326,6 @@ export async function batchChunkOperation( successCount = chunksToDelete.length }) } else { - // Handle enable/disable operations const enabled = operation === 'enable' await db @@ -529,7 +521,6 @@ export async function updateChunk( }) .where(eq(embedding.id, chunkId)) - // Fetch the updated chunk const updatedChunk = await db .select({ id: embedding.id, @@ -588,10 +579,8 @@ export async function deleteChunk( const chunk = chunkToDelete[0] - // Delete the chunk await tx.delete(embedding).where(eq(embedding.id, chunkId)) - // Update document statistics await tx .update(document) .set({ diff --git a/apps/sim/lib/knowledge/documents/types.ts b/apps/sim/lib/knowledge/documents/types.ts index 5ef9c8c71d0..65e165011b0 100644 --- a/apps/sim/lib/knowledge/documents/types.ts +++ b/apps/sim/lib/knowledge/documents/types.ts @@ -1,4 +1,3 @@ -// Document sorting options export type DocumentSortField = | 'filename' | 'fileSize' diff --git a/apps/sim/lib/knowledge/documents/utils.ts b/apps/sim/lib/knowledge/documents/utils.ts index 89f30c7db99..b180e9b4bee 100644 --- a/apps/sim/lib/knowledge/documents/utils.ts +++ b/apps/sim/lib/knowledge/documents/utils.ts @@ -47,7 +47,6 @@ function isRetryableErrorType(error: unknown): error is RetryableError { export function isRetryableError(error: unknown): boolean { if (!isRetryableErrorType(error)) return false - // Check for rate limiting status codes if ( hasStatus(error) && (error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504) @@ -55,7 +54,6 @@ export function isRetryableError(error: unknown): boolean { return true } - // Check for network-level errors (DNS, connection, timeout) const errorMessage = toError(error).message const lowerMessage = errorMessage.toLowerCase() @@ -77,7 +75,6 @@ export function isRetryableError(error: unknown): boolean { return true } - // Check for rate limiting in error messages const rateLimitKeywords = [ 'rate limit', 'rate_limit', @@ -124,13 +121,11 @@ export async function retryWithExponentialBackoff( lastError = toError(error) logger.warn(`Operation failed on attempt ${attempt + 1}`, { error }) - // If this is the last attempt, throw the error if (attempt === maxRetries) { logger.error(`Operation failed after ${maxRetries + 1} attempts`, { error }) throw lastError } - // Check if error is retryable if (!retryCondition(error as RetryableError)) { logger.warn('Error is not retryable, throwing immediately', { error }) throw lastError @@ -187,7 +182,6 @@ export async function fetchWithRetry( return retryWithExponentialBackoff(async () => { const response = await fetch(url, options) - // If response is not ok and status indicates rate limiting, throw an error if (!response.ok && isRetryableError({ status: response.status })) { const errorText = await response.text() const error: HTTPError = new Error( diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index a28a2e78536..9c2867ea857 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -95,7 +95,6 @@ export interface SearchParams { distanceThreshold?: number } -// Use shared embedding utility export { generateSearchEmbedding } from '@/lib/knowledge/embeddings' /** All valid tag slot keys */ @@ -173,7 +172,6 @@ function buildFilterCondition(filter: StructuredFilter, embeddingTable: any) { const column = embeddingTable[tagSlot] if (!column) return null - // Handle text operators if (fieldType === 'text') { const coerced = coerceTagFilterValue(value, 'text') if (!coerced.ok) return null @@ -197,7 +195,6 @@ function buildFilterCondition(filter: StructuredFilter, embeddingTable: any) { } } - // Handle number operators if (fieldType === 'number') { const coerced = coerceTagFilterValue(value, 'number') if (!coerced.ok) return null @@ -228,7 +225,7 @@ function buildFilterCondition(filter: StructuredFilter, embeddingTable: any) { } } - // Handle date operators - expects YYYY-MM-DD format from frontend + // Date values arrive as YYYY-MM-DD strings from the frontend. if (fieldType === 'date') { const coerced = coerceTagFilterValue(value, 'date') if (!coerced.ok) return null @@ -262,7 +259,6 @@ function buildFilterCondition(filter: StructuredFilter, embeddingTable: any) { } } - // Handle boolean operators if (fieldType === 'boolean') { const coerced = coerceTagFilterValue(value, 'boolean') if (!coerced.ok) return null @@ -277,7 +273,6 @@ function buildFilterCondition(filter: StructuredFilter, embeddingTable: any) { } } - // Fallback to equality return sql`${column} = ${value}` } @@ -438,7 +433,6 @@ export async function handleTagOnlySearch(params: SearchParams): Promise { @@ -496,7 +490,6 @@ export async function handleVectorOnlySearch(params: SearchParams): Promise`${embedding.embedding} <=> ${queryVector}::vector`.as('distance') if (strategy.useParallel) { - // Parallel approach for many KBs const parallelLimit = Math.ceil(topK / knowledgeBaseIds.length) + 5 const queryPromises = knowledgeBaseIds.map(async (kbId) => { @@ -734,14 +727,12 @@ export async function handleTagAndVectorSearch(params: SearchParams): Promise r.id), queryVector, diff --git a/apps/sim/lib/knowledge/tags/service.ts b/apps/sim/lib/knowledge/tags/service.ts index bee395eace1..5fe313989a3 100644 --- a/apps/sim/lib/knowledge/tags/service.ts +++ b/apps/sim/lib/knowledge/tags/service.ts @@ -261,34 +261,28 @@ export async function createOrUpdateTagDefinitionsBulk( const updated: DocumentTagDefinition[] = [] const errors: string[] = [] - // Get existing definitions to check for conflicts and determine operations const existingDefinitions = await getDocumentTagDefinitions(knowledgeBaseId) const existingBySlot = new Map(existingDefinitions.map((def) => [def.tagSlot, def])) const existingByDisplayName = new Map(existingDefinitions.map((def) => [def.displayName, def])) - // Process each definition for (const defData of definitions) { try { const { tagSlot, displayName, fieldType, originalDisplayName } = defData - // Validate field type if (!SUPPORTED_FIELD_TYPES.includes(fieldType as (typeof SUPPORTED_FIELD_TYPES)[number])) { errors.push(`Invalid field type: ${fieldType}`) continue } - // Check if this is an update (has originalDisplayName) or create const isUpdate = !!originalDisplayName if (isUpdate) { - // Update existing definition const existingDef = existingByDisplayName.get(originalDisplayName!) if (!existingDef) { errors.push(`Tag definition with display name "${originalDisplayName}" not found`) continue } - // Check if new display name conflicts with another definition if (displayName !== originalDisplayName && existingByDisplayName.has(displayName)) { errors.push(`Display name "${displayName}" already exists`) continue @@ -314,10 +308,8 @@ export async function createOrUpdateTagDefinitionsBulk( updatedAt: now, }) } else { - // Create new definition let finalTagSlot = tagSlot - // If no slot provided or slot is taken, find next available if (!finalTagSlot || existingBySlot.has(finalTagSlot)) { const nextSlot = await getNextAvailableSlot(knowledgeBaseId, fieldType, existingBySlot) if (!nextSlot) { @@ -327,13 +319,11 @@ export async function createOrUpdateTagDefinitionsBulk( finalTagSlot = nextSlot } - // Check slot conflicts if (existingBySlot.has(finalTagSlot)) { errors.push(`Tag slot "${finalTagSlot}" is already in use`) continue } - // Check display name conflicts if (existingByDisplayName.has(displayName)) { errors.push(`Display name "${displayName}" already exists`) continue @@ -660,7 +650,6 @@ export async function getTagUsage( const tagSlot = def.tagSlot validateTagSlot(tagSlot) - // Build WHERE conditions based on field type // Text columns need both IS NOT NULL and != '' checks // Numeric/date/boolean columns only need IS NOT NULL const fieldType = getFieldTypeForSlot(tagSlot) @@ -674,7 +663,6 @@ export async function getTagUsage( isNotNull(sql`${sql.raw(tagSlot)}`), ] - // Only add empty string check for text columns if (isTextColumn) { whereConditions.push(sql`${sql.raw(tagSlot)} != ''`) } diff --git a/apps/sim/lib/knowledge/tags/utils.ts b/apps/sim/lib/knowledge/tags/utils.ts index 58d000a3664..1d00da28889 100644 --- a/apps/sim/lib/knowledge/tags/utils.ts +++ b/apps/sim/lib/knowledge/tags/utils.ts @@ -136,7 +136,6 @@ export function parseNumberValue(value: string): number | null { export function parseDateValue(value: string): Date | null { const stringValue = String(value).trim() - // Must be YYYY-MM-DD format if (!/^\d{4}-\d{2}-\d{2}$/.test(stringValue)) { return null } From 28a6e821e9a775ea4fc16bc36adc93a526bee9e9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 16 Aug 2026 22:01:45 -0700 Subject: [PATCH 4/5] refactor(knowledge): one row reader, one visible-list composition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up from the quality pass. The two list queries had grown into near-copies of each other — same 14-column projection, same document join, same cap check, same row mapping — and the workspace-plus-legacy composition was pasted into both the internal use case and the v1 route, one of which is a surface adapter that should not be composing domain reads at all. Both queries now read through one private projection, so a column added to one list cannot go missing from the other half of the same rendered list, and `listWorkspaceAndLegacyKnowledgeBases` owns the composition both surfaces call. That merge also projects connector types once over the merged set instead of once per source, and skips the copy-and-sort entirely when there are no legacy rows — the common case. Also from the review: the chunk-row memo depends on the two primitives it reads rather than the whole polled document object, the selected chunk resolves in one scan instead of two, an aborted chunk-search pagination throws instead of caching a truncated result as complete, upload cache reconciliation no longer delays the rejected promise, the key-hierarchy rule is stated once on the key factory rather than six times at its call sites, and `TagDefinition` has one declaration. --- apps/sim/app/api/v1/knowledge/route.ts | 21 +- .../knowledge/[id]/[documentId]/document.tsx | 33 +-- .../knowledge/hooks/use-knowledge-upload.ts | 2 +- .../[workspaceId]/knowledge/knowledge.tsx | 4 +- .../kb/use-knowledge-base-tag-definitions.ts | 6 +- apps/sim/hooks/kb/use-knowledge.ts | 9 +- apps/sim/hooks/kb/use-tag-definitions.ts | 21 +- apps/sim/hooks/queries/kb/knowledge.ts | 30 +- .../sim/hooks/queries/utils/knowledge-keys.ts | 8 + .../application/knowledge-bases.test.ts | 33 +-- .../knowledge/application/knowledge-bases.ts | 20 +- apps/sim/lib/knowledge/constants.ts | 7 + apps/sim/lib/knowledge/service.test.ts | 38 +++ apps/sim/lib/knowledge/service.ts | 261 ++++++++++-------- 14 files changed, 250 insertions(+), 243 deletions(-) diff --git a/apps/sim/app/api/v1/knowledge/route.ts b/apps/sim/app/api/v1/knowledge/route.ts index 132be1b0a8a..9593cc8466e 100644 --- a/apps/sim/app/api/v1/knowledge/route.ts +++ b/apps/sim/app/api/v1/knowledge/route.ts @@ -10,10 +10,7 @@ import { } from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { performCreateKnowledgeBase } from '@/lib/knowledge/orchestration' -import { - getLegacyPersonalKnowledgeBases, - getWorkspaceKnowledgeBases, -} from '@/lib/knowledge/service' +import { listWorkspaceAndLegacyKnowledgeBases } from '@/lib/knowledge/service' import { formatKnowledgeBase, handleError } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, @@ -46,19 +43,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId) if (accessError) return accessError - /** - * The workspace's own rows, read after `validateWorkspaceAccess` has authorized this - * caller rather than re-deriving that access inside the row query, plus the caller's - * legacy workspace-less bases, which belong to no workspace and answer only to their - * creator. This is the shape the internal list serves too. - */ - const [workspaceBases, legacyPersonalBases] = await Promise.all([ - getWorkspaceKnowledgeBases(workspaceId), - getLegacyPersonalKnowledgeBases(userId), - ]) - const knowledgeBases = [...workspaceBases.data, ...legacyPersonalBases].sort( - (a, b) => a.createdAt.getTime() - b.createdAt.getTime() - ) + /** Read only after `validateWorkspaceAccess` authorized this caller; same list the + * internal surface serves, from the same place. */ + const knowledgeBases = await listWorkspaceAndLegacyKnowledgeBases(userId, workspaceId) return NextResponse.json({ success: true, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx index 0b99f0710bd..5c9e0e4d99d 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx @@ -300,12 +300,7 @@ export function Document({ const totalPagesRef = useRef(totalPages) totalPagesRef.current = totalPages - const goToPage = useCallback( - async (page: number) => { - await setDocumentParams({ page }) - }, - [setDocumentParams] - ) + const goToPage = useCallback((page: number) => setDocumentParams({ page }), [setDocumentParams]) const updateChunk = showingSearch ? (_id: string, _updates: Record) => {} @@ -360,13 +355,10 @@ export function Document({ const isInEditorView = selectedChunkId !== null || isCreatingNewChunk - const selectedChunk = selectedChunkId - ? (displayChunks.find((c) => c.id === selectedChunkId) ?? null) - : null - - const currentChunkIndex = selectedChunk - ? displayChunks.findIndex((c) => c.id === selectedChunk.id) + const currentChunkIndex = selectedChunkId + ? displayChunks.findIndex((chunk) => chunk.id === selectedChunkId) : -1 + const selectedChunk = currentChunkIndex >= 0 ? displayChunks[currentChunkIndex] : null const canNavigatePrev = currentChunkIndex > 0 || currentPage > 1 const canNavigateNext = currentChunkIndex < displayChunks.length - 1 || currentPage < totalPages @@ -943,13 +935,16 @@ export function Document({ [activeSort, onSortColumn, onClearSort, goToPage] ) + const hasDocumentData = documentData !== null + const processingStatus = documentData?.processingStatus + const chunkRows: ResourceRow[] = useMemo(() => { /** * No document yet is "not known", not "not ready". Falling through to the status row * flashed `Document not ready` on every open, for the frame between mount and the * document query resolving — a claim about a document nothing had read yet. */ - if (!documentData) return [] + if (!hasDocumentData) return [] if (!isCompleted) { return [ @@ -961,12 +956,10 @@ export function Document({
- {documentData.processingStatus === 'pending' && - 'Document processing pending...'} - {documentData.processingStatus === 'processing' && - 'Document processing in progress...'} - {documentData.processingStatus === 'failed' && 'Document processing failed'} - {!documentData.processingStatus && 'Document not ready'} + {processingStatus === 'pending' && 'Document processing pending...'} + {processingStatus === 'processing' && 'Document processing in progress...'} + {processingStatus === 'failed' && 'Document processing failed'} + {!processingStatus && 'Document not ready'}
), @@ -1010,7 +1003,7 @@ export function Document({ }, } }) - }, [isCompleted, documentData, displayChunks, searchQuery]) + }, [isCompleted, hasDocumentData, processingStatus, displayChunks, searchQuery]) const saveLabel = saveStatus === 'saving' diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts index 81f884453be..90ae6736e20 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts @@ -229,7 +229,7 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) { * its staleTime expires. Admission failures create nothing and need no refetch. */ if (err instanceof KnowledgeUploadError && err.code === 'PARTIAL_UPLOAD_FAILURE') { - await invalidateKnowledgeCaches(knowledgeBaseId) + void invalidateKnowledgeCaches(knowledgeBaseId) } const error: UploadError = diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx index 11cb67f02c7..207f8b9599e 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx @@ -1420,7 +1420,7 @@ export function Knowledge() { onOpenChange={(open) => { if (!open) setFolderPendingDelete(null) }} - srTitle='Delete folder' + srTitle='Delete Folder' title='Delete Folder' text={[ 'Are you sure you want to delete ', @@ -1438,7 +1438,7 @@ export function Knowledge() { { - /** - * `document` is a prefix of `detail`, so the wider key added only a refetch of every - * other document page, chunk page, tag definition, and connector row cached for this - * base. Nothing this mutation writes is rendered from those. - */ queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) @@ -440,10 +436,6 @@ export function useDeleteChunk() { queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) - /** - * `exact`, so the base's own `chunkCount`/`tokenCount` refresh without dragging every - * sibling document and chunk page cached beneath `detail` along with them. - */ queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), exact: true, @@ -482,10 +474,6 @@ export function useCreateChunk() { queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) - /** - * `exact`, so the base's own `chunkCount`/`tokenCount` refresh without dragging every - * sibling document and chunk page cached beneath `detail` along with them. - */ queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), exact: true, @@ -524,11 +512,6 @@ export function useUpdateDocument() { return useMutation({ mutationFn: updateDocument, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { - /** - * `document` is a prefix of `detail`, so the wider key added only a refetch of every - * other document page, chunk page, tag definition, and connector row cached for this - * base. Nothing this mutation writes is rendered from those. - */ queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) @@ -770,10 +753,6 @@ export function useBulkChunkOperation() { queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) - /** - * `exact`, so the base's own `chunkCount`/`tokenCount` refresh without dragging every - * sibling document and chunk page cached beneath `detail` along with them. - */ queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), exact: true, @@ -807,11 +786,6 @@ export function useUpdateDocumentTags() { return useMutation({ mutationFn: updateDocumentTags, onSettled: (_data, _error, { knowledgeBaseId, documentId }) => { - /** - * `document` is a prefix of `detail`, so the wider key added only a refetch of every - * other document page, chunk page, tag definition, and connector row cached for this - * base. Nothing this mutation writes is rendered from those. - */ queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) diff --git a/apps/sim/hooks/queries/utils/knowledge-keys.ts b/apps/sim/hooks/queries/utils/knowledge-keys.ts index 12edbd76717..fb98e19d14d 100644 --- a/apps/sim/hooks/queries/utils/knowledge-keys.ts +++ b/apps/sim/hooks/queries/utils/knowledge-keys.ts @@ -15,6 +15,14 @@ export type KnowledgeQueryScope = KnowledgeScope /** Shared with the server prefetch so a hydrated list and a client fetch never disagree. */ export const KNOWLEDGE_BASE_LIST_STALE_TIME = 60 * 1000 +/** + * `document(kb, doc)` is a PREFIX of `detail(kb)`, as are `documents`, `chunks`, + * `tagDefinitions`, and `tagUsage`. A mutation scoped to one document therefore invalidates + * the `document` key alone — the wider key would refetch every sibling document page, chunk + * page, tag definition, and connector row cached under the base. A mutation that also moves + * the base's own `chunkCount`/`tokenCount` invalidates `detail` with `exact: true`, for the + * same reason. + */ export const knowledgeKeys = { all: ['knowledge'] as const, lists: () => [...knowledgeKeys.all, 'list'] as const, diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts index b66fed82129..c92716aa480 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts @@ -16,6 +16,7 @@ const mocks = vi.hoisted(() => ({ deleteRecord: vi.fn(), listRecords: vi.fn(), listLegacyPersonalRecords: vi.fn(), + listVisibleRecords: vi.fn(), getRecord: vi.fn(), getRestorableRecord: vi.fn(), performUpdate: vi.fn(), @@ -82,6 +83,7 @@ vi.mock('@/lib/knowledge/service', () => ({ deleteKnowledgeBase: mocks.deleteRecord, getKnowledgeBaseById: mocks.getRecord, getLegacyPersonalKnowledgeBases: mocks.listLegacyPersonalRecords, + listWorkspaceAndLegacyKnowledgeBases: mocks.listVisibleRecords, getWorkspaceKnowledgeBases: mocks.listRecords, })) @@ -152,6 +154,7 @@ describe('knowledge base application use cases', () => { mocks.createRecord.mockResolvedValue(knowledgeBase) mocks.listRecords.mockResolvedValue({ data: [], nextCursorKeys: null }) mocks.listLegacyPersonalRecords.mockResolvedValue([knowledgeBase]) + mocks.listVisibleRecords.mockResolvedValue([knowledgeBase]) mocks.getRecord.mockResolvedValue(knowledgeBase) mocks.getRestorableRecord.mockResolvedValue(knowledgeBase) mocks.performUpdate.mockResolvedValue({ @@ -207,7 +210,8 @@ describe('knowledge base application use cases', () => { undefined, { forUpdate: undefined } ) - expect(mocks.listRecords).toHaveBeenCalledWith('workspace-1', 'archived') + expect(mocks.listVisibleRecords).toHaveBeenCalledWith('user-1', 'workspace-1', 'archived') + expect(mocks.listLegacyPersonalRecords).not.toHaveBeenCalled() }) /** @@ -217,8 +221,7 @@ describe('knowledge base application use cases', () => { */ it('lists a workspace for an authorized caller who holds no workspace permission row', async () => { mocks.resolvePermission.mockResolvedValue('admin') - mocks.listRecords.mockResolvedValueOnce({ data: [knowledgeBase], nextCursorKeys: null }) - mocks.listLegacyPersonalRecords.mockResolvedValueOnce([]) + mocks.listVisibleRecords.mockResolvedValueOnce([knowledgeBase]) await expect( listInternalKnowledgeBases.execute({ @@ -227,29 +230,7 @@ describe('knowledge base application use cases', () => { }) ).resolves.toEqual({ knowledgeBases: [knowledgeBase] }) - expect(mocks.listRecords).toHaveBeenCalledWith('workspace-1', 'active') - }) - - /** - * Legacy workspace-less bases belong to no workspace, so a workspace list is the only - * place the UI can reach them. They ride along beside the workspace's own rows. - */ - it('includes the caller’s legacy personal bases beside the workspace’s own rows', async () => { - const legacyBase = { - ...knowledgeBase, - id: 'legacy-1', - workspaceId: null, - createdAt: new Date('2025-01-01T00:00:00Z'), - } - mocks.listRecords.mockResolvedValueOnce({ data: [knowledgeBase], nextCursorKeys: null }) - mocks.listLegacyPersonalRecords.mockResolvedValueOnce([legacyBase]) - - await expect( - listInternalKnowledgeBases.execute({ - principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, - input: { workspaceId: 'workspace-1', scope: 'active' }, - }) - ).resolves.toEqual({ knowledgeBases: [legacyBase, knowledgeBase] }) + expect(mocks.listVisibleRecords).toHaveBeenCalledWith('org-admin-1', 'workspace-1', 'active') }) it('loads the active knowledge catalog and tag metadata only after workspace authorization', async () => { diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.ts b/apps/sim/lib/knowledge/application/knowledge-bases.ts index 8ef704f34c0..8884d926d0b 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.ts @@ -59,6 +59,7 @@ import { getLegacyPersonalKnowledgeBases, getWorkspaceKnowledgeBases, type KnowledgeBaseScope, + listWorkspaceAndLegacyKnowledgeBases, updateKnowledgeBase, } from '@/lib/knowledge/service' import type { ChunkingConfig, KnowledgeBaseWithCounts } from '@/lib/knowledge/types' @@ -445,22 +446,11 @@ export const listInternalKnowledgeBases = { } const context = await resolveKnowledgeWorkspaceContext({ workspaceId: input.workspaceId }) await authorizeWorkspaceOperation(principal, knowledgeOperations.list, context) - /** - * Two reads, because this list answers for two authorities. The workspace's own rows are - * read once the operation is authorized — deriving list access a second time from a - * `permissions` row would contradict the authorization that just passed, since workspace - * `admin` can come from an organization role with no such row behind it, and that caller - * could create a knowledge base and then never see it listed. Legacy workspace-less bases - * answer only to their creator and have no workspace to be listed under, so they ride - * along here as they always have — otherwise they are reachable from nowhere in the UI. - */ - const [workspaceBases, legacyPersonalBases] = await Promise.all([ - getWorkspaceKnowledgeBases(context.workspaceId, input.scope), - getLegacyPersonalKnowledgeBases(principal.userId, input.scope), - ]) return { - knowledgeBases: [...workspaceBases.data, ...legacyPersonalBases].sort( - (a, b) => a.createdAt.getTime() - b.createdAt.getTime() + knowledgeBases: await listWorkspaceAndLegacyKnowledgeBases( + principal.userId, + context.workspaceId, + input.scope ), } }, diff --git a/apps/sim/lib/knowledge/constants.ts b/apps/sim/lib/knowledge/constants.ts index 7b75bc6ae17..3e6310eba32 100644 --- a/apps/sim/lib/knowledge/constants.ts +++ b/apps/sim/lib/knowledge/constants.ts @@ -4,6 +4,13 @@ import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' export const KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH = 10_000 /** Hard bound for full-workspace knowledge-base list projections. */ export const MAX_KNOWLEDGE_BASES_PER_WORKSPACE = 10_000 + +/** + * Cap on one caller's legacy workspace-less knowledge bases. Separate from the per-workspace + * cap because it bounds a per-user set governed by no workspace rule — the two limits should + * be free to move independently. + */ +export const MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES = 10_000 /** Hard bound for path-indexed knowledge folder trees and recursive cascades. */ export const MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE = MAX_FOLDERS_PER_WORKSPACE diff --git a/apps/sim/lib/knowledge/service.test.ts b/apps/sim/lib/knowledge/service.test.ts index f7fd7a155fc..584633d9c1a 100644 --- a/apps/sim/lib/knowledge/service.test.ts +++ b/apps/sim/lib/knowledge/service.test.ts @@ -44,6 +44,7 @@ import { getLegacyPersonalKnowledgeBases, getWorkspaceKnowledgeBases, KnowledgeBasePermissionError, + listWorkspaceAndLegacyKnowledgeBases, updateKnowledgeBase, } from '@/lib/knowledge/service' @@ -123,6 +124,43 @@ describe('getLegacyPersonalKnowledgeBases', () => { }) }) +/** + * The workspace list and the legacy personal list are separate reads answering to separate + * authorities, but they render as ONE list — so the merge has to order them together and + * project connectors once over the result, not once per source. + */ +describe('listWorkspaceAndLegacyKnowledgeBases', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('orders both sources as one list and projects connectors once', async () => { + const workspaceRow = { + id: 'kb-workspace', + chunkingConfig: {}, + docCount: 0, + createdAt: new Date('2026-02-01T00:00:00Z'), + } + const legacyRow = { + id: 'kb-legacy', + chunkingConfig: {}, + docCount: 0, + createdAt: new Date('2025-01-01T00:00:00Z'), + } + dbChainMockFns.limit + .mockResolvedValueOnce([workspaceRow]) + .mockResolvedValueOnce([legacyRow]) + .mockResolvedValueOnce([]) + + const result = await listWorkspaceAndLegacyKnowledgeBases('user-a', 'ws-1') + + expect(result.map((kb) => kb.id)).toEqual(['kb-legacy', 'kb-workspace']) + /** Two row reads and ONE connector projection — three chains, never four. */ + expect(dbChainMockFns.limit).toHaveBeenCalledTimes(3) + }) +}) + /** * These tests guard the workspace mass-assignment fix: * a user with write/admin on the *source* workspace must not be able to move a diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index 87110003fa5..2af47ea2525 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -3,6 +3,7 @@ import { document, knowledgeBase, knowledgeConnector, workspaceFiles } from '@si import { createLogger } from '@sim/logger' import { getPostgresErrorCode } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import type { SQL } from 'drizzle-orm' import { and, count, eq, exists, inArray, isNotNull, isNull, ne, sql } from 'drizzle-orm' import type { V2KnowledgeBaseSortBy } from '@/lib/api/contracts/v2/knowledge' import type { CursorKey, KeysetKey, ListSortOrder } from '@/lib/api/list-query' @@ -30,6 +31,7 @@ import { findActiveFolder, resolveRestoredFolderId } from '@/lib/folders/queries import { MAX_KNOWLEDGE_BASES_PER_WORKSPACE, MAX_KNOWLEDGE_CONNECTOR_TYPE_ROWS_PER_LIST, + MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES, } from '@/lib/knowledge/constants' import type { ChunkingConfig, @@ -163,6 +165,63 @@ export interface GetKnowledgeBasesOptions { cursorKeys?: CursorKey[] } +/** `active` hides soft-deleted rows, `archived` shows only them, `all` filters neither. */ +function knowledgeBaseScopeCondition(scope: KnowledgeBaseScope) { + if (scope === 'all') return undefined + return scope === 'archived' + ? sql`${knowledgeBase.deletedAt} IS NOT NULL` + : isNull(knowledgeBase.deletedAt) +} + +/** + * The one projection every knowledge-base list renders: the base's own columns plus its live + * document count. Both list queries read through here so a column added to one list can never + * be missing from the other — they are concatenated into a single rendered list. + */ +async function readKnowledgeBaseRows( + where: SQL | undefined, + orderBy: SQL[], + limit: number +): Promise>> { + const rows = await db + .select({ + id: knowledgeBase.id, + userId: knowledgeBase.userId, + name: knowledgeBase.name, + description: knowledgeBase.description, + tokenCount: sql`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number), + embeddingModel: knowledgeBase.embeddingModel, + embeddingDimension: knowledgeBase.embeddingDimension, + chunkingConfig: knowledgeBase.chunkingConfig, + createdAt: knowledgeBase.createdAt, + updatedAt: knowledgeBase.updatedAt, + deletedAt: knowledgeBase.deletedAt, + workspaceId: knowledgeBase.workspaceId, + folderId: knowledgeBase.folderId, + docCount: count(document.id), + }) + .from(knowledgeBase) + .leftJoin( + document, + and( + eq(document.knowledgeBaseId, knowledgeBase.id), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) + ) + .where(where) + .groupBy(knowledgeBase.id) + .orderBy(...orderBy) + .limit(limit) + + return rows.map((kb) => ({ + ...kb, + chunkingConfig: kb.chunkingConfig as ChunkingConfig, + docCount: Number(kb.docCount), + })) +} + async function attachConnectorTypes( knowledgeBases: Array> ): Promise { @@ -208,11 +267,14 @@ async function attachConnectorTypes( * authorization. Unlike the legacy user-oriented query, this never widens the * scope to workspace-less rows and never depends on a human permission join. */ -export async function getWorkspaceKnowledgeBases( +async function readWorkspaceKnowledgeBaseRows( workspaceId: string, - scope: KnowledgeBaseScope = 'active', + scope: KnowledgeBaseScope, options?: GetKnowledgeBasesOptions -): Promise<{ data: KnowledgeBaseWithCounts[]; nextCursorKeys: CursorKey[] | null }> { +): Promise<{ + data: Array> + nextCursorKeys: CursorKey[] | null +}> { const { folderId, search, @@ -222,63 +284,28 @@ export async function getWorkspaceKnowledgeBases( cursorKeys, } = options ?? {} const keys = KNOWLEDGE_BASE_SORTS[sortBy] - const resumeAfter = resumeKeyset(keys, cursorKeys, sortOrder) /** * An unpaged read still reads one row past the cap so an oversized workspace * is a hard failure rather than a silently truncated list. */ const readLimit = (limit ?? MAX_KNOWLEDGE_BASES_PER_WORKSPACE) + 1 - const scopeCondition = - scope === 'all' - ? undefined - : scope === 'archived' - ? sql`${knowledgeBase.deletedAt} IS NOT NULL` - : isNull(knowledgeBase.deletedAt) - const rows = await db - .select({ - id: knowledgeBase.id, - userId: knowledgeBase.userId, - name: knowledgeBase.name, - description: knowledgeBase.description, - tokenCount: sql`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number), - embeddingModel: knowledgeBase.embeddingModel, - embeddingDimension: knowledgeBase.embeddingDimension, - chunkingConfig: knowledgeBase.chunkingConfig, - createdAt: knowledgeBase.createdAt, - updatedAt: knowledgeBase.updatedAt, - deletedAt: knowledgeBase.deletedAt, - workspaceId: knowledgeBase.workspaceId, - folderId: knowledgeBase.folderId, - docCount: count(document.id), - }) - .from(knowledgeBase) - .leftJoin( - document, - and( - eq(document.knowledgeBaseId, knowledgeBase.id), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .where( - and( - eq(knowledgeBase.workspaceId, workspaceId), - scopeCondition, - folderId === undefined - ? undefined - : folderId === null - ? isNull(knowledgeBase.folderId) - : eq(knowledgeBase.folderId, folderId), - searchFilter(knowledgeBase.name, search), - resumeAfter - ) - ) - .groupBy(knowledgeBase.id) - .orderBy(...listOrderBy(keysetColumns(keys), sortOrder)) - .limit(readLimit) + const rows = await readKnowledgeBaseRows( + and( + eq(knowledgeBase.workspaceId, workspaceId), + knowledgeBaseScopeCondition(scope), + folderId === undefined + ? undefined + : folderId === null + ? isNull(knowledgeBase.folderId) + : eq(knowledgeBase.folderId, folderId), + searchFilter(knowledgeBase.name, search), + resumeKeyset(keys, cursorKeys, sortOrder) + ), + listOrderBy(keysetColumns(keys), sortOrder), + readLimit + ) if (limit === undefined && rows.length > MAX_KNOWLEDGE_BASES_PER_WORKSPACE) { throw new Error( @@ -286,83 +313,91 @@ export async function getWorkspaceKnowledgeBases( ) } - const page = keysetPage(keys, rows, limit) + return keysetPage(keys, rows, limit) +} +export async function getWorkspaceKnowledgeBases( + workspaceId: string, + scope: KnowledgeBaseScope = 'active', + options?: GetKnowledgeBasesOptions +): Promise<{ data: KnowledgeBaseWithCounts[]; nextCursorKeys: CursorKey[] | null }> { + const page = await readWorkspaceKnowledgeBaseRows(workspaceId, scope, options) return { - data: await attachConnectorTypes( - page.data.map((kb) => ({ - ...kb, - chunkingConfig: kb.chunkingConfig as ChunkingConfig, - docCount: Number(kb.docCount), - })) - ), + data: await attachConnectorTypes(page.data), nextCursorKeys: page.nextCursorKeys, } } /** - * Lists the caller's legacy personal knowledge bases — the ones that predate - * workspaces and carry no `workspaceId`, where the creator is the only possible - * authority. Workspace-owned rows are read by - * {@link getWorkspaceKnowledgeBases} after an application use case has + * Lists the caller's legacy personal knowledge bases — the ones that predate workspaces and + * carry no `workspaceId`, where the creator is the only possible authority. Workspace-owned + * rows are read by {@link getWorkspaceKnowledgeBases} after an application use case has * authorized the workspace; nothing here re-derives that access. + * + * @deprecated Nothing creates workspace-less knowledge bases any more, so this population only + * shrinks. Backfill the remaining rows onto a workspace and this function, its branch in + * {@link listWorkspaceAndLegacyKnowledgeBases}, and the concept itself can go. */ -export async function getLegacyPersonalKnowledgeBases( +async function readLegacyPersonalKnowledgeBaseRows( userId: string, - scope: KnowledgeBaseScope = 'active' -): Promise { - const scopeCondition = - scope === 'all' - ? undefined - : scope === 'archived' - ? sql`${knowledgeBase.deletedAt} IS NOT NULL` - : isNull(knowledgeBase.deletedAt) - - const rows = await db - .select({ - id: knowledgeBase.id, - userId: knowledgeBase.userId, - name: knowledgeBase.name, - description: knowledgeBase.description, - tokenCount: sql`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number), - embeddingModel: knowledgeBase.embeddingModel, - embeddingDimension: knowledgeBase.embeddingDimension, - chunkingConfig: knowledgeBase.chunkingConfig, - createdAt: knowledgeBase.createdAt, - updatedAt: knowledgeBase.updatedAt, - deletedAt: knowledgeBase.deletedAt, - workspaceId: knowledgeBase.workspaceId, - folderId: knowledgeBase.folderId, - docCount: count(document.id), - }) - .from(knowledgeBase) - .leftJoin( - document, - and( - eq(document.knowledgeBaseId, knowledgeBase.id), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .where(and(scopeCondition, eq(knowledgeBase.userId, userId), isNull(knowledgeBase.workspaceId))) - .groupBy(knowledgeBase.id) - .orderBy(...listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS.createdAt), 'asc')) - .limit(MAX_KNOWLEDGE_BASES_PER_WORKSPACE + 1) + scope: KnowledgeBaseScope +): Promise>> { + const rows = await readKnowledgeBaseRows( + and( + knowledgeBaseScopeCondition(scope), + eq(knowledgeBase.userId, userId), + isNull(knowledgeBase.workspaceId) + ), + listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS.createdAt), 'asc'), + MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES + 1 + ) /** One row past the cap, so an oversized set fails loudly instead of truncating in silence. */ - if (rows.length > MAX_KNOWLEDGE_BASES_PER_WORKSPACE) { + if (rows.length > MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES) { throw new Error( - `Legacy personal knowledge base list exceeds the ${MAX_KNOWLEDGE_BASES_PER_WORKSPACE} row limit` + `Legacy personal knowledge base list exceeds the ${MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES} row limit` ) } + return rows +} + +export async function getLegacyPersonalKnowledgeBases( + userId: string, + scope: KnowledgeBaseScope = 'active' +): Promise { + return attachConnectorTypes(await readLegacyPersonalKnowledgeBaseRows(userId, scope)) +} + +/** + * Every knowledge base a caller can see under one workspace, as one ordered list. + * + * Two reads, because the list answers to two authorities. The workspace's own rows are read + * once the caller has been authorized FOR that workspace — re-deriving that access from a + * `permissions` row would contradict the authorization that just passed, since workspace + * `admin` can come from an organization role with no such row behind it. Legacy workspace-less + * bases answer only to their creator and belong under no workspace at all, so they ride along + * here; otherwise they are reachable from nowhere. + * + * Callers authorize first. Nothing here decides access. + */ +export async function listWorkspaceAndLegacyKnowledgeBases( + userId: string, + workspaceId: string, + scope: KnowledgeBaseScope = 'active' +): Promise { + const [workspaceRows, legacyPersonalRows] = await Promise.all([ + readWorkspaceKnowledgeBaseRows(workspaceId, scope).then((page) => page.data), + readLegacyPersonalKnowledgeBaseRows(userId, scope), + ]) + + /** One connector projection over the merged set, rather than one per source. */ return attachConnectorTypes( - rows.map((kb) => ({ - ...kb, - chunkingConfig: kb.chunkingConfig as ChunkingConfig, - docCount: Number(kb.docCount), - })) + legacyPersonalRows.length === 0 + ? workspaceRows + : [...workspaceRows, ...legacyPersonalRows].sort( + (a, b) => a.createdAt.getTime() - b.createdAt.getTime() + ) ) } From 29ed28810044d6e442d8cd535a38ccdad670a5c3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 16 Aug 2026 22:13:56 -0700 Subject: [PATCH 5/5] fix(knowledge): refresh the document list pages after a document write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review caught a regression in the invalidation narrowing: `documents` (the list pages) and `document` (one row) are SIBLINGS under `detail`, not parent and child, so scoping a write to the row key left every list rendering the filename, status, tags, `tokenCount`, and `chunkCount` it had just changed. The key factory now exposes a `documentLists` prefix and all six document-scoped mutations invalidate it alongside the row — the chunk mutations included, since every chunk write moves the parent document's `tokenCount`. `detail` stays `exact: true` where only the base's own totals move. Also repoints the shared list-convention test at `getWorkspaceKnowledgeBases`; it exercised the caller-scoped query this branch removed. --- apps/sim/hooks/queries/kb/knowledge.test.ts | 28 ++++++++++++++++++- apps/sim/hooks/queries/kb/knowledge.ts | 24 ++++++++++++++++ .../sim/hooks/queries/utils/knowledge-keys.ts | 20 ++++++++----- apps/sim/lib/api/list-convention.test.ts | 4 +-- 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/apps/sim/hooks/queries/kb/knowledge.test.ts b/apps/sim/hooks/queries/kb/knowledge.test.ts index 4c01ae0fcf5..1fcaaaca386 100644 --- a/apps/sim/hooks/queries/kb/knowledge.test.ts +++ b/apps/sim/hooks/queries/kb/knowledge.test.ts @@ -26,7 +26,12 @@ vi.mock('@/lib/api/client/request', () => ({ requestJson: mocks.requestJson, })) -import { useBulkDocumentOperation, useDeleteDocument } from '@/hooks/queries/kb/knowledge' +import { + useBulkDocumentOperation, + useDeleteDocument, + useUpdateDocument, + useUpdateDocumentTags, +} from '@/hooks/queries/kb/knowledge' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' interface CapturedMutation { @@ -65,6 +70,27 @@ describe('knowledge document mutations', () => { expect(mocks.invalidateQueries).toHaveBeenCalledWith({ queryKey: knowledgeKeys.lists() }) }) + /** + * `documents` (the list pages) and `document` (one row) are siblings under `detail`, so + * invalidating the row's key alone leaves the list rendering the filename, status, tags, and + * counts the write just changed. + */ + it.each([ + ['a document update', () => useUpdateDocument()], + ['a document tag update', () => useUpdateDocumentTags()], + ])('refreshes the document list pages after %s', (_label, build) => { + const mutation = captureMutation(build) + + mutation.onSettled(undefined, undefined, { knowledgeBaseId: 'kb-1', documentId: 'doc-1' }) + + expect(mocks.invalidateQueries).toHaveBeenCalledWith({ + queryKey: knowledgeKeys.documentLists('kb-1'), + }) + expect(mocks.invalidateQueries).toHaveBeenCalledWith({ + queryKey: knowledgeKeys.document('kb-1', 'doc-1'), + }) + }) + it('leaves the knowledge-base lists alone on a bulk enable', () => { const mutation = captureMutation(() => useBulkDocumentOperation()) diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index c67cf47ac05..4c064e479c9 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -407,6 +407,10 @@ export function useUpdateChunk() { queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ + queryClient.invalidateQueries({ + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), + }) }, }) } @@ -436,6 +440,10 @@ export function useDeleteChunk() { queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ + queryClient.invalidateQueries({ + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), + }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), exact: true, @@ -474,6 +482,10 @@ export function useCreateChunk() { queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ + queryClient.invalidateQueries({ + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), + }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), exact: true, @@ -515,6 +527,10 @@ export function useUpdateDocument() { queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ + queryClient.invalidateQueries({ + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), + }) }, }) } @@ -753,6 +769,10 @@ export function useBulkChunkOperation() { queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ + queryClient.invalidateQueries({ + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), + }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), exact: true, @@ -789,6 +809,10 @@ export function useUpdateDocumentTags() { queryClient.invalidateQueries({ queryKey: knowledgeKeys.document(knowledgeBaseId, documentId), }) + /** The document list renders this row's filename, status, tags, and counts. */ + queryClient.invalidateQueries({ + queryKey: knowledgeKeys.documentLists(knowledgeBaseId), + }) }, }) } diff --git a/apps/sim/hooks/queries/utils/knowledge-keys.ts b/apps/sim/hooks/queries/utils/knowledge-keys.ts index fb98e19d14d..743ffc81a81 100644 --- a/apps/sim/hooks/queries/utils/knowledge-keys.ts +++ b/apps/sim/hooks/queries/utils/knowledge-keys.ts @@ -16,12 +16,11 @@ export type KnowledgeQueryScope = KnowledgeScope export const KNOWLEDGE_BASE_LIST_STALE_TIME = 60 * 1000 /** - * `document(kb, doc)` is a PREFIX of `detail(kb)`, as are `documents`, `chunks`, - * `tagDefinitions`, and `tagUsage`. A mutation scoped to one document therefore invalidates - * the `document` key alone — the wider key would refetch every sibling document page, chunk - * page, tag definition, and connector row cached under the base. A mutation that also moves - * the base's own `chunkCount`/`tokenCount` invalidates `detail` with `exact: true`, for the - * same reason. + * `document`, `documents`, `chunks`, `tagDefinitions`, and `tagUsage` all sit UNDER + * `detail(kb)`, so invalidating `detail` non-exactly refetches all of them at once. A mutation + * scoped to one document instead invalidates the two keys that actually render it — its own + * `document` key and the `documentLists` prefix, which are siblings — and, when the base's own + * totals move, `detail` with `exact: true`. */ export const knowledgeKeys = { all: ['knowledge'] as const, @@ -35,8 +34,15 @@ export const knowledgeKeys = { [...knowledgeKeys.detail(knowledgeBaseId), 'tagDefinitions'] as const, tagUsage: (knowledgeBaseId: string) => [...knowledgeKeys.detail(knowledgeBaseId), 'tagUsage'] as const, + /** + * Prefix over every cached page of a base's document list. `documents` and `document` are + * SIBLINGS, not parent and child — a write to one document does not reach the lists that + * render its filename, status, tags, and counts unless this key is invalidated too. + */ + documentLists: (knowledgeBaseId: string) => + [...knowledgeKeys.detail(knowledgeBaseId), 'documents'] as const, documents: (knowledgeBaseId: string, paramsKey: string) => - [...knowledgeKeys.detail(knowledgeBaseId), 'documents', paramsKey] as const, + [...knowledgeKeys.documentLists(knowledgeBaseId), paramsKey] as const, document: (knowledgeBaseId: string, documentId: string) => [...knowledgeKeys.detail(knowledgeBaseId), 'document', documentId] as const, documentTagDefinitions: (knowledgeBaseId: string, documentId: string) => diff --git a/apps/sim/lib/api/list-convention.test.ts b/apps/sim/lib/api/list-convention.test.ts index 80a8257f2a8..77453b972c1 100644 --- a/apps/sim/lib/api/list-convention.test.ts +++ b/apps/sim/lib/api/list-convention.test.ts @@ -56,7 +56,7 @@ vi.mock('@/lib/workflows/skills/builtin-skills', () => ({ import { listVisibleWorkspaceCredentials } from '@/lib/credentials/queries' import { listFoldersForWorkspace } from '@/lib/folders/queries' import { getDocuments } from '@/lib/knowledge/documents/service' -import { getKnowledgeBases } from '@/lib/knowledge/service' +import { getWorkspaceKnowledgeBases } from '@/lib/knowledge/service' import { listWorkspaceMcpServers } from '@/lib/mcp/queries' import { listTables } from '@/lib/table/service' import { listWorkspaceCustomTools } from '@/lib/workflows/custom-tools/operations' @@ -123,7 +123,7 @@ const CASES: ListCase[] = [ column: schemaMock.knowledgeBase.name, table: schemaMock.knowledgeBase, run: ({ search, sortBy, sortOrder }) => - getKnowledgeBases('user-1', WS, 'active', { search, sortBy: sortBy as never, sortOrder }), + getWorkspaceKnowledgeBases(WS, 'active', { search, sortBy: sortBy as never, sortOrder }), sort: { sortBy: 'name', sortOrder: 'asc',