Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions apps/sim/lib/knowledge/application/knowledge-vfs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import {
import { knowledgeOperations } from '@/lib/knowledge/application/operations'
import {
deleteKnowledgeBase,
getWorkspaceKnowledgeBases,
findActiveKnowledgeBasesByExactName,
updateKnowledgeBase,
} from '@/lib/knowledge/service'
import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types'
Expand All @@ -28,11 +28,8 @@ export type DeleteKnowledgeBaseByVfsPathInput = KnowledgeVfsReferenceInput
async function resolveKnowledgeBaseByVfsName(
context: KnowledgeWorkspaceContext,
sourceName: string
): Promise<KnowledgeBaseWithCounts> {
const { data: rows } = await getWorkspaceKnowledgeBases(context.workspaceId, 'active', {
search: sourceName,
})
const matches = rows.filter((row) => row.name === sourceName)
): Promise<Omit<KnowledgeBaseWithCounts, 'connectorTypes'>> {
const matches = await findActiveKnowledgeBasesByExactName(context.workspaceId, sourceName)
if (matches.length > 1) {
throw new OrchestrationError(
'conflict',
Expand Down
10 changes: 0 additions & 10 deletions apps/sim/lib/knowledge/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,7 @@ import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants'

/** Max character length for a knowledge base description, enforced at every layer (UI, internal API, v1 API). */
export const KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH = 10_000
/** Hard bound for full-workspace knowledge-base list projections. */
export const MAX_KNOWLEDGE_BASES_PER_WORKSPACE = 10_000

/**
* Cap on one caller's legacy workspace-less knowledge bases. Separate from the per-workspace
* cap because it bounds a per-user set governed by no workspace rule — the two limits should
* be free to move independently.
*/
export const MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES = 10_000
/** Hard bound for path-indexed knowledge folder trees and recursive cascades. */
export const MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE = MAX_FOLDERS_PER_WORKSPACE

Expand All @@ -20,8 +12,6 @@ export const MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE = MAX_FOLDERS_PER_WORKSPACE
* bound their id arrays without pulling a server-only module into client code.
*/
export const MAX_KNOWLEDGE_BATCH_ITEMS = 100
/** Hard bound for connector-type rows projected onto one knowledge-base list. */
export const MAX_KNOWLEDGE_CONNECTOR_TYPE_ROWS_PER_LIST = 100_000
/** Maximum documents accepted by one internal bulk-create command. */
export const MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE = 100
/** Maximum connector documents mutated atomically by one command. */
Expand Down
112 changes: 88 additions & 24 deletions apps/sim/lib/knowledge/service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,32 +39,58 @@ vi.mock('@/lib/billing/core/usage', () => ({
ensureUserStatsExists: mockEnsureUserStatsExists,
}))

import { MAX_KNOWLEDGE_BASES_PER_WORKSPACE } from '@/lib/knowledge/constants'
import {
findActiveKnowledgeBasesByExactName,
getLegacyPersonalKnowledgeBases,
getWorkspaceKnowledgeBases,
KnowledgeBasePermissionError,
listWorkspaceAndLegacyKnowledgeBases,
updateKnowledgeBase,
} from '@/lib/knowledge/service'

describe('getWorkspaceKnowledgeBases — bounded reads', () => {
/**
* A row cap on this read could only ever fire for a caller that did NOT ask for a page — the
* one kind of caller with no cursor to act on it — so an oversized workspace has to be served,
* not refused.
*/
describe('getWorkspaceKnowledgeBases — paging', () => {
beforeEach(() => {
vi.clearAllMocks()
resetDbChainMock()
})

it('fails before projecting connector data for an oversized workspace list', async () => {
dbChainMockFns.limit.mockResolvedValueOnce(
Array.from({ length: MAX_KNOWLEDGE_BASES_PER_WORKSPACE + 1 }, (_, index) => ({
it('reads unbounded when the caller asked for no page', async () => {
dbChainMockFns.orderBy.mockResolvedValueOnce(
Array.from({ length: 10_001 }, (_, index) => ({
id: `kb-${index}`,
chunkingConfig: {},
docCount: 0,
}))
)

await expect(getWorkspaceKnowledgeBases('ws-1')).rejects.toThrow(
`Knowledge base list exceeds the ${MAX_KNOWLEDGE_BASES_PER_WORKSPACE} row limit`
)
expect(dbChainMockFns.limit).toHaveBeenCalledWith(MAX_KNOWLEDGE_BASES_PER_WORKSPACE + 1)
const result = await getWorkspaceKnowledgeBases('ws-1')

expect(result.data).toHaveLength(10_001)
expect(dbChainMockFns.limit).not.toHaveBeenCalled()
})

it('reads one row past the page so it can report another page', async () => {
dbChainMockFns.limit
.mockResolvedValueOnce(
Array.from({ length: 3 }, (_, index) => ({
id: `kb-${index}`,
chunkingConfig: {},
docCount: 0,
createdAt: new Date('2026-01-01T00:00:00Z'),
}))
)
.mockResolvedValueOnce([])

const result = await getWorkspaceKnowledgeBases('ws-1', 'active', { limit: 2 })

expect(dbChainMockFns.limit).toHaveBeenCalledWith(3)
expect(result.data).toHaveLength(2)
expect(result.nextCursorKeys).not.toBeNull()
})
})

Expand Down Expand Up @@ -110,17 +136,31 @@ describe('getLegacyPersonalKnowledgeBases', () => {
expect(joinedTables).toContain(schemaMock.document)
expect(joinedTables).not.toContain(schemaMock.permissions)
})
})

it('fails before projecting connector data for an oversized set', async () => {
dbChainMockFns.limit.mockResolvedValueOnce(
Array.from({ length: MAX_KNOWLEDGE_BASES_PER_WORKSPACE + 1 }, (_, index) => ({
id: `kb-${index}`,
}))
)
/**
* A VFS path names one knowledge base exactly. Resolving it by reading every base whose name
* merely CONTAINS the term, then filtering in JS, makes a single-row lookup scale with the
* workspace — the sibling `findActiveTablesByExactName` is the shape to match.
*/
describe('findActiveKnowledgeBasesByExactName', () => {
beforeEach(() => {
vi.clearAllMocks()
resetDbChainMock()
})

await expect(getLegacyPersonalKnowledgeBases('user-a')).rejects.toThrow(
`Legacy personal knowledge base list exceeds the ${MAX_KNOWLEDGE_BASES_PER_WORKSPACE} row limit`
)
it('matches the name exactly and reads at most two rows', async () => {
await findActiveKnowledgeBasesByExactName('ws-1', 'Docs')

const [condition] = dbChainMockFns.where.mock.calls[0] ?? []
expect(
hasMockCondition(
condition,
(node) =>
node.type === 'eq' && node.left === schemaMock.knowledgeBase.name && node.right === 'Docs'
)
).toBe(true)
expect(dbChainMockFns.limit).toHaveBeenCalledWith(2)
})
})

Expand All @@ -135,6 +175,29 @@ describe('listWorkspaceAndLegacyKnowledgeBases', () => {
resetDbChainMock()
})

/**
* Soft-delete cleanup only reclaims archived rows past a retention window — and none at all
* for a workspace with no retention configured — so a workspace that archives faster than
* that window crosses any fixed count. This surface has no cursor to page with, so a cap
* here could only mean a 500 on the knowledge page and Recently Deleted, which is exactly
* what it meant on staging.
*/
it('serves a workspace whose archived set is larger than the old row cap', async () => {
const rows = Array.from({ length: 10_001 }, (_, index) => ({
id: `kb-${index}`,
chunkingConfig: {},
docCount: 0,
createdAt: new Date('2026-01-01T00:00:00Z'),
}))
dbChainMockFns.orderBy.mockResolvedValueOnce(rows).mockResolvedValueOnce([])

const result = await listWorkspaceAndLegacyKnowledgeBases('user-a', 'ws-1', 'archived')

expect(result).toHaveLength(10_001)
/** Neither the workspace read nor the legacy read may bound itself. */
expect(dbChainMockFns.limit).not.toHaveBeenCalled()
})

it('orders both sources as one list and projects connectors once', async () => {
const workspaceRow = {
id: 'kb-workspace',
Expand All @@ -148,16 +211,17 @@ describe('listWorkspaceAndLegacyKnowledgeBases', () => {
docCount: 0,
createdAt: new Date('2025-01-01T00:00:00Z'),
}
dbChainMockFns.limit
.mockResolvedValueOnce([workspaceRow])
.mockResolvedValueOnce([legacyRow])
.mockResolvedValueOnce([])
/** Both row reads are unbounded now, so each resolves at `orderBy` rather than `limit`. */
dbChainMockFns.orderBy.mockResolvedValueOnce([workspaceRow]).mockResolvedValueOnce([legacyRow])

const result = await listWorkspaceAndLegacyKnowledgeBases('user-a', 'ws-1')

expect(result.map((kb) => kb.id)).toEqual(['kb-legacy', 'kb-workspace'])
/** Two row reads and ONE connector projection — three chains, never four. */
expect(dbChainMockFns.limit).toHaveBeenCalledTimes(3)
/** ONE connector projection over the merged set, not one per source. */
const connectorReads = dbChainMockFns.from.mock.calls.filter(
([table]) => table === schemaMock.knowledgeConnector
)
expect(connectorReads).toHaveLength(1)
})
})

Expand Down
63 changes: 26 additions & 37 deletions apps/sim/lib/knowledge/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,6 @@ import {
import { OrchestrationError } from '@/lib/core/orchestration/types'
import { generateRestoreName } from '@/lib/core/utils/restore-name'
import { findActiveFolder, resolveRestoredFolderId } from '@/lib/folders/queries'
import {
MAX_KNOWLEDGE_BASES_PER_WORKSPACE,
MAX_KNOWLEDGE_CONNECTOR_TYPE_ROWS_PER_LIST,
MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES,
} from '@/lib/knowledge/constants'
import type {
ChunkingConfig,
CreateKnowledgeBaseData,
Expand Down Expand Up @@ -155,11 +150,7 @@ export interface GetKnowledgeBasesOptions {
search?: string
sortBy?: V2KnowledgeBaseSortBy
sortOrder?: ListSortOrder
/**
* Page size. Omitted reads the whole workspace set as one page, capped by
* {@link MAX_KNOWLEDGE_BASES_PER_WORKSPACE} — what the internal callers that
* need every row still do.
*/
/** Page size. Omitted reads the whole set as one page. */
limit?: number
/** Keyset to resume after, from the previous page's `nextCursorKeys`. */
cursorKeys?: CursorKey[]
Expand All @@ -181,9 +172,9 @@ function knowledgeBaseScopeCondition(scope: KnowledgeBaseScope) {
async function readKnowledgeBaseRows(
where: SQL | undefined,
orderBy: SQL[],
limit: number
limit?: number
): Promise<Array<Omit<KnowledgeBaseWithCounts, 'connectorTypes'>>> {
const rows = await db
const query = db
.select({
id: knowledgeBase.id,
userId: knowledgeBase.userId,
Expand Down Expand Up @@ -213,7 +204,8 @@ async function readKnowledgeBaseRows(
.where(where)
.groupBy(knowledgeBase.id)
.orderBy(...orderBy)
.limit(limit)

const rows = limit === undefined ? await query : await query.limit(limit)

return rows.map((kb) => ({
...kb,
Expand Down Expand Up @@ -241,13 +233,7 @@ async function attachConnectorTypes(
isNull(knowledgeConnector.deletedAt)
)
)
.limit(MAX_KNOWLEDGE_CONNECTOR_TYPE_ROWS_PER_LIST + 1)
: []
if (connectorRows.length > MAX_KNOWLEDGE_CONNECTOR_TYPE_ROWS_PER_LIST) {
throw new Error(
`Knowledge connector projection exceeds the ${MAX_KNOWLEDGE_CONNECTOR_TYPE_ROWS_PER_LIST} row limit`
)
}

const connectorTypesByKb = new Map<string, string[]>()
for (const row of connectorRows) {
Expand Down Expand Up @@ -286,10 +272,11 @@ async function readWorkspaceKnowledgeBaseRows(
const keys = KNOWLEDGE_BASE_SORTS[sortBy]

/**
* An unpaged read still reads one row past the cap so an oversized workspace
* is a hard failure rather than a silently truncated list.
* An unpaged read is unbounded, matching the sibling internal lists (`listTables`, workspace
* files). A row cap could only ever fire for a caller that did not ask for a page — the one
* kind with no cursor to respond with — so it can only turn a slow list into a 500.
*/
const readLimit = (limit ?? MAX_KNOWLEDGE_BASES_PER_WORKSPACE) + 1
const readLimit = limit === undefined ? undefined : limit + 1

const rows = await readKnowledgeBaseRows(
and(
Expand All @@ -307,12 +294,6 @@ async function readWorkspaceKnowledgeBaseRows(
readLimit
)

if (limit === undefined && rows.length > MAX_KNOWLEDGE_BASES_PER_WORKSPACE) {
throw new Error(
`Knowledge base list exceeds the ${MAX_KNOWLEDGE_BASES_PER_WORKSPACE} row limit`
)
}

return keysetPage(keys, rows, limit)
}

Expand Down Expand Up @@ -348,17 +329,9 @@ async function readLegacyPersonalKnowledgeBaseRows(
eq(knowledgeBase.userId, userId),
isNull(knowledgeBase.workspaceId)
),
listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS.createdAt), 'asc'),
MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES + 1
listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS.createdAt), 'asc')
)

/** One row past the cap, so an oversized set fails loudly instead of truncating in silence. */
if (rows.length > MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES) {
throw new Error(
`Legacy personal knowledge base list exceeds the ${MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES} row limit`
)
}

return rows
}

Expand Down Expand Up @@ -401,6 +374,22 @@ export async function listWorkspaceAndLegacyKnowledgeBases(
)
}

/** Loads at most two active exact-name matches so a caller can fail on corrupt ambiguity. */
export async function findActiveKnowledgeBasesByExactName(
workspaceId: string,
name: string
): Promise<Array<Omit<KnowledgeBaseWithCounts, 'connectorTypes'>>> {
return readKnowledgeBaseRows(
and(
eq(knowledgeBase.workspaceId, workspaceId),
eq(knowledgeBase.name, name),
isNull(knowledgeBase.deletedAt)
),
listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS.createdAt), 'asc'),
2
)
}

/**
* Create a new knowledge base
*/
Expand Down
Loading