Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions apps/docs/content/docs/en/integrations/table.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ Tables are created from the **Tables** section in the sidebar. Each table requir

## Usage Instructions

Create and manage custom data tables. Store, query, and manipulate structured data within workflows.
Create and manage custom data tables. Store, query, and manipulate structured data within workflows. Query Rows returns every matching row when Limit is omitted and fails if the result exceeds 5MB.



Expand Down Expand Up @@ -213,7 +213,7 @@ Query rows from a table with filtering, sorting, and pagination
| `tableId` | string | Yes | Table ID |
| `filter` | object | No | Filter conditions \(MongoDB-style operators: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin, $contains, $ncontains, $startsWith, $endsWith, $empty\) |
| `sort` | object | No | Sort order as \{field: "asc"\|"desc"\} |
| `limit` | number | No | Maximum rows to return \(default: $\{TABLE_LIMITS.DEFAULT_QUERY_LIMIT\}, max: $\{TABLE_LIMITS.MAX_QUERY_LIMIT\}\) |
| `limit` | number | No | Maximum rows to return. Omit to return every matching row; the query fails if the result exceeds the 5MB response budget. |
| `offset` | number | No | Number of rows to skip \(default: 0\) |

#### Output
Expand Down
27 changes: 27 additions & 0 deletions apps/sim/app/api/table/[tableId]/rows/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,33 @@ describe('GET /api/table/[tableId]/rows', () => {
expect(body.data.rows[0].data).toEqual({ Name: 'Ada', Age: 36 })
})

it('keeps counts but skips execution metadata for an omitted or expanded limit', async () => {
authAs('internal_jwt')

const omitted = await callGet({ workspaceId: 'workspace-1' })
expect(omitted.status).toBe(200)
expect(mockQueryRows.mock.calls[0][1]).toEqual(
expect.objectContaining({ limit: undefined, includeTotal: true, withExecutions: false })
)

const expanded = await callGet({ workspaceId: 'workspace-1', limit: '1000000' })
expect(expanded.status).toBe(200)
expect(mockQueryRows.mock.calls[1][1]).toEqual(
expect.objectContaining({ limit: 1000000, includeTotal: true, withExecutions: false })
)
})

it('retains metadata loading within the former query limit', async () => {
authAs('internal_jwt')

const res = await callGet({ workspaceId: 'workspace-1', limit: '1000' })

expect(res.status).toBe(200)
expect(mockQueryRows.mock.calls[0][1]).toEqual(
expect.objectContaining({ limit: 1000, includeTotal: true, withExecutions: true })
)
})

it('passes id-keyed filter and rows through untouched for session callers', async () => {
authAs('session')

Expand Down
9 changes: 9 additions & 0 deletions apps/sim/app/api/table/[tableId]/rows/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
validateRowData,
validateRowSize,
} from '@/lib/table'
import { TABLE_LIMITS } from '@/lib/table/constants'
import { TableQueryValidationError } from '@/lib/table/errors'
import { signalTableRowsChanged, signalTableRowsChangedByActor } from '@/lib/table/events'
import { isTablePredicate, predicateToFilter } from '@/lib/table/query-builder/converters'
Expand Down Expand Up @@ -358,6 +359,13 @@ export const GET = withRouteHandler(
}

const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema)
/**
* The newly expanded path can return up to the byte budget, so skip the
* per-row execution-sidecar load. Keep the count behavior unchanged so
* Query Rows continues to return totalCount for workflow callers.
*/
const isExpandedQuery =
validated.limit === undefined || validated.limit > TABLE_LIMITS.MAX_QUERY_LIMIT
const result = await queryRows(
table,
{
Expand All @@ -381,6 +389,7 @@ export const GET = withRouteHandler(
offset: validated.offset,
after: validated.after,
includeTotal: validated.includeTotal,
withExecutions: !isExpandedQuery,
},
requestId
)
Expand Down
30 changes: 30 additions & 0 deletions apps/sim/blocks/blocks/table.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/**
* @vitest-environment node
*/
import { describe, expect, it, vi } from 'vitest'

vi.mock('@/triggers', () => ({
getTrigger: vi.fn(() => ({ subBlocks: [] })),
}))

import { TableBlock } from '@/blocks/blocks/table'

function params(input: Record<string, unknown>): Record<string, unknown> {
return TableBlock.tools.config?.params?.(input as never) as Record<string, unknown>
}

describe('table query_rows transformer', () => {
it('keeps an omitted limit unbounded', () => {
expect(params({ operation: 'query_rows', tableId: 'table-1' }).limit).toBeUndefined()
})

it('parses and validates an explicit limit', () => {
expect(params({ operation: 'query_rows', tableId: 'table-1', limit: '25' }).limit).toBe(25)
expect(params({ operation: 'query_rows', tableId: 'table-1', limit: '1000000' }).limit).toBe(
1000000
)
expect(() => params({ operation: 'query_rows', tableId: 'table-1', limit: 'abc' })).toThrow(
/Invalid number for Limit/
)
})
})
30 changes: 23 additions & 7 deletions apps/sim/blocks/blocks/table.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors'
import { TABLE_LIMITS } from '@/lib/table/constants'
import { filterRulesToFilter, sortRulesToSort } from '@/lib/table/query-builder/converters'
import type { BlockConfig } from '@/blocks/types'
import { parseOptionalNumberInput } from '@/blocks/utils'
import type { TableQueryResponse } from '@/tools/table/types'
import { getTrigger } from '@/triggers'

Expand Down Expand Up @@ -113,7 +114,11 @@ const paramTransformers: Record<string, (params: TableBlockParams) => ParsedPara
tableId: params.tableId,
filter,
data: parseJSON(params.data, 'Row Data'),
limit: params.limit ? Number.parseInt(params.limit) : undefined,
limit: parseOptionalNumberInput(params.limit, 'Limit', {
integer: true,
min: 1,
max: TABLE_LIMITS.MAX_BULK_OPERATION_SIZE,
}),
}
},

Expand All @@ -136,7 +141,11 @@ const paramTransformers: Record<string, (params: TableBlockParams) => ParsedPara
return {
tableId: params.tableId,
filter,
limit: params.limit ? Number.parseInt(params.limit) : undefined,
limit: parseOptionalNumberInput(params.limit, 'Limit', {
integer: true,
min: 1,
max: TABLE_LIMITS.MAX_BULK_OPERATION_SIZE,
}),
}
},

Expand Down Expand Up @@ -171,8 +180,11 @@ const paramTransformers: Record<string, (params: TableBlockParams) => ParsedPara
tableId: params.tableId,
filter,
sort,
limit: params.limit ? Number.parseInt(params.limit) : 100,
offset: params.offset ? Number.parseInt(params.offset) : 0,
limit: parseOptionalNumberInput(params.limit, 'Limit', {
integer: true,
min: 1,
}),
Comment thread
TheodoreSpeaks marked this conversation as resolved.
offset: parseOptionalNumberInput(params.offset, 'Offset', { integer: true, min: 0 }) ?? 0,
}
},
}
Expand All @@ -197,7 +209,7 @@ export const TableBlock: BlockConfig<TableQueryResponse> = {
name: 'Table',
description: 'User-defined data tables',
longDescription:
'Create and manage custom data tables. Store, query, and manipulate structured data within workflows.',
'Create and manage custom data tables. Store, query, and manipulate structured data within workflows. Query Rows returns every matching row when Limit is omitted and fails if the result exceeds 5MB.',
docsLink: 'https://docs.sim.ai/integrations/table',
category: 'blocks',
bgColor: '#10B981',
Expand Down Expand Up @@ -652,7 +664,7 @@ Return ONLY the sort JSON:`,
id: 'limit',
title: 'Limit',
type: 'short-input',
placeholder: '100',
placeholder: 'Leave empty for all rows (fails over 5MB)',
condition: {
field: 'operation',
value: ['query_rows', 'update_rows_by_filter', 'delete_rows_by_filter'],
Expand Down Expand Up @@ -726,7 +738,11 @@ Return ONLY the sort JSON:`,
description: 'Visual filter builder conditions for bulk operations',
},
filter: { type: 'json', description: 'Filter criteria for query/update/delete operations' },
limit: { type: 'number', description: 'Query or bulk operation limit' },
limit: {
type: 'number',
description:
'Optional query row limit; omit to return every matching row (fails over 5MB). Also caps bulk update/delete operations.',
},
builderMode: {
type: 'string',
description: 'Input mode for filter and sort (builder or json)',
Expand Down
14 changes: 14 additions & 0 deletions apps/sim/lib/api/contracts/tables.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,20 @@ describe('tableRowsQuerySchema includeTotal', () => {
})
})

describe('tableRowsQuerySchema limit', () => {
it('leaves an omitted or empty limit unbounded', () => {
expect(tableRowsQuerySchema.parse({ workspaceId: 'ws-1' }).limit).toBeUndefined()
expect(tableRowsQuerySchema.parse({ workspaceId: 'ws-1', limit: '' }).limit).toBeUndefined()
})

it('still parses and validates an explicit limit', () => {
expect(tableRowsQuerySchema.parse({ workspaceId: 'ws-1', limit: '25' }).limit).toBe(25)
expect(tableRowsQuerySchema.parse({ workspaceId: 'ws-1', limit: '1000000' }).limit).toBe(
1000000
)
})
})

describe('tableEventStreamQuerySchema', () => {
it('parses an explicit cursor', () => {
expect(tableEventStreamQuerySchema.parse({ from: '7' })).toEqual({ from: 7 })
Expand Down
25 changes: 14 additions & 11 deletions apps/sim/lib/api/contracts/tables.ts
Original file line number Diff line number Diff line change
Expand Up @@ -820,11 +820,21 @@ export const tableRowsQueryBaseSchema = z.object({
.default(true),
})

export const tableRowsQuerySchema = tableRowsQueryBaseSchema.refine(
(data) => !(data.after && data.sort),
{ message: 'after cursor cannot be combined with sort — cursors paginate the default order' }
const unboundedTableRowsLimitSchema = z.preprocess(
(value) => (value === null || value === undefined || value === '' ? undefined : Number(value)),
z
.number({ error: 'Limit must be a number' })
.int('Limit must be an integer')
.min(1, 'Limit must be at least 1')
.optional()
Comment thread
cursor[bot] marked this conversation as resolved.
)
Comment thread
TheodoreSpeaks marked this conversation as resolved.

export const tableRowsQuerySchema = tableRowsQueryBaseSchema
.extend({ limit: unboundedTableRowsLimitSchema })
.refine((data) => !(data.after && data.sort), {
message: 'after cursor cannot be combined with sort — cursors paginate the default order',
})

export const updateRowsByFilterBodySchema = z.object({
workspaceId: workspaceIdSchema,
filter: bulkFilterSchema,
Expand Down Expand Up @@ -1063,14 +1073,7 @@ export const rowQueryBodySchema = z.object({
// Omitted limit returns the ENTIRE matching result, failing fast (400) when
// it exceeds the response byte budget. An explicit limit caps the page row
// count; the byte budget may still end a page early with nextCursor set.
limit: z.preprocess(
(value) => (value === null || value === undefined || value === '' ? undefined : Number(value)),
z
.number({ error: 'Limit must be a number' })
.int('Limit must be an integer')
.min(1, 'Limit must be at least 1')
.optional()
),
limit: unboundedTableRowsLimitSchema,
cursor: z.string().min(1, 'cursor must be a non-empty token').optional(),
})

Expand Down
25 changes: 25 additions & 0 deletions apps/sim/lib/table/__tests__/service-filter-threading.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,21 @@ describe('queryRows byte budget', () => {
updatedAt: new Date('2024-01-01'),
})

const mockRowsPastFormerBatchSafetyLimit = () => {
const largeRow = row(1, TABLE_LIMITS.MAX_ROW_SIZE_BYTES)
const smallRow = row(2, 0)
const state = { drainBatch: 0 }
dbChainMockFns.limit.mockResolvedValueOnce([])
dbChainMockFns.limit.mockImplementation(async (ask: number) => {
state.drainBatch++
if (state.drainBatch > 1001) return []
const rows = Array.from({ length: ask }, () => smallRow)
if (state.drainBatch === 1) rows[0] = largeRow
return rows
})
return state
}

it('returns an empty page with a null cursor', async () => {
const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1')
expect(result.rows).toEqual([])
Expand Down Expand Up @@ -246,6 +261,16 @@ describe('queryRows byte budget', () => {
expect(result.nextCursor).toBeNull()
})

it('returns an entire under-budget result past the former batch safety limit', async () => {
const state = mockRowsPastFormerBatchSafetyLimit()

const result = await queryRows(TABLE, { includeTotal: false, withExecutions: false }, 'req-1')

expect(state.drainBatch).toBe(1002)
expect(result.rows.length).toBeGreaterThan(TABLE_LIMITS.MAX_QUERY_LIMIT)
expect(result.nextCursor).toBeNull()
})

it('byte-cuts a BOUNDED page and returns a resume cursor instead of throwing', async () => {
const perRow = Math.floor(TABLE_LIMITS.MAX_QUERY_RESULT_BYTES * 0.6)
dbChainMockFns.limit.mockResolvedValueOnce([])
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/lib/table/llm/enrichment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ export function enrichTableToolParameters(
if (enrichedProperties.limit && toolId === 'table_query_rows') {
enrichedProperties.limit = {
...enrichedProperties.limit,
description: `Maximum rows to return (min: 1, max: 1000, default: 100). For ranking queries: use limit=1 for highest/lowest, limit=2 for second highest, etc.`,
description: `Maximum rows to return (min: 1). Omit to return every matching row; the query fails if the result exceeds 5MB. For ranking queries: use limit=1 for highest/lowest, limit=2 for second highest, etc.`,
}
}

Expand Down
13 changes: 1 addition & 12 deletions apps/sim/lib/table/rows/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1270,17 +1270,6 @@ interface BoundedFetchResult {
anchorOffset: number
}

/**
* Belt-and-braces bound on drain iterations.
*
* Unreachable only because every iteration either consumes at least one row or cuts, and a bounded
* page's `limit` is capped at {@link TABLE_LIMITS.MAX_QUERY_LIMIT} — so the limit cut always fires
* first. That makes the two constants exactly tight: raising `MAX_QUERY_LIMIT` above this bound
* would let the loop exit with rows still unread and `hasMore: false`, which clients now trust as
* end-of-table (they terminate on `nextCursor`, which this decides). Raise both together.
*/
const MAX_QUERY_BATCHES = 1000

/**
* Drains rows in adaptively-sized bounded batches until the caller's `limit`
* or the byte ceiling ends the page. Never issues an unbounded SELECT: the
Expand Down Expand Up @@ -1364,7 +1353,7 @@ async function fetchRowsBounded(params: BoundedFetchParams): Promise<BoundedFetc
return withReadGuards(async (trx) => buildQuery(trx), { seqscanOff: sorted })
}

for (let iteration = 0; iteration < MAX_QUERY_BATCHES; iteration++) {
while (true) {
const limitRemaining = limit === undefined ? Number.POSITIVE_INFINITY : limit - rows.length
const target = Math.min(nextBatchRows(), limitRemaining)
const ask = target + 1 // +1 = witness row proving more data exists past a cut
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/tools/generated/tool-metadata.ts

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions apps/sim/tools/table/query_rows.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { TABLE_LIMITS } from '@/lib/table/constants'
import { enrichTableToolSchema } from '@/tools/schema-enrichers'
import type { TableQueryResponse, TableRowQueryParams } from '@/tools/table/types'
import type { ToolConfig } from '@/tools/types'
Expand Down Expand Up @@ -38,7 +37,8 @@ export const tableQueryRowsTool: ToolConfig<TableRowQueryParams, TableQueryRespo
limit: {
type: 'number',
required: false,
description: `Maximum rows to return (default: ${TABLE_LIMITS.DEFAULT_QUERY_LIMIT}, max: ${TABLE_LIMITS.MAX_QUERY_LIMIT})`,
description:
'Maximum rows to return. Omit to return every matching row; the query fails if the result exceeds the 5MB response budget.',
visibility: 'user-or-llm',
},
offset: {
Expand Down
Loading