From 61ceedc5b910c403d51e884fe40f125d79554384 Mon Sep 17 00:00:00 2001 From: Waleed Date: Tue, 7 Jul 2026 13:57:33 -0700 Subject: [PATCH 1/9] fix(notion): align integration with live API docs, add block retrieval coverage (#5491) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(notion): align integration with live API docs, add block retrieval coverage - wire up notion_update_page in the block (was registered but unreachable — no dropdown option or subBlocks) - fix legacy NotionBlock outputs to cover all 17 operations, not just content/metadata - trim ID params (pageId, databaseId, parentId) before use in request URLs across 8 tool files - remove content required:true on create_page (Notion allows title-only pages) - remove dead unused NotionReadDatabaseParams interface - add notion_retrieve_block/_v2 tool for GET /v1/blocks/{id}, filling a gap in block CRUD coverage * fix(notion): second validation pass — pagination gaps, phantom outputs, dead types, regen docs - add missing startCursor/start_cursor pagination param to notion_query_database and notion_search - fix notion_search missing from the shared pageSize/startCursor field's operation condition in NotionBlock - fix NotionV2Block title/url/created_time/last_edited_time outputs rendering unconditionally for every operation instead of only the ones that return them - remove 153 lines of dead unused output-shape constants from types.ts - regenerate integration docs * fix(notion): expose retrieve-block outputs in legacy block picker type/block/archived only conditioned on notion_update_block/notion_delete_block, and has_children was missing entirely — Retrieve Block's payload was invisible in the legacy NotionBlock output picker (Greptile P1 on PR #5491). * fix(notion): document type-specific content gap on retrieve_block output block.properties (BLOCK_OUTPUT_PROPERTIES) only covers common block fields, not the type-specific sub-object (paragraph.rich_text, image.file, etc.) — that varies per block type and isn't enumerable. Documented in the description per Greptile P2 on PR #5491; no functional change, block: data already carries the full object at runtime. --- .../content/docs/en/integrations/notion.mdx | 77 +++-- apps/sim/blocks/blocks/notion.ts | 264 +++++++++++++++++- apps/sim/tools/notion/add_database_row.ts | 2 +- apps/sim/tools/notion/create_database.ts | 2 +- apps/sim/tools/notion/create_page.ts | 2 +- apps/sim/tools/notion/index.ts | 3 + apps/sim/tools/notion/query_database.ts | 13 +- apps/sim/tools/notion/read.ts | 6 +- apps/sim/tools/notion/read_database.ts | 2 +- apps/sim/tools/notion/retrieve_block.ts | 107 +++++++ apps/sim/tools/notion/search.ts | 11 + apps/sim/tools/notion/types.ts | 158 +---------- apps/sim/tools/notion/update_page.ts | 2 +- apps/sim/tools/notion/write.ts | 2 +- apps/sim/tools/registry.ts | 4 + 15 files changed, 458 insertions(+), 197 deletions(-) create mode 100644 apps/sim/tools/notion/retrieve_block.ts diff --git a/apps/docs/content/docs/en/integrations/notion.mdx b/apps/docs/content/docs/en/integrations/notion.mdx index af14a062afc..4f397435ee5 100644 --- a/apps/docs/content/docs/en/integrations/notion.mdx +++ b/apps/docs/content/docs/en/integrations/notion.mdx @@ -145,6 +145,7 @@ Query and filter Notion database entries with advanced filtering | `filter` | string | No | Filter conditions as JSON \(optional\) | | `sorts` | string | No | Sort criteria as JSON array \(optional\) | | `pageSize` | number | No | Number of results to return \(default: 100, max: 100\) | +| `startCursor` | string | No | Pagination cursor returned by a previous request | #### Output @@ -200,6 +201,7 @@ Search across all pages and databases in Notion workspace | `query` | string | No | Search terms to find pages and databases \(leave empty to get all pages\) | | `filterType` | string | No | Filter by object type: "page", "database", or leave empty for all | | `pageSize` | number | No | Number of results to return \(default: 100, max: 100\) | +| `startCursor` | string | No | Pagination cursor returned by a previous request | #### Output @@ -302,8 +304,44 @@ Append new block children (content) to a Notion page or block | `properties` | json | Database properties schema | | `appended` | boolean | Whether content was successfully appended | | `type` | string | Block type | -| `block` | json | The full updated Notion block object | -| `archived` | boolean | Whether the block was archived | +| `block` | json | The full Notion block object | +| `has_children` | boolean | Whether the block has nested blocks | +| `archived` | boolean | Whether the block is archived | +| `discussion_id` | string | Discussion thread ID | +| `name` | string | User display name | +| `avatar_url` | string | User avatar image URL | +| `email` | string | User email address \(person users only\) | + +### `notion_retrieve_block` + +Retrieve a single Notion block by its UUID, including its type-specific content + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `blockId` | string | Yes | The UUID of the block to retrieve | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `content` | string | Page content in markdown format, or comment text for create comment | +| `title` | string | Page or database title | +| `url` | string | Notion URL | +| `id` | string | Page, database, block, comment, or user ID | +| `created_time` | string | Creation timestamp | +| `last_edited_time` | string | Last edit timestamp | +| `results` | array | Array of results \(pages, blocks, comments, or users\) | +| `has_more` | boolean | Whether more results are available | +| `next_cursor` | string | Cursor for pagination | +| `total_results` | number | Number of results returned | +| `properties` | json | Database properties schema | +| `appended` | boolean | Whether content was successfully appended | +| `type` | string | Block type | +| `block` | json | The full Notion block object | +| `has_children` | boolean | Whether the block has nested blocks | +| `archived` | boolean | Whether the block is archived | | `discussion_id` | string | Discussion thread ID | | `name` | string | User display name | | `avatar_url` | string | User avatar image URL | @@ -338,8 +376,9 @@ Retrieve the block children (content) of a Notion page or block | `properties` | json | Database properties schema | | `appended` | boolean | Whether content was successfully appended | | `type` | string | Block type | -| `block` | json | The full updated Notion block object | -| `archived` | boolean | Whether the block was archived | +| `block` | json | The full Notion block object | +| `has_children` | boolean | Whether the block has nested blocks | +| `archived` | boolean | Whether the block is archived | | `discussion_id` | string | Discussion thread ID | | `name` | string | User display name | | `avatar_url` | string | User avatar image URL | @@ -374,8 +413,9 @@ Update the content or archived state of a single Notion block | `properties` | json | Database properties schema | | `appended` | boolean | Whether content was successfully appended | | `type` | string | Block type | -| `block` | json | The full updated Notion block object | -| `archived` | boolean | Whether the block was archived | +| `block` | json | The full Notion block object | +| `has_children` | boolean | Whether the block has nested blocks | +| `archived` | boolean | Whether the block is archived | | `discussion_id` | string | Discussion thread ID | | `name` | string | User display name | | `avatar_url` | string | User avatar image URL | @@ -408,8 +448,9 @@ Delete (move to trash) a single Notion block | `properties` | json | Database properties schema | | `appended` | boolean | Whether content was successfully appended | | `type` | string | Block type | -| `block` | json | The full updated Notion block object | -| `archived` | boolean | Whether the block was archived | +| `block` | json | The full Notion block object | +| `has_children` | boolean | Whether the block has nested blocks | +| `archived` | boolean | Whether the block is archived | | `discussion_id` | string | Discussion thread ID | | `name` | string | User display name | | `avatar_url` | string | User avatar image URL | @@ -444,8 +485,9 @@ Create a comment on a Notion page or within an existing discussion thread | `properties` | json | Database properties schema | | `appended` | boolean | Whether content was successfully appended | | `type` | string | Block type | -| `block` | json | The full updated Notion block object | -| `archived` | boolean | Whether the block was archived | +| `block` | json | The full Notion block object | +| `has_children` | boolean | Whether the block has nested blocks | +| `archived` | boolean | Whether the block is archived | | `discussion_id` | string | Discussion thread ID | | `name` | string | User display name | | `avatar_url` | string | User avatar image URL | @@ -480,8 +522,9 @@ List unresolved comments on a Notion page or block | `properties` | json | Database properties schema | | `appended` | boolean | Whether content was successfully appended | | `type` | string | Block type | -| `block` | json | The full updated Notion block object | -| `archived` | boolean | Whether the block was archived | +| `block` | json | The full Notion block object | +| `has_children` | boolean | Whether the block has nested blocks | +| `archived` | boolean | Whether the block is archived | | `discussion_id` | string | Discussion thread ID | | `name` | string | User display name | | `avatar_url` | string | User avatar image URL | @@ -515,8 +558,9 @@ List all users (members and bots) in the Notion workspace | `properties` | json | Database properties schema | | `appended` | boolean | Whether content was successfully appended | | `type` | string | Block type | -| `block` | json | The full updated Notion block object | -| `archived` | boolean | Whether the block was archived | +| `block` | json | The full Notion block object | +| `has_children` | boolean | Whether the block has nested blocks | +| `archived` | boolean | Whether the block is archived | | `discussion_id` | string | Discussion thread ID | | `name` | string | User display name | | `avatar_url` | string | User avatar image URL | @@ -549,8 +593,9 @@ Retrieve a single Notion user by their UUID | `properties` | json | Database properties schema | | `appended` | boolean | Whether content was successfully appended | | `type` | string | Block type | -| `block` | json | The full updated Notion block object | -| `archived` | boolean | Whether the block was archived | +| `block` | json | The full Notion block object | +| `has_children` | boolean | Whether the block has nested blocks | +| `archived` | boolean | Whether the block is archived | | `discussion_id` | string | Discussion thread ID | | `name` | string | User display name | | `avatar_url` | string | User avatar image URL | diff --git a/apps/sim/blocks/blocks/notion.ts b/apps/sim/blocks/blocks/notion.ts index 03bfa2b8257..db2fecf3e09 100644 --- a/apps/sim/blocks/blocks/notion.ts +++ b/apps/sim/blocks/blocks/notion.ts @@ -30,10 +30,12 @@ export const NotionBlock: BlockConfig = { { label: 'Read Page', id: 'notion_read' }, { label: 'Read Database', id: 'notion_read_database' }, { label: 'Create Page', id: 'notion_create_page' }, + { label: 'Update Page Properties', id: 'notion_update_page' }, { label: 'Create Database', id: 'notion_create_database' }, { label: 'Add Database Row', id: 'notion_add_database_row' }, { label: 'Append Content', id: 'notion_write' }, { label: 'Append Block Children', id: 'notion_append_blocks' }, + { label: 'Retrieve Block', id: 'notion_retrieve_block' }, { label: 'Retrieve Block Children', id: 'notion_retrieve_block_children' }, { label: 'Update Block', id: 'notion_update_block' }, { label: 'Delete Block', id: 'notion_delete_block' }, @@ -77,7 +79,7 @@ export const NotionBlock: BlockConfig = { mode: 'basic', condition: { field: 'operation', - value: ['notion_read', 'notion_write'], + value: ['notion_read', 'notion_write', 'notion_update_page'], }, required: true, }, @@ -91,7 +93,7 @@ export const NotionBlock: BlockConfig = { mode: 'advanced', condition: { field: 'operation', - value: ['notion_read', 'notion_write'], + value: ['notion_read', 'notion_write', 'notion_update_page'], }, required: true, }, @@ -192,7 +194,6 @@ export const NotionBlock: BlockConfig = { field: 'operation', value: 'notion_create_page', }, - required: true, wandConfig: { enabled: true, prompt: @@ -243,6 +244,7 @@ export const NotionBlock: BlockConfig = { 'notion_retrieve_block_children', 'notion_list_comments', 'notion_list_users', + 'notion_search', ], }, }, @@ -318,6 +320,22 @@ export const NotionBlock: BlockConfig = { generationType: 'json-object', }, }, + { + id: 'properties', + title: 'Properties to Update', + type: 'code', + placeholder: 'Enter page properties as JSON object', + condition: { field: 'operation', value: 'notion_update_page' }, + required: true, + wandConfig: { + enabled: true, + prompt: + 'Generate Notion page properties to update in JSON format based on the user\'s description. Properties must match the parent database schema. Common formats: Title: {"Name": {"title": [{"text": {"content": "Value"}}]}}, Text: {"Description": {"rich_text": [{"text": {"content": "Value"}}]}}, Number: {"Price": {"number": 10}}, Select: {"Status": {"select": {"name": "Done"}}}, Multi-select: {"Tags": {"multi_select": [{"name": "Tag1"}]}}, Date: {"Due": {"date": {"start": "2024-01-01"}}}, Checkbox: {"Done": {"checkbox": true}}. Return ONLY valid JSON - no explanations.', + placeholder: + 'Describe the properties to update (e.g., "set status to Done, priority to High")...', + generationType: 'json-object', + }, + }, { id: 'blockId', title: 'Page or Block ID', @@ -328,6 +346,7 @@ export const NotionBlock: BlockConfig = { field: 'operation', value: [ 'notion_append_blocks', + 'notion_retrieve_block', 'notion_retrieve_block_children', 'notion_update_block', 'notion_delete_block', @@ -433,7 +452,13 @@ export const NotionBlock: BlockConfig = { mode: 'advanced', condition: { field: 'operation', - value: ['notion_retrieve_block_children', 'notion_list_comments', 'notion_list_users'], + value: [ + 'notion_retrieve_block_children', + 'notion_list_comments', + 'notion_list_users', + 'notion_query_database', + 'notion_search', + ], }, }, ], @@ -449,6 +474,7 @@ export const NotionBlock: BlockConfig = { 'notion_add_database_row', 'notion_update_page', 'notion_append_blocks', + 'notion_retrieve_block', 'notion_retrieve_block_children', 'notion_update_block', 'notion_delete_block', @@ -480,6 +506,8 @@ export const NotionBlock: BlockConfig = { return 'notion_add_database_row' case 'notion_append_blocks': return 'notion_append_blocks' + case 'notion_retrieve_block': + return 'notion_retrieve_block' case 'notion_retrieve_block_children': return 'notion_retrieve_block_children' case 'notion_update_block': @@ -519,7 +547,8 @@ export const NotionBlock: BlockConfig = { if ( (operation === 'notion_create_page' || operation === 'notion_create_database' || - operation === 'notion_add_database_row') && + operation === 'notion_add_database_row' || + operation === 'notion_update_page') && properties ) { if (typeof properties === 'string') { @@ -651,17 +680,169 @@ export const NotionBlock: BlockConfig = { userId: { type: 'string', description: 'User identifier' }, }, outputs: { - // Common outputs across all Notion operations + // Outputs for the original content/metadata-shaped operations content: { type: 'string', - description: 'Page content, search results, or confirmation messages', + description: 'Page content, comment text, search results, or confirmation messages', + condition: { + field: 'operation', + value: [ + 'notion_read', + 'notion_write', + 'notion_create_page', + 'notion_update_page', + 'notion_query_database', + 'notion_search', + 'notion_create_database', + 'notion_read_database', + 'notion_create_comment', + ], + }, }, - - // Metadata object containing operation-specific information metadata: { type: 'json', description: 'Metadata containing operation-specific details including page/database info, results, and pagination data', + condition: { + field: 'operation', + value: [ + 'notion_read', + 'notion_write', + 'notion_create_page', + 'notion_update_page', + 'notion_query_database', + 'notion_search', + 'notion_create_database', + 'notion_read_database', + ], + }, + }, + + // Outputs for the API-aligned flat-shaped operations added after the legacy block was hidden + id: { + type: 'string', + description: 'Row, block, comment, or user ID', + condition: { + field: 'operation', + value: [ + 'notion_add_database_row', + 'notion_retrieve_block', + 'notion_update_block', + 'notion_delete_block', + 'notion_create_comment', + 'notion_retrieve_user', + ], + }, + }, + url: { + type: 'string', + description: 'Notion page URL', + condition: { field: 'operation', value: 'notion_add_database_row' }, + }, + title: { + type: 'string', + description: 'Row title', + condition: { field: 'operation', value: 'notion_add_database_row' }, + }, + created_time: { + type: 'string', + description: 'Creation timestamp', + condition: { + field: 'operation', + value: ['notion_add_database_row', 'notion_create_comment'], + }, + }, + last_edited_time: { + type: 'string', + description: 'Last edit timestamp', + condition: { field: 'operation', value: 'notion_add_database_row' }, + }, + results: { + type: 'array', + description: 'Array of results (blocks, comments, or users)', + condition: { + field: 'operation', + value: [ + 'notion_append_blocks', + 'notion_retrieve_block_children', + 'notion_list_comments', + 'notion_list_users', + ], + }, + }, + has_more: { + type: 'boolean', + description: 'Whether more results are available', + condition: { + field: 'operation', + value: [ + 'notion_append_blocks', + 'notion_retrieve_block_children', + 'notion_list_comments', + 'notion_list_users', + ], + }, + }, + next_cursor: { + type: 'string', + description: 'Cursor for pagination', + condition: { + field: 'operation', + value: [ + 'notion_append_blocks', + 'notion_retrieve_block_children', + 'notion_list_comments', + 'notion_list_users', + ], + }, + }, + type: { + type: 'string', + description: 'Block type', + condition: { field: 'operation', value: ['notion_retrieve_block', 'notion_update_block'] }, + }, + block: { + type: 'json', + description: 'The full Notion block object', + condition: { field: 'operation', value: ['notion_retrieve_block', 'notion_update_block'] }, + }, + has_children: { + type: 'boolean', + description: 'Whether the block has nested blocks', + condition: { field: 'operation', value: 'notion_retrieve_block' }, + }, + archived: { + type: 'boolean', + description: 'Whether the block is archived', + condition: { + field: 'operation', + value: ['notion_retrieve_block', 'notion_update_block', 'notion_delete_block'], + }, + }, + discussion_id: { + type: 'string', + description: 'Discussion thread ID', + condition: { field: 'operation', value: 'notion_create_comment' }, + }, + rich_text: { + type: 'json', + description: 'Rich text array of the comment', + condition: { field: 'operation', value: 'notion_create_comment' }, + }, + name: { + type: 'string', + description: 'User display name', + condition: { field: 'operation', value: 'notion_retrieve_user' }, + }, + avatar_url: { + type: 'string', + description: 'User avatar image URL', + condition: { field: 'operation', value: 'notion_retrieve_user' }, + }, + email: { + type: 'string', + description: 'User email address (person users only)', + condition: { field: 'operation', value: 'notion_retrieve_user' }, }, }, } @@ -721,6 +902,7 @@ export const NotionV2Block: BlockConfig = { 'notion_create_database_v2', 'notion_add_database_row_v2', 'notion_append_blocks_v2', + 'notion_retrieve_block_v2', 'notion_retrieve_block_children_v2', 'notion_update_block_v2', 'notion_delete_block_v2', @@ -749,10 +931,32 @@ export const NotionV2Block: BlockConfig = { title: { type: 'string', description: 'Page or database title', + condition: { + field: 'operation', + value: [ + 'notion_read', + 'notion_create_page', + 'notion_update_page', + 'notion_create_database', + 'notion_read_database', + 'notion_add_database_row', + ], + }, }, url: { type: 'string', description: 'Notion URL', + condition: { + field: 'operation', + value: [ + 'notion_read', + 'notion_create_page', + 'notion_update_page', + 'notion_create_database', + 'notion_read_database', + 'notion_add_database_row', + ], + }, }, id: { type: 'string', @@ -765,6 +969,7 @@ export const NotionV2Block: BlockConfig = { 'notion_add_database_row', 'notion_read_database', 'notion_update_page', + 'notion_retrieve_block', 'notion_update_block', 'notion_delete_block', 'notion_create_comment', @@ -775,10 +980,31 @@ export const NotionV2Block: BlockConfig = { created_time: { type: 'string', description: 'Creation timestamp', + condition: { + field: 'operation', + value: [ + 'notion_read', + 'notion_create_page', + 'notion_create_database', + 'notion_read_database', + 'notion_add_database_row', + 'notion_create_comment', + ], + }, }, last_edited_time: { type: 'string', description: 'Last edit timestamp', + condition: { + field: 'operation', + value: [ + 'notion_read', + 'notion_create_page', + 'notion_update_page', + 'notion_read_database', + 'notion_add_database_row', + ], + }, }, // List/query/search outputs results: { @@ -843,21 +1069,29 @@ export const NotionV2Block: BlockConfig = { description: 'Whether content was successfully appended', condition: { field: 'operation', value: 'notion_write' }, }, - // Block update/delete outputs + // Block retrieve/update/delete outputs type: { type: 'string', description: 'Block type', - condition: { field: 'operation', value: 'notion_update_block' }, + condition: { field: 'operation', value: ['notion_retrieve_block', 'notion_update_block'] }, }, block: { type: 'json', - description: 'The full updated Notion block object', - condition: { field: 'operation', value: 'notion_update_block' }, + description: 'The full Notion block object', + condition: { field: 'operation', value: ['notion_retrieve_block', 'notion_update_block'] }, + }, + has_children: { + type: 'boolean', + description: 'Whether the block has nested blocks', + condition: { field: 'operation', value: 'notion_retrieve_block' }, }, archived: { type: 'boolean', - description: 'Whether the block was archived', - condition: { field: 'operation', value: ['notion_update_block', 'notion_delete_block'] }, + description: 'Whether the block is archived', + condition: { + field: 'operation', + value: ['notion_retrieve_block', 'notion_update_block', 'notion_delete_block'], + }, }, // Comment outputs discussion_id: { diff --git a/apps/sim/tools/notion/add_database_row.ts b/apps/sim/tools/notion/add_database_row.ts index 53da6718f2f..eb528032480 100644 --- a/apps/sim/tools/notion/add_database_row.ts +++ b/apps/sim/tools/notion/add_database_row.ts @@ -67,7 +67,7 @@ export const notionAddDatabaseRowTool: ToolConfig< return { parent: { type: 'database_id', - database_id: params.databaseId, + database_id: params.databaseId.trim(), }, properties: params.properties, } diff --git a/apps/sim/tools/notion/create_database.ts b/apps/sim/tools/notion/create_database.ts index 2ddda97e316..a67e4020c98 100644 --- a/apps/sim/tools/notion/create_database.ts +++ b/apps/sim/tools/notion/create_database.ts @@ -65,7 +65,7 @@ export const notionCreateDatabaseTool: ToolConfig { - return `https://api.notion.com/v1/databases/${params.databaseId}/query` + return `https://api.notion.com/v1/databases/${params.databaseId.trim()}/query` }, method: 'POST', headers: (params: NotionQueryDatabaseParams) => { @@ -90,6 +96,11 @@ export const notionQueryDatabaseTool: ToolConfig = { request: { url: (params: NotionReadParams) => { - return `https://api.notion.com/v1/pages/${params.pageId}` + return `https://api.notion.com/v1/pages/${params.pageId.trim()}` }, method: 'GET', headers: (params: NotionReadParams) => { @@ -64,7 +64,7 @@ export const notionReadTool: ToolConfig = { } // Now fetch the page content using blocks endpoint - const pageId = params?.pageId + const pageId = params?.pageId?.trim() const accessToken = params?.accessToken if (!pageId || !accessToken) { @@ -212,7 +212,7 @@ export const notionReadV2Tool: ToolConfig { - return `https://api.notion.com/v1/databases/${params.databaseId}` + return `https://api.notion.com/v1/databases/${params.databaseId.trim()}` }, method: 'GET', headers: (params: NotionReadDatabaseParams) => { diff --git a/apps/sim/tools/notion/retrieve_block.ts b/apps/sim/tools/notion/retrieve_block.ts new file mode 100644 index 00000000000..cf4c9efb771 --- /dev/null +++ b/apps/sim/tools/notion/retrieve_block.ts @@ -0,0 +1,107 @@ +import { BLOCK_OUTPUT_PROPERTIES } from '@/tools/notion/types' +import type { ToolConfig } from '@/tools/types' + +export interface NotionRetrieveBlockParams { + blockId: string + accessToken: string +} + +interface NotionRetrieveBlockResponse { + success: boolean + output: { + id: string + type: string + has_children: boolean + archived: boolean + block: Record + } +} + +export const notionRetrieveBlockTool: ToolConfig< + NotionRetrieveBlockParams, + NotionRetrieveBlockResponse +> = { + id: 'notion_retrieve_block', + name: 'Notion Retrieve Block', + description: 'Retrieve a single Notion block by its UUID, including its type-specific content', + version: '1.0.0', + + oauth: { + required: true, + provider: 'notion', + }, + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'hidden', + description: 'Notion OAuth access token', + }, + blockId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'The UUID of the block to retrieve', + }, + }, + + request: { + url: (params: NotionRetrieveBlockParams) => + `https://api.notion.com/v1/blocks/${params.blockId.trim()}`, + method: 'GET', + headers: (params: NotionRetrieveBlockParams) => { + if (!params.accessToken) { + throw new Error('Access token is required') + } + + return { + Authorization: `Bearer ${params.accessToken}`, + 'Notion-Version': '2022-06-28', + 'Content-Type': 'application/json', + } + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + return { + success: response.ok, + output: { + id: data.id, + type: data.type ?? '', + has_children: data.has_children ?? false, + archived: data.archived ?? false, + block: data, + }, + } + }, + + outputs: { + id: BLOCK_OUTPUT_PROPERTIES.id, + type: BLOCK_OUTPUT_PROPERTIES.type, + has_children: BLOCK_OUTPUT_PROPERTIES.has_children, + archived: BLOCK_OUTPUT_PROPERTIES.archived, + block: { + type: 'object', + description: + 'The full Notion block object. Includes a type-specific field (e.g. paragraph, heading_1, image) whose shape varies by block type and is not enumerated below — read it directly off this object.', + properties: BLOCK_OUTPUT_PROPERTIES, + }, + }, +} + +export const notionRetrieveBlockV2Tool: ToolConfig< + NotionRetrieveBlockParams, + NotionRetrieveBlockResponse +> = { + id: 'notion_retrieve_block_v2', + name: 'Notion Retrieve Block', + description: 'Retrieve a single Notion block by its UUID, including its type-specific content', + version: '2.0.0', + oauth: notionRetrieveBlockTool.oauth, + params: notionRetrieveBlockTool.params, + request: notionRetrieveBlockTool.request, + transformResponse: notionRetrieveBlockTool.transformResponse, + outputs: notionRetrieveBlockTool.outputs, +} diff --git a/apps/sim/tools/notion/search.ts b/apps/sim/tools/notion/search.ts index d0fa3e1ec24..a0feba29822 100644 --- a/apps/sim/tools/notion/search.ts +++ b/apps/sim/tools/notion/search.ts @@ -39,6 +39,12 @@ export const notionSearchTool: ToolConfig = visibility: 'user-or-llm', description: 'Number of results to return (default: 100, max: 100)', }, + startCursor: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Pagination cursor returned by a previous request', + }, }, request: { @@ -80,6 +86,11 @@ export const notionSearchTool: ToolConfig = body.page_size = Math.min(Number(params.pageSize), 100) } + // Add pagination cursor if provided + if (params.startCursor) { + body.start_cursor = params.startCursor.trim() + } + return body }, }, diff --git a/apps/sim/tools/notion/types.ts b/apps/sim/tools/notion/types.ts index 448d7f8635d..2dc15428d4d 100644 --- a/apps/sim/tools/notion/types.ts +++ b/apps/sim/tools/notion/types.ts @@ -190,15 +190,6 @@ export const FILE_OUTPUT_PROPERTIES = { }, } as const satisfies Record -/** - * Complete file output definition - */ -export const FILE_OUTPUT: OutputProperty = { - type: 'object', - description: 'File object', - properties: FILE_OUTPUT_PROPERTIES, -} - /** * Emoji object properties. * @see https://developers.notion.com/reference/emoji-object @@ -308,61 +299,6 @@ export const PAGE_OUTPUT: OutputProperty = { properties: PAGE_OUTPUT_PROPERTIES, } -/** - * Simplified page output properties for read operations (flattened structure). - * Used by notion_read and notion_read_v2 tools. - */ -export const PAGE_READ_OUTPUT_PROPERTIES = { - content: { - type: 'string', - description: 'Page content in markdown format with headers, paragraphs, lists, and todos', - }, - title: { type: 'string', description: 'Page title' }, - url: { type: 'string', description: 'Notion page URL' }, - created_time: { type: 'string', description: 'ISO 8601 creation timestamp' }, - last_edited_time: { type: 'string', description: 'ISO 8601 last edit timestamp' }, -} as const satisfies Record - -/** - * Page metadata output properties for legacy tools. - * Used by notion_read, notion_create_page, notion_update_page. - */ -export const PAGE_METADATA_OUTPUT_PROPERTIES = { - title: { type: 'string', description: 'Page title' }, - lastEditedTime: { type: 'string', description: 'ISO 8601 last edit timestamp' }, - createdTime: { type: 'string', description: 'ISO 8601 creation timestamp' }, - url: { type: 'string', description: 'Notion page URL' }, -} as const satisfies Record - -/** - * Page metadata output for create/update operations. - */ -export const PAGE_MUTATION_METADATA_OUTPUT_PROPERTIES = { - title: { type: 'string', description: 'Page title' }, - pageId: { type: 'string', description: 'Page UUID' }, - url: { type: 'string', description: 'Notion page URL' }, - lastEditedTime: { type: 'string', description: 'ISO 8601 last edit timestamp' }, - createdTime: { type: 'string', description: 'ISO 8601 creation timestamp' }, -} as const satisfies Record - -/** - * Complete page metadata output definition - */ -export const PAGE_METADATA_OUTPUT: OutputProperty = { - type: 'object', - description: 'Page metadata including title, URL, and timestamps', - properties: PAGE_METADATA_OUTPUT_PROPERTIES, -} - -/** - * Page mutation metadata output definition (for create/update) - */ -export const PAGE_MUTATION_METADATA_OUTPUT: OutputProperty = { - type: 'object', - description: 'Page metadata including title, page ID, URL, and timestamps', - properties: PAGE_MUTATION_METADATA_OUTPUT_PROPERTIES, -} - /** * Common database object properties from Notion API. * @see https://developers.notion.com/reference/database @@ -400,28 +336,6 @@ export const DATABASE_OUTPUT: OutputProperty = { properties: DATABASE_OUTPUT_PROPERTIES, } -/** - * Database metadata output properties for legacy tools. - * Used by notion_read_database, notion_create_database. - */ -export const DATABASE_METADATA_OUTPUT_PROPERTIES = { - id: { type: 'string', description: 'Database UUID' }, - title: { type: 'string', description: 'Database title' }, - url: { type: 'string', description: 'Notion database URL' }, - createdTime: { type: 'string', description: 'ISO 8601 creation timestamp' }, - lastEditedTime: { type: 'string', description: 'ISO 8601 last edit timestamp' }, - properties: { type: 'object', description: 'Database properties schema' }, -} as const satisfies Record - -/** - * Complete database metadata output definition - */ -export const DATABASE_METADATA_OUTPUT: OutputProperty = { - type: 'object', - description: 'Database metadata including title, ID, URL, timestamps, and properties schema', - properties: DATABASE_METADATA_OUTPUT_PROPERTIES, -} - /** * Block object properties from Notion API. * @see https://developers.notion.com/reference/block @@ -491,25 +405,6 @@ export const SEARCH_RESULTS_OUTPUT: OutputProperty = { }, } -/** - * Search metadata output properties for legacy tools. - */ -export const SEARCH_METADATA_OUTPUT_PROPERTIES = { - totalResults: { type: 'number', description: 'Number of results returned' }, - hasMore: { type: 'boolean', description: 'Whether more results are available' }, - nextCursor: { type: 'string', description: 'Cursor for next page of results', optional: true }, - results: SEARCH_RESULTS_OUTPUT, -} as const satisfies Record - -/** - * Complete search metadata output definition - */ -export const SEARCH_METADATA_OUTPUT: OutputProperty = { - type: 'object', - description: 'Search metadata including total results count, pagination info, and raw results', - properties: SEARCH_METADATA_OUTPUT_PROPERTIES, -} - /** * Database query results array output with page items */ @@ -522,43 +417,6 @@ export const DATABASE_QUERY_RESULTS_OUTPUT: OutputProperty = { }, } -/** - * Query metadata output properties for legacy tools. - */ -export const QUERY_METADATA_OUTPUT_PROPERTIES = { - totalResults: { type: 'number', description: 'Number of results returned' }, - hasMore: { type: 'boolean', description: 'Whether more results are available' }, - nextCursor: { type: 'string', description: 'Cursor for next page of results', optional: true }, - results: DATABASE_QUERY_RESULTS_OUTPUT, -} as const satisfies Record - -/** - * Complete query metadata output definition - */ -export const QUERY_METADATA_OUTPUT: OutputProperty = { - type: 'object', - description: 'Query metadata including total results, pagination info, and raw results array', - properties: QUERY_METADATA_OUTPUT_PROPERTIES, -} - -/** - * Row output properties for add_database_row operations. - */ -export const ROW_OUTPUT_PROPERTIES = { - id: { type: 'string', description: 'Row (page) UUID' }, - url: { type: 'string', description: 'Notion page URL' }, - title: { type: 'string', description: 'Row title' }, - created_time: { type: 'string', description: 'ISO 8601 creation timestamp' }, - last_edited_time: { type: 'string', description: 'ISO 8601 last edit timestamp' }, -} as const satisfies Record - -/** - * Write operation output properties. - */ -export const WRITE_OUTPUT_PROPERTIES = { - appended: { type: 'boolean', description: 'Whether content was successfully appended' }, -} as const satisfies Record - /** * Comment object properties from Notion API. * @see https://developers.notion.com/reference/comment-object @@ -574,15 +432,6 @@ export const COMMENT_OUTPUT_PROPERTIES = { rich_text: RICH_TEXT_ARRAY_OUTPUT, } as const satisfies Record -/** - * Complete comment output definition for array items - */ -export const COMMENT_OUTPUT: OutputProperty = { - type: 'object', - description: 'Notion comment object', - properties: COMMENT_OUTPUT_PROPERTIES, -} - /** * Array of block objects (used by append/retrieve block children). */ @@ -668,6 +517,7 @@ export interface NotionQueryDatabaseParams { filter?: string sorts?: string pageSize?: number + startCursor?: string accessToken: string } @@ -675,6 +525,7 @@ export interface NotionSearchParams { query?: string filterType?: string pageSize?: number + startCursor?: string accessToken: string } @@ -685,11 +536,6 @@ export interface NotionCreateDatabaseParams { accessToken: string } -interface NotionReadDatabaseParams { - databaseId: string - accessToken: string -} - export interface NotionAddDatabaseRowParams { databaseId: string properties: Record diff --git a/apps/sim/tools/notion/update_page.ts b/apps/sim/tools/notion/update_page.ts index 3babd475a8f..7b36c0b9f21 100644 --- a/apps/sim/tools/notion/update_page.ts +++ b/apps/sim/tools/notion/update_page.ts @@ -36,7 +36,7 @@ export const notionUpdatePageTool: ToolConfig { - return `https://api.notion.com/v1/pages/${params.pageId}` + return `https://api.notion.com/v1/pages/${params.pageId.trim()}` }, method: 'PATCH', headers: (params: NotionUpdatePageParams) => { diff --git a/apps/sim/tools/notion/write.ts b/apps/sim/tools/notion/write.ts index d2ac89aaf34..f3437b4fd73 100644 --- a/apps/sim/tools/notion/write.ts +++ b/apps/sim/tools/notion/write.ts @@ -35,7 +35,7 @@ export const notionWriteTool: ToolConfig = { request: { url: (params: NotionWriteParams) => { - return `https://api.notion.com/v1/blocks/${params.pageId}/children` + return `https://api.notion.com/v1/blocks/${params.pageId.trim()}/children` }, method: 'PATCH', headers: (params: NotionWriteParams) => { diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index fa950a2c023..4fcda0f1cd4 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -2489,6 +2489,8 @@ import { notionReadV2Tool, notionRetrieveBlockChildrenTool, notionRetrieveBlockChildrenV2Tool, + notionRetrieveBlockTool, + notionRetrieveBlockV2Tool, notionRetrieveUserTool, notionRetrieveUserV2Tool, notionSearchTool, @@ -5528,6 +5530,7 @@ export const tools: Record = { notion_add_database_row: notionAddDatabaseRowTool, notion_update_page: notionUpdatePageTool, notion_append_blocks: notionAppendBlocksTool, + notion_retrieve_block: notionRetrieveBlockTool, notion_retrieve_block_children: notionRetrieveBlockChildrenTool, notion_update_block: notionUpdateBlockTool, notion_delete_block: notionDeleteBlockTool, @@ -5545,6 +5548,7 @@ export const tools: Record = { notion_update_page_v2: notionUpdatePageV2Tool, notion_add_database_row_v2: notionAddDatabaseRowV2Tool, notion_append_blocks_v2: notionAppendBlocksV2Tool, + notion_retrieve_block_v2: notionRetrieveBlockV2Tool, notion_retrieve_block_children_v2: notionRetrieveBlockChildrenV2Tool, notion_update_block_v2: notionUpdateBlockV2Tool, notion_delete_block_v2: notionDeleteBlockV2Tool, From 56cd2cfa7caca8dcbceb8004c0986338f901a7a7 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 7 Jul 2026 15:58:17 -0700 Subject: [PATCH 2/9] feat(custom-block): audit-log publish, update, and unpublish events (#5493) --- apps/sim/app/api/custom-blocks/[id]/route.ts | 46 +++++++++++++++++-- apps/sim/app/api/custom-blocks/route.ts | 14 ++++++ .../lib/workflows/custom-blocks/operations.ts | 11 +++-- packages/audit/src/types.ts | 6 +++ packages/testing/src/mocks/audit.mock.ts | 4 ++ 5 files changed, 74 insertions(+), 7 deletions(-) diff --git a/apps/sim/app/api/custom-blocks/[id]/route.ts b/apps/sim/app/api/custom-blocks/[id]/route.ts index 3e995a96e72..14d7d06ee9f 100644 --- a/apps/sim/app/api/custom-blocks/[id]/route.ts +++ b/apps/sim/app/api/custom-blocks/[id]/route.ts @@ -1,3 +1,4 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { NextRequest } from 'next/server' @@ -29,19 +30,28 @@ type RouteContext = { params: Promise<{ id: string }> } * different workspace does not, so they cannot alter another workspace's block or * its exposed outputs. */ -async function authorizeManage(userId: string, id: string) { +type ManageContext = NonNullable>> + +async function authorizeManage( + userId: string, + id: string +): Promise<{ error: NextResponse; ctx: null } | { error: null; ctx: ManageContext }> { const ctx = await getCustomBlockManageContext(id) - if (!ctx) return { error: NextResponse.json({ error: 'Not found' }, { status: 404 }) } + if (!ctx) return { error: NextResponse.json({ error: 'Not found' }, { status: 404 }), ctx: null } if (!(await isFeatureEnabled('deploy-as-block', { userId, orgId: ctx.organizationId }))) { return { error: NextResponse.json({ error: 'Deploy as block is not enabled' }, { status: 403 }), + ctx: null, } } if (!ctx.sourceWorkspaceId || !(await hasWorkspaceAdminAccess(userId, ctx.sourceWorkspaceId))) { - return { error: NextResponse.json({ error: 'Admin permissions required' }, { status: 403 }) } + return { + error: NextResponse.json({ error: 'Admin permissions required' }, { status: 403 }), + ctx: null, + } } - return { error: null } + return { error: null, ctx } } export const PATCH = withRouteHandler(async (request: NextRequest, context: RouteContext) => { @@ -56,6 +66,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Rout const { id } = parsed.data.params const authz = await authorizeManage(session.user.id, id) if (authz.error) return authz.error + const { ctx } = authz const { name, description, enabled, iconUrl, inputs, exposedOutputs } = parsed.data.body try { @@ -67,6 +78,19 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: Rout iconUrl, exposedOutputs, }) + recordAudit({ + workspaceId: ctx.sourceWorkspaceId, + actorId: session.user.id, + actorName: session.user.name, + actorEmail: session.user.email, + action: AuditAction.CUSTOM_BLOCK_UPDATED, + resourceType: AuditResourceType.CUSTOM_BLOCK, + resourceId: id, + resourceName: name ?? ctx.name, + description: `Updated custom block "${name ?? ctx.name}"`, + metadata: { organizationId: ctx.organizationId, type: ctx.type }, + request, + }) return NextResponse.json({ success: true as const }) } catch (error) { if (error instanceof CustomBlockValidationError) { @@ -89,7 +113,21 @@ export const DELETE = withRouteHandler(async (request: NextRequest, context: Rou const { id } = parsed.data.params const authz = await authorizeManage(session.user.id, id) if (authz.error) return authz.error + const { ctx } = authz await deleteCustomBlock(id) + recordAudit({ + workspaceId: ctx.sourceWorkspaceId, + actorId: session.user.id, + actorName: session.user.name, + actorEmail: session.user.email, + action: AuditAction.CUSTOM_BLOCK_DELETED, + resourceType: AuditResourceType.CUSTOM_BLOCK, + resourceId: id, + resourceName: ctx.name, + description: `Unpublished custom block "${ctx.name}"`, + metadata: { organizationId: ctx.organizationId, type: ctx.type }, + request, + }) return NextResponse.json({ success: true as const }) }) diff --git a/apps/sim/app/api/custom-blocks/route.ts b/apps/sim/app/api/custom-blocks/route.ts index 12071007881..99b6319aa3d 100644 --- a/apps/sim/app/api/custom-blocks/route.ts +++ b/apps/sim/app/api/custom-blocks/route.ts @@ -1,3 +1,4 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { NextRequest } from 'next/server' @@ -119,6 +120,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => { inputs, exposedOutputs, }) + recordAudit({ + workspaceId, + actorId: userId, + actorName: session.user.name, + actorEmail: session.user.email, + action: AuditAction.CUSTOM_BLOCK_PUBLISHED, + resourceType: AuditResourceType.CUSTOM_BLOCK, + resourceId: block.id, + resourceName: block.name, + description: `Published custom block "${block.name}"`, + metadata: { organizationId, type: block.type, workflowId }, + request, + }) return NextResponse.json({ customBlock: toWire(block) }) } catch (error) { if (error instanceof CustomBlockValidationError) { diff --git a/apps/sim/lib/workflows/custom-blocks/operations.ts b/apps/sim/lib/workflows/custom-blocks/operations.ts index a6732998c30..83df77525ca 100644 --- a/apps/sim/lib/workflows/custom-blocks/operations.ts +++ b/apps/sim/lib/workflows/custom-blocks/operations.ts @@ -203,13 +203,18 @@ export async function getCustomBlockById(id: string) { * (or an org admin, who holds admin on every org workspace) can change its outputs. * `null` when no block matches. */ -export async function getCustomBlockManageContext( - id: string -): Promise<{ organizationId: string; sourceWorkspaceId: string | null } | null> { +export async function getCustomBlockManageContext(id: string): Promise<{ + organizationId: string + sourceWorkspaceId: string | null + type: string + name: string +} | null> { const [row] = await db .select({ organizationId: customBlock.organizationId, sourceWorkspaceId: workflow.workspaceId, + type: customBlock.type, + name: customBlock.name, }) .from(customBlock) .innerJoin(workflow, eq(workflow.id, customBlock.workflowId)) diff --git a/packages/audit/src/types.ts b/packages/audit/src/types.ts index 3ad3ad9dc9a..ab0818a5724 100644 --- a/packages/audit/src/types.ts +++ b/packages/audit/src/types.ts @@ -19,6 +19,11 @@ export const AuditAction = { CHAT_UPDATED: 'chat.updated', CHAT_DELETED: 'chat.deleted', + // Custom Blocks (deploy-as-block) + CUSTOM_BLOCK_PUBLISHED: 'custom_block.published', + CUSTOM_BLOCK_UPDATED: 'custom_block.updated', + CUSTOM_BLOCK_DELETED: 'custom_block.deleted', + // Custom Tools CUSTOM_TOOL_CREATED: 'custom_tool.created', CUSTOM_TOOL_UPDATED: 'custom_tool.updated', @@ -188,6 +193,7 @@ export const AuditResourceType = { CHAT: 'chat', CONNECTOR: 'connector', CREDENTIAL: 'credential', + CUSTOM_BLOCK: 'custom_block', CUSTOM_TOOL: 'custom_tool', DATA_DRAIN: 'data_drain', DOCUMENT: 'document', diff --git a/packages/testing/src/mocks/audit.mock.ts b/packages/testing/src/mocks/audit.mock.ts index 82a59734b8f..65685d9237d 100644 --- a/packages/testing/src/mocks/audit.mock.ts +++ b/packages/testing/src/mocks/audit.mock.ts @@ -47,6 +47,9 @@ export const auditMock = { CREDENTIAL_MEMBER_REMOVED: 'credential_member.removed', CREDENTIAL_MEMBER_ROLE_CHANGED: 'credential_member.role_changed', CREDIT_PURCHASED: 'credit.purchased', + CUSTOM_BLOCK_PUBLISHED: 'custom_block.published', + CUSTOM_BLOCK_UPDATED: 'custom_block.updated', + CUSTOM_BLOCK_DELETED: 'custom_block.deleted', CUSTOM_TOOL_CREATED: 'custom_tool.created', CUSTOM_TOOL_UPDATED: 'custom_tool.updated', CUSTOM_TOOL_DELETED: 'custom_tool.deleted', @@ -156,6 +159,7 @@ export const auditMock = { CHAT: 'chat', CONNECTOR: 'connector', CREDENTIAL: 'credential', + CUSTOM_BLOCK: 'custom_block', CUSTOM_TOOL: 'custom_tool', DATA_DRAIN: 'data_drain', DOCUMENT: 'document', From 60e3509811c9d44c404b9d05b32647b050b953e5 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 7 Jul 2026 16:01:09 -0700 Subject: [PATCH 3/9] fix(tables): server-authoritative run badge, tail SSE from latest, harden Stop-all (#5492) * fix(tables): server-authoritative run badge, tail SSE from latest, harden Stop-all * fix(tables): return null dispatchId when Stop-all cancels a run during prep * improvement(tables): co-locate dispatcher imports, stamp throttle clock only on fetch start * fix(tables): table-wide hasRunning signal for Queueing label, fail fast on seq-read errors * fix(tables): clear hasRunning on table-wide stop, refetch rows when a run resolves without a dispatch * fix(tables): don't let dispatch-cancel cleanup mask the original prep failure * fix(tables): reconcile null-dispatch runs via invalidation, not stale snapshot restore * fix(tables): widen warm-cache remount check to either query cache --- .../table/[tableId]/dispatches/route.test.ts | 129 +++++++++++ .../api/table/[tableId]/dispatches/route.ts | 13 +- .../table/[tableId]/events/stream/route.ts | 20 +- .../run-status-control/run-status-control.tsx | 17 +- .../table-grid/cells/cell-render.tsx | 78 ++++--- .../components/table-grid/table-grid.tsx | 39 +++- .../[tableId]/hooks/use-table-event-stream.ts | 212 ++++++++++-------- .../[workspaceId]/tables/[tableId]/table.tsx | 3 + apps/sim/hooks/queries/tables.ts | 126 ++++++----- apps/sim/lib/api/contracts/tables.test.ts | 24 ++ apps/sim/lib/api/contracts/tables.ts | 26 ++- apps/sim/lib/table/dispatcher.ts | 117 +++++----- apps/sim/lib/table/events.test.ts | 78 +++++++ apps/sim/lib/table/events.ts | 27 +++ apps/sim/lib/table/workflow-columns.ts | 182 +++++++++------ 15 files changed, 765 insertions(+), 326 deletions(-) create mode 100644 apps/sim/app/api/table/[tableId]/dispatches/route.test.ts create mode 100644 apps/sim/lib/api/contracts/tables.test.ts create mode 100644 apps/sim/lib/table/events.test.ts diff --git a/apps/sim/app/api/table/[tableId]/dispatches/route.test.ts b/apps/sim/app/api/table/[tableId]/dispatches/route.test.ts new file mode 100644 index 00000000000..3acddbbe325 --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/dispatches/route.test.ts @@ -0,0 +1,129 @@ +/** + * @vitest-environment node + */ +import { hybridAuthMockFns } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table' + +const { mockCheckAccess, mockListActiveDispatches, mockCountRunningCells } = vi.hoisted(() => ({ + mockCheckAccess: vi.fn(), + mockListActiveDispatches: vi.fn(), + mockCountRunningCells: vi.fn(), +})) + +vi.mock('@/lib/table/dispatcher', () => ({ + listActiveDispatches: mockListActiveDispatches, + countRunningCells: mockCountRunningCells, +})) +vi.mock('@/app/api/table/utils', async () => { + const { NextResponse } = await import('next/server') + return { + checkAccess: mockCheckAccess, + accessError: (result: { status: number }) => + NextResponse.json({ error: 'denied' }, { status: result.status }), + } +}) + +import { GET } from '@/app/api/table/[tableId]/dispatches/route' + +function buildTable(overrides: Partial = {}): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: null, + schema: { columns: [] }, + metadata: null, + rowCount: 0, + maxRows: 1_000_000, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date(), + updatedAt: new Date(), + ...overrides, + } +} + +function makeRequest(tableId = 'tbl_1') { + const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/dispatches`) + return GET(req, { params: Promise.resolve({ tableId }) }) +} + +function buildDispatchRow(overrides: Record = {}) { + return { + id: 'dispatch-1', + tableId: 'tbl_1', + workspaceId: 'workspace-1', + requestId: 'req-1', + mode: 'all', + scope: { groupIds: ['group-1'] }, + status: 'dispatching', + cursor: 4, + limit: null, + isManualRun: true, + processedCount: 5, + ...overrides, + } +} + +describe('GET /api/table/[tableId]/dispatches', () => { + beforeEach(() => { + vi.clearAllMocks() + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType: 'session', + }) + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockListActiveDispatches.mockResolvedValue([]) + mockCountRunningCells.mockResolvedValue({ byRowId: {}, hasRunning: false }) + }) + + it('returns dispatches and the per-row running map, without a total field', async () => { + mockListActiveDispatches.mockResolvedValue([buildDispatchRow()]) + mockCountRunningCells.mockResolvedValue({ + byRowId: { 'row-1': 2, 'row-2': 1 }, + hasRunning: true, + }) + + const response = await makeRequest() + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.data.dispatches).toEqual([ + { + id: 'dispatch-1', + status: 'dispatching', + mode: 'all', + isManualRun: true, + cursor: 4, + scope: { groupIds: ['group-1'] }, + }, + ]) + expect(data.data.runningByRowId).toEqual({ 'row-1': 2, 'row-2': 1 }) + expect(data.data.hasRunning).toBe(true) + expect(data.data).not.toHaveProperty('runningCellCount') + }) + + it('includes unclaimed pre-stamps only while a dispatch is active', async () => { + mockListActiveDispatches.mockResolvedValue([buildDispatchRow()]) + await makeRequest() + expect(mockCountRunningCells).toHaveBeenCalledWith('tbl_1', { + includeUnclaimedPreStamps: true, + }) + + mockListActiveDispatches.mockResolvedValue([]) + await makeRequest() + expect(mockCountRunningCells).toHaveBeenLastCalledWith('tbl_1', { + includeUnclaimedPreStamps: false, + }) + }) + + it('returns 401 when unauthenticated', async () => { + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ success: false }) + const response = await makeRequest() + expect(response.status).toBe(401) + expect(mockListActiveDispatches).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/table/[tableId]/dispatches/route.ts b/apps/sim/app/api/table/[tableId]/dispatches/route.ts index 25b6f871649..a39d3409402 100644 --- a/apps/sim/app/api/table/[tableId]/dispatches/route.ts +++ b/apps/sim/app/api/table/[tableId]/dispatches/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { countActiveRunCells, listActiveDispatches } from '@/lib/table/dispatcher' +import { countRunningCells, listActiveDispatches } from '@/lib/table/dispatcher' import { accessError, checkAccess } from '@/app/api/table/utils' const logger = createLogger('TableDispatchesAPI') @@ -38,7 +38,12 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Rou if (!result.ok) return accessError(result, requestId, tableId) const rows = await listActiveDispatches(tableId) - const running = await countActiveRunCells(tableId, rows) + // Unclaimed `pending` pre-stamps are real queued work while a dispatch is + // active; with none active they're abandoned orphans that would pin the + // "X running" badge above zero forever. + const { byRowId: runningByRowId, hasRunning } = await countRunningCells(tableId, { + includeUnclaimedPreStamps: rows.length > 0, + }) const dispatches: ActiveDispatch[] = rows.map((r) => ({ id: r.id, status: r.status as 'pending' | 'dispatching', @@ -53,8 +58,8 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Rou success: true, data: { dispatches, - runningCellCount: running.total, - runningByRowId: running.byRowId, + runningByRowId, + hasRunning, }, }) } catch (error) { diff --git a/apps/sim/app/api/table/[tableId]/events/stream/route.ts b/apps/sim/app/api/table/[tableId]/events/stream/route.ts index d938c80c3ec..2e9cff2eb37 100644 --- a/apps/sim/app/api/table/[tableId]/events/stream/route.ts +++ b/apps/sim/app/api/table/[tableId]/events/stream/route.ts @@ -8,7 +8,11 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { SSE_HEADERS } from '@/lib/core/utils/sse' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { readTableEventsSince, type TableEventEntry } from '@/lib/table/events' +import { + getLatestTableEventId, + readTableEventsSince, + type TableEventEntry, +} from '@/lib/table/events' import { accessError, checkAccess } from '@/app/api/table/utils' const logger = createLogger('TableEventStreamAPI') @@ -26,10 +30,12 @@ interface RouteContext { /** GET /api/table/[tableId]/events/stream?from= * - * SSE stream of cell-state transitions. Replay-on-reconnect via `from`. + * SSE stream of cell-state transitions. Replay-on-reconnect via `from`; + * absent `from` tails from the latest event id (fresh mount — the client has + * just fetched current state, so replaying history would rewind it). * Pruning (buffer cap exceeded or TTL expired) sends a `pruned` event and * closes; the client responds with a full row-query refetch and reconnects - * from the new earliest. */ + * tailing from latest. */ export const GET = withRouteHandler(async (req: NextRequest, context: RouteContext) => { const requestId = generateRequestId() const parsed = await parseRequest(tableEventStreamContract, req, context) @@ -52,7 +58,7 @@ export const GET = withRouteHandler(async (req: NextRequest, context: RouteConte const stream = new ReadableStream({ async start(controller) { - let lastEventId = fromEventId + let lastEventId = fromEventId ?? 0 const deadline = Date.now() + MAX_STREAM_DURATION_MS let nextHeartbeatAt = Date.now() + HEARTBEAT_INTERVAL_MS @@ -92,6 +98,12 @@ export const GET = withRouteHandler(async (req: NextRequest, context: RouteConte } try { + // No replay cursor → tail from the latest event id. Resolved inside + // the try so a Redis failure errors the stream (client reconnects + // with backoff) rather than silently replaying the whole buffer. + if (fromEventId === undefined) { + lastEventId = await getLatestTableEventId(tableId) + } // Initial replay from buffer. const initial = await readTableEventsSince(tableId, lastEventId) if (initial.status === 'pruned') { diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/run-status-control/run-status-control.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/run-status-control/run-status-control.tsx index 3ef2db128c7..9a834d80e57 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/run-status-control/run-status-control.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/run-status-control/run-status-control.tsx @@ -6,6 +6,10 @@ import { Loader, Square } from '@sim/emcn/icons' interface RunStatusControlProps { running: number + /** No cell has been claimed by a worker yet — everything counted is queued + * or pending, so labeling it "running" would be dishonest. Renders + * "Queueing" without a count instead. */ + queueing: boolean onStopAll: () => void isStopping: boolean } @@ -17,6 +21,7 @@ interface RunStatusControlProps { */ export const RunStatusControl = memo(function RunStatusControl({ running, + queueing, onStopAll, isStopping, }: RunStatusControlProps) { @@ -24,8 +29,14 @@ export const RunStatusControl = memo(function RunStatusControl({
- {running} - running + {queueing ? ( + Queueing + ) : ( + <> + {running} + running + + )}
) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx index be7e6046336..b5529af4e61 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx @@ -15,7 +15,7 @@ export type CellRenderKind = | { kind: 'value'; text: string } | { kind: 'block-error'; message: string } | { kind: 'running' } - | { kind: 'pending-upstream' } + | { kind: 'pending-upstream'; paused: boolean } | { kind: 'queued' } | { kind: 'cancelled' } | { kind: 'error'; message: string | null } @@ -93,9 +93,9 @@ export function resolveCellRender({ exec?.status === 'pending' && typeof exec.jobId === 'string' && exec.jobId.startsWith('paused-') - if (isPaused) return { kind: 'pending-upstream' } + if (isPaused) return { kind: 'pending-upstream', paused: true } if (exec?.status === 'queued' || exec?.status === 'pending') return { kind: 'queued' } - return { kind: 'pending-upstream' } + return { kind: 'pending-upstream', paused: false } } // Waiting wins over a stale terminal status — show the actionable state. @@ -274,48 +274,55 @@ export function CellRender({ kind, isEditing }: CellRenderProps): React.ReactEle case 'running': return ( - + + + ) case 'pending-upstream': return ( - + + + ) case 'cancelled': return ( - + + + ) case 'queued': return ( - - Queued - + + + Queued + + ) case 'waiting': return ( - - - - - Waiting - - - - - Waiting on {kind.labels.map((l) => `"${l}"`).join(', ')} - - + `"${l}"`).join(', ')}`}> + + Waiting + + ) @@ -406,18 +413,22 @@ export function CellRender({ kind, isEditing }: CellRenderProps): React.ReactEle case 'not-found': return ( - - Not found - + + + Not found + + ) case 'no-output': return ( - - No output - + + + No output + + ) @@ -436,6 +447,19 @@ function Wrap({ isEditing, children }: { isEditing: boolean; children: React.Rea return
{children}
} +/** Hover explanation for a status badge. The `` trigger is required — + * Badge/StatusBadge don't forward refs, so the tooltip anchors to a wrapper. */ +function BadgeTooltip({ tip, children }: { tip: string; children: React.ReactNode }) { + return ( + + + {children} + + {tip} + + ) +} + const TYPEWRITER_MS_PER_CHAR = 15 /** diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index 125418b5f45..8bc8c2f0fb7 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -89,6 +89,11 @@ export interface SelectionSnapshot { /** Total running/queued workflow runs across ALL rows. Drives the page-header * RunStatusControl ("N running, Stop all"). */ totalRunning: number + /** Whether any LOADED cell has actually been claimed by a worker + * (`status === 'running'`). False while the in-flight set is all + * queued/pending stamps — the header control then reads "Queueing" + * instead of labeling queued work as running. */ + hasRunningCell: boolean /** Whether any dispatch is active (pending/dispatching). Keeps the RunStatusControl * + Stop-all visible during a run even when the per-row count momentarily reads 0 * (e.g. the first window of an auto-fired/capped dispatch before cells stamp). */ @@ -397,13 +402,27 @@ export function TableGrid({ const { data: tableRunState } = useTableRunState(tableId) const activeDispatches = tableRunState?.dispatches const runningByRowId = tableRunState?.runningByRowId ?? EMPTY_RUNNING_BY_ROW - // Actual in-flight cell count = sum of the live per-row map (kept current by - // applyCell's SSE deltas, and the same source the per-row gutter uses). The - // dispatch-scope `runningCellCount` over-counts already-completed groups on - // rows still inside a dispatch's scope — e.g. a cascade where 3 of 4 columns - // finished would read "4 running" instead of "1". + // In-flight cell count = sum of the server-derived per-row map (refetched on + // a throttle as cell SSE events arrive, stamped optimistically on run-click + // — the same source the per-row gutter uses). const totalRunning = Object.values(runningByRowId).reduce((sum, n) => sum + n, 0) const hasActiveDispatch = (activeDispatches?.length ?? 0) > 0 + // Claimed-cell signal for the "Queueing" vs "N running" label. Two sources + // OR'd: the loaded-rows scan flips instantly via SSE claim events, and the + // server's table-wide flag covers runs whose active window has scrolled + // past the loaded pages (where the scan alone would read "Queueing" for the + // rest of a long run). + const hasRunningLoaded = useMemo(() => { + for (const row of rows) { + const executions = row.executions + if (!executions) continue + for (const exec of Object.values(executions)) { + if (exec?.status === 'running') return true + } + } + return false + }, [rows]) + const hasRunningCell = hasRunningLoaded || (tableRunState?.hasRunning ?? false) // True "select all" total: the filter-scoped COUNT(*) when a filter is active, else the whole // table. Drives the delete-confirm count and the action-bar cell count. @@ -3282,10 +3301,9 @@ export function TableGrid({ }, [rowSelection, rows]) // `runningByRowId` + `totalRunning` come from `useTableRunState` above — - // backend-bootstrapped via `countRunningCells` and kept live by - // `applyCell`'s SSE-driven delta. Counts only cells whose worker has - // actually claimed the cell (`status === 'running'`), ignoring optimistic - // queued/pending stamps. + // server-derived via `countRunningCells` (queued/running/pending), refetched + // on a throttle as cell SSE events arrive, plus optimistic stamps on + // run-click. // Context-menu wrappers: act on `contextMenuRowIds`, then close the menu. // Mirror the action bar's Play / Refresh split: Play fills empty/failed, @@ -3523,6 +3541,7 @@ export function TableGrid({ sameStats && prev.runningInActionBarSelection === runningInActionBarSelection && prev.totalRunning === totalRunning && + prev.hasRunningCell === hasRunningCell && prev.hasActiveDispatch === hasActiveDispatch && prev.hasWorkflowColumns === hasWorkflowColumns && prev.actionBarRowIds.length === actionBarRowIds.length && @@ -3534,6 +3553,7 @@ export function TableGrid({ actionBarRowIds, runningInActionBarSelection, totalRunning, + hasRunningCell, hasActiveDispatch, hasWorkflowColumns, selectedRunScope, @@ -3546,6 +3566,7 @@ export function TableGrid({ actionBarRowIds, runningInActionBarSelection, totalRunning, + hasRunningCell, hasActiveDispatch, hasWorkflowColumns, selectedRunScope, diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts index 45200624ff8..65416e113d6 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts @@ -3,10 +3,10 @@ import { useEffect, useRef } from 'react' import { toast } from '@sim/emcn' import { createLogger } from '@sim/logger' +import { backoffWithJitter } from '@sim/utils/retry' import { useQueryClient } from '@tanstack/react-query' import type { ActiveDispatch } from '@/lib/api/contracts/tables' import type { RowData, RowExecutionMetadata, RowExecutions, TableDefinition } from '@/lib/table' -import { isExecInFlight } from '@/lib/table/deps' import type { TableEvent, TableEventEntry } from '@/lib/table/events' import { consumeInitiatedExport, @@ -22,30 +22,10 @@ interface PrunedEvent { earliestEventId: number | null } -const RECONNECT_BACKOFF_MS = [500, 1_000, 2_000, 5_000, 10_000] -const POINTER_PREFIX = 'table-event-stream-pointer:' -const DISPATCH_INVALIDATE_DEBOUNCE_MS = 250 - -function loadPointer(tableId: string): number { - if (typeof window === 'undefined') return 0 - try { - const raw = window.sessionStorage.getItem(`${POINTER_PREFIX}${tableId}`) - if (!raw) return 0 - const parsed = Number.parseInt(raw, 10) - return Number.isFinite(parsed) && parsed >= 0 ? parsed : 0 - } catch { - return 0 - } -} - -function savePointer(tableId: string, eventId: number): void { - if (typeof window === 'undefined') return - try { - window.sessionStorage.setItem(`${POINTER_PREFIX}${tableId}`, String(eventId)) - } catch { - // sessionStorage can throw under quota / private mode — ignore. - } -} +const RECONNECT_BACKOFF_BASE_MS = 500 +const RECONNECT_BACKOFF_MAX_MS = 10_000 +const RUN_STATE_REFETCH_THROTTLE_MS = 1_000 +const ROWS_INVALIDATE_DEBOUNCE_MS = 250 interface UseTableEventStreamArgs { tableId: string | undefined @@ -60,10 +40,13 @@ interface UseTableEventStreamArgs { * Subscribes to the table's SSE event stream and patches the React Query * cache as cell-state events arrive. * - * Reconnect-resume: on transport error, reconnects with `from=` set to the - * last seen `eventId`; server replays missed events from the Redis-backed - * buffer. If the gap exceeds buffer retention (server emits `pruned`), the - * hook full-refetches the row queries and resumes from the new earliest. + * Fresh mount tails from the latest event — the rows + run-state queries + * fetch current state from the DB, so replaying buffered history would only + * rewind fresh cells through stale intermediate states (queued → running → + * completed churn). Reconnect-resume: on transport error, reconnects with + * `from=` set to the last seen `eventId`; server replays missed events from + * the Redis-backed buffer. If the gap exceeds buffer retention (server emits + * `pruned`), the hook full-refetches and resumes tailing from latest. */ export function useTableEventStream({ tableId, @@ -83,20 +66,73 @@ export function useTableEventStream({ let cancelled = false let eventSource: EventSource | null = null let reconnectTimer: ReturnType | null = null - // Resume from the last seen eventId persisted in sessionStorage. Survives - // tab refresh; if the buffer has rolled past this id the server replies - // `pruned` and we full-refetch + restart from the new earliest. - let lastEventId = loadPointer(tableId) + // `null` = no cursor yet: connect without `from` and tail from latest. + // Advanced in memory per event; within-session reconnects resume from it. + let lastEventId: number | null = null let reconnectAttempt = 0 - // Trailing-edge debounce coalesces window-completion bursts. - let dispatchInvalidateTimer: ReturnType | null = null + // Leading + trailing throttle for run-state refetches. Cell/dispatch SSE + // events arrive in bursts (the server flushes its buffer every 500ms): the + // leading edge keeps the badge stepping promptly on sporadic completions; + // the trailing timer coalesces a burst into one refetch per interval. A + // debounce would starve here — sustained bursts reset it indefinitely. + let runStateInvalidateTimer: ReturnType | null = null + let lastRunStateInvalidateAt = 0 + let runStateFetchInFlight = false + let runStateDirtyDuringFetch = false + const invalidateRunState = async (): Promise => { + // cancelRefetch: false — the default (true) cancels an in-flight refetch + // and restarts it. When the run-state fetch is slower than the throttle + // interval (a busy run congests the server), that livelocks: every + // interval kills the previous fetch before it can land and the badge + // freezes on the last value that ever resolved. Instead, let an + // in-flight fetch complete (slightly stale counts land), remember that + // events arrived meanwhile, and run one follow-up afterwards — without + // the follow-up, a run's final events deduping into a stale fetch would + // freeze the badge non-zero forever. + if (runStateFetchInFlight) { + runStateDirtyDuringFetch = true + return + } + // Stamped only when a fetch actually starts — the coalesced path above + // must not reset the throttle clock, or it delays the follow-up. + lastRunStateInvalidateAt = Date.now() + runStateFetchInFlight = true + try { + await queryClient.invalidateQueries( + { queryKey: tableKeys.activeDispatches(tableId) }, + { cancelRefetch: false } + ) + } finally { + runStateFetchInFlight = false + if (runStateDirtyDuringFetch) { + runStateDirtyDuringFetch = false + scheduleDispatchInvalidate() + } + } + } const scheduleDispatchInvalidate = (): void => { - if (dispatchInvalidateTimer !== null) clearTimeout(dispatchInvalidateTimer) - dispatchInvalidateTimer = setTimeout(() => { - dispatchInvalidateTimer = null - void queryClient.invalidateQueries({ queryKey: tableKeys.activeDispatches(tableId) }) - }, DISPATCH_INVALIDATE_DEBOUNCE_MS) + if (cancelled || runStateInvalidateTimer !== null) return + const elapsed = Date.now() - lastRunStateInvalidateAt + if (elapsed >= RUN_STATE_REFETCH_THROTTLE_MS) { + void invalidateRunState() + return + } + runStateInvalidateTimer = setTimeout(() => { + runStateInvalidateTimer = null + void invalidateRunState() + }, RUN_STATE_REFETCH_THROTTLE_MS - elapsed) + } + /** Urgent resync (usage-limit halt, prune recovery) — skips the throttle. + * Default cancelRefetch here: a fetch started before the halt is stale by + * definition, so kill it and read fresh. One-shot, so no churn risk. */ + const invalidateDispatchesNow = (): void => { + if (runStateInvalidateTimer !== null) { + clearTimeout(runStateInvalidateTimer) + runStateInvalidateTimer = null + } + lastRunStateInvalidateAt = Date.now() + void queryClient.invalidateQueries({ queryKey: tableKeys.activeDispatches(tableId) }) } // Live-fill: import progress ticks arrive every N rows; coalesce the row @@ -107,25 +143,7 @@ export function useTableEventStream({ jobInvalidateTimer = setTimeout(() => { jobInvalidateTimer = null void queryClient.invalidateQueries({ queryKey: tableKeys.rowsRoot(tableId) }) - }, DISPATCH_INVALIDATE_DEBOUNCE_MS) - } - - // Keeps the per-row gutter (`runningByRowId`) live between dispatch events. - // `runningCellCount` (the "X running" badge) is NOT touched here — it's the - // server's dispatch-scope count, seeded optimistically on click and - // re-synced by `applyDispatch` on every window, so live matches reload. - const updateRunningByRow = (rowId: string, wasInFlight: boolean, isInFlight: boolean): void => { - if (wasInFlight === isInFlight) return - const delta = isInFlight ? 1 : -1 - queryClient.setQueryData(tableKeys.activeDispatches(tableId), (prev) => { - if (!prev) return prev - const prevForRow = prev.runningByRowId[rowId] ?? 0 - const nextForRow = Math.max(0, prevForRow + delta) - const nextByRow = { ...prev.runningByRowId } - if (nextForRow === 0) delete nextByRow[rowId] - else nextByRow[rowId] = nextForRow - return { ...prev, runningByRowId: nextByRow } - }) + }, ROWS_INVALIDATE_DEBOUNCE_MS) } const applyCell = (event: Extract): void => { @@ -140,18 +158,12 @@ export function useTableEventStream({ runningBlockIds, blockErrors, } = event - let wasInFlight: boolean | null = null void snapshotAndMutateRows( queryClient, tableId, (row) => { if (row.id !== rowId) return null const prevExec = row.executions?.[groupId] - // In-flight = queued | running | pending. Server's countRunningCells - // counts all three (the gutter Run/Stop button reads this map and - // needs Stop visible during queued too, else clicking Play would - // re-enqueue a cell that's already queued). - if (wasInFlight === null) wasInFlight = isExecInFlight(prevExec) const nextExec: RowExecutionMetadata = { status, executionId: executionId ?? null, @@ -171,13 +183,11 @@ export function useTableEventStream({ }, { cancelInFlight: false } ) - if (wasInFlight === null) { - // Row outside the loaded page slice — can't compute the delta locally. - // Refetch the run-state snapshot from the server. Cheap and rare. - scheduleDispatchInvalidate() - } else { - updateRunningByRow(rowId, wasInFlight, isExecInFlight({ status } as RowExecutionMetadata)) - } + // `runningByRowId` (the "X running" badge + per-row gutter) is + // server-derived: refetch the snapshot on the throttle instead of + // maintaining client-side ±1 deltas, which drift on unloaded rows, + // replays, and races with optimistic stamps. + scheduleDispatchInvalidate() } const applyDispatch = (event: Extract): void => { @@ -185,11 +195,11 @@ export function useTableEventStream({ queryClient.setQueryData(tableKeys.activeDispatches(tableId), (prev) => { // SSE may arrive before the initial fetch lands. Seed an empty // run-state so the dispatch isn't dropped; counters are reconciled - // by the subsequent fetch / per-cell SSE events. + // by the subsequent fetch. const base: TableRunState = prev ?? { dispatches: [], - runningCellCount: 0, runningByRowId: {}, + hasRunning: false, } const list = base.dispatches // Terminal states drop the dispatch from the overlay; client renders @@ -225,9 +235,9 @@ export function useTableEventStream({ return { ...base, dispatches: merged } }) // The dispatcher emits this once per window (after the window's cells - // finish + the cursor advances) and on completion. Re-sync the - // dispatch-scope `runningCellCount` from the server so the badge steps - // down per window and matches a reload exactly. + // finish + the cursor advances) and on completion. Re-sync + // `runningByRowId` from the server so the badge steps down per window + // and matches a reload exactly. scheduleDispatchInvalidate() } @@ -304,9 +314,11 @@ export function useTableEventStream({ } // Blocked cells are left `queued` in the DB with no terminal cell event, // so `runningByRowId` would otherwise stay non-zero (stale "X running"). - // Re-sync the server counts, and refetch rows so cells whose pre-stamps - // the server cleared drop their "Queued" state. - scheduleDispatchInvalidate() + // Re-sync the server counts immediately (the user is being told they're + // over limit — the badge must not linger behind the throttle), and + // refetch rows so cells whose pre-stamps the server cleared drop their + // "Queued" state. + invalidateDispatchesNow() void queryClient.invalidateQueries({ queryKey: tableKeys.rowsRoot(tableId) }) onUsageLimitReachedRef.current?.({ dispatchId: event.dispatchId, message: event.message }) } @@ -314,9 +326,10 @@ export function useTableEventStream({ const handlePrune = (payload: PrunedEvent): void => { logger.info('Table event buffer pruned — full refetch', { tableId, ...payload }) void queryClient.invalidateQueries({ queryKey: tableKeys.rowsRoot(tableId) }) - scheduleDispatchInvalidate() - lastEventId = typeof payload.earliestEventId === 'number' ? payload.earliestEventId : 0 - savePointer(tableId, lastEventId) + invalidateDispatchesNow() + // Tail from latest after the refetch — replaying the surviving buffer + // over freshly-refetched rows would rewind them through stale states. + lastEventId = null // Close proactively so the server's close doesn't fire onerror and route // through the backoff path. Reconnect immediately from the new cursor. eventSource?.close() @@ -327,9 +340,11 @@ export function useTableEventStream({ const scheduleReconnect = (): void => { if (cancelled) return - const idx = Math.min(reconnectAttempt, RECONNECT_BACKOFF_MS.length - 1) - const delay = RECONNECT_BACKOFF_MS[idx] reconnectAttempt++ + const delay = backoffWithJitter(reconnectAttempt, null, { + baseMs: RECONNECT_BACKOFF_BASE_MS, + maxMs: RECONNECT_BACKOFF_MAX_MS, + }) reconnectTimer = setTimeout(() => { reconnectTimer = null connect() @@ -338,7 +353,11 @@ export function useTableEventStream({ const connect = (): void => { if (cancelled) return - const url = `/api/table/${tableId}/events/stream?from=${lastEventId}` + // No cursor → tail from latest (server-side); otherwise replay-resume. + const url = + lastEventId === null + ? `/api/table/${tableId}/events/stream` + : `/api/table/${tableId}/events/stream?from=${lastEventId}` try { eventSource = new EventSource(url) } catch (err) { @@ -354,9 +373,8 @@ export function useTableEventStream({ eventSource.onmessage = (msg: MessageEvent) => { try { const entry = JSON.parse(msg.data) as TableEventEntry - if (entry.eventId <= lastEventId) return + if (lastEventId !== null && entry.eventId <= lastEventId) return lastEventId = entry.eventId - savePointer(tableId, lastEventId) if (entry.event?.kind === 'cell') applyCell(entry.event) else if (entry.event?.kind === 'dispatch') applyDispatch(entry.event) else if (entry.event?.kind === 'job') applyJob(entry.event) @@ -388,12 +406,28 @@ export function useTableEventStream({ } } + // In-SPA remount over a warm cache (table A → B → back to A within + // staleTime): the tail starts at "latest", so transitions that fired while + // unmounted were neither refetched (cache still fresh) nor replayed. + // Reconcile once — either cache being warm is enough (they can evict + // independently). Cold mounts have neither → skip, the queries are + // already fetching. + const hasWarmRunState = + queryClient.getQueryState(tableKeys.activeDispatches(tableId))?.data !== undefined + const hasWarmRows = queryClient + .getQueriesData({ queryKey: tableKeys.rowsRoot(tableId) }) + .some(([, data]) => data !== undefined) + if (hasWarmRunState || hasWarmRows) { + void queryClient.invalidateQueries({ queryKey: tableKeys.rowsRoot(tableId) }) + void invalidateRunState() + } + connect() return () => { cancelled = true if (reconnectTimer !== null) clearTimeout(reconnectTimer) - if (dispatchInvalidateTimer !== null) clearTimeout(dispatchInvalidateTimer) + if (runStateInvalidateTimer !== null) clearTimeout(runStateInvalidateTimer) if (jobInvalidateTimer !== null) clearTimeout(jobInvalidateTimer) eventSource?.close() eventSource = null diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx index 199ec81ed82..0f7485636bb 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/table.tsx @@ -167,6 +167,7 @@ export function Table({ actionBarRowIds: [], runningInActionBarSelection: 0, totalRunning: 0, + hasRunningCell: false, hasActiveDispatch: false, hasWorkflowColumns: false, selectedRunScope: null, @@ -657,6 +658,7 @@ export function Table({ {selection.totalRunning > 0 || selection.hasActiveDispatch ? ( @@ -686,6 +688,7 @@ export function Table({ embedded && (selection.totalRunning > 0 || selection.hasActiveDispatch) ? ( diff --git a/apps/sim/hooks/queries/tables.ts b/apps/sim/hooks/queries/tables.ts index e9e10f908dd..e24ba44f9e0 100644 --- a/apps/sim/hooks/queries/tables.ts +++ b/apps/sim/hooks/queries/tables.ts @@ -271,8 +271,9 @@ export function getTableDetailQueryOptions(workspaceId: string, tableId: string) export interface TableRunState { dispatches: ActiveDispatch[] - runningCellCount: number runningByRowId: Record + /** Any in-flight cell claimed by a worker, table-wide. */ + hasRunning: boolean } async function fetchTableRunState(tableId: string, signal?: AbortSignal): Promise { @@ -282,8 +283,8 @@ async function fetchTableRunState(tableId: string, signal?: AbortSignal): Promis }) return { dispatches: response.data.dispatches, - runningCellCount: response.data.runningCellCount, runningByRowId: response.data.runningByRowId, + hasRunning: response.data.hasRunning, } } @@ -335,46 +336,28 @@ function countNewlyInFlight(before: RowExecutions, after: RowExecutions): number return n } -/** The table's maintained, unfiltered `rowCount` from the detail cache (or - * `null` when the detail hasn't loaded). This is the right scope for a Run-all - * estimate: the dispatcher runs every row regardless of the active view - * filter, whereas the rows query's `totalCount` is filter-scoped. */ -function readTableRowCount( - queryClient: ReturnType, - tableId: string -): number | null { - const def = queryClient.getQueryData(tableKeys.detail(tableId)) - return typeof def?.rowCount === 'number' ? def.rowCount : null -} - /** Optimistically reflect a run on the "X running" badge + per-row gutter Stop - * instantly (the optimistic stamp eats the dispatcher's `pending` SSE, so - * `applyCell` never bumps the count, and the server's dispatch-scope count - * isn't live until the first window). `stampedByRow` drives the per-row gutter - * (loaded rows only); `cellCountDelta` is the badge delta — pass the full run - * scope (rows × groups) for Run-all so it matches the server, or omit to use - * the stamped total. Returns the prior snapshot for rollback. */ -function bumpRunState( + * instantly, ahead of the dispatcher's real pending stamps and the next + * server snapshot refetch. Cancels any in-flight run-state fetch first — a + * fetch started before the click would otherwise resolve after this write + * and clobber the bump back to the pre-run snapshot. Returns the prior + * snapshot for rollback. */ +async function bumpRunState( queryClient: ReturnType, tableId: string, - stampedByRow: Record, - cellCountDelta?: number -): { snapshot: TableRunState | undefined } | null { + stampedByRow: Record +): Promise<{ snapshot: TableRunState | undefined } | null> { const stampedTotal = Object.values(stampedByRow).reduce((s, n) => s + n, 0) - const countDelta = cellCountDelta ?? stampedTotal - if (countDelta === 0 && stampedTotal === 0) return null + if (stampedTotal === 0) return null + await queryClient.cancelQueries({ queryKey: tableKeys.activeDispatches(tableId) }) const snapshot = queryClient.getQueryData(tableKeys.activeDispatches(tableId)) queryClient.setQueryData(tableKeys.activeDispatches(tableId), (prev) => { - const base = prev ?? { dispatches: [], runningCellCount: 0, runningByRowId: {} } + const base = prev ?? { dispatches: [], runningByRowId: {}, hasRunning: false } const nextByRow = { ...base.runningByRowId } for (const [rid, n] of Object.entries(stampedByRow)) { nextByRow[rid] = (nextByRow[rid] ?? 0) + n } - return { - ...base, - runningCellCount: base.runningCellCount + countDelta, - runningByRowId: nextByRow, - } + return { ...base, runningByRowId: nextByRow } }) return { snapshot } } @@ -647,7 +630,7 @@ export function useCreateTableRow({ workspaceId, tableId }: RowMutationContext) }, }) }, - onSuccess: (response) => { + onSuccess: async (response) => { const row = response.data.row if (!row) return @@ -660,7 +643,7 @@ export function useCreateTableRow({ workspaceId, tableId }: RowMutationContext) // the "X running" badge + gutter Stop show immediately (the row had no // prior executions, so the stamped set is the full delta). const stampedCount = countNewlyInFlight({}, stamped.executions ?? {}) - if (stampedCount > 0) bumpRunState(queryClient, tableId, { [row.id]: stampedCount }) + if (stampedCount > 0) await bumpRunState(queryClient, tableId, { [row.id]: stampedCount }) // `reconcileCreatedRow` only patches the default-order view. Filtered / // column-sorted rows queries can't be reconciled from that heuristic @@ -904,7 +887,7 @@ export function useUpdateTableRow({ workspaceId, tableId }: RowMutationContext) } }) - const bumped = bumpRunState(queryClient, tableId, stampedByRow) + const bumped = await bumpRunState(queryClient, tableId, stampedByRow) return { previousQueries, runStateSnapshot: bumped?.snapshot, @@ -995,7 +978,7 @@ export function useBatchUpdateTableRows({ workspaceId, tableId }: RowMutationCon } }) - const bumped = bumpRunState(queryClient, tableId, stampedByRow) + const bumped = await bumpRunState(queryClient, tableId, stampedByRow) return { previousQueries, runStateSnapshot: bumped?.snapshot, @@ -1319,6 +1302,7 @@ export function useCancelTableRuns({ workspaceId, tableId }: RowMutationContext) }) ) : undefined + const touchedRowIds = new Set() const snapshots = await snapshotAndMutateRows( queryClient, tableId, @@ -1350,21 +1334,55 @@ export function useCancelTableRuns({ workspaceId, tableId }: RowMutationContext) } rowTouched = true } - return rowTouched ? { ...r, executions: nextExecutions } : null + if (!rowTouched) return null + touchedRowIds.add(r.id) + return { ...r, executions: nextExecutions } }, { onlyKey } ) - return { snapshots } + + // Zero the badge + per-row gutter for the stopped rows immediately. + // Cancel any in-flight run-state fetch first — one started before the + // server processed the cancel would resolve with stale non-zero counts + // and resurrect the badge until onSettled's refetch lands. + await queryClient.cancelQueries({ queryKey: tableKeys.activeDispatches(tableId) }) + const runStateSnapshot = queryClient.getQueryData( + tableKeys.activeDispatches(tableId) + ) + queryClient.setQueryData(tableKeys.activeDispatches(tableId), (prev) => { + if (!prev) return prev + const nextByRow: Record = {} + for (const [rid, n] of Object.entries(prev.runningByRowId)) { + if (scope === 'all' && !filter) { + // Table-wide stop: everything not explicitly excluded is cancelled, + // including rows outside the loaded page slice. + if (!excludedRowIds?.has(rid)) continue + } else if (touchedRowIds.has(rid)) { + continue + } + nextByRow[rid] = n + } + // An unexcluded table-wide stop cancels every claim, so the stale + // table-wide flag must drop with it (else the header reads "0 + // running" until onSettled refetches). Scoped stops leave other rows' + // claims running — keep it. + const hasRunning = scope === 'all' && !filter && !excludedRowIds ? false : prev.hasRunning + return { ...prev, runningByRowId: nextByRow, hasRunning } + }) + return { snapshots, runStateSnapshot } }, - onError: (_err, _variables, context) => { + onError: (error, _variables, context) => { if (context?.snapshots) restoreCachedWorkflowCells(queryClient, context.snapshots) + queryClient.setQueryData(tableKeys.activeDispatches(tableId), context?.runStateSnapshot) + // A failed Stop must be loud — the optimistic clear above made the run + // look stopped, and silently reverting reads as "the cancel didn't work". + toast.error(`Failed to stop runs: ${error.message}`, { duration: 5000 }) }, onSettled: () => { queryClient.invalidateQueries({ queryKey: tableKeys.rowsRoot(tableId) }) - // Refetch the run-state snapshot — server re-derives runningCellCount + - // runningByRowId from the freshly-updated sidecar via countRunningCells. - // Without this, the counter and row gutter button stay stale until the - // user refetches manually. + // Refetch the run-state snapshot — server re-derives runningByRowId from + // the freshly-updated sidecar via countRunningCells. Reconciles the + // optimistic clear above with whatever the cancel actually stopped. queryClient.invalidateQueries({ queryKey: tableKeys.activeDispatches(tableId) }) }, }) @@ -2062,14 +2080,7 @@ export function useRunColumn({ workspaceId, tableId }: RowMutationContext) { return { ...r, data: nextData, executions: next } }) - // Badge counts the whole run scope (rows × groups), matching the server's - // dispatch-scope count — not just the loaded rows we could stamp. For - // Run-all that's the table's totalCount; for a scoped run, the rowIds. - const scopeRowCount = targetRowIds - ? targetRowIds.size - : (readTableRowCount(queryClient, tableId) ?? Object.keys(stampedByRow).length) - const cellCountDelta = scopeRowCount * targetGroupIds.size - const bumped = bumpRunState(queryClient, tableId, stampedByRow, cellCountDelta) + const bumped = await bumpRunState(queryClient, tableId, stampedByRow) return { snapshots, runStateSnapshot: bumped?.snapshot, didBumpRunState: bumped !== null } }, onError: (_err, _variables, context) => { @@ -2085,14 +2096,21 @@ export function useRunColumn({ workspaceId, tableId }: RowMutationContext) { // optimistic counter to the server's still-zero count. const dispatchId = data?.data?.dispatchId if (!dispatchId) { - // No dispatch created → no SSE to reconcile the bump; roll it back. + // No dispatch created (empty scope, or a Stop-all cancelled the run + // during server prep) → no SSE will reconcile the optimistic state. + // Refetch both caches rather than restoring the onMutate snapshots — + // either restore could clobber fresher state applied since (SSE + // events, throttled refetches, a concurrent Stop-all's clear). if (context?.didBumpRunState) { - queryClient.setQueryData(tableKeys.activeDispatches(tableId), context.runStateSnapshot) + void queryClient.invalidateQueries({ queryKey: tableKeys.activeDispatches(tableId) }) + } + if (context?.snapshots) { + void queryClient.invalidateQueries({ queryKey: tableKeys.rowsRoot(tableId) }) } return } queryClient.setQueryData(tableKeys.activeDispatches(tableId), (prev) => { - const base = prev ?? { dispatches: [], runningCellCount: 0, runningByRowId: {} } + const base = prev ?? { dispatches: [], runningByRowId: {}, hasRunning: false } if (base.dispatches.some((d) => d.id === dispatchId)) return base const dispatch: ActiveDispatch = { id: dispatchId, diff --git a/apps/sim/lib/api/contracts/tables.test.ts b/apps/sim/lib/api/contracts/tables.test.ts new file mode 100644 index 00000000000..0f368363d3b --- /dev/null +++ b/apps/sim/lib/api/contracts/tables.test.ts @@ -0,0 +1,24 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { tableEventStreamQuerySchema } from '@/lib/api/contracts/tables' + +describe('tableEventStreamQuerySchema', () => { + it('parses an explicit cursor', () => { + expect(tableEventStreamQuerySchema.parse({ from: '7' })).toEqual({ from: 7 }) + }) + + it('keeps 0 as an explicit replay-from-start cursor', () => { + expect(tableEventStreamQuerySchema.parse({ from: '0' })).toEqual({ from: 0 }) + }) + + it('yields undefined when absent — the tail-from-latest signal', () => { + expect(tableEventStreamQuerySchema.parse({})).toEqual({ from: undefined }) + }) + + it('yields undefined for invalid values instead of coercing to a full replay', () => { + expect(tableEventStreamQuerySchema.parse({ from: 'abc' })).toEqual({ from: undefined }) + expect(tableEventStreamQuerySchema.parse({ from: '-4' })).toEqual({ from: undefined }) + }) +}) diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index b9d0512b0c0..da9bc6b0214 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -1334,13 +1334,17 @@ export const listActiveDispatchesContract = defineRouteContract({ schema: successResponseSchema( z.object({ dispatches: z.array(activeDispatchSchema), - /** Total cells across the table whose `status === 'running'`. The - * client maintains this incrementally via cell SSE events; this - * field is the bootstrap snapshot on mount. */ - runningCellCount: z.number().int().nonnegative(), - /** Map rowId → number of running cells on that row. Drives the - * per-row badge next to the Stop button. */ + /** Map rowId → number of in-flight (queued/running/pending) cells on + * that row. Sums to the "X running" badge and drives the per-row + * gutter Run/Stop button. Server-authoritative: refetched on a + * throttle as cell SSE events arrive, plus optimistic stamps on + * run-click. */ runningByRowId: z.record(z.string(), z.number().int().positive()), + /** Whether any in-flight cell is actually claimed by a worker + * (`status === 'running'`) — table-wide, unlike the client's + * loaded-rows view. Drives the header's "Queueing" vs "N running" + * label once the run's active window scrolls past the loaded rows. */ + hasRunning: z.boolean(), }) ), }, @@ -1349,11 +1353,15 @@ export const listActiveDispatchesContract = defineRouteContract({ export type ActiveDispatch = z.output export const tableEventStreamQuerySchema = z.object({ + /** Replay cursor: events with `eventId > from` are replayed on connect. + * `0` replays the whole buffer (prune recovery). Absent → the server tails + * from the latest event id — a fresh mount has just fetched current state + * from the DB, so replaying history would only rewind it. */ from: z.preprocess((value) => { - if (typeof value !== 'string') return 0 + if (typeof value !== 'string') return undefined const parsed = Number.parseInt(value, 10) - return Number.isFinite(parsed) && parsed >= 0 ? parsed : 0 - }, z.number().int().min(0)), + return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined + }, z.number().int().min(0).optional()), }) export const tableEventStreamContract = defineRouteContract({ diff --git a/apps/sim/lib/table/dispatcher.ts b/apps/sim/lib/table/dispatcher.ts index 449bc531914..9bba4d27dd5 100644 --- a/apps/sim/lib/table/dispatcher.ts +++ b/apps/sim/lib/table/dispatcher.ts @@ -207,15 +207,10 @@ export async function insertDispatch(input: { return id } -/** Read every dispatch on a table whose status is still `pending` or - * `dispatching`. Drives the client-side "about to run" overlay: rows in an - * active dispatch's scope ahead of its cursor are rendered as queued even - * before the dispatcher has reached them, so refresh during a long Run-all - * doesn't lose the queued indicators. */ -/** Counts in-flight cells (queued / running / pending) across the entire - * table — the authoritative source for the "X running" badge and the per-row - * gutter Run/Stop button. All three statuses are user-cancellable, so the - * gutter must surface Stop whenever any of them are present (else clicking +/** Counts in-flight cells (queued / running / pending) per row across the + * entire table — the authoritative source for the "X running" badge and the + * per-row gutter Run/Stop button. All three statuses are user-cancellable, so + * the gutter must surface Stop whenever any of them are present (else clicking * Play during the queued window would re-run an already-queued cell). * * Excludes orphan pre-stamps — `pending` rows with no `executionId` — which @@ -230,7 +225,7 @@ export async function insertDispatch(input: { export async function countRunningCells( tableId: string, opts?: { includeUnclaimedPreStamps?: boolean } -): Promise<{ total: number; byRowId: Record }> { +): Promise<{ byRowId: Record; hasRunning: boolean }> { // `pending` + null-executionId rows are unclaimed pre-stamps. With an active // dispatch they're real queued work (include); with none they're abandoned // orphans that would pin the badge above zero forever (exclude). @@ -239,6 +234,10 @@ export async function countRunningCells( .select({ rowId: tableRowExecutions.rowId, runningCount: sql`count(*)::int`, + // Cells actually claimed by a worker — drives the header's + // "Queueing" vs "N running" label table-wide (the client can only see + // claims on loaded rows; a long run's active window scrolls past them). + claimedCount: sql`count(*) FILTER (WHERE ${tableRowExecutions.status} = 'running')::int`, }) .from(tableRowExecutions) .where( @@ -251,62 +250,20 @@ export async function countRunningCells( ) ) .groupBy(tableRowExecutions.rowId) - let total = 0 const byRowId: Record = {} + let hasRunning = false for (const r of rows) { - if (r.runningCount > 0) { - byRowId[r.rowId] = r.runningCount - total += r.runningCount - } + if (r.runningCount > 0) byRowId[r.rowId] = r.runningCount + if (r.claimedCount > 0) hasRunning = true } - return { total, byRowId } -} - -/** Authoritative "cells queued or running" count for the table, derived from - * active dispatches so it survives reload and matches the live count. For each - * active dispatch every row in scope ahead of the cursor still has to run each - * targeted group, so remaining work = (rows ahead of cursor) × |groupIds|. - * Exact for Run-all; an upper bound for incomplete/new (rows the eligibility - * filter later skips are still counted). Falls back to the sidecar in-flight - * count when no dispatch is active (orphan stragglers). `byRowId` stays - * sidecar-based — the client overlay renders queued rows ahead of the cursor. */ -export async function countActiveRunCells( - tableId: string, - dispatches?: DispatchRow[] -): Promise<{ total: number; byRowId: Record }> { - const active = dispatches ?? (await listActiveDispatches(tableId)) - if (active.length === 0) return countRunningCells(tableId) - - const countRowsAhead = async (d: DispatchRow): Promise => { - const groupCount = d.scope.groupIds.length - if (groupCount === 0) return 0 - const filters = [eq(userTableRows.tableId, tableId), gt(userTableRows.position, d.cursor)] - if (d.scope.rowIds && d.scope.rowIds.length > 0) { - filters.push(inArray(userTableRows.id, d.scope.rowIds)) - } - const [row] = await db - .select({ rowsAhead: sql`count(*)::int` }) - .from(userTableRows) - .where(and(...filters)) - let rowsAhead = row?.rowsAhead ?? 0 - // A `rows` cap means at most `max - processed` more rows will run, even if - // many more sit ahead of the cursor — clamp so the badge doesn't over-count. - if (d.limit?.type === 'rows') { - rowsAhead = Math.min(rowsAhead, Math.max(0, d.limit.max - d.processedCount)) - } - return rowsAhead * groupCount - } - - // Include pre-stamps so `byRowId` matches the live SSE count (which counts - // `pending`); otherwise the badge flickers 20→0 on each refetch. - const [sidecar, perDispatch] = await Promise.all([ - countRunningCells(tableId, { includeUnclaimedPreStamps: true }), - Promise.all(active.map(countRowsAhead)), - ]) - const total = perDispatch.reduce((sum, n) => sum + n, 0) - return { total, byRowId: sidecar.byRowId } + return { byRowId, hasRunning } } +/** Read every dispatch on a table whose status is still `pending` or + * `dispatching`. Drives the client-side "about to run" overlay: rows in an + * active dispatch's scope ahead of its cursor are rendered as queued even + * before the dispatcher has reached them, so refresh during a long Run-all + * doesn't lose the queued indicators. */ export async function listActiveDispatches(tableId: string): Promise { const rows = await db .select() @@ -714,6 +671,34 @@ export async function markDispatchComplete(dispatchId: string): Promise { .where(eq(tableRunDispatches.id, dispatchId)) } +/** Cancel one dispatch by id (if still active) and emit the terminal SSE so + * the client overlay clears. Used when `runWorkflowColumn` fails between + * inserting its dispatch row and firing the dispatcher — without this the + * orphaned `pending` row would pin the "about to run" overlay forever. */ +export async function cancelDispatchById(dispatchId: string): Promise { + const [row] = await db + .update(tableRunDispatches) + .set({ status: 'cancelled', cancelledAt: new Date() }) + .where( + and( + eq(tableRunDispatches.id, dispatchId), + inArray(tableRunDispatches.status, [...ACTIVE_DISPATCH_STATUSES]) + ) + ) + .returning() + if (!row) return + await appendTableEvent({ + kind: 'dispatch', + tableId: row.tableId, + dispatchId: row.id, + status: 'cancelled', + scope: row.scope as DispatchScope, + cursor: row.cursor, + mode: row.mode as DispatchMode, + isManualRun: row.isManualRun, + }) +} + /** Complete a dispatch only if it's still active, returning whether THIS call * performed the transition. Lets concurrent cells that all hit a hard stop * (e.g. usage limit) elect a single owner — only the winner emits the @@ -752,12 +737,15 @@ export async function markDispatchCancelled(dispatchId: string): Promise { * is that exact filter (a filtered "select all" Stop must not halt * whole-table or differently-filtered runs). Pass `spareExcludedRowIds` * (select-all-minus-deselections Stop) to spare row-scoped dispatches whose - * rows are ALL deselected — that work wasn't in the stopped selection. */ + * rows are ALL deselected — that work wasn't in the stopped selection. Pass + * `spareDispatchId` when the caller is a manual run cancelling *prior* work: + * its own dispatch row is already inserted (so a concurrent Stop-all has + * something to cancel) and must not cancel itself. */ export async function markActiveDispatchesCancelled( tableId: string, - scopeFilter?: Filter, - spareExcludedRowIds?: string[] + opts?: { scopeFilter?: Filter; spareExcludedRowIds?: string[]; spareDispatchId?: string } ): Promise { + const { scopeFilter, spareExcludedRowIds, spareDispatchId } = opts ?? {} const cancelled = await db .update(tableRunDispatches) .set({ status: 'cancelled', cancelledAt: new Date() }) @@ -765,6 +753,7 @@ export async function markActiveDispatchesCancelled( and( eq(tableRunDispatches.tableId, tableId), inArray(tableRunDispatches.status, [...ACTIVE_DISPATCH_STATUSES]), + spareDispatchId ? ne(tableRunDispatches.id, spareDispatchId) : undefined, scopeFilter ? sql`${tableRunDispatches.scope}->'filter' = ${JSON.stringify(scopeFilter)}::jsonb` : undefined, diff --git a/apps/sim/lib/table/events.test.ts b/apps/sim/lib/table/events.test.ts new file mode 100644 index 00000000000..d3bdb6deca3 --- /dev/null +++ b/apps/sim/lib/table/events.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/config/redis', () => ({ + getRedisClient: () => null, +})) + +vi.mock('@/lib/core/config/env', () => ({ + env: { REDIS_URL: undefined }, +})) + +import type { TableEvent } from '@/lib/table/events' +import { appendTableEvent, getLatestTableEventId, readTableEventsSince } from '@/lib/table/events' + +/** Module-level memory buffer can't be reset without vi.resetModules — use a + * unique tableId per test to avoid cross-test bleed. */ +let seq = 0 +function uniqueTableId(): string { + seq++ + return `table-events-test-${seq}` +} + +function cellEvent(tableId: string): TableEvent { + return { + kind: 'cell', + tableId, + rowId: 'row-1', + groupId: 'group-1', + status: 'running', + } +} + +describe('getLatestTableEventId (memory buffer)', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('returns 0 for a table with no events, without allocating a stream', async () => { + const tableId = uniqueTableId() + expect(await getLatestTableEventId(tableId)).toBe(0) + // A pure read must not have created a buffer: appending afterwards still + // starts the sequence at 1. + const entry = await appendTableEvent(cellEvent(tableId)) + expect(entry?.eventId).toBe(1) + }) + + it('returns the latest assigned eventId after appends', async () => { + const tableId = uniqueTableId() + await appendTableEvent(cellEvent(tableId)) + const second = await appendTableEvent(cellEvent(tableId)) + expect(second?.eventId).toBe(2) + expect(await getLatestTableEventId(tableId)).toBe(2) + }) + + it('tailing from the latest id yields no replayed events', async () => { + const tableId = uniqueTableId() + await appendTableEvent(cellEvent(tableId)) + await appendTableEvent(cellEvent(tableId)) + const latest = await getLatestTableEventId(tableId) + const result = await readTableEventsSince(tableId, latest) + expect(result).toEqual({ status: 'ok', events: [] }) + }) + + it('a subsequent append is visible to a reader tailing from the prior latest', async () => { + const tableId = uniqueTableId() + await appendTableEvent(cellEvent(tableId)) + const latest = await getLatestTableEventId(tableId) + await appendTableEvent(cellEvent(tableId)) + const result = await readTableEventsSince(tableId, latest) + expect(result.status).toBe('ok') + if (result.status === 'ok') { + expect(result.events).toHaveLength(1) + expect(result.events[0].eventId).toBe(latest + 1) + } + }) +}) diff --git a/apps/sim/lib/table/events.ts b/apps/sim/lib/table/events.ts index 8b8dc6da93c..7d1c8135fe0 100644 --- a/apps/sim/lib/table/events.ts +++ b/apps/sim/lib/table/events.ts @@ -276,6 +276,33 @@ export async function appendTableEvent(event: TableEvent): Promise { + const redis = getRedisClient() + if (!redis) { + if (canUseMemoryBuffer()) { + // Pure read — getMemoryStream() would allocate a stream as a side effect. + const stream = memoryTableStreams.get(tableId) + return stream ? stream.nextEventId - 1 : 0 + } + return 0 + } + const raw = await redis.get(getSeqKey(tableId)) + if (!raw) return 0 + const parsed = Number.parseInt(raw, 10) + return Number.isFinite(parsed) && parsed > 0 ? parsed : 0 +} + /** * Read events for a table where eventId > afterEventId. Returns 'pruned' if * the caller has fallen off the back of the buffer (TTL expired or cap rolled diff --git a/apps/sim/lib/table/workflow-columns.ts b/apps/sim/lib/table/workflow-columns.ts index c60a04efb48..3cd45940a0f 100644 --- a/apps/sim/lib/table/workflow-columns.ts +++ b/apps/sim/lib/table/workflow-columns.ts @@ -365,7 +365,14 @@ export const TABLE_CONCURRENCY_LIMIT = 20 export async function cancelWorkflowGroupRuns( tableId: string, rowId?: string, - options?: { groupIds?: string[]; filter?: Filter; excludeRowIds?: string[] } + options?: { + groupIds?: string[] + filter?: Filter + excludeRowIds?: string[] + /** Set by a manual run cancelling prior work: its own already-inserted + * dispatch must survive the table-wide dispatch cancel. */ + spareDispatchId?: string + } ): Promise { const { getTableById } = await import('@/lib/table/service') const { updateRow } = await import('@/lib/table/rows/service') @@ -387,7 +394,15 @@ export async function cancelWorkflowGroupRuns( // (its own run); whole-table or differently-scoped dispatches keep running — // their cells cancelled below are skipped via `cancelledAt > requestedAt`. if (!rowId) { - await markActiveDispatchesCancelled(tableId, options?.filter, options?.excludeRowIds) + const cancelledDispatches = await markActiveDispatchesCancelled(tableId, { + scopeFilter: options?.filter, + spareExcludedRowIds: options?.excludeRowIds, + spareDispatchId: options?.spareDispatchId, + }) + logger.info( + `cancelWorkflowGroupRuns: cancelled ${cancelledDispatches.length} active dispatch(es) for table ${tableId}`, + { dispatchIds: cancelledDispatches.map((d) => d.id) } + ) } const allGroups = table.schema.workflowGroups ?? [] @@ -678,67 +693,23 @@ export async function runWorkflowColumn(opts: { if (targetGroups.length === 0) return { dispatchId: null } const targetGroupIds = targetGroups.map((g) => g.id) - const { bulkClearWorkflowGroupCells, insertDispatch, runDispatcherToCompletion } = await import( - './dispatcher' - ) - - // For manual runs (Run all rows / Run column / Refresh-row / Refresh-cell), - // cancel any prior active dispatches AND in-flight cells in scope before - // clearing. Without this: - // - Two dispatcher loops would walk overlapping rows and burn duplicate work. - // - mode:'all' bulk-clear deletes in-flight sidecar rows without aborting - // workers — those would keep writing into the wiped state. - // Scope: table-wide cancel when rowIds is empty (also cancels active - // dispatches via markActiveDispatchesCancelled), per-row cancel otherwise - // (no dispatch cancel — other rows' dispatches keep running). Dep-edit - // cascade in `updateRow` already cancels its own scope before calling, - // so the duplicate work here is a cheap no-op for that caller. - // Auto-fire (`mode:'new'`) is harmless overlap-wise — the NOT EXISTS - // filter excludes already-attempted rows. - const cancelPriorRuns = isManualRun && (mode === 'all' || mode === 'incomplete') - if (cancelPriorRuns) { - if (!rowIds || rowIds.length === 0) { - // Filtered runs cancel only their own scope — a table-wide cancel here - // would stop unrelated work on rows outside the filter (or on deselected rows). - await cancelWorkflowGroupRuns(tableId, undefined, { - groupIds: targetGroupIds, - filter, - excludeRowIds, - }) - } else { - // Per-row cancel — sequential so we don't fan out N parallel - // markActiveDispatchesCancelled calls (it's a no-op when rowId is set, - // but each call still touches the DB). - for (const rowId of rowIds) { - await cancelWorkflowGroupRuns(tableId, rowId, { groupIds: targetGroupIds }) - } - } - } - - // Wipe targeted output cols + executions[gid] before any cells fire so the - // user sees the column flip to empty/Pending instantly. Skipped for capped - // runs: the eager clear can't know which N rows the dispatcher will pick - // (they depend on per-row eligibility as it walks positions), so wiping all - // rows in scope would blank far more than we re-run. `mode: 'all'` re-runs - // completed cells without the clear anyway — the clear is only for instant - // feedback, which the capped rows still get via the dispatcher's pre-stamp. - // Skip the eager clear for a filtered run: `bulkClearWorkflowGroupCells` keys by `rowIds`, and a - // filtered scope has none — clearing table-wide would blank rows that don't match the filter. The - // dispatcher's per-row pre-stamp still provides instant Pending feedback as it walks. - if (!limit && !filter) { - await bulkClearWorkflowGroupCells({ - tableId, - groups: targetGroups.map((g) => ({ id: g.id, outputs: g.outputs })), - rowIds, - excludeRowIds, - mode, - }) - } - - // Always insert a `table_run_dispatches` row. The dispatcher state machine - // is the single source of truth for cursor advancement, SSE emission, and - // cancel — backend (trigger.dev SaaS vs in-process) only affects how each - // window's cells get executed. + const { + bulkClearWorkflowGroupCells, + cancelDispatchById, + insertDispatch, + readDispatch, + runDispatcherToCompletion, + } = await import('./dispatcher') + + // Always insert a `table_run_dispatches` row, and insert it FIRST — before + // the prior-run cancel and the bulk clear below, which can take seconds on + // a large table. The client shows its Stop control optimistically from the + // moment the user clicks Run, so a Stop-all arriving during that prep work + // must find a dispatch row to cancel; inserted-after ordering made an early + // Stop-all a silent no-op and the run proceeded anyway. The dispatcher + // state machine is the single source of truth for cursor advancement, SSE + // emission, and cancel — backend (trigger.dev SaaS vs in-process) only + // affects how each window's cells get executed. const dispatchId = await insertDispatch({ tableId, workspaceId, @@ -757,6 +728,91 @@ export async function runWorkflowColumn(opts: { triggeredByUserId, }) + try { + // For manual runs (Run all rows / Run column / Refresh-row / Refresh-cell), + // cancel any prior active dispatches AND in-flight cells in scope before + // clearing. Without this: + // - Two dispatcher loops would walk overlapping rows and burn duplicate work. + // - mode:'all' bulk-clear deletes in-flight sidecar rows without aborting + // workers — those would keep writing into the wiped state. + // Scope: table-wide cancel when rowIds is empty (also cancels active + // dispatches via markActiveDispatchesCancelled, sparing the one just + // inserted above), per-row cancel otherwise (no dispatch cancel — other + // rows' dispatches keep running). Dep-edit cascade in `updateRow` already + // cancels its own scope before calling, so the duplicate work here is a + // cheap no-op for that caller. Auto-fire (`mode:'new'`) is harmless + // overlap-wise — the NOT EXISTS filter excludes already-attempted rows. + const cancelPriorRuns = isManualRun && (mode === 'all' || mode === 'incomplete') + if (cancelPriorRuns) { + if (!rowIds || rowIds.length === 0) { + // Filtered runs cancel only their own scope — a table-wide cancel here + // would stop unrelated work on rows outside the filter (or on deselected rows). + await cancelWorkflowGroupRuns(tableId, undefined, { + groupIds: targetGroupIds, + filter, + excludeRowIds, + spareDispatchId: dispatchId, + }) + } else { + // Per-row cancel — sequential so we don't fan out N parallel + // markActiveDispatchesCancelled calls (it's a no-op when rowId is set, + // but each call still touches the DB). + for (const rowId of rowIds) { + await cancelWorkflowGroupRuns(tableId, rowId, { groupIds: targetGroupIds }) + } + } + } + + // Wipe targeted output cols + executions[gid] before any cells fire so the + // user sees the column flip to empty/Pending instantly. Skipped for capped + // runs: the eager clear can't know which N rows the dispatcher will pick + // (they depend on per-row eligibility as it walks positions), so wiping all + // rows in scope would blank far more than we re-run. `mode: 'all'` re-runs + // completed cells without the clear anyway — the clear is only for instant + // feedback, which the capped rows still get via the dispatcher's pre-stamp. + // Skip the eager clear for a filtered run: `bulkClearWorkflowGroupCells` keys by `rowIds`, and a + // filtered scope has none — clearing table-wide would blank rows that don't match the filter. The + // dispatcher's per-row pre-stamp still provides instant Pending feedback as it walks. + if (!limit && !filter) { + await bulkClearWorkflowGroupCells({ + tableId, + groups: targetGroups.map((g) => ({ id: g.id, outputs: g.outputs })), + rowIds, + excludeRowIds, + mode, + }) + } + } catch (err) { + // Prep failed after the dispatch row was inserted — cancel it so an + // orphaned `pending` dispatch can't pin the client's "about to run" + // overlay, then fail the request with the ORIGINAL error. The cleanup is + // best-effort: its own failure must not mask the prep failure. + try { + await cancelDispatchById(dispatchId) + } catch (cleanupErr) { + logger.error(`[Cascade] [${requestId}] failed to cancel dispatch after prep failure`, { + dispatchId, + error: toError(cleanupErr).message, + }) + } + throw err + } + + // A Stop-all can land during the prep above; its dispatch cancel is the + // authoritative stop. Don't fire the dispatcher loop for a dead dispatch — + // it would exit on its first status read, but the trigger.dev path would + // still spin up a task for nothing. Return a null dispatchId: the client + // seeds a returned id into its active-dispatch overlay, which would + // resurrect the Run/Stop UI the cancelled SSE event already cleared; null + // takes its "no dispatch created" path and rolls the optimistic bump back. + const current = await readDispatch(dispatchId) + if (!current || current.status === 'cancelled' || current.status === 'complete') { + logger.info( + `[Cascade] [${requestId}] dispatch ${dispatchId} cancelled during prep — not firing` + ) + return { dispatchId: null } + } + logger.info( `[Cascade] [${requestId}] dispatch ${dispatchId} table=${tableId} groups=[${targetGroupIds.join(',')}] rows=${rowIds ? `[${rowIds.join(',')}]` : 'all'} mode=${mode}` ) From 619e9122567df95e6d509df2868e68d31fac0388 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 7 Jul 2026 16:23:02 -0700 Subject: [PATCH 4/9] fix(tables): canonicalize date cells, render times in effective timezone (#5465) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(tables): canonicalize date cells, render times in effective timezone * improvement(tables): render date cells wall-clock-faithful via offset-preserved storage * fix(tables): address review — preserve midnight instants, validate calendar days, effective-zone Today --- .../api/table/[tableId]/import-async/route.ts | 5 +- .../app/api/table/[tableId]/import/route.ts | 18 +- apps/sim/app/api/table/import-async/route.ts | 4 +- apps/sim/app/api/table/import-csv/route.ts | 16 +- .../components/row-modal/row-modal.tsx | 47 ++- .../table-grid/cells/cell-render.tsx | 2 +- .../cells/expanded-cell-popover.tsx | 30 +- .../table-grid/cells/inline-editors.tsx | 109 +++++- .../components/table-grid/table-grid.tsx | 10 +- .../tables/[tableId]/utils.test.ts | 138 ++++++++ .../[workspaceId]/tables/[tableId]/utils.ts | 139 ++++++-- apps/sim/hooks/queries/general-settings.ts | 2 +- apps/sim/hooks/queries/tables.ts | 14 +- apps/sim/lib/api/contracts/tables.ts | 13 + apps/sim/lib/table/dates.test.ts | 158 +++++++++ apps/sim/lib/table/dates.ts | 315 ++++++++++++++++++ apps/sim/lib/table/import-runner.ts | 10 +- apps/sim/lib/table/import.test.ts | 8 +- apps/sim/lib/table/import.ts | 12 +- apps/sim/lib/table/index.ts | 1 + apps/sim/lib/table/validation.ts | 6 +- .../src/components/calendar/calendar.test.ts | 28 +- .../emcn/src/components/calendar/calendar.tsx | 122 ++++++- .../chip-date-picker/chip-date-picker.tsx | 6 + 24 files changed, 1116 insertions(+), 97 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.test.ts create mode 100644 apps/sim/lib/table/dates.test.ts create mode 100644 apps/sim/lib/table/dates.ts diff --git a/apps/sim/app/api/table/[tableId]/import-async/route.ts b/apps/sim/app/api/table/[tableId]/import-async/route.ts index 22f51bfaa93..b440dfaf680 100644 --- a/apps/sim/app/api/table/[tableId]/import-async/route.ts +++ b/apps/sim/app/api/table/[tableId]/import-async/route.ts @@ -10,6 +10,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner' import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service' +import { getUserSettings } from '@/lib/users/queries' import { accessError, checkAccess } from '@/app/api/table/utils' const logger = createLogger('TableImportIntoAsync') @@ -33,7 +34,8 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const parsed = await parseRequest(importIntoTableAsyncContract, request, { params }) if (!parsed.success) return parsed.response const { tableId } = parsed.data.params - const { workspaceId, fileKey, fileName, mode, mapping, createColumns } = parsed.data.body + const { workspaceId, fileKey, fileName, mode, mapping, createColumns, timezone } = + parsed.data.body const access = await checkAccess(tableId, userId, 'write') if (!access.ok) return accessError(access, requestId, tableId) @@ -79,6 +81,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro mode, mapping, createColumns, + timezone: timezone ?? (await getUserSettings(userId)).timezone ?? 'UTC', } if (isTriggerDevEnabled) { // Trigger.dev runs the import outside the web container, so it survives app deploys. diff --git a/apps/sim/app/api/table/[tableId]/import/route.ts b/apps/sim/app/api/table/[tableId]/import/route.ts index 79bb7238ab9..63ed87220fb 100644 --- a/apps/sim/app/api/table/[tableId]/import/route.ts +++ b/apps/sim/app/api/table/[tableId]/import/route.ts @@ -11,6 +11,7 @@ import { csvImportModeSchema, tableIdParamsSchema, } from '@/lib/api/contracts/tables' +import { ianaTimezoneSchema } from '@/lib/api/contracts/user' import { getValidationErrorMessage } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { isMultipartError, readMultipart } from '@/lib/core/utils/multipart' @@ -36,6 +37,7 @@ import { wouldExceedRowLimit, } from '@/lib/table' import { importAppendRows, importReplaceRows } from '@/lib/table/import-data' +import { getUserSettings } from '@/lib/users/queries' import { accessError, checkAccess, @@ -162,6 +164,18 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro createColumns = createColumnsValidation.data } + let timezone = (await getUserSettings(authResult.userId)).timezone ?? 'UTC' + if (fields.timezone) { + const timezoneValidation = ianaTimezoneSchema.safeParse(fields.timezone) + if (!timezoneValidation.success) { + return NextResponse.json( + { error: getValidationErrorMessage(timezoneValidation.error) }, + { status: 400 } + ) + } + timezone = timezoneValidation.data + } + const delimiter = extensionValidation.data === 'tsv' ? '\t' : ',' const parser = createCsvParser(delimiter) // `.pipe` doesn't forward source errors; forward them so the iterator throws. @@ -250,7 +264,9 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro ) } - const coerced = coerceRowsForTable(rows, prospectiveTable.schema, validation.effectiveMap) + const coerced = coerceRowsForTable(rows, prospectiveTable.schema, validation.effectiveMap, { + timezone, + }) // Atomically claim the table before writing. The pre-check above reads a checkAccess snapshot // taken before the parse/validation; a background import could claim the table in that window. diff --git a/apps/sim/app/api/table/import-async/route.ts b/apps/sim/app/api/table/import-async/route.ts index bb0d83d168a..500429075df 100644 --- a/apps/sim/app/api/table/import-async/route.ts +++ b/apps/sim/app/api/table/import-async/route.ts @@ -20,6 +20,7 @@ import { TableConflictError, } from '@/lib/table' import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner' +import { getUserSettings } from '@/lib/users/queries' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' const logger = createLogger('TableImportAsync') @@ -38,7 +39,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const parsed = await parseRequest(importTableAsyncContract, request, {}) if (!parsed.success) return parsed.response - const { workspaceId, fileKey, fileName, deleteSourceFile } = parsed.data.body + const { workspaceId, fileKey, fileName, deleteSourceFile, timezone } = parsed.data.body const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId) if (permission !== 'write' && permission !== 'admin') { @@ -111,6 +112,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { delimiter, mode: 'create', deleteSourceFile, + timezone: timezone ?? (await getUserSettings(userId)).timezone ?? 'UTC', } if (isTriggerDevEnabled) { // Trigger.dev runs the import outside the web container, so it survives app deploys. diff --git a/apps/sim/app/api/table/import-csv/route.ts b/apps/sim/app/api/table/import-csv/route.ts index 91ee680706d..fb29cffab8f 100644 --- a/apps/sim/app/api/table/import-csv/route.ts +++ b/apps/sim/app/api/table/import-csv/route.ts @@ -4,6 +4,7 @@ import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { type NextRequest, NextResponse } from 'next/server' import { csvExtensionSchema, csvImportFormSchema } from '@/lib/api/contracts/tables' +import { ianaTimezoneSchema } from '@/lib/api/contracts/user' import { getValidationErrorMessage } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { isMultipartError, readMultipart } from '@/lib/core/utils/multipart' @@ -25,6 +26,7 @@ import { type TableDefinition, type TableSchema, } from '@/lib/table' +import { getUserSettings } from '@/lib/users/queries' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { csvProxyBodyCapResponse, @@ -85,6 +87,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: 'Access denied' }, { status: 403 }) } + let timezone = (await getUserSettings(userId)).timezone ?? 'UTC' + if (fields.timezone) { + const timezoneResult = ianaTimezoneSchema.safeParse(fields.timezone) + if (!timezoneResult.success) { + return NextResponse.json( + { error: getValidationErrorMessage(timezoneResult.error) }, + { status: 400 } + ) + } + timezone = timezoneResult.data + } + const ext = file.filename.split('.').pop()?.toLowerCase() const extensionResult = csvExtensionSchema.safeParse(ext) if (!extensionResult.success) { @@ -112,7 +126,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { currentRowCount: number ) => { if (rows.length === 0) return 0 - const coerced = coerceRowsForTable(rows, state.schema, state.headerToColumn) + const coerced = coerceRowsForTable(rows, state.schema, state.headerToColumn, { timezone }) const result = await batchInsertRows( { tableId: state.table.id, rows: coerced, workspaceId, userId }, // The created table's rowCount is frozen at 0; pass the running total so the diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx index d0045f4f40a..28c971bda9f 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx @@ -11,14 +11,22 @@ import { ChipModalField, ChipModalFooter, ChipModalHeader, + ChipTimePicker, Label, } from '@sim/emcn' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { useParams } from 'next/navigation' import type { ColumnDefinition, TableInfo, TableRow } from '@/lib/table' +import { useTimezone } from '@/hooks/queries/general-settings' import { useDeleteTableRow, useDeleteTableRows, useUpdateTableRow } from '@/hooks/queries/tables' -import { cleanCellValue, formatValueForInput } from '../../utils' +import { + cleanCellValue, + dateValueToLocalParts, + formatValueForInput, + localPartsToDateValue, + todayLocalCalendarDate, +} from '../../utils' const logger = createLogger('RowModal') @@ -34,14 +42,15 @@ export interface RowModalProps { function cleanRowData( columns: ColumnDefinition[], - rowData: Record + rowData: Record, + timeZone: string ): Record { const cleanData: Record = {} columns.forEach((col) => { const value = rowData[col.name] try { - cleanData[col.name] = cleanCellValue(value, col) + cleanData[col.name] = cleanCellValue(value, col, timeZone) } catch { throw new Error(`Invalid JSON for field: ${col.name}`) } @@ -66,6 +75,7 @@ export function RowModal({ mode, isOpen, onClose, table, row, rowIds, onSuccess const schema = table?.schema const columns = schema?.columns || [] + const timeZone = useTimezone() const [rowData, setRowData] = useState>(() => mode === 'edit' && row ? row.data : {} ) @@ -81,7 +91,7 @@ export function RowModal({ mode, isOpen, onClose, table, row, rowIds, onSuccess setError(null) try { - const cleanData = cleanRowData(columns, rowData) + const cleanData = cleanRowData(columns, rowData, timeZone) if (row) { await updateRowMutation.mutateAsync({ rowId: row.id, data: cleanData }) @@ -189,6 +199,7 @@ interface ColumnFieldProps { function ColumnField({ column, value, onChange }: ColumnFieldProps) { const checkboxId = useId() + const timeZone = useTimezone() const title = ( <> {column.name} @@ -236,14 +247,30 @@ function ColumnField({ column, value, onChange }: ColumnFieldProps) { } if (column.type === 'date') { + const parts = dateValueToLocalParts(formatValueForInput(value, 'date')) return ( - +
+ onChange(localPartsToDateValue(day, parts.time, timeZone))} + placeholder='Select date' + flush + className='flex-1' + /> + + onChange( + localPartsToDateValue(parts.day ?? todayLocalCalendarDate(timeZone), time, timeZone) + ) + } + placeholder='Add time' + flush + className='w-[110px]' + /> +
) } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx index b5529af4e61..4b16f2bceac 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx @@ -354,7 +354,7 @@ export function CellRender({ kind, isEditing }: CellRenderProps): React.ReactEle case 'date': return ( - {storageToDisplay(kind.text)} + {storageToDisplay(kind.text, { seconds: true })} ) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/expanded-cell-popover.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/expanded-cell-popover.tsx index fc1c7daeb07..ce99edc0894 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/expanded-cell-popover.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/expanded-cell-popover.tsx @@ -4,8 +4,14 @@ import type React from 'react' import { useEffect, useEffectEvent, useLayoutEffect, useMemo, useRef, useState } from 'react' import { Button } from '@sim/emcn' import type { TableRow as TableRowType } from '@/lib/table' +import { useTimezone } from '@/hooks/queries/general-settings' import type { EditingCell, SaveReason } from '../../../types' -import { cleanCellValue, displayToStorage, formatValueForInput } from '../../../utils' +import { + cleanCellValue, + displayToStorage, + formatValueForInput, + storageToDisplay, +} from '../../../utils' import type { DisplayColumn } from '../types' interface ExpandedCellPopoverProps { @@ -145,7 +151,11 @@ export function ExpandedCellPopover({ {isEditable ? ( (null) + const timeZone = useTimezone() const handleSave = () => { - // `displayToStorage` only normalizes dates — it returns null for anything else. - // Fall back to the raw draft for non-date columns, matching the inline editor. - const raw = displayToStorage(draftValue) ?? draftValue + // Untouched draft → close without writing. For dates this also avoids + // re-stamping the stored offset with this viewer's zone. + if (draftValue === initialValue) { + onClose() + return + } + // Only date columns go through `displayToStorage` — it now parses many + // date shapes, so a number draft like "2024" must not reach it. + const raw = + column.type === 'date' ? (displayToStorage(draftValue, timeZone) ?? draftValue) : draftValue let cleaned: unknown try { - cleaned = cleanCellValue(raw, column) + cleaned = cleanCellValue(raw, column, timeZone) } catch { setParseError('Invalid JSON') return diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx index bbea94ef95d..20ac51847a2 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx @@ -3,12 +3,16 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { Calendar, cn, Popover, PopoverAnchor, PopoverContent, toast } from '@sim/emcn' import type { ColumnDefinition } from '@/lib/table' +import { isCalendarDateString } from '@/lib/table/dates' +import { useTimezone } from '@/hooks/queries/general-settings' import type { SaveReason } from '../../../types' import { cleanCellValue, + dateValueToLocalParts, displayToStorage, formatValueForInput, storageToDisplay, + todayLocalCalendarDate, } from '../../../utils' interface InlineEditorProps { @@ -28,7 +32,13 @@ function handleEditorWheel(e: React.WheelEvent) { } } -/** Inline editor for `date` columns — text input + popover calendar. */ +/** + * Inline editor for `date` columns — text input + popover with a calendar and + * a time field. Picking a day on a date-only value commits immediately (the + * pick fully determines the value); when the value carries a time, picker + * edits update the draft in place — the day pick keeps the time-of-day + * (including seconds), the time field keeps the day — and Enter/blur commits. + */ function InlineDateEditor({ value, column, @@ -37,16 +47,35 @@ function InlineDateEditor({ onCancel, }: InlineEditorProps) { const inputRef = useRef(null) + const popoverRef = useRef(null) const doneRef = useRef(false) const blurTimeoutRef = useRef | undefined>(undefined) + /** Timestamp of the last pointerdown inside the popover — blur-save skips + * and refocuses while a popover interaction is in flight (covers browsers + * where buttons don't take focus on click). */ + const popoverPointerAtRef = useRef(0) + const timeZone = useTimezone() const storedValue = formatValueForInput(value, column.type) - const [draft, setDraft] = useState(() => - initialCharacter !== undefined ? initialCharacter : storageToDisplay(storedValue) - ) + const initialDraft = + initialCharacter !== undefined + ? initialCharacter + : storageToDisplay(storedValue, { seconds: true }) + const [draft, setDraft] = useState(initialDraft) const [invalid, setInvalid] = useState(false) + /** Picker commits mutate the draft from timeouts/child handlers; reading it + * through a ref keeps the scheduled blur-save from saving a stale draft. */ + const draftRef = useRef(draft) + draftRef.current = draft - const pickerValue = displayToStorage(draft) || storedValue || undefined + /** The calendar works on wall times; feed it the draft's literal wall + * representation. */ + const draftParts = dateValueToLocalParts(displayToStorage(draft, timeZone) ?? storedValue) + const pickerValue = draftParts.day + ? draftParts.time + ? `${draftParts.day}T${draftParts.time}` + : draftParts.day + : undefined useEffect(() => { const input = inputRef.current @@ -66,7 +95,16 @@ function InlineDateEditor({ (reason: SaveReason, storageVal?: string) => { if (doneRef.current) return clearTimeout(blurTimeoutRef.current) - const raw = storageVal ?? displayToStorage(draft) ?? draft + const current = draftRef.current + // Untouched draft → re-save the stored value byte-identical. Re-parsing + // the display form would re-stamp the offset with THIS viewer's zone, + // silently shifting the instant of a value someone else wrote. + if (storageVal === undefined && initialCharacter === undefined && current === initialDraft) { + doneRef.current = true + onSave(storedValue || null, reason) + return + } + const raw = storageVal ?? displayToStorage(current, timeZone) ?? current if (raw && Number.isNaN(Date.parse(raw))) { if (reason === 'blur') { if (!invalid) toast.error('Invalid date') @@ -82,7 +120,7 @@ function InlineDateEditor({ doneRef.current = true onSave(raw || null, reason) }, - [draft, invalid, onSave, onCancel] + [invalid, onSave, onCancel, timeZone, initialDraft, initialCharacter, storedValue] ) const handleKeyDown = useCallback( @@ -103,16 +141,45 @@ function InlineDateEditor({ [doSave, onCancel] ) - const handleBlur = useCallback(() => { - blurTimeoutRef.current = setTimeout(() => doSave('blur'), 200) + const handlePopoverPointerDown = useCallback(() => { + popoverPointerAtRef.current = Date.now() + }, []) + + /** Saves on blur unless focus (or an in-flight pointer interaction) is still + * inside the editor's input/popover system. */ + const scheduleBlurSave = useCallback(() => { + clearTimeout(blurTimeoutRef.current) + blurTimeoutRef.current = setTimeout(() => { + const active = document.activeElement + if (active && (active === inputRef.current || popoverRef.current?.contains(active))) return + if (Date.now() - popoverPointerAtRef.current < 300) { + inputRef.current?.focus() + return + } + doSave('blur') + }, 200) }, [doSave]) + /** + * The calendar (with `showTime`) owns the day/time merge and emits either a + * bare `YYYY-MM-DD` (no time — the pick fully determines the value, commit + * immediately) or a local `YYYY-MM-DDTHH:mm[:ss]` wall time (update the + * draft and keep editing). + */ const handlePickerChange = useCallback( - (dateStr: string) => { + (picked: string) => { clearTimeout(blurTimeoutRef.current) - doSave('enter', dateStr) + if (isCalendarDateString(picked)) { + doSave('enter', picked) + return + } + const canonical = displayToStorage(picked, timeZone) + if (!canonical) return + setDraft(storageToDisplay(canonical, { seconds: true })) + setInvalid(false) + inputRef.current?.focus() }, - [doSave] + [doSave, timeZone] ) const handlePickerOpenChange = useCallback((open: boolean) => { @@ -133,7 +200,7 @@ function InlineDateEditor({ setInvalid(false) }} onKeyDown={handleKeyDown} - onBlur={handleBlur} + onBlur={scheduleBlurSave} placeholder='mm/dd/yyyy' className={cn( 'w-full min-w-0 select-text border-none bg-transparent p-0 text-[var(--text-primary)] text-small outline-none', @@ -142,8 +209,20 @@ function InlineDateEditor({ /> - - + + diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index 8bc8c2f0fb7..2fcbb8a0d1d 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -14,6 +14,7 @@ import type { ColumnDefinition, Filter, TableRow as TableRowType, WorkflowGroup import { getColumnId } from '@/lib/table/column-keys' import { TABLE_LIMITS } from '@/lib/table/constants' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' +import { useTimezone } from '@/hooks/queries/general-settings' import { useAddTableColumn, useBatchCreateTableRows, @@ -505,6 +506,10 @@ export function TableGrid({ const workflowsRef = useRef(workflows) workflowsRef.current = workflows + const timeZone = useTimezone() + const timeZoneRef = useRef(timeZone) + timeZoneRef.current = timeZone + const updateRowMutation = useUpdateTableRow({ workspaceId, tableId }) const createRowMutation = useCreateTableRow({ workspaceId, tableId }) const batchCreateRowsMutation = useBatchCreateTableRows({ workspaceId, tableId }) @@ -1431,7 +1436,7 @@ export function TableGrid({ } } } else if (column.type === 'date') { - text = storageToDisplay(String(val)) + text = storageToDisplay(String(val), { seconds: true }) } else { text = String(val) } @@ -2739,7 +2744,8 @@ export function TableGrid({ try { rowData[currentCols[targetCol].key] = cleanCellValue( pasteRows[r][c], - currentCols[targetCol] + currentCols[targetCol], + timeZoneRef.current ) } catch { /* skip invalid values */ diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.test.ts new file mode 100644 index 00000000000..eace4541f1f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.test.ts @@ -0,0 +1,138 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + cleanCellValue, + dateValueToLocalParts, + displayToStorage, + formatValueForInput, + localPartsToDateValue, + storageToDisplay, +} from '@/app/workspace/[workspaceId]/tables/[tableId]/utils' + +describe('dateValueToLocalParts / localPartsToDateValue', () => { + it('splits calendar dates without a time part and round-trips', () => { + expect(dateValueToLocalParts('2026-07-06')).toEqual({ day: '2026-07-06', time: null }) + expect(localPartsToDateValue('2026-07-06', null)).toBe('2026-07-06') + }) + + it('splits instants into their literal wall day/time — no zone conversion', () => { + expect(dateValueToLocalParts('2026-07-06T16:04:55-07:00')).toEqual({ + day: '2026-07-06', + time: '16:04:55', + }) + expect(dateValueToLocalParts('2026-07-06T23:04:55Z')).toEqual({ + day: '2026-07-06', + time: '23:04:55', + }) + expect(dateValueToLocalParts('2026-07-06T23:04:55.000Z')).toEqual({ + day: '2026-07-06', + time: '23:04:55', + }) + }) + + it('recombines parts stamping the given zone offset, keeping the wall time', () => { + expect(localPartsToDateValue('2026-07-06', '16:04:55', 'America/New_York')).toBe( + '2026-07-06T16:04:55-04:00' + ) + expect(localPartsToDateValue('2026-07-09', '16:04:55', 'America/New_York')).toBe( + '2026-07-09T16:04:55-04:00' + ) + expect(localPartsToDateValue('2026-07-06', '16:04', 'America/New_York')).toBe( + '2026-07-06T16:04:00-04:00' + ) + }) + + it('returns null parts for unparseable values', () => { + expect(dateValueToLocalParts('garbage')).toEqual({ day: null, time: null }) + expect(dateValueToLocalParts('')).toEqual({ day: null, time: null }) + }) +}) + +describe('displayToStorage', () => { + it('parses date-only display formats to calendar dates', () => { + expect(displayToStorage('07/06/2026')).toBe('2026-07-06') + expect(displayToStorage('7/6/2026')).toBe('2026-07-06') + expect(displayToStorage('2026-07-06')).toBe('2026-07-06') + expect(displayToStorage('7/6')).toBe(`${new Date().getFullYear()}-07-06`) + }) + + it('parses M/D/YYYY with a time to a wall time stamped with the given zone', () => { + expect(displayToStorage('07/06/2026 4:04 PM', 'America/New_York')).toBe( + '2026-07-06T16:04:00-04:00' + ) + expect(displayToStorage('07/06/2026 4:04:55 PM', 'America/New_York')).toBe( + '2026-07-06T16:04:55-04:00' + ) + expect(displayToStorage('07/06/2026 16:04', 'America/New_York')).toBe( + '2026-07-06T16:04:00-04:00' + ) + expect(displayToStorage('07/06/2026 12:00 AM', 'America/New_York')).toBe( + '2026-07-06T00:00:00-04:00' + ) + }) + + it('preserves the wall time and offset of canonical and offset strings', () => { + expect(displayToStorage('2026-07-06T16:04:55-07:00')).toBe('2026-07-06T16:04:55-07:00') + expect(displayToStorage('2026-07-06T23:04:55.000Z')).toBe('2026-07-06T23:04:55Z') + expect(displayToStorage('2026-07-06 16:04:55 PDT')).toBe('2026-07-06T16:04:55-07:00') + }) + + it('rejects invalid dates and times', () => { + expect(displayToStorage('13/06/2026')).toBeNull() + expect(displayToStorage('07/06/2026 25:00')).toBeNull() + expect(displayToStorage('07/06/2026 13:00 PM')).toBeNull() + expect(displayToStorage('02/30/2026 5:00 PM')).toBeNull() + expect(displayToStorage('02/30/2026')).toBeNull() + expect(displayToStorage('2/30')).toBeNull() + expect(displayToStorage('garbage')).toBeNull() + }) +}) + +describe('storageToDisplay', () => { + it('renders calendar dates as MM/DD/YYYY', () => { + expect(storageToDisplay('2026-07-06')).toBe('07/06/2026') + }) + + it('renders the literal wall time identically regardless of viewer or offset', () => { + expect(storageToDisplay('2026-07-06T16:04:55-07:00', { seconds: true })).toBe( + '07/06/2026 4:04:55 PM' + ) + expect(storageToDisplay('2026-07-06T16:04:55+09:00', { seconds: true })).toBe( + '07/06/2026 4:04:55 PM' + ) + }) + + it('round-trips an instant through the editor draft format without shifting', () => { + const stored = displayToStorage('07/06/2026 4:04:55 PM', 'America/New_York') as string + const draft = storageToDisplay(stored, { seconds: true }) + expect(draft).toBe('07/06/2026 4:04:55 PM') + expect(displayToStorage(draft, 'America/New_York')).toBe(stored) + }) +}) + +describe('cleanCellValue', () => { + it('normalizes date cells to canonical storage', () => { + const column = { name: 'due', type: 'date' } as const + expect(cleanCellValue('07/06/2026', column)).toBe('2026-07-06') + expect(cleanCellValue('2026-07-06T16:04:55-07:00', column)).toBe('2026-07-06T16:04:55-07:00') + expect(cleanCellValue('nope', column)).toBeNull() + expect(cleanCellValue('', column)).toBeNull() + }) + + it('leaves non-date types on their existing contracts', () => { + expect(cleanCellValue('2024', { name: 'n', type: 'number' } as const)).toBe(2024) + expect(cleanCellValue('true', { name: 'b', type: 'boolean' } as const)).toBe(true) + }) +}) + +describe('formatValueForInput', () => { + it('gives editors the canonical value, surfacing legacy UTC midnights as calendar days', () => { + expect(formatValueForInput('2026-07-06T00:00:00.000Z', 'date')).toBe('2026-07-06') + expect(formatValueForInput('2026-07-06T16:04:55-07:00', 'date')).toBe( + '2026-07-06T16:04:55-07:00' + ) + expect(formatValueForInput('2026-07-06', 'date')).toBe('2026-07-06') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts index 55e310c3630..84e03eefaf0 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts @@ -1,4 +1,10 @@ import type { ColumnDefinition } from '@/lib/table' +import { + formatDateCellDisplay, + getWallClockParts, + normalizeDateCellValue, + storedDateToEditable, +} from '@/lib/table/dates' type BadgeVariant = 'green' | 'blue' | 'purple' | 'orange' | 'teal' | 'gray' @@ -41,7 +47,11 @@ export function getTypeBadgeVariant(type: string): BadgeVariant { * Coerce a raw input value to the appropriate type for a column. * Throws on invalid JSON. */ -export function cleanCellValue(value: unknown, column: ColumnDefinition): unknown { +export function cleanCellValue( + value: unknown, + column: ColumnDefinition, + timeZone?: string +): unknown { if (column.type === 'number') { if (value === '') return null const num = Number(value) @@ -59,8 +69,7 @@ export function cleanCellValue(value: unknown, column: ColumnDefinition): unknow } if (column.type === 'date') { if (value === '' || value === null || value === undefined) return null - const str = String(value) - return Number.isNaN(Date.parse(str)) ? null : str + return displayToStorage(String(value), timeZone) } return value || null } @@ -78,53 +87,113 @@ export function formatValueForInput(value: unknown, type: string): string { return typeof value === 'string' ? value : JSON.stringify(value) } if (type === 'date' && value) { - const str = String(value) - const match = str.match(/^(\d{4})-(\d{2})-(\d{2})/) - if (match) return match[0] - try { - const date = new Date(str) - return date.toISOString().split('T')[0] - } catch { - return str - } + return storedDateToEditable(String(value)) } if (typeof value === 'object') return JSON.stringify(value) return String(value) } +/** A canonical date-cell value split into its wall-clock editing parts. */ +export interface DateCellLocalParts { + /** Calendar day `YYYY-MM-DD`, or null when the value is unparseable. */ + day: string | null + /** Time-of-day `HH:mm:ss`, or null for calendar-date values. */ + time: string | null +} + +/** + * Splits a canonical date-cell value into the day and time the date/time + * pickers edit — the value's **literal wall time**, no timezone conversion + * (display and editing are wall-clock-faithful for every viewer). Calendar + * dates have no time part; legacy strings normalize first. + */ +export function dateValueToLocalParts(value: string): DateCellLocalParts { + if (/^\d{4}-\d{2}-\d{2}$/.test(value)) return { day: value, time: null } + const wall = value.match( + /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/ + ) + if (wall) return { day: wall[1], time: `${wall[2]}:${wall[3] ?? '00'}` } + const canonical = normalizeDateCellValue(value) + if (!canonical || canonical === value) return { day: null, time: null } + return dateValueToLocalParts(canonical) +} + +/** + * Recombines picker-edited parts into a canonical date-cell value: a calendar + * date when there is no time, else that wall time stamped with the given + * zone's offset (runtime-local when omitted). + */ +export function localPartsToDateValue(day: string, time: string | null, timeZone?: string): string { + if (!time) return day + return normalizeDateCellValue(`${day}T${time}`, { timezone: timeZone }) ?? day +} + +/** Today's calendar day as `YYYY-MM-DD` in the given zone (runtime-local when omitted). */ +export function todayLocalCalendarDate(timeZone?: string): string { + const wall = getWallClockParts(new Date(), timeZone) + const pad = (n: number) => String(n).padStart(2, '0') + return `${wall.year}-${pad(wall.month)}-${pad(wall.day)}` +} + /** - * Convert a stored YYYY-MM-DD date string to MM/DD/YYYY display format. + * Format a stored date-cell value for display: calendar dates as MM/DD/YYYY, + * instants as their literal wall time `MM/DD/YYYY h:mm AM/PM` — identical + * for every viewer. Pass `seconds: true` for editor drafts so re-saving an + * untouched cell keeps second precision. */ -export function storageToDisplay(stored: string): string { - const match = stored.match(/^(\d{4})-(\d{2})-(\d{2})/) - if (match) return `${match[2]}/${match[3]}/${match[1]}` - return stored +export function storageToDisplay(stored: string, options?: { seconds?: boolean }): string { + return formatDateCellDisplay(stored, options) } /** - * Convert a MM/DD/YYYY (or MM/DD) display string back to YYYY-MM-DD storage format. + * Parse a date-cell input string to its canonical storage form: `YYYY-MM-DD` + * for date-only inputs (MM/DD/YYYY, MM/DD, ISO), an offset-preserved instant + * for inputs carrying a time. Naive times are stamped with the offset of + * `timeZone` (the writer's effective timezone; the runtime's zone when + * omitted). Returns null when unparseable. */ -export function displayToStorage(display: string): string | null { - const iso = display.match(/^(\d{4})-(\d{2})-(\d{2})$/) - if (iso) { - const month = Number(iso[2]) - const day = Number(iso[3]) - if (month < 1 || month > 12 || day < 1 || day > 31) return null - return display +export function displayToStorage(display: string, timeZone?: string): string | null { + const trimmed = display.trim() + const withTime = trimmed.match( + /^(\d{1,2})\/(\d{1,2})\/(\d{4})[ ,]+(\d{1,2}):(\d{2})(?::(\d{2}))?(?:\s*(AM|PM))?$/i + ) + if (withTime) { + const [, m, d, y, h, min, sec, meridiem] = withTime + let hours = Number(h) + if (meridiem) { + if (hours < 1 || hours > 12) return null + hours = (hours % 12) + (meridiem.toUpperCase() === 'PM' ? 12 : 0) + } else if (hours > 23) { + return null + } + if (Number(min) > 59 || Number(sec ?? 0) > 59) return null + if (!isValidCalendarDay(Number(y), Number(m), Number(d))) return null + const pad = (n: string) => n.padStart(2, '0') + // Route through the shared normalizer so the wall time resolves in the + // effective zone. + return normalizeDateCellValue( + `${y}-${pad(m)}-${pad(d)}T${String(hours).padStart(2, '0')}:${min}:${sec ?? '00'}`, + { timezone: timeZone } + ) } - const full = display.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/) + const full = trimmed.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/) if (full) { - const month = Number(full[1]) - const day = Number(full[2]) - if (month < 1 || month > 12 || day < 1 || day > 31) return null + if (!isValidCalendarDay(Number(full[3]), Number(full[1]), Number(full[2]))) return null return `${full[3]}-${full[1].padStart(2, '0')}-${full[2].padStart(2, '0')}` } - const partial = display.match(/^(\d{1,2})\/(\d{1,2})$/) + const partial = trimmed.match(/^(\d{1,2})\/(\d{1,2})$/) if (partial) { - const month = Number(partial[1]) - const day = Number(partial[2]) - if (month < 1 || month > 12 || day < 1 || day > 31) return null - return `${new Date().getFullYear()}-${partial[1].padStart(2, '0')}-${partial[2].padStart(2, '0')}` + const year = Number(todayLocalCalendarDate(timeZone).slice(0, 4)) + if (!isValidCalendarDay(year, Number(partial[1]), Number(partial[2]))) return null + return `${year}-${partial[1].padStart(2, '0')}-${partial[2].padStart(2, '0')}` } - return null + return normalizeDateCellValue(trimmed, { timezone: timeZone }) +} + +/** True when Y/M/D is a real calendar day — `Date` rolls impossible days over + * (02/30 → 03/02) instead of rejecting them, so compare the round-trip. */ +function isValidCalendarDay(year: number, month: number, day: number): boolean { + if (month < 1 || month > 12 || day < 1 || day > 31) return false + const check = new Date(year, month - 1, day) + return check.getMonth() === month - 1 && check.getDate() === day } diff --git a/apps/sim/hooks/queries/general-settings.ts b/apps/sim/hooks/queries/general-settings.ts index 1c110f49d68..26faf6ebfb7 100644 --- a/apps/sim/hooks/queries/general-settings.ts +++ b/apps/sim/hooks/queries/general-settings.ts @@ -7,7 +7,7 @@ import { type MothershipEnvironment, type UserSettingsApi, updateUserSettingsContract, -} from '@/lib/api/contracts' +} from '@/lib/api/contracts/user' import { syncThemeToNextThemes } from '@/lib/core/utils/theme' import { getBrowserTimezone } from '@/lib/core/utils/timezone' diff --git a/apps/sim/hooks/queries/tables.ts b/apps/sim/hooks/queries/tables.ts index e24ba44f9e0..a33f2e240c9 100644 --- a/apps/sim/hooks/queries/tables.ts +++ b/apps/sim/hooks/queries/tables.ts @@ -95,6 +95,7 @@ import { optimisticallyScheduleNewlyEligibleGroups, } from '@/lib/table/deps' import { runUploadStrategy } from '@/lib/uploads/client/direct-upload' +import { useTimezone } from '@/hooks/queries/general-settings' import { TABLE_LIST_STALE_TIME, type TableQueryScope, @@ -1428,6 +1429,7 @@ interface UploadCsvParams { */ export function useUploadCsvToTable() { const queryClient = useQueryClient() + const timezone = useTimezone() return useMutation({ mutationFn: async ({ workspaceId, file }: UploadCsvParams) => { @@ -1435,6 +1437,7 @@ export function useUploadCsvToTable() { // stream and needs workspaceId before it reaches the (large) file. const formData = new FormData() formData.append('workspaceId', workspaceId) + formData.append('timezone', timezone) formData.append('file', file) // boundary-raw-fetch: multipart/form-data CSV upload, requestJson only supports JSON bodies @@ -1492,11 +1495,12 @@ async function uploadCsvToWorkspaceStorage( */ export function useImportCsvAsync() { const queryClient = useQueryClient() + const timezone = useTimezone() return useMutation({ mutationFn: async ({ workspaceId, file, onProgress }: ImportCsvAsyncParams) => { const fileKey = await uploadCsvToWorkspaceStorage(file, workspaceId, onProgress) const response = await requestJson(importTableAsyncContract, { - body: { workspaceId, fileKey, fileName: file.name }, + body: { workspaceId, fileKey, fileName: file.name, timezone }, }) return response.data }, @@ -1525,10 +1529,11 @@ interface ImportFileAsTableParams { */ export function useImportFileAsTable() { const queryClient = useQueryClient() + const timezone = useTimezone() return useMutation({ mutationFn: async ({ workspaceId, fileKey, fileName }: ImportFileAsTableParams) => { const response = await requestJson(importTableAsyncContract, { - body: { workspaceId, fileKey, fileName, deleteSourceFile: false }, + body: { workspaceId, fileKey, fileName, deleteSourceFile: false, timezone }, }) return response.data }, @@ -1561,6 +1566,7 @@ interface ImportCsvIntoTableAsyncParams { */ export function useImportCsvIntoTableAsync() { const queryClient = useQueryClient() + const timezone = useTimezone() return useMutation({ mutationFn: async ({ workspaceId, @@ -1574,7 +1580,7 @@ export function useImportCsvIntoTableAsync() { const fileKey = await uploadCsvToWorkspaceStorage(file, workspaceId, onProgress) const response = await requestJson(importIntoTableAsyncContract, { params: { tableId }, - body: { workspaceId, fileKey, fileName: file.name, mode, mapping, createColumns }, + body: { workspaceId, fileKey, fileName: file.name, mode, mapping, createColumns, timezone }, }) return response.data }, @@ -1619,6 +1625,7 @@ interface ImportCsvIntoTableResponse { */ export function useImportCsvIntoTable() { const queryClient = useQueryClient() + const timezone = useTimezone() return useMutation({ mutationFn: async ({ @@ -1634,6 +1641,7 @@ export function useImportCsvIntoTable() { const formData = new FormData() formData.append('workspaceId', workspaceId) formData.append('mode', mode) + formData.append('timezone', timezone) if (mapping) { formData.append('mapping', JSON.stringify(mapping)) } diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index da9bc6b0214..d657619f710 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -1,6 +1,7 @@ import { isRecordLike } from '@sim/utils/object' import { z } from 'zod' import { type ContractJsonResponse, defineRouteContract } from '@/lib/api/contracts/types' +import { ianaTimezoneSchema } from '@/lib/api/contracts/user' import type { CsvHeaderMapping, EnrichmentRunDetail, @@ -404,6 +405,12 @@ export const importTableAsyncBodySchema = z.object({ * (e.g. the file viewer's "Import as a table") that must survive the import. */ deleteSourceFile: z.boolean().optional(), + /** + * IANA zone used to interpret naive datetime strings in the file (Excel and + * Sheets exports carry no offset). Defaults to the importing user's saved + * timezone, else UTC. + */ + timezone: ianaTimezoneSchema.optional(), }) export type ImportTableAsyncBody = z.input @@ -686,6 +693,12 @@ export const importIntoTableAsyncBodySchema = z.object({ mode: csvImportModeSchema, mapping: z.record(z.string(), z.string().nullable()).optional(), createColumns: z.array(z.string()).optional(), + /** + * IANA zone used to interpret naive datetime strings in the file (Excel and + * Sheets exports carry no offset). Defaults to the importing user's saved + * timezone, else UTC. + */ + timezone: ianaTimezoneSchema.optional(), }) export type ImportIntoTableAsyncBody = z.input diff --git a/apps/sim/lib/table/dates.test.ts b/apps/sim/lib/table/dates.test.ts new file mode 100644 index 00000000000..3ff51410e15 --- /dev/null +++ b/apps/sim/lib/table/dates.test.ts @@ -0,0 +1,158 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + formatDateCellDisplay, + isCalendarDateString, + normalizeDateCellValue, + storedDateToEditable, +} from '@/lib/table/dates' + +/** The runtime zone's offset suffix at a given local wall time, e.g. `-07:00`. */ +function localOffsetSuffix(local: Date): string { + const minutes = -local.getTimezoneOffset() + if (minutes === 0) return 'Z' + const sign = minutes > 0 ? '+' : '-' + const abs = Math.abs(minutes) + const pad = (n: number) => String(n).padStart(2, '0') + return `${sign}${pad(Math.floor(abs / 60))}:${pad(abs % 60)}` +} + +describe('isCalendarDateString', () => { + it('accepts YYYY-MM-DD and rejects everything else', () => { + expect(isCalendarDateString('2026-07-06')).toBe(true) + expect(isCalendarDateString('2026-13-45')).toBe(false) + expect(isCalendarDateString('2026-07-06T00:00:00Z')).toBe(false) + expect(isCalendarDateString('07/06/2026')).toBe(false) + }) +}) + +describe('normalizeDateCellValue', () => { + it('keeps calendar dates timezone-free', () => { + expect(normalizeDateCellValue('2026-07-06')).toBe('2026-07-06') + expect(normalizeDateCellValue(' 2026-07-06 ')).toBe('2026-07-06') + }) + + it('normalizes date-only inputs in other formats to calendar dates', () => { + expect(normalizeDateCellValue('07/06/2026')).toBe('2026-07-06') + expect(normalizeDateCellValue('7/6/2026')).toBe('2026-07-06') + expect(normalizeDateCellValue('July 6, 2026')).toBe('2026-07-06') + }) + + it('normalizes reduced-precision ISO forms via their UTC day', () => { + expect(normalizeDateCellValue('2026-07')).toBe('2026-07-01') + expect(normalizeDateCellValue('2026')).toBe('2026-01-01') + }) + + it('preserves the wall time and offset of explicit-offset inputs', () => { + expect(normalizeDateCellValue('2026-07-06T16:04:55-07:00')).toBe('2026-07-06T16:04:55-07:00') + expect(normalizeDateCellValue('2026-07-06 16:04:55 PDT')).toBe('2026-07-06T16:04:55-07:00') + expect(normalizeDateCellValue('2026-07-06T23:04:55.000Z')).toBe('2026-07-06T23:04:55Z') + expect(normalizeDateCellValue('2026-07-06 16:04:55+00')).toBe('2026-07-06T16:04:55Z') + expect(normalizeDateCellValue('2026-07-06 16:04:55 EST')).toBe('2026-07-06T16:04:55-05:00') + }) + + it('is idempotent on canonical instants', () => { + const canonical = '2026-07-06T16:04:55-07:00' + expect(normalizeDateCellValue(canonical)).toBe(canonical) + expect(normalizeDateCellValue(canonical, { timezone: 'Asia/Tokyo' })).toBe(canonical) + }) + + it('stamps naive datetimes with the runtime zone offset by default', () => { + const local = new Date(2026, 6, 6, 16, 4, 55) + expect(normalizeDateCellValue('2026-07-06 16:04:55')).toBe( + `2026-07-06T16:04:55${localOffsetSuffix(local)}` + ) + }) + + it('stamps naive datetimes with the provided IANA zone offset', () => { + // July → America/New_York is EDT (UTC-4) + expect(normalizeDateCellValue('2026-07-06 16:04:55', { timezone: 'America/New_York' })).toBe( + '2026-07-06T16:04:55-04:00' + ) + // January → EST (UTC-5); DST resolved per wall date, not per import date + expect(normalizeDateCellValue('2026-01-15 12:00', { timezone: 'America/New_York' })).toBe( + '2026-01-15T12:00:00-05:00' + ) + expect(normalizeDateCellValue('7/6/2026 4:04 PM', { timezone: 'America/Los_Angeles' })).toBe( + '2026-07-06T16:04:00-07:00' + ) + }) + + it('ignores the zone option when the input carries an explicit offset', () => { + expect( + normalizeDateCellValue('2026-07-06T23:04:55.000Z', { timezone: 'America/New_York' }) + ).toBe('2026-07-06T23:04:55Z') + expect( + normalizeDateCellValue('2026-07-06 16:04:55 PDT', { timezone: 'America/New_York' }) + ).toBe('2026-07-06T16:04:55-07:00') + }) + + it('leaves calendar dates untouched by the zone option', () => { + expect(normalizeDateCellValue('2026-07-06', { timezone: 'America/New_York' })).toBe( + '2026-07-06' + ) + }) + + it('throws on an invalid IANA zone', () => { + expect(() => normalizeDateCellValue('2026-07-06 12:00', { timezone: 'Not/AZone' })).toThrow( + RangeError + ) + }) + + it('returns null for unparseable input', () => { + expect(normalizeDateCellValue('not-a-date')).toBeNull() + expect(normalizeDateCellValue('')).toBeNull() + expect(normalizeDateCellValue('2026-13-45')).toBeNull() + expect(normalizeDateCellValue('13/06/2026')).toBeNull() + }) +}) + +describe('formatDateCellDisplay', () => { + it('renders calendar dates as MM/DD/YYYY', () => { + expect(formatDateCellDisplay('2026-07-06')).toBe('07/06/2026') + }) + + it('renders legacy UTC-midnight instants as their UTC calendar day', () => { + expect(formatDateCellDisplay('2026-07-06T00:00:00.000Z')).toBe('07/06/2026') + expect(formatDateCellDisplay('2026-07-06T00:00:00Z')).toBe('07/06/2026') + }) + + it('renders the literal wall time — identical for every viewer', () => { + expect(formatDateCellDisplay('2026-07-06T16:04:55-07:00')).toBe('07/06/2026 4:04 PM') + expect(formatDateCellDisplay('2026-07-06T16:04:55-07:00', { seconds: true })).toBe( + '07/06/2026 4:04:55 PM' + ) + // The offset never shifts the displayed wall time + expect(formatDateCellDisplay('2026-07-06T16:04:55+09:00')).toBe('07/06/2026 4:04 PM') + expect(formatDateCellDisplay('2026-07-06T23:04:55Z')).toBe('07/06/2026 11:04 PM') + expect(formatDateCellDisplay('2026-07-06T00:30:00-07:00')).toBe('07/06/2026 12:30 AM') + }) + + it('omits the seconds suffix when seconds are zero', () => { + expect(formatDateCellDisplay('2026-07-06T23:04:00Z', { seconds: true })).toBe( + '07/06/2026 11:04 PM' + ) + }) + + it('returns unparseable legacy strings as-is', () => { + expect(formatDateCellDisplay('garbage')).toBe('garbage') + }) +}) + +describe('storedDateToEditable', () => { + it('surfaces legacy UTC-midnight instants as their UTC calendar day', () => { + expect(storedDateToEditable('2026-07-06T00:00:00.000Z')).toBe('2026-07-06') + }) + + it('keeps calendar dates and canonicalizes instants', () => { + expect(storedDateToEditable('2026-07-06')).toBe('2026-07-06') + expect(storedDateToEditable('2026-07-06T16:04:55-07:00')).toBe('2026-07-06T16:04:55-07:00') + expect(storedDateToEditable('2026-07-06T23:04:55.000Z')).toBe('2026-07-06T23:04:55Z') + }) + + it('passes unparseable legacy strings through', () => { + expect(storedDateToEditable('garbage')).toBe('garbage') + }) +}) diff --git a/apps/sim/lib/table/dates.ts b/apps/sim/lib/table/dates.ts new file mode 100644 index 00000000000..a38eca7f21e --- /dev/null +++ b/apps/sim/lib/table/dates.ts @@ -0,0 +1,315 @@ +/** + * Canonical date-cell semantics for user tables. + * + * A `date` cell stores exactly one of two shapes: + * + * - **Calendar date** `YYYY-MM-DD` — a timezone-free day. + * - **Instant with preserved offset** — RFC 3339 `YYYY-MM-DDTHH:mm:ss±HH:MM` + * (or `Z`). The wall-time part is what was written and is what every viewer + * sees — display never converts across timezones. The offset suffix carries + * the true instant for machine consumers (SQL `::timestamptz` casts, + * workflows, agents, exports). + * + * The interpretation of an input is determined once, at write time: explicit + * offsets (`Z`, `-07:00`, `PDT`) are preserved as written; naive datetime + * strings are stamped with the offset of the writer's effective timezone + * (via {@link NormalizeDateCellOptions.timezone}), else the runtime's local + * zone — the browser for UI writes, the server (UTC in production) for raw + * API writes. After that the stored value is final: reads render its wall + * time verbatim, identically for everyone. + * + * This module is pure and shared by server coercion and client rendering. + * Client code must import it via this concrete path, never the `@/lib/table` + * barrel (the barrel is server-tainted). + */ + +const CALENDAR_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/ + +/** + * Canonical (or canonical-enough legacy) instant: a literal wall time with an + * optional fractional-seconds part and an optional offset suffix. The capture + * groups are the wall-time fields display renders verbatim. + */ +const WALL_INSTANT_PATTERN = + /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/ + +/** + * Legacy shape: old CSV imports stored date-only columns as UTC-midnight + * instants. Treated as calendar dates so historical rows render as pure days + * rather than a spurious "12:00 AM". + */ +const UTC_MIDNIGHT_PATTERN = /^\d{4}-\d{2}-\d{2}T00:00:00(\.000)?Z$/ + +/** A time-of-day component anywhere in the string (e.g. `16:04`). */ +const TIME_COMPONENT_PATTERN = /\d{1,2}:\d{2}/ + +/** + * ISO reduced-precision date forms (`2026`, `2026-07`) parse as UTC per spec, + * unlike other date-only forms which V8 parses as local time. + */ +const ISO_REDUCED_DATE_PATTERN = /^\d{4}(-\d{2})?$/ + +/** + * Fixed offsets (minutes east of UTC) for the RFC 2822 US timezone + * abbreviations — the only abbreviations `Date.parse` accepts, applied as + * literal offsets exactly as the engine does. + */ +const US_ABBREVIATION_OFFSET_MINUTES: Record = { + EST: -300, + EDT: -240, + CST: -360, + CDT: -300, + MST: -420, + MDT: -360, + PST: -480, + PDT: -420, +} + +/** True when `value` is a canonical timezone-free calendar date. */ +export function isCalendarDateString(value: string): boolean { + return CALENDAR_DATE_PATTERN.test(value) && !Number.isNaN(Date.parse(value)) +} + +/** A wall-clock reading of an instant in some timezone. */ +export interface WallClockParts { + year: number + /** 1-based month. */ + month: number + day: number + hour: number + minute: number + second: number +} + +/** + * The wall-clock reading of `date` in `timeZone` — or in the runtime's local + * zone when omitted. Throws a RangeError on an invalid IANA zone — callers + * validate at the boundary. + */ +export function getWallClockParts(date: Date, timeZone?: string): WallClockParts { + if (!timeZone) { + return { + year: date.getFullYear(), + month: date.getMonth() + 1, + day: date.getDate(), + hour: date.getHours(), + minute: date.getMinutes(), + second: date.getSeconds(), + } + } + const parts = new Intl.DateTimeFormat('en-US', { + timeZone, + hourCycle: 'h23', + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }).formatToParts(date) + const get = (type: string) => Number(parts.find((p) => p.type === type)?.value) + return { + year: get('year'), + month: get('month'), + day: get('day'), + hour: get('hour'), + minute: get('minute'), + second: get('second'), + } +} + +/** Offset of `timeZone` from UTC (ms east) at the moment `at`. */ +function zoneOffsetMs(timeZone: string, at: Date): number { + const wall = getWallClockParts(at, timeZone) + const asUtc = Date.UTC(wall.year, wall.month - 1, wall.day, wall.hour, wall.minute, wall.second) + return asUtc - at.getTime() +} + +/** + * Converts a wall-clock reading in `timeZone` to the UTC instant it denotes. + * Two-pass so readings near a DST transition resolve with the offset in + * force at that wall time. + */ +function wallTimeInZoneToUtc(wall: Date, timeZone: string): Date { + const guess = Date.UTC( + wall.getFullYear(), + wall.getMonth(), + wall.getDate(), + wall.getHours(), + wall.getMinutes(), + wall.getSeconds(), + wall.getMilliseconds() + ) + const adjusted = guess - zoneOffsetMs(timeZone, new Date(guess)) + return new Date(guess - zoneOffsetMs(timeZone, new Date(adjusted))) +} + +function pad(n: number): string { + return String(n).padStart(2, '0') +} + +function toLocalCalendarDate(date: Date): string { + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}` +} + +function toUtcCalendarDate(date: Date): string { + return `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}` +} + +/** `Z` for zero, else `±HH:MM`. */ +function formatOffsetSuffix(offsetMinutes: number): string { + if (offsetMinutes === 0) return 'Z' + const sign = offsetMinutes > 0 ? '+' : '-' + const abs = Math.abs(offsetMinutes) + return `${sign}${pad(Math.floor(abs / 60))}:${pad(abs % 60)}` +} + +/** + * Trailing offset (minutes east of UTC) of a datetime string, or null when + * naive. Recognizes exactly what `Date.parse` recognizes: numeric offsets, + * `Z`/`UT`/`UTC`/`GMT`, and the RFC 2822 US abbreviations. Deliberately does + * not match a trailing `AM`/`PM`. + */ +function extractExplicitOffsetMinutes(value: string): number | null { + const numeric = value.match(/([+-])(\d{1,2}):?(\d{2})?\s*$/) + if (numeric) { + const sign = numeric[1] === '-' ? -1 : 1 + return sign * (Number(numeric[2]) * 60 + Number(numeric[3] ?? 0)) + } + if (/(?:Z|UTC?|GMT)$/i.test(value)) return 0 + const abbreviation = value.match(/([ECMP][SD]T)$/i) + if (abbreviation) return US_ABBREVIATION_OFFSET_MINUTES[abbreviation[1].toUpperCase()] + return null +} + +/** Serializes UTC-read fields of `shifted` as a wall time with `offset`. */ +function formatUtcFieldsAsWall(shifted: Date, offsetMinutes: number): string { + return `${toUtcCalendarDate(shifted)}T${pad(shifted.getUTCHours())}:${pad( + shifted.getUTCMinutes() + )}:${pad(shifted.getUTCSeconds())}${formatOffsetSuffix(offsetMinutes)}` +} + +/** Serializes local-read fields of `parsed` as a wall time with `offset`. */ +function formatLocalFieldsAsWall(parsed: Date, offsetMinutes: number): string { + return `${toLocalCalendarDate(parsed)}T${pad(parsed.getHours())}:${pad( + parsed.getMinutes() + )}:${pad(parsed.getSeconds())}${formatOffsetSuffix(offsetMinutes)}` +} + +export interface NormalizeDateCellOptions { + /** + * IANA zone whose offset stamps naive datetime strings (no explicit + * offset), e.g. a CSV import applying the importing user's timezone. + * Defaults to the runtime's local zone — the author's wall clock in the + * browser, UTC on production servers. Throws a RangeError on an invalid + * zone. + */ + timezone?: string +} + +/** + * Normalizes a raw string to a canonical date-cell value, or `null` when it + * cannot be parsed. Date-only inputs become calendar dates; inputs carrying + * a time become offset-preserved instants: the wall time survives verbatim + * (explicit offsets kept as written, naive readings stamped per + * {@link NormalizeDateCellOptions.timezone}) — see module doc. + */ +export function normalizeDateCellValue( + raw: string, + options?: NormalizeDateCellOptions +): string | null { + const trimmed = raw.trim() + if (!trimmed) return null + if (CALENDAR_DATE_PATTERN.test(trimmed)) { + return Number.isNaN(Date.parse(trimmed)) ? null : trimmed + } + const ms = Date.parse(trimmed) + if (Number.isNaN(ms)) return null + const parsed = new Date(ms) + if (!TIME_COMPONENT_PATTERN.test(trimmed)) { + return ISO_REDUCED_DATE_PATTERN.test(trimmed) + ? toUtcCalendarDate(parsed) + : toLocalCalendarDate(parsed) + } + const explicitOffset = extractExplicitOffsetMinutes(trimmed) + if (explicitOffset !== null) { + // The input's own wall time = the instant shifted east by its offset, + // read as UTC fields. + return formatUtcFieldsAsWall(new Date(ms + explicitOffset * 60_000), explicitOffset) + } + if (options?.timezone) { + // `parsed`'s local getters recover the wall-clock fields V8 read from the + // naive string; stamp them with the requested zone's offset at that time. + const instant = wallTimeInZoneToUtc(parsed, options.timezone) + const offsetMinutes = Math.round(zoneOffsetMs(options.timezone, instant) / 60_000) + return formatLocalFieldsAsWall(parsed, offsetMinutes) + } + return formatLocalFieldsAsWall(parsed, -parsed.getTimezoneOffset()) +} + +/** + * Canonical form a stored date cell should be edited (and re-saved) as. + * Legacy UTC-midnight instants surface as their UTC calendar day (old CSV + * imports stored date-only columns that way). Unparseable legacy strings + * pass through so the editor shows what is actually stored. + */ +export function storedDateToEditable(stored: string): string { + if (UTC_MIDNIGHT_PATTERN.test(stored)) return toUtcCalendarDate(new Date(stored)) + return normalizeDateCellValue(stored) ?? stored +} + +interface FormatDateCellDisplayOptions { + /** Include seconds on instants when non-zero (editor drafts round-trip precision). */ + seconds?: boolean +} + +function formatWallForDisplay( + month: string, + day: string, + year: string, + hour: number, + minute: string, + second: number, + withSeconds: boolean | undefined +): string { + const hours12 = hour % 12 === 0 ? 12 : hour % 12 + const meridiem = hour < 12 ? 'AM' : 'PM' + const secondsPart = withSeconds && second !== 0 ? `:${pad(second)}` : '' + return `${month}/${day}/${year} ${hours12}:${minute}${secondsPart} ${meridiem}` +} + +/** + * Formats a stored date-cell value for display. Calendar dates (and legacy + * UTC-midnight instants) render as `MM/DD/YYYY`; instants render their + * **literal wall time** as `MM/DD/YYYY h:mm AM/PM` — identical for every + * viewer, no timezone conversion. Legacy strings that predate + * canonicalization render via a runtime-local normalization; unparseable + * ones are returned as-is. + */ +export function formatDateCellDisplay( + stored: string, + options?: FormatDateCellDisplayOptions +): string { + const calendar = stored.match(/^(\d{4})-(\d{2})-(\d{2})$/) + if (calendar) return `${calendar[2]}/${calendar[3]}/${calendar[1]}` + if (UTC_MIDNIGHT_PATTERN.test(stored)) { + const date = new Date(stored) + return `${pad(date.getUTCMonth() + 1)}/${pad(date.getUTCDate())}/${date.getUTCFullYear()}` + } + const wall = stored.match(WALL_INSTANT_PATTERN) + if (wall) { + const [, year, month, day, hour, minute, second] = wall + return formatWallForDisplay( + month, + day, + year, + Number(hour), + minute, + Number(second ?? 0), + options?.seconds + ) + } + const canonical = normalizeDateCellValue(stored) + if (!canonical) return stored + return formatDateCellDisplay(canonical, options) +} diff --git a/apps/sim/lib/table/import-runner.ts b/apps/sim/lib/table/import-runner.ts index 13887dd1c10..31086f34743 100644 --- a/apps/sim/lib/table/import-runner.ts +++ b/apps/sim/lib/table/import-runner.ts @@ -68,6 +68,12 @@ export interface TableImportPayload { * survive the import. */ deleteSourceFile?: boolean + /** + * IANA zone used to interpret naive datetime strings in the file. The + * kickoff routes resolve it (request → user setting → UTC) so the detached + * worker never needs a settings lookup. + */ + timezone?: string } /** @@ -199,7 +205,9 @@ export async function runTableImport(payload: TableImportPayload): Promise // may own. Runs per batch (not just at the emit cadence) so we stop within one batch. const owns = await updateJobProgress(tableId, inserted, importId) if (!owns) throw new ImportSupersededError() - const coerced = coerceRowsForTable(rows, schema, headerToColumn) + const coerced = coerceRowsForTable(rows, schema, headerToColumn, { + timezone: payload.timezone, + }) const rowLimit = await assertRowCapacity({ workspaceId, currentRowCount: existingRowCount + inserted, diff --git a/apps/sim/lib/table/import.test.ts b/apps/sim/lib/table/import.test.ts index d25ee031e0e..d4997a123fe 100644 --- a/apps/sim/lib/table/import.test.ts +++ b/apps/sim/lib/table/import.test.ts @@ -137,8 +137,12 @@ describe('import', () => { expect(coerceValue('yes', 'boolean')).toBeNull() }) - it('coerces dates to ISO strings and falls back to the original string', () => { - expect(coerceValue('2024-01-01', 'date')).toBe(new Date('2024-01-01').toISOString()) + it('keeps date-only values as calendar dates, preserves datetime wall times with their offset, and falls back to the original string', () => { + expect(coerceValue('2024-01-01', 'date')).toBe('2024-01-01') + expect(coerceValue('2024-01-01T12:30:00-07:00', 'date')).toBe('2024-01-01T12:30:00-07:00') + expect(coerceValue('2024-01-01 12:30', 'date', { timezone: 'America/New_York' })).toBe( + '2024-01-01T12:30:00-05:00' + ) expect(coerceValue('not-a-date', 'date')).toBe('not-a-date') }) }) diff --git a/apps/sim/lib/table/import.ts b/apps/sim/lib/table/import.ts index a132ba96dd7..104186e32e1 100644 --- a/apps/sim/lib/table/import.ts +++ b/apps/sim/lib/table/import.ts @@ -13,6 +13,7 @@ import { type Options as CsvParseOptions, type Parser, parse as parseCsvStream } from 'csv-parse' import { getColumnId } from '@/lib/table/column-keys' +import { type NormalizeDateCellOptions, normalizeDateCellValue } from '@/lib/table/dates' import type { ColumnDefinition, RowData, TableSchema } from '@/lib/table/types' /** @@ -218,7 +219,8 @@ export function inferSchemaFromCsv( */ export function coerceValue( value: unknown, - colType: CsvColumnType + colType: CsvColumnType, + options?: NormalizeDateCellOptions ): string | number | boolean | null | Record | unknown[] { if (value === null || value === undefined || value === '') return null switch (colType) { @@ -233,8 +235,7 @@ export function coerceValue( return null } case 'date': { - const d = new Date(String(value)) - return Number.isNaN(d.getTime()) ? String(value) : d.toISOString() + return normalizeDateCellValue(String(value), options) ?? String(value) } case 'json': { if (typeof value === 'object') return value as Record | unknown[] @@ -407,7 +408,8 @@ export function buildAutoMapping(csvHeaders: string[], tableSchema: TableSchema) export function coerceRowsForTable( rows: Record[], tableSchema: TableSchema, - headerToColumn: Map + headerToColumn: Map, + options?: NormalizeDateCellOptions ): RowData[] { const colByName = new Map(tableSchema.columns.map((c) => [c.name, c])) @@ -419,7 +421,7 @@ export function coerceRowsForTable( const col = colByName.get(colName) if (!col) continue const colType = (col.type as CsvColumnType) ?? 'string' - coerced[getColumnId(col)] = coerceValue(value, colType) as RowData[string] + coerced[getColumnId(col)] = coerceValue(value, colType, options) as RowData[string] } return coerced }) diff --git a/apps/sim/lib/table/index.ts b/apps/sim/lib/table/index.ts index c79a13f182c..58a73f2313f 100644 --- a/apps/sim/lib/table/index.ts +++ b/apps/sim/lib/table/index.ts @@ -9,6 +9,7 @@ export * from '@/lib/table/billing' export * from '@/lib/table/column-keys' export * from '@/lib/table/columns/service' export * from '@/lib/table/constants' +export * from '@/lib/table/dates' export * from '@/lib/table/import' export * from '@/lib/table/import-data' export * from '@/lib/table/jobs/service' diff --git a/apps/sim/lib/table/validation.ts b/apps/sim/lib/table/validation.ts index 0029f576845..df773160513 100644 --- a/apps/sim/lib/table/validation.ts +++ b/apps/sim/lib/table/validation.ts @@ -8,6 +8,7 @@ import { and, eq, or, type SQL, sql } from 'drizzle-orm' import { NextResponse } from 'next/server' import { getColumnId } from '@/lib/table/column-keys' import { COLUMN_TYPES, getMaxRowSizeBytes, NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants' +import { normalizeDateCellValue } from '@/lib/table/dates' import { withSeqscanOff } from '@/lib/table/planner' import type { ColumnDefinition, @@ -296,7 +297,10 @@ function coerceValueToColumnType( } return { ok: false } case 'date': { - if (typeof value === 'string' && !Number.isNaN(Date.parse(value))) return { ok: true, value } + if (typeof value === 'string') { + const normalized = normalizeDateCellValue(value) + return normalized === null ? { ok: false } : { ok: true, value: normalized } + } // Date instances and epoch numbers may still be out of the representable // range (>±8.64e15ms) — guard `toISOString()`, which throws RangeError on // an Invalid Date, so an over-range value degrades to `{ ok: false }` diff --git a/packages/emcn/src/components/calendar/calendar.test.ts b/packages/emcn/src/components/calendar/calendar.test.ts index f0e3360a21d..e3056173e45 100644 --- a/packages/emcn/src/components/calendar/calendar.test.ts +++ b/packages/emcn/src/components/calendar/calendar.test.ts @@ -2,7 +2,33 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { buildRangeBounds, formatDateRangeLabel, parseDateValue } from './calendar' +import { + buildRangeBounds, + formatDateRangeLabel, + parseDateTimeValue, + parseDateValue, +} from './calendar' + +describe('parseDateTimeValue', () => { + it('reads bare dates as pure days with no time', () => { + expect(parseDateTimeValue('2026-07-06').time).toBeNull() + }) + + it('keeps the time of T datetime strings, even at exactly midnight', () => { + expect(parseDateTimeValue('2026-07-07T00:00:00').time).toBe('00:00') + expect(parseDateTimeValue('2026-07-06T16:04:55').time).toBe('16:04:55') + expect(parseDateTimeValue('2026-07-06T16:04').time).toBe('16:04') + }) + + it('treats a coincidental local midnight as no time for Date instances', () => { + expect(parseDateTimeValue(new Date(2026, 6, 6)).time).toBeNull() + expect(parseDateTimeValue(new Date(2026, 6, 6, 16, 4, 55)).time).toBe('16:04:55') + }) + + it('returns nulls for unparseable input', () => { + expect(parseDateTimeValue('garbage')).toEqual({ date: null, time: null }) + }) +}) describe('parseDateValue', () => { it('parses a YYYY-MM-DD string as a local day', () => { diff --git a/packages/emcn/src/components/calendar/calendar.tsx b/packages/emcn/src/components/calendar/calendar.tsx index cb1af8e0fe7..e47fbd59b51 100644 --- a/packages/emcn/src/components/calendar/calendar.tsx +++ b/packages/emcn/src/components/calendar/calendar.tsx @@ -1,6 +1,6 @@ 'use client' -import { useEffect, useMemo, useState } from 'react' +import { useMemo, useState } from 'react' import { ChevronLeft, ChevronRight } from 'lucide-react' import { cn } from '../../lib/cn' import { Chip, chipVariants } from '../chip/chip' @@ -123,6 +123,44 @@ function extractTime(value: string | Date | undefined, fallback: string): string return typeof value === 'string' && value.includes('T') ? value.slice(11, 16) : fallback } +/** Local `HH:mm` (plus `:ss` when non-zero) time-of-day of a `Date`. */ +function timeOfDayFrom(date: Date): string { + const base = `${pad2(date.getHours())}:${pad2(date.getMinutes())}` + return date.getSeconds() > 0 ? `${base}:${pad2(date.getSeconds())}` : base +} + +/** + * Parses a date value into its local day plus an optional time-of-day. Bare + * `YYYY-MM-DD` strings are pure days (no time). Datetime strings parse through + * `Date` so an explicit offset (`Z`, `-07:00`) resolves to the **local** day — + * unlike {@link parseDateValue}'s date-slice fast path, which would read the + * UTC day. + * + * A `T` datetime string keeps its time even at exactly midnight — it was + * deliberately supplied with a time component, and dropping it would let a + * day-pick silently convert a midnight instant into a bare calendar date. + * The midnight-means-no-time reading applies only to `Date` instances and + * non-`T` strings, where a coincidental local midnight usually denotes a + * pure day. + */ +export function parseDateTimeValue(value: string | Date | undefined): { + date: Date | null + time: string | null +} { + if (!value) return { date: null, time: null } + if (typeof value === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(value)) { + return { date: parseDateValue(value), time: null } + } + const parsed = value instanceof Date ? value : new Date(value) + if (Number.isNaN(parsed.getTime())) return { date: null, time: null } + if (typeof value === 'string' && value.includes('T')) { + return { date: parsed, time: timeOfDayFrom(parsed) } + } + const isMidnight = + parsed.getHours() === 0 && parsed.getMinutes() === 0 && parsed.getSeconds() === 0 + return { date: parsed, time: isMidnight ? null : timeOfDayFrom(parsed) } +} + /** * Orders a start/end pair and serializes the range bounds to the wire format. * Without `showTime` the bounds are bare `YYYY-MM-DD` days; with it, the start @@ -164,10 +202,26 @@ interface CalendarBaseProps { interface CalendarSingleProps extends CalendarBaseProps { mode?: 'single' - /** Selected date as a `YYYY-MM-DD` string or `Date`. */ + /** Selected date as a `YYYY-MM-DD` (or datetime) string or `Date`. */ value?: string | Date - /** Called with the picked date in `YYYY-MM-DD` format. */ + /** + * Called with the picked date in `YYYY-MM-DD` format — or, with `showTime` + * and a set time, the local wall time `YYYY-MM-DDTHH:mm[:ss]`. + */ onChange?: (value: string) => void + /** + * Adds a time-of-day input under the grid. Day picks keep the current time + * (seconds included when the seeded value had them); time edits re-emit on + * the selected (or today's) day. Without a time set, day picks emit bare + * `YYYY-MM-DD` days. + */ + showTime?: boolean + /** + * Today's calendar day (`YYYY-MM-DD`) in the caller's effective timezone; + * drives the Today button and today ring. Defaults to the runtime's local + * day — pass this when the effective zone can differ from the browser's. + */ + today?: string } interface CalendarRangeProps extends CalendarBaseProps { @@ -205,6 +259,9 @@ export type CalendarProps = CalendarSingleProps | CalendarRangeProps * * * @example + * + * + * @example * */ export function Calendar(props: CalendarProps) { @@ -289,19 +346,52 @@ function WeekdayRow() { ) } -function SingleCalendarView({ value, onChange, className }: CalendarSingleProps) { - const selected = useMemo(() => parseDateValue(value), [value]) - const { today, view, setView, goToPrevMonth, goToNextMonth, cells } = useCalendarView(selected) +function SingleCalendarView({ + value, + onChange, + showTime = false, + today: todayValue, + className, +}: CalendarSingleProps) { + const parsed = useMemo(() => parseDateTimeValue(value), [value]) + const selected = parsed.date + const { + today: runtimeToday, + view, + setView, + goToPrevMonth, + goToNextMonth, + cells, + } = useCalendarView(selected) + const today = useMemo( + () => (todayValue ? (parseDateValue(todayValue) ?? runtimeToday) : runtimeToday), + [todayValue, runtimeToday] + ) - useEffect(() => { + const [timeOfDay, setTimeOfDay] = useState(parsed.time) + const [prevValue, setPrevValue] = useState(value) + if (value !== prevValue) { + setPrevValue(value) + setTimeOfDay(parsed.time) if (selected) setView({ month: selected.getMonth(), year: selected.getFullYear() }) - }, [selected, setView]) + } - const selectDay = (day: number) => onChange?.(toDateString(view.year, view.month, day)) + const emit = (year: number, month: number, day: number, time: string | null) => { + const dateStr = toDateString(year, month, day) + onChange?.(showTime && time ? `${dateStr}T${time}` : dateStr) + } + + const selectDay = (day: number) => emit(view.year, view.month, day, timeOfDay) const goToToday = () => { setView({ month: today.getMonth(), year: today.getFullYear() }) - onChange?.(toDateString(today.getFullYear(), today.getMonth(), today.getDate())) + emit(today.getFullYear(), today.getMonth(), today.getDate(), timeOfDay) + } + + const handleTimeChange = (time: string) => { + setTimeOfDay(time) + const base = selected ?? today + emit(base.getFullYear(), base.getMonth(), base.getDate(), time) } return ( @@ -332,6 +422,18 @@ function SingleCalendarView({ value, onChange, className }: CalendarSingleProps) })} + {showTime && ( +
+ Time + +
+ )} +