From 5d7f7e900e6a2652f5e9e43e01cb3110e968c26c Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Wed, 24 Jun 2026 00:04:32 -0700 Subject: [PATCH 01/14] improvement(pi): minor improvements to docs (#5192) --- apps/docs/content/docs/en/workflows/blocks/meta.json | 4 ++-- apps/docs/content/docs/en/workflows/blocks/pi.mdx | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/docs/content/docs/en/workflows/blocks/meta.json b/apps/docs/content/docs/en/workflows/blocks/meta.json index 567a0c3417f..8b5cd58012a 100644 --- a/apps/docs/content/docs/en/workflows/blocks/meta.json +++ b/apps/docs/content/docs/en/workflows/blocks/meta.json @@ -2,7 +2,6 @@ "title": "Core Blocks", "pages": [ "agent", - "pi", "api", "function", "condition", @@ -17,6 +16,7 @@ "human-in-the-loop", "variables", "wait", - "credential" + "credential", + "pi" ] } diff --git a/apps/docs/content/docs/en/workflows/blocks/pi.mdx b/apps/docs/content/docs/en/workflows/blocks/pi.mdx index f98095bdb1a..2bee2901742 100644 --- a/apps/docs/content/docs/en/workflows/blocks/pi.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/pi.mdx @@ -25,7 +25,7 @@ Pick the mode with the **Mode** dropdown. The fields below it change to match. Cloud runs entirely inside a disposable sandbox, so it never touches your machine. It clones the repo, lets the agent work with full read/shell/edit/git, pushes a branch, and opens a PR you review and merge. - Requires sandbox execution to be enabled (the Cloud option only appears when it is). -- Requires **your own provider API key (BYOK)** — the model key is handed to the sandbox, so Sim never injects a hosted key there. +- Requires **your own provider API key (BYOK)** — the model key is handed to the sandbox. - Needs a **GitHub token** with permission to clone, push, and open a PR (see [Setup](#setup-cloud)). - The deliverable is a **pull request** — nothing is committed to your default branch directly. @@ -118,7 +118,7 @@ The one case neither layer can rescue is a *first* prompt that already exceeds t ## Setup -### Cloud +### Cloud [#setup-cloud] Cloud runs in a sandbox image with the Pi CLI and git baked in. @@ -128,7 +128,7 @@ Cloud runs in a sandbox image with the Pi CLI and git baked in. - *Fine-grained:* select the repo, then **Contents: Read and write** + **Pull requests: Read and write**. - *Classic:* the **`repo`** scope. For org repos, authorize the token for SSO. -### Local +### Local [#setup-local] 1. **Enable SSH** on the target machine (on macOS: System Settings → General → Sharing → Remote Login). 2. **Expose it on a public host.** Sim blocks `localhost`/LAN, so use a TCP tunnel — for example `ngrok tcp 22`, which gives a `host:port` to put in **Host** and **Port**. From b212a5d37d05ae05cd2ad35404ab7390e96fdacb Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 00:12:20 -0700 Subject: [PATCH 02/14] improvement(mistral): update OCR pricing to OCR 4 rate ($4/1,000 pages) (#5193) * improvement(mistral): update OCR pricing to OCR 4 rate ($4/1,000 pages) * docs(mistral): document mistral-ocr-latest alias resolves to OCR 4 --- apps/sim/tools/mistral/parser.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/apps/sim/tools/mistral/parser.ts b/apps/sim/tools/mistral/parser.ts index c0e7ee5e11d..948e2753f92 100644 --- a/apps/sim/tools/mistral/parser.ts +++ b/apps/sim/tools/mistral/parser.ts @@ -12,6 +12,19 @@ import type { ToolConfig } from '@/tools/types' const logger = createLogger('MistralParserTool') +/** + * Mistral OCR 4 standard pricing, in USD per page ($4 per 1,000 pages). + * + * This tool calls the synchronous `/v1/ocr` endpoint with the `mistral-ocr-latest` + * alias, which Mistral repointed to OCR 4 (`mistral-ocr-4-0`) on 2026-06-23, so the + * standard (non-batch) OCR 4 rate applies. Document AI / annotation pages are priced + * separately, but this tool does not submit annotation requests. + * + * @see https://mistral.ai/news/ocr-4/ + * @see https://docs.mistral.ai/getting-started/changelog + */ +const MISTRAL_OCR_COST_PER_PAGE = 0.004 + const MISTRAL_OCR_HOSTING = { envKeyPrefix: 'MISTRAL_API_KEY', apiKeyParam: 'apiKey', @@ -19,9 +32,6 @@ const MISTRAL_OCR_HOSTING = { pricing: { type: 'custom' as const, getCost: (_params: unknown, output: Record) => { - // Mistral OCR 3 standard pricing: $2 per 1,000 pages ($0.002/page). - // Annotated pages are priced separately at $3 per 1,000 annotated pages, but this tool does - // not submit annotation requests. Source: https://docs.mistral.ai/models/ocr-3-25-12 const rawUsageInfo = output.usage_info as { pages_processed?: number } | undefined const transformedUsageInfo = ( output.metadata as { usageInfo?: { pagesProcessed?: number } } | undefined @@ -33,7 +43,7 @@ const MISTRAL_OCR_HOSTING = { 'Mistral OCR response missing pages_processed in usage_info or metadata.usageInfo.pagesProcessed' ) } - const cost = pagesProcessed * 0.002 + const cost = pagesProcessed * MISTRAL_OCR_COST_PER_PAGE return { cost, metadata: { pagesProcessed } } }, }, From 86dc04d789640d852ec87d6eeb538490df398b41 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 10:44:57 -0700 Subject: [PATCH 03/14] perf(workspace): server-prefetch home, knowledge, tables, and files list pages (#5196) --- .../workspace/[workspaceId]/files/page.tsx | 18 +- .../workspace/[workspaceId]/files/prefetch.ts | 43 +++++ .../app/workspace/[workspaceId]/home/page.tsx | 20 ++- .../workspace/[workspaceId]/home/prefetch.ts | 48 +++++ .../[workspaceId]/knowledge/page.tsx | 20 ++- .../[workspaceId]/knowledge/prefetch.ts | 28 +++ .../lib/prefetch-internal-fetch.ts | 25 +++ .../[workspaceId]/lib/prefetch.test.ts | 169 ++++++++++++++++++ .../workspace/[workspaceId]/tables/page.tsx | 18 +- .../[workspaceId]/tables/prefetch.ts | 26 +++ apps/sim/hooks/queries/folders.ts | 10 +- 11 files changed, 411 insertions(+), 14 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/prefetch.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/prefetch.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/lib/prefetch-internal-fetch.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts diff --git a/apps/sim/app/workspace/[workspaceId]/files/page.tsx b/apps/sim/app/workspace/[workspaceId]/files/page.tsx index ab21f2f3b72..2ed876e4ba0 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/page.tsx @@ -1,5 +1,8 @@ import { Suspense } from 'react' +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' +import { prefetchFilesBrowser } from '@/app/workspace/[workspaceId]/files/prefetch' import { Files } from './files' import FilesLoading from './loading' @@ -15,10 +18,17 @@ export const metadata: Metadata = { * table headers) so a suspend never shows a blank frame; the route-level * `loading.tsx` covers the navigation/chunk-load transition the same way. */ -export default function FilesPage() { +export default async function FilesPage({ params }: { params: Promise<{ workspaceId: string }> }) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + await prefetchFilesBrowser(queryClient, workspaceId) + return ( - }> - - + + }> + + + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts new file mode 100644 index 00000000000..8780aa537fa --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts @@ -0,0 +1,43 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { WorkspaceFileFolderApi } from '@/lib/api/contracts/workspace-file-folders' +import type { ListWorkspaceFilesResponse } from '@/lib/api/contracts/workspace-files' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' +import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' + +/** + * Prefetches the Files browser's two lists — workspace files and file folders — + * under the same query keys their client hooks (`useWorkspaceFiles`, + * `useWorkspaceFileFolders`) use (scope `active`), so the browser paints + * populated on first render. + * + * Both payloads carry `Date` fields, so they go through their routes and cache + * the serialized wire shape — see {@link prefetchInternalJson}. + */ +export async function prefetchFilesBrowser( + queryClient: QueryClient, + workspaceId: string +): Promise { + await Promise.all([ + queryClient.prefetchQuery({ + queryKey: workspaceFilesKeys.list(workspaceId, 'active'), + queryFn: async () => { + const data = await prefetchInternalJson( + `/api/workspaces/${workspaceId}/files?scope=active` + ) + return data.success ? data.files : [] + }, + staleTime: 30 * 1000, + }), + queryClient.prefetchQuery({ + queryKey: workspaceFileFolderKeys.list(workspaceId, 'active'), + queryFn: async () => { + const data = await prefetchInternalJson<{ folders?: WorkspaceFileFolderApi[] }>( + `/api/workspaces/${workspaceId}/files/folders?scope=active` + ) + return data.folders ?? [] + }, + staleTime: 30 * 1000, + }), + ]) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/page.tsx b/apps/sim/app/workspace/[workspaceId]/home/page.tsx index e29acc640ee..13595d65398 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/page.tsx @@ -1,6 +1,9 @@ import { Suspense } from 'react' +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' import { getSession } from '@/lib/auth' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' +import { prefetchHomeLists } from '@/app/workspace/[workspaceId]/home/prefetch' import { Home } from './home' import { HomeFallback } from './home-fallback' @@ -8,11 +11,20 @@ export const metadata: Metadata = { title: 'New chat', } -export default async function HomePage() { +export default async function HomePage({ params }: { params: Promise<{ workspaceId: string }> }) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + const listsPrefetch = prefetchHomeLists(queryClient, workspaceId) + const session = await getSession() + await listsPrefetch + return ( - }> - - + + }> + + + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts new file mode 100644 index 00000000000..956a9e95555 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts @@ -0,0 +1,48 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { FolderApi } from '@/lib/api/contracts' +import type { ListWorkspaceFilesResponse } from '@/lib/api/contracts/workspace-files' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { FOLDER_LIST_STALE_TIME, mapFolder } from '@/hooks/queries/folders' +import { folderKeys } from '@/hooks/queries/utils/folder-keys' +import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' + +/** + * Prefetches the home page's secondary lists — folders and workspace files — + * under the same query keys their client hooks (`useFolders`, + * `useWorkspaceFiles`) use, so the home view paints populated on first render. + * + * The workflow list (`workflowKeys.list(ws, 'active')`) is already hydrated by + * the workspace sidebar prefetch and is intentionally not repeated here. + * + * Folders are fetched through the route and mapped with the same `mapFolder` + * the hook applies, matching its cached shape (string dates → `Date`). Files + * carry `Date` fields, so they go through the route and cache the serialized + * wire shape — see {@link prefetchInternalJson}. + */ +export async function prefetchHomeLists( + queryClient: QueryClient, + workspaceId: string +): Promise { + await Promise.all([ + queryClient.prefetchQuery({ + queryKey: folderKeys.list(workspaceId, 'active'), + queryFn: async () => { + const { folders } = await prefetchInternalJson<{ folders?: FolderApi[] }>( + `/api/folders?workspaceId=${workspaceId}&scope=active` + ) + return (folders ?? []).map(mapFolder) + }, + staleTime: FOLDER_LIST_STALE_TIME, + }), + queryClient.prefetchQuery({ + queryKey: workspaceFilesKeys.list(workspaceId, 'active'), + queryFn: async () => { + const data = await prefetchInternalJson( + `/api/workspaces/${workspaceId}/files?scope=active` + ) + return data.success ? data.files : [] + }, + staleTime: 30 * 1000, + }), + ]) +} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx index be3743be659..6243dc42035 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx @@ -1,8 +1,26 @@ +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' +import { prefetchKnowledgeBases } from '@/app/workspace/[workspaceId]/knowledge/prefetch' import { Knowledge } from './knowledge' export const metadata: Metadata = { title: 'Knowledge Base', } -export default Knowledge +export default async function KnowledgePage({ + params, +}: { + params: Promise<{ workspaceId: string }> +}) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + await prefetchKnowledgeBases(queryClient, workspaceId) + + return ( + + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts new file mode 100644 index 00000000000..0e8e5578840 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts @@ -0,0 +1,28 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { KnowledgeBaseData } from '@/lib/api/contracts/knowledge' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { knowledgeKeys } from '@/hooks/queries/kb/knowledge' + +/** + * Prefetches the workspace's knowledge-bases list under the same query key the + * client `useKnowledgeBasesQuery` hook uses (scope `active`), so the list paints + * populated on first render. + * + * The list carries `Date` fields, so it goes through the `/api/knowledge` route + * and caches the serialized wire shape — see {@link prefetchInternalJson}. + */ +export async function prefetchKnowledgeBases( + queryClient: QueryClient, + workspaceId: string +): Promise { + await queryClient.prefetchQuery({ + queryKey: knowledgeKeys.list(workspaceId, 'active'), + queryFn: async () => { + const result = await prefetchInternalJson<{ data: KnowledgeBaseData[] }>( + `/api/knowledge?workspaceId=${workspaceId}&scope=active` + ) + return result.data + }, + staleTime: 60 * 1000, + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/lib/prefetch-internal-fetch.ts b/apps/sim/app/workspace/[workspaceId]/lib/prefetch-internal-fetch.ts new file mode 100644 index 00000000000..e48f6064c17 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/lib/prefetch-internal-fetch.ts @@ -0,0 +1,25 @@ +import { headers } from 'next/headers' +import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' + +/** + * Server-side GET against an internal `/api` route, forwarding the incoming + * request's cookie so the route authenticates as the current user. + * + * List prefetches go through the route (rather than the data layer) when the + * payload carries `Date` fields: `NextResponse.json` serializes them to the + * string wire shape the client caches via `requestJson`, so the + * server-hydrated entry byte-matches the client-fetched one through + * dehydration. Calling the data layer directly would cache raw `Date` objects + * and drift from that wire shape. Mirrors the settings/subscription prefetch. + */ +export async function prefetchInternalJson(path: string): Promise { + const cookie = (await headers()).get('cookie') + // boundary-raw-fetch: server-side RSC prefetch forwarding the session cookie to an internal API route; requestJson is client-only and cannot run here + const response = await fetch(`${getInternalApiBaseUrl()}${path}`, { + headers: cookie ? { cookie } : {}, + }) + if (!response.ok) { + throw new Error(`Prefetch failed for ${path}: ${response.status}`) + } + return response.json() as Promise +} diff --git a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts new file mode 100644 index 00000000000..d031c0648ef --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts @@ -0,0 +1,169 @@ +/** + * @vitest-environment node + */ +import { QueryClient } from '@tanstack/react-query' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockPrefetchInternalJson } = vi.hoisted(() => ({ + mockPrefetchInternalJson: vi.fn(), +})) + +vi.mock('@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch', () => ({ + prefetchInternalJson: mockPrefetchInternalJson, +})) + +vi.mock('@/components/emcn', () => ({ + toast: { success: vi.fn(), error: vi.fn() }, +})) + +import { prefetchFilesBrowser } from '@/app/workspace/[workspaceId]/files/prefetch' +import { prefetchHomeLists } from '@/app/workspace/[workspaceId]/home/prefetch' +import { prefetchKnowledgeBases } from '@/app/workspace/[workspaceId]/knowledge/prefetch' +import { prefetchTables } from '@/app/workspace/[workspaceId]/tables/prefetch' +import { knowledgeKeys } from '@/hooks/queries/kb/knowledge' +import { tableKeys } from '@/hooks/queries/tables' +import { folderKeys } from '@/hooks/queries/utils/folder-keys' +import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' +import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' + +const WORKSPACE_ID = 'ws-123' + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }) +} + +describe('workspace list prefetches', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + describe('prefetchKnowledgeBases', () => { + it('primes the exact key useKnowledgeBasesQuery reads and unwraps data', async () => { + const bases = [{ id: 'kb-1' }] + mockPrefetchInternalJson.mockResolvedValue({ data: bases }) + const client = makeClient() + + await prefetchKnowledgeBases(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/knowledge?workspaceId=${WORKSPACE_ID}&scope=active` + ) + expect(client.getQueryData(knowledgeKeys.list(WORKSPACE_ID, 'active'))).toEqual(bases) + }) + }) + + describe('prefetchTables', () => { + it('primes the exact key useTablesList reads and unwraps data.tables', async () => { + const tables = [{ id: 't-1' }] + mockPrefetchInternalJson.mockResolvedValue({ data: { tables } }) + const client = makeClient() + + await prefetchTables(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/table?workspaceId=${WORKSPACE_ID}&scope=active` + ) + expect(client.getQueryData(tableKeys.list(WORKSPACE_ID, 'active'))).toEqual(tables) + }) + }) + + describe('prefetchFilesBrowser', () => { + it('primes both file + folder keys the client hooks read', async () => { + const files = [{ id: 'f-1' }] + const folders = [{ id: 'folder-1' }] + mockPrefetchInternalJson.mockImplementation(async (path: string) => + path.includes('/folders') ? { folders } : { success: true, files } + ) + const client = makeClient() + + await prefetchFilesBrowser(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/workspaces/${WORKSPACE_ID}/files?scope=active` + ) + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/workspaces/${WORKSPACE_ID}/files/folders?scope=active` + ) + expect(client.getQueryData(workspaceFilesKeys.list(WORKSPACE_ID, 'active'))).toEqual(files) + expect(client.getQueryData(workspaceFileFolderKeys.list(WORKSPACE_ID, 'active'))).toEqual( + folders + ) + }) + + it('caches an empty file list when the route reports failure', async () => { + mockPrefetchInternalJson.mockImplementation(async (path: string) => + path.includes('/folders') ? { folders: [] } : { success: false, files: [] } + ) + const client = makeClient() + + await prefetchFilesBrowser(client, WORKSPACE_ID) + + expect(client.getQueryData(workspaceFilesKeys.list(WORKSPACE_ID, 'active'))).toEqual([]) + }) + }) + + describe('prefetchHomeLists', () => { + it('primes folder + file keys, mapping folder rows to the client shape', async () => { + const folderRow = { + id: 'folder-1', + name: 'Docs', + userId: 'u-1', + workspaceId: WORKSPACE_ID, + parentId: null, + color: null, + isExpanded: true, + locked: false, + sortOrder: 0, + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-02T00:00:00.000Z', + archivedAt: null, + } + const files = [{ id: 'f-1' }] + mockPrefetchInternalJson.mockImplementation(async (path: string) => + path.startsWith('/api/folders') ? { folders: [folderRow] } : { success: true, files } + ) + const client = makeClient() + + await prefetchHomeLists(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/folders?workspaceId=${WORKSPACE_ID}&scope=active` + ) + const cachedFolders = client.getQueryData(folderKeys.list(WORKSPACE_ID, 'active')) as Array<{ + id: string + color: string + createdAt: Date + }> + expect(cachedFolders).toHaveLength(1) + expect(cachedFolders[0].color).toBe('#6B7280') + expect(cachedFolders[0].createdAt).toBeInstanceOf(Date) + expect(client.getQueryData(workspaceFilesKeys.list(WORKSPACE_ID, 'active'))).toEqual(files) + }) + }) + + describe('graceful failure', () => { + it.each([ + [ + 'prefetchKnowledgeBases', + prefetchKnowledgeBases, + knowledgeKeys.list(WORKSPACE_ID, 'active'), + ], + ['prefetchTables', prefetchTables, tableKeys.list(WORKSPACE_ID, 'active')], + ['prefetchHomeLists', prefetchHomeLists, folderKeys.list(WORKSPACE_ID, 'active')], + [ + 'prefetchFilesBrowser', + prefetchFilesBrowser, + workspaceFilesKeys.list(WORKSPACE_ID, 'active'), + ], + ] as const)( + '%s does not throw when the fetcher rejects (page still renders, client refetches)', + async (_name, prefetch, queryKey) => { + mockPrefetchInternalJson.mockRejectedValue(new Error('500')) + const client = makeClient() + + await expect(prefetch(client, WORKSPACE_ID)).resolves.toBeUndefined() + expect(client.getQueryData(queryKey)).toBeUndefined() + } + ) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/page.tsx b/apps/sim/app/workspace/[workspaceId]/tables/page.tsx index 15a016a25a2..0e9390a5d95 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/page.tsx @@ -1,6 +1,9 @@ import { Suspense } from 'react' +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' import TablesLoading from '@/app/workspace/[workspaceId]/tables/loading' +import { prefetchTables } from '@/app/workspace/[workspaceId]/tables/prefetch' import { Tables } from './tables' export const metadata: Metadata = { @@ -13,10 +16,17 @@ export const metadata: Metadata = { * fallback renders the real chrome so a suspend never shows a blank frame; the * route-level `loading.tsx` covers the navigation/chunk-load transition. */ -export default function TablesPage() { +export default async function TablesPage({ params }: { params: Promise<{ workspaceId: string }> }) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + await prefetchTables(queryClient, workspaceId) + return ( - }> - - + + }> + + + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts new file mode 100644 index 00000000000..60d6a79a735 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts @@ -0,0 +1,26 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { TableDefinition } from '@/lib/table' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { tableKeys } from '@/hooks/queries/tables' + +/** + * Prefetches the workspace's tables list under the same query key the client + * `useTablesList` hook uses (scope `active`), so the list paints populated on + * first render. + * + * Table definitions carry `Date` fields, so the list goes through the + * `/api/table` route and caches the serialized wire shape — see + * {@link prefetchInternalJson}. + */ +export async function prefetchTables(queryClient: QueryClient, workspaceId: string): Promise { + await queryClient.prefetchQuery({ + queryKey: tableKeys.list(workspaceId, 'active'), + queryFn: async () => { + const response = await prefetchInternalJson<{ data: { tables: TableDefinition[] } }>( + `/api/table?workspaceId=${workspaceId}&scope=active` + ) + return response.data.tables + }, + staleTime: 30 * 1000, + }) +} diff --git a/apps/sim/hooks/queries/folders.ts b/apps/sim/hooks/queries/folders.ts index f0d8913cd05..34b695374fd 100644 --- a/apps/sim/hooks/queries/folders.ts +++ b/apps/sim/hooks/queries/folders.ts @@ -25,7 +25,15 @@ import type { WorkflowFolder } from '@/stores/folders/types' const logger = createLogger('FolderQueries') -function mapFolder(folder: FolderApi): WorkflowFolder { +export const FOLDER_LIST_STALE_TIME = 60 * 1000 + +/** + * Maps a wire folder row to the client `WorkflowFolder` shape (string dates → + * `Date`, color default). Exported so the server-side home prefetch produces + * the exact cached value `useFolders` stores, keeping the hydrated entry in + * sync with a client fetch. + */ +export function mapFolder(folder: FolderApi): WorkflowFolder { return { id: folder.id, name: folder.name, From b52fcc094e060dce715549fcefc7215542f31b88 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 10:45:33 -0700 Subject: [PATCH 04/14] refactor(stores): model execution and workflow-diff state as status enums (#5197) --- apps/sim/hooks/use-undo-redo.ts | 21 +- apps/sim/stores/execution/store.test.ts | 192 +++++++++++++++-- apps/sim/stores/execution/store.ts | 31 ++- apps/sim/stores/execution/types.ts | 77 ++++++- apps/sim/stores/workflow-diff/store.test.ts | 228 ++++++++++++++++++++ apps/sim/stores/workflow-diff/store.ts | 37 ++-- apps/sim/stores/workflow-diff/types.ts | 50 +++++ apps/sim/vitest.setup.ts | 3 + 8 files changed, 569 insertions(+), 70 deletions(-) create mode 100644 apps/sim/stores/workflow-diff/store.test.ts diff --git a/apps/sim/hooks/use-undo-redo.ts b/apps/sim/hooks/use-undo-redo.ts index 025c087de0e..22dfdaa6e0e 100644 --- a/apps/sim/hooks/use-undo-redo.ts +++ b/apps/sim/hooks/use-undo-redo.ts @@ -41,6 +41,7 @@ import { type UpdateParentOperation, useUndoRedoStore, } from '@/stores/undo-redo' +import { deriveDiffFlags } from '@/stores/workflow-diff/types' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { useSubBlockStore } from '@/stores/workflows/subblock/store' import { useWorkflowStore } from '@/stores/workflows/workflow/store' @@ -1234,9 +1235,7 @@ export function useUndoRedo() { // Restore diff state with baseline (local UI only) diffStore._batchedStateUpdate({ - hasActiveDiff: true, - isShowingDiff: true, - isDiffReady: true, + ...deriveDiffFlags('showing'), baselineWorkflow: originalBaseline || null, baselineWorkflowId: activeWorkflowId, diffAnalysis: diffAnalysis, @@ -1285,9 +1284,7 @@ export function useUndoRedo() { // Restore diff state with baseline (local UI only) const diffStore = useWorkflowDiffStore.getState() diffStore._batchedStateUpdate({ - hasActiveDiff: true, - isShowingDiff: true, - isDiffReady: true, + ...deriveDiffFlags('showing'), baselineWorkflow: baselineSnapshot || null, baselineWorkflowId: activeWorkflowId, diffAnalysis: diffAnalysis, @@ -1805,9 +1802,7 @@ export function useUndoRedo() { // Restore diff state with original baseline (local UI only) diffStore._batchedStateUpdate({ - hasActiveDiff: true, - isShowingDiff: true, - isDiffReady: true, + ...deriveDiffFlags('showing'), baselineWorkflow: baselineSnapshot, baselineWorkflowId: activeWorkflowId, diffAnalysis: diffAnalysis, @@ -1834,9 +1829,7 @@ export function useUndoRedo() { // Clear diff state FIRST to prevent flash of colors (local UI only) // Use setState directly to ensure synchronous clearing useWorkflowDiffStore.setState({ - hasActiveDiff: false, - isShowingDiff: false, - isDiffReady: false, + ...deriveDiffFlags('none'), baselineWorkflow: null, baselineWorkflowId: null, diffAnalysis: null, @@ -1886,9 +1879,7 @@ export function useUndoRedo() { // Clear diff state FIRST to prevent flash of colors (local UI only) // Use setState directly to ensure synchronous clearing useWorkflowDiffStore.setState({ - hasActiveDiff: false, - isShowingDiff: false, - isDiffReady: false, + ...deriveDiffFlags('none'), baselineWorkflow: null, baselineWorkflowId: null, diffAnalysis: null, diff --git a/apps/sim/stores/execution/store.test.ts b/apps/sim/stores/execution/store.test.ts index 6888894b80b..c0b11fc87ce 100644 --- a/apps/sim/stores/execution/store.test.ts +++ b/apps/sim/stores/execution/store.test.ts @@ -1,4 +1,6 @@ /** + * @vitest-environment node + * * Tests for the per-workflow execution store. * * These tests cover: @@ -7,12 +9,19 @@ * - Execution lifecycle (start/stop clears run path) * - Block and edge run status tracking * - Active block management - * - Debug state management + * - The {@link ExecutionStatus} enum and its derived `isExecuting` / + * `isDebugging` booleans (exhaustive status → flag mapping + transitions) * - Execution snapshot management * - Store reset * - Immutability guarantees * * @remarks + * The store under test transitively imports the workflow registry store, + * which drags in the block registry and emcn icon CSS. To keep this a true + * unit test that loads under the node environment, the registry store is + * mocked to a minimal stub (the store actions never touch it — only the + * convenience hooks do, which are not exercised here). + * * Most tests use `it.concurrent` with unique workflow IDs per test. * Because the store isolates state by workflow ID, concurrent tests * do not interfere with each other. The `reset` and `immutability` @@ -21,17 +30,30 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/stores/workflows/registry/store', () => ({ + useWorkflowRegistry: Object.assign( + vi.fn(() => null), + { getState: vi.fn(() => ({ activeWorkflowId: null })) } + ), +})) + vi.unmock('@/stores/execution/store') vi.unmock('@/stores/execution/types') import { useExecutionStore } from '@/stores/execution/store' -import { defaultWorkflowExecutionState, initialState } from '@/stores/execution/types' +import { + defaultWorkflowExecutionState, + deriveExecutionFlags, + type ExecutionStatus, + initialState, +} from '@/stores/execution/types' describe('useExecutionStore', () => { describe('getWorkflowExecution', () => { it.concurrent('should return default state for an unknown workflow', () => { const state = useExecutionStore.getState().getWorkflowExecution('wf-get-default') + expect(state.status).toBe('idle') expect(state.isExecuting).toBe(false) expect(state.isDebugging).toBe(false) expect(state.activeBlockIds.size).toBe(0) @@ -63,22 +85,35 @@ describe('useExecutionStore', () => { }) }) + describe('deriveExecutionFlags', () => { + it.concurrent('maps every status to the documented legacy booleans', () => { + const cases: Array<[ExecutionStatus, boolean, boolean]> = [ + ['idle', false, false], + ['running', true, false], + ['debugging', true, true], + ] + for (const [status, isExecuting, isDebugging] of cases) { + expect(deriveExecutionFlags(status)).toEqual({ isExecuting, isDebugging }) + } + }) + }) + describe('setIsExecuting', () => { - it.concurrent('should set isExecuting to true', () => { + it.concurrent('should set isExecuting to true (status running)', () => { useExecutionStore.getState().setIsExecuting('wf-exec-true', true) - expect(useExecutionStore.getState().getWorkflowExecution('wf-exec-true').isExecuting).toBe( - true - ) + const state = useExecutionStore.getState().getWorkflowExecution('wf-exec-true') + expect(state.isExecuting).toBe(true) + expect(state.status).toBe('running') }) - it.concurrent('should set isExecuting to false', () => { + it.concurrent('should set isExecuting to false (status idle)', () => { useExecutionStore.getState().setIsExecuting('wf-exec-false', true) useExecutionStore.getState().setIsExecuting('wf-exec-false', false) - expect(useExecutionStore.getState().getWorkflowExecution('wf-exec-false').isExecuting).toBe( - false - ) + const state = useExecutionStore.getState().getWorkflowExecution('wf-exec-false') + expect(state.isExecuting).toBe(false) + expect(state.status).toBe('idle') }) it.concurrent('should clear lastRunPath and lastRunEdges when starting execution', () => { @@ -107,6 +142,131 @@ describe('useExecutionStore', () => { expect(state.isExecuting).toBe(false) expect(state.lastRunPath.get('block-1')).toBe('success') }) + + it.concurrent('starting a debug run then setIsExecuting(true) clears the run path', () => { + const wf = 'wf-exec-debug-start-clears' + useExecutionStore.getState().setIsExecuting(wf, true) + useExecutionStore.getState().setIsDebugging(wf, true) + useExecutionStore.getState().setBlockRunStatus(wf, 'block-1', 'success') + + useExecutionStore.getState().setIsExecuting(wf, true) + + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('debugging') + expect(state.isExecuting).toBe(true) + expect(state.isDebugging).toBe(true) + expect(state.lastRunPath.size).toBe(0) + expect(state.lastRunEdges.size).toBe(0) + }) + }) + + describe('setIsDebugging', () => { + it.concurrent('should toggle debug mode', () => { + const wf = 'wf-debug-toggle' + useExecutionStore.getState().setIsDebugging(wf, true) + + expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('debugging') + + useExecutionStore.getState().setIsDebugging(wf, false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('running') + }) + + it.concurrent('setIsDebugging(false) while idle is a no-op (stays idle)', () => { + const wf = 'wf-debug-false-idle' + useExecutionStore.getState().setIsDebugging(wf, false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('idle') + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(false) + }) + + it.concurrent('setIsDebugging(false) while running keeps running', () => { + const wf = 'wf-debug-false-running' + useExecutionStore.getState().setIsExecuting(wf, true) + useExecutionStore.getState().setIsDebugging(wf, false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('running') + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(true) + }) + + it.concurrent('does not clear the run path when entering debug mode', () => { + const wf = 'wf-debug-keeps-path' + useExecutionStore.getState().setBlockRunStatus(wf, 'block-1', 'success') + useExecutionStore.getState().setIsDebugging(wf, true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.get('block-1')).toBe( + 'success' + ) + }) + }) + + describe('status enum', () => { + it.concurrent('idle derives both flags false', () => { + const wf = 'wf-status-idle' + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('idle') + expect(state.isExecuting).toBe(false) + expect(state.isDebugging).toBe(false) + }) + + it.concurrent('running derives isExecuting only', () => { + const wf = 'wf-status-running' + useExecutionStore.getState().setStatus(wf, 'running') + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('running') + expect(state.isExecuting).toBe(true) + expect(state.isDebugging).toBe(false) + }) + + it.concurrent('debugging derives both flags true', () => { + const wf = 'wf-status-debugging' + useExecutionStore.getState().setStatus(wf, 'debugging') + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('debugging') + expect(state.isExecuting).toBe(true) + expect(state.isDebugging).toBe(true) + }) + + it.concurrent('setStatus preserves the run path unless clearRunPath is passed', () => { + const wf = 'wf-status-path-rules' + useExecutionStore.getState().setStatus(wf, 'debugging') + useExecutionStore.getState().setBlockRunStatus(wf, 'block-1', 'success') + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.size).toBe(1) + + useExecutionStore.getState().setStatus(wf, 'running') + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.size).toBe(1) + + useExecutionStore.getState().setStatus(wf, 'running', { clearRunPath: true }) + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.size).toBe(0) + }) + + it.concurrent('the derived booleans always agree with the stored status', () => { + const wf = 'wf-status-no-drift' + for (const status of ['idle', 'running', 'debugging', 'idle'] as const) { + useExecutionStore.getState().setStatus(wf, status) + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect({ isExecuting: state.isExecuting, isDebugging: state.isDebugging }).toEqual( + deriveExecutionFlags(status) + ) + } + }) + + it.concurrent('setIsExecuting(true) preserves an active debug session', () => { + const wf = 'wf-status-debug-preserve' + useExecutionStore.getState().setStatus(wf, 'debugging') + useExecutionStore.getState().setIsExecuting(wf, true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('debugging') + }) + + it.concurrent('setIsExecuting(false) returns to idle from any mode', () => { + const wf = 'wf-status-stop' + useExecutionStore.getState().setStatus(wf, 'debugging') + useExecutionStore.getState().setIsExecuting(wf, false) + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('idle') + expect(state.isExecuting).toBe(false) + expect(state.isDebugging).toBe(false) + }) }) describe('setActiveBlocks', () => { @@ -151,18 +311,6 @@ describe('useExecutionStore', () => { }) }) - describe('setIsDebugging', () => { - it.concurrent('should toggle debug mode', () => { - const wf = 'wf-debug-toggle' - useExecutionStore.getState().setIsDebugging(wf, true) - - expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(true) - - useExecutionStore.getState().setIsDebugging(wf, false) - expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(false) - }) - }) - describe('setExecutor', () => { it.concurrent('should store and clear executor', () => { const wf = 'wf-executor' diff --git a/apps/sim/stores/execution/store.ts b/apps/sim/stores/execution/store.ts index 8d9e2827f34..5efa313f021 100644 --- a/apps/sim/stores/execution/store.ts +++ b/apps/sim/stores/execution/store.ts @@ -3,9 +3,11 @@ import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { type BlockRunStatus, defaultWorkflowExecutionState, + deriveExecutionFlags, type EdgeRunStatus, type ExecutionActions, type ExecutionState, + type ExecutionStatus, initialState, type WorkflowExecutionState, } from './types' @@ -78,9 +80,12 @@ export const useExecutionStore = create()((se }) }, - setIsExecuting: (workflowId, isExecuting) => { - const patch: Partial = { isExecuting } - if (isExecuting) { + setStatus: (workflowId, status, options) => { + const patch: Partial = { + status, + ...deriveExecutionFlags(status), + } + if (options?.clearRunPath) { patch.lastRunPath = new Map() patch.lastRunEdges = new Map() } @@ -89,10 +94,24 @@ export const useExecutionStore = create()((se }) }, + setIsExecuting: (workflowId, isExecuting) => { + const current = getOrCreate(get().workflowExecutions, workflowId) + const nextStatus: ExecutionStatus = isExecuting + ? current.status === 'debugging' + ? 'debugging' + : 'running' + : 'idle' + get().setStatus(workflowId, nextStatus, { clearRunPath: isExecuting }) + }, + setIsDebugging: (workflowId, isDebugging) => { - set({ - workflowExecutions: updatedMap(get().workflowExecutions, workflowId, { isDebugging }), - }) + const current = getOrCreate(get().workflowExecutions, workflowId) + const nextStatus: ExecutionStatus = isDebugging + ? 'debugging' + : current.status === 'debugging' + ? 'running' + : current.status + get().setStatus(workflowId, nextStatus) }, setExecutor: (workflowId, executor) => { diff --git a/apps/sim/stores/execution/types.ts b/apps/sim/stores/execution/types.ts index b36ea43a190..bb48c9d45a2 100644 --- a/apps/sim/stores/execution/types.ts +++ b/apps/sim/stores/execution/types.ts @@ -12,6 +12,22 @@ export type BlockRunStatus = 'success' | 'error' */ export type EdgeRunStatus = 'success' | 'error' +/** + * The mutually-exclusive execution mode of a single workflow. + * + * @remarks + * This is the single source of truth for whether a workflow is running. + * The legacy `isExecuting` / `isDebugging` booleans are derived from it + * via {@link deriveExecutionFlags} so illegal combinations — such as + * "debugging while not executing" — are unrepresentable. + * + * - `idle` — not running. + * - `running` — executing normally (derives `isExecuting`). + * - `debugging` — executing in step-by-step debug mode (derives both + * `isExecuting` and `isDebugging`). + */ +export type ExecutionStatus = 'idle' | 'running' | 'debugging' + /** * Execution state scoped to a single workflow. * @@ -19,9 +35,11 @@ export type EdgeRunStatus = 'success' | 'error' * do not interfere with one another. */ export interface WorkflowExecutionState { - /** Whether this workflow is currently executing */ + /** Mutually-exclusive execution mode; the source of truth for run state */ + status: ExecutionStatus + /** Derived from {@link status}: whether this workflow is currently executing */ isExecuting: boolean - /** Whether this workflow is in step-by-step debug mode */ + /** Derived from {@link status}: whether this workflow is in step-by-step debug mode */ isDebugging: boolean /** Block IDs that are currently running (pulsing in the UI) */ activeBlockIds: Set @@ -39,6 +57,24 @@ export interface WorkflowExecutionState { currentExecutionId: string | null } +/** + * Computes the legacy `isExecuting` / `isDebugging` booleans from a status. + * + * @remarks + * Keeping the derived booleans on the stored state object lets existing + * consumers keep reading `state.isExecuting` / `state.isDebugging` + * unchanged while {@link ExecutionStatus} remains the single source of truth. + */ +export function deriveExecutionFlags(status: ExecutionStatus): { + isExecuting: boolean + isDebugging: boolean +} { + return { + isExecuting: status !== 'idle', + isDebugging: status === 'debugging', + } +} + /** * Default values for a workflow that has never been executed. * @@ -48,8 +84,8 @@ export interface WorkflowExecutionState { * re-renders in Zustand selectors that use `Object.is` equality. */ export const defaultWorkflowExecutionState: WorkflowExecutionState = { - isExecuting: false, - isDebugging: false, + status: 'idle', + ...deriveExecutionFlags('idle'), activeBlockIds: new Set(), pendingBlocks: [], executor: null, @@ -83,9 +119,38 @@ export interface ExecutionActions { getWorkflowExecution: (workflowId: string) => WorkflowExecutionState /** Replaces the set of currently-executing block IDs for a workflow */ setActiveBlocks: (workflowId: string, blockIds: Set) => void - /** Marks a workflow as executing or idle. Starting clears the run path */ + /** + * Sets the {@link ExecutionStatus} for a workflow. + * + * @remarks + * Pass `{ clearRunPath: true }` to also reset `lastRunPath` / `lastRunEdges`. + * Run-path clearing is opt-in: it is owned by + * {@link ExecutionActions.setIsExecuting} (which clears on start), matching + * the legacy behavior where only starting execution wiped the run history. + */ + setStatus: ( + workflowId: string, + status: ExecutionStatus, + options?: { clearRunPath?: boolean } + ) => void + /** + * Marks a workflow as executing or idle. Starting (`true`) clears the run path. + * + * @remarks + * Translates to {@link ExecutionActions.setStatus}: `true` preserves an + * active debug session (`debugging`) and otherwise enters `running`, and + * always clears the run path; `false` returns to `idle` and preserves it. + */ setIsExecuting: (workflowId: string, isExecuting: boolean) => void - /** Toggles debug mode for a workflow */ + /** + * Toggles step-by-step debug mode for a workflow. + * + * @remarks + * Translates to {@link ExecutionActions.setStatus}: `true` enters + * `debugging` (which implies executing); `false` returns to `running` only + * when currently `debugging`, otherwise the status is preserved (e.g. calling + * it while `idle` is a no-op). + */ setIsDebugging: (workflowId: string, isDebugging: boolean) => void /** Sets the list of blocks pending execution during debug stepping */ setPendingBlocks: (workflowId: string, blockIds: string[]) => void diff --git a/apps/sim/stores/workflow-diff/store.test.ts b/apps/sim/stores/workflow-diff/store.test.ts new file mode 100644 index 00000000000..c9bca038048 --- /dev/null +++ b/apps/sim/stores/workflow-diff/store.test.ts @@ -0,0 +1,228 @@ +/** + * @vitest-environment node + * + * Tests for the workflow-diff store's status modeling. + * + * Focus: the {@link WorkflowDiffStatus} enum is the single source of truth and + * the legacy `hasActiveDiff` / `isShowingDiff` / `isDiffReady` booleans are + * derived from it, so contradictory combinations are unrepresentable. We assert + * the exhaustive status → boolean mapping and the status transitions driven by + * the tractable actions (`toggleDiffView`, `clearDiff`, `_batchedStateUpdate`). + * + * @remarks + * The store transitively imports the diff engine, serializer, socket + * operations, and the workflow/registry stores, all of which drag in the block + * registry and emcn icon CSS. Every such dependency is mocked so the suite + * loads under the node environment and exercises only the store + its types. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { applyWorkflowStateToStores } = vi.hoisted(() => ({ + applyWorkflowStateToStores: vi.fn(), +})) + +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }), +})) + +vi.mock('@/lib/workflows/diff', () => ({ + WorkflowDiffEngine: class { + clearDiff = vi.fn() + createDiffFromWorkflowState = vi.fn() + }, + stripWorkflowDiffMarkers: vi.fn((s) => s), +})) + +vi.mock('@/lib/workflows/operations/socket-operations', () => ({ + enqueueReplaceWorkflowState: vi.fn().mockResolvedValue(undefined), +})) + +vi.mock('@/lib/workflows/sanitization/validation', () => ({ + validateWorkflowState: vi.fn(() => ({ valid: true, errors: [], sanitizedState: null })), +})) + +vi.mock('@/serializer', () => ({ + Serializer: class { + serializeWorkflow = vi.fn() + deserializeWorkflow = vi.fn() + }, +})) + +vi.mock('@/stores/workflows/registry/store', () => ({ + useWorkflowRegistry: { getState: vi.fn(() => ({ activeWorkflowId: null })) }, +})) + +vi.mock('@/stores/workflows/utils', () => ({ + mergeSubblockState: vi.fn((blocks) => blocks), +})) + +vi.mock('@/stores/workflows/workflow/store', () => ({ + useWorkflowStore: { + getState: vi.fn(() => ({ + getWorkflowState: vi.fn(() => ({ blocks: {}, edges: [], loops: {}, parallels: {} })), + blocks: {}, + lastSaved: 0, + })), + setState: vi.fn(), + }, +})) + +vi.mock('@/stores/workflow-diff/utils', () => ({ + applyWorkflowStateToStores, + captureBaselineSnapshot: vi.fn(), + cloneWorkflowState: vi.fn((s) => s), + createBatchedUpdater: + (set: (u: Record) => void) => (updates: Record) => + set(updates), + getLatestUserMessageId: vi.fn().mockResolvedValue(null), + persistWorkflowStateToServer: vi.fn().mockResolvedValue(true), + WORKFLOW_DIFF_SETTLED_EVENT: 'workflow-diff-settled', +})) + +import { RESET_DIFF_STATE, useWorkflowDiffStore } from '@/stores/workflow-diff/store' +import { + deriveDiffFlags, + type WorkflowDiffState, + type WorkflowDiffStatus, +} from '@/stores/workflow-diff/types' + +function seedStatus(status: WorkflowDiffStatus): void { + useWorkflowDiffStore.setState(deriveDiffFlags(status)) +} + +describe('useWorkflowDiffStore status modeling', () => { + beforeEach(() => { + vi.clearAllMocks() + useWorkflowDiffStore.setState({ + ...RESET_DIFF_STATE, + pendingExternalUpdates: {}, + remoteUpdateVersions: {}, + reconcilingWorkflows: {}, + reconciliationErrors: {}, + } as Partial) + }) + + describe('deriveDiffFlags', () => { + it('maps every status to the documented legacy booleans', () => { + expect(deriveDiffFlags('none')).toEqual({ + status: 'none', + hasActiveDiff: false, + isShowingDiff: false, + isDiffReady: false, + }) + expect(deriveDiffFlags('staged')).toEqual({ + status: 'staged', + hasActiveDiff: true, + isShowingDiff: false, + isDiffReady: true, + }) + expect(deriveDiffFlags('showing')).toEqual({ + status: 'showing', + hasActiveDiff: true, + isShowingDiff: true, + isDiffReady: true, + }) + }) + + it('keeps hasActiveDiff and isDiffReady in lockstep (legacy invariant)', () => { + for (const status of ['none', 'staged', 'showing'] as const) { + const flags = deriveDiffFlags(status) + expect(flags.hasActiveDiff).toBe(flags.isDiffReady) + } + }) + }) + + describe('initial / reset state', () => { + it('starts in the none-derived state', () => { + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('none') + expect(state.hasActiveDiff).toBe(false) + expect(state.isShowingDiff).toBe(false) + expect(state.isDiffReady).toBe(false) + }) + + it('RESET_DIFF_STATE carries the none-derived flags and clears diff payload', () => { + expect(RESET_DIFF_STATE.status).toBe('none') + expect(RESET_DIFF_STATE.hasActiveDiff).toBe(false) + expect(RESET_DIFF_STATE.isShowingDiff).toBe(false) + expect(RESET_DIFF_STATE.isDiffReady).toBe(false) + expect(RESET_DIFF_STATE.baselineWorkflow).toBeNull() + expect(RESET_DIFF_STATE.diffAnalysis).toBeNull() + }) + }) + + describe('toggleDiffView', () => { + it('is a guarded no-op when there is no active diff', () => { + seedStatus('none') + useWorkflowDiffStore.getState().toggleDiffView() + expect(useWorkflowDiffStore.getState().status).toBe('none') + }) + + it('toggles showing → staged (hides the proposed changes)', () => { + seedStatus('showing') + useWorkflowDiffStore.getState().toggleDiffView() + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('staged') + expect(state.hasActiveDiff).toBe(true) + expect(state.isDiffReady).toBe(true) + expect(state.isShowingDiff).toBe(false) + }) + + it('toggles staged → showing (reveals the proposed changes)', () => { + seedStatus('staged') + useWorkflowDiffStore.getState().toggleDiffView() + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('showing') + expect(state.isShowingDiff).toBe(true) + }) + }) + + describe('clearDiff', () => { + it('returns the store to the none status', () => { + seedStatus('showing') + useWorkflowDiffStore.getState().clearDiff({ restoreBaseline: false }) + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('none') + expect(state.hasActiveDiff).toBe(false) + expect(state.isShowingDiff).toBe(false) + expect(state.isDiffReady).toBe(false) + }) + }) + + describe('_batchedStateUpdate (undo/redo writer)', () => { + it('restores the showing status via deriveDiffFlags', () => { + seedStatus('none') + useWorkflowDiffStore.getState()._batchedStateUpdate({ + ...deriveDiffFlags('showing'), + baselineWorkflow: null, + baselineWorkflowId: 'wf-1', + }) + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('showing') + expect(state.hasActiveDiff).toBe(true) + expect(state.isShowingDiff).toBe(true) + expect(state.isDiffReady).toBe(true) + }) + + it('the derived booleans always agree with the stored status', () => { + for (const status of ['none', 'staged', 'showing', 'none'] as const) { + seedStatus(status) + const state = useWorkflowDiffStore.getState() + expect({ + hasActiveDiff: state.hasActiveDiff, + isShowingDiff: state.isShowingDiff, + isDiffReady: state.isDiffReady, + }).toEqual({ + hasActiveDiff: deriveDiffFlags(status).hasActiveDiff, + isShowingDiff: deriveDiffFlags(status).isShowingDiff, + isDiffReady: deriveDiffFlags(status).isDiffReady, + }) + } + }) + }) +}) diff --git a/apps/sim/stores/workflow-diff/store.ts b/apps/sim/stores/workflow-diff/store.ts index 38d08ad3b97..ec46c2192a3 100644 --- a/apps/sim/stores/workflow-diff/store.ts +++ b/apps/sim/stores/workflow-diff/store.ts @@ -8,7 +8,7 @@ import { Serializer } from '@/serializer' import { useWorkflowRegistry } from '../workflows/registry/store' import { mergeSubblockState } from '../workflows/utils' import { useWorkflowStore } from '../workflows/workflow/store' -import type { WorkflowDiffActions, WorkflowDiffState } from './types' +import { deriveDiffFlags, type WorkflowDiffActions, type WorkflowDiffState } from './types' import { applyWorkflowStateToStores, captureBaselineSnapshot, @@ -21,17 +21,20 @@ import { const logger = createLogger('WorkflowDiffStore') const diffEngine = new WorkflowDiffEngine() -const RESET_DIFF_STATE = { - hasActiveDiff: false, - isShowingDiff: false, - isDiffReady: false, + +/** + * Canonical state patch that clears the diff overlay back to `none`: the + * none-derived flags plus a wipe of all diff payload fields. + */ +export const RESET_DIFF_STATE = { + ...deriveDiffFlags('none'), baselineWorkflow: null, baselineWorkflowId: null, diffAnalysis: null, diffMetadata: null, diffError: null, _triggerMessageId: null, -} +} as const /** * Detects when a diff contains no meaningful changes. @@ -70,9 +73,7 @@ export const useWorkflowDiffStore = create { - const { hasActiveDiff, isDiffReady, isShowingDiff } = get() - if (!hasActiveDiff) { - logger.warn('Cannot toggle diff view without active diff') - return - } - if (!isDiffReady) { - logger.warn('Cannot toggle diff view before diff is ready') + const { status } = get() + if (status === 'none') { + logger.warn('Cannot toggle diff view without an active, ready diff') return } - batchedUpdate({ isShowingDiff: !isShowingDiff }) + batchedUpdate(deriveDiffFlags(status === 'showing' ? 'staged' : 'showing')) }, acceptChanges: async (options) => { diff --git a/apps/sim/stores/workflow-diff/types.ts b/apps/sim/stores/workflow-diff/types.ts index 6c3ea3990a1..d4e05e68760 100644 --- a/apps/sim/stores/workflow-diff/types.ts +++ b/apps/sim/stores/workflow-diff/types.ts @@ -1,9 +1,31 @@ import type { DiffAnalysis, WorkflowDiff } from '@/lib/workflows/diff' import type { WorkflowState } from '../workflows/workflow/types' +/** + * The lifecycle stage of the workflow diff overlay. + * + * @remarks + * This is the single source of truth for the diff overlay. The legacy + * `hasActiveDiff` / `isShowingDiff` / `isDiffReady` booleans are derived from + * it via {@link deriveDiffFlags}, which makes contradictory combinations — + * such as "showing a diff that has no active diff" — unrepresentable. + * + * - `none` — no diff staged; the canvas shows the live workflow. + * - `staged` — a diff is staged and ready, but the canvas is showing the + * baseline (proposed changes hidden). + * - `showing` — a diff is staged and ready, and the canvas is showing the + * proposed changes with diff markers. + */ +export type WorkflowDiffStatus = 'none' | 'staged' | 'showing' + export interface WorkflowDiffState { + /** Lifecycle stage of the diff overlay; the source of truth for diff flags */ + status: WorkflowDiffStatus + /** Derived from {@link status}: a diff is staged (`staged` or `showing`) */ hasActiveDiff: boolean + /** Derived from {@link status}: the canvas is rendering the proposed changes */ isShowingDiff: boolean + /** Derived from {@link status}: a staged diff is ready to view/toggle */ isDiffReady: boolean baselineWorkflow: WorkflowState | null baselineWorkflowId: string | null @@ -48,3 +70,31 @@ export interface WorkflowDiffActions { setWorkflowReconciliationError: (workflowId: string, error: string | null) => void _batchedStateUpdate: (updates: Partial) => void } + +/** + * The {@link WorkflowDiffStatus} fields shared by `status` and its derived + * booleans. Spread this into a state patch so the source of truth and the + * legacy flags never drift apart. + */ +export type DiffStatusFlags = Pick< + WorkflowDiffState, + 'status' | 'hasActiveDiff' | 'isShowingDiff' | 'isDiffReady' +> + +/** + * Computes the legacy `hasActiveDiff` / `isShowingDiff` / `isDiffReady` + * booleans (plus the `status` itself) from a {@link WorkflowDiffStatus}. + * + * @remarks + * Keeping the derived booleans on the stored state lets existing consumers + * keep reading `state.hasActiveDiff` etc. unchanged while + * {@link WorkflowDiffStatus} remains the single source of truth. + */ +export function deriveDiffFlags(status: WorkflowDiffStatus): DiffStatusFlags { + return { + status, + hasActiveDiff: status !== 'none', + isShowingDiff: status === 'showing', + isDiffReady: status !== 'none', + } +} diff --git a/apps/sim/vitest.setup.ts b/apps/sim/vitest.setup.ts index 92e945dcca0..b9a9ae7d14a 100644 --- a/apps/sim/vitest.setup.ts +++ b/apps/sim/vitest.setup.ts @@ -41,6 +41,7 @@ vi.mock('@/stores/execution/store', () => ({ useExecutionStore: { getState: vi.fn().mockReturnValue({ getWorkflowExecution: vi.fn().mockReturnValue({ + status: 'idle', isExecuting: false, isDebugging: false, activeBlockIds: new Set(), @@ -50,6 +51,7 @@ vi.mock('@/stores/execution/store', () => ({ lastRunPath: new Map(), lastRunEdges: new Map(), }), + setStatus: vi.fn(), setIsExecuting: vi.fn(), setIsDebugging: vi.fn(), setPendingBlocks: vi.fn(), @@ -61,6 +63,7 @@ vi.mock('@/stores/execution/store', () => ({ }), }, useCurrentWorkflowExecution: vi.fn().mockReturnValue({ + status: 'idle', isExecuting: false, isDebugging: false, activeBlockIds: new Set(), From 038e8f0d84c659886a4b39f7f5c8f2f4bbfbc08e Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 10:46:18 -0700 Subject: [PATCH 05/14] refactor(sse): consolidate client SSE readers behind a single typed primitive (#5195) Replace four hand-rolled client SSE decode loops with two layered primitives in lib/core/utils/sse.ts: - readSSELines: the single byte-stream decode engine. Splits on \n, strips trailing \r, tolerates data: with/without a leading space, skips the [DONE] sentinel, honors an AbortSignal before each chunk and between events, and releases the reader lock only when it acquired it. - readSSEEvents: a thin JSON layer that parses each payload and routes unparseable lines to onParseError (default: skip). An SSESource union accepts a Response, a ReadableStream, or an already-acquired reader so callers that must stash the reader for external cancellation keep ownership of the lock. Migrates use-execution-stream, chat use-chat-streaming, home use-chat (via readSSELines for schema-validated decode), and the workflow chat panel. Legacy server/wand exports (encodeSSE, SSE_HEADERS, readSSEStream) are untouched. Behavior is preserved across abort, RAF batching, TTS, [DONE], delimiter tolerance, and reader-lock ownership. Tests in sse.test.ts pin the prior behavior: \n and \n\n framing, mid-chunk splits, [DONE], data: with/without leading space, \r\n stripping, sync/async early-stop, pre-aborted and mid-stream abort, lock release/non-release per source, lock release on a throwing handler, and Response/stream/reader sources. --- apps/sim/app/chat/hooks/use-chat-streaming.ts | 437 +++++++++--------- .../[workspaceId]/home/hooks/use-chat.ts | 98 ++-- .../w/[workflowId]/components/chat/chat.tsx | 82 ++-- apps/sim/hooks/use-execution-stream.test.ts | 42 ++ apps/sim/hooks/use-execution-stream.ts | 40 +- apps/sim/lib/core/utils/sse.test.ts | 321 ++++++++++++- apps/sim/lib/core/utils/sse.ts | 175 +++++++ 7 files changed, 821 insertions(+), 374 deletions(-) diff --git a/apps/sim/app/chat/hooks/use-chat-streaming.ts b/apps/sim/app/chat/hooks/use-chat-streaming.ts index f4c9fcb9b97..dd315dafe73 100644 --- a/apps/sim/app/chat/hooks/use-chat-streaming.ts +++ b/apps/sim/app/chat/hooks/use-chat-streaming.ts @@ -3,6 +3,7 @@ import { useRef, useState } from 'react' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' +import { readSSEEvents } from '@/lib/core/utils/sse' import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' import type { ChatFile, ChatMessage } from '@/app/chat/components/message/message' import { CHAT_ERROR_MESSAGES } from '@/app/chat/constants' @@ -125,14 +126,12 @@ export function useChatStreaming() { streamingOptions?.voiceSettings?.autoPlayResponses && streamingOptions?.audioStreamHandler - const reader = response.body?.getReader() - if (!reader) { + if (!response.body) { setIsLoading(false) setIsStreamingResponse(false) return } - const decoder = new TextDecoder() let accumulatedText = '' let lastAudioPosition = 0 @@ -192,264 +191,252 @@ export function useChatStreaming() { setIsLoading(false) + let terminated = false + try { - while (true) { - // Check if aborted - if (abortControllerRef.current === null) { - break + await readSSEEvents<{ + blockId?: string + chunk?: string + event?: string + error?: string + data?: { + success: boolean + error?: string | { message?: string } + output?: Record> } - - const { done, value } = await reader.read() - - if (done) { - flushUI() - // Stream any remaining text for TTS - if ( - shouldPlayAudio && - streamingOptions?.audioStreamHandler && - accumulatedText.length > lastAudioPosition - ) { - const remainingText = accumulatedText.substring(lastAudioPosition).trim() - if (remainingText) { - try { - await streamingOptions.audioStreamHandler(remainingText) - } catch (error) { - logger.error('TTS error for remaining text:', error) - } - } + }>(response.body, { + signal: abortControllerRef.current.signal, + onParseError: (_data, parseError) => { + logger.error('Error parsing stream data:', parseError) + }, + onEvent: async (json) => { + const { blockId, chunk: contentChunk, event: eventType } = json + + if (eventType === 'error' || json.event === 'error') { + const errorMessage = json.error || CHAT_ERROR_MESSAGES.GENERIC_ERROR + setMessages((prev) => + prev.map((msg) => + msg.id === messageId + ? { + ...msg, + content: errorMessage, + isStreaming: false, + type: 'assistant' as const, + } + : msg + ) + ) + setIsLoading(false) + terminated = true + return true } - break - } - const chunk = decoder.decode(value, { stream: true }) - const lines = chunk.split('\n\n') + if (eventType === 'final' && json.data) { + flushUI() + const finalData = json.data - for (const line of lines) { - if (line.startsWith('data: ')) { - const data = line.substring(6) + const outputConfigs = streamingOptions?.outputConfigs + const formattedOutputs: string[] = [] + let extractedFiles: ChatFile[] = [] - if (data === '[DONE]') { - continue - } + const formatValue = (value: any): string | null => { + if (value === null || value === undefined) { + return null + } - try { - const json = JSON.parse(data) - const { blockId, chunk: contentChunk, event: eventType } = json - - if (eventType === 'error' || json.event === 'error') { - const errorMessage = json.error || CHAT_ERROR_MESSAGES.GENERIC_ERROR - setMessages((prev) => - prev.map((msg) => - msg.id === messageId - ? { - ...msg, - content: errorMessage, - isStreaming: false, - type: 'assistant' as const, - } - : msg - ) - ) - setIsLoading(false) - return + if (isUserFileWithMetadata(value)) { + return null } - if (eventType === 'final' && json.data) { - flushUI() - const finalData = json.data as { - success: boolean - error?: string | { message?: string } - output?: Record> + if (Array.isArray(value) && value.length === 0) { + return null + } + + if (typeof value === 'string') { + return value + } + + if (typeof value === 'object') { + try { + return `\`\`\`json\n${JSON.stringify(value, null, 2)}\n\`\`\`` + } catch { + return String(value) } + } - const outputConfigs = streamingOptions?.outputConfigs - const formattedOutputs: string[] = [] - let extractedFiles: ChatFile[] = [] + return String(value) + } - const formatValue = (value: any): string | null => { - if (value === null || value === undefined) { - return null - } + const getOutputValue = (blockOutputs: Record, path?: string) => { + if (!path || path === 'content') { + if (blockOutputs.content !== undefined) return blockOutputs.content + if (blockOutputs.result !== undefined) return blockOutputs.result + return blockOutputs + } - if (isUserFileWithMetadata(value)) { - return null - } + if (blockOutputs[path] !== undefined) { + return blockOutputs[path] + } - if (Array.isArray(value) && value.length === 0) { - return null + if (path.includes('.')) { + return path.split('.').reduce((current, segment) => { + if (current && typeof current === 'object' && segment in current) { + return current[segment] } + return undefined + }, blockOutputs) + } - if (typeof value === 'string') { - return value - } + return undefined + } - if (typeof value === 'object') { - try { - return `\`\`\`json\n${JSON.stringify(value, null, 2)}\n\`\`\`` - } catch { - return String(value) - } - } + if (outputConfigs?.length && finalData.output) { + for (const config of outputConfigs) { + const blockOutputs = finalData.output[config.blockId] + if (!blockOutputs) continue + + const value = getOutputValue(blockOutputs, config.path) + + if (isUserFileWithMetadata(value)) { + extractedFiles.push({ + id: value.id, + name: value.name, + url: value.url, + key: value.key, + size: value.size, + type: value.type, + context: value.context, + }) + continue + } - return String(value) + const nestedFiles = extractFilesFromData(value) + if (nestedFiles.length > 0) { + extractedFiles = [...extractedFiles, ...nestedFiles] + continue } - const getOutputValue = (blockOutputs: Record, path?: string) => { - if (!path || path === 'content') { - if (blockOutputs.content !== undefined) return blockOutputs.content - if (blockOutputs.result !== undefined) return blockOutputs.result - return blockOutputs - } + const formatted = formatValue(value) + if (formatted) { + formattedOutputs.push(formatted) + } + } + } - if (blockOutputs[path] !== undefined) { - return blockOutputs[path] - } + let finalContent = accumulatedText - if (path.includes('.')) { - return path.split('.').reduce((current, segment) => { - if (current && typeof current === 'object' && segment in current) { - return current[segment] - } - return undefined - }, blockOutputs) - } + if (formattedOutputs.length > 0) { + const nonEmptyOutputs = formattedOutputs.filter((output) => output.trim()) + if (nonEmptyOutputs.length > 0) { + const combinedOutputs = nonEmptyOutputs.join('\n\n') + finalContent = finalContent + ? `${finalContent.trim()}\n\n${combinedOutputs}` + : combinedOutputs + } + } - return undefined + if (!finalContent && extractedFiles.length === 0) { + if (finalData.error) { + if (typeof finalData.error === 'string') { + finalContent = finalData.error + } else if (typeof finalData.error?.message === 'string') { + finalContent = finalData.error.message } + } else if (finalData.success && finalData.output) { + const fallbackOutput = Object.values(finalData.output) + .map((block) => formatValue(block)?.trim()) + .filter(Boolean)[0] + if (fallbackOutput) { + finalContent = fallbackOutput + } + } + } - if (outputConfigs?.length && finalData.output) { - for (const config of outputConfigs) { - const blockOutputs = finalData.output[config.blockId] - if (!blockOutputs) continue - - const value = getOutputValue(blockOutputs, config.path) - - if (isUserFileWithMetadata(value)) { - extractedFiles.push({ - id: value.id, - name: value.name, - url: value.url, - key: value.key, - size: value.size, - type: value.type, - context: value.context, - }) - continue - } - - const nestedFiles = extractFilesFromData(value) - if (nestedFiles.length > 0) { - extractedFiles = [...extractedFiles, ...nestedFiles] - continue + setMessages((prev) => + prev.map((msg) => + msg.id === messageId + ? { + ...msg, + isStreaming: false, + content: finalContent ?? msg.content, + files: extractedFiles.length > 0 ? extractedFiles : undefined, } + : msg + ) + ) - const formatted = formatValue(value) - if (formatted) { - formattedOutputs.push(formatted) - } - } - } + accumulatedTextRef.current = '' + lastStreamedPositionRef.current = 0 + lastDisplayedPositionRef.current = 0 + audioStreamingActiveRef.current = false - let finalContent = accumulatedText + terminated = true + return true + } - if (formattedOutputs.length > 0) { - const nonEmptyOutputs = formattedOutputs.filter((output) => output.trim()) - if (nonEmptyOutputs.length > 0) { - const combinedOutputs = nonEmptyOutputs.join('\n\n') - finalContent = finalContent - ? `${finalContent.trim()}\n\n${combinedOutputs}` - : combinedOutputs - } - } + if (blockId && contentChunk) { + if (!messageIdMap.has(blockId)) { + messageIdMap.set(blockId, messageId) + } - if (!finalContent && extractedFiles.length === 0) { - if (finalData.error) { - if (typeof finalData.error === 'string') { - finalContent = finalData.error - } else if (typeof finalData.error?.message === 'string') { - finalContent = finalData.error.message - } - } else if (finalData.success && finalData.output) { - const fallbackOutput = Object.values(finalData.output) - .map((block) => formatValue(block)?.trim()) - .filter(Boolean)[0] - if (fallbackOutput) { - finalContent = fallbackOutput - } - } + accumulatedText += contentChunk + accumulatedTextRef.current = accumulatedText + logger.debug('[useChatStreaming] Received chunk', { + blockId, + chunkLength: contentChunk.length, + totalLength: accumulatedText.length, + messageId, + chunk: contentChunk.substring(0, 20), + }) + uiDirty = true + scheduleUIFlush() + + if (shouldPlayAudio && streamingOptions?.audioStreamHandler) { + const newText = accumulatedText.substring(lastAudioPosition) + const sentenceEndings = ['. ', '! ', '? ', '.\n', '!\n', '?\n', '.', '!', '?'] + let sentenceEnd = -1 + + for (const ending of sentenceEndings) { + const index = newText.indexOf(ending) + if (index > 0) { + sentenceEnd = index + ending.length + break } - - setMessages((prev) => - prev.map((msg) => - msg.id === messageId - ? { - ...msg, - isStreaming: false, - content: finalContent ?? msg.content, - files: extractedFiles.length > 0 ? extractedFiles : undefined, - } - : msg - ) - ) - - accumulatedTextRef.current = '' - lastStreamedPositionRef.current = 0 - lastDisplayedPositionRef.current = 0 - audioStreamingActiveRef.current = false - - return } - if (blockId && contentChunk) { - if (!messageIdMap.has(blockId)) { - messageIdMap.set(blockId, messageId) - } - - accumulatedText += contentChunk - accumulatedTextRef.current = accumulatedText - logger.debug('[useChatStreaming] Received chunk', { - blockId, - chunkLength: contentChunk.length, - totalLength: accumulatedText.length, - messageId, - chunk: contentChunk.substring(0, 20), - }) - uiDirty = true - scheduleUIFlush() - - // Real-time TTS for voice mode - if (shouldPlayAudio && streamingOptions?.audioStreamHandler) { - const newText = accumulatedText.substring(lastAudioPosition) - const sentenceEndings = ['. ', '! ', '? ', '.\n', '!\n', '?\n', '.', '!', '?'] - let sentenceEnd = -1 - - for (const ending of sentenceEndings) { - const index = newText.indexOf(ending) - if (index > 0) { - sentenceEnd = index + ending.length - break - } - } - - if (sentenceEnd > 0) { - const sentence = newText.substring(0, sentenceEnd).trim() - if (sentence && sentence.length >= 3) { - try { - await streamingOptions.audioStreamHandler(sentence) - lastAudioPosition += sentenceEnd - } catch (error) { - logger.error('TTS error:', error) - } - } + if (sentenceEnd > 0) { + const sentence = newText.substring(0, sentenceEnd).trim() + if (sentence && sentence.length >= 3) { + try { + await streamingOptions.audioStreamHandler(sentence) + lastAudioPosition += sentenceEnd + } catch (error) { + logger.error('TTS error:', error) } } - } else if (blockId && eventType === 'end') { - setMessages((prev) => - prev.map((msg) => (msg.id === messageId ? { ...msg, isStreaming: false } : msg)) - ) } - } catch (parseError) { - logger.error('Error parsing stream data:', parseError) + } + } else if (blockId && eventType === 'end') { + setMessages((prev) => + prev.map((msg) => (msg.id === messageId ? { ...msg, isStreaming: false } : msg)) + ) + } + }, + }) + + if (!terminated) { + flushUI() + if ( + shouldPlayAudio && + streamingOptions?.audioStreamHandler && + accumulatedText.length > lastAudioPosition + ) { + const remainingText = accumulatedText.substring(lastAudioPosition).trim() + if (remainingText) { + try { + await streamingOptions.audioStreamHandler(remainingText) + } catch (error) { + logger.error('TTS error for remaining text:', error) } } } diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts index b1ec30520af..e81ec0603f9 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts @@ -63,6 +63,7 @@ import { } from '@/lib/copilot/tools/client/run-tool-execution' import { setCurrentChatTraceparent } from '@/lib/copilot/tools/client/trace-context' import { isWorkflowToolName } from '@/lib/copilot/tools/workflow-tools' +import { readSSELines } from '@/lib/core/utils/sse' import { getQueryClient } from '@/app/_shell/providers/get-query-client' import { useFilePreviewController } from '@/app/workspace/[workspaceId]/home/hooks/preview' import { @@ -1934,7 +1935,6 @@ export function useChat( shouldContinue?: () => boolean } ) => { - const decoder = new TextDecoder() const ctx = createStreamLoopContext({ workspaceId, queryClient, @@ -1987,71 +1987,47 @@ export function useChat( return { sawStreamError: false, sawComplete: false } } streamReaderRef.current = reader - let buffer = '' try { - const pendingLines: string[] = [] - - while (true) { - if (pendingLines.length === 0) { - // Don't read another chunk after `complete` has drained. - if (state.sawCompleteEvent) break - const { done, value } = await reader.read() - if (done) break - if (ops.isStale()) continue - - buffer += decoder.decode(value, { stream: true }) - const lines = buffer.split('\n') - buffer = lines.pop() || '' - pendingLines.push(...lines) - if (pendingLines.length === 0) { - continue + await readSSELines(reader, { + onData: (raw) => { + if (state.sawCompleteEvent) return true + if (ops.isStale()) return + + const parsedResult = parsePersistedStreamEventEnvelopeJson(raw) + if (!parsedResult.ok) { + const error = createStreamSchemaValidationError(parsedResult, 'Live SSE event.') + logger.error('Rejected chat SSE event due to client-side schema enforcement', { + reason: parsedResult.reason, + message: parsedResult.message, + errors: parsedResult.errors, + error: error.message, + }) + throw error } - } - - const line = pendingLines.shift() - if (line === undefined) { - continue - } - if (ops.isStale()) { - pendingLines.length = 0 - continue - } - if (!line.startsWith('data: ')) continue - const raw = line.slice(6) - - const parsedResult = parsePersistedStreamEventEnvelopeJson(raw) - if (!parsedResult.ok) { - const error = createStreamSchemaValidationError(parsedResult, 'Live SSE event.') - logger.error('Rejected chat SSE event due to client-side schema enforcement', { - reason: parsedResult.reason, - message: parsedResult.message, - errors: parsedResult.errors, - error: error.message, - }) - throw error - } - const parsed = parsedResult.event + const parsed = parsedResult.event - if (parsed.trace?.requestId && parsed.trace.requestId !== state.streamRequestId) { - state.streamRequestId = parsed.trace.requestId - streamRequestIdRef.current = state.streamRequestId - ops.flush() - } - if (parsed.stream?.streamId) { - streamIdRef.current = parsed.stream.streamId - } - const eventCursor = parsed.stream?.cursor ?? String(parsed.seq) - if (isAlreadyProcessedStreamCursor(eventCursor, lastCursorRef.current)) { - continue - } - if (eventCursor) { - lastCursorRef.current = eventCursor - } + if (parsed.trace?.requestId && parsed.trace.requestId !== state.streamRequestId) { + state.streamRequestId = parsed.trace.requestId + streamRequestIdRef.current = state.streamRequestId + ops.flush() + } + if (parsed.stream?.streamId) { + streamIdRef.current = parsed.stream.streamId + } + const eventCursor = parsed.stream?.cursor ?? String(parsed.seq) + if (isAlreadyProcessedStreamCursor(eventCursor, lastCursorRef.current)) { + return + } + if (eventCursor) { + lastCursorRef.current = eventCursor + } - logger.debug('SSE event received', parsed) - dispatchStreamEvent(ctx, parsed) - } + logger.debug('SSE event received', parsed) + dispatchStreamEvent(ctx, parsed) + if (state.sawCompleteEvent) return true + }, + }) } finally { if (state.sawStreamError && !state.sawCompleteEvent) { applyTurnTerminal(state.model, 'error') diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx index 588ff6da5cc..d4caa19424f 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx @@ -25,6 +25,7 @@ import { extractPathFromOutputId, parseOutputContentSafely, } from '@/lib/core/utils/response-format' +import { readSSEEvents } from '@/lib/core/utils/sse' import { CHAT_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation' import { normalizeInputFormatValue } from '@/lib/workflows/input-format' import { StartBlockPath, TriggerUtils } from '@/lib/workflows/triggers/triggers' @@ -520,12 +521,10 @@ export function Chat() { * @param responseMessageId - ID of the message to update with streamed content */ const processStreamingResponse = useCallback( - async (stream: ReadableStream, responseMessageId: string) => { + async (stream: ReadableStream, responseMessageId: string) => { const reader = stream.getReader() streamReaderRef.current = reader - const decoder = new TextDecoder() let accumulatedContent = '' - let buffer = '' const BATCH_MAX_MS = 50 let pendingChunks = '' @@ -563,63 +562,34 @@ export function Chat() { } try { - while (true) { - const { done, value } = await reader.read() - if (done) { - flushChunks() - finalizeMessageStream(responseMessageId) - break - } - - const chunk = decoder.decode(value, { stream: true }) - buffer += chunk - - const separatorIndex = buffer.lastIndexOf('\n\n') - if (separatorIndex === -1) { - continue - } - - const processable = buffer.slice(0, separatorIndex) - buffer = buffer.slice(separatorIndex + 2) - - const lines = processable.split('\n\n') - - for (const line of lines) { - if (!line.startsWith('data: ')) continue - - const data = line.substring(6) - if (data === '[DONE]') continue - - try { - const json = JSON.parse(data) - const { event, data: eventData, chunk: contentChunk } = json - - if (event === 'final' && eventData) { - const result = eventData as ExecutionResult - - if ('success' in result && !result.success) { - const errorMessage = result.error || 'Workflow execution failed' - flushChunks() - appendMessageContent( - responseMessageId, - `${accumulatedContent ? '\n\n' : ''}Error: ${errorMessage}` - ) - finalizeMessageStream(responseMessageId) - return - } + await readSSEEvents<{ event?: string; data?: ExecutionResult; chunk?: string }>(reader, { + onParseError: (_data, e) => { + logger.error('Error parsing stream data:', e) + }, + onEvent: (json) => { + const { event, data: eventData, chunk: contentChunk } = json + if (event === 'final' && eventData) { + if ('success' in eventData && !eventData.success) { + const errorMessage = eventData.error || 'Workflow execution failed' flushChunks() - finalizeMessageStream(responseMessageId) - } else if (contentChunk) { - accumulatedContent += contentChunk - pendingChunks += contentChunk - scheduleFlush() + appendMessageContent( + responseMessageId, + `${accumulatedContent ? '\n\n' : ''}Error: ${errorMessage}` + ) } - } catch (e) { - logger.error('Error parsing stream data:', e) + return true } - } - } + + if (contentChunk) { + accumulatedContent += contentChunk + pendingChunks += contentChunk + scheduleFlush() + } + }, + }) + flushChunks() + finalizeMessageStream(responseMessageId) } catch (error) { if ((error as Error)?.name !== 'AbortError') { logger.error('Error processing stream:', error) diff --git a/apps/sim/hooks/use-execution-stream.test.ts b/apps/sim/hooks/use-execution-stream.test.ts index da52635ff99..f38f028c805 100644 --- a/apps/sim/hooks/use-execution-stream.test.ts +++ b/apps/sim/hooks/use-execution-stream.test.ts @@ -84,4 +84,46 @@ describe('processSSEStream', () => { expect(onEventId).not.toHaveBeenCalled() }) + + it('releases the reader lock after the stream completes', async () => { + const stream = streamEvents([]) + const reader = stream.getReader() + expect(stream.locked).toBe(true) + + await processSSEStream(reader, {}, 'test') + + expect(stream.locked).toBe(false) + }) + + it('releases the reader lock even when a handler throws', async () => { + const event: ExecutionEvent = { + type: 'block:started', + eventId: 7, + timestamp: new Date().toISOString(), + executionId: 'exec-1', + workflowId: 'wf-1', + data: { + blockId: 'block-1', + blockName: 'Block 1', + blockType: 'function', + executionOrder: 1, + }, + } + const stream = streamEvents([event]) + const reader = stream.getReader() + + await expect( + processSSEStream( + reader, + { + onBlockStarted: () => { + throw new Error('boom') + }, + }, + 'test' + ) + ).rejects.toThrow('boom') + + expect(stream.locked).toBe(false) + }) }) diff --git a/apps/sim/hooks/use-execution-stream.ts b/apps/sim/hooks/use-execution-stream.ts index 24a6e0cad4a..ffe862c4710 100644 --- a/apps/sim/hooks/use-execution-stream.ts +++ b/apps/sim/hooks/use-execution-stream.ts @@ -1,6 +1,7 @@ import { useCallback } from 'react' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { readSSEEvents } from '@/lib/core/utils/sse' import type { BlockChildWorkflowStartedData, BlockCompletedData, @@ -82,36 +83,12 @@ export async function processSSEStream( callbacks: ExecutionStreamCallbacks, logPrefix: string ): Promise { - const decoder = new TextDecoder() - let buffer = '' - try { - while (true) { - const { done, value } = await reader.read() - - if (done) break - - buffer += decoder.decode(value, { stream: true }) - const lines = buffer.split('\n\n') - buffer = lines.pop() || '' - - for (const line of lines) { - if (!line.trim() || !line.startsWith('data: ')) continue - - const data = line.substring(6).trim() - if (data === '[DONE]') { - logger.info(`${logPrefix} stream completed`) - continue - } - - let event: ExecutionEvent - try { - event = JSON.parse(data) as ExecutionEvent - } catch (error) { - logger.error('Failed to parse SSE event:', error, { data }) - continue - } - + await readSSEEvents(reader, { + onParseError: (data, error) => { + logger.error('Failed to parse SSE event:', error, { data }) + }, + onEvent: async (event) => { try { switch (event.type) { case 'execution:started': @@ -168,8 +145,9 @@ export async function processSSEStream( error ) } - } - } + }, + }) + logger.debug(`${logPrefix} stream completed`) } finally { reader.releaseLock() } diff --git a/apps/sim/lib/core/utils/sse.test.ts b/apps/sim/lib/core/utils/sse.test.ts index 524c00b83d3..579e23fd607 100644 --- a/apps/sim/lib/core/utils/sse.test.ts +++ b/apps/sim/lib/core/utils/sse.test.ts @@ -2,7 +2,13 @@ * @vitest-environment node */ import { describe, expect, it, vi } from 'vitest' -import { encodeSSE, readSSEStream, SSE_HEADERS } from '@/lib/core/utils/sse' +import { + encodeSSE, + readSSEEvents, + readSSELines, + readSSEStream, + SSE_HEADERS, +} from '@/lib/core/utils/sse' function createStreamFromChunks(chunks: Uint8Array[]): ReadableStream { let index = 0 @@ -311,3 +317,316 @@ describe('readSSEStream', () => { }) }) }) + +function streamFromStringChunks(chunks: string[]): ReadableStream { + const encoder = new TextEncoder() + return createStreamFromChunks(chunks.map((c) => encoder.encode(c))) +} + +describe('readSSEEvents', () => { + it('parses `\\n\\n`-framed events', async () => { + const stream = streamFromStringChunks([ + 'data: {"n":1}\n\n', + 'data: {"n":2}\n\n', + 'data: {"n":3}\n\n', + ]) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2, 3]) + }) + + it('parses `\\n`-framed events', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\ndata: {"n":2}\ndata: {"n":3}\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2, 3]) + }) + + it('reassembles events split across chunk boundaries', async () => { + const stream = streamFromStringChunks(['data: {"ms', 'g":"hel', 'lo"}\n\n']) + const events: Array<{ msg: string }> = [] + await readSSEEvents<{ msg: string }>(stream, { + onEvent: (e) => { + events.push(e) + }, + }) + expect(events).toEqual([{ msg: 'hello' }]) + }) + + it('skips the [DONE] sentinel', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\n\n', 'data: [DONE]\n\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1]) + }) + + it('accepts `data:` with and without a leading space', async () => { + const stream = streamFromStringChunks(['data:{"n":1}\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('strips trailing carriage returns (\\r\\n framing)', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\r\n\r\n', 'data: {"n":2}\r\n\r\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('routes unparseable payloads to onParseError and continues', async () => { + const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + const onParseError = vi.fn() + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + onParseError, + }) + expect(events).toEqual([2]) + expect(onParseError).toHaveBeenCalledTimes(1) + expect(onParseError).toHaveBeenCalledWith('not-json', expect.any(Error)) + }) + + it('stops early when onEvent returns true', async () => { + const stream = streamFromStringChunks([ + 'data: {"n":1}\n\n', + 'data: {"n":2}\n\n', + 'data: {"n":3}\n\n', + ]) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + return e.n === 2 + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('does not process events once the signal is aborted', async () => { + const controller = new AbortController() + const stream = streamFromStringChunks(['data: {"n":1}\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + signal: controller.signal, + onEvent: (e) => { + events.push(e.n) + controller.abort() + }, + }) + expect(events).toEqual([1]) + }) + + it('returns immediately when the signal is already aborted', async () => { + const controller = new AbortController() + controller.abort() + const stream = streamFromStringChunks(['data: {"n":1}\n\n']) + const onEvent = vi.fn() + await readSSEEvents(stream, { signal: controller.signal, onEvent }) + expect(onEvent).not.toHaveBeenCalled() + }) + + it('releases the reader lock for a stream source', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\n\n']) + await readSSEEvents<{ n: number }>(stream, { onEvent: () => {} }) + expect(() => stream.getReader()).not.toThrow() + }) + + it('does not release the lock for a reader source', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\n\n']) + const reader = stream.getReader() + await readSSEEvents<{ n: number }>(reader, { onEvent: () => {} }) + expect(() => stream.getReader()).toThrow() + reader.releaseLock() + }) + + it('accepts a Response source', async () => { + const response = new Response(streamFromStringChunks(['data: {"n":7}\n\n'])) + const events: number[] = [] + await readSSEEvents<{ n: number }>(response, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([7]) + }) + + it('silently skips unparseable payloads when no onParseError is provided', async () => { + const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await expect( + readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + ).resolves.toBeUndefined() + expect(events).toEqual([2]) + }) + + it('surfaces a fatal parse error when onParseError throws', async () => { + const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await expect( + readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + onParseError: () => { + throw new Error('boom') + }, + }) + ).rejects.toThrow('boom') + expect(events).toEqual([]) + }) + + it('stops early when onEvent resolves true asynchronously', async () => { + const stream = streamFromStringChunks([ + 'data: {"n":1}\n\n', + 'data: {"n":2}\n\n', + 'data: {"n":3}\n\n', + ]) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: async (e) => { + events.push(e.n) + return e.n === 2 + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('throws "No response body" for a Response without a body', async () => { + const response = new Response(null) + await expect(readSSEEvents(response, { onEvent: () => {} })).rejects.toThrow('No response body') + }) +}) + +describe('readSSELines', () => { + it('delivers raw (un-parsed) data payloads', async () => { + const stream = streamFromStringChunks(['data: raw-one\n\n', 'data: {"keep":"asString"}\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['raw-one', '{"keep":"asString"}']) + }) + + it('skips [DONE] and blank separator lines', async () => { + const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: [DONE]\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['a', 'b']) + }) + + it('preserves the raw payload verbatim (no JSON parsing)', async () => { + const stream = streamFromStringChunks(['data: {"unterminated\n\n', 'data:no-space\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['{"unterminated', 'no-space']) + }) + + it('strips a trailing carriage return from each line', async () => { + const stream = streamFromStringChunks(['data: one\r\n\r\ndata: two\r\n\r\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['one', 'two']) + }) + + it('stops early when onData returns true', async () => { + const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: c\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + return raw === 'b' + }, + }) + expect(lines).toEqual(['a', 'b']) + }) + + it('does not deliver any line when the signal is already aborted', async () => { + const controller = new AbortController() + controller.abort() + const stream = streamFromStringChunks(['data: a\n\n']) + const onData = vi.fn() + await readSSELines(stream, { signal: controller.signal, onData }) + expect(onData).not.toHaveBeenCalled() + }) + + it('stops between events in the same chunk once aborted mid-stream', async () => { + const controller = new AbortController() + const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: c\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + signal: controller.signal, + onData: (raw) => { + lines.push(raw) + if (raw === 'a') controller.abort() + }, + }) + expect(lines).toEqual(['a']) + }) + + it('releases the lock for a stream source', async () => { + const stream = streamFromStringChunks(['data: a\n\n']) + await readSSELines(stream, { onData: () => {} }) + expect(() => stream.getReader()).not.toThrow() + }) + + it('does not release the lock for a reader source', async () => { + const stream = streamFromStringChunks(['data: a\n\n']) + const reader = stream.getReader() + await readSSELines(reader, { onData: () => {} }) + expect(() => stream.getReader()).toThrow() + reader.releaseLock() + }) + + it('releases the lock for a stream source even when onData throws', async () => { + const stream = streamFromStringChunks(['data: a\n\n']) + await expect( + readSSELines(stream, { + onData: () => { + throw new Error('handler failed') + }, + }) + ).rejects.toThrow('handler failed') + expect(() => stream.getReader()).not.toThrow() + }) +}) diff --git a/apps/sim/lib/core/utils/sse.ts b/apps/sim/lib/core/utils/sse.ts index 9d9d3f785a5..50c758f0013 100644 --- a/apps/sim/lib/core/utils/sse.ts +++ b/apps/sim/lib/core/utils/sse.ts @@ -20,6 +20,181 @@ export function encodeSSE(data: any): Uint8Array { return new TextEncoder().encode(`data: ${JSON.stringify(data)}\n\n`) } +/** + * The sentinel value servers emit to signal end-of-stream. Lines carrying this + * payload are skipped before reaching the consumer's `onEvent` callback. + */ +const DONE_SENTINEL = '[DONE]' + +/** + * A source the SSE reader can consume: a fetch `Response`, its `ReadableStream` + * body, or an already-acquired reader. Passing a `Response`/stream lets the + * primitive own `getReader()` and the reader lifecycle (lock release); passing a + * reader is supported for callers that must acquire it first (e.g. to stash it + * for external cancellation). + */ +export type SSESource = + | Response + | ReadableStream + | ReadableStreamDefaultReader + +/** + * The result of an SSE event/line callback. Only the literal `true` (returned + * synchronously or resolved from a `Promise`) stops processing and returns + * early — useful for terminal events. Any other value (including the + * `undefined` a handler that returns nothing produces) keeps processing. + * + * Typed as `unknown` rather than `boolean | void | Promise` so + * both sync and `async` handlers — including `async` handlers that return + * nothing (`Promise`) — stay assignable, without the confusing + * `void`-inside-a-`Promise` union that the precise type would require. + */ +export type SSEStopSignal = unknown + +/** + * Options for {@link readSSELines} — the low-level line engine that delivers the + * raw `data:` payload string (no JSON parsing). + */ +export interface ReadSSELinesOptions { + /** Invoked once per SSE `data:` line with the raw (un-parsed) payload string. */ + onData: (rawData: string) => SSEStopSignal + /** Aborts the read; checked before each chunk and between events. */ + signal?: AbortSignal +} + +/** + * Options for {@link readSSEEvents} — the JSON convenience layer over + * {@link readSSELines}. + */ +export interface ReadSSEEventsOptions { + /** + * Invoked once per parsed SSE `data:` event with the JSON-parsed payload. + * Return (or resolve) `true` to stop processing and return early. Callers + * narrow the typed payload. + */ + onEvent: (event: T) => SSEStopSignal + /** + * Invoked for a `data:` line whose payload is not valid JSON. Defaults to + * silently skipping the line. Throw from here to surface a fatal parse error. + */ + onParseError?: (rawData: string, error: unknown) => void + /** Aborts the read; checked before each chunk and between events. */ + signal?: AbortSignal +} + +/** + * Resolves an {@link SSESource} to a reader, reporting whether this call + * acquired the lock (and is therefore responsible for releasing it). + */ +function toReader(source: SSESource): { + reader: ReadableStreamDefaultReader + ownsLock: boolean +} { + if (source instanceof ReadableStream) { + return { reader: source.getReader(), ownsLock: true } + } + if (source instanceof Response) { + if (!source.body) throw new Error('No response body') + return { reader: source.body.getReader(), ownsLock: true } + } + return { reader: source, ownsLock: false } +} + +/** + * Strips an optional trailing carriage return from a single SSE line, so both + * `\n`- and `\r\n`-terminated framings parse identically. + */ +function stripCarriageReturn(line: string): string { + return line.endsWith('\r') ? line.slice(0, -1) : line +} + +/** + * The single client-side SSE decode engine. Reads a byte stream, decodes it + * incrementally, splits it into lines, and invokes `onData` once per `data:` + * line with its raw (un-parsed) payload string. + * + * It splits on `\n` and processes each `data:` line individually, which makes it + * tolerant of BOTH `\n`- and `\n\n`-separated framings (the blank separator + * lines between events are simply ignored). Trailing `\r` is stripped, a single + * optional space after `data:` is consumed, and the `[DONE]` sentinel is + * skipped. The reader's lock is always released on completion, abort, or error + * (only when this function acquired it). + * + * This is the low-level engine. Most callers want {@link readSSEEvents}, which + * adds JSON parsing on top. Reach for `readSSELines` only when the payload needs + * custom parsing (e.g. schema-validated decoding). + * + * @param source - A `Response`, `ReadableStream`, or stream reader. + * @param options - The `onData` callback plus an optional `signal`. + */ +export async function readSSELines(source: SSESource, options: ReadSSELinesOptions): Promise { + const { onData, signal } = options + const { reader, ownsLock } = toReader(source) + const decoder = new TextDecoder() + let buffer = '' + + try { + while (true) { + if (signal?.aborted) break + + const { done, value } = await reader.read() + if (done) break + + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split('\n') + buffer = lines.pop() ?? '' + + for (const rawLine of lines) { + if (signal?.aborted) return + + const line = stripCarriageReturn(rawLine) + if (!line.startsWith('data:')) continue + + let data = line.slice(5) + if (data.startsWith(' ')) data = data.slice(1) + if (data === DONE_SENTINEL) continue + + if ((await onData(data)) === true) return + } + } + } finally { + if (ownsLock) reader.releaseLock() + } +} + +/** + * The JSON convenience layer over {@link readSSELines}: invokes `onEvent` once + * per `data:` event with its JSON-parsed payload. Unparseable lines are passed + * to `onParseError` (default: silently skipped). All framing, `\r`, `[DONE]`, + * abort, and reader-lifecycle behavior is inherited from `readSSELines`. + * + * Higher-level concerns — UI batching, reconnect, error classification, event + * dispatch — belong in the caller's `onEvent`, not here. + * + * @typeParam T - The parsed event type the caller expects (defaults to `unknown`). + * @param source - A `Response`, `ReadableStream`, or stream reader. + * @param options - The `onEvent` callback plus optional `signal`/`onParseError`. + */ +export async function readSSEEvents( + source: SSESource, + options: ReadSSEEventsOptions +): Promise { + const { onEvent, onParseError, signal } = options + await readSSELines(source, { + signal, + onData: (data) => { + let parsed: T + try { + parsed = JSON.parse(data) as T + } catch (error) { + onParseError?.(data, error) + return + } + return onEvent(parsed) + }, + }) +} + /** * Options for reading SSE stream */ From 067f9e9dc99401896fc9837b0d038628fec26a51 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 11:37:06 -0700 Subject: [PATCH 06/14] feat(gitlab): support self-managed GitLab host across tools, block, triggers, webhook, and connector (#5200) * feat(gitlab): support self-managed GitLab host across tools, block, triggers, webhook, and connector Add an optional `host` so the GitLab integration can target a self-managed instance (e.g. gitlab.example.com) instead of gitlab.com. Defaults to gitlab.com everywhere, so existing workflows, blocks, triggers, and stored webhooks are unchanged. - Shared host helper (normalizeGitLabHost/getGitLabApiBase) used by all 19 tools, the block, triggers, the webhook provider, and the connector - SSRF hardening: reject structurally unsafe hosts (userinfo `@`, whitespace, control chars, embedded path/query, empty labels) before the token-bearing request is built; allow self-managed hosts, ports, and IDN punycode - Route the webhook provider's previously-raw fetches through secureFetchWithValidation (DNS + private-IP rejection + IP pinning), matching the tool and connector paths - Add regression tests for the host validator * fix(gitlab): handle unsafe-host errors gracefully in webhook provider Address review feedback: - Validate the optional self-managed host up front in createSubscription and deleteSubscription so a structurally unsafe value surfaces as a clear error (create) or a graceful non-strict skip (delete) instead of an unhandled UnsafeGitLabHostError, mirroring the connector's handling - Document the layered SSRF defense: bare IP literals pass the structural host guard by design and are rejected at the fetch layer (validateUrlWithDNS); add a confirming test group making that intent explicit * fix(slack): drop assistant:write scope pending app review approval Requesting assistant:write before Slack approved it fails the OAuth/install flow for users. Remove it from both request paths until approval lands: - Remove from the user OAuth scope list (oauth.ts), matching the existing users:read.email TODO pattern - Remove the action_assistant capability from the bot manifest generator (capabilities.ts), leaving a TODO to restore it after approval The set_status/set_title/set_suggested_prompts tools remain and surface their existing graceful "reconnect with assistant:write" message until re-enabled. --- apps/sim/blocks/blocks/gitlab.ts | 11 +++ apps/sim/connectors/gitlab/gitlab.ts | 31 +++++--- apps/sim/lib/oauth/oauth.ts | 2 +- apps/sim/lib/webhooks/providers/gitlab.ts | 62 ++++++++++++---- apps/sim/tools/gitlab/cancel_pipeline.ts | 9 ++- apps/sim/tools/gitlab/create_issue.ts | 9 ++- apps/sim/tools/gitlab/create_issue_note.ts | 9 ++- apps/sim/tools/gitlab/create_merge_request.ts | 9 ++- .../tools/gitlab/create_merge_request_note.ts | 9 ++- apps/sim/tools/gitlab/create_pipeline.ts | 9 ++- apps/sim/tools/gitlab/delete_issue.ts | 9 ++- apps/sim/tools/gitlab/get_issue.ts | 9 ++- apps/sim/tools/gitlab/get_merge_request.ts | 9 ++- apps/sim/tools/gitlab/get_pipeline.ts | 9 ++- apps/sim/tools/gitlab/get_project.ts | 9 ++- apps/sim/tools/gitlab/list_issues.ts | 9 ++- apps/sim/tools/gitlab/list_merge_requests.ts | 9 ++- apps/sim/tools/gitlab/list_pipelines.ts | 9 ++- apps/sim/tools/gitlab/list_projects.ts | 9 ++- apps/sim/tools/gitlab/merge_merge_request.ts | 9 ++- apps/sim/tools/gitlab/retry_pipeline.ts | 9 ++- apps/sim/tools/gitlab/types.ts | 5 ++ apps/sim/tools/gitlab/update_issue.ts | 9 ++- apps/sim/tools/gitlab/update_merge_request.ts | 9 ++- apps/sim/tools/gitlab/utils.test.ts | 71 +++++++++++++++++++ apps/sim/tools/gitlab/utils.ts | 68 ++++++++++++++++++ apps/sim/triggers/gitlab/utils.ts | 9 +++ apps/sim/triggers/slack/capabilities.ts | 11 +-- 28 files changed, 388 insertions(+), 53 deletions(-) create mode 100644 apps/sim/tools/gitlab/utils.test.ts create mode 100644 apps/sim/tools/gitlab/utils.ts diff --git a/apps/sim/blocks/blocks/gitlab.ts b/apps/sim/blocks/blocks/gitlab.ts index 59de55f8c26..1cfe24a9e25 100644 --- a/apps/sim/blocks/blocks/gitlab.ts +++ b/apps/sim/blocks/blocks/gitlab.ts @@ -57,6 +57,15 @@ export const GitLabBlock: BlockConfig = { password: true, required: true, }, + // Self-managed GitLab host (defaults to gitlab.com) + { + id: 'host', + title: 'GitLab Host', + type: 'short-input', + placeholder: 'gitlab.com', + mode: 'advanced', + description: 'Self-managed GitLab host. Leave blank for gitlab.com.', + }, // Project ID (required for most operations) { id: 'projectId', @@ -474,6 +483,7 @@ Return ONLY the commit message - no explanations, no extra text.`, params: (params) => { const baseParams: Record = { accessToken: params.accessToken, + host: params.host?.trim() || undefined, } switch (params.operation) { @@ -709,6 +719,7 @@ Return ONLY the commit message - no explanations, no extra text.`, inputs: { operation: { type: 'string', description: 'Operation to perform' }, credential: { type: 'string', description: 'GitLab access token' }, + host: { type: 'string', description: 'Self-managed GitLab host (defaults to gitlab.com)' }, projectId: { type: 'string', description: 'Project ID or URL-encoded path' }, issueIid: { type: 'number', description: 'Issue internal ID' }, mergeRequestIid: { type: 'number', description: 'Merge request internal ID' }, diff --git a/apps/sim/connectors/gitlab/gitlab.ts b/apps/sim/connectors/gitlab/gitlab.ts index 18247214006..99586321f48 100644 --- a/apps/sim/connectors/gitlab/gitlab.ts +++ b/apps/sim/connectors/gitlab/gitlab.ts @@ -14,10 +14,10 @@ import { parseTagDate, sizeLimitSkipReason, } from '@/connectors/utils' +import { normalizeGitLabHost, UnsafeGitLabHostError } from '@/tools/gitlab/utils' const logger = createLogger('GitLabConnector') -const DEFAULT_HOST = 'gitlab.com' const PAGE_SIZE = 100 /** Max repository file size to index. Larger blobs are skipped. */ const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES @@ -175,16 +175,16 @@ interface GitLabProject { } /** - * Normalizes the host config value: trims whitespace, strips any protocol - * prefix and trailing slashes, and falls back to gitlab.com when empty. + * Normalizes the host config value via the shared GitLab host normalizer: + * trims, strips any protocol prefix and trailing slashes, rejects structurally + * unsafe hosts (userinfo, whitespace, embedded path), and falls back to + * gitlab.com when empty. Shared with the GitLab tools and webhook provider so + * every surface resolves and validates hosts identically. + * + * @throws {UnsafeGitLabHostError} when a non-empty host is structurally unsafe. */ function normalizeHost(rawHost: unknown): string { - const host = typeof rawHost === 'string' ? rawHost.trim() : '' - if (!host) return DEFAULT_HOST - return host - .replace(/^https?:\/\//i, '') - .replace(/\/+$/, '') - .trim() + return normalizeGitLabHost(rawHost) } /** @@ -941,7 +941,18 @@ export const gitlabConnector: ConnectorConfig = { return { valid: false, error: 'Max items must be a positive number' } } - const host = normalizeHost(sourceConfig.host) + let host: string + try { + host = normalizeHost(sourceConfig.host) + } catch (error) { + if (error instanceof UnsafeGitLabHostError) { + return { + valid: false, + error: 'Host must be a valid GitLab domain (e.g. gitlab.example.com)', + } + } + throw error + } const apiBase = buildApiBase(host) const encodedProject = encodeProjectId(project) const choice = getContentTypeChoice(sourceConfig) diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index a9a71270042..8f6b17100fe 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -711,7 +711,7 @@ export const OAUTH_PROVIDERS: Record = { 'groups:write', 'chat:write', 'chat:write.public', - 'assistant:write', + // TODO: Add 'assistant:write' once Slack app review is approved 'im:write', 'im:read', 'users:read', diff --git a/apps/sim/lib/webhooks/providers/gitlab.ts b/apps/sim/lib/webhooks/providers/gitlab.ts index 3f6ffcbf12e..d812ee19d92 100644 --- a/apps/sim/lib/webhooks/providers/gitlab.ts +++ b/apps/sim/lib/webhooks/providers/gitlab.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { generateId } from '@sim/utils/id' import { NextResponse } from 'next/server' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -13,17 +14,16 @@ import type { SubscriptionResult, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +import { getGitLabApiBase, UnsafeGitLabHostError } from '@/tools/gitlab/utils' const logger = createLogger('WebhookProvider:GitLab') -const GITLAB_API_BASE = 'https://gitlab.com/api/v4' - function asRecord(value: unknown): Record { return (value as Record) || {} } -function gitlabProjectHooksUrl(projectId: string): string { - return `${GITLAB_API_BASE}/projects/${encodeURIComponent(projectId)}/hooks` +function gitlabProjectHooksUrl(projectId: string, host: unknown): string { + return `${getGitLabApiBase(host)}/projects/${encodeURIComponent(projectId)}/hooks` } /** @@ -33,9 +33,10 @@ function gitlabProjectHooksUrl(projectId: string): string { async function cleanupGitLabHookByUrl( projectId: string, accessToken: string, - url: string + url: string, + host: unknown ): Promise { - const res = await fetch(gitlabProjectHooksUrl(projectId), { + const res = await secureFetchWithValidation(gitlabProjectHooksUrl(projectId, host), { headers: { 'PRIVATE-TOKEN': accessToken }, }).catch(() => null) if (!res || !res.ok) return @@ -47,7 +48,7 @@ async function cleanupGitLabHookByUrl( hooks .filter((hook) => hook.url === url && hook.id != null) .map((hook) => - fetch(`${gitlabProjectHooksUrl(projectId)}/${hook.id}`, { + secureFetchWithValidation(`${gitlabProjectHooksUrl(projectId, host)}/${hook.id}`, { method: 'DELETE', headers: { 'PRIVATE-TOKEN': accessToken }, }).catch(() => null) @@ -113,14 +114,28 @@ export const gitlabHandler: WebhookProviderHandler = { const accessToken = config.accessToken as string | undefined const projectId = config.projectId as string | undefined const triggerId = config.triggerId as string | undefined + const host = config.host as string | undefined if (!accessToken) throw new Error('GitLab Personal Access Token is required to create the webhook.') if (!projectId) throw new Error('GitLab Project ID is required to create the webhook.') + // Validate the optional self-managed host up front so a structurally unsafe + // value surfaces as a clear error instead of an unhandled UnsafeGitLabHostError. + try { + getGitLabApiBase(host) + } catch (error) { + if (error instanceof UnsafeGitLabHostError) { + throw new Error( + 'GitLab host is invalid. Provide a domain like gitlab.example.com (no protocol, path, or credentials).' + ) + } + throw error + } + const { getGitLabEventFlags } = await import('@/triggers/gitlab/utils') const secretToken = generateId() - const res = await fetch(gitlabProjectHooksUrl(projectId), { + const res = await secureFetchWithValidation(gitlabProjectHooksUrl(projectId, host), { method: 'POST', headers: { 'PRIVATE-TOKEN': accessToken, 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -150,7 +165,7 @@ export const gitlabHandler: WebhookProviderHandler = { if (created.id === undefined || created.id === null) { // The hook was created but we can't read its id — delete it by URL so it // is not orphaned in GitLab. - await cleanupGitLabHookByUrl(projectId, accessToken, getNotificationUrl(ctx.webhook)) + await cleanupGitLabHookByUrl(projectId, accessToken, getNotificationUrl(ctx.webhook), host) throw new Error('GitLab webhook created but no hook ID was returned.') } @@ -163,6 +178,7 @@ export const gitlabHandler: WebhookProviderHandler = { const accessToken = config.accessToken as string | undefined const projectId = config.projectId as string | undefined const externalId = config.externalId as string | undefined + const host = config.host as string | undefined if (!accessToken || !projectId || !externalId) { if (ctx.strict) throw new Error('Missing GitLab credentials or hook ID for webhook deletion.') @@ -172,10 +188,30 @@ export const gitlabHandler: WebhookProviderHandler = { return } - const res = await fetch(`${gitlabProjectHooksUrl(projectId)}/${externalId}`, { - method: 'DELETE', - headers: { 'PRIVATE-TOKEN': accessToken }, - }) + // A structurally unsafe host must not abort cleanup in non-strict mode — mirror + // the graceful skip used for missing credentials above. + try { + getGitLabApiBase(host) + } catch (error) { + if (error instanceof UnsafeGitLabHostError) { + if (ctx.strict) { + throw new Error('Cannot delete GitLab webhook: the configured host is invalid.') + } + logger.warn( + `[${ctx.requestId}] Skipping GitLab webhook cleanup — configured host is invalid` + ) + return + } + throw error + } + + const res = await secureFetchWithValidation( + `${gitlabProjectHooksUrl(projectId, host)}/${externalId}`, + { + method: 'DELETE', + headers: { 'PRIVATE-TOKEN': accessToken }, + } + ) if (!res.ok && res.status !== 404) { if (ctx.strict) throw new Error(`Failed to delete GitLab webhook: ${res.status}`) diff --git a/apps/sim/tools/gitlab/cancel_pipeline.ts b/apps/sim/tools/gitlab/cancel_pipeline.ts index 62b9e096b95..9707f758a0e 100644 --- a/apps/sim/tools/gitlab/cancel_pipeline.ts +++ b/apps/sim/tools/gitlab/cancel_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabCancelPipelineParams, GitLabCancelPipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCancelPipelineTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabCancelPipelineTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -34,7 +41,7 @@ export const gitlabCancelPipelineTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines/${params.pipelineId}/cancel` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/cancel` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_issue.ts b/apps/sim/tools/gitlab/create_issue.ts index 6a03e9ff970..cc4475831da 100644 --- a/apps/sim/tools/gitlab/create_issue.ts +++ b/apps/sim/tools/gitlab/create_issue.ts @@ -1,4 +1,5 @@ import type { GitLabCreateIssueParams, GitLabCreateIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateIssueTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabCreateIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_issue_note.ts b/apps/sim/tools/gitlab/create_issue_note.ts index 150bf729d7e..0ad5c218bee 100644 --- a/apps/sim/tools/gitlab/create_issue_note.ts +++ b/apps/sim/tools/gitlab/create_issue_note.ts @@ -1,4 +1,5 @@ import type { GitLabCreateIssueNoteParams, GitLabCreateNoteResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateIssueNoteTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabCreateIssueNoteTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -40,7 +47,7 @@ export const gitlabCreateIssueNoteTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}/notes` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}/notes` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_merge_request.ts b/apps/sim/tools/gitlab/create_merge_request.ts index fdaebbf842a..2c02c2dd0f4 100644 --- a/apps/sim/tools/gitlab/create_merge_request.ts +++ b/apps/sim/tools/gitlab/create_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabCreateMergeRequestParams, GitLabCreateMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabCreateMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -91,7 +98,7 @@ export const gitlabCreateMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_merge_request_note.ts b/apps/sim/tools/gitlab/create_merge_request_note.ts index 599ec569719..f02f5fa35fb 100644 --- a/apps/sim/tools/gitlab/create_merge_request_note.ts +++ b/apps/sim/tools/gitlab/create_merge_request_note.ts @@ -2,6 +2,7 @@ import type { GitLabCreateMergeRequestNoteParams, GitLabCreateNoteResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateMergeRequestNoteTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabCreateMergeRequestNoteTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -43,7 +50,7 @@ export const gitlabCreateMergeRequestNoteTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/notes` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/notes` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_pipeline.ts b/apps/sim/tools/gitlab/create_pipeline.ts index 38a15df006b..a27ed7ba372 100644 --- a/apps/sim/tools/gitlab/create_pipeline.ts +++ b/apps/sim/tools/gitlab/create_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabCreatePipelineParams, GitLabCreatePipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreatePipelineTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabCreatePipelineTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -41,7 +48,7 @@ export const gitlabCreatePipelineTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipeline` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipeline` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/delete_issue.ts b/apps/sim/tools/gitlab/delete_issue.ts index 64fbe881e53..475e52d77a8 100644 --- a/apps/sim/tools/gitlab/delete_issue.ts +++ b/apps/sim/tools/gitlab/delete_issue.ts @@ -1,4 +1,5 @@ import type { GitLabDeleteIssueParams, GitLabDeleteIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabDeleteIssueTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabDeleteIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'DELETE', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_issue.ts b/apps/sim/tools/gitlab/get_issue.ts index 9a3c5821edb..aa87136e552 100644 --- a/apps/sim/tools/gitlab/get_issue.ts +++ b/apps/sim/tools/gitlab/get_issue.ts @@ -1,4 +1,5 @@ import type { GitLabGetIssueParams, GitLabGetIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetIssueTool: ToolConfig = { @@ -14,6 +15,12 @@ export const gitlabGetIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_merge_request.ts b/apps/sim/tools/gitlab/get_merge_request.ts index 6e99d3cfbd3..f228cfba2eb 100644 --- a/apps/sim/tools/gitlab/get_merge_request.ts +++ b/apps/sim/tools/gitlab/get_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabGetMergeRequestParams, GitLabGetMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabGetMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -37,7 +44,7 @@ export const gitlabGetMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_pipeline.ts b/apps/sim/tools/gitlab/get_pipeline.ts index 5f4f25a0eaa..1494e65e4bb 100644 --- a/apps/sim/tools/gitlab/get_pipeline.ts +++ b/apps/sim/tools/gitlab/get_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabGetPipelineParams, GitLabGetPipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetPipelineTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabGetPipelineTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines/${params.pipelineId}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_project.ts b/apps/sim/tools/gitlab/get_project.ts index c49369084f5..5ea42920584 100644 --- a/apps/sim/tools/gitlab/get_project.ts +++ b/apps/sim/tools/gitlab/get_project.ts @@ -1,4 +1,5 @@ import type { GitLabGetProjectParams, GitLabGetProjectResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetProjectTool: ToolConfig = { @@ -14,6 +15,12 @@ export const gitlabGetProjectTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/list_issues.ts b/apps/sim/tools/gitlab/list_issues.ts index 1607920571e..40a016f3b34 100644 --- a/apps/sim/tools/gitlab/list_issues.ts +++ b/apps/sim/tools/gitlab/list_issues.ts @@ -1,4 +1,5 @@ import type { GitLabListIssuesParams, GitLabListIssuesResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListIssuesTool: ToolConfig = { @@ -14,6 +15,12 @@ export const gitlabListIssuesTool: ToolConfig ({ diff --git a/apps/sim/tools/gitlab/list_merge_requests.ts b/apps/sim/tools/gitlab/list_merge_requests.ts index 0296bc3a24f..2cdae3301c4 100644 --- a/apps/sim/tools/gitlab/list_merge_requests.ts +++ b/apps/sim/tools/gitlab/list_merge_requests.ts @@ -2,6 +2,7 @@ import type { GitLabListMergeRequestsParams, GitLabListMergeRequestsResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListMergeRequestsTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabListMergeRequestsTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -91,7 +98,7 @@ export const gitlabListMergeRequestsTool: ToolConfig< if (params.page) queryParams.append('page', String(params.page)) const query = queryParams.toString() - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests${query ? `?${query}` : ''}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests${query ? `?${query}` : ''}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/list_pipelines.ts b/apps/sim/tools/gitlab/list_pipelines.ts index d4aed464736..80294e85f73 100644 --- a/apps/sim/tools/gitlab/list_pipelines.ts +++ b/apps/sim/tools/gitlab/list_pipelines.ts @@ -1,4 +1,5 @@ import type { GitLabListPipelinesParams, GitLabListPipelinesResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListPipelinesTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabListPipelinesTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -75,7 +82,7 @@ export const gitlabListPipelinesTool: ToolConfig< if (params.page) queryParams.append('page', String(params.page)) const query = queryParams.toString() - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines${query ? `?${query}` : ''}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines${query ? `?${query}` : ''}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/list_projects.ts b/apps/sim/tools/gitlab/list_projects.ts index ec8018215a4..b6d6dd4c4a7 100644 --- a/apps/sim/tools/gitlab/list_projects.ts +++ b/apps/sim/tools/gitlab/list_projects.ts @@ -1,4 +1,5 @@ import type { GitLabListProjectsParams, GitLabListProjectsResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListProjectsTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabListProjectsTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, owned: { type: 'boolean', required: false, @@ -80,7 +87,7 @@ export const gitlabListProjectsTool: ToolConfig< if (params.page) queryParams.append('page', String(params.page)) const query = queryParams.toString() - return `https://gitlab.com/api/v4/projects${query ? `?${query}` : ''}` + return `${getGitLabApiBase(params.host)}/projects${query ? `?${query}` : ''}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/merge_merge_request.ts b/apps/sim/tools/gitlab/merge_merge_request.ts index d63686a16f4..500e6ebfd07 100644 --- a/apps/sim/tools/gitlab/merge_merge_request.ts +++ b/apps/sim/tools/gitlab/merge_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabMergeMergeRequestParams, GitLabMergeMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabMergeMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabMergeMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -67,7 +74,7 @@ export const gitlabMergeMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/merge` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/merge` }, method: 'PUT', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/retry_pipeline.ts b/apps/sim/tools/gitlab/retry_pipeline.ts index 3c0fe6f2b4d..48143109c97 100644 --- a/apps/sim/tools/gitlab/retry_pipeline.ts +++ b/apps/sim/tools/gitlab/retry_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabRetryPipelineParams, GitLabRetryPipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabRetryPipelineTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabRetryPipelineTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -34,7 +41,7 @@ export const gitlabRetryPipelineTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines/${params.pipelineId}/retry` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/retry` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/types.ts b/apps/sim/tools/gitlab/types.ts index 9722c16422b..af865ed3ef8 100644 --- a/apps/sim/tools/gitlab/types.ts +++ b/apps/sim/tools/gitlab/types.ts @@ -194,6 +194,11 @@ interface GitLabMilestone { interface GitLabBaseParams { accessToken: string + /** + * Self-managed GitLab host (e.g. `gitlab.example.com`). Optional — defaults to + * `gitlab.com` so existing workflows keep working. + */ + host?: string } // ===== Project Parameters ===== diff --git a/apps/sim/tools/gitlab/update_issue.ts b/apps/sim/tools/gitlab/update_issue.ts index 27c1fb70164..acf7ca25402 100644 --- a/apps/sim/tools/gitlab/update_issue.ts +++ b/apps/sim/tools/gitlab/update_issue.ts @@ -1,4 +1,5 @@ import type { GitLabUpdateIssueParams, GitLabUpdateIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabUpdateIssueTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabUpdateIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'PUT', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/update_merge_request.ts b/apps/sim/tools/gitlab/update_merge_request.ts index c02d4f13b08..69632a637d5 100644 --- a/apps/sim/tools/gitlab/update_merge_request.ts +++ b/apps/sim/tools/gitlab/update_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabUpdateMergeRequestParams, GitLabUpdateMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabUpdateMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabUpdateMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -97,7 +104,7 @@ export const gitlabUpdateMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` }, method: 'PUT', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/utils.test.ts b/apps/sim/tools/gitlab/utils.test.ts new file mode 100644 index 00000000000..f7eca36aef8 --- /dev/null +++ b/apps/sim/tools/gitlab/utils.test.ts @@ -0,0 +1,71 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { getGitLabApiBase, normalizeGitLabHost, UnsafeGitLabHostError } from '@/tools/gitlab/utils' + +describe('normalizeGitLabHost', () => { + it('defaults to gitlab.com when the host is empty, blank, or not a string', () => { + expect(normalizeGitLabHost(undefined)).toBe('gitlab.com') + expect(normalizeGitLabHost(null)).toBe('gitlab.com') + expect(normalizeGitLabHost('')).toBe('gitlab.com') + expect(normalizeGitLabHost(' ')).toBe('gitlab.com') + expect(normalizeGitLabHost(42)).toBe('gitlab.com') + }) + + it('strips protocol and trailing slashes from a self-managed host', () => { + expect(normalizeGitLabHost('gitlab.example.com')).toBe('gitlab.example.com') + expect(normalizeGitLabHost('https://gitlab.example.com')).toBe('gitlab.example.com') + expect(normalizeGitLabHost('http://gitlab.example.com/')).toBe('gitlab.example.com') + expect(normalizeGitLabHost(' https://gitlab.example.com// ')).toBe('gitlab.example.com') + }) + + it('preserves an explicit port and IDN punycode labels', () => { + expect(normalizeGitLabHost('gitlab.example.com:8443')).toBe('gitlab.example.com:8443') + expect(normalizeGitLabHost('xn--80ak6aa92e.com')).toBe('xn--80ak6aa92e.com') + }) + + it('rejects hosts that could redirect the request authority (SSRF / token exfiltration)', () => { + const unsafe = [ + 'legit.com@evil.com', + 'user:pass@evil.com', + 'gitlab.com#@evil.com', + 'gitlab.com /api', + 'line\nbreak.com', + 'evil.com/path', + 'evil.com?x=1', + '[::1]', + 'a..b.com', + '.gitlab.com', + 'gitlab.com.', + ] + for (const host of unsafe) { + expect(() => normalizeGitLabHost(host), host).toThrow(UnsafeGitLabHostError) + } + }) + + it('accepts bare IP literals at the STRUCTURAL layer by design (private/metadata IPs are rejected later by the fetch-layer DNS guard)', () => { + // This guard is structural only — it prevents authority confusion (userinfo, + // path, whitespace). SSRF to private/loopback/metadata addresses is the + // responsibility of validateUrlWithDNS / secureFetchWithValidation at fetch + // time, the single SSRF chokepoint shared by tools, webhooks, and connectors. + // These hosts are therefore structurally valid here, then blocked at fetch. + expect(normalizeGitLabHost('127.0.0.1')).toBe('127.0.0.1') + expect(normalizeGitLabHost('169.254.169.254')).toBe('169.254.169.254') + expect(normalizeGitLabHost('localhost')).toBe('localhost') + }) +}) + +describe('getGitLabApiBase', () => { + it('builds the v4 REST base for the default and self-managed hosts', () => { + expect(getGitLabApiBase(undefined)).toBe('https://gitlab.com/api/v4') + expect(getGitLabApiBase('gitlab.example.com')).toBe('https://gitlab.example.com/api/v4') + expect(getGitLabApiBase('https://gitlab.example.com:8443/')).toBe( + 'https://gitlab.example.com:8443/api/v4' + ) + }) + + it('propagates rejection of unsafe hosts', () => { + expect(() => getGitLabApiBase('legit.com@evil.com')).toThrow(UnsafeGitLabHostError) + }) +}) diff --git a/apps/sim/tools/gitlab/utils.ts b/apps/sim/tools/gitlab/utils.ts new file mode 100644 index 00000000000..6334a7030ee --- /dev/null +++ b/apps/sim/tools/gitlab/utils.ts @@ -0,0 +1,68 @@ +const DEFAULT_GITLAB_HOST = 'gitlab.com' + +/** + * Error thrown when a user-supplied GitLab host is structurally unsafe to use + * as the target of a server-side request that carries the user's access token. + */ +export class UnsafeGitLabHostError extends Error { + constructor(rawHost: string) { + super(`Invalid GitLab host: ${rawHost}`) + this.name = 'UnsafeGitLabHostError' + } +} + +/** + * Rejects a host that is structurally unsafe to fetch with the caller's token. + * + * The host is later interpolated into `https:///api/v4`, so anything that + * could change the request's authority (userinfo `@`, an embedded path/query/ + * fragment, whitespace, or control characters) must be rejected to prevent the + * `PRIVATE-TOKEN` header from being sent to an attacker-controlled origin. The + * allowed alphabet is hostname labels plus an optional `:port`, so self-managed + * hosts such as `gitlab.example.com` or `gitlab.example.com:8443` keep working. + * This is a structural guard only; DNS-based private-IP/SSRF checks remain the + * responsibility of the fetch layer. + */ +function assertSafeGitLabHostString(host: string, rawHost: string): void { + const hostnameWithoutPort = host.replace(/:\d+$/, '') + const allowedHostChars = /^[A-Za-z0-9.-]+$/ + if (!allowedHostChars.test(hostnameWithoutPort)) { + throw new UnsafeGitLabHostError(rawHost) + } + if (hostnameWithoutPort.startsWith('.') || hostnameWithoutPort.endsWith('.')) { + throw new UnsafeGitLabHostError(rawHost) + } + if (hostnameWithoutPort.split('.').some((label) => label.length === 0)) { + throw new UnsafeGitLabHostError(rawHost) + } +} + +/** + * Normalizes a GitLab host value: trims whitespace, strips any protocol prefix + * and trailing slashes, validates that the result is a bare host (optionally + * with a port), and falls back to gitlab.com when empty. Mirrors the GitLab + * connector so tools, triggers, and connectors resolve hosts identically. + * + * @throws {UnsafeGitLabHostError} when a non-empty host is structurally unsafe. + */ +export function normalizeGitLabHost(rawHost: unknown): string { + const raw = typeof rawHost === 'string' ? rawHost.trim() : '' + if (!raw) return DEFAULT_GITLAB_HOST + const host = raw + .replace(/^https?:\/\//i, '') + .replace(/\/+$/, '') + .trim() + if (!host) return DEFAULT_GITLAB_HOST + assertSafeGitLabHostString(host, String(rawHost)) + return host +} + +/** + * Builds the REST API v4 base URL for the configured host. Defaults to + * gitlab.com so existing workflows that never set a host keep working. + * + * @throws {UnsafeGitLabHostError} when a non-empty host is structurally unsafe. + */ +export function getGitLabApiBase(rawHost: unknown): string { + return `https://${normalizeGitLabHost(rawHost)}/api/v4` +} diff --git a/apps/sim/triggers/gitlab/utils.ts b/apps/sim/triggers/gitlab/utils.ts index 6f7848ad2fe..a25d09e3447 100644 --- a/apps/sim/triggers/gitlab/utils.ts +++ b/apps/sim/triggers/gitlab/utils.ts @@ -103,6 +103,15 @@ export function buildGitLabExtraFields(triggerId: string): SubBlockConfig[] { mode: 'trigger', condition: { field: 'selectedTriggerId', value: triggerId }, }, + { + id: 'host', + title: 'GitLab Host', + type: 'short-input', + placeholder: 'gitlab.com', + description: 'Self-managed GitLab host. Leave blank for gitlab.com.', + mode: 'trigger', + condition: { field: 'selectedTriggerId', value: triggerId }, + }, ] } diff --git a/apps/sim/triggers/slack/capabilities.ts b/apps/sim/triggers/slack/capabilities.ts index 1a532e4ebf2..317c240621f 100644 --- a/apps/sim/triggers/slack/capabilities.ts +++ b/apps/sim/triggers/slack/capabilities.ts @@ -105,16 +105,7 @@ export const SLACK_CAPABILITIES: readonly SlackCapability[] = [ scopes: ['channels:history', 'groups:history', 'im:history', 'mpim:history'], events: [], }, - { - id: 'action_assistant', - label: 'Manage assistant threads', - description: - "Let the bot set the status indicator (the 'is thinking…' shimmer), title, and suggested prompts on AI app threads.", - defaultChecked: true, - group: 'action', - scopes: ['assistant:write'], - events: [], - }, + // TODO: Restore the 'action_assistant' capability (scope 'assistant:write') once Slack app review is approved { id: 'action_read_files', label: 'Read file attachments', From e1c3c7f6c97744bc1b193601a056963b32ef6413 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 24 Jun 2026 13:23:37 -0700 Subject: [PATCH 07/14] feat(secrets): ingest env secrets at container runtime instead of fanning into ECS taskdef (#5189) * feat(secrets): ingest env secrets at container runtime instead of fanning into ECS taskdef The app/socket ECS taskdefs were ~42KB, ~93% of which was the secrets[] array: 268 pointer entries each restating the full ~78-char secret ARN, marching toward the 64KB taskdef limit and growing ~150 bytes per hosted key added. The secret blob itself is only ~18KB/268 keys. Move secret delivery to container boot: new @sim/runtime-secrets loadRuntimeSecrets() reads SIM_ENV_SECRET_ID, fetches the combined secret once, and hydrates process.env (no-clobber, no-op when unset, fail-fast). Bootstrap entrypoints for app + realtime await it before importing the real server (env-flags reads env at module load). The app bootstrap is bun-bundled in the Dockerfile builder stage since it runs outside the Next standalone bundle; realtime keeps full node_modules and runs the TS entry. Backward-compatible: with the current fan-out taskdef the loader no-ops and the app reads the injected env vars unchanged. The matching infra change (empty secrets[] + SIM_ENV_SECRET_ID) ships separately, after this image is live. * fix(runtime-secrets): address review feedback - Move the binary-secret guard outside the retry loop (sendWithRetry) so a missing SecretString throws immediately instead of burning 3 attempts + backoff. - Bound each Secrets Manager request with AbortSignal.timeout(5s) so a stalled response can't hang boot indefinitely. - Drop the redundant @aws-sdk/client-secrets-manager pin from apps/realtime; it resolves transitively via @sim/runtime-secrets. - Add a test for the non-retriable binary-secret path. --- apps/realtime/package.json | 1 + apps/realtime/src/bootstrap.ts | 9 ++ apps/sim/bootstrap.ts | 13 +++ apps/sim/package.json | 1 + bun.lock | 24 +++++ docker/app.Dockerfile | 13 ++- docker/realtime.Dockerfile | 2 +- packages/runtime-secrets/package.json | 38 ++++++++ packages/runtime-secrets/src/index.test.ts | 91 ++++++++++++++++++ packages/runtime-secrets/src/index.ts | 102 +++++++++++++++++++++ packages/runtime-secrets/tsconfig.json | 5 + packages/runtime-secrets/vitest.config.ts | 7 ++ 12 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 apps/realtime/src/bootstrap.ts create mode 100644 apps/sim/bootstrap.ts create mode 100644 packages/runtime-secrets/package.json create mode 100644 packages/runtime-secrets/src/index.test.ts create mode 100644 packages/runtime-secrets/src/index.ts create mode 100644 packages/runtime-secrets/tsconfig.json create mode 100644 packages/runtime-secrets/vitest.config.ts diff --git a/apps/realtime/package.json b/apps/realtime/package.json index 99867ef852d..e8b1e1607be 100644 --- a/apps/realtime/package.json +++ b/apps/realtime/package.json @@ -26,6 +26,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", diff --git a/apps/realtime/src/bootstrap.ts b/apps/realtime/src/bootstrap.ts new file mode 100644 index 00000000000..fe786372052 --- /dev/null +++ b/apps/realtime/src/bootstrap.ts @@ -0,0 +1,9 @@ +/** + * Container entrypoint. Hydrates `process.env` from the runtime secret before + * loading the Socket.IO server, whose modules (`@/env`, DB preflight) read env + * at import time. See `@sim/runtime-secrets`. + */ +import { loadRuntimeSecrets } from '@sim/runtime-secrets' + +await loadRuntimeSecrets() +await import('@/index') diff --git a/apps/sim/bootstrap.ts b/apps/sim/bootstrap.ts new file mode 100644 index 00000000000..bc2e92b882c --- /dev/null +++ b/apps/sim/bootstrap.ts @@ -0,0 +1,13 @@ +/** + * Container entrypoint. Hydrates `process.env` from the runtime secret before + * loading the Next.js standalone server, so application modules that read env at + * import time see the full configuration. See `@sim/runtime-secrets`. + */ +import { loadRuntimeSecrets } from '@sim/runtime-secrets' + +await loadRuntimeSecrets() +// `server.js` is the Next standalone build artifact, a sibling of this file in +// the image; it does not exist at type-check time, so the specifier is held in a +// variable to keep it out of static module resolution. +const standaloneServer = './server.js' +await import(standaloneServer) diff --git a/apps/sim/package.json b/apps/sim/package.json index 88e9575836d..dcb9c2bc649 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -99,6 +99,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", diff --git a/bun.lock b/bun.lock index e9bb4f978b3..c20f6bfa669 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "simstudio", @@ -67,6 +68,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", @@ -158,6 +160,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", @@ -400,6 +403,21 @@ "typescript": "^5.7.3", }, }, + "packages/runtime-secrets": { + "name": "@sim/runtime-secrets", + "version": "0.1.0", + "dependencies": { + "@aws-sdk/client-secrets-manager": "3.1032.0", + "@sim/logger": "workspace:*", + "@sim/utils": "workspace:*", + }, + "devDependencies": { + "@sim/tsconfig": "workspace:*", + "@types/node": "24.2.1", + "typescript": "^5.7.3", + "vitest": "^4.1.0", + }, + }, "packages/security": { "name": "@sim/security", "version": "0.1.0", @@ -1469,6 +1487,8 @@ "@sim/realtime-protocol": ["@sim/realtime-protocol@workspace:packages/realtime-protocol"], + "@sim/runtime-secrets": ["@sim/runtime-secrets@workspace:packages/runtime-secrets"], + "@sim/security": ["@sim/security@workspace:packages/security"], "@sim/testing": ["@sim/testing@workspace:packages/testing"], @@ -4237,6 +4257,8 @@ "@sim/realtime/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], + "@sim/runtime-secrets/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], + "@sim/security/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], "@smithy/middleware-compression/fflate": ["fflate@0.8.1", "", {}, "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ=="], @@ -4749,6 +4771,8 @@ "@sim/realtime/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], + "@sim/runtime-secrets/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], + "@sim/security/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], "@trigger.dev/core/@opentelemetry/api-logs/@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index ff0ea1ccc28..e6e7f22bb53 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -81,6 +81,13 @@ RUN --mount=type=cache,id=next-cache-${TARGETPLATFORM},target=/app/apps/sim/.nex --mount=type=cache,id=turbo-cache-${TARGETPLATFORM},target=/app/.turbo \ bun run build +# Bundle the secrets-loading bootstrap into a self-contained entrypoint. It runs +# before (and outside) the Next standalone server, so its dependencies +# (@sim/runtime-secrets, AWS SDK) are inlined here rather than resolved from the +# pruned standalone node_modules. The dynamic import of ./server.js stays a +# runtime import. +RUN bun build apps/sim/bootstrap.ts --target=bun --outfile=apps/sim/bootstrap.js + # ======================================== # Runner Stage: Run the actual app # ======================================== @@ -100,6 +107,10 @@ COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/public ./apps/sim/public COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/standalone ./ COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/static ./apps/sim/.next/static +# Self-contained secrets-loading bootstrap (bundled in the builder stage). Runs +# before the standalone server.js to hydrate process.env from the runtime secret. +COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/bootstrap.js ./apps/sim/bootstrap.js + # Copy blog/author content for runtime filesystem reads (not part of the JS bundle) COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/content ./apps/sim/content @@ -128,4 +139,4 @@ EXPOSE 3000 ENV PORT=3000 \ HOSTNAME="0.0.0.0" -CMD ["bun", "apps/sim/server.js"] +CMD ["bun", "apps/sim/bootstrap.js"] diff --git a/docker/realtime.Dockerfile b/docker/realtime.Dockerfile index 16f3cd1c32f..d403c906462 100644 --- a/docker/realtime.Dockerfile +++ b/docker/realtime.Dockerfile @@ -49,4 +49,4 @@ USER nextjs EXPOSE 3002 -CMD ["bun", "apps/realtime/src/index.ts"] +CMD ["bun", "apps/realtime/src/bootstrap.ts"] diff --git a/packages/runtime-secrets/package.json b/packages/runtime-secrets/package.json new file mode 100644 index 00000000000..ee57201f3d1 --- /dev/null +++ b/packages/runtime-secrets/package.json @@ -0,0 +1,38 @@ +{ + "name": "@sim/runtime-secrets", + "version": "0.1.0", + "private": true, + "sideEffects": false, + "type": "module", + "license": "Apache-2.0", + "engines": { + "bun": ">=1.2.13", + "node": ">=20.0.0" + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "type-check": "tsc --noEmit", + "lint": "biome check --write --unsafe .", + "lint:check": "biome check .", + "format": "biome format --write .", + "format:check": "biome format .", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@aws-sdk/client-secrets-manager": "3.1032.0", + "@sim/logger": "workspace:*", + "@sim/utils": "workspace:*" + }, + "devDependencies": { + "@sim/tsconfig": "workspace:*", + "@types/node": "24.2.1", + "typescript": "^5.7.3", + "vitest": "^4.1.0" + } +} diff --git a/packages/runtime-secrets/src/index.test.ts b/packages/runtime-secrets/src/index.test.ts new file mode 100644 index 00000000000..dfd5cec1f4b --- /dev/null +++ b/packages/runtime-secrets/src/index.test.ts @@ -0,0 +1,91 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSend } = vi.hoisted(() => ({ mockSend: vi.fn() })) + +vi.mock('@aws-sdk/client-secrets-manager', () => ({ + SecretsManagerClient: class SecretsManagerClient { + send = mockSend + }, + GetSecretValueCommand: class GetSecretValueCommand { + constructor(public input: unknown) {} + }, +})) + +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }), +})) + +vi.mock('@sim/utils/helpers', () => ({ + sleep: vi.fn().mockResolvedValue(undefined), +})) + +import { loadRuntimeSecrets } from './index' + +const TOUCHED = ['SIM_ENV_SECRET_ID', 'FOO', 'BAZ'] as const + +describe('loadRuntimeSecrets', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const key of TOUCHED) delete process.env[key] + }) + + afterEach(() => { + for (const key of TOUCHED) delete process.env[key] + }) + + it('no-ops when SIM_ENV_SECRET_ID is unset', async () => { + await loadRuntimeSecrets() + expect(mockSend).not.toHaveBeenCalled() + }) + + it('hydrates process.env from the parsed secret JSON', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({ SecretString: JSON.stringify({ FOO: 'bar', BAZ: 'qux' }) }) + + await loadRuntimeSecrets() + + expect(process.env.FOO).toBe('bar') + expect(process.env.BAZ).toBe('qux') + }) + + it('never overwrites an already-set env var', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + process.env.FOO = 'existing' + mockSend.mockResolvedValue({ SecretString: JSON.stringify({ FOO: 'new', BAZ: 'qux' }) }) + + await loadRuntimeSecrets() + + expect(process.env.FOO).toBe('existing') + expect(process.env.BAZ).toBe('qux') + }) + + it('throws when the secret is not valid JSON', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({ SecretString: 'not json' }) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/not valid JSON/) + }) + + it('throws when the secret JSON is not an object', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({ SecretString: JSON.stringify(['a', 'b']) }) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/must be a JSON object/) + }) + + it('throws immediately on a binary secret (no SecretString), without retrying', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({}) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/binary secrets/) + expect(mockSend).toHaveBeenCalledTimes(1) + }) + + it('retries then throws when the fetch keeps failing', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockRejectedValue(new Error('boom')) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/Failed to fetch runtime secrets/) + expect(mockSend).toHaveBeenCalledTimes(3) + }) +}) diff --git a/packages/runtime-secrets/src/index.ts b/packages/runtime-secrets/src/index.ts new file mode 100644 index 00000000000..86c79e7952d --- /dev/null +++ b/packages/runtime-secrets/src/index.ts @@ -0,0 +1,102 @@ +import type { GetSecretValueCommandOutput } from '@aws-sdk/client-secrets-manager' +import { GetSecretValueCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { backoffWithJitter } from '@sim/utils/retry' + +const logger = createLogger('RuntimeSecrets') + +/** Plaintext env var (set in the ECS task definition) naming the secret to ingest. */ +const SECRET_ID_ENV = 'SIM_ENV_SECRET_ID' + +const MAX_ATTEMPTS = 3 + +/** Bounds each Secrets Manager request so a stalled response can't hang boot. */ +const REQUEST_TIMEOUT_MS = 5000 + +/** + * Fetches the combined `/{env}/sim/env-vars` secret once at container boot and + * hydrates `process.env`, so secrets no longer have to be fanned out into the + * ECS task definition (which is approaching the 64 KB rendered-document limit). + * + * Must run before any application module that reads env at import time. No-ops + * when {@link SECRET_ID_ENV} is unset (local dev / self-hosted keep using their + * own env). Existing `process.env` keys are never overwritten, so explicit + * task-definition `environment` entries win. Throws on any fetch/parse failure + * so a misconfigured container crashes instead of booting without its config. + */ +export async function loadRuntimeSecrets(): Promise { + const secretId = process.env[SECRET_ID_ENV] + if (!secretId) { + logger.info(`${SECRET_ID_ENV} not set; skipping runtime secret ingestion`) + return + } + + const client = new SecretsManagerClient( + process.env.AWS_REGION ? { region: process.env.AWS_REGION } : {} + ) + + const secretString = await fetchSecretString(client, secretId) + const entries = parseSecretJson(secretString) + + let loaded = 0 + let skipped = 0 + for (const [key, value] of Object.entries(entries)) { + if (key in process.env) { + skipped++ + continue + } + process.env[key] = typeof value === 'string' ? value : JSON.stringify(value) + loaded++ + } + + logger.info('Runtime secrets ingested', { secretId, loaded, skipped }) +} + +async function fetchSecretString(client: SecretsManagerClient, secretId: string): Promise { + const response = await sendWithRetry(client, secretId) + if (!response.SecretString) { + // Non-retriable: a binary secret will never become a string between attempts. + throw new Error('Secret has no SecretString (binary secrets are not supported)') + } + return response.SecretString +} + +async function sendWithRetry( + client: SecretsManagerClient, + secretId: string +): Promise { + let lastError: unknown + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + return await client.send(new GetSecretValueCommand({ SecretId: secretId }), { + abortSignal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }) + } catch (error) { + lastError = error + if (attempt < MAX_ATTEMPTS) { + const delay = backoffWithJitter(attempt, null, { baseMs: 200, maxMs: 2000 }) + logger.warn( + `Failed to fetch runtime secrets (attempt ${attempt}/${MAX_ATTEMPTS}), retrying`, + { error: getErrorMessage(error) } + ) + await sleep(delay) + } + } + } + throw new Error(`Failed to fetch runtime secrets from ${secretId}: ${getErrorMessage(lastError)}`) +} + +function parseSecretJson(secretString: string): Record { + let parsed: unknown + try { + parsed = JSON.parse(secretString) + } catch (error) { + throw new Error(`Runtime secret is not valid JSON: ${getErrorMessage(error)}`) + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('Runtime secret must be a JSON object of key/value pairs') + } + return parsed as Record +} diff --git a/packages/runtime-secrets/tsconfig.json b/packages/runtime-secrets/tsconfig.json new file mode 100644 index 00000000000..1ffa3d2e844 --- /dev/null +++ b/packages/runtime-secrets/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "@sim/tsconfig/library.json", + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/runtime-secrets/vitest.config.ts b/packages/runtime-secrets/vitest.config.ts new file mode 100644 index 00000000000..2b1c323fe22 --- /dev/null +++ b/packages/runtime-secrets/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + environment: 'node', + }, +}) From 5a938e5d564eaaf0a519f137a45aa6a8f3c5828a Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 24 Jun 2026 16:29:00 -0700 Subject: [PATCH 08/14] improvement(sandbox): mount workspace files by presigned URL instead of buffering bytes (#5202) * improvement(sandbox): mount workspace files by presigned URL instead of buffering bytes Files and directories mounted into the function_execute sandbox were downloaded into the web process, re-encoded, and shipped inline. Mirror the table-snapshot path: under cloud storage, presign each file and let the sandbox curl it directly (no web-heap transit). Local storage keeps the buffered fallback. Add a count cap on the inputFiles list and a generous aggregate URL-mount byte ceiling so oversized requests fail fast instead of filling sandbox disk. * improvement(sandbox): use mount path in size-limit errors, display GB, add directory local-fallback test --- .../tools/handlers/function-execute.test.ts | 149 +++++++++++++++++- .../tools/handlers/function-execute.ts | 147 +++++++++++------ 2 files changed, 244 insertions(+), 52 deletions(-) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts index b47286a03b2..f6494b14aa0 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts @@ -13,6 +13,11 @@ const { mockGeneratePresignedDownloadUrl, mockHasCloudStorage, mockExecuteTool, + mockListWorkspaceFiles, + mockFindWorkspaceFileRecord, + mockFetchWorkspaceFileBuffer, + mockGetSandboxWorkspaceFilePath, + mockListWorkspaceFileFolders, } = vi.hoisted(() => ({ mockIsFeatureEnabled: vi.fn(), mockGetTableById: vi.fn(), @@ -23,6 +28,11 @@ const { mockGeneratePresignedDownloadUrl: vi.fn(), mockHasCloudStorage: vi.fn(), mockExecuteTool: vi.fn(), + mockListWorkspaceFiles: vi.fn(), + mockFindWorkspaceFileRecord: vi.fn(), + mockFetchWorkspaceFileBuffer: vi.fn(), + mockGetSandboxWorkspaceFilePath: vi.fn(), + mockListWorkspaceFileFolders: vi.fn(), })) vi.mock('@/lib/core/config/feature-flags', () => ({ isFeatureEnabled: mockIsFeatureEnabled })) @@ -41,15 +51,14 @@ vi.mock('@/lib/uploads/core/storage-service', () => ({ hasCloudStorage: mockHasCloudStorage, })) vi.mock('@/tools', () => ({ executeTool: mockExecuteTool })) -// Workspace-file + VFS surfaces are unused on the tables-only path; stub to avoid heavy loads. vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ - fetchWorkspaceFileBuffer: vi.fn(), - findWorkspaceFileRecord: vi.fn(), - getSandboxWorkspaceFilePath: vi.fn(), - listWorkspaceFiles: vi.fn(), + fetchWorkspaceFileBuffer: mockFetchWorkspaceFileBuffer, + findWorkspaceFileRecord: mockFindWorkspaceFileRecord, + getSandboxWorkspaceFilePath: mockGetSandboxWorkspaceFilePath, + listWorkspaceFiles: mockListWorkspaceFiles, })) vi.mock('@/lib/uploads/contexts/workspace/workspace-file-folder-manager', () => ({ - listWorkspaceFileFolders: vi.fn(), + listWorkspaceFileFolders: mockListWorkspaceFileFolders, })) vi.mock('@/lib/copilot/vfs/path-utils', () => ({ decodeVfsPathSegments: (p: string) => p.split('/'), @@ -247,3 +256,131 @@ describe('executeFunctionExecute table mounts', () => { expect(mockGetOrCreateTableSnapshot).not.toHaveBeenCalled() }) }) + +const fileRecord = { + id: 'file_1', + workspaceId: 'ws_1', + name: 'data.csv', + key: 'workspace/ws_1/data.csv', + path: '/api/files/serve/workspace%2Fws_1%2Fdata.csv', + size: 100, + type: 'text/csv', + storageContext: 'workspace' as const, +} + +describe('executeFunctionExecute file mounts', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecuteTool.mockResolvedValue({ success: true }) + mockIsFeatureEnabled.mockResolvedValue(false) + mockHasCloudStorage.mockReturnValue(true) + mockGeneratePresignedDownloadUrl.mockResolvedValue('https://s3.example/file?sig=abc') + mockListWorkspaceFiles.mockResolvedValue([fileRecord]) + mockFindWorkspaceFileRecord.mockReturnValue(fileRecord) + mockGetSandboxWorkspaceFilePath.mockReturnValue('/home/user/files/data.csv') + }) + + it('cloud storage: mounts by presigned URL with the record context, no bytes through web', async () => { + await executeFunctionExecute({ inputFiles: ['files/data.csv'] }, context as never) + + expect(mockFetchWorkspaceFileBuffer).not.toHaveBeenCalled() + expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledWith( + 'workspace/ws_1/data.csv', + 'workspace', + expect.any(Number) + ) + expect(mountedFiles()[0]).toEqual({ + type: 'url', + path: '/home/user/files/data.csv', + url: 'https://s3.example/file?sig=abc', + }) + }) + + it('local storage: falls back to a buffered inline content mount', async () => { + mockHasCloudStorage.mockReturnValue(false) + mockFetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('name\nAda\n')) + + await executeFunctionExecute({ inputFiles: ['files/data.csv'] }, context as never) + + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + const file = mountedFiles()[0] + expect(file.path).toBe('/home/user/files/data.csv') + expect(file.content).toBe('name\nAda\n') + expect(file.type).toBeUndefined() + }) + + it('cloud storage: throws when a file exceeds the per-file URL mount limit', async () => { + mockFindWorkspaceFileRecord.mockReturnValue({ ...fileRecord, size: 600 * 1024 * 1024 }) + + await expect( + executeFunctionExecute({ inputFiles: ['files/data.csv'] }, context as never) + ).rejects.toThrow(/per-file mount limit/) + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + }) + + it('cloud storage: throws when mounts exceed the aggregate URL mount limit', async () => { + // Each file is at the 500MB per-file cap; the 5th pushes the running total past 2GB. + mockFindWorkspaceFileRecord.mockReturnValue({ ...fileRecord, size: 500 * 1024 * 1024 }) + const paths = Array.from({ length: 5 }, (_, i) => `files/big-${i}.csv`) + + await expect(executeFunctionExecute({ inputFiles: paths }, context as never)).rejects.toThrow( + /total mount limit/ + ) + expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledTimes(4) + }) + + it('throws when the inputFiles list exceeds the mounted-file count cap', async () => { + const paths = Array.from({ length: 501 }, (_, i) => `files/f-${i}.csv`) + + await expect(executeFunctionExecute({ inputFiles: paths }, context as never)).rejects.toThrow( + /Too many input files/ + ) + expect(mockListWorkspaceFiles).not.toHaveBeenCalled() + }) + + it('cloud storage: mounts each directory descendant by presigned URL', async () => { + mockListWorkspaceFileFolders.mockResolvedValue([{ path: 'Reports' }]) + const descendant = { + ...fileRecord, + name: 'q1.csv', + key: 'workspace/ws_1/q1.csv', + folderPath: 'Reports', + } + mockListWorkspaceFiles.mockResolvedValue([descendant]) + + await executeFunctionExecute({ inputs: { directories: ['files/Reports'] } }, context as never) + + expect(mockFetchWorkspaceFileBuffer).not.toHaveBeenCalled() + expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledWith( + 'workspace/ws_1/q1.csv', + 'workspace', + expect.any(Number) + ) + expect(mountedFiles()[0]).toEqual({ + type: 'url', + path: '/home/user/files/Reports/q1.csv', + url: 'https://s3.example/file?sig=abc', + }) + }) + + it('local storage: buffers directory descendants via inline content', async () => { + mockHasCloudStorage.mockReturnValue(false) + mockListWorkspaceFileFolders.mockResolvedValue([{ path: 'Reports' }]) + const descendant = { + ...fileRecord, + name: 'q1.csv', + key: 'workspace/ws_1/q1.csv', + folderPath: 'Reports', + } + mockListWorkspaceFiles.mockResolvedValue([descendant]) + mockFetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('a,b\n1,2\n')) + + await executeFunctionExecute({ inputs: { directories: ['files/Reports'] } }, context as never) + + expect(mockGeneratePresignedDownloadUrl).not.toHaveBeenCalled() + const file = mountedFiles()[0] + expect(file.path).toBe('/home/user/files/Reports/q1.csv') + expect(file.content).toBe('a,b\n1,2\n') + expect(file.type).toBeUndefined() + }) +}) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index 2550b63dde1..bc32699d66a 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -14,6 +14,7 @@ import { findWorkspaceFileRecord, getSandboxWorkspaceFilePath, listWorkspaceFiles, + type WorkspaceFileRecord, } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { downloadFile, @@ -37,15 +38,98 @@ const MAX_MOUNTED_FILES = 500 const SNAPSHOT_MIN_ROWS = 500 /** - * Lifetime of the presigned URL handed to the sandbox to fetch a snapshot. Long enough to download - * a large file at sandbox startup; the URL grants read to only that one version-pinned object. + * Lifetime of a presigned URL handed to the sandbox to fetch a mounted object (table snapshot or + * workspace file). Long enough to download a large file at sandbox startup; the URL grants read to + * only that one object. */ -const SNAPSHOT_URL_TTL_SECONDS = 600 +const MOUNT_URL_TTL_SECONDS = 600 + +/** + * Per-file ceiling for URL-mounted workspace files. The bytes never transit the web process — the + * sandbox curls them straight from storage — so the bound is sandbox disk, not web heap (unlike the + * inline MAX_FILE_SIZE path). + */ +const MOUNT_URL_MAX_BYTES = 500 * 1024 * 1024 + +/** + * Aggregate ceiling across all URL-mounted files in one request. URL mounts bypass the web heap (so + * they don't count against MAX_TOTAL_SIZE), but the sandbox still curls every byte onto its disk — + * this rejects an oversized request up front instead of filling the sandbox disk one slow curl at a + * time. Generous vs MAX_TOTAL_SIZE since the bytes never transit web memory. + */ +const MAX_TOTAL_URL_BYTES = 2 * 1024 * 1024 * 1024 type SandboxFile = | { type?: 'content'; path: string; content: string; encoding?: 'base64' } | { type: 'url'; path: string; url: string } +/** + * Running byte totals for one resolveInputFiles call. `buffered` bytes pass through the web process + * (capped by MAX_TOTAL_SIZE); `url` bytes are curled straight into the sandbox (capped by + * MAX_TOTAL_URL_BYTES). Tracked separately because the two ceilings protect different resources — + * web heap vs sandbox disk. + */ +interface MountedBytes { + buffered: number + url: number +} + +/** + * Mounts a stored workspace file into the sandbox and records its bytes against the running totals. + * With cloud storage the sandbox fetches the bytes itself from a presigned URL (no web-heap transit, + * per-file ceiling MOUNT_URL_MAX_BYTES, aggregate ceiling MAX_TOTAL_URL_BYTES); with local storage a + * presigned URL is an app-internal serve path a remote sandbox can't reach, so we buffer the bytes + * through the web process under the inline MAX_FILE_SIZE / MAX_TOTAL_SIZE guards. + */ +async function pushWorkspaceFileMount( + sandboxFiles: SandboxFile[], + record: WorkspaceFileRecord, + mountPath: string, + mounted: MountedBytes +): Promise { + if (hasCloudStorage()) { + if (record.size > MOUNT_URL_MAX_BYTES) { + throw new Error( + `Input file "${mountPath}" is ${Math.round(record.size / 1024 / 1024)}MB, over the ${MOUNT_URL_MAX_BYTES / 1024 / 1024}MB per-file mount limit.` + ) + } + if (mounted.url + record.size > MAX_TOTAL_URL_BYTES) { + throw new Error( + `Mounting "${mountPath}" would exceed the ${MAX_TOTAL_URL_BYTES / 1024 / 1024 / 1024}GB total mount limit. Mount fewer or smaller files.` + ) + } + const url = await generatePresignedDownloadUrl( + record.key, + record.storageContext ?? 'workspace', + MOUNT_URL_TTL_SECONDS + ) + sandboxFiles.push({ type: 'url', path: mountPath, url }) + mounted.url += record.size + return + } + + if (record.size > MAX_FILE_SIZE) { + throw new Error( + `Input file "${mountPath}" is ${Math.round(record.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.` + ) + } + if (mounted.buffered + record.size > MAX_TOTAL_SIZE) { + throw new Error( + `Mounting "${mountPath}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller files.` + ) + } + const buffer = await fetchWorkspaceFileBuffer(record) + const isText = /^text\/|application\/json|application\/xml|application\/csv/.test( + record.type || '' + ) + sandboxFiles.push({ + path: mountPath, + content: isText ? buffer.toString('utf-8') : buffer.toString('base64'), + encoding: isText ? undefined : 'base64', + }) + mounted.buffered += buffer.length +} + interface CanonicalFileInput { path: string sandboxPath?: string @@ -89,10 +173,15 @@ export async function resolveInputFiles( inputDirectories?: unknown[] ): Promise { const sandboxFiles: SandboxFile[] = [] - let totalSize = 0 + const mounted: MountedBytes = { buffered: 0, url: 0 } const betaEnabled = await isFeatureEnabled('mothership-beta') if (inputFiles?.length && workspaceId) { + if (inputFiles.length > MAX_MOUNTED_FILES) { + throw new Error( + `Too many input files (${inputFiles.length}). Maximum is ${MAX_MOUNTED_FILES}. Mount fewer files.` + ) + } const allFiles = await listWorkspaceFiles(workspaceId, { includeReservedSystemFiles: betaEnabled, }) @@ -124,33 +213,14 @@ export async function resolveInputFiles( `Input file not found: "${filePath}". Pass the exact canonical VFS path copied from glob/read (e.g. "files/Reports/data.csv").` ) } - if (record.size > MAX_FILE_SIZE) { - throw new Error( - `Input file "${filePath}" is ${Math.round(record.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.` - ) - } - if (totalSize + record.size > MAX_TOTAL_SIZE) { - throw new Error( - `Mounting "${filePath}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller files.` - ) - } - const buffer = await fetchWorkspaceFileBuffer(record) - totalSize += buffer.length - const isText = /^text\/|application\/json|application\/xml|application\/csv/.test( - record.type || '' - ) - const content = isText ? buffer.toString('utf-8') : buffer.toString('base64') const explicitSandboxPath = typeof fileRef === 'object' && fileRef !== null ? (fileRef as CanonicalFileInput).sandboxPath : undefined - sandboxFiles.push({ - path: - explicitSandboxPath || - (alias ? workflowAliasSandboxPath(alias.aliasPath) : getSandboxWorkspaceFilePath(record)), - content, - encoding: isText ? undefined : 'base64', - }) + const mountPath = + explicitSandboxPath || + (alias ? workflowAliasSandboxPath(alias.aliasPath) : getSandboxWorkspaceFilePath(record)) + await pushWorkspaceFileMount(sandboxFiles, record, mountPath, mounted) } } @@ -228,17 +298,6 @@ export async function resolveInputFiles( } } for (const record of descendants) { - if (record.size > MAX_FILE_SIZE) { - throw new Error(`Input file exceeds size limit: ${record.name}`) - } - if (totalSize + record.size > MAX_TOTAL_SIZE) { - throw new Error('Total input size limit exceeded while mounting directory') - } - const buffer = await fetchWorkspaceFileBuffer(record) - totalSize += buffer.length - const isText = /^text\/|application\/json|application\/xml|application\/csv/.test( - record.type || '' - ) const relativeFolder = record.folderPath?.slice(folder.path.length).replace(/^\/+/, '') ?? '' const relativePath = alias @@ -246,11 +305,7 @@ export async function resolveInputFiles( [relativeFolder, record.name].filter(Boolean).join('/').split('/') ) : [relativeFolder, record.name].filter(Boolean).join('/') - sandboxFiles.push({ - path: `${mountRoot}/${relativePath}`, - content: isText ? buffer.toString('utf-8') : buffer.toString('base64'), - encoding: isText ? undefined : 'base64', - }) + await pushWorkspaceFileMount(sandboxFiles, record, `${mountRoot}/${relativePath}`, mounted) } } } @@ -305,7 +360,7 @@ export async function resolveInputFiles( const url = await generatePresignedDownloadUrl( snapshot.key, 'execution', - SNAPSHOT_URL_TTL_SECONDS + MOUNT_URL_TTL_SECONDS ) sandboxFiles.push({ type: 'url', path: mountPath, url }) continue @@ -318,7 +373,7 @@ export async function resolveInputFiles( `Input table "${tableId}" is ${Math.round(snapshot.size / 1024 / 1024)}MB, over the ${MAX_FILE_SIZE / 1024 / 1024}MB per-file mount limit.` ) } - if (totalSize + snapshot.size > MAX_TOTAL_SIZE) { + if (mounted.buffered + snapshot.size > MAX_TOTAL_SIZE) { throw new Error( `Mounting "${tableId}" would exceed the ${MAX_TOTAL_SIZE / 1024 / 1024}MB total mount limit. Mount fewer or smaller tables.` ) @@ -328,7 +383,7 @@ export async function resolveInputFiles( context: 'execution', maxBytes: MAX_FILE_SIZE, }) - totalSize += buffer.length + mounted.buffered += buffer.length sandboxFiles.push({ path: mountPath, content: buffer.toString('utf-8') }) continue } From c3a09694aba809f91021b5bf3246c95b4a2d7eaa Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 17:37:37 -0700 Subject: [PATCH 09/14] fix(tables): SSR crash from tableKeys in a 'use client' module + drop redundant flushChunks (#5204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(tables): move tableKeys to a non-client module so the SSR prefetch works The tables list page crashed at SSR ('tableKeys.list is not a function') because tables/prefetch.ts (a server component) imported tableKeys from hooks/queries/tables.ts — a 'use client' module whose exports resolve to client-reference stubs on the server. Extract the key factory into hooks/queries/utils/table-keys.ts (no 'use client'), mirroring folder-keys.ts, and import it from there in the prefetch, hook, trigger, and consumers. * refactor(chat): drop redundant flushChunks on the SSE error path On an error 'final' event the reader stops via return true, so the post-loop flush is the single flush point. Defer the error append to after that flush (single flush, correct ordering) instead of flushing inside onEvent and again post-loop. No behavior change. * fix(sse): process the final unterminated line on stream end readSSELines broke out of the read loop on 'done' without flushing the TextDecoder or processing the trailing buffer, so a final 'data:' line not terminated by a newline (and any buffered multi-byte character) was dropped. Flush the decoder on end-of-stream and process the remaining buffer. Addresses a Cursor Medium finding on the consolidated SSE reader. --- .../resource-registry/resource-registry.tsx | 2 +- .../[workspaceId]/lib/prefetch.test.ts | 2 +- .../[tableId]/hooks/use-table-event-stream.ts | 2 +- .../[workspaceId]/tables/prefetch.ts | 2 +- .../w/[workflowId]/components/chat/chat.tsx | 14 ++++---- apps/sim/hooks/queries/tables.test.ts | 2 +- apps/sim/hooks/queries/tables.ts | 26 +------------- apps/sim/hooks/queries/utils/table-keys.ts | 34 +++++++++++++++++++ apps/sim/lib/core/utils/sse.test.ts | 26 ++++++++++++++ apps/sim/lib/core/utils/sse.ts | 7 ++-- apps/sim/triggers/table/poller.ts | 2 +- 11 files changed, 79 insertions(+), 40 deletions(-) create mode 100644 apps/sim/hooks/queries/utils/table-keys.ts diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-registry.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-registry.tsx index 460950dfa18..7c3f6a4db5e 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-registry.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-registry.tsx @@ -25,9 +25,9 @@ import { knowledgeKeys } from '@/hooks/queries/kb/knowledge' import { logKeys } from '@/hooks/queries/logs' import { mothershipChatKeys } from '@/hooks/queries/mothership-chats' import { scheduleKeys } from '@/hooks/queries/schedules' -import { tableKeys } from '@/hooks/queries/tables' import { folderKeys } from '@/hooks/queries/utils/folder-keys' import { invalidateWorkflowLists } from '@/hooks/queries/utils/invalidate-workflow-lists' +import { tableKeys } from '@/hooks/queries/utils/table-keys' import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' diff --git a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts index d031c0648ef..2f8375c836f 100644 --- a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts @@ -21,8 +21,8 @@ import { prefetchHomeLists } from '@/app/workspace/[workspaceId]/home/prefetch' import { prefetchKnowledgeBases } from '@/app/workspace/[workspaceId]/knowledge/prefetch' import { prefetchTables } from '@/app/workspace/[workspaceId]/tables/prefetch' import { knowledgeKeys } from '@/hooks/queries/kb/knowledge' -import { tableKeys } from '@/hooks/queries/tables' import { folderKeys } from '@/hooks/queries/utils/folder-keys' +import { tableKeys } from '@/hooks/queries/utils/table-keys' import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts index 34789dff546..6a0efe9695d 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/hooks/use-table-event-stream.ts @@ -13,8 +13,8 @@ import { downloadExportResult, snapshotAndMutateRows, type TableRunState, - tableKeys, } from '@/hooks/queries/tables' +import { tableKeys } from '@/hooks/queries/utils/table-keys' const logger = createLogger('useTableEventStream') diff --git a/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts index 60d6a79a735..8d41a1d6680 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts @@ -1,7 +1,7 @@ import type { QueryClient } from '@tanstack/react-query' import type { TableDefinition } from '@/lib/table' import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' -import { tableKeys } from '@/hooks/queries/tables' +import { tableKeys } from '@/hooks/queries/utils/table-keys' /** * Prefetches the workspace's tables list under the same query key the client diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx index d4caa19424f..fe3892ec05c 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx @@ -561,6 +561,7 @@ export function Chat() { } } + let finalError: string | null = null try { await readSSEEvents<{ event?: string; data?: ExecutionResult; chunk?: string }>(reader, { onParseError: (_data, e) => { @@ -571,12 +572,7 @@ export function Chat() { if (event === 'final' && eventData) { if ('success' in eventData && !eventData.success) { - const errorMessage = eventData.error || 'Workflow execution failed' - flushChunks() - appendMessageContent( - responseMessageId, - `${accumulatedContent ? '\n\n' : ''}Error: ${errorMessage}` - ) + finalError = eventData.error || 'Workflow execution failed' } return true } @@ -589,6 +585,12 @@ export function Chat() { }, }) flushChunks() + if (finalError) { + appendMessageContent( + responseMessageId, + `${accumulatedContent ? '\n\n' : ''}Error: ${finalError}` + ) + } finalizeMessageStream(responseMessageId) } catch (error) { if ((error as Error)?.name !== 'AbortError') { diff --git a/apps/sim/hooks/queries/tables.test.ts b/apps/sim/hooks/queries/tables.test.ts index da4c8c1cc04..d8245e50c31 100644 --- a/apps/sim/hooks/queries/tables.test.ts +++ b/apps/sim/hooks/queries/tables.test.ts @@ -83,13 +83,13 @@ vi.mock('@/components/emcn', () => ({ })) import { - tableKeys, tableRowsInfiniteOptions, tableRowsParamsKey, useDeleteColumn, useRestoreTable, useUpdateColumn, } from '@/hooks/queries/tables' +import { tableKeys } from '@/hooks/queries/utils/table-keys' const TABLE_ID = 'tbl-1' const WORKSPACE_ID = 'ws-1' diff --git a/apps/sim/hooks/queries/tables.ts b/apps/sim/hooks/queries/tables.ts index df49de32b17..cbf69d8c57a 100644 --- a/apps/sim/hooks/queries/tables.ts +++ b/apps/sim/hooks/queries/tables.ts @@ -96,34 +96,10 @@ import { optimisticallyScheduleNewlyEligibleGroups, } from '@/lib/table/deps' import { runUploadStrategy } from '@/lib/uploads/client/direct-upload' +import { type TableQueryScope, tableKeys } from '@/hooks/queries/utils/table-keys' const logger = createLogger('TableQueries') -type TableQueryScope = 'active' | 'archived' | 'all' - -export const tableKeys = { - all: ['tables'] as const, - lists: () => [...tableKeys.all, 'list'] as const, - list: (workspaceId?: string, scope: TableQueryScope = 'active') => - [...tableKeys.lists(), workspaceId ?? '', scope] as const, - details: () => [...tableKeys.all, 'detail'] as const, - detail: (tableId: string) => [...tableKeys.details(), tableId] as const, - exportJobs: (workspaceId?: string) => - [...tableKeys.all, 'export-jobs', workspaceId ?? ''] as const, - rowsRoot: (tableId: string) => [...tableKeys.detail(tableId), 'rows'] as const, - infiniteRows: (tableId: string, paramsKey: string) => - [...tableKeys.rowsRoot(tableId), 'infinite', paramsKey] as const, - rowWrites: (tableId: string) => [...tableKeys.rowsRoot(tableId), 'write'] as const, - find: (tableId: string, paramsKey: string) => - [...tableKeys.rowsRoot(tableId), 'find', paramsKey] as const, - activeDispatches: (tableId: string) => - [...tableKeys.detail(tableId), 'active-dispatches'] as const, - enrichmentDetails: (tableId: string) => - [...tableKeys.detail(tableId), 'enrichment-detail'] as const, - enrichmentDetail: (tableId: string, rowId: string, groupId: string) => - [...tableKeys.enrichmentDetails(tableId), rowId, groupId] as const, -} - type TableRowsParams = Omit & TableIdParamsInput & { filter?: Filter | null diff --git a/apps/sim/hooks/queries/utils/table-keys.ts b/apps/sim/hooks/queries/utils/table-keys.ts new file mode 100644 index 00000000000..cf27bddd013 --- /dev/null +++ b/apps/sim/hooks/queries/utils/table-keys.ts @@ -0,0 +1,34 @@ +/** + * React Query key factory for user-defined tables. + * + * Lives in this standalone (non-`'use client'`) module — like + * {@link file://./folder-keys.ts} — so it can be imported from server + * components (e.g. the tables page prefetch) without pulling in the + * `'use client'` `@/hooks/queries/tables` module, whose exports would + * otherwise resolve to client-reference stubs on the server. + */ + +export type TableQueryScope = 'active' | 'archived' | 'all' + +export const tableKeys = { + all: ['tables'] as const, + lists: () => [...tableKeys.all, 'list'] as const, + list: (workspaceId?: string, scope: TableQueryScope = 'active') => + [...tableKeys.lists(), workspaceId ?? '', scope] as const, + details: () => [...tableKeys.all, 'detail'] as const, + detail: (tableId: string) => [...tableKeys.details(), tableId] as const, + exportJobs: (workspaceId?: string) => + [...tableKeys.all, 'export-jobs', workspaceId ?? ''] as const, + rowsRoot: (tableId: string) => [...tableKeys.detail(tableId), 'rows'] as const, + infiniteRows: (tableId: string, paramsKey: string) => + [...tableKeys.rowsRoot(tableId), 'infinite', paramsKey] as const, + rowWrites: (tableId: string) => [...tableKeys.rowsRoot(tableId), 'write'] as const, + find: (tableId: string, paramsKey: string) => + [...tableKeys.rowsRoot(tableId), 'find', paramsKey] as const, + activeDispatches: (tableId: string) => + [...tableKeys.detail(tableId), 'active-dispatches'] as const, + enrichmentDetails: (tableId: string) => + [...tableKeys.detail(tableId), 'enrichment-detail'] as const, + enrichmentDetail: (tableId: string, rowId: string, groupId: string) => + [...tableKeys.enrichmentDetails(tableId), rowId, groupId] as const, +} diff --git a/apps/sim/lib/core/utils/sse.test.ts b/apps/sim/lib/core/utils/sse.test.ts index 579e23fd607..7ddf99cbd3b 100644 --- a/apps/sim/lib/core/utils/sse.test.ts +++ b/apps/sim/lib/core/utils/sse.test.ts @@ -361,6 +361,32 @@ describe('readSSEEvents', () => { expect(events).toEqual([{ msg: 'hello' }]) }) + it('emits a final data: line that has no trailing newline (stream tail)', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\n', 'data: {"n":2}']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('flushes a multi-byte character in the final unterminated line', async () => { + const encoder = new TextEncoder() + const euro = encoder.encode('€') + const chunk1 = new Uint8Array([...encoder.encode('data: {"s":"'), euro[0], euro[1]]) + const chunk2 = new Uint8Array([euro[2], ...encoder.encode('"}')]) + const stream = createStreamFromChunks([chunk1, chunk2]) + const events: Array<{ s: string }> = [] + await readSSEEvents<{ s: string }>(stream, { + onEvent: (e) => { + events.push(e) + }, + }) + expect(events).toEqual([{ s: '€' }]) + }) + it('skips the [DONE] sentinel', async () => { const stream = streamFromStringChunks(['data: {"n":1}\n\n', 'data: [DONE]\n\n']) const events: number[] = [] diff --git a/apps/sim/lib/core/utils/sse.ts b/apps/sim/lib/core/utils/sse.ts index 50c758f0013..2651147136e 100644 --- a/apps/sim/lib/core/utils/sse.ts +++ b/apps/sim/lib/core/utils/sse.ts @@ -138,11 +138,10 @@ export async function readSSELines(source: SSESource, options: ReadSSELinesOptio if (signal?.aborted) break const { done, value } = await reader.read() - if (done) break - buffer += decoder.decode(value, { stream: true }) + buffer += done ? decoder.decode() : decoder.decode(value, { stream: true }) const lines = buffer.split('\n') - buffer = lines.pop() ?? '' + buffer = done ? '' : (lines.pop() ?? '') for (const rawLine of lines) { if (signal?.aborted) return @@ -156,6 +155,8 @@ export async function readSSELines(source: SSESource, options: ReadSSELinesOptio if ((await onData(data)) === true) return } + + if (done) break } } finally { if (ownsLock) reader.releaseLock() diff --git a/apps/sim/triggers/table/poller.ts b/apps/sim/triggers/table/poller.ts index 6fe6ad17f81..e922c0502ec 100644 --- a/apps/sim/triggers/table/poller.ts +++ b/apps/sim/triggers/table/poller.ts @@ -3,7 +3,7 @@ import { requestJson } from '@/lib/api/client/request' import { listTablesContract } from '@/lib/api/contracts/tables' import type { TableDefinition } from '@/lib/table' import { getQueryClient } from '@/app/_shell/providers/get-query-client' -import { tableKeys } from '@/hooks/queries/tables' +import { tableKeys } from '@/hooks/queries/utils/table-keys' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { useSubBlockStore } from '@/stores/workflows/subblock/store' import type { TriggerConfig } from '@/triggers/types' From ae4bc05e607d3fe69e7b1ffc2723647df810743a Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 18:05:40 -0700 Subject: [PATCH 10/14] feat(gitlab): add repository, code-review, and CI job tools + validation fixes (#5205) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(gitlab): add repository, code-review, and CI job tools + validation fixes Expand the GitLab integration with 12 new tools (all host-aware via getGitLabApiBase, wired through types/index/registry/block): - Repository: list_repository_tree, get_file, create_file, update_file, create_branch, list_branches, list_commits - Code review: get_merge_request_changes, approve_merge_request - CI jobs: list_pipeline_jobs, get_job_log, play_job Validation fixes from /validate-integration: - Correct the block inputs key (credential -> accessToken) so it matches the subBlock id and the params the block reads - Trim projectId before encoding in all tool request URLs (input hygiene) /validate-connector and /validate-trigger passed clean against the GitLab REST API v4 docs — no changes required. * fix(gitlab): address review feedback + regen docs - get_merge_request_changes: use the /diffs endpoint (/changes was removed in GitLab 18.0); return the diff array + count (drops the MR envelope that /diffs no longer provides), fetch max page size in a single call - create_file/update_file: send explicit `encoding: 'text'` for clarity - Remove `// =====` separator comments from types.ts (repo convention) - Regenerate GitLab integration docs + catalog for the 12 new tools --- .../content/docs/en/integrations/gitlab.mdx | 464 ++++++++++++++++++ apps/sim/blocks/blocks/gitlab.ts | 347 ++++++++++++- apps/sim/lib/integrations/integrations.json | 152 +++++- .../sim/tools/gitlab/approve_merge_request.ts | 105 ++++ apps/sim/tools/gitlab/cancel_pipeline.ts | 2 +- apps/sim/tools/gitlab/create_branch.ts | 102 ++++ apps/sim/tools/gitlab/create_file.ts | 106 ++++ apps/sim/tools/gitlab/create_issue.ts | 2 +- apps/sim/tools/gitlab/create_issue_note.ts | 2 +- apps/sim/tools/gitlab/create_merge_request.ts | 2 +- .../tools/gitlab/create_merge_request_note.ts | 2 +- apps/sim/tools/gitlab/create_pipeline.ts | 2 +- apps/sim/tools/gitlab/delete_issue.ts | 2 +- apps/sim/tools/gitlab/get_file.ts | 114 +++++ apps/sim/tools/gitlab/get_issue.ts | 2 +- apps/sim/tools/gitlab/get_job_log.ts | 75 +++ apps/sim/tools/gitlab/get_merge_request.ts | 2 +- .../tools/gitlab/get_merge_request_changes.ts | 98 ++++ apps/sim/tools/gitlab/get_pipeline.ts | 2 +- apps/sim/tools/gitlab/get_project.ts | 2 +- apps/sim/tools/gitlab/index.ts | 28 ++ apps/sim/tools/gitlab/list_branches.ts | 103 ++++ apps/sim/tools/gitlab/list_commits.ts | 129 +++++ apps/sim/tools/gitlab/list_issues.ts | 2 +- apps/sim/tools/gitlab/list_merge_requests.ts | 2 +- apps/sim/tools/gitlab/list_pipeline_jobs.ts | 119 +++++ apps/sim/tools/gitlab/list_pipelines.ts | 2 +- apps/sim/tools/gitlab/list_repository_tree.ts | 120 +++++ apps/sim/tools/gitlab/merge_merge_request.ts | 2 +- apps/sim/tools/gitlab/play_job.ts | 90 ++++ apps/sim/tools/gitlab/retry_pipeline.ts | 2 +- apps/sim/tools/gitlab/types.ts | 234 ++++++++- apps/sim/tools/gitlab/update_file.ts | 118 +++++ apps/sim/tools/gitlab/update_issue.ts | 2 +- apps/sim/tools/gitlab/update_merge_request.ts | 2 +- apps/sim/tools/registry.ts | 50 +- 36 files changed, 2521 insertions(+), 69 deletions(-) create mode 100644 apps/sim/tools/gitlab/approve_merge_request.ts create mode 100644 apps/sim/tools/gitlab/create_branch.ts create mode 100644 apps/sim/tools/gitlab/create_file.ts create mode 100644 apps/sim/tools/gitlab/get_file.ts create mode 100644 apps/sim/tools/gitlab/get_job_log.ts create mode 100644 apps/sim/tools/gitlab/get_merge_request_changes.ts create mode 100644 apps/sim/tools/gitlab/list_branches.ts create mode 100644 apps/sim/tools/gitlab/list_commits.ts create mode 100644 apps/sim/tools/gitlab/list_pipeline_jobs.ts create mode 100644 apps/sim/tools/gitlab/list_repository_tree.ts create mode 100644 apps/sim/tools/gitlab/play_job.ts create mode 100644 apps/sim/tools/gitlab/update_file.ts diff --git a/apps/docs/content/docs/en/integrations/gitlab.mdx b/apps/docs/content/docs/en/integrations/gitlab.mdx index 6d7340bac8e..eb37db7210b 100644 --- a/apps/docs/content/docs/en/integrations/gitlab.mdx +++ b/apps/docs/content/docs/en/integrations/gitlab.mdx @@ -41,6 +41,7 @@ List GitLab projects accessible to the authenticated user | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `owned` | boolean | No | Limit to projects owned by the current user | | `membership` | boolean | No | Limit to projects the current user is a member of | | `search` | string | No | Search projects by name | @@ -65,6 +66,7 @@ Get details of a specific GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path \(e.g., "namespace/project"\) | #### Output @@ -81,6 +83,7 @@ List issues in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `state` | string | No | Filter by state \(opened, closed, all\) | | `labels` | string | No | Comma-separated list of label names | @@ -107,6 +110,7 @@ Get details of a specific GitLab issue | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `issueIid` | number | Yes | Issue number within the project \(the # shown in GitLab UI\) | @@ -124,6 +128,7 @@ Create a new issue in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `title` | string | Yes | Issue title | | `description` | string | No | Issue description \(Markdown supported\) | @@ -147,6 +152,7 @@ Update an existing issue in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `issueIid` | number | Yes | Issue internal ID \(IID\) | | `title` | string | No | New issue title | @@ -172,6 +178,7 @@ Delete an issue from a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `issueIid` | number | Yes | Issue internal ID \(IID\) | @@ -189,6 +196,7 @@ Add a comment to a GitLab issue | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `issueIid` | number | Yes | Issue internal ID \(IID\) | | `body` | string | Yes | Comment body \(Markdown supported\) | @@ -207,6 +215,7 @@ List merge requests in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `state` | string | No | Filter by state \(opened, closed, merged, all\) | | `labels` | string | No | Comma-separated list of label names | @@ -232,6 +241,7 @@ Get details of a specific GitLab merge request | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | @@ -249,6 +259,7 @@ Create a new merge request in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `sourceBranch` | string | Yes | Source branch name | | `targetBranch` | string | Yes | Target branch name | @@ -275,6 +286,7 @@ Update an existing merge request in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | | `title` | string | No | New merge request title | @@ -302,6 +314,7 @@ Merge a merge request in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | | `mergeCommitMessage` | string | No | Custom merge commit message | @@ -324,6 +337,7 @@ Add a comment to a GitLab merge request | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | | `body` | string | Yes | Comment body \(Markdown supported\) | @@ -342,6 +356,7 @@ List pipelines in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `ref` | string | No | Filter by ref \(branch or tag\) | | `status` | string | No | Filter by status \(created, waiting_for_resource, preparing, pending, running, success, failed, canceled, skipped, manual, scheduled\) | @@ -365,6 +380,7 @@ Get details of a specific GitLab pipeline | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `pipelineId` | number | Yes | Pipeline ID | @@ -382,6 +398,7 @@ Trigger a new pipeline in a GitLab project | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `ref` | string | Yes | Branch or tag to run the pipeline on | | `variables` | array | No | Array of variables for the pipeline \(each with key, value, and optional variable_type\) | @@ -400,6 +417,7 @@ Retry a failed GitLab pipeline | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `pipelineId` | number | Yes | Pipeline ID | @@ -417,6 +435,7 @@ Cancel a running GitLab pipeline | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | | `projectId` | string | Yes | Project ID or URL-encoded path | | `pipelineId` | number | Yes | Pipeline ID | @@ -426,4 +445,449 @@ Cancel a running GitLab pipeline | --------- | ---- | ----------- | | `pipeline` | object | The cancelled GitLab pipeline | +### `gitlab_list_repository_tree` + +List files and directories in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `path` | string | No | Path inside the repository to list | +| `ref` | string | No | Branch, tag, or commit SHA to list from | +| `recursive` | boolean | No | Whether to list files recursively | +| `perPage` | number | No | Number of results per page \(default 20, max 100\) | +| `page` | number | No | Page number for pagination | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `tree` | array | List of repository tree entries | +| `total` | number | Total number of tree entries | + +### `gitlab_get_file` + +Get the contents of a file from a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `filePath` | string | Yes | Path to the file in the repository | +| `ref` | string | Yes | Branch, tag, or commit SHA | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `filePath` | string | The file path | +| `fileName` | string | The file name | +| `size` | number | The file size in bytes | +| `ref` | string | The branch, tag, or commit SHA | +| `blobId` | string | The blob ID | +| `lastCommitId` | string | The last commit ID that modified the file | +| `content` | string | The decoded file content | + +### `gitlab_create_file` + +Create a new file in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `filePath` | string | Yes | Path to the file in the repository | +| `branch` | string | Yes | Branch to commit the new file to | +| `content` | string | Yes | File content | +| `commitMessage` | string | Yes | Commit message | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `filePath` | string | The created file path | +| `branch` | string | The branch the file was committed to | + +### `gitlab_update_file` + +Update an existing file in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `filePath` | string | Yes | Path to the file in the repository | +| `branch` | string | Yes | Branch to commit the update to | +| `content` | string | Yes | New file content | +| `commitMessage` | string | Yes | Commit message | +| `lastCommitId` | string | No | Last known commit ID for the file \(optimistic locking\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `filePath` | string | The updated file path | +| `branch` | string | The branch the update was committed to | + +### `gitlab_create_branch` + +Create a new branch in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `branch` | string | Yes | Name of the new branch | +| `ref` | string | Yes | Source branch/tag/SHA | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `name` | string | The created branch name | +| `webUrl` | string | The web URL of the branch | +| `protected` | boolean | Whether the branch is protected | +| `commit` | object | The commit the branch points to | + +### `gitlab_list_branches` + +List branches in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `search` | string | No | Filter branches by name | +| `perPage` | number | No | Number of results per page \(default 20, max 100\) | +| `page` | number | No | Page number for pagination | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `branches` | array | List of branches | +| `total` | number | Total number of branches | + +### `gitlab_list_commits` + +List commits in a GitLab project repository + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `refName` | string | No | Branch, tag, or revision range to list commits from | +| `since` | string | No | Only commits after this ISO 8601 date | +| `until` | string | No | Only commits before this ISO 8601 date | +| `path` | string | No | Only commits affecting this file path | +| `author` | string | No | Filter commits by author | +| `perPage` | number | No | Number of results per page \(default 20, max 100\) | +| `page` | number | No | Page number for pagination | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `commits` | array | List of commits | +| `total` | number | Total number of commits | + +### `gitlab_get_merge_request_changes` + +Get the file changes (diffs) of a GitLab merge request + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `mergeRequestIid` | number | The merge request internal ID \(IID\) | +| `changes` | array | List of file changes \(diffs\) | +| `changesCount` | number | Number of changed files returned | + +### `gitlab_approve_merge_request` + +Approve a GitLab merge request + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `mergeRequestIid` | number | Yes | Merge request internal ID \(IID\) | +| `sha` | string | No | HEAD SHA of the merge request to approve | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `approvalsRequired` | number | Number of approvals required | +| `approvalsLeft` | number | Number of approvals still needed | +| `approvedBy` | array | List of approvers | + +### `gitlab_list_pipeline_jobs` + +List jobs for a GitLab pipeline + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `pipelineId` | number | Yes | Pipeline ID | +| `scope` | string | No | Filter jobs by scope \(e.g. created, running, success, failed\) | +| `includeRetried` | boolean | No | Whether to include retried jobs | +| `perPage` | number | No | Number of results per page \(default 20, max 100\) | +| `page` | number | No | Page number for pagination | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `jobs` | array | List of pipeline jobs | +| `total` | number | Total number of jobs | + +### `gitlab_get_job_log` + +Get the log (trace) of a GitLab job + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `jobId` | number | Yes | Job ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `log` | string | The job log \(trace\) output | + +### `gitlab_play_job` + +Trigger (play) a manual GitLab job + +#### Input + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `projectId` | string | Yes | Project ID or URL-encoded path | +| `jobId` | number | Yes | Job ID | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `id` | number | The job ID | +| `name` | string | The job name | +| `status` | string | The job status | +| `webUrl` | string | The web URL of the job | + + + +## Triggers + +A **Trigger** is a block that starts a workflow when an event happens in this service. + +### GitLab Comment + +Trigger workflow when a comment is added on a commit, merge request, or issue + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(note\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `object_attributes` | object | object_attributes output from the tool | +| ↳ `id` | number | Comment ID | +| ↳ `note` | string | Comment body | +| ↳ `noteable_type` | string | What the comment is on \(Commit, MergeRequest, Issue, Snippet\) | +| ↳ `action` | string | Action \(create, update\) | +| ↳ `url` | string | Comment URL | + + +--- + +### GitLab Event + +Trigger workflow from any GitLab webhook event + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(push, merge_request, issue, etc.\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `user` | json | Actor that triggered the event \(when present\) | +| `object_attributes` | json | Event-specific attributes \(varies by object_kind\) | + + +--- + +### GitLab Issue + +Trigger workflow when an issue is opened, updated, or closed in GitLab + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(issue\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `object_attributes` | object | object_attributes output from the tool | +| ↳ `id` | number | Global issue ID | +| ↳ `iid` | number | Project-scoped issue number | +| ↳ `title` | string | Issue title | +| ↳ `state` | string | State \(opened, closed\) | +| ↳ `action` | string | Action \(open, close, reopen, update\) | +| ↳ `description` | string | Issue description | +| ↳ `confidential` | boolean | Whether the issue is confidential | +| ↳ `url` | string | Issue URL | + + +--- + +### GitLab Merge Request + +Trigger workflow when a merge request is opened, updated, or merged in GitLab + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(merge_request\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `object_attributes` | object | object_attributes output from the tool | +| ↳ `id` | number | Global merge request ID | +| ↳ `iid` | number | Project-scoped merge request number | +| ↳ `title` | string | Merge request title | +| ↳ `state` | string | State \(opened, closed, merged, locked\) | +| ↳ `action` | string | Action \(open, close, reopen, update, merge, etc.\) | +| ↳ `source_branch` | string | Source branch | +| ↳ `target_branch` | string | Target branch | +| ↳ `merge_status` | string | Merge status | +| ↳ `draft` | boolean | Whether the merge request is a draft | +| ↳ `url` | string | Merge request URL | + + +--- + +### GitLab Pipeline + +Trigger workflow when a pipeline status changes in GitLab + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(pipeline\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `object_attributes` | object | object_attributes output from the tool | +| ↳ `id` | number | Pipeline ID | +| ↳ `status` | string | Pipeline status \(success, failed, running, etc.\) | +| ↳ `detailed_status` | string | Detailed pipeline status | +| ↳ `ref` | string | Ref the pipeline ran on | +| ↳ `sha` | string | Commit SHA | +| ↳ `source` | string | Pipeline source \(push, web, schedule, etc.\) | +| ↳ `duration` | number | Pipeline duration in seconds | +| ↳ `url` | string | Pipeline URL | + + +--- + +### GitLab Push + +Trigger workflow when commits are pushed to a GitLab project + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `accessToken` | string | Yes | Used to create the webhook in your project. Requires the Maintainer or Owner role. | +| `projectId` | string | Yes | The GitLab project to register the webhook on. | +| `host` | string | No | Self-managed GitLab host. Leave blank for gitlab.com. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `object_kind` | string | Event kind \(push\) | +| `event_type` | string | GitLab event type from the X-Gitlab-Event header | +| `ref` | string | Git ref that was pushed \(e.g. refs/heads/main\) | +| `branch` | string | Branch name derived from ref | +| `before` | string | SHA before the push | +| `after` | string | SHA after the push | +| `checkout_sha` | string | SHA of the most recent commit | +| `user_username` | string | Username of the pusher | +| `user_name` | string | Display name of the pusher | +| `user_email` | string | Email of the pusher | +| `total_commits_count` | number | Number of commits in the push | +| `commits` | json | Array of commit objects included in this push | diff --git a/apps/sim/blocks/blocks/gitlab.ts b/apps/sim/blocks/blocks/gitlab.ts index 1cfe24a9e25..078422bedac 100644 --- a/apps/sim/blocks/blocks/gitlab.ts +++ b/apps/sim/blocks/blocks/gitlab.ts @@ -46,6 +46,21 @@ export const GitLabBlock: BlockConfig = { { label: 'Create Pipeline', id: 'gitlab_create_pipeline' }, { label: 'Retry Pipeline', id: 'gitlab_retry_pipeline' }, { label: 'Cancel Pipeline', id: 'gitlab_cancel_pipeline' }, + // Repository Operations + { label: 'List Repository Tree', id: 'gitlab_list_repository_tree' }, + { label: 'Get File', id: 'gitlab_get_file' }, + { label: 'Create File', id: 'gitlab_create_file' }, + { label: 'Update File', id: 'gitlab_update_file' }, + { label: 'List Commits', id: 'gitlab_list_commits' }, + { label: 'List Branches', id: 'gitlab_list_branches' }, + { label: 'Create Branch', id: 'gitlab_create_branch' }, + // Additional Merge Request Operations + { label: 'Get MR Changes', id: 'gitlab_get_merge_request_changes' }, + { label: 'Approve Merge Request', id: 'gitlab_approve_merge_request' }, + // Job Operations + { label: 'List Pipeline Jobs', id: 'gitlab_list_pipeline_jobs' }, + { label: 'Get Job Log', id: 'gitlab_get_job_log' }, + { label: 'Play Job', id: 'gitlab_play_job' }, ], value: () => 'gitlab_list_projects', }, @@ -94,6 +109,18 @@ export const GitLabBlock: BlockConfig = { 'gitlab_create_pipeline', 'gitlab_retry_pipeline', 'gitlab_cancel_pipeline', + 'gitlab_list_repository_tree', + 'gitlab_get_file', + 'gitlab_create_file', + 'gitlab_update_file', + 'gitlab_list_commits', + 'gitlab_list_branches', + 'gitlab_create_branch', + 'gitlab_get_merge_request_changes', + 'gitlab_approve_merge_request', + 'gitlab_list_pipeline_jobs', + 'gitlab_get_job_log', + 'gitlab_play_job', ], }, }, @@ -128,6 +155,8 @@ export const GitLabBlock: BlockConfig = { 'gitlab_update_merge_request', 'gitlab_merge_merge_request', 'gitlab_create_merge_request_note', + 'gitlab_get_merge_request_changes', + 'gitlab_approve_merge_request', ], }, }, @@ -140,7 +169,12 @@ export const GitLabBlock: BlockConfig = { required: true, condition: { field: 'operation', - value: ['gitlab_get_pipeline', 'gitlab_retry_pipeline', 'gitlab_cancel_pipeline'], + value: [ + 'gitlab_get_pipeline', + 'gitlab_retry_pipeline', + 'gitlab_cancel_pipeline', + 'gitlab_list_pipeline_jobs', + ], }, }, // Title (for issue/MR creation) @@ -247,7 +281,135 @@ Return ONLY the comment text - no explanations, no extra formatting.`, required: true, condition: { field: 'operation', - value: ['gitlab_create_pipeline'], + value: ['gitlab_create_pipeline', 'gitlab_get_file', 'gitlab_create_branch'], + }, + }, + // File Path + { + id: 'filePath', + title: 'File Path', + type: 'short-input', + placeholder: 'Path to file (e.g., src/index.ts)', + required: true, + condition: { + field: 'operation', + value: ['gitlab_get_file', 'gitlab_create_file', 'gitlab_update_file'], + }, + }, + // Branch + { + id: 'branch', + title: 'Branch', + type: 'short-input', + placeholder: 'Branch name', + required: true, + condition: { + field: 'operation', + value: ['gitlab_create_file', 'gitlab_update_file', 'gitlab_create_branch'], + }, + }, + // File Content + { + id: 'content', + title: 'File Content', + type: 'long-input', + placeholder: 'File content', + required: true, + condition: { + field: 'operation', + value: ['gitlab_create_file', 'gitlab_update_file'], + }, + }, + // Commit Message + { + id: 'commitMessage', + title: 'Commit Message', + type: 'short-input', + placeholder: 'Commit message', + required: true, + condition: { + field: 'operation', + value: ['gitlab_create_file', 'gitlab_update_file'], + }, + }, + // Job ID + { + id: 'jobId', + title: 'Job ID', + type: 'short-input', + placeholder: 'Enter job ID', + required: true, + condition: { + field: 'operation', + value: ['gitlab_get_job_log', 'gitlab_play_job'], + }, + }, + // Subdirectory path (for repository tree) + { + id: 'path', + title: 'Path', + type: 'short-input', + placeholder: 'Subdirectory path (optional)', + mode: 'advanced', + condition: { + field: 'operation', + value: ['gitlab_list_repository_tree'], + }, + }, + // Recursive tree listing + { + id: 'recursive', + title: 'Recursive', + type: 'switch', + mode: 'advanced', + condition: { + field: 'operation', + value: ['gitlab_list_repository_tree'], + }, + }, + // Ref name filter (for list commits) + { + id: 'refName', + title: 'Ref (branch/tag)', + type: 'short-input', + placeholder: 'Branch or tag (optional)', + mode: 'advanced', + condition: { + field: 'operation', + value: ['gitlab_list_commits'], + }, + }, + // Job scope filter (for list pipeline jobs) + { + id: 'scope', + title: 'Job Scope', + type: 'dropdown', + options: [ + { label: 'All', id: '' }, + { label: 'Failed', id: 'failed' }, + { label: 'Success', id: 'success' }, + { label: 'Running', id: 'running' }, + { label: 'Pending', id: 'pending' }, + { label: 'Canceled', id: 'canceled' }, + { label: 'Manual', id: 'manual' }, + ], + value: () => '', + mode: 'advanced', + condition: { + field: 'operation', + value: ['gitlab_list_pipeline_jobs'], + }, + }, + // Commit SHA (for approve merge request) + { + id: 'sha', + title: 'Commit SHA', + type: 'short-input', + placeholder: 'Optional HEAD SHA to approve', + mode: 'advanced', + condition: { + field: 'operation', + value: ['gitlab_approve_merge_request'], }, }, // Labels @@ -427,6 +589,10 @@ Return ONLY the commit message - no explanations, no extra text.`, 'gitlab_list_issues', 'gitlab_list_merge_requests', 'gitlab_list_pipelines', + 'gitlab_list_repository_tree', + 'gitlab_list_branches', + 'gitlab_list_commits', + 'gitlab_list_pipeline_jobs', ], }, }, @@ -444,6 +610,10 @@ Return ONLY the commit message - no explanations, no extra text.`, 'gitlab_list_issues', 'gitlab_list_merge_requests', 'gitlab_list_pipelines', + 'gitlab_list_repository_tree', + 'gitlab_list_branches', + 'gitlab_list_commits', + 'gitlab_list_pipeline_jobs', ], }, }, @@ -475,6 +645,18 @@ Return ONLY the commit message - no explanations, no extra text.`, 'gitlab_create_pipeline', 'gitlab_retry_pipeline', 'gitlab_cancel_pipeline', + 'gitlab_list_repository_tree', + 'gitlab_get_file', + 'gitlab_create_file', + 'gitlab_update_file', + 'gitlab_create_branch', + 'gitlab_list_branches', + 'gitlab_list_commits', + 'gitlab_get_merge_request_changes', + 'gitlab_approve_merge_request', + 'gitlab_list_pipeline_jobs', + 'gitlab_get_job_log', + 'gitlab_play_job', ], config: { tool: (params) => { @@ -710,6 +892,140 @@ Return ONLY the commit message - no explanations, no extra text.`, pipelineId: Number(params.pipelineId), } + case 'gitlab_list_repository_tree': + if (!params.projectId?.trim()) { + throw new Error('Project ID is required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + path: params.path?.trim() || undefined, + recursive: params.recursive || undefined, + perPage: params.perPage ? Number(params.perPage) : undefined, + page: params.page ? Number(params.page) : undefined, + } + + case 'gitlab_get_file': + if (!params.projectId?.trim() || !params.filePath?.trim() || !params.ref?.trim()) { + throw new Error('Project ID, file path, and ref are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + filePath: params.filePath.trim(), + ref: params.ref.trim(), + } + + case 'gitlab_create_file': + case 'gitlab_update_file': + if ( + !params.projectId?.trim() || + !params.filePath?.trim() || + !params.branch?.trim() || + !params.content || + !params.commitMessage?.trim() + ) { + throw new Error( + 'Project ID, file path, branch, content, and commit message are required.' + ) + } + return { + ...baseParams, + projectId: params.projectId.trim(), + filePath: params.filePath.trim(), + branch: params.branch.trim(), + content: params.content, + commitMessage: params.commitMessage.trim(), + } + + case 'gitlab_create_branch': + if (!params.projectId?.trim() || !params.branch?.trim() || !params.ref?.trim()) { + throw new Error('Project ID, branch name, and source ref are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + branch: params.branch.trim(), + ref: params.ref.trim(), + } + + case 'gitlab_list_branches': + if (!params.projectId?.trim()) { + throw new Error('Project ID is required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + perPage: params.perPage ? Number(params.perPage) : undefined, + page: params.page ? Number(params.page) : undefined, + } + + case 'gitlab_list_commits': + if (!params.projectId?.trim()) { + throw new Error('Project ID is required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + refName: params.refName?.trim() || undefined, + perPage: params.perPage ? Number(params.perPage) : undefined, + page: params.page ? Number(params.page) : undefined, + } + + case 'gitlab_get_merge_request_changes': + if (!params.projectId?.trim() || !params.mergeRequestIid) { + throw new Error('Project ID and Merge Request IID are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + mergeRequestIid: Number(params.mergeRequestIid), + } + + case 'gitlab_approve_merge_request': + if (!params.projectId?.trim() || !params.mergeRequestIid) { + throw new Error('Project ID and Merge Request IID are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + mergeRequestIid: Number(params.mergeRequestIid), + sha: params.sha?.trim() || undefined, + } + + case 'gitlab_list_pipeline_jobs': + if (!params.projectId?.trim() || !params.pipelineId) { + throw new Error('Project ID and Pipeline ID are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + pipelineId: Number(params.pipelineId), + scope: params.scope || undefined, + perPage: params.perPage ? Number(params.perPage) : undefined, + page: params.page ? Number(params.page) : undefined, + } + + case 'gitlab_get_job_log': + if (!params.projectId?.trim() || !params.jobId) { + throw new Error('Project ID and Job ID are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + jobId: Number(params.jobId), + } + + case 'gitlab_play_job': + if (!params.projectId?.trim() || !params.jobId) { + throw new Error('Project ID and Job ID are required.') + } + return { + ...baseParams, + projectId: params.projectId.trim(), + jobId: Number(params.jobId), + } + default: return baseParams } @@ -718,7 +1034,7 @@ Return ONLY the commit message - no explanations, no extra text.`, }, inputs: { operation: { type: 'string', description: 'Operation to perform' }, - credential: { type: 'string', description: 'GitLab access token' }, + accessToken: { type: 'string', description: 'GitLab Personal Access Token' }, host: { type: 'string', description: 'Self-managed GitLab host (defaults to gitlab.com)' }, projectId: { type: 'string', description: 'Project ID or URL-encoded path' }, issueIid: { type: 'number', description: 'Issue internal ID' }, @@ -745,6 +1061,16 @@ Return ONLY the commit message - no explanations, no extra text.`, mergeCommitMessage: { type: 'string', description: 'Custom merge commit message' }, perPage: { type: 'number', description: 'Results per page' }, page: { type: 'number', description: 'Page number' }, + filePath: { type: 'string', description: 'Path to file in the repository' }, + branch: { type: 'string', description: 'Branch name' }, + content: { type: 'string', description: 'File content' }, + commitMessage: { type: 'string', description: 'Commit message' }, + jobId: { type: 'number', description: 'Job ID' }, + path: { type: 'string', description: 'Subdirectory path for repository tree' }, + recursive: { type: 'boolean', description: 'Recursively list repository tree' }, + refName: { type: 'string', description: 'Branch or tag name filter' }, + scope: { type: 'string', description: 'Job scope filter' }, + sha: { type: 'string', description: 'Commit SHA' }, }, outputs: { // Project outputs @@ -761,6 +1087,21 @@ Return ONLY the commit message - no explanations, no extra text.`, pipeline: { type: 'json', description: 'Pipeline details' }, // Note outputs note: { type: 'json', description: 'Comment/note details' }, + // Repository outputs + tree: { type: 'json', description: 'Repository tree entries' }, + content: { type: 'string', description: 'File contents (decoded)' }, + fileName: { type: 'string', description: 'File name' }, + branches: { type: 'json', description: 'List of branches' }, + commits: { type: 'json', description: 'List of commits' }, + name: { type: 'string', description: 'Created branch name' }, + webUrl: { type: 'string', description: 'Web URL' }, + // Merge request change outputs + changes: { type: 'json', description: 'Merge request file changes/diffs' }, + approvalsRequired: { type: 'number', description: 'Approvals required' }, + approvalsLeft: { type: 'number', description: 'Approvals remaining' }, + // Job outputs + jobs: { type: 'json', description: 'Pipeline jobs' }, + log: { type: 'string', description: 'Job log output' }, // Success indicator success: { type: 'boolean', description: 'Operation success status' }, }, diff --git a/apps/sim/lib/integrations/integrations.json b/apps/sim/lib/integrations/integrations.json index cf2683bc099..12543d0f629 100644 --- a/apps/sim/lib/integrations/integrations.json +++ b/apps/sim/lib/integrations/integrations.json @@ -1,5 +1,5 @@ { - "updatedAt": "2026-06-18", + "updatedAt": "2026-06-25", "integrations": [ { "type": "onepassword", @@ -5628,11 +5628,90 @@ { "name": "Cancel Pipeline", "description": "Cancel a running GitLab pipeline" + }, + { + "name": "List Repository Tree", + "description": "List files and directories in a GitLab project repository" + }, + { + "name": "Get File", + "description": "Get the contents of a file from a GitLab project repository" + }, + { + "name": "Create File", + "description": "Create a new file in a GitLab project repository" + }, + { + "name": "Update File", + "description": "Update an existing file in a GitLab project repository" + }, + { + "name": "List Commits", + "description": "List commits in a GitLab project repository" + }, + { + "name": "List Branches", + "description": "List branches in a GitLab project repository" + }, + { + "name": "Create Branch", + "description": "Create a new branch in a GitLab project repository" + }, + { + "name": "Get MR Changes", + "description": "Get the file changes (diffs) of a GitLab merge request" + }, + { + "name": "Approve Merge Request", + "description": "Approve a GitLab merge request" + }, + { + "name": "List Pipeline Jobs", + "description": "List jobs for a GitLab pipeline" + }, + { + "name": "Get Job Log", + "description": "Get the log (trace) of a GitLab job" + }, + { + "name": "Play Job", + "description": "Trigger (play) a manual GitLab job" } ], - "operationCount": 19, - "triggers": [], - "triggerCount": 0, + "operationCount": 31, + "triggers": [ + { + "id": "gitlab_push", + "name": "GitLab Push", + "description": "Trigger workflow when commits are pushed to a GitLab project" + }, + { + "id": "gitlab_merge_request", + "name": "GitLab Merge Request", + "description": "Trigger workflow when a merge request is opened, updated, or merged in GitLab" + }, + { + "id": "gitlab_issue", + "name": "GitLab Issue", + "description": "Trigger workflow when an issue is opened, updated, or closed in GitLab" + }, + { + "id": "gitlab_pipeline", + "name": "GitLab Pipeline", + "description": "Trigger workflow when a pipeline status changes in GitLab" + }, + { + "id": "gitlab_comment", + "name": "GitLab Comment", + "description": "Trigger workflow when a comment is added on a commit, merge request, or issue" + }, + { + "id": "gitlab_webhook", + "name": "GitLab Event", + "description": "Trigger workflow from any GitLab webhook event" + } + ], + "triggerCount": 6, "authType": "api-key", "category": "tools", "integrationType": "devops", @@ -11233,8 +11312,39 @@ } ], "operationCount": 6, - "triggers": [], - "triggerCount": 0, + "triggers": [ + { + "id": "pagerduty_incident_triggered", + "name": "PagerDuty Incident Triggered", + "description": "Trigger workflow when a new incident is triggered in PagerDuty" + }, + { + "id": "pagerduty_incident_acknowledged", + "name": "PagerDuty Incident Acknowledged", + "description": "Trigger workflow when an incident is acknowledged in PagerDuty" + }, + { + "id": "pagerduty_incident_resolved", + "name": "PagerDuty Incident Resolved", + "description": "Trigger workflow when an incident is resolved in PagerDuty" + }, + { + "id": "pagerduty_incident_escalated", + "name": "PagerDuty Incident Escalated", + "description": "Trigger workflow when an incident is escalated in PagerDuty" + }, + { + "id": "pagerduty_incident_reassigned", + "name": "PagerDuty Incident Reassigned", + "description": "Trigger workflow when an incident is reassigned in PagerDuty" + }, + { + "id": "pagerduty_webhook", + "name": "PagerDuty Incident Event", + "description": "Trigger workflow from any PagerDuty incident event" + } + ], + "triggerCount": 6, "authType": "api-key", "category": "tools", "integrationType": "observability", @@ -18289,8 +18399,34 @@ } ], "operationCount": 26, - "triggers": [], - "triggerCount": 0, + "triggers": [ + { + "id": "zendesk_ticket_created", + "name": "Zendesk Ticket Created", + "description": "Trigger workflow when a new ticket is created in Zendesk" + }, + { + "id": "zendesk_ticket_status_changed", + "name": "Zendesk Ticket Status Changed", + "description": "Trigger workflow when a ticket status changes in Zendesk" + }, + { + "id": "zendesk_ticket_comment_added", + "name": "Zendesk Ticket Comment Added", + "description": "Trigger workflow when a comment is added to a Zendesk ticket" + }, + { + "id": "zendesk_ticket_priority_changed", + "name": "Zendesk Ticket Priority Changed", + "description": "Trigger workflow when a ticket priority changes in Zendesk" + }, + { + "id": "zendesk_webhook", + "name": "Zendesk Ticket Event", + "description": "Trigger workflow from any Zendesk ticket event" + } + ], + "triggerCount": 5, "authType": "api-key", "category": "tools", "integrationType": "support", diff --git a/apps/sim/tools/gitlab/approve_merge_request.ts b/apps/sim/tools/gitlab/approve_merge_request.ts new file mode 100644 index 00000000000..0db63c53539 --- /dev/null +++ b/apps/sim/tools/gitlab/approve_merge_request.ts @@ -0,0 +1,105 @@ +import type { + GitLabApproveMergeRequestParams, + GitLabApproveMergeRequestResponse, +} from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabApproveMergeRequestTool: ToolConfig< + GitLabApproveMergeRequestParams, + GitLabApproveMergeRequestResponse +> = { + id: 'gitlab_approve_merge_request', + name: 'GitLab Approve Merge Request', + description: 'Approve a GitLab merge request', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + mergeRequestIid: { + type: 'number', + required: true, + visibility: 'user-or-llm', + description: 'Merge request internal ID (IID)', + }, + sha: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'HEAD SHA of the merge request to approve', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/approve` + }, + method: 'POST', + headers: (params) => ({ + 'Content-Type': 'application/json', + 'PRIVATE-TOKEN': params.accessToken, + }), + body: (params) => { + const body: Record = {} + + if (params.sha) body.sha = params.sha + + return body + }, + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + + return { + success: true, + output: { + approvalsRequired: data.approvals_required ?? null, + approvalsLeft: data.approvals_left ?? null, + approvedBy: data.approved_by ?? [], + }, + } + }, + + outputs: { + approvalsRequired: { + type: 'number', + description: 'Number of approvals required', + }, + approvalsLeft: { + type: 'number', + description: 'Number of approvals still needed', + }, + approvedBy: { + type: 'array', + description: 'List of approvers', + }, + }, +} diff --git a/apps/sim/tools/gitlab/cancel_pipeline.ts b/apps/sim/tools/gitlab/cancel_pipeline.ts index 9707f758a0e..b16ef0534e9 100644 --- a/apps/sim/tools/gitlab/cancel_pipeline.ts +++ b/apps/sim/tools/gitlab/cancel_pipeline.ts @@ -40,7 +40,7 @@ export const gitlabCancelPipelineTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/cancel` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/create_branch.ts b/apps/sim/tools/gitlab/create_branch.ts new file mode 100644 index 00000000000..8fe6a8697c2 --- /dev/null +++ b/apps/sim/tools/gitlab/create_branch.ts @@ -0,0 +1,102 @@ +import type { GitLabCreateBranchParams, GitLabCreateBranchResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabCreateBranchTool: ToolConfig< + GitLabCreateBranchParams, + GitLabCreateBranchResponse +> = { + id: 'gitlab_create_branch', + name: 'GitLab Create Branch', + description: 'Create a new branch in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + branch: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Name of the new branch', + }, + ref: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Source branch/tag/SHA', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const queryParams = new URLSearchParams() + queryParams.append('branch', String(params.branch)) + queryParams.append('ref', String(params.ref)) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/branches?${queryParams.toString()}` + }, + method: 'POST', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + + return { + success: true, + output: { + name: data.name ?? null, + webUrl: data.web_url ?? null, + protected: data.protected ?? null, + commit: data.commit ?? null, + }, + } + }, + + outputs: { + name: { + type: 'string', + description: 'The created branch name', + }, + webUrl: { + type: 'string', + description: 'The web URL of the branch', + }, + protected: { + type: 'boolean', + description: 'Whether the branch is protected', + }, + commit: { + type: 'object', + description: 'The commit the branch points to', + }, + }, +} diff --git a/apps/sim/tools/gitlab/create_file.ts b/apps/sim/tools/gitlab/create_file.ts new file mode 100644 index 00000000000..f0468959b4a --- /dev/null +++ b/apps/sim/tools/gitlab/create_file.ts @@ -0,0 +1,106 @@ +import type { GitLabCreateFileParams, GitLabCreateFileResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabCreateFileTool: ToolConfig = { + id: 'gitlab_create_file', + name: 'GitLab Create File', + description: 'Create a new file in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + filePath: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Path to the file in the repository', + }, + branch: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Branch to commit the new file to', + }, + content: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'File content', + }, + commitMessage: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Commit message', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const encodedPath = encodeURIComponent(String(params.filePath)) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/files/${encodedPath}` + }, + method: 'POST', + headers: (params) => ({ + 'Content-Type': 'application/json', + 'PRIVATE-TOKEN': params.accessToken, + }), + body: (params) => ({ + branch: params.branch, + content: params.content, + commit_message: params.commitMessage, + encoding: 'text', + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + + return { + success: true, + output: { + filePath: data.file_path ?? null, + branch: data.branch ?? null, + }, + } + }, + + outputs: { + filePath: { + type: 'string', + description: 'The created file path', + }, + branch: { + type: 'string', + description: 'The branch the file was committed to', + }, + }, +} diff --git a/apps/sim/tools/gitlab/create_issue.ts b/apps/sim/tools/gitlab/create_issue.ts index cc4475831da..f3830a81f0a 100644 --- a/apps/sim/tools/gitlab/create_issue.ts +++ b/apps/sim/tools/gitlab/create_issue.ts @@ -74,7 +74,7 @@ export const gitlabCreateIssueTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/create_issue_note.ts b/apps/sim/tools/gitlab/create_issue_note.ts index 0ad5c218bee..c1ef8447405 100644 --- a/apps/sim/tools/gitlab/create_issue_note.ts +++ b/apps/sim/tools/gitlab/create_issue_note.ts @@ -46,7 +46,7 @@ export const gitlabCreateIssueNoteTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}/notes` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/create_merge_request.ts b/apps/sim/tools/gitlab/create_merge_request.ts index 2c02c2dd0f4..9c47556553e 100644 --- a/apps/sim/tools/gitlab/create_merge_request.ts +++ b/apps/sim/tools/gitlab/create_merge_request.ts @@ -97,7 +97,7 @@ export const gitlabCreateMergeRequestTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/create_merge_request_note.ts b/apps/sim/tools/gitlab/create_merge_request_note.ts index f02f5fa35fb..7f364efba51 100644 --- a/apps/sim/tools/gitlab/create_merge_request_note.ts +++ b/apps/sim/tools/gitlab/create_merge_request_note.ts @@ -49,7 +49,7 @@ export const gitlabCreateMergeRequestNoteTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/notes` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/create_pipeline.ts b/apps/sim/tools/gitlab/create_pipeline.ts index a27ed7ba372..6a9777ea53b 100644 --- a/apps/sim/tools/gitlab/create_pipeline.ts +++ b/apps/sim/tools/gitlab/create_pipeline.ts @@ -47,7 +47,7 @@ export const gitlabCreatePipelineTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipeline` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/delete_issue.ts b/apps/sim/tools/gitlab/delete_issue.ts index 475e52d77a8..7521b826629 100644 --- a/apps/sim/tools/gitlab/delete_issue.ts +++ b/apps/sim/tools/gitlab/delete_issue.ts @@ -38,7 +38,7 @@ export const gitlabDeleteIssueTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'DELETE', diff --git a/apps/sim/tools/gitlab/get_file.ts b/apps/sim/tools/gitlab/get_file.ts new file mode 100644 index 00000000000..7c394b2591f --- /dev/null +++ b/apps/sim/tools/gitlab/get_file.ts @@ -0,0 +1,114 @@ +import type { GitLabGetFileParams, GitLabGetFileResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabGetFileTool: ToolConfig = { + id: 'gitlab_get_file', + name: 'GitLab Get File', + description: 'Get the contents of a file from a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + filePath: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Path to the file in the repository', + }, + ref: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Branch, tag, or commit SHA', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const encodedPath = encodeURIComponent(String(params.filePath)) + const ref = encodeURIComponent(String(params.ref)) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/files/${encodedPath}?ref=${ref}` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + const decoded = Buffer.from(data.content ?? '', 'base64').toString('utf-8') + + return { + success: true, + output: { + filePath: data.file_path ?? null, + fileName: data.file_name ?? null, + size: data.size ?? null, + ref: data.ref ?? null, + blobId: data.blob_id ?? null, + lastCommitId: data.last_commit_id ?? null, + content: decoded, + }, + } + }, + + outputs: { + filePath: { + type: 'string', + description: 'The file path', + }, + fileName: { + type: 'string', + description: 'The file name', + }, + size: { + type: 'number', + description: 'The file size in bytes', + }, + ref: { + type: 'string', + description: 'The branch, tag, or commit SHA', + }, + blobId: { + type: 'string', + description: 'The blob ID', + }, + lastCommitId: { + type: 'string', + description: 'The last commit ID that modified the file', + }, + content: { + type: 'string', + description: 'The decoded file content', + }, + }, +} diff --git a/apps/sim/tools/gitlab/get_issue.ts b/apps/sim/tools/gitlab/get_issue.ts index aa87136e552..edd41e86f6c 100644 --- a/apps/sim/tools/gitlab/get_issue.ts +++ b/apps/sim/tools/gitlab/get_issue.ts @@ -37,7 +37,7 @@ export const gitlabGetIssueTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'GET', diff --git a/apps/sim/tools/gitlab/get_job_log.ts b/apps/sim/tools/gitlab/get_job_log.ts new file mode 100644 index 00000000000..f3cd4b58314 --- /dev/null +++ b/apps/sim/tools/gitlab/get_job_log.ts @@ -0,0 +1,75 @@ +import type { GitLabGetJobLogParams, GitLabGetJobLogResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabGetJobLogTool: ToolConfig = { + id: 'gitlab_get_job_log', + name: 'GitLab Get Job Log', + description: 'Get the log (trace) of a GitLab job', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + jobId: { + type: 'number', + required: true, + visibility: 'user-or-llm', + description: 'Job ID', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/jobs/${params.jobId}/trace` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const log = await response.text() + + return { + success: true, + output: { + log, + }, + } + }, + + outputs: { + log: { + type: 'string', + description: 'The job log (trace) output', + }, + }, +} diff --git a/apps/sim/tools/gitlab/get_merge_request.ts b/apps/sim/tools/gitlab/get_merge_request.ts index f228cfba2eb..b9b45b0d98d 100644 --- a/apps/sim/tools/gitlab/get_merge_request.ts +++ b/apps/sim/tools/gitlab/get_merge_request.ts @@ -43,7 +43,7 @@ export const gitlabGetMergeRequestTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` }, method: 'GET', diff --git a/apps/sim/tools/gitlab/get_merge_request_changes.ts b/apps/sim/tools/gitlab/get_merge_request_changes.ts new file mode 100644 index 00000000000..4bd6ba78e5e --- /dev/null +++ b/apps/sim/tools/gitlab/get_merge_request_changes.ts @@ -0,0 +1,98 @@ +import type { + GitLabGetMergeRequestChangesParams, + GitLabGetMergeRequestChangesResponse, +} from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabGetMergeRequestChangesTool: ToolConfig< + GitLabGetMergeRequestChangesParams, + GitLabGetMergeRequestChangesResponse +> = { + id: 'gitlab_get_merge_request_changes', + name: 'GitLab Get Merge Request Changes', + description: 'Get the file changes (diffs) of a GitLab merge request', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + mergeRequestIid: { + type: 'number', + required: true, + visibility: 'user-or-llm', + description: 'Merge request internal ID (IID)', + }, + }, + + request: { + /** + * Uses the `/diffs` endpoint (the `/changes` endpoint was deprecated in + * GitLab 15.7 and removed in 18.0). `/diffs` returns the diff array directly + * and is paginated; we request the max page size (100) to return the changes + * in a single call, which covers the vast majority of merge requests. + */ + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/diffs?per_page=100` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response, params) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + const changes = Array.isArray(data) ? data : [] + + return { + success: true, + output: { + mergeRequestIid: params?.mergeRequestIid ?? null, + changes, + changesCount: changes.length, + }, + } + }, + + outputs: { + mergeRequestIid: { + type: 'number', + description: 'The merge request internal ID (IID)', + }, + changes: { + type: 'array', + description: 'List of file changes (diffs)', + }, + changesCount: { + type: 'number', + description: 'Number of changed files returned', + }, + }, +} diff --git a/apps/sim/tools/gitlab/get_pipeline.ts b/apps/sim/tools/gitlab/get_pipeline.ts index 1494e65e4bb..c69cfd61185 100644 --- a/apps/sim/tools/gitlab/get_pipeline.ts +++ b/apps/sim/tools/gitlab/get_pipeline.ts @@ -38,7 +38,7 @@ export const gitlabGetPipelineTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}` }, method: 'GET', diff --git a/apps/sim/tools/gitlab/get_project.ts b/apps/sim/tools/gitlab/get_project.ts index 5ea42920584..882da20092a 100644 --- a/apps/sim/tools/gitlab/get_project.ts +++ b/apps/sim/tools/gitlab/get_project.ts @@ -31,7 +31,7 @@ export const gitlabGetProjectTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}` }, method: 'GET', diff --git a/apps/sim/tools/gitlab/index.ts b/apps/sim/tools/gitlab/index.ts index 0133af843ab..25d7ba9e6cd 100644 --- a/apps/sim/tools/gitlab/index.ts +++ b/apps/sim/tools/gitlab/index.ts @@ -1,20 +1,32 @@ +import { gitlabApproveMergeRequestTool } from '@/tools/gitlab/approve_merge_request' import { gitlabCancelPipelineTool } from '@/tools/gitlab/cancel_pipeline' +import { gitlabCreateBranchTool } from '@/tools/gitlab/create_branch' +import { gitlabCreateFileTool } from '@/tools/gitlab/create_file' import { gitlabCreateIssueTool } from '@/tools/gitlab/create_issue' import { gitlabCreateIssueNoteTool } from '@/tools/gitlab/create_issue_note' import { gitlabCreateMergeRequestTool } from '@/tools/gitlab/create_merge_request' import { gitlabCreateMergeRequestNoteTool } from '@/tools/gitlab/create_merge_request_note' import { gitlabCreatePipelineTool } from '@/tools/gitlab/create_pipeline' import { gitlabDeleteIssueTool } from '@/tools/gitlab/delete_issue' +import { gitlabGetFileTool } from '@/tools/gitlab/get_file' import { gitlabGetIssueTool } from '@/tools/gitlab/get_issue' +import { gitlabGetJobLogTool } from '@/tools/gitlab/get_job_log' import { gitlabGetMergeRequestTool } from '@/tools/gitlab/get_merge_request' +import { gitlabGetMergeRequestChangesTool } from '@/tools/gitlab/get_merge_request_changes' import { gitlabGetPipelineTool } from '@/tools/gitlab/get_pipeline' import { gitlabGetProjectTool } from '@/tools/gitlab/get_project' +import { gitlabListBranchesTool } from '@/tools/gitlab/list_branches' +import { gitlabListCommitsTool } from '@/tools/gitlab/list_commits' import { gitlabListIssuesTool } from '@/tools/gitlab/list_issues' import { gitlabListMergeRequestsTool } from '@/tools/gitlab/list_merge_requests' +import { gitlabListPipelineJobsTool } from '@/tools/gitlab/list_pipeline_jobs' import { gitlabListPipelinesTool } from '@/tools/gitlab/list_pipelines' import { gitlabListProjectsTool } from '@/tools/gitlab/list_projects' +import { gitlabListRepositoryTreeTool } from '@/tools/gitlab/list_repository_tree' import { gitlabMergeMergeRequestTool } from '@/tools/gitlab/merge_merge_request' +import { gitlabPlayJobTool } from '@/tools/gitlab/play_job' import { gitlabRetryPipelineTool } from '@/tools/gitlab/retry_pipeline' +import { gitlabUpdateFileTool } from '@/tools/gitlab/update_file' import { gitlabUpdateIssueTool } from '@/tools/gitlab/update_issue' import { gitlabUpdateMergeRequestTool } from '@/tools/gitlab/update_merge_request' @@ -36,10 +48,26 @@ export { gitlabUpdateMergeRequestTool, gitlabMergeMergeRequestTool, gitlabCreateMergeRequestNoteTool, + gitlabGetMergeRequestChangesTool, + gitlabApproveMergeRequestTool, // Pipelines gitlabListPipelinesTool, gitlabGetPipelineTool, gitlabCreatePipelineTool, gitlabRetryPipelineTool, gitlabCancelPipelineTool, + // Jobs + gitlabListPipelineJobsTool, + gitlabGetJobLogTool, + gitlabPlayJobTool, + // Repository Files & Tree + gitlabListRepositoryTreeTool, + gitlabGetFileTool, + gitlabCreateFileTool, + gitlabUpdateFileTool, + // Branches + gitlabListBranchesTool, + gitlabCreateBranchTool, + // Commits + gitlabListCommitsTool, } diff --git a/apps/sim/tools/gitlab/list_branches.ts b/apps/sim/tools/gitlab/list_branches.ts new file mode 100644 index 00000000000..2943c250285 --- /dev/null +++ b/apps/sim/tools/gitlab/list_branches.ts @@ -0,0 +1,103 @@ +import type { GitLabListBranchesParams, GitLabListBranchesResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabListBranchesTool: ToolConfig< + GitLabListBranchesParams, + GitLabListBranchesResponse +> = { + id: 'gitlab_list_branches', + name: 'GitLab List Branches', + description: 'List branches in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + search: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Filter branches by name', + }, + perPage: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Number of results per page (default 20, max 100)', + }, + page: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Page number for pagination', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const queryParams = new URLSearchParams() + + if (params.search) queryParams.append('search', params.search) + if (params.perPage) queryParams.append('per_page', String(params.perPage)) + if (params.page) queryParams.append('page', String(params.page)) + + const query = queryParams.toString() + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/branches${query ? `?${query}` : ''}` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const branches = await response.json() + const total = response.headers.get('x-total') + + return { + success: true, + output: { + branches: branches ?? [], + total: total ? Number.parseInt(total, 10) : (branches?.length ?? 0), + }, + } + }, + + outputs: { + branches: { + type: 'array', + description: 'List of branches', + }, + total: { + type: 'number', + description: 'Total number of branches', + }, + }, +} diff --git a/apps/sim/tools/gitlab/list_commits.ts b/apps/sim/tools/gitlab/list_commits.ts new file mode 100644 index 00000000000..b3be01049d2 --- /dev/null +++ b/apps/sim/tools/gitlab/list_commits.ts @@ -0,0 +1,129 @@ +import type { GitLabListCommitsParams, GitLabListCommitsResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabListCommitsTool: ToolConfig = + { + id: 'gitlab_list_commits', + name: 'GitLab List Commits', + description: 'List commits in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + refName: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Branch, tag, or revision range to list commits from', + }, + since: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Only commits after this ISO 8601 date', + }, + until: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Only commits before this ISO 8601 date', + }, + path: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Only commits affecting this file path', + }, + author: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Filter commits by author', + }, + perPage: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Number of results per page (default 20, max 100)', + }, + page: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Page number for pagination', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const queryParams = new URLSearchParams() + + if (params.refName) queryParams.append('ref_name', params.refName) + if (params.since) queryParams.append('since', params.since) + if (params.until) queryParams.append('until', params.until) + if (params.path) queryParams.append('path', params.path) + if (params.author) queryParams.append('author', params.author) + if (params.perPage) queryParams.append('per_page', String(params.perPage)) + if (params.page) queryParams.append('page', String(params.page)) + + const query = queryParams.toString() + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/commits${query ? `?${query}` : ''}` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const commits = await response.json() + const total = response.headers.get('x-total') + + return { + success: true, + output: { + commits: commits ?? [], + total: total ? Number.parseInt(total, 10) : (commits?.length ?? 0), + }, + } + }, + + outputs: { + commits: { + type: 'array', + description: 'List of commits', + }, + total: { + type: 'number', + description: 'Total number of commits', + }, + }, + } diff --git a/apps/sim/tools/gitlab/list_issues.ts b/apps/sim/tools/gitlab/list_issues.ts index 40a016f3b34..8a46a915314 100644 --- a/apps/sim/tools/gitlab/list_issues.ts +++ b/apps/sim/tools/gitlab/list_issues.ts @@ -85,7 +85,7 @@ export const gitlabListIssuesTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) const queryParams = new URLSearchParams() if (params.state) queryParams.append('state', params.state) diff --git a/apps/sim/tools/gitlab/list_merge_requests.ts b/apps/sim/tools/gitlab/list_merge_requests.ts index 2cdae3301c4..65ad7efe797 100644 --- a/apps/sim/tools/gitlab/list_merge_requests.ts +++ b/apps/sim/tools/gitlab/list_merge_requests.ts @@ -85,7 +85,7 @@ export const gitlabListMergeRequestsTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) const queryParams = new URLSearchParams() if (params.state) queryParams.append('state', params.state) diff --git a/apps/sim/tools/gitlab/list_pipeline_jobs.ts b/apps/sim/tools/gitlab/list_pipeline_jobs.ts new file mode 100644 index 00000000000..db63ccb8df6 --- /dev/null +++ b/apps/sim/tools/gitlab/list_pipeline_jobs.ts @@ -0,0 +1,119 @@ +import type { + GitLabListPipelineJobsParams, + GitLabListPipelineJobsResponse, +} from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabListPipelineJobsTool: ToolConfig< + GitLabListPipelineJobsParams, + GitLabListPipelineJobsResponse +> = { + id: 'gitlab_list_pipeline_jobs', + name: 'GitLab List Pipeline Jobs', + description: 'List jobs for a GitLab pipeline', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + pipelineId: { + type: 'number', + required: true, + visibility: 'user-or-llm', + description: 'Pipeline ID', + }, + scope: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Filter jobs by scope (e.g. created, running, success, failed)', + }, + includeRetried: { + type: 'boolean', + required: false, + visibility: 'user-or-llm', + description: 'Whether to include retried jobs', + }, + perPage: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Number of results per page (default 20, max 100)', + }, + page: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Page number for pagination', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const queryParams = new URLSearchParams() + + if (params.scope) queryParams.append('scope', params.scope) + if (params.includeRetried) queryParams.append('include_retried', 'true') + if (params.perPage) queryParams.append('per_page', String(params.perPage)) + if (params.page) queryParams.append('page', String(params.page)) + + const query = queryParams.toString() + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/jobs${query ? `?${query}` : ''}` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const jobs = await response.json() + const total = response.headers.get('x-total') + + return { + success: true, + output: { + jobs: jobs ?? [], + total: total ? Number.parseInt(total, 10) : (jobs?.length ?? 0), + }, + } + }, + + outputs: { + jobs: { + type: 'array', + description: 'List of pipeline jobs', + }, + total: { + type: 'number', + description: 'Total number of jobs', + }, + }, +} diff --git a/apps/sim/tools/gitlab/list_pipelines.ts b/apps/sim/tools/gitlab/list_pipelines.ts index 80294e85f73..bb21081e2ea 100644 --- a/apps/sim/tools/gitlab/list_pipelines.ts +++ b/apps/sim/tools/gitlab/list_pipelines.ts @@ -71,7 +71,7 @@ export const gitlabListPipelinesTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) const queryParams = new URLSearchParams() if (params.ref) queryParams.append('ref', params.ref) diff --git a/apps/sim/tools/gitlab/list_repository_tree.ts b/apps/sim/tools/gitlab/list_repository_tree.ts new file mode 100644 index 00000000000..efbc439b852 --- /dev/null +++ b/apps/sim/tools/gitlab/list_repository_tree.ts @@ -0,0 +1,120 @@ +import type { + GitLabListRepositoryTreeParams, + GitLabListRepositoryTreeResponse, +} from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabListRepositoryTreeTool: ToolConfig< + GitLabListRepositoryTreeParams, + GitLabListRepositoryTreeResponse +> = { + id: 'gitlab_list_repository_tree', + name: 'GitLab List Repository Tree', + description: 'List files and directories in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + path: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Path inside the repository to list', + }, + ref: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Branch, tag, or commit SHA to list from', + }, + recursive: { + type: 'boolean', + required: false, + visibility: 'user-or-llm', + description: 'Whether to list files recursively', + }, + perPage: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Number of results per page (default 20, max 100)', + }, + page: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Page number for pagination', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const queryParams = new URLSearchParams() + + if (params.path) queryParams.append('path', params.path) + if (params.ref) queryParams.append('ref', params.ref) + if (params.recursive) queryParams.append('recursive', 'true') + if (params.perPage) queryParams.append('per_page', String(params.perPage)) + if (params.page) queryParams.append('page', String(params.page)) + + const query = queryParams.toString() + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/tree${query ? `?${query}` : ''}` + }, + method: 'GET', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const tree = await response.json() + const total = response.headers.get('x-total') + + return { + success: true, + output: { + tree: tree ?? [], + total: total ? Number.parseInt(total, 10) : (tree?.length ?? 0), + }, + } + }, + + outputs: { + tree: { + type: 'array', + description: 'List of repository tree entries', + }, + total: { + type: 'number', + description: 'Total number of tree entries', + }, + }, +} diff --git a/apps/sim/tools/gitlab/merge_merge_request.ts b/apps/sim/tools/gitlab/merge_merge_request.ts index 500e6ebfd07..4991e67e209 100644 --- a/apps/sim/tools/gitlab/merge_merge_request.ts +++ b/apps/sim/tools/gitlab/merge_merge_request.ts @@ -73,7 +73,7 @@ export const gitlabMergeMergeRequestTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/merge` }, method: 'PUT', diff --git a/apps/sim/tools/gitlab/play_job.ts b/apps/sim/tools/gitlab/play_job.ts new file mode 100644 index 00000000000..bd8a37effca --- /dev/null +++ b/apps/sim/tools/gitlab/play_job.ts @@ -0,0 +1,90 @@ +import type { GitLabPlayJobParams, GitLabPlayJobResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabPlayJobTool: ToolConfig = { + id: 'gitlab_play_job', + name: 'GitLab Play Job', + description: 'Trigger (play) a manual GitLab job', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + jobId: { + type: 'number', + required: true, + visibility: 'user-or-llm', + description: 'Job ID', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/jobs/${params.jobId}/play` + }, + method: 'POST', + headers: (params) => ({ + 'PRIVATE-TOKEN': params.accessToken, + }), + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + + return { + success: true, + output: { + id: data.id ?? null, + name: data.name ?? null, + status: data.status ?? null, + webUrl: data.web_url ?? null, + }, + } + }, + + outputs: { + id: { + type: 'number', + description: 'The job ID', + }, + name: { + type: 'string', + description: 'The job name', + }, + status: { + type: 'string', + description: 'The job status', + }, + webUrl: { + type: 'string', + description: 'The web URL of the job', + }, + }, +} diff --git a/apps/sim/tools/gitlab/retry_pipeline.ts b/apps/sim/tools/gitlab/retry_pipeline.ts index 48143109c97..470ade6d9f0 100644 --- a/apps/sim/tools/gitlab/retry_pipeline.ts +++ b/apps/sim/tools/gitlab/retry_pipeline.ts @@ -40,7 +40,7 @@ export const gitlabRetryPipelineTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/retry` }, method: 'POST', diff --git a/apps/sim/tools/gitlab/types.ts b/apps/sim/tools/gitlab/types.ts index af865ed3ef8..d87a10d67b6 100644 --- a/apps/sim/tools/gitlab/types.ts +++ b/apps/sim/tools/gitlab/types.ts @@ -1,7 +1,5 @@ import type { ToolResponse } from '@/tools/types' -// ===== Core Types ===== - interface GitLabProject { id: number name: string @@ -124,6 +122,46 @@ interface GitLabPipeline { } } +interface GitLabTreeEntry { + id: string + name: string + type: string + path: string + mode: string +} + +interface GitLabCommit { + id: string + short_id: string + title: string + message: string + author_name: string + authored_date: string + created_at: string + web_url: string +} + +interface GitLabJob { + id: number + name: string + stage: string + status: string + started_at?: string | null + finished_at?: string | null + duration?: number | null + web_url: string + ref?: string +} + +interface GitLabMergeRequestChange { + old_path: string + new_path: string + diff: string + new_file: boolean + deleted_file: boolean + renamed_file: boolean +} + interface GitLabBranch { name: string merged: boolean @@ -190,8 +228,6 @@ interface GitLabMilestone { web_url: string } -// ===== Common Parameters ===== - interface GitLabBaseParams { accessToken: string /** @@ -201,8 +237,6 @@ interface GitLabBaseParams { host?: string } -// ===== Project Parameters ===== - export interface GitLabListProjectsParams extends GitLabBaseParams { owned?: boolean membership?: boolean @@ -218,8 +252,6 @@ export interface GitLabGetProjectParams extends GitLabBaseParams { projectId: string | number } -// ===== Issue Parameters ===== - export interface GitLabListIssuesParams extends GitLabBaseParams { projectId: string | number state?: 'opened' | 'closed' | 'all' @@ -267,8 +299,6 @@ export interface GitLabDeleteIssueParams extends GitLabBaseParams { issueIid: number } -// ===== Merge Request Parameters ===== - export interface GitLabListMergeRequestsParams extends GitLabBaseParams { projectId: string | number state?: 'opened' | 'closed' | 'merged' | 'all' @@ -325,8 +355,6 @@ export interface GitLabMergeMergeRequestParams extends GitLabBaseParams { mergeWhenPipelineSucceeds?: boolean } -// ===== Pipeline Parameters ===== - export interface GitLabListPipelinesParams extends GitLabBaseParams { projectId: string | number ref?: string @@ -369,9 +397,7 @@ export interface GitLabCancelPipelineParams extends GitLabBaseParams { pipelineId: number } -// ===== Branch Parameters ===== - -interface GitLabListBranchesParams extends GitLabBaseParams { +export interface GitLabListBranchesParams extends GitLabBaseParams { projectId: string | number search?: string perPage?: number @@ -383,7 +409,7 @@ interface GitLabGetBranchParams extends GitLabBaseParams { branch: string } -interface GitLabCreateBranchParams extends GitLabBaseParams { +export interface GitLabCreateBranchParams extends GitLabBaseParams { projectId: string | number branch: string ref: string @@ -394,7 +420,78 @@ interface GitLabDeleteBranchParams extends GitLabBaseParams { branch: string } -// ===== Note/Comment Parameters ===== +export interface GitLabListRepositoryTreeParams extends GitLabBaseParams { + projectId: string | number + path?: string + ref?: string + recursive?: boolean + perPage?: number + page?: number +} + +export interface GitLabGetFileParams extends GitLabBaseParams { + projectId: string | number + filePath: string + ref: string +} + +export interface GitLabCreateFileParams extends GitLabBaseParams { + projectId: string | number + filePath: string + branch: string + content: string + commitMessage: string +} + +export interface GitLabUpdateFileParams extends GitLabBaseParams { + projectId: string | number + filePath: string + branch: string + content: string + commitMessage: string + lastCommitId?: string +} + +export interface GitLabListCommitsParams extends GitLabBaseParams { + projectId: string | number + refName?: string + since?: string + until?: string + path?: string + author?: string + perPage?: number + page?: number +} + +export interface GitLabGetMergeRequestChangesParams extends GitLabBaseParams { + projectId: string | number + mergeRequestIid: number +} + +export interface GitLabApproveMergeRequestParams extends GitLabBaseParams { + projectId: string | number + mergeRequestIid: number + sha?: string +} + +export interface GitLabListPipelineJobsParams extends GitLabBaseParams { + projectId: string | number + pipelineId: number + scope?: string + includeRetried?: boolean + perPage?: number + page?: number +} + +export interface GitLabGetJobLogParams extends GitLabBaseParams { + projectId: string | number + jobId: number +} + +export interface GitLabPlayJobParams extends GitLabBaseParams { + projectId: string | number + jobId: number +} interface GitLabListIssueNotesParams extends GitLabBaseParams { projectId: string | number @@ -426,8 +523,6 @@ export interface GitLabCreateMergeRequestNoteParams extends GitLabBaseParams { body: string } -// ===== Label Parameters ===== - interface GitLabListLabelsParams extends GitLabBaseParams { projectId: string | number search?: string @@ -442,8 +537,6 @@ interface GitLabCreateLabelParams extends GitLabBaseParams { description?: string } -// ===== User Parameters ===== - interface GitLabGetCurrentUserParams extends GitLabBaseParams {} interface GitLabListUsersParams extends GitLabBaseParams { @@ -452,8 +545,6 @@ interface GitLabListUsersParams extends GitLabBaseParams { page?: number } -// ===== Response Types ===== - export interface GitLabListProjectsResponse extends ToolResponse { output: { projects?: GitLabProject[] @@ -560,7 +651,7 @@ export interface GitLabCancelPipelineResponse extends ToolResponse { } } -interface GitLabListBranchesResponse extends ToolResponse { +export interface GitLabListBranchesResponse extends ToolResponse { output: { branches?: GitLabBranch[] total?: number @@ -573,9 +664,12 @@ interface GitLabGetBranchResponse extends ToolResponse { } } -interface GitLabCreateBranchResponse extends ToolResponse { +export interface GitLabCreateBranchResponse extends ToolResponse { output: { - branch?: GitLabBranch + name?: string | null + webUrl?: string | null + protected?: boolean | null + commit?: GitLabBranch['commit'] | null } } @@ -624,7 +718,83 @@ interface GitLabListUsersResponse extends ToolResponse { } } -// ===== Union Response Type ===== +export interface GitLabListRepositoryTreeResponse extends ToolResponse { + output: { + tree?: GitLabTreeEntry[] + total?: number + } +} + +export interface GitLabGetFileResponse extends ToolResponse { + output: { + filePath?: string | null + fileName?: string | null + size?: number | null + ref?: string | null + blobId?: string | null + lastCommitId?: string | null + content?: string + } +} + +export interface GitLabCreateFileResponse extends ToolResponse { + output: { + filePath?: string | null + branch?: string | null + } +} + +export interface GitLabUpdateFileResponse extends ToolResponse { + output: { + filePath?: string | null + branch?: string | null + } +} + +export interface GitLabListCommitsResponse extends ToolResponse { + output: { + commits?: GitLabCommit[] + total?: number + } +} + +export interface GitLabGetMergeRequestChangesResponse extends ToolResponse { + output: { + mergeRequestIid?: number | null + changes?: GitLabMergeRequestChange[] + changesCount?: number + } +} + +export interface GitLabApproveMergeRequestResponse extends ToolResponse { + output: { + approvalsRequired?: number | null + approvalsLeft?: number | null + approvedBy?: unknown[] + } +} + +export interface GitLabListPipelineJobsResponse extends ToolResponse { + output: { + jobs?: GitLabJob[] + total?: number + } +} + +export interface GitLabGetJobLogResponse extends ToolResponse { + output: { + log?: string + } +} + +export interface GitLabPlayJobResponse extends ToolResponse { + output: { + id?: number | null + name?: string | null + status?: string | null + webUrl?: string | null + } +} export type GitLabResponse = | GitLabListProjectsResponse @@ -654,3 +824,13 @@ export type GitLabResponse = | GitLabCreateLabelResponse | GitLabGetCurrentUserResponse | GitLabListUsersResponse + | GitLabListRepositoryTreeResponse + | GitLabGetFileResponse + | GitLabCreateFileResponse + | GitLabUpdateFileResponse + | GitLabListCommitsResponse + | GitLabGetMergeRequestChangesResponse + | GitLabApproveMergeRequestResponse + | GitLabListPipelineJobsResponse + | GitLabGetJobLogResponse + | GitLabPlayJobResponse diff --git a/apps/sim/tools/gitlab/update_file.ts b/apps/sim/tools/gitlab/update_file.ts new file mode 100644 index 00000000000..38a7beece3d --- /dev/null +++ b/apps/sim/tools/gitlab/update_file.ts @@ -0,0 +1,118 @@ +import type { GitLabUpdateFileParams, GitLabUpdateFileResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' +import type { ToolConfig } from '@/tools/types' + +export const gitlabUpdateFileTool: ToolConfig = { + id: 'gitlab_update_file', + name: 'GitLab Update File', + description: 'Update an existing file in a GitLab project repository', + version: '1.0.0', + + params: { + accessToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'GitLab Personal Access Token', + }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, + projectId: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Project ID or URL-encoded path', + }, + filePath: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Path to the file in the repository', + }, + branch: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Branch to commit the update to', + }, + content: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'New file content', + }, + commitMessage: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Commit message', + }, + lastCommitId: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Last known commit ID for the file (optimistic locking)', + }, + }, + + request: { + url: (params) => { + const encodedId = encodeURIComponent(String(params.projectId).trim()) + const encodedPath = encodeURIComponent(String(params.filePath)) + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/repository/files/${encodedPath}` + }, + method: 'PUT', + headers: (params) => ({ + 'Content-Type': 'application/json', + 'PRIVATE-TOKEN': params.accessToken, + }), + body: (params) => { + const body: Record = { + branch: params.branch, + content: params.content, + commit_message: params.commitMessage, + encoding: 'text', + } + + if (params.lastCommitId) body.last_commit_id = params.lastCommitId + + return body + }, + }, + + transformResponse: async (response) => { + if (!response.ok) { + const errorText = await response.text() + return { + success: false, + error: `GitLab API error: ${response.status} ${errorText}`, + output: {}, + } + } + + const data = await response.json() + + return { + success: true, + output: { + filePath: data.file_path ?? null, + branch: data.branch ?? null, + }, + } + }, + + outputs: { + filePath: { + type: 'string', + description: 'The updated file path', + }, + branch: { + type: 'string', + description: 'The branch the update was committed to', + }, + }, +} diff --git a/apps/sim/tools/gitlab/update_issue.ts b/apps/sim/tools/gitlab/update_issue.ts index acf7ca25402..b2e8b8503aa 100644 --- a/apps/sim/tools/gitlab/update_issue.ts +++ b/apps/sim/tools/gitlab/update_issue.ts @@ -86,7 +86,7 @@ export const gitlabUpdateIssueTool: ToolConfig { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'PUT', diff --git a/apps/sim/tools/gitlab/update_merge_request.ts b/apps/sim/tools/gitlab/update_merge_request.ts index 69632a637d5..a0930567db9 100644 --- a/apps/sim/tools/gitlab/update_merge_request.ts +++ b/apps/sim/tools/gitlab/update_merge_request.ts @@ -103,7 +103,7 @@ export const gitlabUpdateMergeRequestTool: ToolConfig< request: { url: (params) => { - const encodedId = encodeURIComponent(String(params.projectId)) + const encodedId = encodeURIComponent(String(params.projectId).trim()) return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` }, method: 'PUT', diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index df8a070ec5c..7f8f537c98d 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -1082,23 +1082,35 @@ import { githubUpdateReleaseV2Tool, } from '@/tools/github' import { + gitlabApproveMergeRequestTool, gitlabCancelPipelineTool, + gitlabCreateBranchTool, + gitlabCreateFileTool, gitlabCreateIssueNoteTool, gitlabCreateIssueTool, gitlabCreateMergeRequestNoteTool, gitlabCreateMergeRequestTool, gitlabCreatePipelineTool, gitlabDeleteIssueTool, + gitlabGetFileTool, gitlabGetIssueTool, + gitlabGetJobLogTool, + gitlabGetMergeRequestChangesTool, gitlabGetMergeRequestTool, gitlabGetPipelineTool, gitlabGetProjectTool, + gitlabListBranchesTool, + gitlabListCommitsTool, gitlabListIssuesTool, gitlabListMergeRequestsTool, + gitlabListPipelineJobsTool, gitlabListPipelinesTool, gitlabListProjectsTool, + gitlabListRepositoryTreeTool, gitlabMergeMergeRequestTool, + gitlabPlayJobTool, gitlabRetryPipelineTool, + gitlabUpdateFileTool, gitlabUpdateIssueTool, gitlabUpdateMergeRequestTool, } from '@/tools/gitlab' @@ -5477,25 +5489,37 @@ export const tools: Record = { github_check_star_v2: githubCheckStarV2Tool, github_list_stargazers: githubListStargazersTool, github_list_stargazers_v2: githubListStargazersV2Tool, - gitlab_list_projects: gitlabListProjectsTool, - gitlab_get_project: gitlabGetProjectTool, - gitlab_list_issues: gitlabListIssuesTool, - gitlab_get_issue: gitlabGetIssueTool, + gitlab_approve_merge_request: gitlabApproveMergeRequestTool, + gitlab_cancel_pipeline: gitlabCancelPipelineTool, + gitlab_create_branch: gitlabCreateBranchTool, + gitlab_create_file: gitlabCreateFileTool, gitlab_create_issue: gitlabCreateIssueTool, - gitlab_update_issue: gitlabUpdateIssueTool, - gitlab_delete_issue: gitlabDeleteIssueTool, gitlab_create_issue_note: gitlabCreateIssueNoteTool, - gitlab_list_merge_requests: gitlabListMergeRequestsTool, - gitlab_get_merge_request: gitlabGetMergeRequestTool, gitlab_create_merge_request: gitlabCreateMergeRequestTool, - gitlab_update_merge_request: gitlabUpdateMergeRequestTool, - gitlab_merge_merge_request: gitlabMergeMergeRequestTool, gitlab_create_merge_request_note: gitlabCreateMergeRequestNoteTool, - gitlab_list_pipelines: gitlabListPipelinesTool, - gitlab_get_pipeline: gitlabGetPipelineTool, gitlab_create_pipeline: gitlabCreatePipelineTool, + gitlab_delete_issue: gitlabDeleteIssueTool, + gitlab_get_file: gitlabGetFileTool, + gitlab_get_issue: gitlabGetIssueTool, + gitlab_get_job_log: gitlabGetJobLogTool, + gitlab_get_merge_request: gitlabGetMergeRequestTool, + gitlab_get_merge_request_changes: gitlabGetMergeRequestChangesTool, + gitlab_get_pipeline: gitlabGetPipelineTool, + gitlab_get_project: gitlabGetProjectTool, + gitlab_list_branches: gitlabListBranchesTool, + gitlab_list_commits: gitlabListCommitsTool, + gitlab_list_issues: gitlabListIssuesTool, + gitlab_list_merge_requests: gitlabListMergeRequestsTool, + gitlab_list_pipeline_jobs: gitlabListPipelineJobsTool, + gitlab_list_pipelines: gitlabListPipelinesTool, + gitlab_list_projects: gitlabListProjectsTool, + gitlab_list_repository_tree: gitlabListRepositoryTreeTool, + gitlab_merge_merge_request: gitlabMergeMergeRequestTool, + gitlab_play_job: gitlabPlayJobTool, gitlab_retry_pipeline: gitlabRetryPipelineTool, - gitlab_cancel_pipeline: gitlabCancelPipelineTool, + gitlab_update_file: gitlabUpdateFileTool, + gitlab_update_issue: gitlabUpdateIssueTool, + gitlab_update_merge_request: gitlabUpdateMergeRequestTool, grain_list_recordings: grainListRecordingsTool, grain_get_recording: grainGetRecordingTool, grain_get_transcript: grainGetTranscriptTool, From cff7a4931008a893c270dcf7fa5a21f654361153 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 24 Jun 2026 18:07:39 -0700 Subject: [PATCH 11/14] feat(file): workspace-scoped inline images + public-share cascade (#5203) * feat(file): workspace-scoped inline images + public-share cascade Embedded markdown images now resolve only within the document's workspace, and public file shares cascade to the images the shared document embeds. - New /api/workspaces/[id]/files/inline (in-app, workspace-scoped) and /api/files/public/[token]/inline (public cascade) routes; the public one serves an embed only when it is referenced-by-doc, same-workspace, and passes a magic-byte image sniff - Embed srcs (serve-key and view-id forms) rewrite through one scoped inline route; one shared isomorphic parser owns the embed grammar for both the frontend renderer and the server doc scan - Accept wf_ file ids on the view/export routes (were 400ing on .uuid()) * feat(file): add Image command to the markdown editor slash menu - New /Image slash command uploads an image via a file picker and inserts it at the caret (same upload+insert path as paste/drop) - Inserted src is the workspace serve URL, so it renders in-app and cascades to public shares like any other embed - Per-editor handler wired through slash-command storage (the extension set is a shared singleton); only active when the editor is editable * fix(file): export rewrites all embed forms; cap embedded refs combined Addresses PR review: - Markdown export now rewrites the in-app `/workspace//files/` embed form too (not just `/api/files/view/`), so a bundled asset never leaves a broken link in an offline export (Bugbot) - extractEmbeddedFileRefs bounds total references (keys + ids) to 50 combined rather than 50 each, matching MAX_EMBEDDED_IMAGES intent --- apps/sim/app/api/files/export/[id]/route.ts | 18 ++- .../files/public/[token]/inline/route.test.ts | 116 ++++++++++++++++++ .../api/files/public/[token]/inline/route.ts | 99 +++++++++++++++ apps/sim/app/api/files/serve-inline-image.ts | 44 +++++++ .../[id]/files/inline/route.test.ts | 77 ++++++++++++ .../api/workspaces/[id]/files/inline/route.ts | 59 +++++++++ apps/sim/app/f/[token]/public-file-view.tsx | 19 ++- .../components/file-viewer/file-viewer.tsx | 26 +++- .../rich-markdown-editor/image.test.ts | 45 ++++--- .../rich-markdown-editor/image.tsx | 22 +--- .../rich-markdown-editor.tsx | 34 +++++ .../slash-command/commands.test.ts | 36 +++++- .../slash-command/commands.ts | 22 ++++ .../slash-command/slash-command.ts | 19 ++- apps/sim/hooks/use-file-content-source.tsx | 77 ++++++++++-- apps/sim/lib/api/contracts/primitives.ts | 27 ++++ apps/sim/lib/api/contracts/public-shares.ts | 18 ++- .../sim/lib/api/contracts/storage-transfer.ts | 5 +- apps/sim/lib/api/contracts/workspace-files.ts | 16 +++ .../server/files/embedded-image-refs.test.ts | 43 +++++++ .../tools/server/files/embedded-image-refs.ts | 20 ++- .../lib/uploads/server/inline-image.test.ts | 63 ++++++++++ apps/sim/lib/uploads/server/inline-image.ts | 41 +++++++ .../uploads/utils/embedded-image-ref.test.ts | 60 +++++++++ .../lib/uploads/utils/embedded-image-ref.ts | 73 +++++++++++ apps/sim/lib/uploads/utils/validation.test.ts | 31 +++++ apps/sim/lib/uploads/utils/validation.ts | 28 +++++ scripts/check-api-validation-contracts.ts | 4 +- 28 files changed, 1073 insertions(+), 69 deletions(-) create mode 100644 apps/sim/app/api/files/public/[token]/inline/route.test.ts create mode 100644 apps/sim/app/api/files/public/[token]/inline/route.ts create mode 100644 apps/sim/app/api/files/serve-inline-image.ts create mode 100644 apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts create mode 100644 apps/sim/app/api/workspaces/[id]/files/inline/route.ts create mode 100644 apps/sim/lib/copilot/tools/server/files/embedded-image-refs.test.ts create mode 100644 apps/sim/lib/uploads/server/inline-image.test.ts create mode 100644 apps/sim/lib/uploads/server/inline-image.ts create mode 100644 apps/sim/lib/uploads/utils/embedded-image-ref.test.ts create mode 100644 apps/sim/lib/uploads/utils/embedded-image-ref.ts diff --git a/apps/sim/app/api/files/export/[id]/route.ts b/apps/sim/app/api/files/export/[id]/route.ts index 18c8aafb563..26dc06abe0d 100644 --- a/apps/sim/app/api/files/export/[id]/route.ts +++ b/apps/sim/app/api/files/export/[id]/route.ts @@ -7,6 +7,7 @@ import { NextResponse } from 'next/server' import { fileExportContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { extractEmbeddedImageIds } from '@/lib/copilot/tools/server/files/embedded-image-refs' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { StorageContext } from '@/lib/uploads/config' import { USE_BLOB_STORAGE } from '@/lib/uploads/config' @@ -19,9 +20,6 @@ const logger = createLogger('FilesExportAPI') const MARKDOWN_MIME_TYPES = new Set(['text/markdown', 'text/x-markdown']) const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown']) -const VIEW_URL_RE = - /\/api\/files\/view\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi -const MAX_EMBEDDED_IMAGES = 50 function isMarkdown(originalName: string, contentType: string): boolean { if (MARKDOWN_MIME_TYPES.has(contentType)) return true @@ -82,10 +80,7 @@ export const GET = withRouteHandler( }) let mdContent = mdBuffer.toString('utf-8') - const imageIds = [...new Set([...mdContent.matchAll(VIEW_URL_RE)].map((m) => m[1]))].slice( - 0, - MAX_EMBEDDED_IMAGES - ) + const imageIds = extractEmbeddedImageIds(mdContent) logger.info('Exporting markdown', { id, imageCount: imageIds.length }) @@ -139,10 +134,11 @@ export const GET = withRouteHandler( for (const [imageId, asset] of assetMap) { const escapedId = imageId.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') const replacement = `./assets/${asset.filename}` - mdContent = mdContent.replace( - new RegExp(`/api/files/view/${escapedId}`, 'g'), - () => replacement - ) + // Rewrite both embed spellings the extractor resolves to this id — the view URL and the in-app + // `/workspace//files/` path — so a bundled asset never leaves a broken link in the export. + mdContent = mdContent + .replace(new RegExp(`/api/files/view/${escapedId}`, 'g'), () => replacement) + .replace(new RegExp(`/workspace/[A-Za-z0-9-]+/files/${escapedId}`, 'g'), () => replacement) } const zip = new JSZip() diff --git a/apps/sim/app/api/files/public/[token]/inline/route.test.ts b/apps/sim/app/api/files/public/[token]/inline/route.test.ts new file mode 100644 index 00000000000..3f2b654bda0 --- /dev/null +++ b/apps/sim/app/api/files/public/[token]/inline/route.test.ts @@ -0,0 +1,116 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockResolveShare, mockRateLimit, mockValidateAuth, mockDownloadFile, mockResolveImage } = + vi.hoisted(() => ({ + mockResolveShare: vi.fn(), + mockRateLimit: vi.fn(), + mockValidateAuth: vi.fn(), + mockDownloadFile: vi.fn(), + mockResolveImage: vi.fn(), + })) + +vi.mock('@/lib/public-shares/share-manager', () => ({ + resolveActiveShareByToken: mockResolveShare, +})) +vi.mock('@/lib/public-shares/rate-limit', () => ({ enforcePublicFileRateLimit: mockRateLimit })) +vi.mock('@/lib/core/security/deployment-auth', () => ({ validateDeploymentAuth: mockValidateAuth })) +vi.mock('@/lib/uploads/core/storage-service', () => ({ downloadFile: mockDownloadFile })) +vi.mock('@/lib/uploads/server/inline-image', () => ({ + resolveWorkspaceInlineImage: mockResolveImage, +})) + +import { GET } from '@/app/api/files/public/[token]/inline/route' + +const TOKEN = 'tok_share_123456' +const DOC_KEY = 'workspace/ws-1/doc.md' +const IMG_KEY = 'workspace/ws-1/photo.png' +const FILE_ID = 'wf_YwDXi8eWOkTxn0sbgChlB' +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]) + +const params = { params: Promise.resolve({ token: TOKEN }) } +const req = (q: string) => new NextRequest(`http://localhost/api/files/public/${TOKEN}/inline?${q}`) + +const share = { + share: { id: 'sh_1', token: TOKEN, authType: 'public' }, + file: { id: 'wf_doc', key: DOC_KEY, workspaceId: 'ws-1', originalName: 'doc.md' }, + workspaceName: 'Acme', + ownerName: 'Jane', +} + +/** doc bytes embed the image via the view form; image bytes are a real PNG */ +function downloadByKey(docContent = `![a](/api/files/view/${FILE_ID})`) { + return ({ key }: { key: string }) => + Promise.resolve(key === DOC_KEY ? Buffer.from(docContent, 'utf-8') : PNG) +} + +describe('GET /api/files/public/[token]/inline', () => { + beforeEach(() => { + vi.clearAllMocks() + mockRateLimit.mockResolvedValue(null) + mockResolveShare.mockResolvedValue(share) + mockValidateAuth.mockResolvedValue({ authorized: true }) + mockResolveImage.mockResolvedValue({ + key: IMG_KEY, + contentType: 'image/png', + filename: 'photo.png', + }) + mockDownloadFile.mockImplementation(downloadByKey()) + }) + + it('serves a same-workspace image referenced by the doc, typed from its bytes', async () => { + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(200) + expect(res.headers.get('content-type')).toBe('image/png') + }) + + it('serves a key-referenced image', async () => { + mockDownloadFile.mockImplementation( + downloadByKey(`![a](/api/files/serve/${encodeURIComponent(IMG_KEY)}?context=workspace)`) + ) + const res = await GET(req(`key=${encodeURIComponent(IMG_KEY)}`), params) + expect(res.status).toBe(200) + }) + + it('404s when the reference is not embedded in the shared document', async () => { + mockDownloadFile.mockImplementation(downloadByKey('no images here')) + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(404) + expect(mockResolveImage).not.toHaveBeenCalled() + }) + + it('404s when the referenced file is not in the document workspace', async () => { + mockResolveImage.mockResolvedValue(null) + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(404) + }) + + it('404s when the bytes are not a renderable image', async () => { + mockDownloadFile.mockImplementation(({ key }: { key: string }) => + Promise.resolve( + key === DOC_KEY + ? Buffer.from(`![a](/api/files/view/${FILE_ID})`, 'utf-8') + : Buffer.from('', 'utf-8') + ) + ) + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(404) + }) + + it('401s and never reads storage when the share is unauthorized', async () => { + mockValidateAuth.mockResolvedValue({ authorized: false, error: 'auth_required_password' }) + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(401) + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('404s for an unknown or inactive token', async () => { + mockResolveShare.mockResolvedValue(null) + const res = await GET(req(`fileId=${FILE_ID}`), params) + expect(res.status).toBe(404) + expect(mockDownloadFile).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/files/public/[token]/inline/route.ts b/apps/sim/app/api/files/public/[token]/inline/route.ts new file mode 100644 index 00000000000..87c343a26a8 --- /dev/null +++ b/apps/sim/app/api/files/public/[token]/inline/route.ts @@ -0,0 +1,99 @@ +import { createLogger } from '@sim/logger' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { getPublicInlineFileContract } from '@/lib/api/contracts/public-shares' +import { parseRequest } from '@/lib/api/server' +import { + extractEmbeddedImageIds, + extractEmbeddedImageKeys, +} from '@/lib/copilot/tools/server/files/embedded-image-refs' +import { validateDeploymentAuth } from '@/lib/core/security/deployment-auth' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { enforcePublicFileRateLimit } from '@/lib/public-shares/rate-limit' +import { resolveActiveShareByToken } from '@/lib/public-shares/share-manager' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import { resolveWorkspaceInlineImage } from '@/lib/uploads/server/inline-image' +import { serveInlineImage } from '@/app/api/files/serve-inline-image' +import { createErrorResponse, FileNotFoundError } from '@/app/api/files/utils' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('PublicInlineFileAPI') + +/** + * GET /api/files/public/[token]/inline?key=|fileId= + * + * Cascades a markdown document's public share to the images it embeds, so a logged-out viewer sees them + * instead of broken icons. The share grants the document bytes; this route extends that grant to the + * document's referenced images only, behind three gates that together hold the security boundary: + * + * 1. Referenced-by-doc — the requested key/id must appear in the shared document's current bytes. The + * token is a capability for the document and its embeds, never an arbitrary workspace file. + * 2. Same-workspace — the referenced file must be a `workspace` file in the document's own workspace + * ({@link resolveWorkspaceInlineImage}). This blocks any cross-workspace reference (which an author + * can write but must never resolve) from loading. + * 3. Content-truth — the served content type is sniffed from the bytes, not the client-declared type, + * and only genuine raster images are served. A file spoofing `image/png` while holding HTML/SVG is + * refused rather than rendered inline. + */ +export const GET = withRouteHandler( + async (request: NextRequest, context: { params: Promise<{ token: string }> }) => { + const requestId = generateRequestId() + + try { + const limited = await enforcePublicFileRateLimit(request, 'content') + if (limited) return limited + + const parsed = await parseRequest(getPublicInlineFileContract, request, context) + if (!parsed.success) return parsed.response + const { token } = parsed.data.params + const ref = parsed.data.query + + const resolved = await resolveActiveShareByToken(token) + if (!resolved) { + throw new FileNotFoundError('Not found') + } + + const auth = await validateDeploymentAuth( + requestId, + resolved.share, + request, + undefined, + 'file' + ) + if (!auth.authorized) { + return NextResponse.json({ error: auth.error ?? 'auth_required_password' }, { status: 401 }) + } + + const { file: doc } = resolved + if (!doc.workspaceId) { + throw new FileNotFoundError('Not found') + } + + // Referenced-by-doc gate: the share grants exactly the images the document embeds. + const docText = (await downloadFile({ key: doc.key, context: 'workspace' })).toString('utf-8') + const referenced = ref.fileId + ? extractEmbeddedImageIds(docText).includes(ref.fileId) + : extractEmbeddedImageKeys(docText).includes(ref.key as string) + if (!referenced) { + throw new FileNotFoundError('Not found') + } + + // Same-workspace gate: resolve scoped to the document's own workspace. + const image = await resolveWorkspaceInlineImage(doc.workspaceId, ref) + if (!image) { + throw new FileNotFoundError('Not found') + } + + // Content-truth gate (`sniff`): render only genuine raster image bytes. + return await serveInlineImage(image, { sniff: true }) + } catch (error) { + if (error instanceof FileNotFoundError) { + return createErrorResponse(error) + } + logger.error('Error serving public inline image:', error) + return createErrorResponse(error instanceof Error ? error : new Error('Failed to serve file')) + } + } +) diff --git a/apps/sim/app/api/files/serve-inline-image.ts b/apps/sim/app/api/files/serve-inline-image.ts new file mode 100644 index 00000000000..88c3383d961 --- /dev/null +++ b/apps/sim/app/api/files/serve-inline-image.ts @@ -0,0 +1,44 @@ +import { createLogger } from '@sim/logger' +import type { NextResponse } from 'next/server' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import type { ResolvedInlineImage } from '@/lib/uploads/server/inline-image' +import { sniffImageContentType } from '@/lib/uploads/utils/validation' +import { createFileResponse, FileNotFoundError } from '@/app/api/files/utils' + +const logger = createLogger('InlineImageServe') + +/** + * A shared/edited/deleted file must never serve stale bytes from its fixed inline URL, so every inline + * image revalidates on each request. + */ +const INLINE_CACHE_CONTROL = 'private, no-cache, must-revalidate' + +/** + * Download and respond with an already-workspace-scoped inline image — the single serving tail for both + * the in-app and public inline routes. When `sniff` is set (public shares, a less-trusted audience), the + * served content type is derived from the bytes and non-raster content is refused with 404; otherwise the + * stored content type is served, matching the in-app serve route. + */ +export async function serveInlineImage( + image: ResolvedInlineImage, + { sniff }: { sniff: boolean } +): Promise { + const buffer = await downloadFile({ key: image.key, context: 'workspace' }) + + let contentType = image.contentType + if (sniff) { + const sniffed = sniffImageContentType(buffer) + if (!sniffed) { + logger.warn('Embedded reference is not a renderable image', { key: image.key }) + throw new FileNotFoundError('Not found') + } + contentType = sniffed + } + + return createFileResponse({ + buffer, + contentType, + filename: image.filename, + cacheControl: INLINE_CACHE_CONTROL, + }) +} diff --git a/apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts new file mode 100644 index 00000000000..3bb2a8a06ba --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts @@ -0,0 +1,77 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetSession, mockGetPerms, mockResolveImage, mockDownloadFile } = vi.hoisted(() => ({ + mockGetSession: vi.fn(), + mockGetPerms: vi.fn(), + mockResolveImage: vi.fn(), + mockDownloadFile: vi.fn(), +})) + +vi.mock('@/lib/auth', () => ({ + auth: { api: { getSession: vi.fn() } }, + getSession: mockGetSession, +})) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ getUserEntityPermissions: mockGetPerms })) +vi.mock('@/lib/uploads/server/inline-image', () => ({ + resolveWorkspaceInlineImage: mockResolveImage, +})) +vi.mock('@/lib/uploads/core/storage-service', () => ({ downloadFile: mockDownloadFile })) + +import { GET } from '@/app/api/workspaces/[id]/files/inline/route' + +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]) +const params = { params: Promise.resolve({ id: 'ws-1' }) } +const req = (q: string) => new NextRequest(`http://localhost/api/workspaces/ws-1/files/inline?${q}`) + +describe('GET /api/workspaces/[id]/files/inline', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetSession.mockResolvedValue({ user: { id: 'u1' } }) + mockGetPerms.mockResolvedValue('read') + mockResolveImage.mockResolvedValue({ + key: 'workspace/ws-1/x-photo.png', + contentType: 'image/png', + filename: 'photo.png', + }) + mockDownloadFile.mockResolvedValue(PNG) + }) + + it('serves a workspace-scoped image by fileId', async () => { + const res = await GET(req('fileId=wf_abc'), params) + expect(res.status).toBe(200) + expect(mockResolveImage).toHaveBeenCalledWith('ws-1', { fileId: 'wf_abc' }) + }) + + it('serves a workspace-scoped image by key', async () => { + const res = await GET(req(`key=${encodeURIComponent('workspace/ws-1/x-photo.png')}`), params) + expect(res.status).toBe(200) + }) + + it('404s when the reference does not resolve in the workspace (cross-workspace)', async () => { + mockResolveImage.mockResolvedValue(null) + const res = await GET(req('fileId=wf_other'), params) + expect(res.status).toBe(404) + }) + + it('404s without workspace membership, before resolving the file', async () => { + mockGetPerms.mockResolvedValue(null) + const res = await GET(req('fileId=wf_abc'), params) + expect(res.status).toBe(404) + expect(mockResolveImage).not.toHaveBeenCalled() + }) + + it('401s without a session', async () => { + mockGetSession.mockResolvedValue(null) + const res = await GET(req('fileId=wf_abc'), params) + expect(res.status).toBe(401) + }) + + it('400s when neither key nor fileId is provided', async () => { + const res = await GET(req(''), params) + expect(res.status).toBe(400) + }) +}) diff --git a/apps/sim/app/api/workspaces/[id]/files/inline/route.ts b/apps/sim/app/api/workspaces/[id]/files/inline/route.ts new file mode 100644 index 00000000000..245fb5731d8 --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/files/inline/route.ts @@ -0,0 +1,59 @@ +import { createLogger } from '@sim/logger' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { getInlineWorkspaceFileContract } from '@/lib/api/contracts/workspace-files' +import { parseRequest } from '@/lib/api/server' +import { getSession } from '@/lib/auth' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { resolveWorkspaceInlineImage } from '@/lib/uploads/server/inline-image' +import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' +import { serveInlineImage } from '@/app/api/files/serve-inline-image' +import { createErrorResponse, FileNotFoundError } from '@/app/api/files/utils' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('WorkspaceInlineFileAPI') + +/** + * GET /api/workspaces/[id]/files/inline?key=|fileId= + * + * Serves an image embedded in a workspace markdown document, **scoped to the workspace in the path**. + * The markdown editor rewrites its embedded `/api/files/serve/` and `/api/files/view/` srcs to + * this route so a referenced file resolves only within the document's workspace — a cross-workspace + * reference returns 404 and does not render, even for a viewer who belongs to the other workspace. Read + * access to the workspace is required; disposition/content-type handling mirrors the serve route. + */ +export const GET = withRouteHandler( + async (request: NextRequest, context: { params: Promise<{ id: string }> }) => { + try { + const parsed = await parseRequest(getInlineWorkspaceFileContract, request, context) + if (!parsed.success) return parsed.response + const { id: workspaceId } = parsed.data.params + const ref = parsed.data.query + + const session = await getSession() + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + // Authorize before disclosing anything; deny with 404 so a non-member can't probe existence. + const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) + if (!permission) { + throw new FileNotFoundError('Not found') + } + + const image = await resolveWorkspaceInlineImage(workspaceId, ref) + if (!image) { + throw new FileNotFoundError('Not found') + } + + return await serveInlineImage(image, { sniff: false }) + } catch (error) { + if (error instanceof FileNotFoundError) { + return createErrorResponse(error) + } + logger.error('Error serving workspace inline image:', error) + return createErrorResponse(error instanceof Error ? error : new Error('Failed to serve file')) + } + } +) diff --git a/apps/sim/app/f/[token]/public-file-view.tsx b/apps/sim/app/f/[token]/public-file-view.tsx index f27b63df65d..360119e4945 100644 --- a/apps/sim/app/f/[token]/public-file-view.tsx +++ b/apps/sim/app/f/[token]/public-file-view.tsx @@ -9,7 +9,7 @@ import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { buildProvenance } from '@/app/f/[token]/utils' import { FileViewer } from '@/app/workspace/[workspaceId]/files/components/file-viewer' import { useBrandConfig } from '@/ee/whitelabeling' -import { type FileContentSource, FileContentSourceProvider } from '@/hooks/use-file-content-source' +import { createPublicFileContentSource } from '@/hooks/use-file-content-source' interface PublicFileViewProps { token: string @@ -41,7 +41,12 @@ export function PublicFileView({ // `updatedAt` fold in the content version so the React Query caches (keyed on the // storage key + `updatedAt`) refetch when the shared file changes — even when its // size is unchanged. - const source = useMemo(() => ({ buildUrl: () => contentUrl }), [contentUrl]) + // Embedded images route through the token-scoped cascade endpoint, which serves them only when the + // shared document actually references them and they live in its workspace. + const source = useMemo( + () => createPublicFileContentSource(token, contentUrl), + [token, contentUrl] + ) const file = useMemo( () => ({ id: token, @@ -116,9 +121,13 @@ export function PublicFileView({
- - - +
) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index 2e1d4d834ea..3d4f4b5d3e9 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -6,6 +6,11 @@ import dynamic from 'next/dynamic' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { getFileExtension } from '@/lib/uploads/utils/file-utils' import { useWorkspaceFileBinary, useWorkspaceFileContent } from '@/hooks/queries/workspace-files' +import { + createWorkspaceFileContentSource, + type FileContentSource, + FileContentSourceProvider, +} from '@/hooks/use-file-content-source' import { CsvTablePreview } from './csv-table-preview' import { DocxPreview } from './docx-preview' import { resolveFileCategory } from './file-category' @@ -78,6 +83,12 @@ export type PreviewMode = 'editor' | 'split' | 'preview' interface FileViewerProps { file: WorkspaceFileRecord workspaceId: string + /** + * Content source for this view. Defaults to a workspace-scoped source derived from `workspaceId`; + * the public share page passes a token-scoped source. Provided to descendants (renderers, embedded + * images) via {@link FileContentSourceProvider}. + */ + contentSource?: FileContentSource canEdit: boolean /** * Render a read-only preview with no editing affordances. Text files render @@ -97,7 +108,20 @@ interface FileViewerProps { previewContextKey?: string } -export function FileViewer({ +export function FileViewer(props: FileViewerProps) { + const { contentSource, workspaceId } = props + const source = useMemo( + () => contentSource ?? createWorkspaceFileContentSource(workspaceId), + [contentSource, workspaceId] + ) + return ( + + + + ) +} + +function FileViewerContent({ file, workspaceId, canEdit, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts index 41e2f888408..a2879b6da6f 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts @@ -2,26 +2,41 @@ * @vitest-environment jsdom */ import { describe, expect, it } from 'vitest' -import { resolveDisplaySrc } from './image' +import { + createPublicFileContentSource, + createWorkspaceFileContentSource, +} from '@/hooks/use-file-content-source' -describe('resolveDisplaySrc', () => { - it('rewrites an in-app workspace file path to its serving endpoint (display only)', () => { - expect(resolveDisplaySrc('/workspace/W1/files/F123')).toBe('/api/files/view/F123') - expect(resolveDisplaySrc('/workspace/any-ws-id/files/abc-def')).toBe('/api/files/view/abc-def') +const KEY = 'workspace/W1/1700000000000-deadbeefdeadbeef-photo.png' +const ENCODED = encodeURIComponent(KEY) + +describe('content-source resolveImageSrc', () => { + it('in-app source rewrites embeds to the workspace-scoped inline route', () => { + const src = createWorkspaceFileContentSource('ws-1') + expect(src.resolveImageSrc(`/api/files/serve/${ENCODED}?context=workspace`)).toBe( + `/api/workspaces/ws-1/files/inline?key=${encodeURIComponent(KEY)}` + ) + expect(src.resolveImageSrc('/api/files/view/wf_abc')).toBe( + '/api/workspaces/ws-1/files/inline?fileId=wf_abc' + ) }) - it('leaves absolute and non-workspace URLs untouched', () => { - expect(resolveDisplaySrc('https://cdn.example.com/a.png')).toBe('https://cdn.example.com/a.png') - expect(resolveDisplaySrc('http://localhost/workspace/W1/files/F1')).toBe( - 'http://localhost/workspace/W1/files/F1' + it('public source rewrites embeds to the token-scoped inline route', () => { + const src = createPublicFileContentSource('tok_1', '/api/files/public/tok_1/content') + expect(src.resolveImageSrc('/api/files/view/wf_abc')).toBe( + '/api/files/public/tok_1/inline?fileId=wf_abc' ) - expect(resolveDisplaySrc('/other/path/files/x')).toBe('/other/path/files/x') - expect(resolveDisplaySrc('relative/image.png')).toBe('relative/image.png') }) - it('passes through empty/undefined and unparseable values', () => { - expect(resolveDisplaySrc(undefined)).toBeUndefined() - expect(resolveDisplaySrc('')).toBe('') - expect(resolveDisplaySrc('/workspace/W1/files/')).toBe('/workspace/W1/files/') + it('passes external/data srcs through unchanged in both sources', () => { + const ws = createWorkspaceFileContentSource('ws-1') + const pub = createPublicFileContentSource('tok_1', '/c') + expect(ws.resolveImageSrc('https://cdn.example.com/a.png')).toBe( + 'https://cdn.example.com/a.png' + ) + expect(pub.resolveImageSrc('https://cdn.example.com/a.png')).toBe( + 'https://cdn.example.com/a.png' + ) + expect(ws.resolveImageSrc(undefined)).toBeUndefined() }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx index 8e76a4244bb..5809af9625a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx @@ -3,6 +3,7 @@ import type { JSONContent } from '@tiptap/core' import { Image } from '@tiptap/extension-image' import type { ReactNodeViewProps } from '@tiptap/react' import { NodeViewWrapper, ReactNodeViewRenderer } from '@tiptap/react' +import { useFileContentSource } from '@/hooks/use-file-content-source' import { normalizeLinkHref } from './markdown-fidelity' const MIN_WIDTH = 64 @@ -26,24 +27,6 @@ function escapeAttr(value: string): string { .replace(/>/g, '>') } -/** - * Rewrite an in-app workspace file path (`/workspace/{id}/files/{fileId}`) to its serving endpoint - * (`/api/files/view/{fileId}`) for display only — the stored `src` attribute keeps the original path - * so markdown round-trips unchanged. Absolute and non-workspace URLs pass through untouched. - */ -export function resolveDisplaySrc(src: string | undefined): string | undefined { - if (!src) return src - try { - const parsed = new URL(src, 'http://placeholder') - if (parsed.origin !== 'http://placeholder') return src - const [, seg1, , seg3, fileId] = parsed.pathname.split('/') - if (seg1 === 'workspace' && seg3 === 'files' && fileId) return `/api/files/view/${fileId}` - } catch { - // not a parseable URL — render as-is - } - return src -} - /** * Serialize an image to markdown when it has no explicit size, and to an HTML `` tag when * it does — standard markdown has no width syntax, so a resized image must round-trip as HTML to @@ -174,6 +157,7 @@ export const MarkdownImage = Image.extend({ * commits the new pixel width to the `width` attribute, which serializes to ``. */ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactNodeViewProps) { + const source = useFileContentSource() const imageRef = useRef(null) const dragAbortRef = useRef(null) const [dragging, setDragging] = useState(false) @@ -232,7 +216,7 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN const image = ( {attrs.alt(null) + // The `/Image` slash command opens this hidden picker; `pendingImagePosRef` holds the caret position + // captured when the command ran, so the upload inserts where `/Image` was typed. + const imageInputRef = useRef(null) + const pendingImagePosRef = useRef(null) + // Upload then insert each image at `at` (paste caret / drop point), sequentially; held in a ref so handlers reach the latest. const insertImagesRef = useRef<(images: File[], at: number) => Promise>(() => Promise.resolve() @@ -293,6 +298,19 @@ export function LoadedRichMarkdownEditor({ }) editorInstanceRef.current = editor + // Wire the `/Image` slash command to the hidden picker (per-editor storage, since the extension set is + // shared across instances). Reads only refs, so the handler stays stable across the editor's life. + useEffect(() => { + if (!editor) return + editor.storage.slashCommand.insertImage = (at: number) => { + pendingImagePosRef.current = at + imageInputRef.current?.click() + } + return () => { + editor.storage.slashCommand.insertImage = null + } + }, [editor]) + const wasStreamingRef = useRef(streamingAtMountRef.current) const pendingStreamBodyRef = useRef(null) @@ -386,6 +404,22 @@ export function LoadedRichMarkdownEditor({ > {editor && } {editor && } + { + const input = event.currentTarget + const images = Array.from(input.files ?? []).filter((f) => f.type.startsWith('image/')) + const at = + pendingImagePosRef.current ?? editorInstanceRef.current?.state.selection.from ?? 0 + pendingImagePosRef.current = null + input.value = '' + if (images.length > 0) void insertImagesRef.current(images, at) + }} + /> { @@ -48,4 +49,37 @@ describe('SLASH_COMMANDS registry', () => { const titles = SLASH_COMMANDS.map((c) => c.title) expect(new Set(titles).size).toBe(titles.length) }) + + it('Image command replaces the trigger and hands the caret to the host insertImage handler', () => { + const insertImage = vi.fn() + const deleteRange = vi.fn(() => chain) + const chain = { focus: () => chain, deleteRange, run: () => true } + const editor = { + chain: () => chain, + storage: { slashCommand: { insertImage } }, + state: { selection: { from: 7 } }, + } as unknown as Editor + + const image = SLASH_COMMANDS.find((c) => c.title === 'Image') + expect(image).toBeDefined() + image?.run({ editor, range: { from: 5, to: 6 } as Range }) + + expect(deleteRange).toHaveBeenCalledWith({ from: 5, to: 6 }) + expect(insertImage).toHaveBeenCalledWith(7) + }) + + it('Image command is a no-op when no handler is wired', () => { + const chain = { focus: () => chain, deleteRange: () => chain, run: () => true } + const editor = { + chain: () => chain, + storage: { slashCommand: { insertImage: null } }, + state: { selection: { from: 0 } }, + } as unknown as Editor + expect(() => + SLASH_COMMANDS.find((c) => c.title === 'Image')?.run({ + editor, + range: { from: 0, to: 1 } as Range, + }) + ).not.toThrow() + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/commands.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/commands.ts index acf945017d9..a3bdd960bc8 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/commands.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/commands.ts @@ -4,6 +4,7 @@ import { Heading1, Heading2, Heading3, + Image as ImageIcon, List, ListChecks, ListOrdered, @@ -19,6 +20,15 @@ export interface SlashCommandContext { range: Range } +/** + * Per-editor storage on the `slashCommand` extension. The host editor component sets `insertImage` + * after mount; it opens an image file picker and uploads + inserts the chosen image(s) at `at`. Null + * in headless/read-only contexts, where the Image command is a no-op. + */ +export interface SlashCommandStorage { + insertImage: ((at: number) => void) | null +} + export interface SlashCommandItem { title: string /** Group heading the item is shown under in the menu. */ @@ -131,6 +141,18 @@ export const SLASH_COMMANDS: readonly SlashCommandItem[] = [ aliases: ['hr', 'horizontal rule', 'separator'], run: ({ editor, range }) => editor.chain().focus().deleteRange(range).setHorizontalRule().run(), }, + { + title: 'Image', + group: 'Media', + icon: ImageIcon, + aliases: ['picture', 'photo', 'upload', 'img'], + run: ({ editor, range }) => { + // Replace the typed `/query`, then hand off to the host component's picker, which uploads and + // inserts the image at the caret (the same path as paste/drop). No-op when no handler is wired. + editor.chain().focus().deleteRange(range).run() + editor.storage.slashCommand.insertImage?.(editor.state.selection.from) + }, + }, ] /** diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/slash-command.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/slash-command.ts index 2a5118ec1cf..1ceff0c9eed 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/slash-command.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/slash-command/slash-command.ts @@ -3,9 +3,20 @@ import type { Editor } from '@tiptap/core' import { Extension } from '@tiptap/core' import { ReactRenderer } from '@tiptap/react' import Suggestion, { type SuggestionOptions, type SuggestionProps } from '@tiptap/suggestion' -import { filterSlashCommands, type SlashCommandContext, type SlashCommandItem } from './commands' +import { + filterSlashCommands, + type SlashCommandContext, + type SlashCommandItem, + type SlashCommandStorage, +} from './commands' import { SlashCommandList, type SlashCommandListHandle } from './slash-command-list' +declare module '@tiptap/core' { + interface Storage { + slashCommand: SlashCommandStorage + } +} + type SlashSuggestionProps = SuggestionProps function positionPopup(element: HTMLElement, getRect: SlashSuggestionProps['clientRect']) { @@ -76,9 +87,13 @@ function renderSlashSuggestion(): ReturnType, SlashCommandStorage>({ name: 'slashCommand', + addStorage() { + return { insertImage: null } + }, + addProseMirrorPlugins() { return [ Suggestion({ diff --git a/apps/sim/hooks/use-file-content-source.tsx b/apps/sim/hooks/use-file-content-source.tsx index ee09819d7c8..bd9552d8e7e 100644 --- a/apps/sim/hooks/use-file-content-source.tsx +++ b/apps/sim/hooks/use-file-content-source.tsx @@ -1,6 +1,10 @@ 'use client' import { createContext, useContext } from 'react' +import { + type EmbeddedFileRef, + extractEmbeddedFileRef, +} from '@/lib/uploads/utils/embedded-image-ref' export interface FileContentUrlOptions { /** Request the uncompiled source instead of the rendered/compiled bytes. */ @@ -11,6 +15,12 @@ export interface FileContentUrlOptions { bust?: boolean } +function inlineRefQuery(ref: NonNullable): string { + return 'key' in ref + ? `key=${encodeURIComponent(ref.key)}` + : `fileId=${encodeURIComponent(ref.fileId)}` +} + /** * Seam for "where do a file's bytes come from". The in-app viewer resolves the * auth-gated workspace serve URL; the public share page swaps in a token-scoped @@ -19,18 +29,67 @@ export interface FileContentUrlOptions { */ export interface FileContentSource { buildUrl: (key: string, opts?: FileContentUrlOptions) => string + /** + * Map an embedded image `src` to a display URL scoped to the current context: the in-app source + * points at the workspace-scoped inline route, the public source at the token-scoped cascade route. + * Non-workspace srcs (external, `data:`, public assets) pass through unchanged. + */ + resolveImageSrc: (src: string | undefined) => string | undefined +} + +function buildServeUrl(key: string, opts?: FileContentUrlOptions): string { + const base = `/api/files/serve/${encodeURIComponent(key)}?context=workspace` + const params: string[] = [] + if (opts?.version != null) params.push(`v=${encodeURIComponent(String(opts.version))}`) + else if (opts?.bust) params.push(`t=${Date.now()}`) + if (opts?.raw) params.push('raw=1') + return params.length > 0 ? `${base}&${params.join('&')}` : base +} + +/** Build a source whose embeds resolve through `inlineBase` (the workspace- or token-scoped inline route). */ +function inlineImageSource( + buildUrl: FileContentSource['buildUrl'], + inlineBase: string +): FileContentSource { + return { + buildUrl, + resolveImageSrc: (src) => { + if (!src) return src + const ref = extractEmbeddedFileRef(src) + return ref ? `${inlineBase}?${inlineRefQuery(ref)}` : src + }, + } +} + +/** + * In-app source scoped to one workspace. Direct file bytes come from the workspace serve URL; embedded + * images route through `/api/workspaces/{workspaceId}/files/inline`, which resolves a reference only + * within this workspace — a cross-workspace embed 404s and does not render. + */ +export function createWorkspaceFileContentSource(workspaceId: string): FileContentSource { + return inlineImageSource(buildServeUrl, `/api/workspaces/${workspaceId}/files/inline`) +} + +/** + * Public share source. Direct file bytes come from the token content URL; embedded images route through + * `/api/files/public/{token}/inline`, which serves them only when referenced by the shared document and + * in its workspace. + */ +export function createPublicFileContentSource( + token: string, + contentUrl: string +): FileContentSource { + return inlineImageSource(() => contentUrl, `/api/files/public/${token}/inline`) } -/** Default source: the auth-gated workspace serve URL (the historical behavior). */ +/** + * Context default for components rendered outside a {@link FileContentSourceProvider}: serve URLs for + * direct bytes, embeds passed through unchanged. The file viewer always provides a workspace- or + * token-scoped source, so embeds resolve through the scoped inline routes there. + */ export const workspaceFileContentSource: FileContentSource = { - buildUrl: (key, opts) => { - const base = `/api/files/serve/${encodeURIComponent(key)}?context=workspace` - const params: string[] = [] - if (opts?.version != null) params.push(`v=${encodeURIComponent(String(opts.version))}`) - else if (opts?.bust) params.push(`t=${Date.now()}`) - if (opts?.raw) params.push('raw=1') - return params.length > 0 ? `${base}&${params.join('&')}` : base - }, + buildUrl: buildServeUrl, + resolveImageSrc: (src) => src, } const FileContentSourceContext = createContext(workspaceFileContentSource) diff --git a/apps/sim/lib/api/contracts/primitives.ts b/apps/sim/lib/api/contracts/primitives.ts index 3f5391b5408..7e73175abc3 100644 --- a/apps/sim/lib/api/contracts/primitives.ts +++ b/apps/sim/lib/api/contracts/primitives.ts @@ -50,6 +50,33 @@ export const organizationIdSchema = z.string().min(1, 'Organization ID is requir */ export const workflowIdSchema = z.string().min(1, 'Workflow ID is required') +/** + * A `workspace_files.id` value. The column is a free-form `text` primary key, so + * ids come in two shapes: UUID v4 (legacy rows and the `insertFileMetadata` + * default) and the current `wf_` form minted by the workspace upload + * path. Both are drawn from `[A-Za-z0-9_-]`, so accept that charset rather than a + * UUID-only schema — a `.uuid()` constraint here silently 400s every `wf_` file. + */ +export const workspaceFileIdSchema = z + .string() + .min(1, 'File ID is required') + .max(128, 'File ID is too long') + .regex(/^[A-Za-z0-9_-]+$/, 'Invalid file id') + +/** + * Reference to an image embedded in a document: either a workspace storage `key` + * (serve-URL embeds) or a workspace file `id` (view-URL embeds) — exactly one. Shared + * by the in-app and public inline-image routes, which resolve it within a workspace. + */ +export const inlineFileRefQuerySchema = z + .object({ + key: z.string().min(1).max(512).optional(), + fileId: workspaceFileIdSchema.optional(), + }) + .refine((q) => (q.key ? 1 : 0) + (q.fileId ? 1 : 0) === 1, { + message: 'Provide exactly one of `key` or `fileId`', + }) + /** * Boolean query-string primitive that correctly handles the literal strings * `"true"` / `"false"` (case-insensitive) in addition to real booleans. diff --git a/apps/sim/lib/api/contracts/public-shares.ts b/apps/sim/lib/api/contracts/public-shares.ts index aa623324558..a839d372ead 100644 --- a/apps/sim/lib/api/contracts/public-shares.ts +++ b/apps/sim/lib/api/contracts/public-shares.ts @@ -1,5 +1,5 @@ import { z } from 'zod' -import { workspaceIdSchema } from '@/lib/api/contracts/primitives' +import { inlineFileRefQuerySchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' export const shareResourceTypeSchema = z.enum(['file', 'folder']) @@ -126,6 +126,22 @@ export const getPublicFileContentContract = defineRouteContract({ }, }) +/** + * Binary stream of an image embedded in a shared document. Authorized by the parent + * document's active share — the route serves the bytes only when the reference is + * actually embedded in the shared document AND the file lives in the same workspace, + * and only when the bytes are a renderable raster image. + */ +export const getPublicInlineFileContract = defineRouteContract({ + method: 'GET', + path: '/api/files/public/[token]/inline', + params: publicFileTokenParamsSchema, + query: inlineFileRefQuerySchema, + response: { + mode: 'binary', + }, +}) + const authenticatePublicFileBodySchema = z.object({ password: z.string().min(1, 'Password is required').max(1024, 'Password is too long'), }) diff --git a/apps/sim/lib/api/contracts/storage-transfer.ts b/apps/sim/lib/api/contracts/storage-transfer.ts index 57de6d75b29..e5fef522c20 100644 --- a/apps/sim/lib/api/contracts/storage-transfer.ts +++ b/apps/sim/lib/api/contracts/storage-transfer.ts @@ -3,6 +3,7 @@ import { batchPresignedUploadResponseSchema, presignedUploadResponseSchema, } from '@/lib/api/contracts/file-uploads' +import { workspaceFileIdSchema } from '@/lib/api/contracts/primitives' import { type ContractBodyInput, type ContractJsonResponse, @@ -465,11 +466,11 @@ export const fileServeQuerySchema = z.object({ }) export const fileViewParamsSchema = z.object({ - id: z.string().uuid('File ID must be a valid UUID'), + id: workspaceFileIdSchema, }) export const fileExportParamsSchema = z.object({ - id: z.string().uuid('File ID must be a valid UUID'), + id: workspaceFileIdSchema, }) export const boxUploadContract = defineRouteContract({ diff --git a/apps/sim/lib/api/contracts/workspace-files.ts b/apps/sim/lib/api/contracts/workspace-files.ts index c7f1f6d5366..b47b5710528 100644 --- a/apps/sim/lib/api/contracts/workspace-files.ts +++ b/apps/sim/lib/api/contracts/workspace-files.ts @@ -1,4 +1,5 @@ import { z } from 'zod' +import { inlineFileRefQuerySchema } from '@/lib/api/contracts/primitives' import { shareRecordSchema } from '@/lib/api/contracts/public-shares' import { defineRouteContract } from '@/lib/api/contracts/types' @@ -16,6 +17,21 @@ export const listWorkspaceFilesQuerySchema = z.object({ scope: workspaceFileScopeSchema.default('active'), }) +/** + * Binary stream of an image embedded in a workspace markdown document, scoped to the + * workspace in the path. The route serves the bytes only when the referenced file is a + * `workspace` file belonging to `[id]` — cross-workspace references do not resolve. + */ +export const getInlineWorkspaceFileContract = defineRouteContract({ + method: 'GET', + path: '/api/workspaces/[id]/files/inline', + params: workspaceFilesParamsSchema, + query: inlineFileRefQuerySchema, + response: { + mode: 'binary', + }, +}) + const workspaceFileNameSchema = z .string({ error: 'Name is required' }) .trim() diff --git a/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.test.ts b/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.test.ts new file mode 100644 index 00000000000..d4c33793f3f --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { + extractEmbeddedImageIds, + extractEmbeddedImageKeys, +} from '@/lib/copilot/tools/server/files/embedded-image-refs' + +const KEY = 'workspace/W1/1700000000000-deadbeefdeadbeef-photo.png' + +describe('extractEmbeddedImageIds', () => { + it('extracts unique ids from view-url and in-app-path embeds (wf_ and uuid)', () => { + const a = 'wf_YwDXi8eWOkTxn0sbgChlB' + const b = '4bdaf6c4-072e-464e-891d-b6af3b5fe2cc' + const content = `![x](/api/files/view/${a}) ![y](/workspace/W1/files/${b}) ![dup](/api/files/view/${a})` + expect(extractEmbeddedImageIds(content).sort()).toEqual([b, a].sort()) + }) + + it('ignores serve-url, external, and plain content', () => { + expect( + extractEmbeddedImageIds(`![a](/api/files/serve/${encodeURIComponent(KEY)}) plain`) + ).toEqual([]) + }) + + it('caps the result at 50 ids', () => { + const content = Array.from( + { length: 60 }, + (_, i) => `/api/files/view/wf_${String(i).padStart(6, '0')}` + ).join(' ') + expect(extractEmbeddedImageIds(content)).toHaveLength(50) + }) +}) + +describe('extractEmbeddedImageKeys', () => { + it('extracts decoded workspace keys from serve-url embeds (encoded + s3/blob prefixed)', () => { + const content = `![a](/api/files/serve/${encodeURIComponent(KEY)}?context=workspace) ![b](/api/files/serve/s3/${encodeURIComponent(KEY)})` + expect(extractEmbeddedImageKeys(content)).toEqual([KEY]) + }) + + it('drops non-workspace keys (e.g. public profile pictures) and view-url embeds', () => { + const content = + '![a](/api/files/serve/profile-pictures%2Fu1%2Favatar.png) ![b](/api/files/view/wf_abc)' + expect(extractEmbeddedImageKeys(content)).toEqual([]) + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.ts b/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.ts index fb7a0eb5967..f5f7b9fbbad 100644 --- a/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.ts +++ b/apps/sim/lib/copilot/tools/server/files/embedded-image-refs.ts @@ -1,8 +1,26 @@ import { getFileMetadataById } from '@/lib/uploads/server/metadata' +import { extractEmbeddedFileRefs } from '@/lib/uploads/utils/embedded-image-ref' -/** The canonical embed form the file agent writes for workspace images: `/api/files/view/`. */ +/** View-URL embed (`/api/files/view/`) — the only form the file agent writes; see {@link findUnembeddableImageRefs}. */ const VIEW_EMBED_RE = /\/api\/files\/view\/([A-Za-z0-9_-]+)/g +/** + * De-duplicated workspace file **ids** embedded in `content` (view URL or in-app workspace path). + * Shares the {@link extractEmbeddedFileRefs} grammar with the frontend renderer so the referenced-by-doc + * gate authorizes exactly what the client links. Resolution and access are checked by the caller. + */ +export function extractEmbeddedImageIds(content: string): string[] { + return extractEmbeddedFileRefs(content).ids +} + +/** + * De-duplicated workspace storage **keys** (`workspace//…`) embedded in `content` via the serve URL. + * Same shared grammar as {@link extractEmbeddedImageIds}. + */ +export function extractEmbeddedImageKeys(content: string): string[] { + return extractEmbeddedFileRefs(content).keys +} + /** * Returns the ids of `/api/files/view/` image embeds in `content` that will not render or survive a * workspace export. An embed is valid only when its id resolves to a workspace file in this same diff --git a/apps/sim/lib/uploads/server/inline-image.test.ts b/apps/sim/lib/uploads/server/inline-image.test.ts new file mode 100644 index 00000000000..ba774a3e8f1 --- /dev/null +++ b/apps/sim/lib/uploads/server/inline-image.test.ts @@ -0,0 +1,63 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetWorkspaceFile, mockGetFileMetadataByKey } = vi.hoisted(() => ({ + mockGetWorkspaceFile: vi.fn(), + mockGetFileMetadataByKey: vi.fn(), +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ getWorkspaceFile: mockGetWorkspaceFile })) +vi.mock('@/lib/uploads/server/metadata', () => ({ getFileMetadataByKey: mockGetFileMetadataByKey })) + +import { resolveWorkspaceInlineImage } from '@/lib/uploads/server/inline-image' + +describe('resolveWorkspaceInlineImage', () => { + beforeEach(() => vi.clearAllMocks()) + + it('resolves by fileId scoped to the workspace (getWorkspaceFile already enforces scope)', async () => { + mockGetWorkspaceFile.mockResolvedValue({ + key: 'workspace/ws-1/x.png', + type: 'image/png', + name: 'x.png', + }) + const out = await resolveWorkspaceInlineImage('ws-1', { fileId: 'wf_a' }) + expect(mockGetWorkspaceFile).toHaveBeenCalledWith('ws-1', 'wf_a') + expect(out).toEqual({ + key: 'workspace/ws-1/x.png', + contentType: 'image/png', + filename: 'x.png', + }) + }) + + it('returns null when getWorkspaceFile finds nothing (cross-workspace / deleted / non-workspace)', async () => { + mockGetWorkspaceFile.mockResolvedValue(null) + expect(await resolveWorkspaceInlineImage('ws-1', { fileId: 'wf_a' })).toBeNull() + }) + + it('resolves by key only when the row belongs to the workspace', async () => { + mockGetFileMetadataByKey.mockResolvedValue({ + key: 'workspace/ws-1/x.png', + workspaceId: 'ws-1', + contentType: 'image/png', + originalName: 'x.png', + }) + const out = await resolveWorkspaceInlineImage('ws-1', { key: 'workspace/ws-1/x.png' }) + expect(out).toEqual({ + key: 'workspace/ws-1/x.png', + contentType: 'image/png', + filename: 'x.png', + }) + }) + + it('returns null when the keyed row belongs to a different workspace', async () => { + mockGetFileMetadataByKey.mockResolvedValue({ + key: 'workspace/ws-2/x.png', + workspaceId: 'ws-2', + contentType: 'image/png', + originalName: 'x.png', + }) + expect(await resolveWorkspaceInlineImage('ws-1', { key: 'workspace/ws-2/x.png' })).toBeNull() + }) +}) diff --git a/apps/sim/lib/uploads/server/inline-image.ts b/apps/sim/lib/uploads/server/inline-image.ts new file mode 100644 index 00000000000..b44b9e08e4a --- /dev/null +++ b/apps/sim/lib/uploads/server/inline-image.ts @@ -0,0 +1,41 @@ +import { getWorkspaceFile } from '@/lib/uploads/contexts/workspace' +import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' + +/** + * A markdown-embedded image reference: either a workspace file `fileId` (view-URL embeds) or a + * workspace storage `key` (serve-URL embeds). Exactly one is set — enforced at the route boundary. + */ +export interface InlineImageRef { + key?: string + fileId?: string +} + +/** The fields a serve handler needs to return an embedded image. */ +export interface ResolvedInlineImage { + key: string + contentType: string + filename: string +} + +/** + * Resolve an embedded-image reference to its storage key + metadata, **scoped to `workspaceId`**. + * Returns null whenever the reference is not a live `workspace` file in that workspace — a + * cross-workspace, non-workspace, missing, or deleted file. This is the single workspace-scope gate + * shared by the in-app inline route and the public-share cascade, mirroring how the user-facing file + * view resolves a file within its workspace ({@link getWorkspaceFile}). + */ +export async function resolveWorkspaceInlineImage( + workspaceId: string, + ref: InlineImageRef +): Promise { + if (ref.fileId) { + const file = await getWorkspaceFile(workspaceId, ref.fileId) + return file ? { key: file.key, contentType: file.type, filename: file.name } : null + } + if (ref.key) { + const record = await getFileMetadataByKey(ref.key, 'workspace') + if (!record || record.workspaceId !== workspaceId) return null + return { key: record.key, contentType: record.contentType, filename: record.originalName } + } + return null +} diff --git a/apps/sim/lib/uploads/utils/embedded-image-ref.test.ts b/apps/sim/lib/uploads/utils/embedded-image-ref.test.ts new file mode 100644 index 00000000000..c8773883b70 --- /dev/null +++ b/apps/sim/lib/uploads/utils/embedded-image-ref.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest' +import { + extractEmbeddedFileRef, + extractEmbeddedFileRefs, +} from '@/lib/uploads/utils/embedded-image-ref' + +const KEY = 'workspace/W1/1700000000000-deadbeefdeadbeef-photo.png' +const ENCODED = encodeURIComponent(KEY) + +describe('extractEmbeddedFileRef', () => { + it('parses serve-url embeds (encoded, raw, and s3/blob prefixed) to the workspace key', () => { + expect(extractEmbeddedFileRef(`/api/files/serve/${ENCODED}?context=workspace`)).toEqual({ + key: KEY, + }) + expect(extractEmbeddedFileRef(`/api/files/serve/s3/${ENCODED}`)).toEqual({ key: KEY }) + expect(extractEmbeddedFileRef(`/api/files/serve/blob/${ENCODED}`)).toEqual({ key: KEY }) + }) + + it('parses view-url and in-app-path embeds to the file id', () => { + expect(extractEmbeddedFileRef('/api/files/view/wf_YwDXi8eWOkTxn0sbgChlB')).toEqual({ + fileId: 'wf_YwDXi8eWOkTxn0sbgChlB', + }) + expect(extractEmbeddedFileRef('/workspace/W1/files/wf_abc')).toEqual({ fileId: 'wf_abc' }) + }) + + it('returns null for external, data, and non-workspace serve urls', () => { + expect(extractEmbeddedFileRef('https://cdn.example.com/a.png')).toBeNull() + expect(extractEmbeddedFileRef('data:image/png;base64,AAAA')).toBeNull() + expect(extractEmbeddedFileRef('/api/files/serve/profile-pictures%2Fu1%2Favatar.png')).toBeNull() + }) +}) + +describe('extractEmbeddedFileRefs', () => { + it('collects de-duplicated keys and ids from a document via the shared parser', () => { + const content = ` + ![a](/api/files/serve/${ENCODED}?context=workspace) + ![b](/api/files/view/wf_abc) + ![c](/workspace/W1/files/4bdaf6c4-072e-464e-891d-b6af3b5fe2cc) + ![dup](/api/files/serve/s3/${ENCODED}) + ![ext](https://cdn.example.com/x.png) + ![pub](/api/files/serve/profile-pictures%2Fu1%2Favatar.png) + ` + const { keys, ids } = extractEmbeddedFileRefs(content) + expect(keys).toEqual([KEY]) + expect(ids.sort()).toEqual(['4bdaf6c4-072e-464e-891d-b6af3b5fe2cc', 'wf_abc'].sort()) + }) + + it('caps total references (keys + ids) at 50 combined', () => { + const ids = Array.from( + { length: 40 }, + (_, i) => `/api/files/view/wf_${String(i).padStart(6, '0')}` + ) + const keys = Array.from( + { length: 40 }, + (_, i) => `/api/files/serve/${encodeURIComponent(`workspace/W1/k${i}.png`)}` + ) + const { keys: k, ids: d } = extractEmbeddedFileRefs([...ids, ...keys].join(' ')) + expect(k.length + d.length).toBe(50) + }) +}) diff --git a/apps/sim/lib/uploads/utils/embedded-image-ref.ts b/apps/sim/lib/uploads/utils/embedded-image-ref.ts new file mode 100644 index 00000000000..0161af11ea7 --- /dev/null +++ b/apps/sim/lib/uploads/utils/embedded-image-ref.ts @@ -0,0 +1,73 @@ +/** + * The grammar of a markdown-embedded workspace image reference, shared by the frontend renderer + * (which rewrites one `src` at a time) and the server (which scans a whole document for the + * referenced-by-doc gate and the export bundler). Both go through {@link extractEmbeddedFileRef} so + * the set the client links and the set the server authorizes can never drift apart. + * + * Pure and isomorphic — no DOM, Node, or DB imports — so it is safe to import from both client and + * server code. + */ + +/** A reference parsed from an embed `src`: a workspace storage key, a workspace file id, or neither. */ +export type EmbeddedFileRef = { key: string } | { fileId: string } | null + +/** Hard cap on embedded images resolved from one document — bounds export bundles and the cascade. */ +export const MAX_EMBEDDED_IMAGES = 50 + +/** + * Candidate embed URL substrings in document text: a serve URL, a view URL, or the in-app workspace + * path. The captured run stops at whitespace/quote/paren/angle/query so authoritative parsing is left + * to {@link extractEmbeddedFileRef}. + */ +const EMBED_URL_RE = + /(?:\/api\/files\/(?:serve|view)\/|\/workspace\/[A-Za-z0-9-]+\/files\/)[^\s)"'<>?]*/g + +/** + * Parse a single embed `src` into the workspace file it references, normalizing the spellings the + * editor and file agent produce: `/api/files/serve/` (incl. `s3/`/`blob/` prefixes), `/api/files/view/`, + * and the in-app path `/workspace//files/`. Returns null for absolute, `data:`, or non-workspace + * URLs (e.g. public `profile-pictures/` assets), which render as-is. + */ +export function extractEmbeddedFileRef(src: string): EmbeddedFileRef { + try { + const parsed = new URL(src, 'http://placeholder') + if (parsed.origin !== 'http://placeholder') return null + const segs = parsed.pathname.split('/') + if (segs[1] === 'api' && segs[2] === 'files' && segs[3] === 'serve') { + let keySegs = segs.slice(4) + if (keySegs[0] === 's3' || keySegs[0] === 'blob') keySegs = keySegs.slice(1) + const raw = keySegs.join('/') + if (!raw) return null + const key = decodeURIComponent(raw) + return key.startsWith('workspace/') ? { key } : null + } + if (segs[1] === 'api' && segs[2] === 'files' && segs[3] === 'view' && segs[4]) { + return { fileId: segs[4] } + } + if (segs[1] === 'workspace' && segs[3] === 'files' && segs[4]) { + return { fileId: segs[4] } + } + return null + } catch { + return null + } +} + +/** + * The de-duplicated keys and ids embedded in `content`, bounded to {@link MAX_EMBEDDED_IMAGES} unique + * references **combined** (keys + ids). Every candidate URL is interpreted by {@link extractEmbeddedFileRef}, + * so this is exactly the set the frontend rewrites — the server's referenced-by-doc gate and the export + * bundler share one grammar. + */ +export function extractEmbeddedFileRefs(content: string): { keys: string[]; ids: string[] } { + const keys = new Set() + const ids = new Set() + for (const match of content.matchAll(EMBED_URL_RE)) { + const ref = extractEmbeddedFileRef(match[0]) + if (!ref) continue + if ('key' in ref) keys.add(ref.key) + else ids.add(ref.fileId) + if (keys.size + ids.size >= MAX_EMBEDDED_IMAGES) break + } + return { keys: [...keys], ids: [...ids] } +} diff --git a/apps/sim/lib/uploads/utils/validation.test.ts b/apps/sim/lib/uploads/utils/validation.test.ts index f5db99cbd09..9d5d31ea1d6 100644 --- a/apps/sim/lib/uploads/utils/validation.test.ts +++ b/apps/sim/lib/uploads/utils/validation.test.ts @@ -1,9 +1,40 @@ import { describe, expect, it } from 'vitest' import { SUPPORTED_ATTACHMENT_EXTENSIONS, + sniffImageContentType, validateAttachmentFileType, } from '@/lib/uploads/utils/validation' +describe('sniffImageContentType', () => { + const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]) + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00]) + const gif87 = Buffer.from('GIF87a....', 'latin1') + const gif89 = Buffer.from('GIF89a....', 'latin1') + const webp = Buffer.concat([ + Buffer.from('RIFF', 'latin1'), + Buffer.from([0x00, 0x00, 0x00, 0x00]), + Buffer.from('WEBP', 'latin1'), + ]) + + it('detects real raster image formats from magic bytes', () => { + expect(sniffImageContentType(png)).toBe('image/png') + expect(sniffImageContentType(jpeg)).toBe('image/jpeg') + expect(sniffImageContentType(gif87)).toBe('image/gif') + expect(sniffImageContentType(gif89)).toBe('image/gif') + expect(sniffImageContentType(webp)).toBe('image/webp') + }) + + it('rejects non-image content, including image-shaped strings and SVG', () => { + expect( + sniffImageContentType(Buffer.from('', 'utf-8')) + ).toBeNull() + expect(sniffImageContentType(Buffer.from('', 'utf-8'))).toBeNull() + expect(sniffImageContentType(Buffer.from('RIFFxxxxAVI ', 'latin1'))).toBeNull() + expect(sniffImageContentType(Buffer.alloc(0))).toBeNull() + expect(sniffImageContentType(Buffer.from([0x89, 0x50]))).toBeNull() + }) +}) + describe('validateAttachmentFileType', () => { it('accepts image files (png, jpg, gif, webp, svg)', () => { expect(validateAttachmentFileType('screenshot.png')).toBeNull() diff --git a/apps/sim/lib/uploads/utils/validation.ts b/apps/sim/lib/uploads/utils/validation.ts index 4f46d67516a..b4e27684f63 100644 --- a/apps/sim/lib/uploads/utils/validation.ts +++ b/apps/sim/lib/uploads/utils/validation.ts @@ -335,6 +335,34 @@ export function isValidPng(buffer: Buffer): boolean { return buffer.length >= 8 && buffer.subarray(0, 8).equals(PNG_MAGIC_BYTES) } +/** + * Detect a renderable raster image from its leading bytes, returning the canonical MIME type or + * `null` when the content is not one of the inline-renderable image formats (PNG, JPEG, GIF, WebP). + * + * The stored `contentType` is client-declared and never sniffed at upload time, so any path that + * renders a file inline for a less-trusted audience (e.g. images embedded in a public share) must + * derive the served type from the bytes themselves — a file claiming `image/png` could be HTML, SVG, + * or a script. SVG is deliberately excluded: it can carry script and is not a raster format. + */ +export function sniffImageContentType(buffer: Buffer): string | null { + if (isValidPng(buffer)) return 'image/png' + if (buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { + return 'image/jpeg' + } + if (buffer.length >= 6) { + const header = buffer.toString('latin1', 0, 6) + if (header === 'GIF87a' || header === 'GIF89a') return 'image/gif' + } + if ( + buffer.length >= 12 && + buffer.toString('latin1', 0, 4) === 'RIFF' && + buffer.toString('latin1', 8, 12) === 'WEBP' + ) { + return 'image/webp' + } + return null +} + export function validateMediaFileType( fileName: string, mimeType: string diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 17f0a25fa29..a45c2d84914 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 860, - zodRoutes: 860, + totalRoutes: 862, + zodRoutes: 862, nonZodRoutes: 0, } as const From 6260eda22668dc8fced33452365ac8a6605d149d Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 18:46:18 -0700 Subject: [PATCH 12/14] fix(ssr): harden credential query-key factory + fetchers against the 'use client' stub bug (#5206) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ssr): move credential query-key factory + fetchers to non-client modules Preventively closes the same 'use client' SSR client-reference-stub class that crashed the tables page. Server-evaluated modules (the credential block def, the workflow-comparison helpers) imported workspaceCredentialKeys / fetchWorkspaceCredentialList / fetchCredentialSetById from 'use client' hook modules, where they resolve to client-reference stubs on the server (a future server call path would throw 'X is not a function'). Extract them into non-client hooks/queries/utils/{credential-keys, fetch-workspace-credentials,fetch-credential-set}.ts (mirroring folder-keys.ts / fetch-workflow-envelope.ts) and import from there. No behavior change — these values were only ever called from browser paths. * docs+ci: codify the 'use client' server-import rule + add check:client-boundary Document the Next.js rule that server code can only render a 'use client' export as a component, never call it (server imports resolve to client-reference stubs that throw — the tables-page crash). Add the rule to .claude/rules/sim-queries.md + a cross-ref in sim-architecture.md. Add scripts/check-client-boundary-imports.ts (wired into CI as check:client-boundary) that flags any value import from a 'use client' module in a server-evaluated, non-JSX surface (prefetch / route handler / trigger / block definition), so this class can't silently recur. Escape hatch: // client-boundary-allow: . --- .claude/rules/sim-architecture.md | 4 + .claude/rules/sim-queries.md | 11 + .github/workflows/test-build.yml | 3 + .../secrets-manager/secrets-manager.tsx | 7 +- apps/sim/blocks/blocks/credential.ts | 3 +- apps/sim/hooks/queries/credential-sets.ts | 14 +- apps/sim/hooks/queries/credentials.ts | 34 +-- apps/sim/hooks/queries/invitations.ts | 2 +- apps/sim/hooks/queries/organization.ts | 2 +- .../hooks/queries/utils/credential-keys.ts | 24 ++ .../queries/utils/fetch-credential-set.ts | 21 ++ .../utils/fetch-workspace-credentials.ts | 20 ++ .../comparison/format-description.test.ts | 2 +- .../workflows/comparison/resolve-values.ts | 2 +- package.json | 1 + scripts/check-client-boundary-imports.ts | 233 ++++++++++++++++++ 16 files changed, 328 insertions(+), 55 deletions(-) create mode 100644 apps/sim/hooks/queries/utils/credential-keys.ts create mode 100644 apps/sim/hooks/queries/utils/fetch-credential-set.ts create mode 100644 apps/sim/hooks/queries/utils/fetch-workspace-credentials.ts create mode 100644 scripts/check-client-boundary-imports.ts diff --git a/.claude/rules/sim-architecture.md b/.claude/rules/sim-architecture.md index bc52fd37001..a0cfbfcd050 100644 --- a/.claude/rules/sim-architecture.md +++ b/.claude/rules/sim-architecture.md @@ -38,6 +38,10 @@ packages/ # @sim/* — audit, auth, db, logger, realtime-protocol - `apps/* → packages/*` only. Packages never import from `apps/*`. - `apps/realtime` avoids Next.js, React, the block/tool registry, provider SDKs, and the executor; never add `@/lib/webhooks/providers/*`, `@/executor/*`, `@/blocks/*`, or `@/tools/*` imports to any package it consumes. CI enforces this via `scripts/check-monorepo-boundaries.ts` and `scripts/check-realtime-prune-graph.ts`. +## The `'use client'` server boundary + +Every export of a `'use client'` module becomes a *client reference* on the server — server-evaluated code (RSC pages/layouts, `prefetch.ts`, route handlers, block definitions, triggers) can only *render* it as a component or pass it as a prop, never *call* it (doing so throws at runtime, e.g. `tableKeys.list is not a function`; `next build` does not catch it). Keep server-importable query primitives (key factories, fetchers, mappers, constants) in non-`'use client'` modules — see `.claude/rules/sim-queries.md`. Enforced by `scripts/check-client-boundary-imports.ts`. + ## Feature Organization Features live under `app/workspace/[workspaceId]/`: diff --git a/.claude/rules/sim-queries.md b/.claude/rules/sim-queries.md index 25707c740af..d1db9437ff0 100644 --- a/.claude/rules/sim-queries.md +++ b/.claude/rules/sim-queries.md @@ -27,6 +27,17 @@ Never use inline query keys — always use the factory. **Every identifier the `queryFn` forwards into the fetch MUST appear in the `queryKey`.** (Query-machinery identifiers — `signal`, `pageParam` — are exempt; they aren't fetch-scoping args.) If the fetch is scoped by `workspaceId`, `cursor`, `limit`, an org id, etc., those values must be part of the key — otherwise distinct fetch args share one cache entry (a cross-tenant / per-param cache collision). The lone exception is a globally-unique id used as the key while a second fetch arg is only an authz scope that cannot collide; annotate those with `// rq-lint-allow: `. Enforced by the `key-fetch-arg-drift` check in `scripts/check-react-query-patterns.ts`. +## Server-importable query primitives must NOT live in a `'use client'` module + +Next.js rewrites **every** export of a `'use client'` module into a *client reference* in the server bundle. Server-evaluated code — RSC `page.tsx`/`layout.tsx`, `prefetch.ts`, route handlers, **block definitions**, triggers/workers — can only *render* such an export as a component or pass it as a prop; **calling** one throws at runtime (`Attempted to call X from the server but X is on the client` — for an object export it surfaces as `X.list is not a function`). `next build` does **not** catch this — only SSR/runtime does. + +So any **query-key factory, standalone `requestJson` fetcher, mapper, or constant** that a server module imports must live in a **non-`'use client'`** module: + +- key factories → `hooks/queries/utils/-keys.ts` (see `folder-keys.ts`, `table-keys.ts`, `credential-keys.ts`) +- standalone fetchers/mappers → `hooks/queries/utils/fetch-*.ts` / `*-list-query.ts` (see `fetch-workflow-envelope.ts`, `fetch-credential-set.ts`) + +The `'use client'` hook module then imports these back for its hooks. **Never** define a server-imported factory/fetcher directly in a `'use client'` hooks file — it crashes SSR (this caused the tables-page crash). Enforced for prefetch/route/trigger/block files by `scripts/check-client-boundary-imports.ts` (`bun run check:client-boundary`, run in CI). Escape hatch for a genuinely browser-only path: `// client-boundary-allow: ` on the line above the import. + ## File Structure ```typescript diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 40f83645146..c4ffd7449ea 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -122,6 +122,9 @@ jobs: - name: React Query pattern audit run: bun run check:react-query + - name: Client boundary import audit + run: bun run check:client-boundary + - name: Verify realtime prune graph run: bun run check:realtime-prune diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secrets-manager/secrets-manager.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secrets-manager/secrets-manager.tsx index e2581174df8..ffb2becd85a 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secrets-manager/secrets-manager.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secrets-manager/secrets-manager.tsx @@ -17,11 +17,7 @@ import type { WorkspaceEnvironmentData } from '@/lib/environment/api' import { UnsavedChangesModal } from '@/app/workspace/[workspaceId]/components/credential-detail' import { SecretValueField } from '@/app/workspace/[workspaceId]/settings/components/secrets/components/secret-value-field' import { isValidEnvVarName } from '@/executor/constants' -import { - useWorkspaceCredentials, - type WorkspaceCredential, - workspaceCredentialKeys, -} from '@/hooks/queries/credentials' +import { useWorkspaceCredentials, type WorkspaceCredential } from '@/hooks/queries/credentials' import { usePersonalEnvironment, useRemoveWorkspaceEnvironment, @@ -29,6 +25,7 @@ import { useUpsertWorkspaceEnvironment, useWorkspaceEnvironment, } from '@/hooks/queries/environment' +import { workspaceCredentialKeys } from '@/hooks/queries/utils/credential-keys' import { useWorkspacePermissionsQuery } from '@/hooks/queries/workspace' import { useSettingsDirtyStore } from '@/stores/settings/dirty/store' diff --git a/apps/sim/blocks/blocks/credential.ts b/apps/sim/blocks/blocks/credential.ts index fc324d721a4..830a8b65b65 100644 --- a/apps/sim/blocks/blocks/credential.ts +++ b/apps/sim/blocks/blocks/credential.ts @@ -2,7 +2,8 @@ import { CredentialIcon } from '@/components/icons' import { getServiceConfigByProviderId } from '@/lib/oauth/utils' import { getQueryClient } from '@/app/_shell/providers/get-query-client' import type { BlockConfig } from '@/blocks/types' -import { fetchWorkspaceCredentialList, workspaceCredentialKeys } from '@/hooks/queries/credentials' +import { workspaceCredentialKeys } from '@/hooks/queries/utils/credential-keys' +import { fetchWorkspaceCredentialList } from '@/hooks/queries/utils/fetch-workspace-credentials' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' interface CredentialBlockOutput { diff --git a/apps/sim/hooks/queries/credential-sets.ts b/apps/sim/hooks/queries/credential-sets.ts index d828e42544b..2a1086444a5 100644 --- a/apps/sim/hooks/queries/credential-sets.ts +++ b/apps/sim/hooks/queries/credential-sets.ts @@ -19,7 +19,6 @@ import { createCredentialSetContract, createCredentialSetInvitationContract, deleteCredentialSetContract, - getCredentialSetContract, leaveCredentialSetContract, listCredentialSetInvitationDetailsContract, listCredentialSetInvitationsContract, @@ -29,6 +28,7 @@ import { removeCredentialSetMemberContract, resendCredentialSetInvitationContract, } from '@/lib/api/contracts' +import { fetchCredentialSetById } from '@/hooks/queries/utils/fetch-credential-set' export type { CreateCredentialSetData, @@ -76,18 +76,6 @@ export function useCredentialSets(organizationId?: string, enabled = true) { }) } -export async function fetchCredentialSetById( - id: string, - signal?: AbortSignal -): Promise { - if (!id) return null - const data = await requestJson(getCredentialSetContract, { - params: { id }, - signal, - }) - return data.credentialSet ?? null -} - export function useCredentialSetDetail(id?: string, enabled = true) { return useQuery({ queryKey: credentialSetKeys.detail(id), diff --git a/apps/sim/hooks/queries/credentials.ts b/apps/sim/hooks/queries/credentials.ts index a2dda84d906..9e20147faab 100644 --- a/apps/sim/hooks/queries/credentials.ts +++ b/apps/sim/hooks/queries/credentials.ts @@ -20,6 +20,8 @@ import { type WorkspaceCredentialType, } from '@/lib/api/contracts' import { environmentKeys } from '@/hooks/queries/environment' +import { workspaceCredentialKeys } from '@/hooks/queries/utils/credential-keys' +import { fetchWorkspaceCredentialList } from '@/hooks/queries/utils/fetch-workspace-credentials' /** * Key prefix for OAuth credential queries. @@ -34,38 +36,6 @@ export type { WorkspaceCredentialType, } -export const workspaceCredentialKeys = { - all: ['workspaceCredentials'] as const, - lists: () => [...workspaceCredentialKeys.all, 'list'] as const, - list: (workspaceId?: string, type?: string, providerId?: string) => - [ - ...workspaceCredentialKeys.lists(), - workspaceId ?? 'none', - type ?? 'all', - providerId ?? 'all', - ] as const, - details: () => [...workspaceCredentialKeys.all, 'detail'] as const, - detail: (credentialId?: string) => - [...workspaceCredentialKeys.details(), credentialId ?? 'none'] as const, - members: (credentialId?: string) => - [...workspaceCredentialKeys.detail(credentialId), 'members'] as const, -} - -/** - * Fetch workspace credential list from API. - * Used by the prefetch function for hover-based cache warming. - */ -export async function fetchWorkspaceCredentialList( - workspaceId: string, - signal?: AbortSignal -): Promise { - const data = await requestJson(listWorkspaceCredentialsContract, { - query: { workspaceId }, - signal, - }) - return data.credentials ?? [] -} - /** * Prefetch workspace credentials into a QueryClient cache. * Use on hover to warm data before navigation. diff --git a/apps/sim/hooks/queries/invitations.ts b/apps/sim/hooks/queries/invitations.ts index 17e0bfe3963..86a772a5e70 100644 --- a/apps/sim/hooks/queries/invitations.ts +++ b/apps/sim/hooks/queries/invitations.ts @@ -11,8 +11,8 @@ import { resendInvitationContract, } from '@/lib/api/contracts/invitations' import { updateWorkspacePermissionsContract } from '@/lib/api/contracts/workspaces' -import { workspaceCredentialKeys } from '@/hooks/queries/credentials' import { organizationKeys } from '@/hooks/queries/organization' +import { workspaceCredentialKeys } from '@/hooks/queries/utils/credential-keys' import { workspaceKeys } from '@/hooks/queries/workspace' export const invitationKeys = { diff --git a/apps/sim/hooks/queries/organization.ts b/apps/sim/hooks/queries/organization.ts index 11352d88b97..f005c7eaeed 100644 --- a/apps/sim/hooks/queries/organization.ts +++ b/apps/sim/hooks/queries/organization.ts @@ -42,8 +42,8 @@ import { import { client } from '@/lib/auth/auth-client' import { isEnterprise, isPaid, isTeam } from '@/lib/billing/plan-helpers' import { hasPaidSubscriptionStatus } from '@/lib/billing/subscriptions/utils' -import { workspaceCredentialKeys } from '@/hooks/queries/credentials' import { subscriptionKeys } from '@/hooks/queries/subscription' +import { workspaceCredentialKeys } from '@/hooks/queries/utils/credential-keys' import { workspaceKeys } from '@/hooks/queries/workspace' const logger = createLogger('OrganizationQueries') diff --git a/apps/sim/hooks/queries/utils/credential-keys.ts b/apps/sim/hooks/queries/utils/credential-keys.ts new file mode 100644 index 00000000000..728cbd34ceb --- /dev/null +++ b/apps/sim/hooks/queries/utils/credential-keys.ts @@ -0,0 +1,24 @@ +/** + * React Query key factory for workspace credentials. + * + * Lives in this standalone (non-`'use client'`) module — like + * {@link file://./folder-keys.ts} — so server-evaluated code (block + * definitions, server prefetch) can import it without pulling client-reference + * stubs from the `'use client'` `@/hooks/queries/credentials` module. + */ +export const workspaceCredentialKeys = { + all: ['workspaceCredentials'] as const, + lists: () => [...workspaceCredentialKeys.all, 'list'] as const, + list: (workspaceId?: string, type?: string, providerId?: string) => + [ + ...workspaceCredentialKeys.lists(), + workspaceId ?? 'none', + type ?? 'all', + providerId ?? 'all', + ] as const, + details: () => [...workspaceCredentialKeys.all, 'detail'] as const, + detail: (credentialId?: string) => + [...workspaceCredentialKeys.details(), credentialId ?? 'none'] as const, + members: (credentialId?: string) => + [...workspaceCredentialKeys.detail(credentialId), 'members'] as const, +} diff --git a/apps/sim/hooks/queries/utils/fetch-credential-set.ts b/apps/sim/hooks/queries/utils/fetch-credential-set.ts new file mode 100644 index 00000000000..c9523d53633 --- /dev/null +++ b/apps/sim/hooks/queries/utils/fetch-credential-set.ts @@ -0,0 +1,21 @@ +import { requestJson } from '@/lib/api/client/request' +import { type CredentialSet, getCredentialSetContract } from '@/lib/api/contracts' + +/** + * Fetches a credential set by id (returns `null` for an empty id). + * + * Lives in this standalone (non-`'use client'`) module so server-reachable + * workflow-comparison helpers can import it without pulling client-reference + * stubs from the `'use client'` `@/hooks/queries/credential-sets` module. + */ +export async function fetchCredentialSetById( + id: string, + signal?: AbortSignal +): Promise { + if (!id) return null + const data = await requestJson(getCredentialSetContract, { + params: { id }, + signal, + }) + return data.credentialSet ?? null +} diff --git a/apps/sim/hooks/queries/utils/fetch-workspace-credentials.ts b/apps/sim/hooks/queries/utils/fetch-workspace-credentials.ts new file mode 100644 index 00000000000..9fd8efd7b6f --- /dev/null +++ b/apps/sim/hooks/queries/utils/fetch-workspace-credentials.ts @@ -0,0 +1,20 @@ +import { requestJson } from '@/lib/api/client/request' +import { listWorkspaceCredentialsContract, type WorkspaceCredential } from '@/lib/api/contracts' + +/** + * Fetches the workspace credential list. + * + * Lives in this standalone (non-`'use client'`) module so block definitions and + * server prefetch can import it without pulling client-reference stubs from the + * `'use client'` `@/hooks/queries/credentials` module. + */ +export async function fetchWorkspaceCredentialList( + workspaceId: string, + signal?: AbortSignal +): Promise { + const data = await requestJson(listWorkspaceCredentialsContract, { + query: { workspaceId }, + signal, + }) + return data.credentials ?? [] +} diff --git a/apps/sim/lib/workflows/comparison/format-description.test.ts b/apps/sim/lib/workflows/comparison/format-description.test.ts index f186a9d5a4f..22fb5b2fa05 100644 --- a/apps/sim/lib/workflows/comparison/format-description.test.ts +++ b/apps/sim/lib/workflows/comparison/format-description.test.ts @@ -36,7 +36,7 @@ vi.mock('@/lib/workflows/subblocks/context', () => ({ buildSelectorContextFromBlock: vi.fn(() => ({})), })) -vi.mock('@/hooks/queries/credential-sets', () => ({ +vi.mock('@/hooks/queries/utils/fetch-credential-set', () => ({ fetchCredentialSetById: vi.fn(), })) diff --git a/apps/sim/lib/workflows/comparison/resolve-values.ts b/apps/sim/lib/workflows/comparison/resolve-values.ts index bc9d6c7a753..fa9dbb37f9a 100644 --- a/apps/sim/lib/workflows/comparison/resolve-values.ts +++ b/apps/sim/lib/workflows/comparison/resolve-values.ts @@ -4,8 +4,8 @@ import { buildSelectorContextFromBlock } from '@/lib/workflows/subblocks/context import { getBlock } from '@/blocks/registry' import { SELECTOR_TYPES_HYDRATION_REQUIRED, type SubBlockConfig } from '@/blocks/types' import { CREDENTIAL_SET, isUuid } from '@/executor/constants' -import { fetchCredentialSetById } from '@/hooks/queries/credential-sets' import { fetchOAuthCredentialDetail } from '@/hooks/queries/oauth/oauth-credentials' +import { fetchCredentialSetById } from '@/hooks/queries/utils/fetch-credential-set' import { getSelectorDefinition, loadAllSelectorOptions } from '@/hooks/selectors/registry' import { resolveSelectorForSubBlock } from '@/hooks/selectors/resolution' import type { SelectorContext, SelectorKey } from '@/hooks/selectors/types' diff --git a/package.json b/package.json index ce15ab78107..54f25a8da65 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "check:realtime-prune": "bun run scripts/check-realtime-prune-graph.ts", "check:zustand-v5": "bun run scripts/check-zustand-v5-selectors.ts", "check:react-query": "bun run scripts/check-react-query-patterns.ts --check", + "check:client-boundary": "bun run scripts/check-client-boundary-imports.ts --check", "check:utils": "bun run scripts/check-utils-enforcement.ts", "check:migrations": "bun run scripts/check-migrations-safety.ts", "mship-contracts:generate": "bun run scripts/sync-mothership-stream-contract.ts", diff --git a/scripts/check-client-boundary-imports.ts b/scripts/check-client-boundary-imports.ts new file mode 100644 index 00000000000..6d0ec2d6758 --- /dev/null +++ b/scripts/check-client-boundary-imports.ts @@ -0,0 +1,233 @@ +#!/usr/bin/env bun +/** + * Guards against the Next.js `'use client'` server-import foot-gun. + * + * Next.js rewrites EVERY export of a `'use client'` module into a client + * reference in the server bundle. Server-evaluated code can only *render* such + * an export as a component or pass it as a prop — *calling* one throws at + * runtime ("Attempted to call X from the server but X is on the client"). The + * crash for an object export looks like `tableKeys.list is not a function`. + * `next build` does NOT catch this; only SSR/runtime does. + * + * This script flags any **value** import (not `import type`) that resolves to a + * `'use client'` module from a server-evaluated, non-JSX surface — the places + * that never legitimately render a client component and so only ever import a + * client module to (illegally) call its values: + * + * - `apps/sim/app/** /prefetch*.ts` (RSC server prefetch) + * - `apps/sim/app/api/** /route.ts(x)` (route handlers) + * - `apps/sim/triggers/**` (trigger.dev tasks/pollers/webhooks) + * - `apps/sim/blocks/**` (block definitions — evaluated server-side) + * + * Fix: move the imported query-key factory / standalone fetcher / mapper / + * constant into a non-`'use client'` module (e.g. `hooks/queries/utils/*-keys.ts` + * or `hooks/queries/utils/fetch-*.ts`) and import it from there. See the rule in + * `.claude/rules/sim-queries.md`. + * + * Escape hatch: `// client-boundary-allow: ` on the line directly above + * the import (reason required). Use only for a genuinely browser-only code path. + * + * Usage: + * bun run scripts/check-client-boundary-imports.ts # report + * bun run scripts/check-client-boundary-imports.ts --check # CI gate (fail on any) + */ +import { readdir, readFile } from 'node:fs/promises' +import path from 'node:path' + +const ROOT = path.resolve(import.meta.dir, '..') +const APP_DIR = path.join(ROOT, 'apps/sim') + +/** Server-evaluated, non-JSX surfaces. A file matches if its path passes one. */ +function isServerSurface(rel: string): boolean { + if (/(^|\/)prefetch[^/]*\.ts$/.test(rel)) return true + if (/^app\/api\/.+\/route\.tsx?$/.test(rel)) return true + if (/^triggers\//.test(rel)) return true + if (/^blocks\//.test(rel)) return true + return false +} + +const SOURCE_EXTENSIONS = ['.ts', '.tsx'] +const ALLOW_DIRECTIVE = 'client-boundary-allow' + +async function listFiles(dir: string): Promise { + const out: string[] = [] + let entries: Awaited> + try { + entries = await readdir(dir, { withFileTypes: true }) + } catch { + return out + } + for (const entry of entries) { + const full = path.join(dir, entry.name) + if (entry.isDirectory()) { + if (entry.name === 'node_modules' || entry.name === '.next') continue + out.push(...(await listFiles(full))) + } else if (SOURCE_EXTENSIONS.includes(path.extname(entry.name))) { + out.push(full) + } + } + return out +} + +const useClientCache = new Map() + +async function isUseClientModule(absFile: string): Promise { + const cached = useClientCache.get(absFile) + if (cached !== undefined) return cached + let content: string + try { + content = await readFile(absFile, 'utf8') + } catch { + useClientCache.set(absFile, false) + return false + } + // The directive must be the first statement (comments/blank lines may precede it). + let isClient = false + for (const raw of content.split('\n')) { + const line = raw.trim() + if (line === '' || line.startsWith('//') || line.startsWith('/*') || line.startsWith('*')) { + continue + } + isClient = line === "'use client'" || line === '"use client"' + break + } + useClientCache.set(absFile, isClient) + return isClient +} + +/** Resolve an import specifier to an absolute source file, or null if external/unresolved. */ +async function resolveSpecifier(spec: string, fromFile: string): Promise { + let base: string + if (spec.startsWith('@/')) { + base = path.join(APP_DIR, spec.slice(2)) + } else if (spec.startsWith('./') || spec.startsWith('../')) { + base = path.resolve(path.dirname(fromFile), spec) + } else { + return null // external package + } + const candidates = [ + base, + ...SOURCE_EXTENSIONS.map((ext) => base + ext), + ...SOURCE_EXTENSIONS.map((ext) => path.join(base, `index${ext}`)), + ] + for (const candidate of candidates) { + if (!SOURCE_EXTENSIONS.includes(path.extname(candidate))) continue + try { + await readFile(candidate, 'utf8') + return candidate + } catch {} + } + return null +} + +interface ImportInfo { + line: number + specifier: string + clause: string +} + +/** Parse `import ... from '...'` statements, skipping side-effect-only imports. */ +function parseImports(content: string): ImportInfo[] { + const lines = content.split('\n') + const imports: ImportInfo[] = [] + const re = /^\s*import\s+([\s\S]*?)\s+from\s+['"]([^'"]+)['"]/ + for (let i = 0; i < lines.length; i++) { + if (!/^\s*import\b/.test(lines[i]) || !lines[i].includes('import')) continue + // Join up to 12 following lines to capture multi-line import clauses. + const block = lines.slice(i, i + 12).join('\n') + const match = re.exec(block) + if (!match) continue + imports.push({ line: i + 1, clause: match[1], specifier: match[2] }) + } + return imports +} + +/** True when the import brings in at least one runtime VALUE (not purely types). */ +function importsAValue(clause: string): boolean { + const trimmed = clause.trim() + if (trimmed.startsWith('type ')) return false // `import type { ... }` / `import type X` + const braceStart = trimmed.indexOf('{') + // A default or namespace binding outside the braces is always a value. + const beforeBrace = braceStart === -1 ? trimmed : trimmed.slice(0, braceStart) + if (beforeBrace.replace(/[,\s]/g, '').length > 0) return true + if (braceStart === -1) return true + const inner = trimmed.slice(braceStart + 1, trimmed.lastIndexOf('}')) + // A named import is a value unless every member is `type`-prefixed. + return inner + .split(',') + .map((s) => s.trim()) + .filter(Boolean) + .some((member) => !member.startsWith('type ')) +} + +function hasAllowDirective(content: string, importLine: number): boolean { + const lines = content.split('\n') + for (let i = importLine - 2; i >= 0 && i >= importLine - 5; i--) { + const line = lines[i]?.trim() ?? '' + if (line === '' || line.startsWith('//') || line.startsWith('*') || line.startsWith('/*')) { + if (line.includes(ALLOW_DIRECTIVE)) { + const reason = + line + .split(ALLOW_DIRECTIVE)[1] + ?.replace(/^[:\s]+/, '') + .trim() ?? '' + return reason.length > 0 + } + continue + } + break + } + return false +} + +interface Violation { + file: string + line: number + specifier: string +} + +async function main() { + const checkMode = process.argv.includes('--check') + const allFiles = await listFiles(APP_DIR) + const violations: Violation[] = [] + + for (const absFile of allFiles) { + const rel = path.relative(APP_DIR, absFile) + if (!isServerSurface(rel)) continue + // A server file that is itself `'use client'` is a client component — out of scope. + if (await isUseClientModule(absFile)) continue + + const content = await readFile(absFile, 'utf8') + for (const imp of parseImports(content)) { + if (!importsAValue(imp.clause)) continue + const resolved = await resolveSpecifier(imp.specifier, absFile) + if (!resolved) continue + if (!(await isUseClientModule(resolved))) continue + if (hasAllowDirective(content, imp.line)) continue + violations.push({ file: rel, line: imp.line, specifier: imp.specifier }) + } + } + + if (violations.length === 0) { + console.log( + "✓ Client-boundary import check passed (no server file imports a value from a 'use client' module)." + ) + return + } + + console.error( + `\n✗ ${violations.length} server file(s) import a runtime value from a 'use client' module.\n` + + ` On the server these resolve to client-reference stubs and throw when called (e.g. 'X.list is not a function').\n` + + ` Move the imported factory/fetcher/constant into a non-'use client' module (hooks/queries/utils/*-keys.ts or fetch-*.ts).\n` + + ` See .claude/rules/sim-queries.md. Escape hatch: // ${ALLOW_DIRECTIVE}: above the import.\n` + ) + for (const v of violations) { + console.error(` ${v.file}:${v.line} imports from '${v.specifier}'`) + } + if (checkMode) process.exit(1) +} + +main().catch((error) => { + console.error(error) + process.exit(1) +}) From 9bd8f149a52bf36b57e0df93391f3a21f45a45f2 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 20:38:04 -0700 Subject: [PATCH 13/14] fix(workspace): add granular error boundaries to 7 more workspace segments (#5207) Adds error.tsx (reusing the shared ErrorState) to home, integrations, knowledge/[id], skills, settings, scheduled-tasks, and chat/[chatId] so a crash in any of these panels stays scoped to the panel and offers a retry, instead of bubbling to the generic workspace-level boundary. --- .../[workspaceId]/chat/[chatId]/error.tsx | 15 +++++++++++++++ .../app/workspace/[workspaceId]/home/error.tsx | 15 +++++++++++++++ .../[workspaceId]/integrations/error.tsx | 15 +++++++++++++++ .../[workspaceId]/knowledge/[id]/error.tsx | 15 +++++++++++++++ .../[workspaceId]/scheduled-tasks/error.tsx | 15 +++++++++++++++ .../workspace/[workspaceId]/settings/error.tsx | 15 +++++++++++++++ .../app/workspace/[workspaceId]/skills/error.tsx | 15 +++++++++++++++ 7 files changed, 105 insertions(+) create mode 100644 apps/sim/app/workspace/[workspaceId]/chat/[chatId]/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/home/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/integrations/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/knowledge/[id]/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/scheduled-tasks/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/error.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/skills/error.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/chat/[chatId]/error.tsx b/apps/sim/app/workspace/[workspaceId]/chat/[chatId]/error.tsx new file mode 100644 index 00000000000..1ebfc47facb --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/chat/[chatId]/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function ChatError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/error.tsx b/apps/sim/app/workspace/[workspaceId]/home/error.tsx new file mode 100644 index 00000000000..03d205fdcd0 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function HomeError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/error.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/error.tsx new file mode 100644 index 00000000000..706d7eea055 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/integrations/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function IntegrationsError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/error.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/error.tsx new file mode 100644 index 00000000000..91dbaadb2c5 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function KnowledgeBaseError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/scheduled-tasks/error.tsx b/apps/sim/app/workspace/[workspaceId]/scheduled-tasks/error.tsx new file mode 100644 index 00000000000..4fc75c7b937 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/scheduled-tasks/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function ScheduledTasksError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/error.tsx b/apps/sim/app/workspace/[workspaceId]/settings/error.tsx new file mode 100644 index 00000000000..02a18fd3362 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function SettingsError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/skills/error.tsx b/apps/sim/app/workspace/[workspaceId]/skills/error.tsx new file mode 100644 index 00000000000..9a860682257 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/skills/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function SkillsError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} From e748a64ff4f3a83675882a529f5b355f668fa50d Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 24 Jun 2026 20:43:52 -0700 Subject: [PATCH 14/14] refactor(realtime): type the socket event-handler boundary with @sim/realtime-protocol (#5208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(realtime): type the socket event-handler boundary with @sim/realtime-protocol Replace the (data: any) event-handler types in socket-provider.tsx with precise broadcast types that mirror the exact payloads emitted by the realtime Socket.IO server (apps/realtime/src/handlers/** and rooms/**). Add @sim/realtime-protocol/events with the canonical wire types for the broadcast/confirmation events the server emits: WorkflowOperationBroadcast, SubblockUpdateBroadcast, VariableUpdateBroadcast, CursorUpdateBroadcast, SelectionUpdateBroadcast, the four workflow-lifecycle broadcasts, and OperationConfirmed/Failed. Typing change only; zero runtime/logic changes. Store-internal any (rehydrate state, subblock map, emit payloads) is left untouched as out of scope. * fix(realtime): type cursor-update broadcast cursor as nullable The client emits 'cursor-update' with { cursor: null } when a remote user's cursor leaves the canvas, and the server re-broadcasts it verbatim, so receivers genuinely get cursor: null. Type CursorUpdateBroadcast.cursor as CursorPosition | null to match the wire. (selection stays non-null — it signals absence via type: 'none', never null.) --- .../workspace/providers/socket-provider.tsx | 110 +++++++++------- packages/realtime-protocol/package.json | 4 + packages/realtime-protocol/src/events.ts | 122 ++++++++++++++++++ 3 files changed, 189 insertions(+), 47 deletions(-) create mode 100644 packages/realtime-protocol/src/events.ts diff --git a/apps/sim/app/workspace/providers/socket-provider.tsx b/apps/sim/app/workspace/providers/socket-provider.tsx index 47587fe4f0e..3e5d6fa9040 100644 --- a/apps/sim/app/workspace/providers/socket-provider.tsx +++ b/apps/sim/app/workspace/providers/socket-provider.tsx @@ -11,6 +11,19 @@ import { useState, } from 'react' import { createLogger } from '@sim/logger' +import type { + CursorUpdateBroadcast, + OperationConfirmedBroadcast, + OperationFailedBroadcast, + SelectionUpdateBroadcast, + SubblockUpdateBroadcast, + VariableUpdateBroadcast, + WorkflowDeletedBroadcast, + WorkflowDeployedBroadcast, + WorkflowOperationBroadcast, + WorkflowRevertedBroadcast, + WorkflowUpdatedBroadcast, +} from '@sim/realtime-protocol/events' import { generateId } from '@sim/utils/id' import { backoffWithJitter } from '@sim/utils/retry' import { useParams } from 'next/navigation' @@ -92,18 +105,18 @@ interface SocketContextType { emitCursorUpdate: (cursor: { x: number; y: number } | null) => void emitSelectionUpdate: (selection: { type: 'block' | 'edge' | 'none'; id?: string }) => void - onWorkflowOperation: (handler: (data: any) => void) => void - onSubblockUpdate: (handler: (data: any) => void) => void - onVariableUpdate: (handler: (data: any) => void) => void - - onCursorUpdate: (handler: (data: any) => void) => void - onSelectionUpdate: (handler: (data: any) => void) => void - onWorkflowDeleted: (handler: (data: any) => void) => void - onWorkflowReverted: (handler: (data: any) => void) => void - onWorkflowUpdated: (handler: (data: any) => void) => void - onWorkflowDeployed: (handler: (data: any) => void) => void - onOperationConfirmed: (handler: (data: any) => void) => void - onOperationFailed: (handler: (data: any) => void) => void + onWorkflowOperation: (handler: (data: WorkflowOperationBroadcast) => void) => void + onSubblockUpdate: (handler: (data: SubblockUpdateBroadcast) => void) => void + onVariableUpdate: (handler: (data: VariableUpdateBroadcast) => void) => void + + onCursorUpdate: (handler: (data: CursorUpdateBroadcast) => void) => void + onSelectionUpdate: (handler: (data: SelectionUpdateBroadcast) => void) => void + onWorkflowDeleted: (handler: (data: WorkflowDeletedBroadcast) => void) => void + onWorkflowReverted: (handler: (data: WorkflowRevertedBroadcast) => void) => void + onWorkflowUpdated: (handler: (data: WorkflowUpdatedBroadcast) => void) => void + onWorkflowDeployed: (handler: (data: WorkflowDeployedBroadcast) => void) => void + onOperationConfirmed: (handler: (data: OperationConfirmedBroadcast) => void) => void + onOperationFailed: (handler: (data: OperationFailedBroadcast) => void) => void } const SocketContext = createContext({ @@ -173,17 +186,17 @@ export function SocketProvider({ children, user }: SocketProviderProps) { explicitWorkflowIdRef.current = explicitWorkflowId const eventHandlers = useRef<{ - workflowOperation?: (data: any) => void - subblockUpdate?: (data: any) => void - variableUpdate?: (data: any) => void - cursorUpdate?: (data: any) => void - selectionUpdate?: (data: any) => void - workflowDeleted?: (data: any) => void - workflowReverted?: (data: any) => void - workflowUpdated?: (data: any) => void - workflowDeployed?: (data: any) => void - operationConfirmed?: (data: any) => void - operationFailed?: (data: any) => void + workflowOperation?: (data: WorkflowOperationBroadcast) => void + subblockUpdate?: (data: SubblockUpdateBroadcast) => void + variableUpdate?: (data: VariableUpdateBroadcast) => void + cursorUpdate?: (data: CursorUpdateBroadcast) => void + selectionUpdate?: (data: SelectionUpdateBroadcast) => void + workflowDeleted?: (data: WorkflowDeletedBroadcast) => void + workflowReverted?: (data: WorkflowRevertedBroadcast) => void + workflowUpdated?: (data: WorkflowUpdatedBroadcast) => void + workflowDeployed?: (data: WorkflowDeployedBroadcast) => void + operationConfirmed?: (data: OperationConfirmedBroadcast) => void + operationFailed?: (data: OperationFailedBroadcast) => void }>({}) const positionUpdateTimeouts = useRef>(new Map()) @@ -555,19 +568,19 @@ export function SocketProvider({ children, user }: SocketProviderProps) { executeJoinCommands(result.commands) }) - socketInstance.on('workflow-operation', (data) => { + socketInstance.on('workflow-operation', (data: WorkflowOperationBroadcast) => { eventHandlers.current.workflowOperation?.(data) }) - socketInstance.on('subblock-update', (data) => { + socketInstance.on('subblock-update', (data: SubblockUpdateBroadcast) => { eventHandlers.current.subblockUpdate?.(data) }) - socketInstance.on('variable-update', (data) => { + socketInstance.on('variable-update', (data: VariableUpdateBroadcast) => { eventHandlers.current.variableUpdate?.(data) }) - socketInstance.on('workflow-deleted', (data) => { + socketInstance.on('workflow-deleted', (data: WorkflowDeletedBroadcast) => { logger.warn(`Workflow ${data.workflowId} has been deleted`) const result = joinControllerRef.current.handleWorkflowDeleted(data.workflowId) if (result.shouldClearCurrent) { @@ -577,17 +590,17 @@ export function SocketProvider({ children, user }: SocketProviderProps) { eventHandlers.current.workflowDeleted?.(data) }) - socketInstance.on('workflow-reverted', (data) => { + socketInstance.on('workflow-reverted', (data: WorkflowRevertedBroadcast) => { logger.info(`Workflow ${data.workflowId} has been reverted to deployed state`) eventHandlers.current.workflowReverted?.(data) }) - socketInstance.on('workflow-updated', (data) => { + socketInstance.on('workflow-updated', (data: WorkflowUpdatedBroadcast) => { logger.info(`Workflow ${data.workflowId} has been updated externally`) eventHandlers.current.workflowUpdated?.(data) }) - socketInstance.on('workflow-deployed', (data) => { + socketInstance.on('workflow-deployed', (data: WorkflowDeployedBroadcast) => { logger.info(`Workflow ${data.workflowId} deployment state changed`) eventHandlers.current.workflowDeployed?.(data) }) @@ -647,17 +660,17 @@ export function SocketProvider({ children, user }: SocketProviderProps) { return true } - socketInstance.on('operation-confirmed', (data) => { + socketInstance.on('operation-confirmed', (data: OperationConfirmedBroadcast) => { logger.debug('Operation confirmed', { operationId: data.operationId }) eventHandlers.current.operationConfirmed?.(data) }) - socketInstance.on('operation-failed', (data) => { + socketInstance.on('operation-failed', (data: OperationFailedBroadcast) => { logger.warn('Operation failed', { operationId: data.operationId, error: data.error }) eventHandlers.current.operationFailed?.(data) }) - socketInstance.on('cursor-update', (data) => { + socketInstance.on('cursor-update', (data: CursorUpdateBroadcast) => { if (!isWorkflowVisible()) { return } @@ -675,7 +688,7 @@ export function SocketProvider({ children, user }: SocketProviderProps) { eventHandlers.current.cursorUpdate?.(data) }) - socketInstance.on('selection-update', (data) => { + socketInstance.on('selection-update', (data: SelectionUpdateBroadcast) => { if (!isWorkflowVisible()) { return } @@ -1045,47 +1058,50 @@ export function SocketProvider({ children, user }: SocketProviderProps) { [socket, currentWorkflowId, isWorkflowVisible] ) - const onWorkflowOperation = useCallback((handler: (data: any) => void) => { + const onWorkflowOperation = useCallback((handler: (data: WorkflowOperationBroadcast) => void) => { eventHandlers.current.workflowOperation = handler }, []) - const onSubblockUpdate = useCallback((handler: (data: any) => void) => { + const onSubblockUpdate = useCallback((handler: (data: SubblockUpdateBroadcast) => void) => { eventHandlers.current.subblockUpdate = handler }, []) - const onVariableUpdate = useCallback((handler: (data: any) => void) => { + const onVariableUpdate = useCallback((handler: (data: VariableUpdateBroadcast) => void) => { eventHandlers.current.variableUpdate = handler }, []) - const onCursorUpdate = useCallback((handler: (data: any) => void) => { + const onCursorUpdate = useCallback((handler: (data: CursorUpdateBroadcast) => void) => { eventHandlers.current.cursorUpdate = handler }, []) - const onSelectionUpdate = useCallback((handler: (data: any) => void) => { + const onSelectionUpdate = useCallback((handler: (data: SelectionUpdateBroadcast) => void) => { eventHandlers.current.selectionUpdate = handler }, []) - const onWorkflowDeleted = useCallback((handler: (data: any) => void) => { + const onWorkflowDeleted = useCallback((handler: (data: WorkflowDeletedBroadcast) => void) => { eventHandlers.current.workflowDeleted = handler }, []) - const onWorkflowReverted = useCallback((handler: (data: any) => void) => { + const onWorkflowReverted = useCallback((handler: (data: WorkflowRevertedBroadcast) => void) => { eventHandlers.current.workflowReverted = handler }, []) - const onWorkflowUpdated = useCallback((handler: (data: any) => void) => { + const onWorkflowUpdated = useCallback((handler: (data: WorkflowUpdatedBroadcast) => void) => { eventHandlers.current.workflowUpdated = handler }, []) - const onWorkflowDeployed = useCallback((handler: (data: any) => void) => { + const onWorkflowDeployed = useCallback((handler: (data: WorkflowDeployedBroadcast) => void) => { eventHandlers.current.workflowDeployed = handler }, []) - const onOperationConfirmed = useCallback((handler: (data: any) => void) => { - eventHandlers.current.operationConfirmed = handler - }, []) + const onOperationConfirmed = useCallback( + (handler: (data: OperationConfirmedBroadcast) => void) => { + eventHandlers.current.operationConfirmed = handler + }, + [] + ) - const onOperationFailed = useCallback((handler: (data: any) => void) => { + const onOperationFailed = useCallback((handler: (data: OperationFailedBroadcast) => void) => { eventHandlers.current.operationFailed = handler }, []) diff --git a/packages/realtime-protocol/package.json b/packages/realtime-protocol/package.json index 1bf2411e848..4f026e9d2ac 100644 --- a/packages/realtime-protocol/package.json +++ b/packages/realtime-protocol/package.json @@ -14,6 +14,10 @@ "types": "./src/constants.ts", "default": "./src/constants.ts" }, + "./events": { + "types": "./src/events.ts", + "default": "./src/events.ts" + }, "./schemas": { "types": "./src/schemas.ts", "default": "./src/schemas.ts" diff --git a/packages/realtime-protocol/src/events.ts b/packages/realtime-protocol/src/events.ts new file mode 100644 index 00000000000..95f630de80e --- /dev/null +++ b/packages/realtime-protocol/src/events.ts @@ -0,0 +1,122 @@ +import type { OperationTarget, SocketOperation } from './constants' + +/** + * Wire types for the broadcast/confirmation events the realtime Socket.IO server + * emits to clients. These mirror the exact object literals emitted by + * `apps/realtime/src/handlers/**` and `apps/realtime/src/rooms/**`, and are the + * canonical types consumed by the client socket transport + * (`apps/sim/app/workspace/providers/socket-provider.tsx`). + * + * Payload bodies that the transport forwards opaquely are typed `unknown` rather + * than a concrete operation union, because the transport never narrows them — the + * collaborative-workflow consumer dispatches on `operation`/`target` itself. + */ + +/** A live-presence cursor position broadcast over the socket. */ +export interface CursorPosition { + x: number + y: number +} + +/** A live-presence selection broadcast over the socket. */ +export interface PresenceSelection { + type: 'block' | 'edge' | 'none' + id?: string +} + +/** + * `workflow-operation` broadcast. The server re-broadcasts the originating + * operation envelope plus sender identity and operation metadata. + */ +export interface WorkflowOperationBroadcast { + operation: SocketOperation | string + target: OperationTarget | string + payload: unknown + timestamp: number + senderId: string + userId: string + userName: string + metadata: { + workflowId: string + operationId: string + isPositionUpdate?: boolean + isBatchPositionUpdate?: boolean + } +} + +/** `subblock-update` broadcast. */ +export interface SubblockUpdateBroadcast { + workflowId: string + blockId: string + subblockId: string + value: unknown + timestamp: number +} + +/** `variable-update` broadcast. */ +export interface VariableUpdateBroadcast { + workflowId: string + variableId: string + field: string + value: unknown + timestamp: number +} + +/** `cursor-update` presence broadcast for a single remote user. */ +export interface CursorUpdateBroadcast { + socketId: string + userId: string + userName: string + avatarUrl?: string | null + /** `null` when the remote user's cursor leaves the canvas (the client emits `{ cursor: null }`). */ + cursor: CursorPosition | null +} + +/** `selection-update` presence broadcast for a single remote user. */ +export interface SelectionUpdateBroadcast { + socketId: string + userId: string + userName: string + avatarUrl?: string | null + selection: PresenceSelection +} + +/** `workflow-deleted` lifecycle broadcast. */ +export interface WorkflowDeletedBroadcast { + workflowId: string + message: string + timestamp: number +} + +/** `workflow-reverted` lifecycle broadcast. */ +export interface WorkflowRevertedBroadcast { + workflowId: string + message: string + timestamp: number +} + +/** `workflow-updated` lifecycle broadcast. */ +export interface WorkflowUpdatedBroadcast { + workflowId: string + message: string + timestamp: number +} + +/** `workflow-deployed` lifecycle broadcast. */ +export interface WorkflowDeployedBroadcast { + workflowId: string + timestamp: number +} + +/** `operation-confirmed` ack for a previously-emitted operation. */ +export interface OperationConfirmedBroadcast { + operationId: string + serverTimestamp: number +} + +/** `operation-failed` rejection for a previously-emitted operation. */ +export interface OperationFailedBroadcast { + operationId: string + error: string + retryable?: boolean +}