diff --git a/.agents/skills/memory-load-check/SKILL.md b/.agents/skills/memory-load-check/SKILL.md new file mode 100644 index 00000000000..5a46fce78ca --- /dev/null +++ b/.agents/skills/memory-load-check/SKILL.md @@ -0,0 +1,138 @@ +--- +name: memory-load-check +description: Review PRs and diffs for unbounded memory loading, concurrency explosions, oversized payload materialization, and missing pagination or byte caps. Use when reviewing cleanup jobs, background jobs, data imports/exports, file parsing, API fan-out, workflow execution payloads, large arrays/files, or any change that reads many rows, files, responses, logs, or external API pages into process memory. +--- + +# Memory Load Check + +Use this skill when a PR or diff could load unbounded data into a Node/Bun process, especially in cron routes, background tasks, API routes, workflow execution, file parsing, cleanup jobs, migrations, import/export flows, and external API integrations. + +## Review Goal + +Prove each changed path has explicit bounds for: +- rows held in memory +- bytes held in memory +- concurrent promises, DB queries, HTTP calls, storage operations, and jobs +- number of pages, batches, chunks, retries, and retained intermediate objects + +If any bound depends only on current production size or "probably small" data, treat it as a finding. + +## References + +Read these when doing a deeper pass: +- Node.js streams/backpressure: https://nodejs.org/learn/modules/backpressuring-in-streams +- Node.js stream usage: https://nodejs.org/en/learn/modules/how-to-use-streams +- Keyset/cursor pagination over offset scans: https://blog.sequinstream.com/keyset-cursors-not-offsets-for-postgres-pagination/ +- Postgres pagination tradeoffs: https://www.citusdata.com/blog/2016/03/30/five-ways-to-paginate/ + +## Sim Helpers To Prefer + +- `apps/sim/lib/cleanup/batch-delete.ts` + - `chunkedBatchDelete`: bounded SELECT -> optional side effect -> DELETE loop. + - `batchDeleteByWorkspaceAndTimestamp`: common workspace/timestamp cleanup wrapper. + - `selectRowsByIdChunks`: chunks large ID sets and enforces an overall row cap. + - `chunkArray`: use only after the input set itself is already bounded. +- `apps/sim/lib/core/utils/stream-limits.ts` + - `PayloadSizeLimitError` + - `assertKnownSizeWithinLimit` + - `assertContentLengthWithinLimit` + - `readStreamToBufferWithLimit` + - `readNodeStreamToBufferWithLimit` + - `readResponseToBufferWithLimit` + - `readResponseTextWithLimit` +- Cleanup dispatcher pattern in `apps/sim/lib/billing/cleanup-dispatcher.ts` + - page active workspaces with `WHERE id > afterId ORDER BY id LIMIT N` + - dispatch concrete chunks (`workspaceIds`, retention, label) instead of one giant scope + - prefer Trigger.dev queue/concurrency keys when available + - execute inline fallback chunks sequentially, not with unbounded `Promise.all` +- File parse route pattern in `apps/sim/app/api/files/parse/route.ts` + - cap downloads and parsed output separately + - preserve partial results when a later item exceeds the cap + - never read untrusted response bodies without a byte cap +- Large workflow value payloads + - prefer durable references/manifests over inlining large arrays or files + - materialize refs only behind an explicit byte budget + +## Review Workflow + +1. Identify every changed data source: + - database queries + - storage lists/downloads/uploads + - external API pagination + - file reads and HTTP responses + - workflow logs, snapshots, payloads, arrays, and manifests + - queues, cron routes, and background jobs +2. For each source, write down the maximum cardinality and maximum bytes. If the code does not enforce one, it is unbounded. +3. Trace whether data is processed incrementally or accumulated: + - arrays from `select`, `findMany`, `Promise.all`, `map`, `filter`, `flatMap` + - maps/sets keyed by all users, workspaces, executions, files, or rows + - `Buffer.concat`, `response.arrayBuffer()`, `response.text()`, `JSON.stringify`, `JSON.parse` + - queues of promises or job payloads built before dispatch +4. Check concurrency separately from memory: + - no `Promise.all(items.map(...))` unless `items` is already small and bounded + - use chunks, sequential loops, queue concurrency, or a concurrency limiter + - align concurrency with DB pool size, storage/API limits, and task queue semantics +5. Verify SQL shape: + - every bulk query has `LIMIT` + - large pagination uses cursor/keyset style (`id > afterId`, timestamps plus unique ID), not deep `OFFSET` + - `IN (...)` lists are chunked + - side-effect rows selected before delete have per-batch and per-run caps +6. Verify byte safety: + - check `Content-Length` when available + - stream with cumulative byte accounting + - cap both input bytes and expanded output bytes + - reject or reference oversized values before serializing large JSON responses +7. Confirm failure behavior: + - exceeding a cap should stop before loading more data + - partial successful work should be preserved when the API contract expects it + - retries should not duplicate huge in-memory state + - cleanup jobs should make progress over future runs instead of widening one run + +## Red Flags + +- loads all active workspaces, users, executions, logs, files, messages, or subscriptions before filtering +- builds a full `Map` or `Set` for a platform-wide scope +- uses `Promise.all` over rows from an unbounded query +- fetches all pages from an external API before processing +- reads an entire file, HTTP response, or stream without a max byte budget +- checks size only after `Buffer.concat`, `arrayBuffer`, `text`, `JSON.parse`, or parse expansion +- chunks only after loading the complete dataset +- paginates with unbounded/deep `OFFSET` on a mutable or large table +- creates one queue job per row without batching or a queue-level concurrency key +- accumulates per-row errors/results with no maximum +- adds a cache, singleton, or module-level collection without eviction or size limits + +## Preferred Fixes + +- Move filters into SQL/API requests and select only needed columns. +- Replace full-table loads with cursor/keyset pagination and a deterministic order. +- Process one page/batch at a time; do not keep previous pages unless needed. +- Add per-batch and per-run row caps so long backlogs drain across repeated jobs. +- Split large ID lists with `selectRowsByIdChunks` or `chunkArray` after bounding the source. +- Use `chunkedBatchDelete` for cleanup loops with row side effects. +- Use stream-limit helpers for file/HTTP/body reads. +- Store large workflow values as refs/manifests and materialize only within a caller budget. +- Replace unbounded `Promise.all` with sequential chunk loops, queue concurrency, or a small limiter. +- Include tests that prove caps stop work early and partial results or progress are preserved. + +## Findings Format + +Lead with concrete findings, ordered by risk: + +```markdown +## Findings + +- **P1 Unbounded workspace load in cleanup dispatch** (`path/to/file.ts`) + The new path calls `select().from(workspace)` without a limit, then builds maps for every row before dispatch. In production this scales with all active workspaces and can exhaust the app process. Page by `workspace.id` with a fixed limit and dispatch bounded chunks. + +## Good Signals + +- Uses `readResponseToBufferWithLimit` for external downloads. +- Inline fallback processes chunks sequentially. + +## Residual Risk + +- The row cap is explicit, but no test currently proves the loop stops at the cap. +``` + +Only say "good to go" when every changed source has explicit row, byte, and concurrency bounds or the boundedness is proven by a stable invariant. diff --git a/.agents/skills/validate-integration/SKILL.md b/.agents/skills/validate-integration/SKILL.md index d8d243c5012..7a5ea8e7caf 100644 --- a/.agents/skills/validate-integration/SKILL.md +++ b/.agents/skills/validate-integration/SKILL.md @@ -232,13 +232,23 @@ If any tools support pagination: - [ ] Pagination response fields (`nextToken`, `cursor`, etc.) are included in tool outputs - [ ] Pagination subBlocks are set to `mode: 'advanced'` -## Step 7: Validate Error Handling +## Step 7: Validate Memory Load Safety + +If any tool lists, searches, exports, imports, downloads, uploads, paginates, batches, transforms arrays, or reads file/HTTP bodies, read `.agents/skills/memory-load-check/SKILL.md` and apply it to the integration. + +- [ ] List/search tools expose API limits and do not auto-fetch every page into memory +- [ ] Transform logic does not build unbounded arrays, maps, sets, or `Promise.all` fan-outs +- [ ] File and HTTP body reads use explicit byte caps or existing stream-limit helpers +- [ ] Large result payloads are summarized, paginated, referenced, or capped rather than raw-dumped +- [ ] Pagination and download tests cover caps, early stop behavior, or partial-result preservation when relevant + +## Step 8: Validate Error Handling - [ ] `transformResponse` checks for error conditions before accessing data - [ ] Error responses include meaningful messages (not just generic "failed") - [ ] HTTP error status codes are handled (check `response.ok` or status codes) -## Step 8: Report and Fix +## Step 9: Report and Fix ### Report Format @@ -297,6 +307,7 @@ After fixing, confirm: - [ ] Validated OAuth scopes use centralized utilities (getScopesForService, getCanonicalScopesForProvider) — no hardcoded arrays - [ ] Validated scope descriptions exist in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` for all scopes - [ ] Validated pagination consistency across tools and block +- [ ] Validated memory load safety using `.agents/skills/memory-load-check/SKILL.md` when tools list/search/download/import/export/batch data - [ ] Validated error handling (error checks, meaningful messages) - [ ] Validated registry entries (tools and block, alphabetical, correct imports) - [ ] Reported all issues grouped by severity diff --git a/.gitignore b/.gitignore index c0532fd4492..c38b288a683 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,6 @@ i18n.cache .claude/worktrees/ .claude/scheduled_tasks.lock .deepsec/ + +# Personal Cursor Skills +.cursor/skills/ask-sim/ diff --git a/apps/sim/app/(landing)/blog/layout.tsx b/apps/sim/app/(landing)/blog/layout.tsx index 96b81a7dca5..a82cb763a4e 100644 --- a/apps/sim/app/(landing)/blog/layout.tsx +++ b/apps/sim/app/(landing)/blog/layout.tsx @@ -2,6 +2,7 @@ import { getNavBlogPosts } from '@/lib/blog/registry' import { SITE_URL } from '@/lib/core/utils/urls' import Footer from '@/app/(landing)/components/footer/footer' import Navbar from '@/app/(landing)/components/navbar/navbar' +import { ScrollToTop } from '@/app/(landing)/components/scroll-to-top' export default async function StudioLayout({ children }: { children: React.ReactNode }) { const blogPosts = await getNavBlogPosts() @@ -29,6 +30,7 @@ export default async function StudioLayout({ children }: { children: React.React return (
+ ` + return new NextResponse(body, { + headers: { 'Content-Type': 'text/html; charset=utf-8' }, + }) +} + +export const GET = withRouteHandler(async (request: NextRequest) => { + const parsed = await parseRequest(mcpOauthCallbackContract, request, {}) + if (!parsed.success) { + return htmlClose('Malformed authorization callback.', false, 'missing_params') + } + const { state, code, error: errorParam } = parsed.data.query + + const initialRow = state ? await loadOauthRowByState(state).catch(() => null) : null + const stateRowServerId = initialRow?.mcpServerId + + if (errorParam) { + logger.warn(`MCP OAuth callback received error: ${errorParam}`) + if (initialRow) await clearState(initialRow.id).catch(() => {}) + return htmlClose( + `Authorization failed: ${errorParam}`, + false, + 'provider_error', + stateRowServerId + ) + } + if (!state || !code) { + return htmlClose( + 'Missing state or code in callback URL.', + false, + 'missing_params', + stateRowServerId + ) + } + + let serverId: string | undefined + try { + const session = await getSession() + if (!session?.user?.id) { + return htmlClose( + 'You must be signed in to complete authorization.', + false, + 'unauthenticated', + stateRowServerId + ) + } + + const row = initialRow + if (!row) { + return htmlClose('Invalid or expired authorization state.', false, 'invalid_state') + } + serverId = row.mcpServerId + + if (session.user.id !== row.userId) { + return htmlClose( + 'You must be signed in as the same user that initiated the flow.', + false, + 'user_mismatch', + serverId + ) + } + + const [server] = await db + .select({ id: mcpServers.id, url: mcpServers.url, workspaceId: mcpServers.workspaceId }) + .from(mcpServers) + .where(and(eq(mcpServers.id, row.mcpServerId), isNull(mcpServers.deletedAt))) + .limit(1) + if (!server || !server.url) { + return htmlClose('Server no longer exists.', false, 'server_gone', serverId) + } + if (server.workspaceId !== row.workspaceId) { + return htmlClose( + 'Workspace mismatch on authorization callback.', + false, + 'invalid_state', + serverId + ) + } + try { + assertSafeOauthServerUrl(server.url) + } catch { + return htmlClose( + 'MCP OAuth requires https (or http://localhost for development).', + false, + 'insecure_url', + serverId + ) + } + + // Burn state before token exchange so a replayed callback cannot reuse it. + await clearState(row.id) + + const preregistered = await loadPreregisteredClient(server.id) + const provider = new SimMcpOauthProvider({ row, preregistered }) + let result: Awaited> + try { + result = await mcpAuth(provider, { + serverUrl: server.url, + authorizationCode: code, + }) + } catch (e) { + logger.error('Token exchange failed during MCP OAuth callback', e) + return htmlClose( + 'Token exchange failed. Please try again.', + false, + 'token_exchange_failed', + server.id + ) + } finally { + await clearVerifier(row.id) + } + + if (result !== 'AUTHORIZED') { + return htmlClose('Authorization did not complete.', false, 'token_exchange_failed', server.id) + } + + try { + // discoverServerTools writes the result to this server's cache so the UI's + // immediate refetch hits it instead of re-fetching live. + await mcpService.discoverServerTools(session.user.id, server.id, server.workspaceId) + } catch (e) { + logger.warn('Post-auth tools refresh failed', toError(e).message) + } + + return htmlClose('Connected. You can close this window.', true, 'authorized', server.id) + } catch (error) { + logger.error('MCP OAuth callback failed', error) + return htmlClose('Authorization failed. Please try again.', false, 'unknown', serverId) + } +}) diff --git a/apps/sim/app/api/mcp/oauth/start/route.test.ts b/apps/sim/app/api/mcp/oauth/start/route.test.ts new file mode 100644 index 00000000000..fa8ebf26dc7 --- /dev/null +++ b/apps/sim/app/api/mcp/oauth/start/route.test.ts @@ -0,0 +1,198 @@ +/** + * @vitest-environment node + */ +import { + dbChainMock, + dbChainMockFns, + hybridAuthMock, + hybridAuthMockFns, + McpOauthRedirectRequiredMock, + mcpOauthMock, + mcpOauthMockFns, + permissionsMock, + permissionsMockFns, + resetDbChainMock, + schemaMock, +} from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockMcpAuth } = vi.hoisted(() => ({ + mockMcpAuth: vi.fn(), +})) + +vi.mock('@sim/db', () => dbChainMock) +vi.mock('@sim/db/schema', () => schemaMock) +vi.mock('drizzle-orm', () => ({ + and: vi.fn(), + eq: vi.fn(), + isNull: vi.fn(), +})) +vi.mock('@modelcontextprotocol/sdk/client/auth.js', () => ({ + auth: mockMcpAuth, +})) +vi.mock('@/lib/auth/hybrid', () => hybridAuthMock) +vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) +vi.mock('@/lib/mcp/oauth', () => mcpOauthMock) + +import { GET, surfaceOauthError } from './route' + +describe('MCP OAuth start route', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-2', + userName: 'User Two', + userEmail: 'user2@example.com', + authType: 'session', + }) + permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') + dbChainMockFns.limit.mockResolvedValue([ + { + id: 'server-1', + name: 'Exa', + url: 'https://mcp.exa.ai/mcp', + workspaceId: 'workspace-1', + authType: 'oauth', + deletedAt: null, + }, + ]) + mcpOauthMockFns.mockGetOrCreateOauthRow.mockResolvedValue({ + id: 'oauth-row-1', + mcpServerId: 'server-1', + userId: 'user-1', + workspaceId: 'workspace-1', + clientInformation: null, + tokens: null, + codeVerifier: null, + state: null, + stateCreatedAt: null, + updatedAt: new Date(), + }) + mcpOauthMockFns.mockLoadPreregisteredClient.mockResolvedValue(undefined) + mockMcpAuth.mockRejectedValue(new McpOauthRedirectRequiredMock('https://mcp.exa.ai/authorize')) + }) + + it('requires workspace write permission via MCP auth middleware', async () => { + const request = new NextRequest( + 'http://localhost:3000/api/mcp/oauth/start?workspaceId=workspace-1&serverId=server-1' + ) + + await GET(request) + + expect(permissionsMockFns.mockGetUserEntityPermissions).toHaveBeenCalledWith( + 'user-2', + 'workspace', + 'workspace-1' + ) + }) + + it('uses a workspace-scoped OAuth row and stamps the latest authorizing user', async () => { + const request = new NextRequest( + 'http://localhost:3000/api/mcp/oauth/start?workspaceId=workspace-1&serverId=server-1' + ) + + const response = await GET(request) + const body = await response.json() + + expect(response.status).toBe(200) + expect(body).toEqual({ + status: 'redirect', + authorizationUrl: 'https://mcp.exa.ai/authorize', + }) + expect(mcpOauthMockFns.mockGetOrCreateOauthRow).toHaveBeenCalledWith({ + mcpServerId: 'server-1', + userId: 'user-2', + workspaceId: 'workspace-1', + }) + expect(mcpOauthMockFns.mockSetOauthRowUser).toHaveBeenCalledWith('oauth-row-1', 'user-2') + }) + + it('rejects a second user starting OAuth while another authorization is active', async () => { + mcpOauthMockFns.mockGetOrCreateOauthRow.mockResolvedValueOnce({ + id: 'oauth-row-1', + mcpServerId: 'server-1', + userId: 'user-1', + workspaceId: 'workspace-1', + clientInformation: null, + tokens: null, + codeVerifier: null, + state: 'hashed-active-state', + stateCreatedAt: new Date(), + updatedAt: new Date(), + }) + const request = new NextRequest( + 'http://localhost:3000/api/mcp/oauth/start?workspaceId=workspace-1&serverId=server-1' + ) + + const response = await GET(request) + const body = await response.json() + + expect(response.status).toBe(409) + expect(body.error).toBe('OAuth authorization already in progress for this server') + expect(mockMcpAuth).not.toHaveBeenCalled() + }) + + it('does not leak non-OAuth internal error details to the client', async () => { + mcpOauthMockFns.mockGetOrCreateOauthRow.mockRejectedValueOnce( + new Error('connect ECONNREFUSED 10.0.0.5:5432 (internal-db-host)') + ) + const request = new NextRequest( + 'http://localhost:3000/api/mcp/oauth/start?workspaceId=workspace-1&serverId=server-1' + ) + + const response = await GET(request) + const body = await response.json() + + expect(response.status).toBe(500) + expect(body.error).toBe('Failed to start OAuth flow') + expect(body.error).not.toContain('ECONNREFUSED') + expect(body.error).not.toContain('internal-db-host') + }) +}) + +describe('surfaceOauthError', () => { + it('uses typed OAuthError errorCode and message for spec-compliant errors', async () => { + const { InvalidGrantError } = await import('@modelcontextprotocol/sdk/server/auth/errors.js') + const err = new InvalidGrantError('Refresh token expired') + expect(surfaceOauthError(err)).toBe('invalid_grant: Refresh token expired') + }) + + it('parses Raw body envelope for ServerError fallbacks (non-spec vendors)', async () => { + const { ServerError } = await import('@modelcontextprotocol/sdk/server/auth/errors.js') + const err = new ServerError( + 'HTTP 400: Invalid OAuth error response: zod error. Raw body: {"code":400,"message":"redirect URI https://example.com/cb is not allowed","retryable":false}' + ) + expect(surfaceOauthError(err)).toBe( + 'Authorization server: redirect URI https://example.com/cb is not allowed' + ) + }) + + it('prefers error_description over message over error in fallback envelope', async () => { + const { ServerError } = await import('@modelcontextprotocol/sdk/server/auth/errors.js') + const err = new ServerError( + 'HTTP 400: Invalid OAuth error response: zod. Raw body: {"error":"invalid_grant","error_description":"the description","message":"the message"}' + ) + expect(surfaceOauthError(err)).toBe('Authorization server: the description') + }) + + it('returns first line of generic errors', () => { + const err = new Error('Network blip\n at fetch (...)') + expect(surfaceOauthError(err)).toBe('Network blip') + }) + + it('truncates messages longer than 250 chars with ellipsis', async () => { + const { InvalidGrantError } = await import('@modelcontextprotocol/sdk/server/auth/errors.js') + const longMessage = 'x'.repeat(300) + const result = surfaceOauthError(new InvalidGrantError(longMessage)) + expect(result.endsWith('…')).toBe(true) + expect(result.length).toBe(251) + }) + + it('returns generic fallback for non-Error values', () => { + expect(surfaceOauthError(null)).toBe('Failed to start OAuth flow') + expect(surfaceOauthError(undefined)).toBe('Failed to start OAuth flow') + }) +}) diff --git a/apps/sim/app/api/mcp/oauth/start/route.ts b/apps/sim/app/api/mcp/oauth/start/route.ts new file mode 100644 index 00000000000..c7619b9d555 --- /dev/null +++ b/apps/sim/app/api/mcp/oauth/start/route.ts @@ -0,0 +1,160 @@ +import { auth as mcpAuth } from '@modelcontextprotocol/sdk/client/auth.js' +import { OAuthError, ServerError } from '@modelcontextprotocol/sdk/server/auth/errors.js' +import { db } from '@sim/db' +import { mcpServers } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { and, eq, isNull } from 'drizzle-orm' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { startMcpOauthContract } from '@/lib/api/contracts/mcp' +import { parseRequest } from '@/lib/api/server' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { withMcpAuth } from '@/lib/mcp/middleware' +import { + assertSafeOauthServerUrl, + getOrCreateOauthRow, + loadPreregisteredClient, + McpOauthInsecureUrlError, + McpOauthRedirectRequired, + SimMcpOauthProvider, + setOauthRowUser, +} from '@/lib/mcp/oauth' +import { createMcpErrorResponse } from '@/lib/mcp/utils' + +const logger = createLogger('McpOauthStartAPI') +const OAUTH_START_TTL_MS = 10 * 60 * 1000 +const MAX_SURFACED_ERROR_LENGTH = 250 + +export function surfaceOauthError(error: unknown): string { + // Spec-compliant OAuth servers throw typed subclasses with clean RFC 6749 fields. + if (error instanceof OAuthError && !(error instanceof ServerError)) { + return truncate(`${error.errorCode}: ${error.message}`) + } + + // ServerError wraps non-spec response bodies as "HTTP N: Invalid OAuth error + // response: ... Raw body: {...}". Dig the vendor message out of the JSON tail. + if (error instanceof Error) { + const rawBodyMatch = error.message.match(/Raw body:\s*(\{[\s\S]*\})\s*$/) + if (rawBodyMatch) { + try { + const body = JSON.parse(rawBodyMatch[1]) as Record + const vendorMessage = + (typeof body.error_description === 'string' && body.error_description) || + (typeof body.message === 'string' && body.message) || + (typeof body.error === 'string' && body.error) || + null + if (vendorMessage) return truncate(`Authorization server: ${vendorMessage}`) + } catch {} + } + return truncate(error.message.split('\n')[0] || 'Failed to start OAuth flow') + } + return 'Failed to start OAuth flow' +} + +function truncate(message: string): string { + return message.length > MAX_SURFACED_ERROR_LENGTH + ? `${message.slice(0, MAX_SURFACED_ERROR_LENGTH)}…` + : message +} + +export const dynamic = 'force-dynamic' + +export const GET = withRouteHandler( + withMcpAuth('write')(async (request: NextRequest, { userId, workspaceId }) => { + try { + const parsed = await parseRequest(startMcpOauthContract, request, {}) + if (!parsed.success) return parsed.response + const { serverId } = parsed.data.query + + const [server] = await db + .select() + .from(mcpServers) + .where( + and( + eq(mcpServers.id, serverId), + eq(mcpServers.workspaceId, workspaceId), + isNull(mcpServers.deletedAt) + ) + ) + .limit(1) + + if (!server) { + return createMcpErrorResponse(new Error('Server not found'), 'Server not found', 404) + } + if (server.authType !== 'oauth') { + return createMcpErrorResponse( + new Error(`Server authType is "${server.authType}", not oauth`), + 'Server is not configured for OAuth', + 400 + ) + } + if (!server.url) { + return createMcpErrorResponse(new Error('Server has no URL'), 'Missing server URL', 400) + } + try { + assertSafeOauthServerUrl(server.url) + } catch (e) { + if (e instanceof McpOauthInsecureUrlError) { + return createMcpErrorResponse( + e, + 'MCP OAuth requires https (or http://localhost for development)', + 400 + ) + } + throw e + } + + const row = await getOrCreateOauthRow({ + mcpServerId: server.id, + userId, + workspaceId, + }) + const hasActiveFlow = + !!row.state && + !!row.stateCreatedAt && + row.stateCreatedAt.getTime() > Date.now() - OAUTH_START_TTL_MS + if (hasActiveFlow && row.userId && row.userId !== userId) { + return createMcpErrorResponse( + new Error('OAuth authorization already in progress'), + 'OAuth authorization already in progress for this server', + 409 + ) + } + if (row.userId !== userId) { + await setOauthRowUser(row.id, userId) + row.userId = userId + } + const preregistered = await loadPreregisteredClient(server.id) + const provider = new SimMcpOauthProvider({ row, preregistered }) + + try { + const result = await mcpAuth(provider, { serverUrl: server.url }) + if (result === 'AUTHORIZED') { + return NextResponse.json({ status: 'already_authorized' }) + } + return createMcpErrorResponse( + new Error('Provider did not capture redirect URL'), + 'Failed to start OAuth flow', + 500 + ) + } catch (e) { + if (e instanceof McpOauthRedirectRequired) { + logger.info(`OAuth redirect for server ${serverId}`) + return NextResponse.json({ + status: 'redirect', + authorizationUrl: e.authorizationUrl, + }) + } + throw e + } + } catch (error) { + logger.error('Error starting MCP OAuth flow:', error) + // Only surface OAuth-flow errors verbatim; everything else (DB, decryption, + // network) gets a generic message to avoid leaking internal details. + const userMessage = + error instanceof OAuthError ? surfaceOauthError(error) : 'Failed to start OAuth flow' + return createMcpErrorResponse(toError(error), userMessage, 500) + } + }) +) diff --git a/apps/sim/app/api/mcp/servers/[id]/route.ts b/apps/sim/app/api/mcp/servers/[id]/route.ts index 6795f6383e1..4242fdef119 100644 --- a/apps/sim/app/api/mcp/servers/[id]/route.ts +++ b/apps/sim/app/api/mcp/servers/[id]/route.ts @@ -45,23 +45,25 @@ export const PATCH = withRouteHandler( } ) - // Remove workspaceId from body to prevent it from being updated - const { workspaceId: _, ...updateData } = body - const result = await performUpdateMcpServer({ workspaceId, userId, actorName: userName, actorEmail: userEmail, serverId, - name: updateData.name, - description: updateData.description, - transport: updateData.transport, - url: updateData.url, - headers: updateData.headers, - timeout: updateData.timeout, - retries: updateData.retries, - enabled: updateData.enabled, + name: body.name, + description: body.description, + transport: body.transport, + url: body.url, + headers: body.headers, + timeout: body.timeout, + retries: body.retries, + enabled: body.enabled, + authType: body.authType, + oauthClientId: body.oauthClientId || null, + oauthClientIdProvided: body.oauthClientId !== undefined, + oauthClientSecret: body.oauthClientSecret, + oauthClientSecretProvided: body.oauthClientSecret !== undefined, request, }) if (!result.success || !result.server) { @@ -75,7 +77,10 @@ export const PATCH = withRouteHandler( logger.info(`[${requestId}] Successfully updated MCP server: ${serverId}`) - return createMcpSuccessResponse({ server: updatedServer }) + const { oauthClientSecret: _secret, ...rest } = updatedServer + return createMcpSuccessResponse({ + server: { ...rest, hasOauthClientSecret: !!_secret }, + }) } catch (error) { logger.error(`[${requestId}] Error updating MCP server:`, error) return createMcpErrorResponse(toError(error), 'Failed to update MCP server', 500) diff --git a/apps/sim/app/api/mcp/servers/route.ts b/apps/sim/app/api/mcp/servers/route.ts index f0f2744b053..1d02caeef74 100644 --- a/apps/sim/app/api/mcp/servers/route.ts +++ b/apps/sim/app/api/mcp/servers/route.ts @@ -27,11 +27,16 @@ export const GET = withRouteHandler( try { logger.info(`[${requestId}] Listing MCP servers for workspace ${workspaceId}`) - const servers = await db + const rows = await db .select() .from(mcpServers) .where(and(eq(mcpServers.workspaceId, workspaceId), isNull(mcpServers.deletedAt))) + const servers = rows.map(({ oauthClientSecret: _secret, ...rest }) => ({ + ...rest, + hasOauthClientSecret: !!_secret, + })) + logger.info( `[${requestId}] Listed ${servers.length} MCP servers for workspace ${workspaceId}` ) @@ -45,13 +50,6 @@ export const GET = withRouteHandler( /** * POST - Register a new MCP server for the workspace (requires write permission) - * - * Uses deterministic server IDs based on URL hash to ensure that re-adding - * the same server produces the same ID. This prevents "server not found" errors - * when workflows reference the old server ID after delete/re-add cycles. - * - * If a server with the same ID already exists (same URL in same workspace), - * it will be updated instead of creating a duplicate. */ export const POST = withRouteHandler( withMcpAuth('write')( @@ -96,6 +94,11 @@ export const POST = withRouteHandler( retries: body.retries, enabled: body.enabled, source, + authType: body.authType, + oauthClientId: body.oauthClientId || null, + oauthClientIdProvided: body.oauthClientId !== undefined, + oauthClientSecret: body.oauthClientSecret, + oauthClientSecretProvided: body.oauthClientSecret !== undefined, request, }) if (!result.success || !result.serverId) { @@ -112,8 +115,8 @@ export const POST = withRouteHandler( return createMcpSuccessResponse( result.updated - ? { serverId: result.serverId, updated: true } - : { serverId: result.serverId }, + ? { serverId: result.serverId, updated: true, authType: result.authType } + : { serverId: result.serverId, authType: result.authType }, result.updated ? 200 : 201 ) } catch (error) { diff --git a/apps/sim/app/api/mcp/servers/test-connection/route.ts b/apps/sim/app/api/mcp/servers/test-connection/route.ts index bd88be77aad..c017de7a34c 100644 --- a/apps/sim/app/api/mcp/servers/test-connection/route.ts +++ b/apps/sim/app/api/mcp/servers/test-connection/route.ts @@ -12,8 +12,9 @@ import { validateMcpServerSsrf, } from '@/lib/mcp/domain-check' import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware' +import { detectMcpAuthType } from '@/lib/mcp/oauth' import { resolveMcpConfigEnvVars } from '@/lib/mcp/resolve-config' -import type { McpTransport } from '@/lib/mcp/types' +import type { McpAuthType, McpTransport } from '@/lib/mcp/types' import { createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils' const logger = createLogger('McpServerTestAPI') @@ -31,6 +32,8 @@ function isUrlBasedTransport(transport: McpTransport): boolean { interface TestConnectionResult { success: boolean error?: string + authRequired?: boolean + authType?: McpAuthType serverInfo?: { name: string version: string @@ -163,6 +166,18 @@ export const POST = withRouteHandler( } const result: TestConnectionResult = { success: false } + + // Skip unauth connect when the server returns an RFC 9728 OAuth challenge. + if (testConfig.url) { + const detectedAuthType = await detectMcpAuthType(testConfig.url) + if (detectedAuthType === 'oauth') { + result.authRequired = true + result.authType = 'oauth' + return createMcpSuccessResponse(result, 200) + } + result.authType = detectedAuthType + } + let client: McpClient | null = null try { diff --git a/apps/sim/app/api/mcp/tools/discover/route.ts b/apps/sim/app/api/mcp/tools/discover/route.ts index e94f2f56328..b125fa7ff2b 100644 --- a/apps/sim/app/api/mcp/tools/discover/route.ts +++ b/apps/sim/app/api/mcp/tools/discover/route.ts @@ -1,3 +1,4 @@ +import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js' import { createLogger } from '@sim/logger' import type { NextRequest } from 'next/server' import { mcpToolDiscoveryQuerySchema, refreshMcpToolsBodySchema } from '@/lib/api/contracts/mcp' @@ -5,7 +6,7 @@ import { validationErrorResponse } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware' import { mcpService } from '@/lib/mcp/service' -import type { McpToolDiscoveryResponse } from '@/lib/mcp/types' +import { McpOauthAuthorizationRequiredError, type McpToolDiscoveryResponse } from '@/lib/mcp/types' import { categorizeError, createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils' const logger = createLogger('McpToolDiscoveryAPI') @@ -46,6 +47,12 @@ export const GET = withRouteHandler( ) return createMcpSuccessResponse(responseData) } catch (error) { + if ( + error instanceof McpOauthAuthorizationRequiredError || + error instanceof UnauthorizedError + ) { + return createMcpErrorResponse(error, 'OAuth re-authorization required', 401) + } logger.error(`[${requestId}] Error discovering MCP tools:`, error) const { message, status } = categorizeError(error) return createMcpErrorResponse(new Error(message), 'Failed to discover MCP tools', status) @@ -100,6 +107,12 @@ export const POST = withRouteHandler( }, }) } catch (error) { + if ( + error instanceof McpOauthAuthorizationRequiredError || + error instanceof UnauthorizedError + ) { + return createMcpErrorResponse(error, 'OAuth re-authorization required', 401) + } logger.error(`[${requestId}] Error refreshing tool discovery:`, error) const { message, status } = categorizeError(error) return createMcpErrorResponse(new Error(message), 'Failed to refresh tool discovery', status) diff --git a/apps/sim/app/api/mcp/tools/execute/route.ts b/apps/sim/app/api/mcp/tools/execute/route.ts index d9458deceab..8599a5fcadf 100644 --- a/apps/sim/app/api/mcp/tools/execute/route.ts +++ b/apps/sim/app/api/mcp/tools/execute/route.ts @@ -1,5 +1,7 @@ +import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js' import { createLogger } from '@sim/logger' import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' import { mcpToolExecutionBodySchema } from '@/lib/api/contracts/mcp' import { getHighestPrioritySubscription } from '@/lib/billing/core/plan' import { getExecutionTimeout } from '@/lib/core/execution-limits' @@ -7,8 +9,14 @@ import type { SubscriptionPlan } from '@/lib/core/rate-limiter/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { SIM_VIA_HEADER } from '@/lib/execution/call-chain' import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware' +import { McpOauthRedirectRequired } from '@/lib/mcp/oauth' import { mcpService } from '@/lib/mcp/service' -import type { McpTool, McpToolCall, McpToolResult } from '@/lib/mcp/types' +import { + McpOauthAuthorizationRequiredError, + type McpTool, + type McpToolCall, + type McpToolResult, +} from '@/lib/mcp/types' import { categorizeError, createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils' import { assertPermissionsAllowed, @@ -43,6 +51,7 @@ function hasType(prop: unknown): prop is SchemaProperty { */ export const POST = withRouteHandler( withMcpAuth('read')(async (request: NextRequest, { userId, workspaceId, requestId }) => { + let serverId: string | undefined try { const rawBody = getParsedBody(request) ?? (await request.json()) const parsedBody = mcpToolExecutionBodySchema.safeParse(rawBody) @@ -63,7 +72,8 @@ export const POST = withRouteHandler( userId: userId, }) - const { serverId, toolName, arguments: rawArgs } = body + const { toolName, arguments: rawArgs } = body + serverId = body.serverId const args = rawArgs || {} try { @@ -101,7 +111,8 @@ export const POST = withRouteHandler( if (tool.inputSchema?.properties) { for (const [paramName, paramSchema] of Object.entries(tool.inputSchema.properties)) { - const schema = paramSchema as any + const schema = hasType(paramSchema) ? paramSchema : null + if (!schema) continue const value = args[paramName] if (value === undefined || value === null) { @@ -185,12 +196,18 @@ export const POST = withRouteHandler( extraHeaders[SIM_VIA_HEADER] = simViaHeader } + let timeoutHandle: ReturnType | undefined const result = await Promise.race([ mcpService.executeTool(userId, serverId, toolCall, workspaceId, extraHeaders), - new Promise((_, reject) => - setTimeout(() => reject(new Error('Tool execution timeout')), executionTimeout) - ), - ]) + new Promise((_, reject) => { + timeoutHandle = setTimeout( + () => reject(new Error('Tool execution timeout')), + executionTimeout + ) + }), + ]).finally(() => { + if (timeoutHandle !== undefined) clearTimeout(timeoutHandle) + }) const transformedResult = transformToolResult(result) @@ -218,6 +235,27 @@ export const POST = withRouteHandler( return createMcpSuccessResponse(transformedResult) } catch (error) { + if ( + error instanceof McpOauthAuthorizationRequiredError || + error instanceof McpOauthRedirectRequired || + error instanceof UnauthorizedError + ) { + const errorServerId = + error instanceof McpOauthAuthorizationRequiredError ? error.serverId : serverId + logger.warn(`[${requestId}] OAuth re-authorization required for MCP tool execution`, { + serverId: errorServerId, + }) + return NextResponse.json( + { + success: false, + error: 'OAuth re-authorization required', + code: 'reauth_required', + serverId: errorServerId, + }, + { status: 401 } + ) + } + logger.error(`[${requestId}] Error executing MCP tool:`, error) const { message, status } = categorizeError(error) diff --git a/apps/sim/app/api/resume/poll/route.ts b/apps/sim/app/api/resume/poll/route.ts index ad07ea009b5..86edf2f218c 100644 --- a/apps/sim/app/api/resume/poll/route.ts +++ b/apps/sim/app/api/resume/poll/route.ts @@ -139,13 +139,21 @@ async function dispatchRow(row: DueRow, now: Date): Promise { }) if (enqueueResult.status === 'starting') { - PauseResumeManager.startResumeExecution({ + // Route through `executeResumeJob` (not `PauseResumeManager.startResumeExecution` + // directly) so cell-context restoration + cascade-loop continuation + // fires. This is the same primitive the trigger.dev `resumeExecutionTask` + // wraps — calling it directly handles both trigger.dev-disabled local + // dev and trigger.dev-enabled prod identically. + const { executeResumeJob } = await import('@/background/resume-execution') + void executeResumeJob({ resumeEntryId: enqueueResult.resumeEntryId, resumeExecutionId: enqueueResult.resumeExecutionId, - pausedExecution: enqueueResult.pausedExecution, + pausedExecutionId: enqueueResult.pausedExecution.id, contextId: enqueueResult.contextId, resumeInput: enqueueResult.resumeInput, userId: enqueueResult.userId, + workflowId: row.workflowId, + parentExecutionId: row.executionId, }).catch((error) => { logger.error('Background time-pause resume failed', { executionId: row.executionId, diff --git a/apps/sim/app/api/table/[tableId]/columns/run/route.ts b/apps/sim/app/api/table/[tableId]/columns/run/route.ts index eddfc416e0a..2b96981d115 100644 --- a/apps/sim/app/api/table/[tableId]/columns/run/route.ts +++ b/apps/sim/app/api/table/[tableId]/columns/run/route.ts @@ -1,5 +1,4 @@ import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' import { runColumnContract } from '@/lib/api/contracts/tables' import { parseRequest } from '@/lib/api/server' @@ -30,21 +29,16 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const access = await checkAccess(tableId, auth.userId, 'write') if (!access.ok) return accessError(access, requestId, tableId) - // Dispatch in the background — large fan-outs (thousands of rows) issue - // sequential trigger.dev calls and would otherwise hold the HTTP response - // open for minutes, blocking the AI/copilot tool span and the UI mutation. - void runWorkflowColumn({ + const { dispatchId } = await runWorkflowColumn({ tableId, workspaceId, groupIds, mode: runMode, rowIds, requestId, - }).catch((err) => { - logger.error(`[${requestId}] run-column dispatch failed:`, toError(err).message) }) - return NextResponse.json({ success: true, data: { triggered: null } }) + return NextResponse.json({ success: true, data: { dispatchId } }) } catch (error) { if (error instanceof Error && error.message === 'Invalid workspace ID') { return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) diff --git a/apps/sim/app/api/table/[tableId]/dispatches/route.ts b/apps/sim/app/api/table/[tableId]/dispatches/route.ts new file mode 100644 index 00000000000..7682ba82994 --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/dispatches/route.ts @@ -0,0 +1,63 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { type ActiveDispatch, listActiveDispatchesContract } from '@/lib/api/contracts/tables' +import { parseRequest } from '@/lib/api/server' +import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { countActiveRunCells, listActiveDispatches } from '@/lib/table/dispatcher' +import { accessError, checkAccess } from '@/app/api/table/utils' + +const logger = createLogger('TableDispatchesAPI') + +interface RouteParams { + params: Promise<{ tableId: string }> +} + +/** + * GET /api/table/[tableId]/dispatches + * + * Returns active (`pending` / `dispatching`) dispatches for the table. Drives + * the client's "about to run" overlay so refresh during a long Run-all keeps + * the queued indicators on rows the dispatcher hasn't reached yet. + */ +export const GET = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => { + const requestId = generateRequestId() + + try { + const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success || !authResult.userId) { + return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) + } + + const parsed = await parseRequest(listActiveDispatchesContract, request, { params }) + if (!parsed.success) return parsed.response + const { tableId } = parsed.data.params + + const result = await checkAccess(tableId, authResult.userId, 'read') + if (!result.ok) return accessError(result, requestId, tableId) + + const rows = await listActiveDispatches(tableId) + const running = await countActiveRunCells(tableId, rows) + const dispatches: ActiveDispatch[] = rows.map((r) => ({ + id: r.id, + status: r.status as 'pending' | 'dispatching', + mode: r.mode, + isManualRun: r.isManualRun, + cursor: r.cursor, + scope: r.scope, + })) + + return NextResponse.json({ + success: true, + data: { + dispatches, + runningCellCount: running.total, + runningByRowId: running.byRowId, + }, + }) + } catch (error) { + logger.error(`[${requestId}] list-dispatches failed:`, error) + return NextResponse.json({ error: 'Failed to list active dispatches' }, { status: 500 }) + } +}) diff --git a/apps/sim/app/api/table/[tableId]/import/route.test.ts b/apps/sim/app/api/table/[tableId]/import/route.test.ts index b821961cb6d..b51b35ecece 100644 --- a/apps/sim/app/api/table/[tableId]/import/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/import/route.test.ts @@ -112,6 +112,7 @@ function buildTable(overrides: Partial = {}): TableDefinition { async function callPost(form: FormData, { tableId }: { tableId: string } = { tableId: 'tbl_1' }) { const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/import`, { method: 'POST', + headers: { 'content-length': '1024' }, body: form, }) return POST(req, { params: Promise.resolve({ tableId }) }) @@ -182,6 +183,26 @@ describe('POST /api/table/[tableId]/import', () => { expect(data.error).toMatch(/archived/i) }) + it('returns 413 for oversized CSV files before reading their contents', async () => { + const file = createCsvFile('name,age\nAlice,30') + Object.defineProperty(file, 'size', { + value: 26 * 1024 * 1024, + }) + const arrayBufferSpy = vi.spyOn(file, 'arrayBuffer') + + const req = { + formData: async () => createFormData(file), + } as unknown as NextRequest + + const response = await POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) + expect(response.status).toBe(413) + const data = await response.json() + expect(data.error).toMatch(/CSV import file exceeds maximum size/) + expect(arrayBufferSpy).not.toHaveBeenCalled() + expect(mockBatchInsertRowsWithTx).not.toHaveBeenCalled() + expect(mockReplaceTableRowsWithTx).not.toHaveBeenCalled() + }) + it('returns 400 when the CSV is missing a required column', async () => { const response = await callPost(createFormData(createCsvFile('age\n30'))) expect(response.status).toBe(400) @@ -208,6 +229,21 @@ describe('POST /api/table/[tableId]/import', () => { expect(mockReplaceTableRowsWithTx).not.toHaveBeenCalled() }) + it('accepts chunked multipart imports without a content-length header', async () => { + const form = createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'append' }) + const req = new NextRequest('http://localhost:3000/api/table/tbl_1/import', { + method: 'POST', + body: form, + }) + + expect(req.headers.get('content-length')).toBeNull() + + const response = await POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) + + expect(response.status).toBe(200) + expect(mockBatchInsertRowsWithTx).toHaveBeenCalledTimes(1) + }) + it('rejects append when it would exceed maxRows', async () => { mockCheckAccess.mockResolvedValueOnce({ ok: true, diff --git a/apps/sim/app/api/table/[tableId]/import/route.ts b/apps/sim/app/api/table/[tableId]/import/route.ts index c51cde1b2ab..9d9ddcfd96d 100644 --- a/apps/sim/app/api/table/[tableId]/import/route.ts +++ b/apps/sim/app/api/table/[tableId]/import/route.ts @@ -14,12 +14,18 @@ import { import { getValidationErrorMessage } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { + isPayloadSizeLimitError, + readFileToBufferWithLimit, + readFormDataWithLimit, +} from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { addTableColumnsWithTx, batchInsertRowsWithTx, buildAutoMapping, CSV_MAX_BATCH_SIZE, + CSV_MAX_FILE_SIZE_BYTES, type CsvHeaderMapping, CsvImportValidationError, coerceRowsForTable, @@ -34,6 +40,7 @@ import { import { accessError, checkAccess } from '@/app/api/table/utils' const logger = createLogger('TableImportCSVExisting') +const MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024 interface RouteParams { params: Promise<{ tableId: string }> @@ -49,7 +56,10 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - const formData = await request.formData() + const formData = await readFormDataWithLimit(request, { + maxBytes: CSV_MAX_FILE_SIZE_BYTES + MAX_MULTIPART_OVERHEAD_BYTES, + label: 'CSV import body', + }) const formValidation = csvImportFormSchema.safeParse({ file: formData.get('file'), workspaceId: formData.get('workspaceId'), @@ -59,9 +69,11 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const rawCreateColumns = formData.get('createColumns') if (!formValidation.success) { + const message = getValidationErrorMessage(formValidation.error) + const isSizeLimit = message.includes('File exceeds maximum allowed size') return NextResponse.json( - { error: getValidationErrorMessage(formValidation.error) }, - { status: 400 } + { error: isSizeLimit ? 'CSV import file exceeds maximum size' : message }, + { status: isSizeLimit ? 413 : 400 } ) } @@ -125,7 +137,10 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro createColumns = createColumnsValidation.data } - const buffer = Buffer.from(await file.arrayBuffer()) + const buffer = await readFileToBufferWithLimit(file, { + maxBytes: CSV_MAX_FILE_SIZE_BYTES, + label: 'CSV import file', + }) const delimiter = extensionValidation.data === 'tsv' ? '\t' : ',' const { headers, rows } = await parseCsvBuffer(buffer, delimiter) @@ -343,14 +358,19 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro const message = toError(error).message logger.error(`[${requestId}] CSV import into existing table failed:`, error) + const isSizeLimitError = + isPayloadSizeLimitError(error) || message.includes('CSV import file exceeds maximum size') const isClientError = message.includes('CSV file has no') || message.includes('already exists') || - message.includes('Invalid column name') + message.includes('Invalid column name') || + isSizeLimitError return NextResponse.json( { error: isClientError ? message : 'Failed to import CSV' }, - { status: isClientError ? 400 : 500 } + { + status: isSizeLimitError ? 413 : isClientError ? 400 : 500, + } ) } }) diff --git a/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts b/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts index 9a4a988bc25..fe7452230ee 100644 --- a/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts @@ -136,6 +136,11 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: RowR // Only `null` when a `cancellationGuard` is supplied and the SQL guard // rejects the write — this route doesn't pass one, so reaching null is a bug. if (!updatedRow) throw new Error('updateRow returned null without a cancellationGuard') + // Auto-dispatch for user edits is handled inside `updateRow` (mode: 'new'). + // Firing a second mode: 'incomplete' dispatch here would race with the + // `mode: 'new'` one AND bulk-clear sibling-group outputs (the incomplete + // bulk-clear wipes ALL targeted columns when any one column on the row + // is empty). return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 414ac57d41a..8e29e12005c 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -1,8 +1,8 @@ import { db } from '@sim/db' -import { userTableRows } from '@sim/db/schema' +import { tableRowExecutions, userTableRows } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { and, eq, sql } from 'drizzle-orm' +import { and, eq, inArray, sql } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { type BatchInsertTableRowsBodyInput, @@ -17,7 +17,14 @@ import { isZodError, validationErrorResponse } from '@/lib/api/server/validation import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { Filter, RowData, Sort, TableSchema } from '@/lib/table' +import type { + Filter, + RowData, + RowExecutionMetadata, + RowExecutions, + Sort, + TableSchema, +} from '@/lib/table' import { batchInsertRows, batchUpdateRows, @@ -283,7 +290,6 @@ export const GET = withRouteHandler( .select({ id: userTableRows.id, data: userTableRows.data, - executions: userTableRows.executions, position: userTableRows.position, createdAt: userTableRows.createdAt, updatedAt: userTableRows.updatedAt, @@ -313,6 +319,41 @@ export const GET = withRouteHandler( const rows = await query.limit(validated.limit).offset(validated.offset) + // Sidecar: fetch per-(row, group) execution state and group into a map + // so the response preserves the legacy `row.executions[groupId]` wire + // shape. One indexed-IN scan against table_row_executions. + const executionsByRow = new Map() + if (rows.length > 0) { + const execRows = await db + .select() + .from(tableRowExecutions) + .where( + inArray( + tableRowExecutions.rowId, + rows.map((r) => r.id) + ) + ) + for (const e of execRows) { + const existing = executionsByRow.get(e.rowId) ?? {} + const meta: RowExecutionMetadata = { + status: e.status as RowExecutionMetadata['status'], + executionId: e.executionId ?? null, + jobId: e.jobId ?? null, + workflowId: e.workflowId, + error: e.error ?? null, + ...(e.runningBlockIds && e.runningBlockIds.length > 0 + ? { runningBlockIds: e.runningBlockIds } + : {}), + ...(e.blockErrors && Object.keys(e.blockErrors as Record).length > 0 + ? { blockErrors: e.blockErrors as Record } + : {}), + ...(e.cancelledAt ? { cancelledAt: e.cancelledAt.toISOString() } : {}), + } + existing[e.groupId] = meta + executionsByRow.set(e.rowId, existing) + } + } + logger.info( `[${requestId}] Queried ${rows.length} rows from table ${tableId} (total: ${totalCount ?? 'n/a'})` ) @@ -323,7 +364,7 @@ export const GET = withRouteHandler( rows: rows.map((r) => ({ id: r.id, data: r.data, - executions: r.executions ?? {}, + executions: executionsByRow.get(r.id) ?? {}, position: r.position, createdAt: r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt), diff --git a/apps/sim/app/api/table/import-csv/route.test.ts b/apps/sim/app/api/table/import-csv/route.test.ts new file mode 100644 index 00000000000..9844bf69664 --- /dev/null +++ b/apps/sim/app/api/table/import-csv/route.test.ts @@ -0,0 +1,104 @@ +/** + * @vitest-environment node + */ +import { hybridAuthMockFns, permissionsMock, permissionsMockFns } from '@sim/testing' +import type { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCreateTable, mockParseCsvBuffer, mockGetWorkspaceTableLimits } = vi.hoisted(() => ({ + mockCreateTable: vi.fn(), + mockParseCsvBuffer: vi.fn(), + mockGetWorkspaceTableLimits: vi.fn(), +})) + +vi.mock('@sim/utils/id', () => ({ + generateId: vi.fn().mockReturnValue('deadbeefcafef00d'), + generateShortId: vi.fn().mockReturnValue('short-id'), +})) + +vi.mock('@/lib/table', () => ({ + batchInsertRows: vi.fn(), + CSV_MAX_BATCH_SIZE: 1000, + CSV_MAX_FILE_SIZE_BYTES: 25 * 1024 * 1024, + coerceRowsForTable: vi.fn(), + createTable: mockCreateTable, + deleteTable: vi.fn(), + getWorkspaceTableLimits: mockGetWorkspaceTableLimits, + inferSchemaFromCsv: vi.fn(), + parseCsvBuffer: mockParseCsvBuffer, + sanitizeName: vi.fn((name: string) => name), + TABLE_LIMITS: { + MAX_TABLE_NAME_LENGTH: 64, + }, +})) + +vi.mock('@/app/api/table/utils', () => ({ + normalizeColumn: vi.fn((column) => column), +})) + +vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) + +import { POST } from '@/app/api/table/import-csv/route' + +function createCsvFile(contents: string, name = 'data.csv', type = 'text/csv'): File { + return new File([contents], name, { type }) +} + +function createFormData(file: File): FormData { + const form = new FormData() + form.append('file', file) + form.append('workspaceId', 'workspace-1') + return form +} + +async function callPost(form: FormData) { + const req = { + formData: async () => form, + } as unknown as NextRequest + return POST(req) +} + +describe('POST /api/table/import-csv', () => { + beforeEach(() => { + vi.clearAllMocks() + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType: 'session', + }) + permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') + mockGetWorkspaceTableLimits.mockResolvedValue({ + maxRowsPerTable: 1000, + maxTables: 10, + }) + }) + + it('returns 413 for oversized CSV files before reading their contents or creating a table', async () => { + const file = createCsvFile('name,age\nAlice,30') + Object.defineProperty(file, 'size', { + value: 26 * 1024 * 1024, + }) + const arrayBufferSpy = vi.spyOn(file, 'arrayBuffer') + + const response = await callPost(createFormData(file)) + const data = await response.json() + + expect(response.status).toBe(413) + expect(data.error).toMatch(/CSV import file exceeds maximum size/) + expect(arrayBufferSpy).not.toHaveBeenCalled() + expect(mockParseCsvBuffer).not.toHaveBeenCalled() + expect(mockCreateTable).not.toHaveBeenCalled() + }) + + it('accepts chunked multipart requests without a content-length header', async () => { + const req = { + headers: new Headers({ 'transfer-encoding': 'chunked' }), + formData: vi.fn(async () => createFormData(createCsvFile('name\nAlice'))), + } as unknown as NextRequest + + const response = await POST(req) + + expect(response.status).not.toBe(411) + expect(req.formData).toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/table/import-csv/route.ts b/apps/sim/app/api/table/import-csv/route.ts index 11951d0cb20..31927889202 100644 --- a/apps/sim/app/api/table/import-csv/route.ts +++ b/apps/sim/app/api/table/import-csv/route.ts @@ -6,10 +6,16 @@ import { csvExtensionSchema, csvImportFormSchema } from '@/lib/api/contracts/tab import { getValidationErrorMessage } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' +import { + isPayloadSizeLimitError, + readFileToBufferWithLimit, + readFormDataWithLimit, +} from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { batchInsertRows, CSV_MAX_BATCH_SIZE, + CSV_MAX_FILE_SIZE_BYTES, coerceRowsForTable, createTable, deleteTable, @@ -24,6 +30,7 @@ import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { normalizeColumn } from '@/app/api/table/utils' const logger = createLogger('TableImportCSV') +const MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024 export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() @@ -34,16 +41,21 @@ export const POST = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) } - const formData = await request.formData() + const formData = await readFormDataWithLimit(request, { + maxBytes: CSV_MAX_FILE_SIZE_BYTES + MAX_MULTIPART_OVERHEAD_BYTES, + label: 'CSV import body', + }) const validation = csvImportFormSchema.safeParse({ file: formData.get('file'), workspaceId: formData.get('workspaceId'), }) if (!validation.success) { + const message = getValidationErrorMessage(validation.error) + const isSizeLimit = message.includes('File exceeds maximum allowed size') return NextResponse.json( - { error: getValidationErrorMessage(validation.error) }, - { status: 400 } + { error: isSizeLimit ? 'CSV import file exceeds maximum size' : message }, + { status: isSizeLimit ? 413 : 400 } ) } @@ -63,7 +75,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const buffer = Buffer.from(await file.arrayBuffer()) + const buffer = await readFileToBufferWithLimit(file, { + maxBytes: CSV_MAX_FILE_SIZE_BYTES, + label: 'CSV import file', + }) const delimiter = extensionValidation.data === 'tsv' ? '\t' : ',' const { headers, rows } = await parseCsvBuffer(buffer, delimiter) @@ -132,16 +147,21 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const message = toError(error).message logger.error(`[${requestId}] CSV import failed:`, error) + const isSizeLimitError = + isPayloadSizeLimitError(error) || message.includes('CSV import file exceeds maximum size') const isClientError = message.includes('maximum table limit') || message.includes('CSV file has no') || message.includes('Invalid table name') || message.includes('Invalid schema') || - message.includes('already exists') + message.includes('already exists') || + isSizeLimitError return NextResponse.json( { error: isClientError ? message : 'Failed to import CSV' }, - { status: isClientError ? 400 : 500 } + { + status: isSizeLimitError ? 413 : isClientError ? 400 : 500, + } ) } }) diff --git a/apps/sim/app/api/templates/[id]/route.ts b/apps/sim/app/api/templates/[id]/route.ts index bb2e4a48c97..8f4c0e367c5 100644 --- a/apps/sim/app/api/templates/[id]/route.ts +++ b/apps/sim/app/api/templates/[id]/route.ts @@ -311,7 +311,17 @@ export const DELETE = withRouteHandler( return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } - const existing = await db.select().from(templates).where(eq(templates.id, id)).limit(1) + const existing = await db + .select({ + name: templates.name, + workflowId: templates.workflowId, + creatorId: templates.creatorId, + status: templates.status, + tags: templates.tags, + }) + .from(templates) + .where(eq(templates.id, id)) + .limit(1) if (existing.length === 0) { logger.warn(`[${requestId}] Template not found for delete: ${id}`) return NextResponse.json({ error: 'Template not found' }, { status: 404 }) diff --git a/apps/sim/app/api/tools/docusign/route.ts b/apps/sim/app/api/tools/docusign/route.ts index c88878bb73b..594587fc510 100644 --- a/apps/sim/app/api/tools/docusign/route.ts +++ b/apps/sim/app/api/tools/docusign/route.ts @@ -4,30 +4,92 @@ import { type NextRequest, NextResponse } from 'next/server' import { docusignToolContract } from '@/lib/api/contracts/tools/docusign' import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + assertKnownSizeWithinLimit, + DEFAULT_MAX_ERROR_BODY_BYTES, + isPayloadSizeLimitError, + readResponseJsonWithLimit, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { assertToolFileAccess } from '@/app/api/files/authorization' const logger = createLogger('DocuSignAPI') +const MAX_DOCUSIGN_DOCUMENT_BYTES = 25 * 1024 * 1024 +const MAX_LEGACY_INLINE_DOCUMENT_BYTES = 7 * 1024 * 1024 +const MAX_DOCUSIGN_JSON_BYTES = 2 * 1024 * 1024 +const DOCUSIGN_FETCH_TIMEOUT_MS = 30_000 interface DocuSignAccountInfo { accountId: string baseUri: string } +async function readDocusignJson( + response: Response, + label: string +): Promise> { + return readResponseJsonWithLimit>(response, { + maxBytes: MAX_DOCUSIGN_JSON_BYTES, + label, + }) +} + +function docusignError(data: Record, fallback: string): string { + return ( + (typeof data.message === 'string' && data.message) || + (typeof data.errorCode === 'string' && data.errorCode) || + fallback + ) +} + +async function fetchDocusign( + input: string, + init: RequestInit = {}, + parentSignal?: AbortSignal +): Promise { + const controller = new AbortController() + const timeout = setTimeout(() => { + controller.abort(new Error('DocuSign request timed out')) + }, DOCUSIGN_FETCH_TIMEOUT_MS) + const abort = () => controller.abort(parentSignal?.reason ?? new Error('Request aborted')) + parentSignal?.addEventListener('abort', abort, { once: true }) + + try { + return await fetch(input, { ...init, signal: controller.signal }) + } finally { + clearTimeout(timeout) + parentSignal?.removeEventListener('abort', abort) + } +} + /** * Resolves the user's DocuSign account info from their access token * by calling the DocuSign userinfo endpoint. */ -async function resolveAccount(accessToken: string): Promise { - const response = await fetch('https://account-d.docusign.com/oauth/userinfo', { - headers: { Authorization: `Bearer ${accessToken}` }, - }) +async function resolveAccount( + accessToken: string, + signal?: AbortSignal +): Promise { + const response = await fetchDocusign( + 'https://account-d.docusign.com/oauth/userinfo', + { + headers: { Authorization: `Bearer ${accessToken}` }, + }, + signal + ) if (!response.ok) { - const errorText = await response.text() + const errorText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'DocuSign account error response', + }).catch(() => '') logger.error('Failed to resolve DocuSign account', { status: response.status, error: errorText, @@ -35,10 +97,16 @@ async function resolveAccount(accessToken: string): Promise throw new Error(`Failed to resolve DocuSign account: ${response.status}`) } - const data = await response.json() - const accounts = data.accounts ?? [] + const data = await readDocusignJson(response, 'DocuSign account response') + const accounts = Array.isArray(data.accounts) + ? (data.accounts as Array<{ + is_default?: boolean + base_uri?: string + account_id?: string + }>) + : [] - const defaultAccount = accounts.find((a: { is_default: boolean }) => a.is_default) ?? accounts[0] + const defaultAccount = accounts.find((account) => account.is_default) ?? accounts[0] if (!defaultAccount) { throw new Error('No DocuSign accounts found for this user') } @@ -47,9 +115,13 @@ async function resolveAccount(accessToken: string): Promise if (!baseUri) { throw new Error('DocuSign account is missing base_uri') } + const accountId = defaultAccount.account_id + if (!accountId) { + throw new Error('DocuSign account is missing account_id') + } return { - accountId: defaultAccount.account_id, + accountId, baseUri, } } @@ -77,7 +149,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const { accessToken, operation, ...params } = parsed.data.body try { - const account = await resolveAccount(accessToken) + const account = await resolveAccount(accessToken, request.signal) const apiBase = `${account.baseUri}/restapi/v2.1/accounts/${account.accountId}` const headers: Record = { Authorization: `Bearer ${accessToken}`, @@ -86,21 +158,27 @@ export const POST = withRouteHandler(async (request: NextRequest) => { switch (operation) { case 'send_envelope': - return await handleSendEnvelope(apiBase, headers, params, authResult.userId) + return await handleSendEnvelope(apiBase, headers, params, authResult.userId, request.signal) case 'create_from_template': - return await handleCreateFromTemplate(apiBase, headers, params) + return await handleCreateFromTemplate(apiBase, headers, params, request.signal) case 'get_envelope': - return await handleGetEnvelope(apiBase, headers, params) + return await handleGetEnvelope(apiBase, headers, params, request.signal) case 'list_envelopes': - return await handleListEnvelopes(apiBase, headers, params) + return await handleListEnvelopes(apiBase, headers, params, request.signal) case 'void_envelope': - return await handleVoidEnvelope(apiBase, headers, params) + return await handleVoidEnvelope(apiBase, headers, params, request.signal) case 'download_document': - return await handleDownloadDocument(apiBase, headers, params) + return await handleDownloadDocument( + apiBase, + headers, + params, + authResult.userId, + request.signal + ) case 'list_templates': - return await handleListTemplates(apiBase, headers, params) + return await handleListTemplates(apiBase, headers, params, request.signal) case 'list_recipients': - return await handleListRecipients(apiBase, headers, params) + return await handleListRecipients(apiBase, headers, params, request.signal) default: return NextResponse.json( { success: false, error: `Unknown operation: ${operation}` }, @@ -110,7 +188,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { logger.error('DocuSign API error', { operation, error }) const message = getErrorMessage(error, 'Internal server error') - return NextResponse.json({ success: false, error: message }, { status: 500 }) + return NextResponse.json( + { success: false, error: message }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) } }) @@ -118,7 +199,8 @@ async function handleSendEnvelope( apiBase: string, headers: Record, params: Record, - userId: string + userId: string, + signal?: AbortSignal ) { const { signerEmail, signerName, emailSubject, emailBody, ccEmail, ccName, file, status } = params @@ -140,15 +222,29 @@ async function handleSendEnvelope( const userFile = userFiles[0] const denied = await assertToolFileAccess(userFile.key, userId, 'docusign-send', logger) if (denied) return denied - const buffer = await downloadFileFromStorage(userFile, 'docusign-send', logger) + if (userFile.size > MAX_DOCUSIGN_DOCUMENT_BYTES) { + return NextResponse.json( + { success: false, error: 'Document is too large to send through DocuSign' }, + { status: 413 } + ) + } + const buffer = await downloadFileFromStorage(userFile, 'docusign-send', logger, { + maxBytes: MAX_DOCUSIGN_DOCUMENT_BYTES, + }) + assertKnownSizeWithinLimit(buffer.length, MAX_DOCUSIGN_DOCUMENT_BYTES, 'DocuSign document') documentBase64 = buffer.toString('base64') documentName = userFile.name } } catch (fileError) { logger.error('Failed to process file for DocuSign envelope', { fileError }) return NextResponse.json( - { success: false, error: 'Failed to process uploaded file' }, - { status: 400 } + { + success: false, + error: isPayloadSizeLimitError(fileError) + ? getErrorMessage(fileError, 'Document is too large to send through DocuSign') + : 'Failed to process uploaded file', + }, + { status: isPayloadSizeLimitError(fileError) ? 413 : 400 } ) } } @@ -216,17 +312,21 @@ async function handleSendEnvelope( ) } - const response = await fetch(`${apiBase}/envelopes`, { - method: 'POST', - headers, - body: JSON.stringify(envelopeBody), - }) + const response = await fetchDocusign( + `${apiBase}/envelopes`, + { + method: 'POST', + headers, + body: JSON.stringify(envelopeBody), + }, + signal + ) - const data = await response.json() + const data = await readDocusignJson(response, 'DocuSign send envelope response') if (!response.ok) { logger.error('DocuSign send envelope failed', { data, status: response.status }) return NextResponse.json( - { success: false, error: data.message || data.errorCode || 'Failed to send envelope' }, + { success: false, error: docusignError(data, 'Failed to send envelope') }, { status: response.status } ) } @@ -237,7 +337,8 @@ async function handleSendEnvelope( async function handleCreateFromTemplate( apiBase: string, headers: Record, - params: Record + params: Record, + signal?: AbortSignal ) { const { templateId, emailSubject, emailBody, templateRoles, status } = params @@ -270,19 +371,23 @@ async function handleCreateFromTemplate( if (emailSubject) envelopeBody.emailSubject = emailSubject if (emailBody) envelopeBody.emailBlurb = emailBody - const response = await fetch(`${apiBase}/envelopes`, { - method: 'POST', - headers, - body: JSON.stringify(envelopeBody), - }) + const response = await fetchDocusign( + `${apiBase}/envelopes`, + { + method: 'POST', + headers, + body: JSON.stringify(envelopeBody), + }, + signal + ) - const data = await response.json() + const data = await readDocusignJson(response, 'DocuSign create from template response') if (!response.ok) { logger.error('DocuSign create from template failed', { data, status: response.status }) return NextResponse.json( { success: false, - error: data.message || data.errorCode || 'Failed to create envelope from template', + error: docusignError(data, 'Failed to create envelope from template'), }, { status: response.status } ) @@ -294,22 +399,24 @@ async function handleCreateFromTemplate( async function handleGetEnvelope( apiBase: string, headers: Record, - params: Record + params: Record, + signal?: AbortSignal ) { const { envelopeId } = params if (!envelopeId) { return NextResponse.json({ success: false, error: 'envelopeId is required' }, { status: 400 }) } - const response = await fetch( + const response = await fetchDocusign( `${apiBase}/envelopes/${(envelopeId as string).trim()}?include=recipients,documents`, - { headers } + { headers }, + signal ) - const data = await response.json() + const data = await readDocusignJson(response, 'DocuSign envelope response') if (!response.ok) { return NextResponse.json( - { success: false, error: data.message || data.errorCode || 'Failed to get envelope' }, + { success: false, error: docusignError(data, 'Failed to get envelope') }, { status: response.status } ) } @@ -320,7 +427,8 @@ async function handleGetEnvelope( async function handleListEnvelopes( apiBase: string, headers: Record, - params: Record + params: Record, + signal?: AbortSignal ) { const queryParams = new URLSearchParams() @@ -338,12 +446,12 @@ async function handleListEnvelopes( if (params.searchText) queryParams.append('search_text', params.searchText as string) if (params.count) queryParams.append('count', params.count as string) - const response = await fetch(`${apiBase}/envelopes?${queryParams}`, { headers }) - const data = await response.json() + const response = await fetchDocusign(`${apiBase}/envelopes?${queryParams}`, { headers }, signal) + const data = await readDocusignJson(response, 'DocuSign envelope list response') if (!response.ok) { return NextResponse.json( - { success: false, error: data.message || data.errorCode || 'Failed to list envelopes' }, + { success: false, error: docusignError(data, 'Failed to list envelopes') }, { status: response.status } ) } @@ -354,7 +462,8 @@ async function handleListEnvelopes( async function handleVoidEnvelope( apiBase: string, headers: Record, - params: Record + params: Record, + signal?: AbortSignal ) { const { envelopeId, voidedReason } = params if (!envelopeId) { @@ -364,16 +473,20 @@ async function handleVoidEnvelope( return NextResponse.json({ success: false, error: 'voidedReason is required' }, { status: 400 }) } - const response = await fetch(`${apiBase}/envelopes/${(envelopeId as string).trim()}`, { - method: 'PUT', - headers, - body: JSON.stringify({ status: 'voided', voidedReason }), - }) + const response = await fetchDocusign( + `${apiBase}/envelopes/${(envelopeId as string).trim()}`, + { + method: 'PUT', + headers, + body: JSON.stringify({ status: 'voided', voidedReason }), + }, + signal + ) - const data = await response.json() + const data = await readDocusignJson(response, 'DocuSign void envelope response') if (!response.ok) { return NextResponse.json( - { success: false, error: data.message || data.errorCode || 'Failed to void envelope' }, + { success: false, error: docusignError(data, 'Failed to void envelope') }, { status: response.status } ) } @@ -384,7 +497,9 @@ async function handleVoidEnvelope( async function handleDownloadDocument( apiBase: string, headers: Record, - params: Record + params: Record, + userId: string, + signal?: AbortSignal ) { const { envelopeId, documentId } = params if (!envelopeId) { @@ -393,17 +508,21 @@ async function handleDownloadDocument( const docId = (documentId as string) || 'combined' - const response = await fetch( + const response = await fetchDocusign( `${apiBase}/envelopes/${(envelopeId as string).trim()}/documents/${docId}`, { headers: { Authorization: headers.Authorization }, - } + }, + signal ) if (!response.ok) { let errorText = '' try { - errorText = await response.text() + errorText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'DocuSign document error response', + }) } catch { // ignore } @@ -422,16 +541,50 @@ async function handleDownloadDocument( fileName = filenameMatch[1].replace(/['"]/g, '') } - const buffer = Buffer.from(await response.arrayBuffer()) - const base64Content = buffer.toString('base64') + const buffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_DOCUSIGN_DOCUMENT_BYTES, + label: 'DocuSign document download', + }) + + const workspaceId = typeof params.workspaceId === 'string' ? params.workspaceId : undefined + const workflowId = typeof params.workflowId === 'string' ? params.workflowId : undefined + const executionId = typeof params.executionId === 'string' ? params.executionId : undefined + const legacyInlineContent = + buffer.length <= MAX_LEGACY_INLINE_DOCUMENT_BYTES + ? { base64Content: buffer.toString('base64') } + : {} + + if (workspaceId && workflowId && executionId) { + const file = await uploadExecutionFile( + { workspaceId, workflowId, executionId }, + buffer, + fileName, + contentType, + userId + ) + return NextResponse.json({ + file, + mimeType: contentType, + fileName, + ...legacyInlineContent, + }) + } + + const file = await uploadCopilotFile({ + buffer, + fileName, + contentType, + userId, + }) - return NextResponse.json({ base64Content, mimeType: contentType, fileName }) + return NextResponse.json({ file, mimeType: contentType, fileName, ...legacyInlineContent }) } async function handleListTemplates( apiBase: string, headers: Record, - params: Record + params: Record, + signal?: AbortSignal ) { const queryParams = new URLSearchParams() if (params.searchText) queryParams.append('search_text', params.searchText as string) @@ -440,12 +593,12 @@ async function handleListTemplates( const queryString = queryParams.toString() const url = queryString ? `${apiBase}/templates?${queryString}` : `${apiBase}/templates` - const response = await fetch(url, { headers }) - const data = await response.json() + const response = await fetchDocusign(url, { headers }, signal) + const data = await readDocusignJson(response, 'DocuSign template list response') if (!response.ok) { return NextResponse.json( - { success: false, error: data.message || data.errorCode || 'Failed to list templates' }, + { success: false, error: docusignError(data, 'Failed to list templates') }, { status: response.status } ) } @@ -456,21 +609,26 @@ async function handleListTemplates( async function handleListRecipients( apiBase: string, headers: Record, - params: Record + params: Record, + signal?: AbortSignal ) { const { envelopeId } = params if (!envelopeId) { return NextResponse.json({ success: false, error: 'envelopeId is required' }, { status: 400 }) } - const response = await fetch(`${apiBase}/envelopes/${(envelopeId as string).trim()}/recipients`, { - headers, - }) - const data = await response.json() + const response = await fetchDocusign( + `${apiBase}/envelopes/${(envelopeId as string).trim()}/recipients`, + { + headers, + }, + signal + ) + const data = await readDocusignJson(response, 'DocuSign recipients response') if (!response.ok) { return NextResponse.json( - { success: false, error: data.message || data.errorCode || 'Failed to list recipients' }, + { success: false, error: docusignError(data, 'Failed to list recipients') }, { status: response.status } ) } diff --git a/apps/sim/app/api/tools/google_slides/export-presentation/route.ts b/apps/sim/app/api/tools/google_slides/export-presentation/route.ts new file mode 100644 index 00000000000..5c36785f8eb --- /dev/null +++ b/apps/sim/app/api/tools/google_slides/export-presentation/route.ts @@ -0,0 +1,176 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { type NextRequest, NextResponse } from 'next/server' +import { googleSlidesExportPresentationContract } from '@/lib/api/contracts/tools/google' +import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + isPayloadSizeLimitError, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +import { presentationUrl } from '@/tools/google_slides/utils' + +const logger = createLogger('GoogleSlidesExportAPI') +const MAX_GOOGLE_SLIDES_EXPORT_BYTES = 10 * 1024 * 1024 +const MAX_LEGACY_INLINE_EXPORT_BYTES = 7 * 1024 * 1024 + +const FORMAT_TO_MIME = { + PDF: 'application/pdf', + PPTX: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + ODP: 'application/vnd.oasis.opendocument.presentation', + TXT: 'text/plain', + PNG: 'image/png', + JPEG: 'image/jpeg', + SVG: 'image/svg+xml', +} as const + +export const dynamic = 'force-dynamic' + +function buildExportUrl(presentationId: string, exportFormat: keyof typeof FORMAT_TO_MIME): string { + const mimeType = FORMAT_TO_MIME[exportFormat] + return `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(presentationId)}/export?mimeType=${encodeURIComponent(mimeType)}` +} + +function buildExportFilename( + presentationId: string, + exportFormat: keyof typeof FORMAT_TO_MIME +): string { + return `${presentationId}.${exportFormat.toLowerCase()}` +} + +export const POST = withRouteHandler(async (request: NextRequest) => { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success || !authResult.userId) { + return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + + const parsed = await parseRequest( + googleSlidesExportPresentationContract, + request, + {}, + { + validationErrorResponse: (error) => + NextResponse.json( + { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, + { status: 400 } + ), + } + ) + if (!parsed.success) return parsed.response + + try { + const body = parsed.data.body + const exportFormat = body.exportFormat ?? 'PDF' + const mimeType = FORMAT_TO_MIME[exportFormat] + const exportUrl = buildExportUrl(body.presentationId, exportFormat) + const urlValidation = await validateUrlWithDNS(exportUrl, 'googleSlidesExportUrl') + if (!urlValidation.isValid) { + return NextResponse.json( + { success: false, error: urlValidation.error || 'Invalid Google Slides export URL' }, + { status: 400 } + ) + } + + const response = await secureFetchWithPinnedIP(exportUrl, urlValidation.resolvedIP!, { + headers: { Authorization: `Bearer ${body.accessToken}` }, + maxResponseBytes: MAX_GOOGLE_SLIDES_EXPORT_BYTES, + }) + + if (!response.ok) { + const errorText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Google Slides export error response', + }).catch(() => '') + return NextResponse.json( + { + success: false, + error: `Failed to export presentation: ${response.status} ${errorText}`, + }, + { status: response.status } + ) + } + + const buffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_GOOGLE_SLIDES_EXPORT_BYTES, + label: 'Google Slides export response', + }) + const filename = buildExportFilename(body.presentationId, exportFormat) + const legacyInlineContent = + buffer.length <= MAX_LEGACY_INLINE_EXPORT_BYTES + ? { contentBase64: buffer.toString('base64') } + : {} + const executionContext = + body.workspaceId && body.workflowId && body.executionId + ? { + workspaceId: body.workspaceId, + workflowId: body.workflowId, + executionId: body.executionId, + } + : undefined + + if (executionContext) { + const file = await uploadExecutionFile( + executionContext, + buffer, + filename, + mimeType, + authResult.userId + ) + return NextResponse.json({ + success: true, + output: { + file: { ...file, mimeType }, + exportFormat, + mimeType, + sizeBytes: buffer.length, + exportUrl: file.url, + ...legacyInlineContent, + metadata: { + presentationId: body.presentationId, + url: presentationUrl(body.presentationId), + exportFormat, + }, + }, + }) + } + + const file = await uploadCopilotFile({ + buffer, + fileName: filename, + contentType: mimeType, + userId: authResult.userId, + }) + + return NextResponse.json({ + success: true, + output: { + file, + exportUrl: file.url, + exportFormat, + mimeType, + sizeBytes: buffer.length, + ...legacyInlineContent, + metadata: { + presentationId: body.presentationId, + url: presentationUrl(body.presentationId), + exportFormat, + }, + }, + }) + } catch (error) { + logger.error('Google Slides export failed', { error }) + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Failed to export presentation') }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) + } +}) diff --git a/apps/sim/app/api/tools/image/route.ts b/apps/sim/app/api/tools/image/route.ts index d48e5dffd80..b1643542402 100644 --- a/apps/sim/app/api/tools/image/route.ts +++ b/apps/sim/app/api/tools/image/route.ts @@ -21,10 +21,21 @@ import { validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' +import { + assertKnownSizeWithinLimit, + consumeOrCancelBody, + DEFAULT_MAX_ERROR_BODY_BYTES, + isPayloadSizeLimitError, + readResponseJsonWithLimit, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' import { getBaseUrl } from '@/lib/core/utils/urls' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' const logger = createLogger('ImageProxyAPI') +const MAX_IMAGE_BYTES = 25 * 1024 * 1024 +const MAX_IMAGE_JSON_BYTES = Math.ceil((MAX_IMAGE_BYTES * 4) / 3) + 256 * 1024 export const dynamic = 'force-dynamic' export const maxDuration = 600 @@ -116,7 +127,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { logger.error(`[${requestId}] Image generation failed:`, error) const errorMessage = getErrorMessage(error, 'Image generation failed') - return NextResponse.json({ error: errorMessage }, { status: 500 }) + return NextResponse.json( + { error: errorMessage }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) } const storedImage = await storeGeneratedImage(imageResult, body, authResult.userId, requestId) @@ -131,7 +145,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { logger.error(`[${requestId}] Image generation route error:`, error) const errorMessage = getErrorMessage(error, 'Unknown error') - return NextResponse.json({ error: errorMessage }, { status: 500 }) + return NextResponse.json( + { error: errorMessage }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) } }) @@ -172,6 +189,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { try { const imageResponse = await secureFetchWithPinnedIP(imageUrl, urlValidation.resolvedIP!, { method: 'GET', + maxResponseBytes: MAX_IMAGE_BYTES, headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', @@ -186,6 +204,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { }) if (!imageResponse.ok) { + await consumeOrCancelBody(imageResponse) logger.error(`[${requestId}] Image fetch failed:`, { status: imageResponse.status, statusText: imageResponse.statusText, @@ -197,14 +216,17 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const contentType = imageResponse.headers.get('content-type') || 'image/jpeg' - const imageArrayBuffer = await imageResponse.arrayBuffer() + const imageBuffer = await readResponseToBufferWithLimit(imageResponse, { + maxBytes: MAX_IMAGE_BYTES, + label: 'image proxy response', + }) - if (imageArrayBuffer.byteLength === 0) { + if (imageBuffer.length === 0) { logger.error(`[${requestId}] Empty image received`) return new NextResponse('Empty image received', { status: 404 }) } - return new NextResponse(imageArrayBuffer, { + return new NextResponse(new Uint8Array(imageBuffer), { headers: { 'Content-Type': contentType, 'Access-Control-Allow-Origin': '*', @@ -216,7 +238,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { logger.error(`[${requestId}] Image proxy error:`, { error: errorMessage }) return new NextResponse(`Failed to proxy image: ${errorMessage}`, { - status: 500, + status: isPayloadSizeLimitError(error) ? 413 : 500, }) } }) @@ -458,9 +480,11 @@ async function bufferFromImageUrl(url: string): Promise<{ buffer: Buffer; conten if (url.startsWith('data:')) { const match = /^data:([^;]+);base64,(.+)$/u.exec(url) if (!match) throw new Error('Invalid data URI image response') + const buffer = Buffer.from(match[2], 'base64') + assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'inline image response') return { contentType: match[1], - buffer: Buffer.from(match[2], 'base64'), + buffer, } } @@ -471,15 +495,22 @@ async function bufferFromImageUrl(url: string): Promise<{ buffer: Buffer; conten const imageResponse = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, { method: 'GET', + maxResponseBytes: MAX_IMAGE_BYTES, }) if (!imageResponse.ok) { - await imageResponse.text().catch(() => {}) + await readResponseTextWithLimit(imageResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'generated image error response', + }).catch(() => '') throw new Error(`Failed to download generated image: ${imageResponse.status}`) } const contentType = imageResponse.headers.get('content-type') || 'image/png' - const arrayBuffer = await imageResponse.arrayBuffer() - return { buffer: Buffer.from(arrayBuffer), contentType } + const buffer = await readResponseToBufferWithLimit(imageResponse, { + maxBytes: MAX_IMAGE_BYTES, + label: 'generated image download', + }) + return { buffer, contentType } } async function generateWithOpenAI( @@ -524,11 +555,17 @@ async function generateWithOpenAI( }) if (!openaiResponse.ok) { - const error = await openaiResponse.text() + const error = await readResponseTextWithLimit(openaiResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'OpenAI image error response', + }) throw new Error(`OpenAI API error: ${openaiResponse.status} - ${error}`) } - const data = (await openaiResponse.json()) as unknown + const data = await readResponseJsonWithLimit(openaiResponse, { + maxBytes: MAX_IMAGE_JSON_BYTES, + label: 'OpenAI image response', + }) if (!isRecord(data)) { throw new Error('Invalid OpenAI image response') } @@ -542,6 +579,7 @@ async function generateWithOpenAI( if (base64Image) { buffer = Buffer.from(base64Image, 'base64') + assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'OpenAI image response') } else if (imageUrl) { const downloaded = await bufferFromImageUrl(imageUrl) buffer = downloaded.buffer @@ -611,11 +649,17 @@ async function generateWithGemini( ) if (!geminiResponse.ok) { - const error = await geminiResponse.text() + const error = await readResponseTextWithLimit(geminiResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Gemini image error response', + }) throw new Error(`Gemini API error: ${geminiResponse.status} - ${error}`) } - const data = (await geminiResponse.json()) as unknown + const data = await readResponseJsonWithLimit(geminiResponse, { + maxBytes: MAX_IMAGE_JSON_BYTES, + label: 'Gemini image response', + }) if (!isRecord(data)) { throw new Error('Invalid Gemini image response') } @@ -650,7 +694,11 @@ async function generateWithGemini( } return { - buffer: Buffer.from(base64Image, 'base64'), + buffer: (() => { + const buffer = Buffer.from(base64Image, 'base64') + assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'Gemini image response') + return buffer + })(), contentType, fileName: `gemini-${model}.${extensionFromContentType(contentType)}`, provider: 'gemini', @@ -767,11 +815,17 @@ async function generateWithFalAI( }) if (!createResponse.ok) { - const error = await createResponse.text() + const error = await readResponseTextWithLimit(createResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Fal.ai create error response', + }) throw new Error(`Fal.ai API error: ${createResponse.status} - ${error}`) } - const createData = (await createResponse.json()) as unknown + const createData = await readResponseJsonWithLimit(createResponse, { + maxBytes: MAX_IMAGE_JSON_BYTES, + label: 'Fal.ai create response', + }) if (!isRecord(createData)) { throw new Error('Invalid Fal.ai queue response') } @@ -804,11 +858,17 @@ async function generateWithFalAI( }) if (!statusResponse.ok) { - await statusResponse.text().catch(() => {}) + await readResponseTextWithLimit(statusResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Fal.ai status error response', + }).catch(() => '') throw new Error(`Fal.ai status check failed: ${statusResponse.status}`) } - const statusData = (await statusResponse.json()) as unknown + const statusData = await readResponseJsonWithLimit(statusResponse, { + maxBytes: MAX_IMAGE_JSON_BYTES, + label: 'Fal.ai status response', + }) if (!isRecord(statusData)) { throw new Error('Invalid Fal.ai status response') } @@ -830,11 +890,17 @@ async function generateWithFalAI( ) if (!resultResponse.ok) { - await resultResponse.text().catch(() => {}) + await readResponseTextWithLimit(resultResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Fal.ai result error response', + }).catch(() => '') throw new Error(`Failed to fetch Fal.ai result: ${resultResponse.status}`) } - const resultData = (await resultResponse.json()) as unknown + const resultData = await readResponseJsonWithLimit(resultResponse, { + maxBytes: MAX_IMAGE_JSON_BYTES, + label: 'Fal.ai result response', + }) if (!isRecord(resultData)) { throw new Error('Invalid Fal.ai result response') } diff --git a/apps/sim/app/api/tools/tts/route.ts b/apps/sim/app/api/tools/tts/route.ts index 7a82e57eebf..366d2ee03ee 100644 --- a/apps/sim/app/api/tools/tts/route.ts +++ b/apps/sim/app/api/tools/tts/route.ts @@ -7,11 +7,16 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import { + isPayloadSizeLimitError, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' import { getBaseUrl } from '@/lib/core/utils/urls' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { StorageService } from '@/lib/uploads' const logger = createLogger('ProxyTTSAPI') +const MAX_TTS_AUDIO_BYTES = 25 * 1024 * 1024 export const POST = withRouteHandler(async (request: NextRequest) => { try { @@ -35,8 +40,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) if (!parsed.success) return parsed.response - const { text, voiceId, apiKey, modelId, workspaceId, workflowId, executionId } = - parsed.data.body + const { + text, + voiceId, + apiKey, + modelId, + stability, + similarityBoost, + workspaceId, + workflowId, + executionId, + } = parsed.data.body const voiceIdValidation = validateAlphanumericId(voiceId, 'voiceId', 255) if (!voiceIdValidation.isValid) { @@ -57,6 +71,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const endpoint = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}` + const hasVoiceSetting = stability !== undefined || similarityBoost !== undefined + const voiceSettings = hasVoiceSetting + ? { + stability: stability ?? 0.5, + similarity_boost: similarityBoost ?? 0.75, + } + : undefined + const response = await fetch(endpoint, { method: 'POST', headers: { @@ -67,6 +89,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { body: JSON.stringify({ text, model_id: modelId, + ...(voiceSettings ? { voice_settings: voiceSettings } : {}), }), signal: AbortSignal.timeout(DEFAULT_EXECUTION_TIMEOUT_MS), }) @@ -80,14 +103,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const audioBlob = await response.blob() + const audioBuffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_TTS_AUDIO_BYTES, + label: 'TTS audio response', + signal: request.signal, + }) - if (audioBlob.size === 0) { + if (audioBuffer.length === 0) { logger.error('Empty audio received from ElevenLabs') return NextResponse.json({ error: 'Empty audio received' }, { status: 422 }) } - const audioBuffer = Buffer.from(await audioBlob.arrayBuffer()) const timestamp = Date.now() // Use execution storage for workflow tool calls, copilot for chat UI @@ -142,7 +168,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { error: `Internal Server Error: ${getErrorMessage(error, 'Unknown error')}`, }, - { status: 500 } + { status: isPayloadSizeLimitError(error) ? 413 : 500 } ) } }) diff --git a/apps/sim/app/api/tools/tts/unified/route.ts b/apps/sim/app/api/tools/tts/unified/route.ts index 3b6f0a78707..80cc10db05b 100644 --- a/apps/sim/app/api/tools/tts/unified/route.ts +++ b/apps/sim/app/api/tools/tts/unified/route.ts @@ -10,6 +10,13 @@ import { import { getValidationErrorMessage, parseRequest, validationErrorResponse } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import { + assertKnownSizeWithinLimit, + isPayloadSizeLimitError, + readResponseJsonWithLimit, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' import { getBaseUrl } from '@/lib/core/utils/urls' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { StorageService } from '@/lib/uploads' @@ -26,6 +33,36 @@ import type { import { getFileExtension, getMimeType } from '@/tools/tts/types' const logger = createLogger('TtsUnifiedProxyAPI') +const MAX_TTS_AUDIO_BYTES = 25 * 1024 * 1024 +const MAX_TTS_ERROR_BYTES = 64 * 1024 +const MAX_TTS_JSON_BYTES = Math.ceil((MAX_TTS_AUDIO_BYTES * 4) / 3) + 256 * 1024 + +async function readTtsErrorJson( + response: Response, + label: string +): Promise> { + return readResponseJsonWithLimit>(response, { + maxBytes: MAX_TTS_ERROR_BYTES, + label, + }).catch(() => ({})) +} + +function getTtsErrorMessage(error: Record, fallback: string): string { + const nested = error.error + if (typeof nested === 'object' && nested !== null && 'message' in nested) { + const message = (nested as { message?: unknown }).message + if (typeof message === 'string') return message + } + for (const key of ['message', 'err_msg', 'error_message', 'error', 'detail']) { + const value = error[key] + if (typeof value === 'string') return value + if (typeof value === 'object' && value !== null && 'message' in value) { + const message = (value as { message?: unknown }).message + if (typeof message === 'string') return message + } + } + return fallback +} export const dynamic = 'force-dynamic' export const maxDuration = 60 // 1 minute @@ -208,7 +245,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { logger.error(`[${requestId}] TTS synthesis failed:`, error) const errorMessage = getErrorMessage(error, 'TTS synthesis failed') - return NextResponse.json({ error: errorMessage }, { status: 500 }) + return NextResponse.json( + { error: errorMessage }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) } const timestamp = Date.now() @@ -277,7 +317,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { logger.error(`[${requestId}] TTS unified proxy error:`, error) const errorMessage = getErrorMessage(error, 'Unknown error') - return NextResponse.json({ error: errorMessage }, { status: 500 }) + return NextResponse.json( + { error: errorMessage }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) } }) @@ -303,13 +346,15 @@ async function synthesizeWithOpenAi( }) if (!response.ok) { - const error = await response.json().catch(() => ({})) - const errorMessage = error.error?.message || error.message || response.statusText + const error = await readTtsErrorJson(response, 'OpenAI TTS error response') + const errorMessage = getTtsErrorMessage(error, response.statusText) throw new Error(`OpenAI TTS API error: ${errorMessage}`) } - const arrayBuffer = await response.arrayBuffer() - const audioBuffer = Buffer.from(arrayBuffer) + const audioBuffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_TTS_AUDIO_BYTES, + label: 'OpenAI TTS audio response', + }) const mimeType = getMimeType(responseFormat) return { @@ -359,13 +404,15 @@ async function synthesizeWithDeepgram( }) if (!response.ok) { - const error = await response.json().catch(() => ({})) - const errorMessage = error.err_msg || error.message || response.statusText + const error = await readTtsErrorJson(response, 'Deepgram TTS error response') + const errorMessage = getTtsErrorMessage(error, response.statusText) throw new Error(`Deepgram TTS API error: ${errorMessage}`) } - const arrayBuffer = await response.arrayBuffer() - const audioBuffer = Buffer.from(arrayBuffer) + const audioBuffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_TTS_AUDIO_BYTES, + label: 'Deepgram TTS audio response', + }) let finalFormat: string = encoding if (container === 'wav') { @@ -422,16 +469,15 @@ async function synthesizeWithElevenLabs( }) if (!response.ok) { - const error = await response.json().catch(() => ({})) - const errorMessage = - typeof error.detail === 'string' - ? error.detail - : error.detail?.message || error.message || response.statusText + const error = await readTtsErrorJson(response, 'ElevenLabs TTS error response') + const errorMessage = getTtsErrorMessage(error, response.statusText) throw new Error(`ElevenLabs TTS API error: ${errorMessage}`) } - const arrayBuffer = await response.arrayBuffer() - const audioBuffer = Buffer.from(arrayBuffer) + const audioBuffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_TTS_AUDIO_BYTES, + label: 'ElevenLabs TTS audio response', + }) return { audioBuffer, @@ -509,9 +555,9 @@ async function synthesizeWithCartesia( }) if (!response.ok) { - const error = await response.json().catch(() => ({})) - const errorMessage = error.error || error.message || response.statusText - const errorDetail = error.detail || '' + const error = await readTtsErrorJson(response, 'Cartesia TTS error response') + const errorMessage = getTtsErrorMessage(error, response.statusText) + const errorDetail = typeof error.detail === 'string' ? error.detail : '' logger.error('Cartesia API error details:', { status: response.status, error: errorMessage, @@ -523,8 +569,10 @@ async function synthesizeWithCartesia( ) } - const arrayBuffer = await response.arrayBuffer() - const audioBuffer = Buffer.from(arrayBuffer) + const audioBuffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_TTS_AUDIO_BYTES, + label: 'Cartesia TTS audio response', + }) const format = outputFormat && typeof outputFormat === 'object' && 'container' in outputFormat @@ -616,12 +664,15 @@ async function synthesizeWithGoogle( ) if (!response.ok) { - const error = await response.json().catch(() => ({})) - const errorMessage = error.error?.message || error.message || response.statusText + const error = await readTtsErrorJson(response, 'Google TTS error response') + const errorMessage = getTtsErrorMessage(error, response.statusText) throw new Error(`Google Cloud TTS API error: ${errorMessage}`) } - const data = await response.json() + const data = await readResponseJsonWithLimit<{ audioContent?: string }>(response, { + maxBytes: MAX_TTS_JSON_BYTES, + label: 'Google TTS JSON response', + }) const audioContent = data.audioContent if (!audioContent) { @@ -629,6 +680,7 @@ async function synthesizeWithGoogle( } const audioBuffer = Buffer.from(audioContent, 'base64') + assertKnownSizeWithinLimit(audioBuffer.length, MAX_TTS_AUDIO_BYTES, 'Google TTS audio response') const format = audioEncoding.toLowerCase().replace('_', '') const mimeType = getMimeType(format) @@ -706,12 +758,17 @@ async function synthesizeWithAzure( }) if (!response.ok) { - const error = await response.text() + const error = await readResponseTextWithLimit(response, { + maxBytes: MAX_TTS_ERROR_BYTES, + label: 'Azure TTS error response', + }) throw new Error(`Azure TTS API error: ${error || response.statusText}`) } - const arrayBuffer = await response.arrayBuffer() - const audioBuffer = Buffer.from(arrayBuffer) + const audioBuffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_TTS_AUDIO_BYTES, + label: 'Azure TTS audio response', + }) const format = outputFormat.includes('mp3') ? 'mp3' : 'wav' const mimeType = getMimeType(format) @@ -768,13 +825,15 @@ async function synthesizeWithPlayHT( }) if (!response.ok) { - const error = await response.json().catch(() => ({})) - const errorMessage = error.error_message || error.message || response.statusText + const error = await readTtsErrorJson(response, 'PlayHT TTS error response') + const errorMessage = getTtsErrorMessage(error, response.statusText) throw new Error(`PlayHT TTS API error: ${errorMessage}`) } - const arrayBuffer = await response.arrayBuffer() - const audioBuffer = Buffer.from(arrayBuffer) + const audioBuffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_TTS_AUDIO_BYTES, + label: 'PlayHT TTS audio response', + }) const format = outputFormat || 'mp3' const mimeType = getMimeType(format) diff --git a/apps/sim/app/api/tools/typeform/files/route.ts b/apps/sim/app/api/tools/typeform/files/route.ts new file mode 100644 index 00000000000..f4ded92ff92 --- /dev/null +++ b/apps/sim/app/api/tools/typeform/files/route.ts @@ -0,0 +1,168 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { type NextRequest, NextResponse } from 'next/server' +import { typeformFilesContract } from '@/lib/api/contracts/tools/typeform' +import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' +import { checkInternalAuth } from '@/lib/auth/hybrid' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + isPayloadSizeLimitError, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' + +const logger = createLogger('TypeformFilesAPI') +const MAX_TYPEFORM_FILE_BYTES = 10 * 1024 * 1024 + +export const dynamic = 'force-dynamic' + +function buildTypeformFileUrl({ + formId, + responseId, + fieldId, + filename, + inline, +}: { + formId: string + responseId: string + fieldId: string + filename: string + inline?: boolean +}): string { + const encodedFormId = encodeURIComponent(formId) + const encodedResponseId = encodeURIComponent(responseId) + const encodedFieldId = encodeURIComponent(fieldId) + const encodedFilename = encodeURIComponent(filename) + const url = new URL( + `https://api.typeform.com/forms/${encodedFormId}/responses/${encodedResponseId}/fields/${encodedFieldId}/files/${encodedFilename}` + ) + if (inline !== undefined) { + url.searchParams.set('inline', String(inline)) + } + return url.toString() +} + +function getFilename( + response: { headers: { get(name: string): string | null } }, + fallback: string +): string { + const contentDisposition = response.headers.get('content-disposition') || '' + const filenameMatch = contentDisposition.match(/filename="(.+?)"/) + return filenameMatch?.[1] || fallback || 'typeform-file' +} + +export const POST = withRouteHandler(async (request: NextRequest) => { + const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) + if (!authResult.success || !authResult.userId) { + return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + + const parsed = await parseRequest( + typeformFilesContract, + request, + {}, + { + validationErrorResponse: (error) => + NextResponse.json( + { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, + { status: 400 } + ), + } + ) + if (!parsed.success) return parsed.response + + try { + const body = parsed.data.body + const fileUrl = buildTypeformFileUrl(body) + const urlValidation = await validateUrlWithDNS(fileUrl, 'typeformFileUrl') + if (!urlValidation.isValid) { + return NextResponse.json( + { success: false, error: urlValidation.error || 'Invalid Typeform file URL' }, + { status: 400 } + ) + } + + const response = await secureFetchWithPinnedIP(fileUrl, urlValidation.resolvedIP!, { + headers: { Authorization: `Bearer ${body.apiKey}` }, + maxResponseBytes: MAX_TYPEFORM_FILE_BYTES, + }) + + if (!response.ok) { + const errorText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Typeform file error response', + }).catch(() => '') + return NextResponse.json( + { + success: false, + error: `Failed to download Typeform file: ${response.status} ${errorText}`, + }, + { status: response.status } + ) + } + + const buffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_TYPEFORM_FILE_BYTES, + label: 'Typeform file download', + }) + const contentType = response.headers.get('content-type') || 'application/octet-stream' + const filename = getFilename(response, body.filename) + const executionContext = + body.workspaceId && body.workflowId && body.executionId + ? { + workspaceId: body.workspaceId, + workflowId: body.workflowId, + executionId: body.executionId, + } + : undefined + + if (executionContext) { + const file = await uploadExecutionFile( + executionContext, + buffer, + filename, + contentType, + authResult.userId + ) + return NextResponse.json({ + success: true, + output: { + fileUrl: file.url, + file: { ...file, mimeType: contentType }, + contentType, + filename, + }, + }) + } + + const file = await uploadCopilotFile({ + buffer, + fileName: filename, + contentType, + userId: authResult.userId, + }) + + return NextResponse.json({ + success: true, + output: { + fileUrl: file.url || fileUrl, + file, + contentType, + filename, + }, + }) + } catch (error) { + logger.error('Typeform file download failed', { error }) + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Failed to download Typeform file') }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) + } +}) diff --git a/apps/sim/app/api/tools/video/route.ts b/apps/sim/app/api/tools/video/route.ts index 693a6e192c2..0a6f3ddead4 100644 --- a/apps/sim/app/api/tools/video/route.ts +++ b/apps/sim/app/api/tools/video/route.ts @@ -7,16 +7,52 @@ import { videoProviders, videoToolContract } from '@/lib/api/contracts/tools/med import { getValidationErrorMessage, parseRequest, validationErrorResponse } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { + assertKnownSizeWithinLimit, + DEFAULT_MAX_ERROR_BODY_BYTES, + isPayloadSizeLimitError, + PayloadSizeLimitError, + readResponseJsonWithLimit, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { assertToolFileAccess } from '@/app/api/files/authorization' import type { UserFile } from '@/executor/types' const logger = createLogger('VideoProxyAPI') +const MAX_VIDEO_OUTPUT_BYTES = 250 * 1024 * 1024 +const MAX_VIDEO_REFERENCE_IMAGE_BYTES = 25 * 1024 * 1024 +const MAX_VIDEO_JSON_BYTES = 2 * 1024 * 1024 export const dynamic = 'force-dynamic' export const maxDuration = 600 // 10 minutes for video generation +async function readVideoResponseBuffer(response: Response, label: string): Promise { + return readResponseToBufferWithLimit(response, { + maxBytes: MAX_VIDEO_OUTPUT_BYTES, + label, + }) +} + +async function readVideoJson>( + response: Response, + label: string +): Promise { + return readResponseJsonWithLimit(response, { + maxBytes: MAX_VIDEO_JSON_BYTES, + label, + }) +} + +async function readVideoErrorText(response: Response, label: string): Promise { + return readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label, + }).catch(() => '') +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateId() logger.info(`[${requestId}] Video generation request started`) @@ -214,7 +250,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { logger.error(`[${requestId}] Video generation failed:`, error) const errorMessage = getErrorMessage(error, 'Video generation failed') - return NextResponse.json({ error: errorMessage }, { status: 500 }) + return NextResponse.json( + { error: errorMessage }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) } const executionContext = @@ -298,7 +337,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } catch (error) { logger.error(`[${requestId}] Video proxy error:`, error) const errorMessage = getErrorMessage(error, 'Unknown error') - return NextResponse.json({ error: errorMessage }, { status: 500 }) + return NextResponse.json( + { error: errorMessage }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) } }) @@ -333,7 +375,21 @@ async function generateWithRunway( } if (visualReference) { - const refBuffer = await downloadFileFromStorage(visualReference, requestId, logger) + if (visualReference.size > MAX_VIDEO_REFERENCE_IMAGE_BYTES) { + throw new PayloadSizeLimitError({ + label: 'video visual reference', + maxBytes: MAX_VIDEO_REFERENCE_IMAGE_BYTES, + observedBytes: visualReference.size, + }) + } + const refBuffer = await downloadFileFromStorage(visualReference, requestId, logger, { + maxBytes: MAX_VIDEO_REFERENCE_IMAGE_BYTES, + }) + assertKnownSizeWithinLimit( + refBuffer.length, + MAX_VIDEO_REFERENCE_IMAGE_BYTES, + 'video visual reference' + ) const refBase64 = refBuffer.toString('base64') createPayload.promptImage = `data:${visualReference.type};base64,${refBase64}` // Use promptImage } @@ -349,11 +405,11 @@ async function generateWithRunway( }) if (!createResponse.ok) { - const error = await createResponse.text() + const error = await readVideoErrorText(createResponse, 'Runway create error response') throw new Error(`Runway API error: ${createResponse.status} - ${error}`) } - const createData = await createResponse.json() + const createData = await readVideoJson<{ id: string }>(createResponse, 'Runway create response') const taskId = createData.id logger.info(`[${requestId}] Runway task created: ${taskId}`) @@ -373,24 +429,32 @@ async function generateWithRunway( }) if (!statusResponse.ok) { - await statusResponse.text().catch(() => {}) + await readVideoErrorText(statusResponse, 'Runway status error response') throw new Error(`Runway status check failed: ${statusResponse.status}`) } - const statusData = await statusResponse.json() + const statusData = await readVideoJson<{ + status?: string + output?: string[] + failure?: string + }>(statusResponse, 'Runway status response') if (statusData.status === 'SUCCEEDED') { logger.info(`[${requestId}] Runway generation completed after ${attempts * 5}s`) - const videoResponse = await fetch(statusData.output[0]) + const videoUrl = statusData.output?.[0] + if (!videoUrl) { + throw new Error('No video URL in response') + } + + const videoResponse = await fetch(videoUrl) if (!videoResponse.ok) { - await videoResponse.text().catch(() => {}) + await readVideoErrorText(videoResponse, 'Runway video error response') throw new Error(`Failed to download video: ${videoResponse.status}`) } - const arrayBuffer = await videoResponse.arrayBuffer() return { - buffer: Buffer.from(arrayBuffer), + buffer: await readVideoResponseBuffer(videoResponse, 'Runway video response'), width: dimensions.width, height: dimensions.height, jobId: taskId, @@ -455,11 +519,11 @@ async function generateWithVeo( ) if (!createResponse.ok) { - const error = await createResponse.text() + const error = await readVideoErrorText(createResponse, 'Veo create error response') throw new Error(`Veo API error: ${createResponse.status} - ${error}`) } - const createData = await createResponse.json() + const createData = await readVideoJson<{ name: string }>(createResponse, 'Veo create response') const operationName = createData.name logger.info(`[${requestId}] Veo operation created: ${operationName}`) @@ -481,11 +545,17 @@ async function generateWithVeo( ) if (!statusResponse.ok) { - await statusResponse.text().catch(() => {}) + await readVideoErrorText(statusResponse, 'Veo status error response') throw new Error(`Veo status check failed: ${statusResponse.status}`) } - const statusData = await statusResponse.json() + const statusData = await readVideoJson<{ + done?: boolean + error?: { message?: string } + response?: { + generateVideoResponse?: { generatedSamples?: Array<{ video?: { uri?: string } }> } + } + }>(statusResponse, 'Veo status response') if (statusData.done) { if (statusData.error) { @@ -506,13 +576,12 @@ async function generateWithVeo( }) if (!videoResponse.ok) { - await videoResponse.text().catch(() => {}) + await readVideoErrorText(videoResponse, 'Veo video error response') throw new Error(`Failed to download video: ${videoResponse.status}`) } - const arrayBuffer = await videoResponse.arrayBuffer() return { - buffer: Buffer.from(arrayBuffer), + buffer: await readVideoResponseBuffer(videoResponse, 'Veo video response'), width: dimensions.width, height: dimensions.height, jobId: operationName, @@ -570,11 +639,11 @@ async function generateWithLuma( }) if (!createResponse.ok) { - const error = await createResponse.text() + const error = await readVideoErrorText(createResponse, 'Luma create error response') throw new Error(`Luma API error: ${createResponse.status} - ${error}`) } - const createData = await createResponse.json() + const createData = await readVideoJson<{ id: string }>(createResponse, 'Luma create response') const generationId = createData.id logger.info(`[${requestId}] Luma generation created: ${generationId}`) @@ -596,11 +665,15 @@ async function generateWithLuma( ) if (!statusResponse.ok) { - await statusResponse.text().catch(() => {}) + await readVideoErrorText(statusResponse, 'Luma status error response') throw new Error(`Luma status check failed: ${statusResponse.status}`) } - const statusData = await statusResponse.json() + const statusData = await readVideoJson<{ + state?: string + failure_reason?: string + assets?: { video?: string } + }>(statusResponse, 'Luma status response') if (statusData.state === 'completed') { logger.info(`[${requestId}] Luma generation completed after ${attempts * 5}s`) @@ -612,13 +685,12 @@ async function generateWithLuma( const videoResponse = await fetch(videoUrl) if (!videoResponse.ok) { - await videoResponse.text().catch(() => {}) + await readVideoErrorText(videoResponse, 'Luma video error response') throw new Error(`Failed to download video: ${videoResponse.status}`) } - const arrayBuffer = await videoResponse.arrayBuffer() return { - buffer: Buffer.from(arrayBuffer), + buffer: await readVideoResponseBuffer(videoResponse, 'Luma video response'), width: dimensions.width, height: dimensions.height, jobId: generationId, @@ -677,7 +749,7 @@ async function generateWithMiniMax( }) if (!createResponse.ok) { - const errorText = await createResponse.text() + const errorText = await readVideoErrorText(createResponse, 'MiniMax create error response') if (createResponse.status === 401 || createResponse.status === 1004) { throw new Error( `MiniMax API authentication failed (${createResponse.status}). Please ensure you're using a valid MiniMax API key from platform.minimax.io. Error: ${errorText}` @@ -686,7 +758,10 @@ async function generateWithMiniMax( throw new Error(`MiniMax API error: ${createResponse.status} - ${errorText}`) } - const createData = await createResponse.json() + const createData = await readVideoJson<{ + base_resp?: { status_code?: number; status_msg?: string } + task_id?: string + }>(createResponse, 'MiniMax create response') // Check for error in response if (createData.base_resp?.status_code !== 0) { @@ -694,6 +769,9 @@ async function generateWithMiniMax( } const taskId = createData.task_id + if (!taskId) { + throw new Error('MiniMax response missing task_id') + } logger.info(`[${requestId}] MiniMax task created: ${taskId}`) @@ -714,11 +792,16 @@ async function generateWithMiniMax( ) if (!statusResponse.ok) { - await statusResponse.text().catch(() => {}) + await readVideoErrorText(statusResponse, 'MiniMax status error response') throw new Error(`MiniMax status check failed: ${statusResponse.status}`) } - const statusData = await statusResponse.json() + const statusData = await readVideoJson<{ + base_resp?: { status_code?: number; status_msg?: string } + status?: string + file_id?: string + error?: string + }>(statusResponse, 'MiniMax status response') if ( statusData.base_resp?.status_code !== 0 && @@ -748,11 +831,14 @@ async function generateWithMiniMax( ) if (!fileResponse.ok) { - await fileResponse.text().catch(() => {}) + await readVideoErrorText(fileResponse, 'MiniMax file error response') throw new Error(`Failed to download video: ${fileResponse.status}`) } - const fileData = await fileResponse.json() + const fileData = await readVideoJson<{ file?: { download_url?: string } }>( + fileResponse, + 'MiniMax file response' + ) const videoUrl = fileData.file?.download_url if (!videoUrl) { @@ -762,13 +848,12 @@ async function generateWithMiniMax( // Download the actual video file const videoResponse = await fetch(videoUrl) if (!videoResponse.ok) { - await videoResponse.text().catch(() => {}) + await readVideoErrorText(videoResponse, 'MiniMax video error response') throw new Error(`Failed to download video from URL: ${videoResponse.status}`) } - const arrayBuffer = await videoResponse.arrayBuffer() return { - buffer: Buffer.from(arrayBuffer), + buffer: await readVideoResponseBuffer(videoResponse, 'MiniMax video response'), width: dimensions.width, height: dimensions.height, jobId: taskId, @@ -1125,11 +1210,11 @@ async function generateWithFalAI( }) if (!createResponse.ok) { - const error = await createResponse.text() + const error = await readVideoErrorText(createResponse, 'Fal.ai create error response') throw new Error(`Fal.ai API error: ${createResponse.status} - ${error}`) } - const createData = (await createResponse.json()) as unknown + const createData = await readVideoJson(createResponse, 'Fal.ai queue response') if (!isRecord(createData)) { throw new Error('Invalid Fal.ai queue response') } @@ -1162,11 +1247,11 @@ async function generateWithFalAI( }) if (!statusResponse.ok) { - await statusResponse.text().catch(() => {}) + await readVideoErrorText(statusResponse, 'Fal.ai status error response') throw new Error(`Fal.ai status check failed: ${statusResponse.status}`) } - const statusData = (await statusResponse.json()) as unknown + const statusData = await readVideoJson(statusResponse, 'Fal.ai status response') if (!isRecord(statusData)) { throw new Error('Invalid Fal.ai status response') } @@ -1189,11 +1274,11 @@ async function generateWithFalAI( ) if (!resultResponse.ok) { - await resultResponse.text().catch(() => {}) + await readVideoErrorText(resultResponse, 'Fal.ai result error response') throw new Error(`Failed to fetch result: ${resultResponse.status}`) } - const resultData = (await resultResponse.json()) as unknown + const resultData = await readVideoJson(resultResponse, 'Fal.ai result response') if (!isRecord(resultData)) { throw new Error('Invalid Fal.ai result response') } @@ -1208,12 +1293,10 @@ async function generateWithFalAI( const videoResponse = await fetch(videoUrl) if (!videoResponse.ok) { - await videoResponse.text().catch(() => {}) + await readVideoErrorText(videoResponse, 'Fal.ai video error response') throw new Error(`Failed to download video: ${videoResponse.status}`) } - const arrayBuffer = await videoResponse.arrayBuffer() - let width = getNumberProperty(videoOutput, 'width') || 1920 let height = getNumberProperty(videoOutput, 'height') || 1080 @@ -1224,7 +1307,7 @@ async function generateWithFalAI( } return { - buffer: Buffer.from(arrayBuffer), + buffer: await readVideoResponseBuffer(videoResponse, 'Fal.ai video response'), width, height, jobId: requestIdFal, diff --git a/apps/sim/app/api/users/me/settings/route.ts b/apps/sim/app/api/users/me/settings/route.ts index 0ad9cfa1865..47e0ac452eb 100644 --- a/apps/sim/app/api/users/me/settings/route.ts +++ b/apps/sim/app/api/users/me/settings/route.ts @@ -39,7 +39,24 @@ export const GET = withRouteHandler(async () => { } const userId = session.user.id - const result = await db.select().from(settings).where(eq(settings.userId, userId)).limit(1) + const result = await db + .select({ + theme: settings.theme, + autoConnect: settings.autoConnect, + telemetryEnabled: settings.telemetryEnabled, + emailPreferences: settings.emailPreferences, + billingUsageNotificationsEnabled: settings.billingUsageNotificationsEnabled, + showTrainingControls: settings.showTrainingControls, + superUserModeEnabled: settings.superUserModeEnabled, + mothershipEnvironment: settings.mothershipEnvironment, + errorNotificationsEnabled: settings.errorNotificationsEnabled, + snapToGridSize: settings.snapToGridSize, + showActionBar: settings.showActionBar, + lastActiveWorkspaceId: settings.lastActiveWorkspaceId, + }) + .from(settings) + .where(eq(settings.userId, userId)) + .limit(1) if (!result.length) { return NextResponse.json({ data: defaultSettings }, { status: 200 }) diff --git a/apps/sim/app/api/v1/admin/organizations/[id]/billing/route.ts b/apps/sim/app/api/v1/admin/organizations/[id]/billing/route.ts index df91e08bf25..9dd083a30a2 100644 --- a/apps/sim/app/api/v1/admin/organizations/[id]/billing/route.ts +++ b/apps/sim/app/api/v1/admin/organizations/[id]/billing/route.ts @@ -55,7 +55,11 @@ export const GET = withRouteHandler( try { if (!isBillingEnabled) { const [[orgData], [memberCount]] = await Promise.all([ - db.select().from(organization).where(eq(organization.id, organizationId)).limit(1), + db + .select({ id: organization.id, name: organization.name }) + .from(organization) + .where(eq(organization.id, organizationId)) + .limit(1), db .select({ count: count() }) .from(member) diff --git a/apps/sim/app/api/v1/admin/organizations/route.ts b/apps/sim/app/api/v1/admin/organizations/route.ts index 9e3b8b59834..07ee15890e8 100644 --- a/apps/sim/app/api/v1/admin/organizations/route.ts +++ b/apps/sim/app/api/v1/admin/organizations/route.ts @@ -71,7 +71,22 @@ export const GET = withRouteHandler( try { const [countResult, organizations] = await Promise.all([ db.select({ total: count() }).from(organization), - db.select().from(organization).orderBy(organization.name).limit(limit).offset(offset), + db + .select({ + id: organization.id, + name: organization.name, + slug: organization.slug, + logo: organization.logo, + orgUsageLimit: organization.orgUsageLimit, + storageUsedBytes: organization.storageUsedBytes, + departedMemberUsage: organization.departedMemberUsage, + createdAt: organization.createdAt, + updatedAt: organization.updatedAt, + }) + .from(organization) + .orderBy(organization.name) + .limit(limit) + .offset(offset), ]) const total = countResult[0].total diff --git a/apps/sim/app/api/v1/admin/types.ts b/apps/sim/app/api/v1/admin/types.ts index 525f22671b2..3cdbfc46d27 100644 --- a/apps/sim/app/api/v1/admin/types.ts +++ b/apps/sim/app/api/v1/admin/types.ts @@ -198,7 +198,23 @@ export interface AdminWorkflowDetail extends AdminWorkflow { edgeCount: number } -export function toAdminWorkflow(dbWorkflow: DbWorkflow): AdminWorkflow { +export type AdminWorkflowSource = Pick< + DbWorkflow, + | 'id' + | 'name' + | 'description' + | 'color' + | 'workspaceId' + | 'folderId' + | 'isDeployed' + | 'deployedAt' + | 'runCount' + | 'lastRunAt' + | 'createdAt' + | 'updatedAt' +> + +export function toAdminWorkflow(dbWorkflow: AdminWorkflowSource): AdminWorkflow { return { id: dbWorkflow.id, name: dbWorkflow.name, @@ -443,7 +459,20 @@ export interface AdminOrganizationDetail extends AdminOrganization { subscription: AdminSubscription | null } -export function toAdminOrganization(dbOrg: DbOrganization): AdminOrganization { +export type AdminOrganizationSource = Pick< + DbOrganization, + | 'id' + | 'name' + | 'slug' + | 'logo' + | 'orgUsageLimit' + | 'storageUsedBytes' + | 'departedMemberUsage' + | 'createdAt' + | 'updatedAt' +> + +export function toAdminOrganization(dbOrg: AdminOrganizationSource): AdminOrganization { return { id: dbOrg.id, name: dbOrg.name, diff --git a/apps/sim/app/api/v1/admin/workflows/route.ts b/apps/sim/app/api/v1/admin/workflows/route.ts index 1b8477df165..0a85dd78345 100644 --- a/apps/sim/app/api/v1/admin/workflows/route.ts +++ b/apps/sim/app/api/v1/admin/workflows/route.ts @@ -33,7 +33,25 @@ export const GET = withRouteHandler( try { const [countResult, workflows] = await Promise.all([ db.select({ total: count() }).from(workflow), - db.select().from(workflow).orderBy(workflow.name).limit(limit).offset(offset), + db + .select({ + id: workflow.id, + name: workflow.name, + description: workflow.description, + color: workflow.color, + workspaceId: workflow.workspaceId, + folderId: workflow.folderId, + isDeployed: workflow.isDeployed, + deployedAt: workflow.deployedAt, + runCount: workflow.runCount, + lastRunAt: workflow.lastRunAt, + createdAt: workflow.createdAt, + updatedAt: workflow.updatedAt, + }) + .from(workflow) + .orderBy(workflow.name) + .limit(limit) + .offset(offset), ]) const total = countResult[0].total diff --git a/apps/sim/app/api/v1/admin/workspaces/[id]/workflows/route.ts b/apps/sim/app/api/v1/admin/workspaces/[id]/workflows/route.ts index 1184b781ed9..8a841ee6ba9 100644 --- a/apps/sim/app/api/v1/admin/workspaces/[id]/workflows/route.ts +++ b/apps/sim/app/api/v1/admin/workspaces/[id]/workflows/route.ts @@ -63,7 +63,20 @@ export const GET = withRouteHandler( .from(workflow) .where(and(eq(workflow.workspaceId, workspaceId), isNull(workflow.archivedAt))), db - .select() + .select({ + id: workflow.id, + name: workflow.name, + description: workflow.description, + color: workflow.color, + workspaceId: workflow.workspaceId, + folderId: workflow.folderId, + isDeployed: workflow.isDeployed, + deployedAt: workflow.deployedAt, + runCount: workflow.runCount, + lastRunAt: workflow.lastRunAt, + createdAt: workflow.createdAt, + updatedAt: workflow.updatedAt, + }) .from(workflow) .where(and(eq(workflow.workspaceId, workspaceId), isNull(workflow.archivedAt))) .orderBy(workflow.name) diff --git a/apps/sim/app/api/v1/files/route.ts b/apps/sim/app/api/v1/files/route.ts index a286a655de6..06d965ac028 100644 --- a/apps/sim/app/api/v1/files/route.ts +++ b/apps/sim/app/api/v1/files/route.ts @@ -5,6 +5,11 @@ import { type NextRequest, NextResponse } from 'next/server' import { v1ListFilesContract, v1UploadFileFormFieldsSchema } from '@/lib/api/contracts/v1/files' import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' +import { + isPayloadSizeLimitError, + readFileToBufferWithLimit, + readFormDataWithLimit, +} from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { FileConflictError, @@ -25,6 +30,7 @@ export const dynamic = 'force-dynamic' export const revalidate = 0 const MAX_FILE_SIZE = 100 * 1024 * 1024 // 100MB +const MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024 /** GET /api/v1/files — List all files in a workspace. */ export const GET = withRouteHandler(async (request: NextRequest) => { @@ -83,8 +89,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let formData: FormData try { - formData = await request.formData() - } catch { + formData = await readFormDataWithLimit(request, { + maxBytes: MAX_FILE_SIZE + MAX_MULTIPART_OVERHEAD_BYTES, + label: 'workspace file upload body', + }) + } catch (error) { + if (isPayloadSizeLimitError(error)) { + return NextResponse.json({ error: error.message }, { status: 413 }) + } return NextResponse.json( { error: 'Request body must be valid multipart form data' }, { status: 400 } @@ -117,14 +129,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { error: `File size exceeds 100MB limit (${(file.size / (1024 * 1024)).toFixed(2)}MB)`, }, - { status: 400 } + { status: 413 } ) } const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId, 'write') if (accessError) return accessError - const buffer = Buffer.from(await file.arrayBuffer()) + const buffer = await readFileToBufferWithLimit(file, { + maxBytes: MAX_FILE_SIZE, + label: 'workspace upload file', + }) const userFile = await uploadWorkspaceFile( workspaceId, @@ -172,6 +187,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { }, }) } catch (error) { + if (isPayloadSizeLimitError(error)) { + return NextResponse.json({ error: error.message }, { status: 413 }) + } + const errorMessage = getErrorMessage(error, 'Failed to upload file') const isDuplicate = error instanceof FileConflictError || errorMessage.includes('already exists') diff --git a/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts b/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts index 810bb0dfc65..4724b39b247 100644 --- a/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts +++ b/apps/sim/app/api/v1/tables/[tableId]/rows/[rowId]/route.ts @@ -144,6 +144,9 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: RowR if (!updatedRow) { return NextResponse.json({ error: 'Row not found' }, { status: 404 }) } + // Auto-dispatch for user edits is handled inside `updateRow` (mode: 'new'). + // Firing a second mode: 'incomplete' dispatch here would race with it AND + // bulk-clear sibling-group outputs. return NextResponse.json({ success: true, @@ -227,7 +230,7 @@ export const DELETE = withRouteHandler(async (request: NextRequest, context: Row eq(userTableRows.workspaceId, workspaceId) ) ) - .returning() + .returning({ id: userTableRows.id }) if (!deletedRow) { return NextResponse.json({ error: 'Row not found' }, { status: 404 }) diff --git a/apps/sim/app/api/webhooks/[id]/route.ts b/apps/sim/app/api/webhooks/[id]/route.ts index ab65529dfaa..ea79ee38f69 100644 --- a/apps/sim/app/api/webhooks/[id]/route.ts +++ b/apps/sim/app/api/webhooks/[id]/route.ts @@ -137,13 +137,23 @@ export const PATCH = withRouteHandler( } await assertWorkflowMutable(webhookData.workflow.id) + const setClause: Partial = {} + if (isActive !== undefined && isActive !== webhooks[0].webhook.isActive) { + setClause.isActive = isActive + } + if (failedCount !== undefined && failedCount !== webhooks[0].webhook.failedCount) { + setClause.failedCount = failedCount + } + + if (Object.keys(setClause).length === 0) { + logger.info(`[${requestId}] No-op webhook PATCH (no field changes): ${id}`) + return NextResponse.json({ webhook: webhooks[0].webhook }, { status: 200 }) + } + + setClause.updatedAt = new Date() const updatedWebhook = await db .update(webhook) - .set({ - isActive: isActive !== undefined ? isActive : webhooks[0].webhook.isActive, - failedCount: failedCount !== undefined ? failedCount : webhooks[0].webhook.failedCount, - updatedAt: new Date(), - }) + .set(setClause) .where(eq(webhook.id, id)) .returning() diff --git a/apps/sim/app/page.tsx b/apps/sim/app/page.tsx index c12a4a75e3d..58c64b1d8f0 100644 --- a/apps/sim/app/page.tsx +++ b/apps/sim/app/page.tsx @@ -31,7 +31,7 @@ export const metadata: Metadata = { locale: 'en_US', images: [ { - url: '/logo/426-240/primary/small.png', + url: '/logo/426-240/reverse/small.png', width: 2130, height: 1200, alt: 'Sim — The AI Workspace for Teams', @@ -47,7 +47,7 @@ export const metadata: Metadata = { description: 'Sim is the open-source AI workspace where teams build, deploy, and manage AI agents. Connect 1,000+ integrations and every major LLM to create agents that automate real work.', images: { - url: '/logo/426-240/primary/small.png', + url: '/logo/426-240/reverse/small.png', alt: 'Sim — The AI Workspace for Teams', }, }, diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx index 4bda12945a5..07a3cf78181 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx +++ b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/components/trace-view/trace-view.tsx @@ -32,6 +32,7 @@ import { import { cn } from '@/lib/core/utils/cn' import type { TraceSpan } from '@/lib/logs/types' import { + DEFAULT_BLOCK_COLOR, formatCostAmount, formatTokenCount, formatTps, @@ -120,6 +121,21 @@ function iconColorClass(bgColor: string): string { return r * 299 + g * 587 + b * 114 > 160_000 ? 'text-[#111111]' : 'text-white' } +/** + * Near-black bgColors disappear against the dark-mode surface (--bg: #1b1b1b). + * Below the luminance threshold we fall back to the neutral block color used + * for blocks with no distinct identity; everything brighter passes through. + */ +function adjustBgForContrast(bgColor: string): string { + const hex = bgColor.replace('#', '') + if (hex.length !== 6) return bgColor + const r = Number.parseInt(hex.slice(0, 2), 16) + const g = Number.parseInt(hex.slice(2, 4), 16) + const b = Number.parseInt(hex.slice(4, 6), 16) + if (r * 299 + g * 587 + b * 114 < 30_000) return DEFAULT_BLOCK_COLOR + return bgColor +} + /** * Flattens the visible (expanded) span tree into a linear list for keyboard * navigation, carrying depth, the chain of parent ids for indent drawing, and @@ -268,7 +284,12 @@ const TraceTreeRow = memo(function TraceTreeRow({ const duration = span.duration || endMs - startMs const isRootWorkflow = depth === 0 && span.type?.toLowerCase() === 'workflow' const hasError = isRootWorkflow ? hasUnhandledErrorInTree(span) : hasErrorInTree(span) - const { icon: BlockIcon, bgColor } = getBlockIconAndColor(span.type, span.name, span.provider) + const { icon: BlockIcon, bgColor: rawBgColor } = getBlockIconAndColor( + span.type, + span.name, + span.provider + ) + const bgColor = adjustBgForContrast(rawBgColor) const nameMatches = !!matchQuery && spanMatchesQuery(span, matchQuery) const offsetMs = runStartMs > 0 ? Math.max(0, startMs - runStartMs) : 0 @@ -651,7 +672,12 @@ const TraceDetailPane = memo(function TraceDetailPane({ span }: { span: TraceSpa } const duration = span.duration || parseTime(span.endTime) - parseTime(span.startTime) - const { icon: BlockIcon, bgColor } = getBlockIconAndColor(span.type, span.name, span.provider) + const { icon: BlockIcon, bgColor: rawBgColor } = getBlockIconAndColor( + span.type, + span.name, + span.provider + ) + const bgColor = adjustBgForContrast(rawBgColor) const isRootWorkflow = span.type?.toLowerCase() === 'workflow' const hasError = isRootWorkflow ? hasUnhandledErrorInTree(span) : hasErrorInTree(span) const isDirectError = span.status === 'error' diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/utils.ts b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/utils.ts index afdf52bdaab..4f142b0c67d 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/utils.ts @@ -8,6 +8,20 @@ import { getBlock, getBlockByToolName } from '@/blocks' import { PROVIDER_DEFINITIONS } from '@/providers/models' import { normalizeToolId } from '@/tools/normalize' +/** + * Extracts the bare tool name from an MCP tool id of the form + * `mcp-{serverId}-{toolName}`. Returns null when the id is not MCP-shaped. + * Kept local to avoid importing from `@/lib/mcp/utils`, which pulls in + * `next/server` and breaks client bundles. + */ +function tryParseMcpToolName(toolId: string): string | null { + if (!toolId.startsWith('mcp-')) return null + const parts = toolId.split('-') + if (parts.length < 3) return null + const toolName = parts.slice(2).join('-') + return toolName.length > 0 ? toolName : null +} + export const DEFAULT_BLOCK_COLOR = '#6b7280' export interface BlockIconAndColor { @@ -41,6 +55,10 @@ export function getBlockIconAndColor( ): BlockIconAndColor { const lowerType = type.toLowerCase() if (lowerType === 'tool' && toolName) { + if (tryParseMcpToolName(toolName)) { + const mcpBlock = getBlock('mcp') + if (mcpBlock) return { icon: mcpBlock.icon, bgColor: mcpBlock.bgColor } + } const normalized = normalizeToolId(toolName) if (normalized === 'load_skill') return { icon: AgentSkillsIcon, bgColor: '#8B5CF6' } const toolBlock = getBlockByToolName(normalized) @@ -90,7 +108,11 @@ export function formatTps( } export function getDisplayName(span: TraceSpan): string { - if (span.type?.toLowerCase() === 'tool') return normalizeToolId(span.name) + if (span.type?.toLowerCase() === 'tool') { + const mcpToolName = tryParseMcpToolName(span.name) + if (mcpToolName) return mcpToolName + return normalizeToolId(span.name) + } return span.name } diff --git a/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx b/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx index 7ddf9eec8f2..aaa32744da5 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx +++ b/apps/sim/app/workspace/[workspaceId]/logs/logs.tsx @@ -103,6 +103,8 @@ import { const LOGS_PER_PAGE = 50 as const const SORTABLE_COLUMNS: readonly LogSortBy[] = ['date', 'duration', 'cost', 'status'] as const const REFRESH_SPINNER_DURATION_MS = 1000 as const +const LIVE_REFRESH_INTERVAL_MS = 10_000 as const +const ACTIVE_RUN_DETAIL_REFRESH_MS = 3_000 as const const LOG_COLUMNS: ResourceColumn[] = [ { id: 'workflow', header: 'Workflow' }, @@ -317,7 +319,7 @@ export default function Logs() { (query: { state: { data?: WorkflowLogDetail } }) => { if (!isLive) return false const status = query.state.data?.status - return status === 'running' || status === 'pending' ? 3000 : false + return status === 'running' || status === 'pending' ? ACTIVE_RUN_DETAIL_REFRESH_MS : false }, [isLive] ) @@ -365,7 +367,7 @@ export default function Logs() { ) const logsQuery = useLogsList(workspaceId, logFilters, { - refetchInterval: isLive ? 3000 : false, + refetchInterval: isLive ? LIVE_REFRESH_INTERVAL_MS : false, }) const dashboardFilters = useMemo( @@ -383,7 +385,7 @@ export default function Logs() { ) const dashboardStatsQuery = useDashboardStats(workspaceId, dashboardFilters, { - refetchInterval: isLive ? 3000 : false, + refetchInterval: isLive ? LIVE_REFRESH_INTERVAL_MS : false, }) const logs = useMemo(() => { diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/components/form-field/form-field.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/components/form-field/form-field.tsx index 04beeb1484a..cad5381d1d2 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/components/form-field/form-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/components/form-field/form-field.tsx @@ -9,7 +9,7 @@ interface FormFieldProps { export function FormField({ label, children, optional }: FormFieldProps) { return (
-
) : ( -
+
+ + -
- Headers +
{(formData.headers || []).map((header, index) => ( ))}
-
+ + + + {showAdvanced && ( +
+ + { + if (testResult) clearTestResult() + if (submitError) setSubmitError(null) + setFormData((prev) => ({ ...prev, oauthClientId: e.target.value })) + }} + className='h-9' + /> + + + { + if (testResult) clearTestResult() + if (submitError) setSubmitError(null) + setOauthClientSecretTouched(value.length > 0) + setFormData((prev) => ({ ...prev, oauthClientSecret: value })) + }} + className='h-9' + /> + +

+ Only needed for servers that don't support automatic client registration. +

+
+ )}
)} - + {submitError && ( -

{submitError}

+

{submitError}

)}
@@ -716,7 +808,7 @@ export function McpServerFormModal({ )}
- {formMode === 'json' ? ( diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/mcp.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/mcp.tsx index a21da0f563e..e8bc927a284 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/mcp.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/mcp.tsx @@ -1,6 +1,6 @@ 'use client' -import { useCallback, useEffect, useMemo, useState } from 'react' +import { useEffect, useState } from 'react' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { ChevronDown, Plus, Search } from 'lucide-react' @@ -27,6 +27,7 @@ import { type McpToolIssue, } from '@/lib/mcp/tool-validation' import type { McpTransport } from '@/lib/mcp/types' +import { useMcpOauthPopup } from '@/hooks/mcp/use-mcp-oauth-popup' import { type McpServer, type McpTool, @@ -102,7 +103,10 @@ function ServerListItem({ ({transportLabel})

{isRefreshing ? 'Refreshing...' @@ -123,14 +127,29 @@ function ServerListItem({ ) } +function buildEditInitialData(server: McpServer) { + const entries: { key: string; value: string }[] = server.headers + ? Object.entries(server.headers).map(([key, value]) => ({ key, value })) + : [] + if (entries.length === 0) entries.push({ key: '', value: '' }) + const last = entries[entries.length - 1] + if (last.key !== '' || last.value !== '') entries.push({ key: '', value: '' }) + + return { + name: server.name || '', + transport: (server.transport as McpTransport) || 'streamable-http', + url: server.url || '', + timeout: 30000, + headers: entries, + oauthClientId: server.oauthClientId || undefined, + hasOauthClientSecret: server.hasOauthClientSecret === true, + } +} + interface MCPProps { initialServerId?: string | null } -/** - * MCP Settings component for managing Model Context Protocol servers. - * Handles server CRUD operations, connection testing, and environment variable integration. - */ export function MCP({ initialServerId }: MCPProps) { const params = useParams() const workspaceId = params.workspaceId as string @@ -147,7 +166,8 @@ export function MCP({ initialServerId }: MCPProps) { isFetching: toolsFetching, } = useMcpToolsQuery(workspaceId) const { data: storedTools = [], refetch: refetchStoredTools } = useStoredMcpTools(workspaceId) - const forceRefreshTools = useForceRefreshMcpTools() + const forceRefreshToolsMutation = useForceRefreshMcpTools() + const forceRefreshTools = forceRefreshToolsMutation.mutate const createServerMutation = useCreateMcpServer() const deleteServerMutation = useDeleteMcpServer() const refreshServerMutation = useRefreshMcpServer() @@ -156,23 +176,16 @@ export function MCP({ initialServerId }: MCPProps) { const { data: allowedMcpDomains = null } = useAllowedMcpDomains() const [showAddModal, setShowAddModal] = useState(false) - const [showEditModal, setShowEditModal] = useState(false) - const [editInitialData, setEditInitialData] = useState< - | { - name: string - transport: McpTransport - url?: string - timeout?: number - headers?: { key: string; value: string }[] - } - | undefined - >(undefined) + const [editingServerId, setEditingServerId] = useState(null) const [searchTerm, setSearchTerm] = useState('') const [deletingServers, setDeletingServers] = useState>(() => new Set()) + const { connectingServers: connectingOauthServers, startOauthForServer } = useMcpOauthPopup({ + workspaceId, + }) - const [showDeleteDialog, setShowDeleteDialog] = useState(false) - const [serverToDelete, setServerToDelete] = useState<{ id: string; name: string } | null>(null) + const [serverToDeleteId, setServerToDeleteId] = useState(null) + const showDeleteDialog = serverToDeleteId !== null const [selectedServerId, setSelectedServerId] = useState(initialServerId ?? null) @@ -185,28 +198,23 @@ export function MCP({ initialServerId }: MCPProps) { } }, []) - const [refreshingServers, setRefreshingServers] = useState< - Record - >({}) const [expandedTools, setExpandedTools] = useState>(() => new Set()) - const handleRemoveServer = useCallback((serverId: string, serverName: string) => { - setServerToDelete({ id: serverId, name: serverName }) - setShowDeleteDialog(true) - }, []) + const handleRemoveServer = (serverId: string) => { + setServerToDeleteId(serverId) + } - const confirmDeleteServer = useCallback(async () => { - if (!serverToDelete) return + const confirmDeleteServer = async () => { + if (!serverToDeleteId) return - setShowDeleteDialog(false) - const { id: serverId, name: serverName } = serverToDelete - setServerToDelete(null) + const serverId = serverToDeleteId + setServerToDeleteId(null) setDeletingServers((prev) => new Set(prev).add(serverId)) try { await deleteServerMutation.mutateAsync({ workspaceId, serverId }) - logger.info(`Removed MCP server: ${serverName}`) + logger.info(`Removed MCP server: ${serverId}`) } catch (error) { logger.error('Failed to remove MCP server:', error) } finally { @@ -216,43 +224,36 @@ export function MCP({ initialServerId }: MCPProps) { return newSet }) } - }, [serverToDelete, deleteServerMutation, workspaceId]) - - const toolsByServer = useMemo(() => { - return (mcpToolsData || []).reduce( - (acc, tool) => { - if (!tool?.serverId) return acc - if (!acc[tool.serverId]) { - acc[tool.serverId] = [] - } - acc[tool.serverId].push(tool) - return acc - }, - {} as Record - ) - }, [mcpToolsData]) - - const filteredServers = useMemo(() => { - return (servers || []).filter((server) => - server.name?.toLowerCase().includes(searchTerm.toLowerCase()) - ) - }, [servers, searchTerm]) + } - const handleViewDetails = useCallback( - (serverId: string) => { - setSelectedServerId(serverId) - forceRefreshTools(workspaceId) - refetchStoredTools() + const toolsByServer = (mcpToolsData || []).reduce( + (acc, tool) => { + if (!tool?.serverId) return acc + if (!acc[tool.serverId]) { + acc[tool.serverId] = [] + } + acc[tool.serverId].push(tool) + return acc }, - [workspaceId, forceRefreshTools, refetchStoredTools] + {} as Record + ) + + const filteredServers = (servers || []).filter((server) => + server.name?.toLowerCase().includes(searchTerm.toLowerCase()) ) - const handleBackToList = useCallback(() => { + const handleViewDetails = (serverId: string) => { + setSelectedServerId(serverId) + forceRefreshTools(workspaceId) + refetchStoredTools() + } + + const handleBackToList = () => { setSelectedServerId(null) setExpandedTools(new Set()) - }, []) + } - const toggleToolExpanded = useCallback((toolName: string) => { + const toggleToolExpanded = (toolName: string) => { setExpandedTools((prev) => { const newSet = new Set(prev) if (newSet.has(toolName)) { @@ -262,131 +263,109 @@ export function MCP({ initialServerId }: MCPProps) { } return newSet }) - }, []) + } - const handleRefreshServer = useCallback( - async (serverId: string) => { - try { - setRefreshingServers((prev) => ({ ...prev, [serverId]: { status: 'refreshing' } })) - const result = await refreshServerMutation.mutateAsync({ workspaceId, serverId }) - logger.info( - `Refreshed MCP server: ${serverId}, workflows updated: ${result.workflowsUpdated}` - ) - - const activeWorkflowId = useWorkflowRegistry.getState().activeWorkflowId - if (activeWorkflowId && result.updatedWorkflowIds?.includes(activeWorkflowId)) { - logger.info(`Active workflow ${activeWorkflowId} was updated, reloading subblock values`) - try { - const { data: workflowData } = await requestJson(getWorkflowStateContract, { - params: { id: activeWorkflowId }, - }) - if (workflowData?.state?.blocks) { - useSubBlockStore - .getState() - .initializeFromWorkflow( - activeWorkflowId, - workflowData.state.blocks as Record - ) - } - } catch (reloadError) { - logger.warn('Failed to reload workflow subblock values:', reloadError) - } - } + const handleRefreshServer = async (serverId: string) => { + try { + const result = await refreshServerMutation.mutateAsync({ workspaceId, serverId }) + logger.info( + `Refreshed MCP server: ${serverId}, workflows updated: ${result.workflowsUpdated}` + ) - setRefreshingServers((prev) => ({ - ...prev, - [serverId]: { status: 'refreshed', workflowsUpdated: result.workflowsUpdated }, - })) - setTimeout(() => { - setRefreshingServers((prev) => { - const newState = { ...prev } - delete newState[serverId] - return newState + const activeWorkflowId = useWorkflowRegistry.getState().activeWorkflowId + if (activeWorkflowId && result.updatedWorkflowIds?.includes(activeWorkflowId)) { + logger.info(`Active workflow ${activeWorkflowId} was updated, reloading subblock values`) + try { + const { data: workflowData } = await requestJson(getWorkflowStateContract, { + params: { id: activeWorkflowId }, }) - }, 3000) - } catch (error) { - logger.error('Failed to refresh MCP server:', error) - setRefreshingServers((prev) => { - const newState = { ...prev } - delete newState[serverId] - return newState - }) + if (workflowData?.state?.blocks) { + useSubBlockStore + .getState() + .initializeFromWorkflow( + activeWorkflowId, + workflowData.state.blocks as Record + ) + } + } catch (reloadError) { + logger.warn('Failed to reload workflow subblock values:', reloadError) + } } - }, - [refreshServerMutation, workspaceId] - ) - - const handleOpenEditModal = useCallback((server: McpServer) => { - const headers: { key: string; value: string }[] = server.headers - ? Object.entries(server.headers).map(([key, value]) => ({ key, value })) - : [{ key: '', value: '' }] - if (headers.length === 0) headers.push({ key: '', value: '' }) - - const lastHeader = headers[headers.length - 1] - if (lastHeader.key !== '' || lastHeader.value !== '') { - headers.push({ key: '', value: '' }) + } catch (error) { + logger.error('Failed to refresh MCP server:', error) } + } - setEditInitialData({ - name: server.name || '', - transport: (server.transport as McpTransport) || 'streamable-http', - url: server.url || '', - timeout: 30000, - headers, - }) - setShowEditModal(true) - }, []) - - const selectedServer = useMemo(() => { + useEffect(() => { + if (!refreshServerMutation.isSuccess) return + const timeout = window.setTimeout(() => refreshServerMutation.reset(), 3000) + return () => window.clearTimeout(timeout) + // eslint-disable-next-line react-hooks/exhaustive-deps -- mutation object is unstable; isSuccess flag is the trigger + }, [refreshServerMutation.isSuccess]) + + const refreshingServerId = refreshServerMutation.isPending + ? refreshServerMutation.variables?.serverId + : null + const refreshedServerId = refreshServerMutation.isSuccess + ? refreshServerMutation.variables?.serverId + : null + const refreshedWorkflowsUpdated = refreshServerMutation.data?.workflowsUpdated + + const editingServer = editingServerId + ? (servers.find((s) => s.id === editingServerId) as McpServer | undefined) + : undefined + const editInitialData = editingServer ? buildEditInitialData(editingServer) : undefined + + const selectedServer = (() => { if (!selectedServerId) return null const server = servers.find((s) => s.id === selectedServerId) as McpServer | undefined if (!server) return null const serverTools = (toolsByServer[selectedServerId] || []) as McpTool[] return { server, tools: serverTools } - }, [selectedServerId, servers, toolsByServer]) + })() + + const getStoredToolIssues = ( + serverId: string, + toolName: string + ): { issue: McpToolIssue; workflowName: string }[] => { + const relevantStoredTools = storedTools.filter( + (st) => st.serverId === serverId && st.toolName === toolName + ) - const getStoredToolIssues = useCallback( - (serverId: string, toolName: string): { issue: McpToolIssue; workflowName: string }[] => { - const relevantStoredTools = storedTools.filter( - (st) => st.serverId === serverId && st.toolName === toolName + const serverStates = servers.map((s) => ({ + id: s.id, + url: s.url, + connectionStatus: s.connectionStatus, + lastError: s.lastError || undefined, + })) + + const discoveredTools = mcpToolsData.map((t) => ({ + serverId: t.serverId, + name: t.name, + inputSchema: t.inputSchema, + })) + + const issues: { issue: McpToolIssue; workflowName: string }[] = [] + + for (const storedTool of relevantStoredTools) { + const issue = getMcpToolIssue( + { + serverId: storedTool.serverId, + serverUrl: storedTool.serverUrl, + toolName: storedTool.toolName, + schema: storedTool.schema, + }, + serverStates, + discoveredTools ) - const serverStates = servers.map((s) => ({ - id: s.id, - url: s.url, - connectionStatus: s.connectionStatus, - lastError: s.lastError || undefined, - })) - - const discoveredTools = mcpToolsData.map((t) => ({ - serverId: t.serverId, - name: t.name, - inputSchema: t.inputSchema, - })) - - const issues: { issue: McpToolIssue; workflowName: string }[] = [] - - for (const storedTool of relevantStoredTools) { - const issue = getMcpToolIssue( - { - serverId: storedTool.serverId, - serverUrl: storedTool.serverUrl, - toolName: storedTool.toolName, - schema: storedTool.schema, - }, - serverStates, - discoveredTools - ) - - if (issue) { - issues.push({ issue, workflowName: storedTool.workflowName }) - } + if (issue) { + issues.push({ issue, workflowName: storedTool.workflowName }) } + } - return issues - }, - [storedTools, servers, mcpToolsData] - ) + return issues + } const error = toolsError || serversError const hasServers = servers && servers.length > 0 @@ -422,12 +401,32 @@ export function MCP({ initialServerId }: MCPProps) { {server.connectionStatus === 'error' && (

Status -

+

{server.lastError || 'Unable to connect'}

)} + {server.authType === 'oauth' && server.connectionStatus !== 'connected' && ( +
+ + Authentication + +
+ +
+
+ )} +
Tools ({tools.length}) @@ -450,11 +449,12 @@ export function MCP({ initialServerId }: MCPProps) { key={tool.name} className='overflow-hidden rounded-md border bg-[var(--surface-3)]' > - + {isExpanded && hasParams && (
@@ -563,25 +563,27 @@ export function MCP({ initialServerId }: MCPProps) { -
{ + if (!open) setEditingServerId(null) + }} mode='edit' initialData={editInitialData} onSubmit={async (config) => { @@ -620,7 +622,7 @@ export function MCP({ initialServerId }: MCPProps) { />
@@ -628,7 +630,7 @@ export function MCP({ initialServerId }: MCPProps) {
{error ? (
-

+

{getErrorMessage(error, 'Failed to load MCP servers')}

@@ -656,8 +658,8 @@ export function MCP({ initialServerId }: MCPProps) { tools={tools} isDeleting={deletingServers.has(server.id)} isLoadingTools={isLoadingTools} - isRefreshing={refreshingServers[server.id]?.status === 'refreshing'} - onRemove={() => handleRemoveServer(server.id, server.name || 'this server')} + isRefreshing={refreshingServerId === server.id} + onRemove={() => handleRemoveServer(server.id)} onViewDetails={() => handleViewDetails(server.id)} /> ) @@ -677,28 +679,38 @@ export function MCP({ initialServerId }: MCPProps) { onOpenChange={setShowAddModal} mode='add' onSubmit={async (config) => { - await createServerMutation.mutateAsync({ + const result = await createServerMutation.mutateAsync({ workspaceId, config: { ...config, enabled: true }, }) + if (result.authType === 'oauth') { + await startOauthForServer(result.serverId) + } }} workspaceId={workspaceId} availableEnvVars={availableEnvVars} allowedMcpDomains={allowedMcpDomains} /> - + { + if (!open) setServerToDeleteId(null) + }} + > Delete MCP Server Are you sure you want to delete{' '} - {serverToDelete?.name} + + {servers.find((s) => s.id === serverToDeleteId)?.name || 'this server'} + ? This action cannot be undone. -