{title}

+ {/* Header container - stays in DOM to preserve height */} +

+ {/* Header - fades out when sidebar is collapsed */} - +

{title}

+ + + + {/* Divider - absolutely positioned, visible when sidebar is collapsed but section is expanded */} +

) { const ref = React.useRef(null); useShortcutKeys.useShortcutKeys({ @@ -176,14 +178,14 @@ function PopoverSideMenuTrigger({ {...props} ref={ref} className={cn( - "flex h-[1.8rem] shrink-0 select-none items-center gap-x-1.5 rounded-sm bg-transparent px-[0.4rem] text-center font-sans text-2sm font-normal text-text-bright transition duration-150 focus-custom hover:bg-charcoal-750", - shortcut ? "justify-between" : "", + "flex h-[1.8rem] shrink-0 select-none items-center rounded-sm bg-transparent pl-[0.4rem] pr-2.5 text-center font-sans text-2sm font-normal text-text-bright transition duration-150 focus-custom hover:bg-charcoal-750", + shortcut && !hideShortcutKey ? "justify-between gap-x-1.5" : "", className )} > {children} - {shortcut && ( - + {shortcut && !hideShortcutKey && ( + )} ); diff --git a/apps/webapp/app/hooks/useUser.ts b/apps/webapp/app/hooks/useUser.ts index e31455cf92a..fd9938fdb9a 100644 --- a/apps/webapp/app/hooks/useUser.ts +++ b/apps/webapp/app/hooks/useUser.ts @@ -1,10 +1,12 @@ -import { UIMatch } from "@remix-run/react"; -import type { User } from "~/models/user.server"; -import { loader } from "~/root"; +import { type UIMatch } from "@remix-run/react"; +import type { UserWithDashboardPreferences } from "~/models/user.server"; +import { type loader } from "~/root"; import { useChanged } from "./useChanged"; import { useTypedMatchesData } from "./useTypedMatchData"; import { useIsImpersonating } from "./useOrganizations"; +export type User = UserWithDashboardPreferences; + export function useOptionalUser(matches?: UIMatch[]): User | undefined { const routeMatch = useTypedMatchesData({ id: "root", diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam/route.tsx index aea6b345ee7..3f4be602aa7 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam/route.tsx @@ -19,7 +19,7 @@ export default function Project() { return ( <> -

(); const fetchIncidents = useCallback(() => { @@ -36,36 +34,50 @@ export function IncidentStatusPanel() { }, [fetcher]); useEffect(() => { + if (!isManagedCloud) return; + fetchIncidents(); const interval = setInterval(fetchIncidents, 60 * 1000); // 1 minute return () => clearInterval(interval); - }, []); + }, [isManagedCloud, fetchIncidents]); const operational = fetcher.data?.operational ?? true; + if (!isManagedCloud || operational) { + return null; + } + return ( - <> - {!operational && ( + +

+ {/* Expanded panel - animated height and opacity */}

+ {/* Header */}

Active incident

+ + {/* Description */} Our team is working on resolving the issue. Check our status page for more information. + + {/* Button */}

- )} - + + {/* Collapsed button - animated height and opacity */} + + + + + } + content="Active incident" + side="right" + sideOffset={8} + disableHoverableContent + asChild + /> + +

+ + + + + ); +} + +function IncidentPopoverContent() { + return ( +

+ + + Active incident + +

+ + Our team is working on resolving the issue. Check our status page for more information. + + + View status page + +

); } diff --git a/apps/webapp/app/routes/resources.preferences.sidemenu.tsx b/apps/webapp/app/routes/resources.preferences.sidemenu.tsx new file mode 100644 index 00000000000..9c3edc66be1 --- /dev/null +++ b/apps/webapp/app/routes/resources.preferences.sidemenu.tsx @@ -0,0 +1,35 @@ +import { json, type ActionFunctionArgs } from "@remix-run/node"; +import { z } from "zod"; +import { updateSideMenuPreferences } from "~/services/dashboardPreferences.server"; +import { requireUser } from "~/services/session.server"; + +// Transforms form data string "true"/"false" to boolean, or undefined if not present +const booleanFromFormData = z + .enum(["true", "false"]) + .transform((val) => val === "true") + .optional(); + +const RequestSchema = z.object({ + isCollapsed: booleanFromFormData, + manageSectionCollapsed: booleanFromFormData, +}); + +export async function action({ request }: ActionFunctionArgs) { + const user = await requireUser(request); + + const formData = await request.formData(); + const rawData = Object.fromEntries(formData); + + const result = RequestSchema.safeParse(rawData); + if (!result.success) { + return json({ success: false, error: "Invalid request data" }, { status: 400 }); + } + + await updateSideMenuPreferences({ + user, + isCollapsed: result.data.isCollapsed, + manageSectionCollapsed: result.data.manageSectionCollapsed, + }); + + return json({ success: true }); +} diff --git a/apps/webapp/app/services/dashboardPreferences.server.ts b/apps/webapp/app/services/dashboardPreferences.server.ts index 3649704b811..323b96fa88b 100644 --- a/apps/webapp/app/services/dashboardPreferences.server.ts +++ b/apps/webapp/app/services/dashboardPreferences.server.ts @@ -3,6 +3,13 @@ import { prisma } from "~/db.server"; import { logger } from "./logger.server"; import { type UserFromSession } from "./session.server"; +const SideMenuPreferences = z.object({ + isCollapsed: z.boolean().default(false), + manageSectionCollapsed: z.boolean().default(false), +}); + +export type SideMenuPreferences = z.infer; + const DashboardPreferences = z.object({ version: z.literal("1"), currentProjectId: z.string().optional(), @@ -12,6 +19,7 @@ const DashboardPreferences = z.object({ currentEnvironment: z.object({ id: z.string() }), }) ), + sideMenu: SideMenuPreferences.optional(), }); export type DashboardPreferences = z.infer; @@ -99,3 +107,47 @@ export async function clearCurrentProject({ user }: { user: UserFromSession }) { }, }); } + +export async function updateSideMenuPreferences({ + user, + isCollapsed, + manageSectionCollapsed, +}: { + user: UserFromSession; + isCollapsed?: boolean; + manageSectionCollapsed?: boolean; +}) { + if (user.isImpersonating) { + return; + } + + // Parse with schema to apply defaults, then overlay any new values + const currentSideMenu = SideMenuPreferences.parse(user.dashboardPreferences.sideMenu ?? {}); + const updatedSideMenu = SideMenuPreferences.parse({ + ...currentSideMenu, + ...(isCollapsed !== undefined && { isCollapsed }), + ...(manageSectionCollapsed !== undefined && { manageSectionCollapsed }), + }); + + // Only update if something changed + if ( + updatedSideMenu.isCollapsed === currentSideMenu.isCollapsed && + updatedSideMenu.manageSectionCollapsed === currentSideMenu.manageSectionCollapsed + ) { + return; + } + + const updatedPreferences: DashboardPreferences = { + ...user.dashboardPreferences, + sideMenu: updatedSideMenu, + }; + + return prisma.user.update({ + where: { + id: user.id, + }, + data: { + dashboardPreferences: updatedPreferences, + }, + }); +} From d4e4fbd7fc32cff50cde653d0af5373bc96e0451 Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Tue, 27 Jan 2026 18:26:43 +0000 Subject: [PATCH 004/400] fix(webapp): Truncate long branch names to prevent breaking the onboarding layout (#2954) ### Before CleanShot 2026-01-21 at 20 11 08@2x

### After

---

--- apps/webapp/app/components/BlankStatePanels.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/components/BlankStatePanels.tsx b/apps/webapp/app/components/BlankStatePanels.tsx index d0e798f1685..7423bd61ac8 100644 --- a/apps/webapp/app/components/BlankStatePanels.tsx +++ b/apps/webapp/app/components/BlankStatePanels.tsx @@ -599,9 +599,9 @@ function DeploymentOnboardingSteps() { return (

- - Deploy your tasks to {environmentFullTitle(environment)} +

+ + Deploy your tasks to {environmentFullTitle(environment)}

Date: Tue, 27 Jan 2026 18:33:36 +0000 Subject: [PATCH 005/400] Fix(webapp) prevent incidents url being called every frame (#2956) --- apps/webapp/app/routes/resources.incidents.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/webapp/app/routes/resources.incidents.tsx b/apps/webapp/app/routes/resources.incidents.tsx index fbdb00a1c5c..532038d4f99 100644 --- a/apps/webapp/app/routes/resources.incidents.tsx +++ b/apps/webapp/app/routes/resources.incidents.tsx @@ -31,7 +31,7 @@ export function IncidentStatusPanel({ isCollapsed = false }: { isCollapsed?: boo if (fetcher.state === "idle") { fetcher.load("/resources/incidents"); } - }, [fetcher]); + }, []); useEffect(() => { if (!isManagedCloud) return; From e29e1c86d9b9cdc01c36b5fbd083c857c5909180 Mon Sep 17 00:00:00 2001 From: Mihai Popescu Date: Thu, 29 Jan 2026 12:47:59 +0200 Subject: [PATCH 006/400] Fix/tri 7032 logs page feedback (#2947) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## ✨ Changes ### UI & UX - Normalized log level display across table and detail view - Fixed table header scroll behavior and sidebar positioning - Improved loading state with taller segment and disabled resizing - Added "no more logs" message with count - Enhanced keyboard shortcuts ### Filtering & Search - Streamlined filters: RunId and Task only (removed run filters) - Side panel closes when filters change - Fixed logs from previous search remaining in table - Fixed table scroll position when changing filters ### Backend - Added performance indexes on message and attributes (`014_add_task_runs_v2_search_indexes.sql`) - Added DEBUG level logging by default - Removed internal logs from display - Fixed ServiceValidationError forwarding to frontend - Removed v1 logs API support --- .../app/components/LogLevelTooltipInfo.tsx | 63 +++ apps/webapp/app/components/Shortcuts.tsx | 31 ++ .../app/components/logs/LogDetailView.tsx | 64 +-- .../app/components/logs/LogsLevelFilter.tsx | 125 ++---- .../app/components/logs/LogsRunIdFilter.tsx | 2 +- .../app/components/logs/LogsSearchInput.tsx | 14 +- apps/webapp/app/components/logs/LogsTable.tsx | 69 ++-- .../app/components/logs/LogsTaskFilter.tsx | 144 +++++++ .../app/components/primitives/Table.tsx | 19 +- apps/webapp/app/hooks/useCanViewLogsPage.ts | 16 + .../v3/LogDetailPresenter.server.ts | 5 +- .../presenters/v3/LogsListPresenter.server.ts | 276 ++++--------- .../route.tsx | 384 ++++++++++++------ .../route.tsx | 53 +++ ...ojects.$projectParam.env.$envParam.logs.ts | 34 +- .../route.tsx | 81 ++-- apps/webapp/app/utils/logUtils.ts | 50 +-- .../014_add_task_runs_v2_serch_indexes.sql | 20 + internal-packages/clickhouse/src/index.ts | 4 - .../clickhouse/src/taskEvents.ts | 54 +-- 20 files changed, 875 insertions(+), 633 deletions(-) create mode 100644 apps/webapp/app/components/LogLevelTooltipInfo.tsx create mode 100644 apps/webapp/app/components/logs/LogsTaskFilter.tsx create mode 100644 apps/webapp/app/hooks/useCanViewLogsPage.ts create mode 100644 apps/webapp/app/routes/resources.orgs.$organizationSlug.can-view-logs-page/route.tsx create mode 100644 internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql diff --git a/apps/webapp/app/components/LogLevelTooltipInfo.tsx b/apps/webapp/app/components/LogLevelTooltipInfo.tsx new file mode 100644 index 00000000000..6f967af70d1 --- /dev/null +++ b/apps/webapp/app/components/LogLevelTooltipInfo.tsx @@ -0,0 +1,63 @@ +import { BookOpenIcon } from "@heroicons/react/20/solid"; +import { LinkButton } from "./primitives/Buttons"; +import { Header3 } from "./primitives/Headers"; +import { Paragraph } from "./primitives/Paragraph"; + +export function LogLevelTooltipInfo() { + return ( +

+ Log Levels + + Structured logging helps you debug and monitor your tasks. + +

+ Info +

+ + General informational messages about task execution. + +

+ Warn +

+ + Warning messages indicating potential issues that don't prevent execution. + +

+ Error +

+ + Error messages for failures and exceptions during task execution. + +

+ Debug +

+ + Detailed diagnostic information for development and debugging. + +

+ Tracing & Spans + + Automatically track the flow of your code through task triggers, attempts, and HTTP + requests. Create custom traces to monitor specific operations. + +

+ + Read docs + +

+ ); +} diff --git a/apps/webapp/app/components/Shortcuts.tsx b/apps/webapp/app/components/Shortcuts.tsx index cf1b01a703f..e3e4d6fe957 100644 --- a/apps/webapp/app/components/Shortcuts.tsx +++ b/apps/webapp/app/components/Shortcuts.tsx @@ -161,6 +161,37 @@ function ShortcutContent() {

+ Logs page + + + + + + + + + + + + + to + + + + + + + + + + + + + + + +

Schedules page diff --git a/apps/webapp/app/components/logs/LogDetailView.tsx b/apps/webapp/app/components/logs/LogDetailView.tsx index a367e75495a..22e2e288ac4 100644 --- a/apps/webapp/app/components/logs/LogDetailView.tsx +++ b/apps/webapp/app/components/logs/LogDetailView.tsx @@ -21,7 +21,7 @@ import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; import type { LogEntry } from "~/presenters/v3/LogsListPresenter.server"; -import { getLevelColor, getKindColor, getKindLabel } from "~/utils/logUtils"; +import { getLevelColor } from "~/utils/logUtils"; import { v3RunSpanPath, v3RunsPath, v3DeploymentVersionPath } from "~/utils/pathBuilder"; import type { loader as logDetailLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId"; import { TaskRunStatusCombo, descriptionForTaskRunStatus } from "~/components/runs/v3/TaskRunStatus"; @@ -94,16 +94,34 @@ export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDet const isLoading = fetcher.state === "loading"; const log = fetcher.data ?? initialLog; - // Handle Escape key to close panel + const runPath = v3RunSpanPath( + organization, + project, + environment, + { friendlyId: log?.runId ?? "" }, + { spanId: log?.spanId ?? "" } + ); + + // Handle keyboard shortcuts useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { + const target = e.target as HTMLElement; + if (target && ( + target.tagName === "INPUT" || + target.tagName === "TEXTAREA" || + target.tagName === "SELECT" || + target.contentEditable === "true" + )) { + return; + } + if (e.key === "Escape") { onClose(); } }; window.addEventListener("keydown", handleKeyDown); return () => window.removeEventListener("keydown", handleKeyDown); - }, [onClose]); + }, [onClose, log, runPath, isLoading]); if (isLoading && !log) { return ( @@ -129,36 +147,18 @@ export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDet ); } - const runPath = v3RunSpanPath( - organization, - project, - environment, - { friendlyId: log.runId }, - { spanId: log.spanId } - ); - return (

{/* Header */} -

- - {getKindLabel(log.kind)} - - - {log.level} - -

+ + {log.level} + @@ -185,8 +185,8 @@ export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDet -

diff --git a/apps/webapp/app/components/logs/LogsLevelFilter.tsx b/apps/webapp/app/components/logs/LogsLevelFilter.tsx index 4ec7d957304..8c2abf64f25 100644 --- a/apps/webapp/app/components/logs/LogsLevelFilter.tsx +++ b/apps/webapp/app/components/logs/LogsLevelFilter.tsx @@ -1,9 +1,8 @@ import * as Ariakit from "@ariakit/react"; -import { ExclamationTriangleIcon } from "@heroicons/react/20/solid"; -import { type ReactNode, useMemo } from "react"; +import { IconListTree } from "@tabler/icons-react"; +import { type ReactNode } from "react"; import { AppliedFilter } from "~/components/primitives/AppliedFilter"; import { - ComboBox, SelectItem, SelectList, SelectPopover, @@ -12,24 +11,20 @@ import { shortcutFromIndex, } from "~/components/primitives/Select"; import { useSearchParams } from "~/hooks/useSearchParam"; -import { FilterMenuProvider, appliedSummary } from "~/components/runs/v3/SharedFilters"; +import { appliedSummary } from "~/components/runs/v3/SharedFilters"; import type { LogLevel } from "~/presenters/v3/LogsListPresenter.server"; import { cn } from "~/utils/cn"; const allLogLevels: { level: LogLevel; label: string; color: string }[] = [ - { level: "ERROR", label: "Error", color: "text-error" }, - { level: "WARN", label: "Warning", color: "text-warning" }, { level: "INFO", label: "Info", color: "text-blue-400" }, - { level: "CANCELLED", label: "Cancelled", color: "text-charcoal-400" }, + { level: "WARN", label: "Warning", color: "text-warning" }, + { level: "ERROR", label: "Error", color: "text-error" }, { level: "DEBUG", label: "Debug", color: "text-charcoal-400" }, - { level: "TRACE", label: "Trace", color: "text-charcoal-500" }, ]; -function getAvailableLevels(showDebug: boolean): typeof allLogLevels { - if (showDebug) { - return allLogLevels; - } - return allLogLevels.filter((level) => level.level !== "DEBUG"); +// In the future we might add other levels or change which are available +function getAvailableLevels(): typeof allLogLevels { + return allLogLevels; } function getLevelBadgeColor(level: LogLevel): string { @@ -42,10 +37,6 @@ function getLevelBadgeColor(level: LogLevel): string { return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; case "INFO": return "text-blue-400 bg-blue-500/10 border-blue-500/20"; - case "TRACE": - return "text-charcoal-500 bg-charcoal-800 border-charcoal-700"; - case "CANCELLED": - return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; default: return "text-text-dimmed bg-charcoal-750 border-charcoal-700"; } @@ -53,81 +44,50 @@ function getLevelBadgeColor(level: LogLevel): string { const shortcut = { key: "l" }; -export function LogsLevelFilter({ showDebug = false }: { showDebug?: boolean }) { +export function LogsLevelFilter() { const { values } = useSearchParams(); const selectedLevels = values("levels"); const hasLevels = selectedLevels.length > 0 && selectedLevels.some((v) => v !== ""); if (hasLevels) { - return ; + return ; } return ( - - {(search, setSearch) => ( - } - variant="secondary/small" - shortcut={shortcut} - tooltipTitle="Filter by level" - > - Level - - } - searchValue={search} - clearSearchValue={() => setSearch("")} - showDebug={showDebug} - /> - )} - + } + variant="secondary/small" + shortcut={shortcut} + tooltipTitle="Filter by level" + > + Level + + } + /> ); } function LevelDropdown({ trigger, - clearSearchValue, - searchValue, - onClose, - showDebug = false, }: { trigger: ReactNode; - clearSearchValue: () => void; - searchValue: string; - onClose?: () => void; - showDebug?: boolean; }) { const { values, replace } = useSearchParams(); const handleChange = (values: string[]) => { - clearSearchValue(); replace({ levels: values, cursor: undefined, direction: undefined }); }; - const availableLevels = getAvailableLevels(showDebug); - const filtered = useMemo(() => { - return availableLevels.filter((item) => - item.label.toLowerCase().includes(searchValue.toLowerCase()) - ); - }, [searchValue, availableLevels]); + const availableLevels = getAvailableLevels(); return ( {trigger} - { - if (onClose) { - onClose(); - return false; - } - return true; - }} - > - + - {filtered.map((item, index) => ( + {availableLevels.map((item, index) => ( - {(search, setSearch) => ( - }> - } - value={appliedSummary(levels)} - onRemove={() => del(["levels", "cursor", "direction"])} - variant="secondary/small" - /> - - } - searchValue={search} - clearSearchValue={() => setSearch("")} - showDebug={showDebug} - /> - )} - + }> + } + value={appliedSummary(levels)} + onRemove={() => del(["levels", "cursor", "direction"])} + variant="secondary/small" + /> + + } + /> ); } diff --git a/apps/webapp/app/components/logs/LogsRunIdFilter.tsx b/apps/webapp/app/components/logs/LogsRunIdFilter.tsx index 5c23d1a1929..857e623d7c9 100644 --- a/apps/webapp/app/components/logs/LogsRunIdFilter.tsx +++ b/apps/webapp/app/components/logs/LogsRunIdFilter.tsx @@ -14,7 +14,7 @@ import { import { useSearchParams } from "~/hooks/useSearchParam"; import { FilterMenuProvider } from "~/components/runs/v3/SharedFilters"; -const shortcut = { key: "r" }; +const shortcut = { key: "i" }; export function LogsRunIdFilter() { const { value } = useSearchParams(); diff --git a/apps/webapp/app/components/logs/LogsSearchInput.tsx b/apps/webapp/app/components/logs/LogsSearchInput.tsx index 41871722b91..fd539f66ae2 100644 --- a/apps/webapp/app/components/logs/LogsSearchInput.tsx +++ b/apps/webapp/app/components/logs/LogsSearchInput.tsx @@ -1,5 +1,6 @@ import { MagnifyingGlassIcon, XMarkIcon } from "@heroicons/react/20/solid"; import { useNavigate } from "@remix-run/react"; +import { motion } from "framer-motion"; import { useCallback, useEffect, useRef, useState } from "react"; import { Input } from "~/components/primitives/Input"; import { ShortcutKey } from "~/components/primitives/ShortcutKey"; @@ -52,7 +53,16 @@ export function LogsSearchInput() { return (

+ 0 ? "24rem" : "auto" }} + transition={{ + type: "spring", + stiffness: 300, + damping: 30, + }} + className="relative h-6 min-w-52" + > -

+ {text.length > 0 && (

+ {list?.retention?.wasClamped && ( + + )} + {isAdmin && ( + + )} +

+ ); +} + +function LogsList({ + list, }: { list: Exclude["data"]>, { error: string }>; //exclude error, it is handled - rootOnlyDefault: boolean; isAdmin: boolean; showDebug: boolean; defaultPeriod?: string; @@ -253,37 +382,50 @@ function LogsList({ // Selected log state - managed locally to avoid triggering navigation const [selectedLogId, setSelectedLogId] = useState(); - const handleDebugToggle = useCallback( - (checked: boolean) => { - const url = new URL(window.location.href); - if (checked) { - url.searchParams.set("showDebug", "true"); - } else { - url.searchParams.delete("showDebug"); - } - window.location.href = url.toString(); - }, - [] - ); + // Track which filter state (search params) the current fetcher request corresponds to + const fetcherFilterStateRef = useRef(location.search); + // Clear accumulated logs immediately when filters change (for instant visual feedback) + useEffect(() => { + setAccumulatedLogs([]); + setNextCursor(undefined); + // Close side panel when filters change to avoid showing a log that's no longer visible + setSelectedLogId(undefined); + }, [location.search]); - // Reset accumulated logs when the initial list changes (e.g., filters change) + // Populate accumulated logs when new data arrives useEffect(() => { setAccumulatedLogs(list.logs); setNextCursor(list.pagination.next); }, [list.logs, list.pagination.next]); + // Clear log parameter from URL when selectedLogId is cleared + useEffect(() => { + if (!selectedLogId) { + const url = new URL(window.location.href); + if (url.searchParams.has("log")) { + url.searchParams.delete("log"); + window.history.replaceState(null, "", url.toString()); + } + } + }, [selectedLogId]); + // Append new logs when fetcher completes (with deduplication) useEffect(() => { if (fetcher.data && fetcher.state === "idle") { + // Ignore fetcher data if it was loaded for a different filter state + if (fetcherFilterStateRef.current !== location.search) { + return; + } + const existingIds = new Set(accumulatedLogs.map((log) => log.id)); const newLogs = fetcher.data.logs.filter((log) => !existingIds.has(log.id)); if (newLogs.length > 0) { setAccumulatedLogs((prev) => [...prev, ...newLogs]); - setNextCursor(fetcher.data.pagination.next); } + setNextCursor(fetcher.data.pagination.next); } - }, [fetcher.data, fetcher.state, accumulatedLogs]); + }, [fetcher.data, fetcher.state, accumulatedLogs, location.search]); // Build resource URL for loading more const loadMoreUrl = useMemo(() => { @@ -297,27 +439,26 @@ function LogsList({ const handleLoadMore = useCallback(() => { if (loadMoreUrl && fetcher.state === "idle") { + // Store the current filter state before loading + fetcherFilterStateRef.current = location.search; fetcher.load(loadMoreUrl); } - }, [loadMoreUrl, fetcher]); + }, [loadMoreUrl, fetcher, location.search]); const selectedLog = useMemo(() => { if (!selectedLogId) return undefined; return accumulatedLogs.find((log) => log.id === selectedLogId); }, [selectedLogId, accumulatedLogs]); - const updateUrlWithLog = useCallback( - (logId: string | undefined) => { - const url = new URL(window.location.href); - if (logId) { - url.searchParams.set("log", logId); - } else { - url.searchParams.delete("log"); - } - window.history.replaceState(null, "", url.toString()); - }, - [] - ); + const updateUrlWithLog = useCallback((logId: string | undefined) => { + const url = new URL(window.location.href); + if (logId) { + url.searchParams.set("log", logId); + } else { + url.searchParams.delete("log"); + } + window.history.replaceState(null, "", url.toString()); + }, []); const handleLogSelect = useCallback( (logId: string) => { @@ -339,51 +480,30 @@ function LogsList({ return ( -

- {/* Filters */} -

- - - -

- {isAdmin && ( - - )} -

- - {/* Table */} - -

+ - {/* Side panel for log details */} {selectedLogId && ( <> -

}> + + +

+ } + > { + if (isAdmin || isImpersonating) { + return true; + } + + const organization = await prisma.organization.findFirst({ + where: { + slug: organizationSlug, + members: { some: { userId } }, + }, + select: { + featureFlags: true, + }, + }); + + if (!organization?.featureFlags) { + return false; + } + + const flags = organization.featureFlags as Record; + const hasLogsPageAccessResult = validateFeatureFlagValue( + FEATURE_FLAG.hasLogsPageAccess, + flags.hasLogsPageAccess + ); + + return hasLogsPageAccessResult.success && hasLogsPageAccessResult.data === true; +} + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const user = await requireUser(request); + const { organizationSlug } = OrganizationParamsSchema.parse(params); + + const canViewLogsPage = user.admin || user.isImpersonating || await hasLogsPageAccess( + user.id, + user.admin, + user.isImpersonating, + organizationSlug + ); + + return typedjson({ canViewLogsPage }); +}; diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts index fd6f1c1a6a2..656e20472e3 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts @@ -4,13 +4,13 @@ import { requireUser, requireUserId } from "~/services/session.server"; import { EnvironmentParamSchema } from "~/utils/pathBuilder"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; -import { getRunFiltersFromRequest } from "~/presenters/RunFilters.server"; import { LogsListPresenter, type LogLevel, LogsListOptionsSchema } from "~/presenters/v3/LogsListPresenter.server"; import { $replica } from "~/db.server"; import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getCurrentPlan } from "~/services/platform.v3.server"; // Valid log levels for filtering -const validLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "CANCELLED"]; +const validLevels: LogLevel[] = ["DEBUG", "INFO", "WARN", "ERROR"]; function parseLevelsFromUrl(url: URL): LogLevel[] | undefined { const levelParams = url.searchParams.getAll("levels").filter((v) => v.length > 0); @@ -19,7 +19,10 @@ function parseLevelsFromUrl(url: URL): LogLevel[] | undefined { } export const loader = async ({ request, params }: LoaderFunctionArgs) => { - const userId = await requireUserId(request); + const user = await requireUser(request); + const userId = user.id; + const isAdmin = user?.admin || user?.isImpersonating; + const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params); const project = await findProjectBySlug(organizationSlug, projectParam, userId); @@ -32,28 +35,41 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { throw new Response("Environment not found", { status: 404 }); } - const user = await requireUser(request); - const isAdmin = user?.admin || user?.isImpersonating; - - const filters = await getRunFiltersFromRequest(request); + // Get the user's plan to determine log retention limit + const plan = await getCurrentPlan(project.organizationId); + const retentionLimitDays = plan?.v3Subscription?.plan?.limits.logRetentionDays.number ?? 30; - // Get search term, cursor, levels, and showDebug from query params + // Get filters from query params const url = new URL(request.url); + const tasks = url.searchParams.getAll("tasks").filter((t) => t.length > 0); + const runId = url.searchParams.get("runId") ?? undefined; const search = url.searchParams.get("search") ?? undefined; const cursor = url.searchParams.get("cursor") ?? undefined; const levels = parseLevelsFromUrl(url); const showDebug = url.searchParams.get("showDebug") === "true"; + const period = url.searchParams.get("period") ?? undefined; + const fromStr = url.searchParams.get("from"); + const toStr = url.searchParams.get("to"); + let from = fromStr ? parseInt(fromStr, 10) : undefined; + let to = toStr ? parseInt(toStr, 10) : undefined; + if (Number.isNaN(from)) from = undefined; + if (Number.isNaN(to)) to = undefined; const options = LogsListOptionsSchema.parse({ userId, projectId: project.id, - ...filters, + tasks: tasks.length > 0 ? tasks : undefined, + runId, search, cursor, + period, + from, + to, levels, includeDebugLogs: isAdmin && showDebug, defaultPeriod: "1h", + retentionLimitDays, }) as any; // Validated by LogsListOptionsSchema at runtime const presenter = new LogsListPresenter($replica, clickhouseClient); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index 6c6222e7c3f..bd186dcea4d 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -73,6 +73,7 @@ import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; import { useSearchParams } from "~/hooks/useSearchParam"; import { useHasAdminAccess } from "~/hooks/useUser"; +import { useCanViewLogsPage } from "~/hooks/useCanViewLogsPage"; import { redirectWithErrorMessage } from "~/models/message.server"; import { type Span, SpanPresenter, type SpanRun } from "~/presenters/v3/SpanPresenter.server"; import { logger } from "~/services/logger.server"; @@ -319,6 +320,7 @@ function RunBody({ const { value, replace } = useSearchParams(); const tab = value("tab"); const resetFetcher = useTypedFetcher(); + const canViewLogsPage = useCanViewLogsPage(); return (

@@ -1012,44 +1014,55 @@ function RunBody({

{run.logsDeletedAt === null ? ( -

+ canViewLogsPage ? ( +

+ + View logs + + + + + + + + + + +

+ ) : ( - View logs + Download logs - - - - - - - - - -

+ ) ) : null}

diff --git a/apps/webapp/app/utils/logUtils.ts b/apps/webapp/app/utils/logUtils.ts index b4a130b8e56..cad9bbc9070 100644 --- a/apps/webapp/app/utils/logUtils.ts +++ b/apps/webapp/app/utils/logUtils.ts @@ -1,10 +1,10 @@ import { createElement, Fragment, type ReactNode } from "react"; import { z } from "zod"; -export const LogLevelSchema = z.enum(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "CANCELLED"]); +export const LogLevelSchema = z.enum(["DEBUG", "INFO", "WARN", "ERROR"]); export type LogLevel = z.infer; -export const validLogLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "CANCELLED"]; +export const validLogLevels: LogLevel[] = ["DEBUG", "INFO", "WARN", "ERROR",]; // Default styles for search highlighting const DEFAULT_HIGHLIGHT_STYLES: React.CSSProperties = { @@ -71,10 +71,6 @@ export function highlightSearchText( // Convert ClickHouse kind to display level export function kindToLevel(kind: string, status: string): LogLevel { - if (status === "CANCELLED") { - return "CANCELLED"; - } - // ERROR can come from either kind or status if (kind === "LOG_ERROR" || status === "ERROR") { return "ERROR"; @@ -94,7 +90,7 @@ export function kindToLevel(kind: string, status: string): LogLevel { case "ANCESTOR_OVERRIDE": case "SPAN_EVENT": default: - return "TRACE"; + return "INFO"; } } @@ -109,47 +105,7 @@ export function getLevelColor(level: LogLevel): string { return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; case "INFO": return "text-blue-400 bg-blue-500/10 border-blue-500/20"; - case "TRACE": - return "text-charcoal-500 bg-charcoal-800 border-charcoal-700"; - case "CANCELLED": - return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; default: return "text-text-dimmed bg-charcoal-750 border-charcoal-700"; } } - -// Event kind badge color styles -export function getKindColor(kind: string): string { - if (kind === "SPAN") { - return "text-purple-400 bg-purple-500/10 border-purple-500/20"; - } - if (kind === "SPAN_EVENT") { - return "text-amber-400 bg-amber-500/10 border-amber-500/20"; - } - if (kind.startsWith("LOG_")) { - return "text-blue-400 bg-blue-500/10 border-blue-500/20"; - } - return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; -} - -// Get human readable kind label -export function getKindLabel(kind: string): string { - switch (kind) { - case "SPAN": - return "Span"; - case "SPAN_EVENT": - return "Event"; - case "LOG_DEBUG": - case "LOG_INFO": - case "LOG_WARN": - case "LOG_ERROR": - case "LOG_LOG": - return "Log"; - case "DEBUG_EVENT": - return "Debug"; - case "ANCESTOR_OVERRIDE": - return "Override"; - default: - return kind; - } -} diff --git a/internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql b/internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql new file mode 100644 index 00000000000..1d6af329451 --- /dev/null +++ b/internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql @@ -0,0 +1,20 @@ +-- +goose Up + +-- Add indexes for text search on task task_events_v2 tables for message and attributes fields +ALTER TABLE trigger_dev.task_events_v2 + ADD INDEX IF NOT EXISTS idx_attributes_text_search lower(attributes_text) + TYPE ngrambf_v1(3, 32768, 2, 0) + GRANULARITY 1; + +ALTER TABLE trigger_dev.task_events_v2 + ADD INDEX IF NOT EXISTS idx_message_text_search lower(message) + TYPE ngrambf_v1(3, 32768, 2, 0) + GRANULARITY 1; + +-- +goose Down + +ALTER TABLE trigger_dev.task_events_v2 +DROP INDEX idx_attributes_text_search; + +ALTER TABLE trigger_dev.task_events_v2 +DROP INDEX idx_message_text_search; diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index acb3c56a8aa..4f4cb5e3b16 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -25,8 +25,6 @@ import { insertTaskEventsV2, getLogsListQueryBuilderV2, getLogDetailQueryBuilderV2, - getLogsListQueryBuilderV1, - getLogDetailQueryBuilderV1, } from "./taskEvents.js"; import { Logger, type LogLevel } from "@trigger.dev/core/logger"; import type { Agent as HttpAgent } from "http"; @@ -213,8 +211,6 @@ export class ClickHouse { traceSummaryQueryBuilder: getTraceSummaryQueryBuilder(this.reader), traceDetailedSummaryQueryBuilder: getTraceDetailedSummaryQueryBuilder(this.reader), spanDetailsQueryBuilder: getSpanDetailsQueryBuilder(this.reader), - logsListQueryBuilder: getLogsListQueryBuilderV1(this.reader, this.logsQuerySettings?.list), - logDetailQueryBuilder: getLogDetailQueryBuilderV1(this.reader, this.logsQuerySettings?.detail), }; } diff --git a/internal-packages/clickhouse/src/taskEvents.ts b/internal-packages/clickhouse/src/taskEvents.ts index fa64a908dd9..890eab9cc78 100644 --- a/internal-packages/clickhouse/src/taskEvents.ts +++ b/internal-packages/clickhouse/src/taskEvents.ts @@ -320,56 +320,4 @@ export function getLogDetailQueryBuilderV2(ch: ClickhouseReader, settings?: Clic ], settings, }); -} - -// ============================================================================ -// Logs List Query Builders for V1 (task_events_v1) -// ============================================================================ - -export function getLogsListQueryBuilderV1(ch: ClickhouseReader, settings?: ClickHouseSettings) { - return ch.queryBuilderFast({ - name: "getLogsListV1", - table: "trigger_dev.task_events_v1", - columns: [ - "environment_id", - "organization_id", - "project_id", - "task_identifier", - "run_id", - "start_time", - "trace_id", - "span_id", - "parent_span_id", - { name: "message", expression: "LEFT(message, 512)" }, - "kind", - "status", - "duration", - "attributes_text" - ], - settings, - }); -} - -export function getLogDetailQueryBuilderV1(ch: ClickhouseReader, settings?: ClickHouseSettings) { - return ch.queryBuilderFast({ - name: "getLogDetailV1", - table: "trigger_dev.task_events_v1", - columns: [ - "environment_id", - "organization_id", - "project_id", - "task_identifier", - "run_id", - "start_time", - "trace_id", - "span_id", - "parent_span_id", - "message", - "kind", - "status", - "duration", - "attributes_text", - ], - settings, - }); -} +} \ No newline at end of file From c0b86efbd3d9f0577c1b3a5803bce0f43c131c12 Mon Sep 17 00:00:00 2001 From: Oskar Otwinowski Date: Thu, 29 Jan 2026 12:51:58 +0100 Subject: [PATCH 007/400] feat(webapp): Add MiddleTruncate component for long task names (#2946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes # ## ✅ Checklist - [ ] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [ ] The PR title follows the convention. - [ ] I ran and tested the code works --- ## Testing Tested the MiddleTruncate component in the TasksDropdown by: 1. Verifying that long task names (e.g., "namespace:category:subcategory:task-name") are truncated in the middle 2. Confirming the full text appears in a tooltip on hover 3. Testing responsive behavior - truncation adjusts when the container is resized 4. Verifying that short task names that fit within the container are displayed in full without truncation --- ## Changelog Added a new `MiddleTruncate` primitive component that intelligently truncates text in the middle while preserving the beginning and end portions. This is particularly useful for long hierarchical identifiers like task slugs. **Key features:** - Truncates text in the middle with an ellipsis (…) when it exceeds available width - Shows full text in a tooltip on hover when truncated - Responsive - recalculates truncation on container resize using ResizeObserver - Maintains minimum character visibility (4 chars minimum on each side for readability) - Integrated into TasksDropdown to handle long task names **Changes:** - Created new `MiddleTruncate.tsx` component with binary search algorithm for optimal character distribution - Updated TasksDropdown to use MiddleTruncate for task slug display - Increased TasksDropdown popover width from 240px to 360px to provide better space for truncated text --- ## Screenshots 💯 https://github.com/user-attachments/assets/a7a2191a-2e36-437e-ab3f-517fe7620b93 ---

--------- Co-authored-by: Claude --- .../components/primitives/MiddleTruncate.tsx | 168 ++++++++++++++++++ .../app/components/runs/v3/RunFilters.tsx | 5 +- 2 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 apps/webapp/app/components/primitives/MiddleTruncate.tsx diff --git a/apps/webapp/app/components/primitives/MiddleTruncate.tsx b/apps/webapp/app/components/primitives/MiddleTruncate.tsx new file mode 100644 index 00000000000..c116205aed9 --- /dev/null +++ b/apps/webapp/app/components/primitives/MiddleTruncate.tsx @@ -0,0 +1,168 @@ +import { useRef, useState, useLayoutEffect, useCallback } from "react"; +import { cn } from "~/utils/cn"; +import { SimpleTooltip } from "./Tooltip"; + +type MiddleTruncateProps = { + text: string; + className?: string; +}; + +/** + * A component that truncates text in the middle, showing the beginning and end. + * Shows the full text in a tooltip on hover when truncated. + * + * Example: "namespace:category:subcategory:task-name" becomes "namespace:cat…task-name" + */ +export function MiddleTruncate({ text, className }: MiddleTruncateProps) { + const containerRef = useRef(null); + const measureRef = useRef(null); + const [displayText, setDisplayText] = useState(text); + const [isTruncated, setIsTruncated] = useState(false); + + const calculateTruncation = useCallback(() => { + const container = containerRef.current; + const measure = measureRef.current; + if (!container || !measure) return; + + const parent = container.parentElement; + if (!parent) return; + + // Get the available width from the parent container + const parentStyle = getComputedStyle(parent); + const availableWidth = + parent.clientWidth - + parseFloat(parentStyle.paddingLeft) - + parseFloat(parentStyle.paddingRight); + + // Measure full text width + measure.textContent = text; + const fullTextWidth = measure.offsetWidth; + + // If text fits, no truncation needed + if (fullTextWidth <= availableWidth) { + setDisplayText(text); + setIsTruncated(false); + return; + } + + // Text needs truncation - find optimal split + const ellipsis = "…"; + measure.textContent = ellipsis; + const ellipsisWidth = measure.offsetWidth; + + const targetWidth = availableWidth - ellipsisWidth - 4; // small buffer + + if (targetWidth <= 0) { + setDisplayText(ellipsis); + setIsTruncated(true); + return; + } + + // Incrementally find the optimal character counts + let startChars = 0; + let endChars = 0; + + // Alternate adding characters from start and end + while (startChars + endChars < text.length) { + // Try adding to start + const testStart = text.slice(0, startChars + 1); + const testEnd = endChars > 0 ? text.slice(-endChars) : ""; + measure.textContent = testStart + ellipsis + testEnd; + + if (measure.offsetWidth > targetWidth) break; + startChars++; + + if (startChars + endChars >= text.length) break; + + // Try adding to end + const newTestEnd = text.slice(-(endChars + 1)); + measure.textContent = text.slice(0, startChars) + ellipsis + newTestEnd; + + if (measure.offsetWidth > targetWidth) break; + endChars++; + } + + // Ensure minimum characters on each side for readability + const minChars = 4; + const prevStartChars = startChars; + const prevEndChars = endChars; + + if (startChars < minChars && text.length > minChars * 2 + 1) { + startChars = minChars; + } + if (endChars < minChars && text.length > minChars * 2 + 1) { + endChars = minChars; + } + + // Re-measure after enforcing minChars to prevent overflow + if (startChars !== prevStartChars || endChars !== prevEndChars) { + measure.textContent = text.slice(0, startChars) + ellipsis + text.slice(-endChars); + if (measure.offsetWidth > targetWidth) { + // Revert to previous values if minChars enforcement causes overflow + startChars = prevStartChars; + endChars = prevEndChars; + } + } + + // If combined chars would exceed text length, show full text + if (startChars + endChars >= text.length) { + setDisplayText(text); + setIsTruncated(false); + return; + } + + const result = text.slice(0, startChars) + ellipsis + text.slice(-endChars); + setDisplayText(result); + setIsTruncated(true); + }, [text]); + + useLayoutEffect(() => { + calculateTruncation(); + + // Recalculate on resize (guard for jsdom/older browsers) + if (typeof ResizeObserver === "undefined") { + return; + } + + const resizeObserver = new ResizeObserver(() => { + calculateTruncation(); + }); + + const container = containerRef.current; + if (container?.parentElement) { + resizeObserver.observe(container.parentElement); + } + + return () => { + resizeObserver.disconnect(); + }; + }, [calculateTruncation]); + + const content = ( + + {/* Hidden span for measuring text width */} + + ); + + if (isTruncated) { + return ( + {text}} + side="top" + asChild + /> + ); + } + + return content; +} diff --git a/apps/webapp/app/components/runs/v3/RunFilters.tsx b/apps/webapp/app/components/runs/v3/RunFilters.tsx index 5695081816a..cff56573ad7 100644 --- a/apps/webapp/app/components/runs/v3/RunFilters.tsx +++ b/apps/webapp/app/components/runs/v3/RunFilters.tsx @@ -31,6 +31,7 @@ import { DateTime } from "~/components/primitives/DateTime"; import { FormError } from "~/components/primitives/FormError"; import { Input } from "~/components/primitives/Input"; import { Label } from "~/components/primitives/Label"; +import { MiddleTruncate } from "~/components/primitives/MiddleTruncate"; import { Paragraph } from "~/components/primitives/Paragraph"; import { ComboBox, @@ -634,7 +635,7 @@ function TasksDropdown({ {trigger} { if (onClose) { onClose(); @@ -654,7 +655,7 @@ function TasksDropdown({ } > - {item.slug} + ))} From f53db6fd164aa4c591d09523b8316786d20531fb Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 29 Jan 2026 13:02:47 +0000 Subject: [PATCH 008/400] Query: time limits, performance improvements, styling (#2953) Summary - Query: add time limits, performance improvements, and styling updates Changes - Add ClickHouse output_text and error_text columns with indexes - Automatically use _text columns for JSON based on query pattern; support JSON column data prefixes - Add idempotency key and scope columns - Add enforcedWhereClause for tenant and time restrictions, instead of the old tenant stuff. - Implement basic time filter limiting and set default time period based on plan; show message when results are clipped - UX: resizable code area (including vertical splits), collapsible sidebar, fix table/chart vertical sizing, max height for chart legend in fullscreen - Styling and UI tweaks: improved chart legend styling, more chart colours, thinner line chart stroke, pricing callout color, improved layout for callouts - Features: generate and save AI titles ---

--- .../app/components/code/QueryResultsChart.tsx | 32 +- .../app/components/code/TSQLResultsTable.tsx | 5 +- .../components/primitives/AnimatedNumber.tsx | 61 +- .../app/components/primitives/Callout.tsx | 9 +- .../app/components/primitives/Resizable.tsx | 35 +- .../app/components/primitives/charts/Card.tsx | 4 +- .../primitives/charts/ChartLegendCompound.tsx | 119 +-- .../primitives/charts/ChartLine.tsx | 4 +- .../primitives/charts/ChartRoot.tsx | 7 + .../app/components/runs/v3/SharedFilters.tsx | 156 ++-- apps/webapp/app/env.server.ts | 2 +- .../presenters/v3/QueryPresenter.server.ts | 4 + .../ExamplesContent.tsx | 3 +- .../QueryHelpSidebar.tsx | 89 ++- .../QueryHistoryPopover.tsx | 34 +- .../route.tsx | 530 ++++++++++--- .../utils.ts | 32 - ...jectParam.env.$envParam.query.ai-title.tsx | 81 ++ .../app/services/queryService.server.ts | 101 ++- apps/webapp/app/v3/querySchemas.ts | 14 +- .../v3/services/aiQueryTitleService.server.ts | 71 ++ apps/webapp/package.json | 2 +- ...date_output_error_text_to_extract_data.sql | 45 ++ .../clickhouse/src/client/tsql.ts | 86 ++- internal-packages/clickhouse/src/index.ts | 2 +- internal-packages/clickhouse/src/tsql.test.ts | 250 ++++--- .../migration.sql | 7 + .../database/prisma/schema.prisma | 4 +- internal-packages/tsql/src/index.test.ts | 373 ++++++++- internal-packages/tsql/src/index.ts | 105 +-- .../tsql/src/query/printer.test.ts | 707 +++++++++++++++--- internal-packages/tsql/src/query/printer.ts | 522 +++++++++++-- .../tsql/src/query/printer_context.ts | 86 ++- internal-packages/tsql/src/query/schema.ts | 36 + .../tsql/src/query/security.test.ts | 147 +++- pnpm-lock.yaml | 10 +- 36 files changed, 2982 insertions(+), 793 deletions(-) delete mode 100644 apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/utils.ts create mode 100644 apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title.tsx create mode 100644 apps/webapp/app/v3/services/aiQueryTitleService.server.ts create mode 100644 internal-packages/clickhouse/schema/014_update_output_error_text_to_extract_data.sql create mode 100644 internal-packages/database/prisma/migrations/20260124203524_customer_query_add_title_remove_cost/migration.sql diff --git a/apps/webapp/app/components/code/QueryResultsChart.tsx b/apps/webapp/app/components/code/QueryResultsChart.tsx index bde823af191..26a34722de5 100644 --- a/apps/webapp/app/components/code/QueryResultsChart.tsx +++ b/apps/webapp/app/components/code/QueryResultsChart.tsx @@ -5,9 +5,10 @@ import { Chart } from "~/components/primitives/charts/ChartCompound"; import { Paragraph } from "../primitives/Paragraph"; import type { AggregationType, ChartConfiguration } from "./ChartConfigPanel"; -// Color palette for chart series +// Color palette for chart series - 30 distinct colors for large datasets const CHART_COLORS = [ - "#7655fd", // Primary purple + // Primary colors + "#7655fd", // Purple "#22c55e", // Green "#f59e0b", // Amber "#ef4444", // Red @@ -17,6 +18,28 @@ const CHART_COLORS = [ "#14b8a6", // Teal "#f97316", // Orange "#6366f1", // Indigo + // Extended palette + "#84cc16", // Lime + "#0ea5e9", // Sky + "#f43f5e", // Rose + "#a855f7", // Fuchsia + "#eab308", // Yellow + "#10b981", // Emerald + "#3b82f6", // Blue + "#d946ef", // Magenta + "#78716c", // Stone + "#facc15", // Gold + // Additional distinct colors + "#2dd4bf", // Turquoise + "#fb923c", // Light orange + "#a3e635", // Yellow-green + "#38bdf8", // Light blue + "#c084fc", // Light purple + "#4ade80", // Light green + "#fbbf24", // Light amber + "#f472b6", // Light pink + "#67e8f9", // Light cyan + "#818cf8", // Light indigo ]; function getSeriesColor(index: number): string { @@ -30,6 +53,8 @@ interface QueryResultsChartProps { fullLegend?: boolean; /** Callback when "View all" legend button is clicked */ onViewAllLegendItems?: () => void; + /** When true, constrains legend to max 50% height with scrolling */ + legendScrollable?: boolean; } interface TransformedData { @@ -702,6 +727,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({ config, fullLegend = false, onViewAllLegendItems, + legendScrollable = false, }: QueryResultsChartProps) { const { xAxisColumn, @@ -872,6 +898,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({ minHeight="300px" fillContainer onViewAllLegendItems={onViewAllLegendItems} + legendScrollable={legendScrollable} > MAX_STRING_DISPLAY_LENGTH; if (isTruncated) { @@ -1137,6 +1139,7 @@ export const TSQLResultsTable = memo(function TSQLResultsTable({ height: `${rowVirtualizer.getTotalSize()}px`, position: "relative", }} + className="bg-background-dimmed divide-y divide-charcoal-700" > {rowVirtualizer.getVirtualItems().map((virtualRow) => { const row = tableRows[virtualRow.index]; diff --git a/apps/webapp/app/components/primitives/AnimatedNumber.tsx b/apps/webapp/app/components/primitives/AnimatedNumber.tsx index 2d1ff7ea7b1..fea0f9d8900 100644 --- a/apps/webapp/app/components/primitives/AnimatedNumber.tsx +++ b/apps/webapp/app/components/primitives/AnimatedNumber.tsx @@ -1,9 +1,64 @@ import { animate, motion, useMotionValue, useTransform } from "framer-motion"; -import { useEffect } from "react"; +import { useEffect, useMemo } from "react"; -export function AnimatedNumber({ value, duration = 0.5 }: { value: number; duration?: number }) { +/** + * Determines the number of decimal places to display based on the value. + * - For integers or large numbers (>=100), no decimals + * - For numbers >= 10, 1 decimal place + * - For numbers >= 1, 2 decimal places + * - For smaller numbers, up to 4 decimal places + */ +function getDecimalPlaces(value: number): number { + if (Number.isInteger(value)) return 0; + + const absValue = Math.abs(value); + if (absValue >= 100) return 0; + if (absValue >= 10) return 1; + if (absValue >= 1) return 2; + if (absValue >= 0.1) return 3; + return 4; +} + +/** + * Sanitizes a decimal places value to ensure it's valid for toLocaleString. + * - Coerces to a finite number (handles NaN, Infinity, -Infinity) + * - Rounds to an integer + * - Clamps to the valid 0-20 range for toLocaleString options + */ +function sanitizeDecimals(decimals: number): number { + if (!Number.isFinite(decimals)) { + return 0; + } + return Math.min(20, Math.max(0, Math.round(decimals))); +} + +export function AnimatedNumber({ + value, + duration = 0.5, + decimalPlaces, +}: { + value: number; + duration?: number; + /** Number of decimal places to display. If not provided, auto-detects based on value. */ + decimalPlaces?: number; +}) { const motionValue = useMotionValue(value); - let display = useTransform(motionValue, (current) => Math.round(current).toLocaleString()); + + // Determine decimal places - use provided value or auto-detect, then sanitize + const safeDecimals = useMemo(() => { + const rawDecimals = decimalPlaces !== undefined ? decimalPlaces : getDecimalPlaces(value); + return sanitizeDecimals(rawDecimals); + }, [decimalPlaces, value]); + + const display = useTransform(motionValue, (current) => { + if (safeDecimals === 0) { + return Math.round(current).toLocaleString(); + } + return current.toLocaleString(undefined, { + minimumFractionDigits: safeDecimals, + maximumFractionDigits: safeDecimals, + }); + }); useEffect(() => { animate(motionValue, value, { diff --git a/apps/webapp/app/components/primitives/Callout.tsx b/apps/webapp/app/components/primitives/Callout.tsx index 207ad134b64..da2d2ea760e 100644 --- a/apps/webapp/app/components/primitives/Callout.tsx +++ b/apps/webapp/app/components/primitives/Callout.tsx @@ -1,4 +1,5 @@ import { + CreditCardIcon, ExclamationCircleIcon, ExclamationTriangleIcon, InformationCircleIcon, @@ -60,10 +61,10 @@ export const variantClasses = { linkClassName: "transition hover:bg-blue-400/20", }, pricing: { - className: "border-charcoal-700 bg-charcoal-800", - icon: , - textColor: "text-text-bright", - linkClassName: "transition hover:bg-charcoal-750", + className: "border-indigo-400/20 bg-indigo-800/30", + icon: , + textColor: "text-indigo-300", + linkClassName: "transition hover:bg-indigo-400/20", }, } as const; diff --git a/apps/webapp/app/components/primitives/Resizable.tsx b/apps/webapp/app/components/primitives/Resizable.tsx index 22fb38d358a..830cd01184a 100644 --- a/apps/webapp/app/components/primitives/Resizable.tsx +++ b/apps/webapp/app/components/primitives/Resizable.tsx @@ -26,19 +26,40 @@ const ResizableHandle = ({ }) => ( div]:rotate-90", + // Base styles + "group relative flex items-center justify-center focus-custom", + // Horizontal orientation (default) + "w-0.75 after:absolute after:inset-y-0 after:left-1/2 after:w-1 after:-translate-x-1/2", + // Vertical orientation + "data-[handle-orientation=vertical]:h-0.75 data-[handle-orientation=vertical]:w-full", + "data-[handle-orientation=vertical]:after:inset-x-0 data-[handle-orientation=vertical]:after:inset-y-auto", + "data-[handle-orientation=vertical]:after:top-1/2 data-[handle-orientation=vertical]:after:left-0", + "data-[handle-orientation=vertical]:after:h-1 data-[handle-orientation=vertical]:after:w-full", + "data-[handle-orientation=vertical]:after:-translate-y-1/2 data-[handle-orientation=vertical]:after:translate-x-0", className )} size="3px" {...props} > -

+ {/* Horizontal orientation line indicator */} +

+ {/* Vertical orientation line indicator */} +

{withHandle && ( -

- {Array.from({ length: 3 }).map((_, index) => ( -

- ))} -

+ <> + {/* Horizontal orientation dots (vertical arrangement) */} +

+ {Array.from({ length: 3 }).map((_, index) => ( +

+ ))} +

+ {/* Vertical orientation dots (horizontal arrangement) */} +

+ {Array.from({ length: 3 }).map((_, index) => ( +

+ ))} +

+ )} ); diff --git a/apps/webapp/app/components/primitives/charts/Card.tsx b/apps/webapp/app/components/primitives/charts/Card.tsx index 429c51e3312..c618b51d016 100644 --- a/apps/webapp/app/components/primitives/charts/Card.tsx +++ b/apps/webapp/app/components/primitives/charts/Card.tsx @@ -6,7 +6,7 @@ export const Card = ({ children, className }: { children: ReactNode; className?: return (

@@ -17,7 +17,7 @@ export const Card = ({ children, className }: { children: ReactNode; className?: const CardHeader = ({ children }: { children: ReactNode }) => { return ( - {children} + {children} ); }; diff --git a/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx b/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx index 08a5abc9830..1ab1bb855fa 100644 --- a/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx @@ -17,6 +17,8 @@ export type ChartLegendCompoundProps = { totalLabel?: string; /** Callback when "View all" button is clicked */ onViewAllLegendItems?: () => void; + /** When true, constrains legend to max 50% height with scrolling */ + scrollable?: boolean; }; /** @@ -37,6 +39,7 @@ export function ChartLegendCompound({ className, totalLabel = "Total", onViewAllLegendItems, + scrollable = false, }: ChartLegendCompoundProps) { const { config, dataKey, dataKeys, highlight, labelFormatter } = useChartContext(); const totals = useSeriesTotal(); @@ -128,11 +131,17 @@ export function ChartLegendCompound({ const isHovering = (highlight.activePayload?.length ?? 0) > 0; return ( -

{/* Total row */}

@@ -143,62 +152,68 @@ export function ChartLegendCompound({

{/* Separator */} -

- {legendItems.visible.map((item) => { - const total = currentData[item.dataKey] ?? 0; - const isActive = highlight.activeBarKey === item.dataKey; + {/* Legend items - scrollable when scrollable prop is true */} +

+ {legendItems.visible.map((item) => { + const total = currentData[item.dataKey] ?? 0; + const isActive = highlight.activeBarKey === item.dataKey; - return ( -

highlight.setHoveredLegendItem(item.dataKey)} - onMouseLeave={() => highlight.reset()} - > - {/* Active highlight background */} - {isActive && item.color && ( -

- )} -

- {item.color && ( -

- )} - - {item.label} + return ( +

highlight.setHoveredLegendItem(item.dataKey)} + onMouseLeave={() => highlight.reset()} + > + {/* Active highlight background */} + {isActive && item.color && ( +

+ )} +

+ {item.color && ( +

+ )} + + {item.label} + +

+ +

- - -

- ); - })} + ); + })} - {/* View more row - replaced by hovered hidden item when applicable */} - {legendItems.remaining > 0 && - (legendItems.hoveredHiddenItem ? ( - - ) : ( - - ))} + {/* View more row - replaced by hovered hidden item when applicable */} + {legendItems.remaining > 0 && + (legendItems.hoveredHiddenItem ? ( + + ) : ( + + ))} +

); } diff --git a/apps/webapp/app/components/primitives/charts/ChartLine.tsx b/apps/webapp/app/components/primitives/charts/ChartLine.tsx index f6bc220af9a..9148727cad0 100644 --- a/apps/webapp/app/components/primitives/charts/ChartLine.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartLine.tsx @@ -172,7 +172,7 @@ export function ChartLineRenderer({ stroke={config[key]?.color} fill={config[key]?.color} fillOpacity={0.6} - strokeWidth={2} + strokeWidth={1} stackId="stack" isAnimationActive={false} /> @@ -220,7 +220,7 @@ export function ChartLineRenderer({ dataKey={key} type={lineType} stroke={config[key]?.color} - strokeWidth={2} + strokeWidth={1} dot={false} activeDot={{ r: 4 }} isAnimationActive={false} diff --git a/apps/webapp/app/components/primitives/charts/ChartRoot.tsx b/apps/webapp/app/components/primitives/charts/ChartRoot.tsx index 8e3810adc79..d1496a2ffa8 100644 --- a/apps/webapp/app/components/primitives/charts/ChartRoot.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartRoot.tsx @@ -31,6 +31,8 @@ export type ChartRootProps = { legendTotalLabel?: string; /** Callback when "View all" legend button is clicked */ onViewAllLegendItems?: () => void; + /** When true, constrains legend to max 50% height with scrolling */ + legendScrollable?: boolean; /** When true, chart fills its parent container height and distributes space between chart and legend */ fillContainer?: boolean; children: React.ComponentProps["children"]; @@ -72,6 +74,7 @@ export function ChartRoot({ maxLegendItems = 5, legendTotalLabel, onViewAllLegendItems, + legendScrollable = false, fillContainer = false, children, }: ChartRootProps) { @@ -94,6 +97,7 @@ export function ChartRoot({ maxLegendItems={maxLegendItems} legendTotalLabel={legendTotalLabel} onViewAllLegendItems={onViewAllLegendItems} + legendScrollable={legendScrollable} fillContainer={fillContainer} > {children} @@ -109,6 +113,7 @@ type ChartRootInnerProps = { maxLegendItems?: number; legendTotalLabel?: string; onViewAllLegendItems?: () => void; + legendScrollable?: boolean; fillContainer?: boolean; children: React.ComponentProps["children"]; }; @@ -120,6 +125,7 @@ function ChartRootInner({ maxLegendItems = 5, legendTotalLabel, onViewAllLegendItems, + legendScrollable = false, fillContainer = false, children, }: ChartRootInnerProps) { @@ -160,6 +166,7 @@ function ChartRootInner({ maxItems={maxLegendItems} totalLabel={legendTotalLabel} onViewAllLegendItems={onViewAllLegendItems} + scrollable={legendScrollable} /> )}

diff --git a/apps/webapp/app/components/runs/v3/SharedFilters.tsx b/apps/webapp/app/components/runs/v3/SharedFilters.tsx index bf4497ca876..b7675eb83a0 100644 --- a/apps/webapp/app/components/runs/v3/SharedFilters.tsx +++ b/apps/webapp/app/components/runs/v3/SharedFilters.tsx @@ -4,36 +4,31 @@ import { endOfDay, endOfMonth, endOfWeek, - isSaturday, - isSunday, - previousSaturday, startOfDay, startOfMonth, startOfWeek, - startOfYear, subDays, - subMonths, - subWeeks, + subWeeks } from "date-fns"; import parse from "parse-duration"; import { startTransition, useCallback, useEffect, useState, type ReactNode } from "react"; +import simplur from "simplur"; import { AppliedFilter } from "~/components/primitives/AppliedFilter"; +import { Callout } from "~/components/primitives/Callout"; import { DateTime } from "~/components/primitives/DateTime"; import { DateTimePicker } from "~/components/primitives/DateTimePicker"; import { Label } from "~/components/primitives/Label"; import { Paragraph } from "~/components/primitives/Paragraph"; import { RadioButtonCircle } from "~/components/primitives/RadioButton"; import { ComboboxProvider, SelectPopover, SelectProvider } from "~/components/primitives/Select"; +import { useOptionalOrganization } from "~/hooks/useOrganizations"; import { useSearchParams } from "~/hooks/useSearchParam"; import { type ShortcutDefinition } from "~/hooks/useShortcutKeys"; import { cn } from "~/utils/cn"; -import { Button } from "../../primitives/Buttons"; +import { organizationBillingPath } from "~/utils/pathBuilder"; +import { Button, LinkButton } from "../../primitives/Buttons"; import { filterIcon } from "./RunFilters"; -export type DisplayableEnvironment = Pick & { - userName?: string; -}; - export function FilterMenuProvider({ children, onClose, @@ -95,6 +90,10 @@ const timePeriods = [ label: "3 days", value: "3d", }, + { + label: "5 days", + value: "5d", + }, { label: "7 days", value: "7d", @@ -106,11 +105,7 @@ const timePeriods = [ { label: "30 days", value: "30d", - }, - { - label: "90 days", - value: "90d", - }, + } ]; const timeUnits = [ @@ -128,6 +123,22 @@ function parsePeriodString(period: string): { value: number; unit: string } | nu return null; } +const MS_PER_DAY = 1000 * 60 * 60 * 24; + +// Convert a period string to days using parse-duration +function periodToDays(period: string): number { + const ms = parse(period); + if (!ms) return 0; + return ms / MS_PER_DAY; +} + +// Calculate the number of days a date range spans from now +function dateRangeToDays(from?: Date): number { + if (!from) return 0; + const now = new Date(); + return Math.ceil((now.getTime() - from.getTime()) / MS_PER_DAY); +} + const DEFAULT_PERIOD = "7d"; const defaultPeriodMs = parse(DEFAULT_PERIOD); if (!defaultPeriodMs) { @@ -292,6 +303,8 @@ export interface TimeFilterProps { applyShortcut?: ShortcutDefinition | undefined; /** Callback when the user applies a time filter selection, receives the applied values */ onValueChange?: (values: TimeFilterApplyValues) => void; + /** When set an upgrade message will be shown if you select a period further back than this number of days */ + maxPeriodDays?: number; } export function TimeFilter({ @@ -303,6 +316,7 @@ export function TimeFilter({ hideLabel = false, applyShortcut, onValueChange, + maxPeriodDays, }: TimeFilterProps = {}) { const { value } = useSearchParams(); const periodValue = period ?? value("period"); @@ -339,6 +353,7 @@ export function TimeFilter({ labelName={labelName} applyShortcut={applyShortcut} onValueChange={onValueChange} + maxPeriodDays={maxPeriodDays} /> )} @@ -356,6 +371,8 @@ function getInitialCustomDuration(period?: string): { value: string; unit: strin return { value: "", unit: "m" }; } +type SectionType = "duration" | "dateRange"; + export function TimeDropdown({ trigger, period, @@ -366,6 +383,7 @@ export function TimeDropdown({ applyShortcut, onApply, onValueChange, + maxPeriodDays, }: { trigger: ReactNode; period?: string; @@ -377,14 +395,16 @@ export function TimeDropdown({ onApply?: (values: TimeFilterApplyValues) => void; /** When provided, the component operates in controlled mode and skips URL navigation */ onValueChange?: (values: TimeFilterApplyValues) => void; + /** When set an upgrade message will be shown if you select a period further back than this number of days */ + maxPeriodDays?: number; }) { + const organization = useOptionalOrganization(); const [open, setOpen] = useState(); const { replace } = useSearchParams(); const [fromValue, setFromValue] = useState(from); const [toValue, setToValue] = useState(to); // Section selection state: "duration" or "dateRange" - type SectionType = "duration" | "dateRange"; const initialSection: SectionType = from || to ? "dateRange" : "duration"; const [activeSection, setActiveSection] = useState(initialSection); const [validationError, setValidationError] = useState(null); @@ -418,9 +438,28 @@ export function TimeDropdown({ return !isNaN(value) && value > 0; })(); + // Calculate if the current selection exceeds maxPeriodDays + const exceedsMaxPeriod = (() => { + if (!maxPeriodDays) return false; + + if (activeSection === "duration") { + const periodToCheck = selectedPeriod === "custom" ? `${customValue}${customUnit}` : selectedPeriod; + if (!periodToCheck) return false; + return periodToDays(periodToCheck) > maxPeriodDays; + } else { + // For date range, check if fromValue is further back than maxPeriodDays + return dateRangeToDays(fromValue) > maxPeriodDays; + } + })(); + const applySelection = useCallback(() => { setValidationError(null); + if (exceedsMaxPeriod) { + setValidationError(`Your plan allows a maximum of ${maxPeriodDays} days. Upgrade for longer retention.`); + return; + } + if (activeSection === "duration") { // Validate custom duration if (selectedPeriod === "custom" && !isCustomDurationValid) { @@ -498,6 +537,8 @@ export function TimeDropdown({ replace, onApply, onValueChange, + exceedsMaxPeriod, + maxPeriodDays ]); return ( @@ -683,7 +724,7 @@ export function TimeDropdown({ />

{/* Quick select date ranges */} -

e.stopPropagation()}> +

e.stopPropagation()}> { const today = new Date(); setFromValue(startOfDay(today)); - setToValue(today); + setToValue(endOfDay(today)); setActiveSection("dateRange"); setValidationError(null); setSelectedQuickDate("today"); }} /> +

e.stopPropagation()}> { const now = new Date(); setFromValue(startOfWeek(now, { weekStartsOn: 1 })); - setToValue(now); + setToValue(endOfWeek(now, { weekStartsOn: 1 })); setActiveSection("dateRange"); setValidationError(null); setSelectedQuickDate("thisWeek"); }} /> - { - const now = new Date(); - let saturday: Date; - if (isSaturday(now)) { - saturday = subDays(now, 7); - } else if (isSunday(now)) { - saturday = subDays(now, 8); - } else { - saturday = previousSaturday(now); - } - const sunday = endOfDay(subDays(saturday, -1)); - setFromValue(startOfDay(saturday)); - setToValue(sunday); - setActiveSection("dateRange"); - setValidationError(null); - setSelectedQuickDate("lastWeekend"); - }} - /> - { - const lastWeek = subWeeks(new Date(), 1); - const monday = startOfWeek(lastWeek, { weekStartsOn: 1 }); - const friday = endOfDay(subDays(monday, -4)); // Monday + 4 days = Friday - setFromValue(startOfDay(monday)); - setToValue(friday); - setActiveSection("dateRange"); - setValidationError(null); - setSelectedQuickDate("lastWeekdays"); - }} - /> - { - const lastMonth = subMonths(new Date(), 1); - setFromValue(startOfMonth(lastMonth)); - setToValue(endOfMonth(lastMonth)); - setActiveSection("dateRange"); - setValidationError(null); - setSelectedQuickDate("lastMonth"); - }} - /> { const now = new Date(); setFromValue(startOfMonth(now)); - setToValue(now); + setToValue(endOfMonth(now)); setActiveSection("dateRange"); setValidationError(null); setSelectedQuickDate("thisMonth"); }} /> - { - const now = new Date(); - setFromValue(startOfYear(now)); - setToValue(now); - setActiveSection("dateRange"); - setValidationError(null); - setSelectedQuickDate("yearToDate"); - }} - />

{validationError && activeSection === "dateRange" && ( @@ -812,6 +796,17 @@ export function TimeDropdown({

+ {/* Upgrade callout when exceeding maxPeriodDays */} + {exceedsMaxPeriod && organization && ( + Upgrade} + className="items-center" + > + {simplur`Your plan allows a maximum of ${maxPeriodDays} day[|s].`} + + )} + {/* Action buttons */}

diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 1dc3091f16c..98c04b6f953 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -521,7 +521,6 @@ const EnvironmentSchema = z PROD_USAGE_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(), CENTS_PER_RUN: z.coerce.number().default(0), - CENTS_PER_QUERY_BYTE_SECOND: z.coerce.number().default(0), EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"), RESOURCE_MONITOR_ENABLED: z.string().default("0"), @@ -1197,6 +1196,7 @@ const EnvironmentSchema = z QUERY_CLICKHOUSE_MAX_AST_ELEMENTS: z.coerce.number().int().default(4_000_000), QUERY_CLICKHOUSE_MAX_EXPANDED_AST_ELEMENTS: z.coerce.number().int().default(4_000_000), QUERY_CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY: z.coerce.number().int().default(0), + QUERY_CLICKHOUSE_MAX_RETURNED_ROWS: z.coerce.number().int().default(10_000), // Query page concurrency limits QUERY_DEFAULT_ORG_CONCURRENCY_LIMIT: z.coerce.number().int().default(3), diff --git a/apps/webapp/app/presenters/v3/QueryPresenter.server.ts b/apps/webapp/app/presenters/v3/QueryPresenter.server.ts index ef33e45fdff..53ebb3ccdff 100644 --- a/apps/webapp/app/presenters/v3/QueryPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueryPresenter.server.ts @@ -8,6 +8,8 @@ export type QueryHistoryItem = { scope: QueryScope; createdAt: Date; userName: string | null; + /** AI-generated title summarizing the query */ + title: string | null; /** Time filter settings */ filterPeriod: string | null; filterFrom: Date | null; @@ -24,6 +26,7 @@ export class QueryPresenter extends BasePresenter { id: true, query: true, scope: true, + title: true, createdAt: true, filterPeriod: true, filterFrom: true, @@ -43,6 +46,7 @@ export class QueryPresenter extends BasePresenter { scope: q.scope.toLowerCase() as QueryScope, createdAt: q.createdAt, userName: q.user?.displayName ?? q.user?.name ?? null, + title: q.title, filterPeriod: q.filterPeriod, filterFrom: q.filterFrom, filterTo: q.filterTo, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx index b172e4d35ee..0238efd5bca 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx @@ -48,7 +48,7 @@ LIMIT 20`, total_cost, usage_duration, machine, - created_at + triggered_at FROM runs WHERE triggered_at > now() - INTERVAL 7 DAY ORDER BY total_cost DESC @@ -79,4 +79,3 @@ export function ExamplesContent({

); } - diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHelpSidebar.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHelpSidebar.tsx index f7c94d1ef38..daa7187acf2 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHelpSidebar.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHelpSidebar.tsx @@ -36,53 +36,82 @@ export function QueryHelpSidebar({ onValueChange={onTabChange} className="flex min-h-0 flex-col overflow-hidden pt-1" > - - -

- AI -

- - - Writing TRQL - - - Table schema - - - Examples - - +

+ + +

+ AI +

+ + + Writing TRQL + + + Table schema + + + Examples + + +

- +

+ +

- +

+ +

- +

+ +

- +

+ +

); } - diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHistoryPopover.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHistoryPopover.tsx index b5121706756..66492ca944f 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHistoryPopover.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHistoryPopover.tsx @@ -36,9 +36,14 @@ const SQL_KEYWORDS = [ ]; function highlightSQL(query: string): React.ReactNode[] { - // Normalize whitespace for display (let CSS line-clamp handle truncation) - const normalized = query.replace(/\s+/g, " ").slice(0, 200); - const suffix = ""; + // Normalize: collapse multiple spaces/tabs to single space, but preserve newlines + // Then trim each line and limit total length + const normalized = query + .split("\n") + .map((line) => line.replace(/[ \t]+/g, " ").trim()) + .filter((line) => line.length > 0) + .join("\n") + .slice(0, 500); // Create a regex pattern that matches keywords as whole words (case insensitive) const keywordPattern = new RegExp( @@ -69,10 +74,6 @@ function highlightSQL(query: string): React.ReactNode[] { parts.push(normalized.slice(lastIndex)); } - if (suffix) { - parts.push(suffix); - } - return parts; } @@ -118,10 +119,21 @@ export function QueryHistoryPopover({ }} className="flex w-full items-center gap-2 rounded-sm px-2 py-2 outline-none transition-colors focus-custom hover:bg-charcoal-900" > -

- {highlightSQL(item.query)} -

+ {item.title ? ( + <> +

+ {item.title} +

+ {highlightSQL(item.query)} +

+ + ) : ( +

+ {highlightSQL(item.query)} +

+ )}

{item.scope} {valueLabel && · {valueLabel}} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx index 422d8cc2f4b..996149a4697 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx @@ -1,13 +1,23 @@ -import { ArrowDownTrayIcon, ArrowsPointingInIcon, ArrowsPointingOutIcon, ArrowTrendingUpIcon, ClipboardIcon } from "@heroicons/react/20/solid"; -import type { OutputColumnMetadata, WhereClauseFallback } from "@internal/clickhouse"; +import { + ArrowDownTrayIcon, + ArrowsPointingOutIcon, + ArrowTrendingUpIcon, + ClipboardIcon, + TableCellsIcon, +} from "@heroicons/react/20/solid"; +import type { OutputColumnMetadata } from "@internal/clickhouse"; +import { type WhereClauseCondition } from "@internal/tsql"; +import { useFetcher } from "@remix-run/react"; import { redirect, type ActionFunctionArgs, type LoaderFunctionArgs, } from "@remix-run/server-runtime"; +import parse from "parse-duration"; import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef, useState } from "react"; import { flushSync } from "react-dom"; import { typedjson, useTypedFetcher, useTypedLoaderData } from "remix-typedjson"; +import simplur from "simplur"; import { z } from "zod"; import { AISparkleIcon } from "~/assets/icons/AISparkleIcon"; import { AlphaTitle } from "~/components/AlphaBadge"; @@ -21,9 +31,7 @@ import { autoFormatSQL, TSQLEditor } from "~/components/code/TSQLEditor"; import { TSQLResultsTable } from "~/components/code/TSQLResultsTable"; import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel"; import { PageBody, PageContainer } from "~/components/layout/AppLayout"; -import { TimeFilter, timeFilters } from "~/components/runs/v3/SharedFilters"; -import { useSearchParams } from "~/hooks/useSearchParam"; -import { Button } from "~/components/primitives/Buttons"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; import { Callout } from "~/components/primitives/Callout"; import { Card } from "~/components/primitives/charts/Card"; import { @@ -32,6 +40,7 @@ import { ClientTabsList, ClientTabsTrigger, } from "~/components/primitives/ClientTabs"; +import { Dialog, DialogContent, DialogHeader } from "~/components/primitives/Dialog"; import { Header3 } from "~/components/primitives/Headers"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { Paragraph } from "~/components/primitives/Paragraph"; @@ -49,27 +58,30 @@ import { import { Select, SelectItem } from "~/components/primitives/Select"; import { Spinner } from "~/components/primitives/Spinner"; import { Switch } from "~/components/primitives/Switch"; +import { SimpleTooltip } from "~/components/primitives/Tooltip"; +import { TimeFilter, timeFilters } from "~/components/runs/v3/SharedFilters"; import { prisma } from "~/db.server"; +import { env } from "~/env.server"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; +import { useSearchParams } from "~/hooks/useSearchParam"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { QueryPresenter, type QueryHistoryItem } from "~/presenters/v3/QueryPresenter.server"; +import type { action as titleAction } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title"; +import { getLimit } from "~/services/platform.v3.server"; import { executeQuery, type QueryScope } from "~/services/queryService.server"; +import { requireUser } from "~/services/session.server"; import { downloadFile, rowsToCSV, rowsToJSON } from "~/utils/dataExport"; -import { EnvironmentParamSchema } from "~/utils/pathBuilder"; +import { EnvironmentParamSchema, organizationBillingPath } from "~/utils/pathBuilder"; import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags.server"; import { querySchemas } from "~/v3/querySchemas"; +import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; import { QueryHelpSidebar } from "./QueryHelpSidebar"; import { QueryHistoryPopover } from "./QueryHistoryPopover"; import type { AITimeFilter } from "./types"; -import { formatQueryStats } from "./utils"; -import { requireUser } from "~/services/session.server"; -import parse from "parse-duration"; -import { SimpleTooltip } from "~/components/primitives/Tooltip"; -import { Dialog, DialogContent, DialogHeader, DialogPortal, DialogTrigger } from "~/components/primitives/Dialog"; -import { DialogOverlay } from "@radix-ui/react-dialog"; +import { formatDurationNanoseconds } from "@trigger.dev/core/v3"; /** Convert a Date or ISO string to ISO string format */ function toISOString(value: Date | string): string { @@ -159,12 +171,21 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { return typedjson({ defaultQuery, + defaultPeriod: await getDefaultPeriod(project.organizationId), history, isAdmin, + maxRows: env.QUERY_CLICKHOUSE_MAX_RETURNED_ROWS, }); }; -const DEFAULT_PERIOD = "7d"; +async function getDefaultPeriod(organizationId: string): Promise { + const idealDefaultPeriodDays = 7; + const maxQueryPeriod = await getLimit(organizationId, "queryPeriodDays", 30); + if (maxQueryPeriod < idealDefaultPeriodDays) { + return `${maxQueryPeriod}d`; + } + return `${idealDefaultPeriodDays}d`; +} const ActionSchema = z.object({ query: z.string().min(1, "Query is required"), @@ -193,8 +214,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + periodClipped: null, }, { status: 403 } ); @@ -209,8 +232,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + periodClipped: null, }, { status: 404 } ); @@ -225,8 +250,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + periodClipped: null, }, { status: 404 } ); @@ -250,8 +277,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + periodClipped: null, }, { status: 400 } ); @@ -263,31 +292,54 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const explain = explainParam === "true" && isAdmin; // Build time filter fallback for triggered_at column + const defaultPeriod = await getDefaultPeriod(project.organizationId); const timeFilter = timeFilters({ period: period ?? undefined, from: from ?? undefined, to: to ?? undefined, - defaultPeriod: DEFAULT_PERIOD, + defaultPeriod, }); - let triggeredAtFallback: WhereClauseFallback; + // Calculate the effective "from" date the user is requesting (for period clipping check) + // This is null only when the user specifies just a "to" date (rare case) + let requestedFromDate: Date | null = null; + if (timeFilter.from) { + requestedFromDate = new Date(timeFilter.from); + } else if (!timeFilter.to) { + // Period specified (or default) - calculate from now + const periodMs = parse(timeFilter.period ?? defaultPeriod) ?? 7 * 24 * 60 * 60 * 1000; + requestedFromDate = new Date(Date.now() - periodMs); + } + + // Build the fallback WHERE condition based on what the user specified + let triggeredAtFallback: WhereClauseCondition; if (timeFilter.from && timeFilter.to) { - // Both from and to specified - use BETWEEN triggeredAtFallback = { op: "between", low: timeFilter.from, high: timeFilter.to }; } else if (timeFilter.from) { - // Only from specified triggeredAtFallback = { op: "gte", value: timeFilter.from }; } else if (timeFilter.to) { - // Only to specified triggeredAtFallback = { op: "lte", value: timeFilter.to }; } else { - // Period specified (or default) - calculate from now - const periodMs = parse(timeFilter.period ?? DEFAULT_PERIOD) ?? 7 * 24 * 60 * 60 * 1000; - triggeredAtFallback = { op: "gte", value: new Date(Date.now() - periodMs) }; + triggeredAtFallback = { op: "gte", value: requestedFromDate! }; } + const maxQueryPeriod = await getLimit(project.organizationId, "queryPeriodDays", 30); + const maxQueryPeriodDate = new Date(Date.now() - maxQueryPeriod * 24 * 60 * 60 * 1000); + + // Check if the requested time period exceeds the plan limit + const periodClipped = requestedFromDate !== null && requestedFromDate < maxQueryPeriodDate; + + // Force tenant isolation and time period limits + const enforcedWhereClause = { + organization_id: { op: "eq", value: project.organizationId }, + project_id: + scope === "project" || scope === "environment" ? { op: "eq", value: project.id } : undefined, + environment_id: scope === "environment" ? { op: "eq", value: environment.id } : undefined, + triggered_at: { op: "gte", value: maxQueryPeriodDate }, + } satisfies Record; + try { - const [error, result] = await executeQuery({ + const [error, result, queryId] = await executeQuery({ name: "query-page", query, schema: z.record(z.any()), @@ -298,6 +350,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { projectId: project.id, environmentId: environment.id, explain, + enforcedWhereClause, whereClauseFallback: { triggered_at: triggeredAtFallback, }, @@ -323,8 +376,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + queryId: null, + periodClipped: null, }, { status: 400 } ); @@ -336,8 +392,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: result.columns, stats: result.stats, hiddenColumns: result.hiddenColumns ?? null, + reachedMaxRows: result.reachedMaxRows, explainOutput: result.explainOutput ?? null, generatedSql: result.generatedSql ?? null, + queryId, + periodClipped: periodClipped ? maxQueryPeriod : null, }); } catch (err) { const errorMessage = err instanceof Error ? err.message : "Unknown error executing query"; @@ -348,8 +407,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + queryId: null, + periodClipped: null, }, { status: 500 } ); @@ -368,18 +430,45 @@ interface QueryEditorFormHandle { const QueryEditorForm = forwardRef< QueryEditorFormHandle, { + defaultPeriod: string; defaultQuery: string; defaultScope: QueryScope; defaultTimeFilter?: { period?: string; from?: string; to?: string }; history: QueryHistoryItem[]; fetcher: ReturnType>; isAdmin: boolean; + onQuerySubmit?: () => void; + onHistorySelected?: (item: QueryHistoryItem) => void; } ->(function QueryEditorForm({ defaultQuery, defaultScope, defaultTimeFilter, history, fetcher, isAdmin }, ref) { +>(function QueryEditorForm( + { + defaultPeriod, + defaultQuery, + defaultScope, + defaultTimeFilter, + history, + fetcher, + isAdmin, + onQuerySubmit, + onHistorySelected, + }, + ref +) { const isLoading = fetcher.state === "submitting" || fetcher.state === "loading"; const [query, setQuery] = useState(defaultQuery); const [scope, setScope] = useState(defaultScope); const formRef = useRef(null); + const prevFetcherState = useRef(fetcher.state); + const plan = useCurrentPlan(); + const maxPeriodDays = plan?.v3Subscription?.plan?.limits?.queryPeriodDays?.number; + + // Notify parent when query is submitted (for title generation) + useEffect(() => { + if (prevFetcherState.current !== "submitting" && fetcher.state === "submitting") { + onQuerySubmit?.(); + } + prevFetcherState.current = fetcher.state; + }, [fetcher.state, onQuerySubmit]); // Get time filter values - initialize from props (which may come from history) const [period, setPeriod] = useState(defaultTimeFilter?.period); @@ -406,18 +495,23 @@ const QueryEditorForm = forwardRef< [query] ); - const handleHistorySelected = useCallback((item: QueryHistoryItem) => { - setQuery(item.query); - setScope(item.scope); - // Apply time filter from history item - // Note: filterFrom/filterTo might be Date objects or ISO strings depending on serialization - setPeriod(item.filterPeriod ?? undefined); - setFrom(item.filterFrom ? toISOString(item.filterFrom) : undefined); - setTo(item.filterTo ? toISOString(item.filterTo) : undefined); - }, []); + const handleHistorySelected = useCallback( + (item: QueryHistoryItem) => { + setQuery(item.query); + setScope(item.scope); + // Apply time filter from history item + // Note: filterFrom/filterTo might be Date objects or ISO strings depending on serialization + setPeriod(item.filterPeriod ?? undefined); + setFrom(item.filterFrom ? toISOString(item.filterFrom) : undefined); + setTo(item.filterTo ? toISOString(item.filterTo) : undefined); + // Notify parent about history selection (for title) + onHistorySelected?.(item); + }, + [onHistorySelected] + ); return ( -

- + {/* Pass time filter values to action */} @@ -468,14 +565,16 @@ const QueryEditorForm = forwardRef< {queryHasTriggeredAt ? ( - Set in query - } + button={ + + } content="Your query includes a WHERE clause with triggered_at so this filter is disabled." /> ) : ( )}

- - - - - - + return ( + <> + + +

+ + +

+ + {titleContent} +

+ +

+ + + + + + diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/utils.ts b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/utils.ts deleted file mode 100644 index 41fc6e31aee..00000000000 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/utils.ts +++ /dev/null @@ -1,32 +0,0 @@ -export function formatQueryStats(stats: { - read_rows: string; - read_bytes: string; - elapsed_ns: string; - byte_seconds: string; -}): string { - const readRows = parseInt(stats.read_rows, 10); - const readBytes = parseInt(stats.read_bytes, 10); - const elapsedNs = parseInt(stats.elapsed_ns, 10); - const byteSeconds = parseFloat(stats.byte_seconds); - - const elapsedMs = elapsedNs / 1_000_000; - const formattedTime = - elapsedMs < 1000 ? `${elapsedMs.toFixed(1)}ms` : `${(elapsedMs / 1000).toFixed(2)}s`; - const formattedBytes = formatBytes(readBytes); - - return `${readRows.toLocaleString()} rows read · ${formattedBytes} · ${formattedTime} · ${formatBytes( - byteSeconds - )}s`; -} - -export function formatBytes(bytes: number): string { - if (bytes === 0) return "0 B"; - if (bytes < 0) return "-" + formatBytes(-bytes); - const k = 1024; - const sizes = ["B", "KB", "MB", "GB"]; - const i = Math.max( - 0, - Math.min(Math.floor(Math.log(bytes) / Math.log(k)), sizes.length - 1) - ); - return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`; -} diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title.tsx new file mode 100644 index 00000000000..9fa57d7fbe8 --- /dev/null +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title.tsx @@ -0,0 +1,81 @@ +import { openai } from "@ai-sdk/openai"; +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { tryCatch } from "@trigger.dev/core/utils"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { findProjectBySlug } from "~/models/project.server"; +import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { requireUserId } from "~/services/session.server"; +import { EnvironmentParamSchema } from "~/utils/pathBuilder"; +import { AIQueryTitleService } from "~/v3/services/aiQueryTitleService.server"; + +const RequestSchema = z.object({ + query: z.string().min(1, "Query is required"), + queryId: z.string().optional(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + + // Parse the request body + const [error, data] = await tryCatch(request.json()); + if (error) { + return json({ success: false as const, error: error.message, title: null }, { status: 400 }); + } + const submission = RequestSchema.safeParse(data); + + if (!submission.success) { + return json( + { success: false as const, error: "Invalid request data", title: null }, + { status: 400 } + ); + } + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + return json( + { success: false as const, error: "Project not found", title: null }, + { status: 404 } + ); + } + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { + return json( + { success: false as const, error: "Environment not found", title: null }, + { status: 404 } + ); + } + + if (!env.OPENAI_API_KEY) { + return json( + { success: false as const, error: "OpenAI API key is not configured", title: null }, + { status: 400 } + ); + } + + const { query, queryId } = submission.data; + + const service = new AIQueryTitleService(openai(env.AI_RUN_FILTER_MODEL ?? "gpt-4o-mini")); + + const result = await service.generateTitle(query); + + if (!result.success) { + return json({ success: false as const, error: result.error, title: null }, { status: 500 }); + } + + // Strip leading/trailing quotes that AI sometimes adds + const title = result.title.replace(/^["']|["']$/g, ""); + + // If a queryId was provided, update the CustomerQuery record with the title + if (queryId) { + await prisma.customerQuery.update({ + where: { id: queryId, organizationId: project.organizationId }, + data: { title }, + }); + } + + return json({ success: true as const, title, error: null }); +} diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts index 6673caf4431..1d5af9e0011 100644 --- a/apps/webapp/app/services/queryService.server.ts +++ b/apps/webapp/app/services/queryService.server.ts @@ -7,7 +7,7 @@ import { type TSQLQueryResult, } from "@internal/clickhouse"; import type { CustomerQuerySource } from "@trigger.dev/database"; -import type { TableSchema } from "@internal/tsql"; +import type { TableSchema, WhereClauseCondition } from "@internal/tsql"; import { type z } from "zod"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; @@ -56,17 +56,14 @@ function getDefaultClickhouseSettings(): ClickHouseSettings { export type ExecuteQueryOptions = Omit< ExecuteTSQLOptions, - "tableSchema" | "organizationId" | "projectId" | "environmentId" | "fieldMappings" + "tableSchema" | "fieldMappings" > & { + organizationId: string; + projectId?: string; + environmentId?: string; tableSchema: TableSchema[]; /** The scope of the query - determines tenant isolation */ scope: QueryScope; - /** Organization ID (required) */ - organizationId: string; - /** Project ID (required for project/environment scope) */ - projectId: string; - /** Environment ID (required for environment scope) */ - environmentId: string; /** History options for saving query to billing/audit */ history?: { /** Where the query originated from */ @@ -89,18 +86,27 @@ export type ExecuteQueryOptions = Omit< customOrgConcurrencyLimit?: number; }; +/** + * Extended result type that includes the optional queryId when saved to history + */ +export type ExecuteQueryResult = + | [error: Error, result: null, queryId: null] + | [error: null, result: T, queryId: string | null]; + /** * Execute a TSQL query against ClickHouse with tenant isolation * Handles building tenant options, field mappings, and optionally saves to history + * Returns [error, result, queryId] where queryId is the CustomerQuery ID if saved to history */ export async function executeQuery( options: ExecuteQueryOptions -): Promise>> { +): Promise>[1], null>>> { const { scope, organizationId, projectId, environmentId, + enforcedWhereClause, history, customOrgConcurrencyLimit, whereClauseFallback, @@ -112,39 +118,22 @@ export async function executeQuery( const orgLimit = customOrgConcurrencyLimit ?? DEFAULT_ORG_CONCURRENCY_LIMIT; // Acquire concurrency slot - const acquireResult = await queryConcurrencyLimiter.acquire({ - key: organizationId, - requestId, - keyLimit: orgLimit, - globalLimit: GLOBAL_CONCURRENCY_LIMIT, - }); - - if (!acquireResult.success) { - const errorMessage = - acquireResult.reason === "key_limit" - ? `You've exceeded your query concurrency of ${orgLimit} for this organization. Please try again later.` - : "We're experiencing a lot of queries at the moment. Please try again later."; - return [new QueryError(errorMessage, { query: options.query }), null]; - } - - try { - // Build tenant IDs based on scope - const tenantOptions: { - organizationId: string; - projectId?: string; - environmentId?: string; - } = { - organizationId, - }; - - if (scope === "project" || scope === "environment") { - tenantOptions.projectId = projectId; - } + const acquireResult = await queryConcurrencyLimiter.acquire({ + key: organizationId, + requestId, + keyLimit: orgLimit, + globalLimit: GLOBAL_CONCURRENCY_LIMIT, + }); - if (scope === "environment") { - tenantOptions.environmentId = environmentId; + if (!acquireResult.success) { + const errorMessage = + acquireResult.reason === "key_limit" + ? `You've exceeded your query concurrency of ${orgLimit} for this organization. Please try again later.` + : "We're experiencing a lot of queries at the moment. Please try again later."; + return [new QueryError(errorMessage, { query: options.query }), null, null]; } + try { // Build field mappings for project_ref → project_id and environment_id → slug translation const projects = await prisma.project.findMany({ where: { organizationId }, @@ -163,18 +152,29 @@ export async function executeQuery( const result = await executeTSQL(clickhouseClient.reader, { ...baseOptions, - ...tenantOptions, + enforcedWhereClause, fieldMappings, whereClauseFallback, clickhouseSettings: { ...getDefaultClickhouseSettings(), ...baseOptions.clickhouseSettings, // Allow caller overrides if needed }, + querySettings: { + maxRows: env.QUERY_CLICKHOUSE_MAX_RETURNED_ROWS, + ...baseOptions.querySettings, // Allow caller overrides if needed + }, }); + // If query failed, return early with no queryId + if (result[0] !== null) { + return [result[0], null, null]; + } + + let queryId: string | null = null; + // If query succeeded and history options provided, save to history // Skip history for EXPLAIN queries (admin debugging) and when explicitly skipped (e.g., impersonating) - if (result[0] === null && history && !history.skip && !baseOptions.explain) { + if (history && !history.skip && !baseOptions.explain) { // Check if this query is the same as the last one saved (avoid duplicate history entries) const lastQuery = await prisma.customerQuery.findFirst({ where: { @@ -183,7 +183,7 @@ export async function executeQuery( userId: history.userId ?? null, }, orderBy: { createdAt: "desc" }, - select: { query: true, scope: true, filterPeriod: true, filterFrom: true, filterTo: true }, + select: { id: true, query: true, scope: true, filterPeriod: true, filterFrom: true, filterTo: true }, }); const timeFilter = history.timeFilter; @@ -195,17 +195,15 @@ export async function executeQuery( lastQuery.filterFrom?.getTime() === (timeFilter?.from?.getTime() ?? undefined) && lastQuery.filterTo?.getTime() === (timeFilter?.to?.getTime() ?? undefined); - if (!isDuplicate) { - const stats = result[1].stats; - const byteSeconds = parseFloat(stats.byte_seconds) || 0; - const costInCents = byteSeconds * env.CENTS_PER_QUERY_BYTE_SECOND; - - await prisma.customerQuery.create({ + if (isDuplicate && lastQuery) { + // Return the existing query's ID for duplicate queries + queryId = lastQuery.id; + } else { + const created = await prisma.customerQuery.create({ data: { query: options.query, scope: scopeToEnum[scope], - stats: { ...stats }, - costInCents, + stats: { ...result[1].stats }, source: history.source, organizationId, projectId: scope === "project" || scope === "environment" ? projectId : null, @@ -216,10 +214,11 @@ export async function executeQuery( filterTo: history.timeFilter?.to ?? null, }, }); + queryId = created.id; } } - return result; + return [null, result[1], queryId]; } finally { // Always release the concurrency slot await queryConcurrencyLimiter.release({ diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index cb574cfd9c2..fe7005974d5 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -167,10 +167,14 @@ export const runsSchema: TableSchema = { expression: "if(depth > 0, true, false)", }, - // Useless until we show the user-provided key idempotency_key: { name: "idempotency_key", - ...column("String", { description: "Idempotency key", example: "user-123-action-456" }), + clickhouseName: "idempotency_key_user", + ...column("String", { description: "Idempotency key (available from 4.3.3)", example: "user-123-action-456" }), + }, + idempotency_key_scope: { + name: "idempotency_key_scope", + ...column("String", { description: "The idempotency key scope determines whether a task should be considered unique within a parent run, a specific attempt, or globally. An empty value means there's no idempotency key set (available from 4.3.3).", example: "run", allowedValues: ["global", "run", "attempt"], }), }, region: { name: "region", @@ -325,6 +329,8 @@ export const runsSchema: TableSchema = { // Output & error (JSON columns) // For JSON columns, NULL checks are transformed to check for empty object '{}' // So `error IS NULL` becomes `error = '{}'` and `error IS NOT NULL` becomes `error != '{}'` + // textColumn uses the pre-materialized text columns for better performance + // dataPrefix handles the internal {"data": ...} wrapper transparently output: { name: "output", ...column("JSON", { @@ -332,6 +338,8 @@ export const runsSchema: TableSchema = { example: '{"result": "success"}', }), nullValue: "'{}'", // Transform NULL checks to compare against empty object + textColumn: "output_text", // Use output_text for full JSON value queries + dataPrefix: "data", // Internal data is wrapped in {"data": ...} }, error: { name: "error", @@ -341,6 +349,8 @@ export const runsSchema: TableSchema = { example: '{"message": "Task failed"}', }), nullValue: "'{}'", // Transform NULL checks to compare against empty object + textColumn: "error_text", // Use error_text for full JSON value queries + dataPrefix: "data", // Internal data is wrapped in {"data": ...} }, // Tags & versions diff --git a/apps/webapp/app/v3/services/aiQueryTitleService.server.ts b/apps/webapp/app/v3/services/aiQueryTitleService.server.ts new file mode 100644 index 00000000000..983f732453a --- /dev/null +++ b/apps/webapp/app/v3/services/aiQueryTitleService.server.ts @@ -0,0 +1,71 @@ +import { openai } from "@ai-sdk/openai"; +import { generateText, type LanguageModelV1 } from "ai"; +import { env } from "~/env.server"; + +/** + * Result type for title generation + */ +export type AIQueryTitleResult = + | { success: true; title: string } + | { success: false; error: string }; + +/** + * Service for generating concise titles for SQL queries using AI + */ +export class AIQueryTitleService { + constructor(private readonly model: LanguageModelV1 = openai("gpt-4o-mini")) {} + + /** + * Generate a concise title for a SQL query + */ + async generateTitle(query: string): Promise { + if (!env.OPENAI_API_KEY) { + return { success: false, error: "OpenAI API key is not configured" }; + } + + try { + const result = await generateText({ + model: this.model, + system: `You are a helpful assistant that generates concise titles for SQL queries. + +Your task is to create a short, descriptive title (5-10 words) that summarizes what the query does. + +Guidelines: +- Focus on the main purpose/intent of the query +- Use plain language, not technical SQL terms +- Start with an action verb when appropriate (e.g., "Count", "List", "Show", "Find") +- Be specific about what data is being retrieved +- Do not include quotes around the title +- Do not include punctuation at the end + +Examples: +- "Failed runs by hour over 7 days" +- "Top 50 most expensive task runs" +- "Run counts grouped by status" +- "Average execution time by task" +- "Recent runs with errors"`, + prompt: `Generate a concise title for this SQL query:\n\n${query}`, + maxTokens: 50, + experimental_telemetry: { + isEnabled: true, + metadata: { + feature: "ai-query-title", + }, + }, + }); + + const title = result.text.trim(); + + if (!title) { + return { success: false, error: "No title generated" }; + } + + return { success: true, title }; + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : "Failed to generate title", + }; + } + } +} diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 97f4533348f..51a468b50c0 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -121,7 +121,7 @@ "@trigger.dev/core": "workspace:*", "@trigger.dev/database": "workspace:*", "@trigger.dev/otlp-importer": "workspace:*", - "@trigger.dev/platform": "1.0.21", + "@trigger.dev/platform": "1.0.22", "@trigger.dev/redis-worker": "workspace:*", "@trigger.dev/sdk": "workspace:*", "@types/pg": "8.6.6", diff --git a/internal-packages/clickhouse/schema/014_update_output_error_text_to_extract_data.sql b/internal-packages/clickhouse/schema/014_update_output_error_text_to_extract_data.sql new file mode 100644 index 00000000000..c66a1492a55 --- /dev/null +++ b/internal-packages/clickhouse/schema/014_update_output_error_text_to_extract_data.sql @@ -0,0 +1,45 @@ +-- +goose Up +-- Update the materialized columns to extract the 'data' field if it exists +-- This avoids the {"data": ...} wrapper in the text representation +-- Note: Direct JSON path access (output.data) returns null for nested objects, +-- so we use JSONExtractRaw on the stringified JSON instead +ALTER TABLE trigger_dev.task_runs_v2 +ADD COLUMN output_text String MATERIALIZED if ( + toJSONString (output) = '{}', + '', + if ( + length (JSONExtractRaw (toJSONString (output), 'data')) > 0, + JSONExtractRaw (toJSONString (output), 'data'), + toJSONString (output) + ) +); + +-- For error: extract error.data if it exists +ALTER TABLE trigger_dev.task_runs_v2 +ADD COLUMN error_text String MATERIALIZED if ( + toJSONString (error) = '{}', + '', + if ( + length (JSONExtractRaw (toJSONString (error), 'data')) > 0, + JSONExtractRaw (toJSONString (error), 'data'), + toJSONString (error) + ) +); + +-- Add the indexes +ALTER TABLE trigger_dev.task_runs_v2 ADD INDEX idx_output_text output_text TYPE ngrambf_v1 (3, 131072, 3, 0) GRANULARITY 4; + +ALTER TABLE trigger_dev.task_runs_v2 ADD INDEX idx_error_text error_text TYPE ngrambf_v1 (3, 131072, 3, 0) GRANULARITY 4; + +-- +goose Down +ALTER TABLE trigger_dev.task_runs_v2 +DROP INDEX IF EXISTS idx_output_text; + +ALTER TABLE trigger_dev.task_runs_v2 +DROP INDEX IF EXISTS idx_error_text; + +ALTER TABLE trigger_dev.task_runs_v2 +DROP COLUMN IF EXISTS output_text; + +ALTER TABLE trigger_dev.task_runs_v2 +DROP COLUMN IF EXISTS error_text; \ No newline at end of file diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts index c68923369db..61868b610fe 100644 --- a/internal-packages/clickhouse/src/client/tsql.ts +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -2,7 +2,7 @@ * TSQL Query Execution for ClickHouse * * This module provides a safe interface for executing TSQL queries against ClickHouse - * with automatic tenant isolation and SQL injection protection. + * with enforced WHERE clause conditions (tenant isolation + plan limits) and SQL injection protection. */ import type { ClickHouseSettings } from "@clickhouse/client"; @@ -14,7 +14,7 @@ import { type TableSchema, type QuerySettings, type FieldMappings, - type WhereClauseFallback, + type WhereClauseCondition } from "@internal/tsql"; import type { ClickhouseReader, QueryStats } from "./types.js"; import { QueryError } from "./errors.js"; @@ -25,7 +25,7 @@ const logger = new Logger("tsql", "info"); export type { QueryStats }; -export type { TableSchema, QuerySettings, FieldMappings, WhereClauseFallback }; +export type { TableSchema, QuerySettings, FieldMappings, WhereClauseCondition }; /** * Options for executing a TSQL query @@ -37,14 +37,26 @@ export interface ExecuteTSQLOptions { query: string; /** The Zod schema for validating output rows */ schema: TOut; - /** The organization ID for tenant isolation (required) */ - organizationId: string; - /** The project ID for tenant isolation (optional - omit to query across all projects) */ - projectId?: string; - /** The environment ID for tenant isolation (optional - omit to query across all environments) */ - environmentId?: string; /** Schema registry defining allowed tables and columns */ tableSchema: TableSchema[]; + /** + * REQUIRED: Conditions always applied at the table level. + * Must include tenant columns (e.g., organization_id) for multi-tenant tables. + * Applied to every table reference including subqueries, CTEs, and JOINs. + * + * @example + * ```typescript + * { + * // Tenant isolation + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * environment_id: { op: "eq", value: "env_789" }, + * // Plan-based time limit + * triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) } + * } + * ``` + */ + enforcedWhereClause: Record; /** Optional ClickHouse query settings */ clickhouseSettings?: ClickHouseSettings; /** Optional TSQL query settings (maxRows, timezone, etc.) */ @@ -78,6 +90,7 @@ export interface ExecuteTSQLOptions { /** * Fallback WHERE conditions to apply when the user hasn't filtered on a column. * Key is the column name, value is the fallback condition. + * These are applied at the AST level (top-level query only). * * @example * ```typescript @@ -87,7 +100,7 @@ export interface ExecuteTSQLOptions { * } * ``` */ - whereClauseFallback?: Record; + whereClauseFallback?: Record; } /** @@ -102,6 +115,11 @@ export interface TSQLQuerySuccess { * Only populated when SELECT * is transformed to core columns only. */ hiddenColumns?: string[]; + /** + * Whether the result count equals the maxRows limit. + * When true, the results may be truncated and more rows may exist. + */ + reachedMaxRows: boolean; /** * The raw EXPLAIN output from ClickHouse. * Only populated when `explain: true` is passed. @@ -123,7 +141,7 @@ export type TSQLQueryResult = [QueryError, null] | [null, TSQLQuerySuccess * Execute a TSQL query against ClickHouse * * This function: - * 1. Compiles the TSQL query to ClickHouse SQL (parse, validate, inject tenant guards) + * 1. Compiles the TSQL query to ClickHouse SQL (parse, validate, inject enforced WHERE clauses) * 2. Executes the query and returns validated results * * @example @@ -132,10 +150,12 @@ export type TSQLQueryResult = [QueryError, null] | [null, TSQLQuerySuccess * name: "get_task_runs", * query: "SELECT id, status FROM task_runs WHERE status = 'completed' ORDER BY created_at DESC LIMIT 100", * schema: z.object({ id: z.string(), status: z.string() }), - * organizationId: "org_123", - * projectId: "proj_456", - * environmentId: "env_789", * tableSchema: [taskRunsSchema], + * enforcedWhereClause: { + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * environment_id: { op: "eq", value: "env_789" }, + * }, * }); * ``` */ @@ -145,18 +165,22 @@ export async function executeTSQL( ): Promise>> { const shouldTransformValues = options.transformValues ?? true; const isExplain = options.explain ?? false; + const maxRows = options.querySettings?.maxRows; let generatedSql: string | undefined; let generatedParams: Record | undefined; try { // 1. Compile the TSQL query to ClickHouse SQL + // Pass maxRows + 1 to fetch one extra row for overflow detection + const compiledSettings = maxRows !== undefined + ? { ...options.querySettings, maxRows: maxRows + 1 } + : options.querySettings; + const { sql, params, columns, hiddenColumns } = compileTSQL(options.query, { - organizationId: options.organizationId, - projectId: options.projectId, - environmentId: options.environmentId, tableSchema: options.tableSchema, - settings: options.querySettings, + enforcedWhereClause: options.enforcedWhereClause, + settings: compiledSettings, fieldMappings: options.fieldMappings, whereClauseFallback: options.whereClauseFallback, }); @@ -231,26 +255,36 @@ export async function executeTSQL( columns: [], stats, hiddenColumns, + reachedMaxRows: false, explainOutput: combinedOutput, generatedSql, }, ]; } + // Determine if we exceeded maxRows (we fetched maxRows + 1 to detect overflow) + const reachedMaxRows = maxRows !== undefined && rows !== undefined && rows.length > maxRows; + + // Remove the overflow row if we got one (pop is O(1), slice would be O(n)) + const finalRows = rows ?? []; + if (reachedMaxRows) { + finalRows.pop(); + } + // Build the result, including hiddenColumns if present - const baseResult = { columns, stats, hiddenColumns }; + const baseResult = { columns, stats, hiddenColumns, reachedMaxRows }; // 3. Transform result values if enabled - if (shouldTransformValues && rows) { + if (shouldTransformValues && finalRows.length > 0) { const transformedRows = transformResults( - rows as Record[], + finalRows as Record[], options.tableSchema, { fieldMappings: options.fieldMappings } ); return [null, { rows: transformedRows as z.output[], ...baseResult }]; } - return [null, { rows: rows ?? [], ...baseResult }]; + return [null, { rows: finalRows as z.output[], ...baseResult }]; } catch (error) { const errorMessage = error instanceof Error ? error.message : "Unknown error"; @@ -284,9 +318,11 @@ export async function executeTSQL( * name: "get_task_runs", * query: "SELECT * FROM task_runs LIMIT 10", * schema: taskRunRowSchema, - * organizationId: "org_123", - * projectId: "proj_456", - * environmentId: "env_789", + * enforcedWhereClause: { + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * environment_id: { op: "eq", value: "env_789" }, + * }, * }); * ``` */ diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index 4f4cb5e3b16..50b39d35a79 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -54,7 +54,7 @@ export { type TSQLQuerySuccess, type QueryStats, type FieldMappings, - type WhereClauseFallback, + type WhereClauseCondition, } from "./client/tsql.js"; export type { OutputColumnMetadata } from "@internal/tsql"; diff --git a/internal-packages/clickhouse/src/tsql.test.ts b/internal-packages/clickhouse/src/tsql.test.ts index fd33ed51062..5c1d6ed8022 100644 --- a/internal-packages/clickhouse/src/tsql.test.ts +++ b/internal-packages/clickhouse/src/tsql.test.ts @@ -106,9 +106,11 @@ describe("TSQL Integration Tests", () => { name: "test-simple-select", query: "SELECT run_id, status FROM task_runs", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -145,9 +147,11 @@ describe("TSQL Integration Tests", () => { name: "test-where-clause", query: "SELECT run_id, status FROM task_runs WHERE status = 'COMPLETED_SUCCESSFULLY'", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -197,9 +201,11 @@ describe("TSQL Integration Tests", () => { name: "test-tenant-isolation-1", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -212,9 +218,11 @@ describe("TSQL Integration Tests", () => { name: "test-tenant-isolation-2", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant2", - projectId: "proj_tenant2", - environmentId: "env_tenant2", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant2" }, + project_id: { op: "eq", value: "proj_tenant2" }, + environment_id: { op: "eq", value: "env_tenant2" }, + }, tableSchema: [taskRunsSchema], }); @@ -254,9 +262,11 @@ describe("TSQL Integration Tests", () => { name: "test-cross-tenant-attack", query: "SELECT run_id, status FROM task_runs WHERE status = 'COMPLETED' OR 1=1", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_attacker", - projectId: "proj_attacker", - environmentId: "env_attacker", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_attacker" }, + project_id: { op: "eq", value: "proj_attacker" }, + environment_id: { op: "eq", value: "env_attacker" }, + }, tableSchema: [taskRunsSchema], }); @@ -288,9 +298,11 @@ describe("TSQL Integration Tests", () => { query: "SELECT status, count(*) as cnt FROM task_runs GROUP BY status ORDER BY cnt DESC, status ASC", schema: z.object({ status: z.string(), cnt: z.coerce.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -325,9 +337,11 @@ describe("TSQL Integration Tests", () => { name: "test-order-limit", query: "SELECT run_id FROM task_runs ORDER BY created_at DESC LIMIT 2", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -347,9 +361,11 @@ describe("TSQL Integration Tests", () => { name: "test-unknown-table", query: "SELECT * FROM unknown_table", schema: z.object({ id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -378,9 +394,11 @@ describe("TSQL Integration Tests", () => { name: "test-executor", query: "SELECT run_id, status FROM task_runs WHERE status = 'PENDING'", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, }); expect(error).toBeNull(); @@ -406,9 +424,11 @@ describe("TSQL Integration Tests", () => { name: "test-injection", query: "SELECT run_id, status FROM task_runs WHERE status = 'DROP TABLE task_runs'", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -438,9 +458,11 @@ describe("TSQL Integration Tests", () => { query: "SELECT run_id, status FROM task_runs WHERE status IN ('COMPLETED_SUCCESSFULLY', 'FAILED')", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -467,9 +489,11 @@ describe("TSQL Integration Tests", () => { name: "test-like-query", query: "SELECT run_id, task_identifier FROM task_runs WHERE task_identifier LIKE 'email%'", schema: z.object({ run_id: z.string(), task_identifier: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -530,8 +554,10 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-cross-project-query", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_multi", - // projectId and environmentId omitted - query across all + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_multi" }, + // project_id and environment_id omitted - query across all + }, tableSchema: [taskRunsSchema], }); @@ -590,9 +616,11 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-cross-env-query", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_envtest", - projectId: "proj_envtest", - // environmentId omitted - query across all environments + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_envtest" }, + project_id: { op: "eq", value: "proj_envtest" }, + // environment_id omitted - query across all environments + }, tableSchema: [taskRunsSchema], }); @@ -649,8 +677,10 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-org-isolation-1", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_isolation_1", - // projectId and environmentId omitted + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_isolation_1" }, + // project_id and environment_id omitted + }, tableSchema: [taskRunsSchema], }); @@ -663,8 +693,10 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-org-isolation-2", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_isolation_2", - // projectId and environmentId omitted + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_isolation_2" }, + // project_id and environment_id omitted + }, tableSchema: [taskRunsSchema], }); @@ -706,8 +738,10 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-or-bypass-attempt", query: "SELECT run_id, status FROM task_runs WHERE status = 'COMPLETED' OR 1=1", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_attacker", - // No project/env filter - but org filter should still protect + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_attacker" }, + // No project/env filter - but org filter should still protect + }, tableSchema: [taskRunsSchema], }); @@ -751,8 +785,10 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-executor-optional", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_executor_test", - // projectId and environmentId omitted + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_executor_test" }, + // project_id and environment_id omitted + }, }); expect(error).toBeNull(); @@ -839,9 +875,11 @@ describe("TSQL Virtual Column Tests", () => { execution_duration: z.number().nullable(), usage_duration_seconds: z.number(), }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [virtualColumnSchema], }); @@ -889,9 +927,11 @@ describe("TSQL Virtual Column Tests", () => { name: "test-virtual-column-where", query: "SELECT run_id FROM task_runs WHERE execution_duration > 5000", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [virtualColumnSchema], }); @@ -935,9 +975,11 @@ describe("TSQL Virtual Column Tests", () => { run_id: z.string(), usage_duration_seconds: z.number(), }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [virtualColumnSchema], }); @@ -977,9 +1019,11 @@ describe("TSQL Virtual Column Tests", () => { run_id: z.string(), dur_sec: z.number(), }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [virtualColumnSchema], }); @@ -1013,9 +1057,11 @@ describe("TSQL Virtual Column Tests", () => { run_id: z.string(), execution_duration: z.number().nullable(), }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [virtualColumnSchema], }); @@ -1110,9 +1156,11 @@ describe("TSQL Virtual Column Tests", () => { name: "test-expression-division-where", query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost > 1.0", schema: z.object({ run_id: z.string(), invocation_cost: z.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1153,9 +1201,11 @@ describe("TSQL Virtual Column Tests", () => { name: "test-expression-gte-where", query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost >= 1.0", schema: z.object({ run_id: z.string(), invocation_cost: z.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1192,9 +1242,11 @@ describe("TSQL Virtual Column Tests", () => { name: "test-expression-lt-where", query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost < 1.0", schema: z.object({ run_id: z.string(), invocation_cost: z.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1236,9 +1288,11 @@ describe("TSQL Virtual Column Tests", () => { query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost BETWEEN 1.0 AND 2.0", schema: z.object({ run_id: z.string(), invocation_cost: z.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1282,9 +1336,11 @@ describe("TSQL Virtual Column Tests", () => { query: "SELECT run_id FROM task_runs WHERE status = 'COMPLETED_SUCCESSFULLY' AND invocation_cost > 2.0", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1328,9 +1384,11 @@ describe("TSQL Virtual Column Tests", () => { name: "test-expression-large-integer-where", query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost > 100", schema: z.object({ run_id: z.string(), invocation_cost: z.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1393,9 +1451,11 @@ describe("Field Mapping Tests", () => { name: "test-field-mapping-select", query: "SELECT run_id, project_ref FROM task_runs", schema: z.object({ run_id: z.string(), project_ref: z.string().nullable() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [fieldMappingSchema], fieldMappings: { project: { @@ -1434,9 +1494,11 @@ describe("Field Mapping Tests", () => { name: "test-field-mapping-unmapped", query: "SELECT run_id, project_ref FROM task_runs WHERE run_id = 'run_fm_unmapped'", schema: z.object({ run_id: z.string(), project_ref: z.string().nullable() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [fieldMappingSchema], fieldMappings: { project: { @@ -1481,9 +1543,11 @@ describe("Field Mapping Tests", () => { name: "test-field-mapping-where", query: "SELECT run_id FROM task_runs WHERE project_ref = 'my-project-ref'", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [fieldMappingSchema], fieldMappings: { project: { @@ -1530,7 +1594,9 @@ describe("Field Mapping Tests", () => { query: "SELECT run_id FROM task_runs WHERE project_ref IN ('my-project-ref', 'other-project')", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, tableSchema: [fieldMappingSchema], fieldMappings: { project: { diff --git a/internal-packages/database/prisma/migrations/20260124203524_customer_query_add_title_remove_cost/migration.sql b/internal-packages/database/prisma/migrations/20260124203524_customer_query_add_title_remove_cost/migration.sql new file mode 100644 index 00000000000..ab542e6090d --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260124203524_customer_query_add_title_remove_cost/migration.sql @@ -0,0 +1,7 @@ +-- AlterTable +ALTER TABLE "CustomerQuery" +ADD COLUMN IF NOT EXISTS "title" TEXT; + +-- AlterTable +ALTER TABLE "CustomerQuery" +DROP COLUMN IF EXISTS "costInCents"; \ No newline at end of file diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 163445e3b13..c76b411412c 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2452,8 +2452,8 @@ model CustomerQuery { /// Query execution statistics from ClickHouse stats Json - /// Cost of the query in cents (for Stripe metering) - costInCents Float @default(0) + /// AI-generated title summarizing the query + title String? /// Where the query originated from source CustomerQuerySource @default(DASHBOARD) diff --git a/internal-packages/tsql/src/index.test.ts b/internal-packages/tsql/src/index.test.ts index 8621325c8c9..7a5182668d9 100644 --- a/internal-packages/tsql/src/index.test.ts +++ b/internal-packages/tsql/src/index.test.ts @@ -5,12 +5,12 @@ import { isColumnReferencedInExpression, createFallbackExpression, injectFallbackConditions, - type WhereClauseFallback, + type WhereClauseCondition, } from "./index.js"; import { column, type TableSchema } from "./query/schema.js"; /** - * Test table schema for whereClauseFallback tests + * Test table schema for enforcedWhereClause tests */ const taskRunsSchema: TableSchema = { name: "task_runs", @@ -21,6 +21,7 @@ const taskRunsSchema: TableSchema = { created_at: { name: "created_at", ...column("DateTime64") }, updated_at: { name: "updated_at", ...column("DateTime64") }, time: { name: "time", ...column("DateTime64") }, + triggered_at: { name: "triggered_at", ...column("DateTime64") }, organization_id: { name: "organization_id", ...column("String") }, project_id: { name: "project_id", ...column("String") }, environment_id: { name: "environment_id", ...column("String") }, @@ -32,6 +33,46 @@ const taskRunsSchema: TableSchema = { }, }; +/** + * Test table schema with tenant columns (lookup table with tenant isolation) + */ +const lookupTableSchema: TableSchema = { + name: "lookup_table", + clickhouseName: "trigger_dev.lookup_table", + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + columns: { + id: { name: "id", ...column("String") }, + name: { name: "name", ...column("String") }, + }, +}; + +/** + * Test table schema WITHOUT tenant columns (e.g., global reference data) + */ +// @ts-expect-error - tenant columns are required but not set +const nonTenantTableSchema: TableSchema = { + name: "reference_data", + clickhouseName: "trigger_dev.reference_data", + // No tenantColumns - this is a global table + columns: { + id: { name: "id", ...column("String") }, + value: { name: "value", ...column("String") }, + }, +}; + +/** + * Base options with tenant isolation for tests + */ +const baseEnforcedWhereClause: Record = { + organization_id: { op: "eq", value: "org_test123" }, + project_id: { op: "eq", value: "proj_test456" }, + environment_id: { op: "eq", value: "env_test789" }, +}; + describe("isColumnReferencedInExpression", () => { it("should detect column in simple WHERE clause", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-01-01'"); @@ -126,7 +167,7 @@ describe("createFallbackExpression", () => { describe("injectFallbackConditions", () => { it("should inject fallback when column is not in WHERE", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE status = 'completed'"); - const fallbacks: Record = { + const fallbacks: Record = { time: { op: "gte", value: "2024-01-01" }, }; @@ -140,7 +181,7 @@ describe("injectFallbackConditions", () => { it("should NOT inject fallback when column is already in WHERE", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-06-01'"); - const fallbacks: Record = { + const fallbacks: Record = { time: { op: "gte", value: "2024-01-01" }, }; @@ -154,7 +195,7 @@ describe("injectFallbackConditions", () => { it("should inject fallback when query has no WHERE clause", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs LIMIT 10"); - const fallbacks: Record = { + const fallbacks: Record = { time: { op: "gte", value: "2024-01-01" }, }; @@ -167,7 +208,7 @@ describe("injectFallbackConditions", () => { it("should inject multiple fallbacks", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs LIMIT 10"); - const fallbacks: Record = { + const fallbacks: Record = { time: { op: "gte", value: "2024-01-01" }, status: { op: "eq", value: "completed" }, }; @@ -182,7 +223,7 @@ describe("injectFallbackConditions", () => { it("should only inject fallbacks for unreferenced columns", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-06-01'"); - const fallbacks: Record = { + const fallbacks: Record = { time: { op: "gte", value: "2024-01-01" }, // Should NOT be injected status: { op: "eq", value: "completed" }, // Should be injected }; @@ -197,10 +238,8 @@ describe("injectFallbackConditions", () => { describe("compileTSQL with whereClauseFallback", () => { const baseOptions = { - organizationId: "org_test123", - projectId: "proj_test456", - environmentId: "env_test789", tableSchema: [taskRunsSchema], + enforcedWhereClause: baseEnforcedWhereClause, }; describe("simple comparison fallbacks", () => { @@ -474,3 +513,317 @@ describe("compileTSQL with whereClauseFallback", () => { }); }); +describe("compileTSQL with enforcedWhereClause", () => { + describe("validation tests", () => { + it("should throw error when required tenant column is missing", () => { + expect(() => + compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: {}, // Missing organization_id + }) + ).toThrow("Table 'task_runs' requires 'organization_id' in enforcedWhereClause"); + }); + + it("should throw error when organization_id is missing but other tenant columns are present", () => { + expect(() => + compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + project_id: { op: "eq", value: "proj_123" }, + environment_id: { op: "eq", value: "env_456" }, + }, + }) + ).toThrow("Table 'task_runs' requires 'organization_id' in enforcedWhereClause"); + }); + + it("should work with non-tenant table and empty enforcedWhereClause", () => { + const { sql } = compileTSQL("SELECT id FROM reference_data", { + tableSchema: [nonTenantTableSchema], + enforcedWhereClause: {}, + }); + + expect(sql).toContain("SELECT"); + expect(sql).toContain("FROM"); + }); + + it("should work with only organization_id (project and env are optional)", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + }, + }); + + expect(sql).toContain("organization_id"); + expect(sql).not.toContain("project_id"); + expect(sql).not.toContain("environment_id"); + }); + }); + + describe("basic functionality", () => { + it("should apply single enforced condition", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + }, + }); + + expect(sql).toContain("equals("); + expect(sql).toContain("organization_id"); + }); + + it("should apply multiple enforced conditions", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + project_id: { op: "eq", value: "proj_456" }, + environment_id: { op: "eq", value: "env_789" }, + }, + }); + + expect(sql).toContain("organization_id"); + expect(sql).toContain("project_id"); + expect(sql).toContain("environment_id"); + }); + + it("should apply enforced condition even when user filters on same field", () => { + const { sql } = compileTSQL( + "SELECT id FROM task_runs WHERE triggered_at > '2025-01-01'", + { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-01-01" }, + }, + } + ); + + // Should have BOTH the user's condition AND the enforced condition + // User's condition: greater(triggered_at, '2025-01-01') + // Enforced condition: greaterOrEquals(triggered_at, '2024-01-01') + const triggeredAtMatches = sql.match(/triggered_at/g) || []; + expect(triggeredAtMatches.length).toBeGreaterThanOrEqual(2); + }); + + it("should apply different comparison operators", () => { + const { sql: sqlGt } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + time: { op: "gt", value: "2024-01-01" }, + }, + }); + expect(sqlGt).toContain("greater("); + + const { sql: sqlLt } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + time: { op: "lt", value: "2024-12-31" }, + }, + }); + expect(sqlLt).toContain("less("); + + const { sql: sqlNeq } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + status: { op: "neq", value: "deleted" }, + }, + }); + expect(sqlNeq).toContain("notEquals("); + }); + + it("should apply BETWEEN condition", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + time: { op: "between", low: "2024-01-01", high: "2024-12-31" }, + }, + }); + + expect(sql).toContain("time BETWEEN"); + }); + + it("should handle Date values in enforced conditions", () => { + const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + const { sql, params } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: sevenDaysAgo }, + }, + }); + + expect(sql).toContain("triggered_at"); + expect(sql).toContain("toDateTime64"); + }); + }); + + describe("enforcedWhereClause + whereClauseFallback interaction", () => { + it("should apply both enforced and fallback conditions when user doesn't filter", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-01-01" }, + }, + whereClauseFallback: { + status: { op: "eq", value: "completed" }, + }, + }); + + // Should have both enforced (triggered_at) and fallback (status) + expect(sql).toContain("triggered_at"); + expect(sql).toContain("status"); + }); + + it("should apply enforced but not fallback when user filters on fallback column", () => { + const { sql, params } = compileTSQL( + "SELECT id FROM task_runs WHERE status = 'failed'", + { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-01-01" }, + }, + whereClauseFallback: { + status: { op: "eq", value: "completed" }, + }, + } + ); + + // Enforced triggered_at should be applied + expect(sql).toContain("triggered_at"); + // User's status = 'failed' should be there (as a parameter) + expect(Object.values(params)).toContain("failed"); + // The fallback 'completed' should NOT be applied since user filtered on status + expect(Object.values(params)).not.toContain("completed"); + }); + + it("should apply both enforced and fallback on same field (enforced always, fallback only if not filtered)", () => { + // User doesn't filter on triggered_at, so BOTH enforced AND fallback apply + const { sql } = compileTSQL("SELECT id FROM task_runs WHERE status = 'completed'", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-06-01" }, // Enforced: last 6 months + }, + whereClauseFallback: { + triggered_at: { op: "gte", value: "2024-01-01" }, // Fallback: last year + }, + }); + + // Both should be applied (enforced at printer level, fallback at AST level) + const triggeredAtMatches = sql.match(/triggered_at/g) || []; + expect(triggeredAtMatches.length).toBeGreaterThanOrEqual(2); + }); + + it("should skip fallback but keep enforced when user filters on same field", () => { + const { sql } = compileTSQL( + "SELECT id FROM task_runs WHERE triggered_at > '2025-01-01'", + { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-06-01" }, // Enforced: always applied + }, + whereClauseFallback: { + triggered_at: { op: "gte", value: "2024-01-01" }, // Fallback: skipped since user filtered + }, + } + ); + + // User's condition + enforced should be present + // Fallback should NOT be applied since user filtered on triggered_at + // Count distinct triggered_at conditions + const triggeredAtMatches = sql.match(/triggered_at/g) || []; + // Should be 2: user's condition + enforced condition (NOT 3, no fallback) + expect(triggeredAtMatches.length).toBe(2); + }); + }); + + describe("security tests", () => { + it("should apply enforced conditions to UNION queries", () => { + const { sql } = compileTSQL( + "SELECT id FROM task_runs WHERE status = 'completed' UNION ALL SELECT id FROM task_runs WHERE status = 'failed'", + { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-01-01" }, + }, + } + ); + + // Both parts of the UNION should have the enforced conditions + const orgMatches = sql.match(/organization_id/g) || []; + expect(orgMatches.length).toBe(2); + + const triggeredAtMatches = sql.match(/triggered_at/g) || []; + expect(triggeredAtMatches.length).toBe(2); + }); + + it("should NOT be bypassable via OR clause", () => { + const { sql } = compileTSQL( + "SELECT id FROM task_runs WHERE status = 'completed' OR 1=1", + { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-01-01" }, + }, + } + ); + + // The enforced conditions should be ANDed with the entire user WHERE clause + // So the structure should be: (enforced AND enforced AND ...) AND (user_where) + expect(sql).toContain("organization_id"); + expect(sql).toContain("triggered_at"); + // The 1=1 should be within the user's OR clause, not affecting enforced conditions + }); + + it("should skip enforced conditions for columns that don't exist in table", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + nonexistent_column: { op: "eq", value: "test" }, + }, + }); + + // Should not contain nonexistent_column + expect(sql).not.toContain("nonexistent_column"); + // Should still have organization_id + expect(sql).toContain("organization_id"); + }); + }); + + describe("edge cases", () => { + it("should handle empty enforced conditions for non-tenant table", () => { + const { sql } = compileTSQL("SELECT id FROM reference_data", { + tableSchema: [nonTenantTableSchema], + enforcedWhereClause: {}, + }); + + expect(sql).toContain("SELECT"); + expect(sql).not.toContain("WHERE"); // No WHERE clause needed + }); + + it("should properly format numeric values", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + }, + }); + + // org_123 should be parameterized, not inlined + expect(sql).toContain("tsql_val_"); + }); + }); +}); + diff --git a/internal-packages/tsql/src/index.ts b/internal-packages/tsql/src/index.ts index 9a7b3ddb20d..e0f061eef9f 100644 --- a/internal-packages/tsql/src/index.ts +++ b/internal-packages/tsql/src/index.ts @@ -23,7 +23,13 @@ import { CompareOperationOp } from "./query/ast.js"; import { SyntaxError as TSQLSyntaxError } from "./query/errors.js"; import { TSQLParseTreeConverter } from "./query/parser.js"; import { printToClickHouse, type PrintResult } from "./query/printer.js"; -import { createPrinterContext, type QuerySettings } from "./query/printer_context.js"; +import { + createPrinterContext, + type BetweenCondition, + type QuerySettings, + type SimpleComparisonCondition, + type WhereClauseCondition, +} from "./query/printer_context.js"; import { createSchemaRegistry, type FieldMappings, type TableSchema } from "./query/schema.js"; /** @@ -113,9 +119,12 @@ export { createPrinterContext, DEFAULT_QUERY_SETTINGS, PrinterContext, + type BetweenCondition, type PrinterContextOptions, type QueryNotice, type QuerySettings, + type SimpleComparisonCondition, + type WhereClauseCondition, } from "./query/printer_context.js"; // Re-export printer @@ -304,7 +313,7 @@ function createValueExpression(value: Date | string | number): Expression { /** * Map fallback operator to CompareOperationOp */ -function mapFallbackOpToCompareOp(op: SimpleComparisonFallback["op"]): CompareOperationOp { +function mapFallbackOpToCompareOp(op: SimpleComparisonCondition["op"]): CompareOperationOp { switch (op) { case "eq": return CompareOperationOp.Eq; @@ -330,7 +339,7 @@ function mapFallbackOpToCompareOp(op: SimpleComparisonFallback["op"]): CompareOp */ export function createFallbackExpression( column: string, - fallback: WhereClauseFallback + fallback: WhereClauseCondition ): Expression { const fieldExpr: Field = { expression_type: "field", @@ -367,7 +376,7 @@ export function createFallbackExpression( */ export function injectFallbackConditions( ast: SelectQuery | SelectSetQuery, - fallbacks: Record + fallbacks: Record ): SelectQuery | SelectSetQuery { // Handle SelectSetQuery (UNION, etc.) - apply to each query in the set if (ast.expression_type === "select_set_query") { @@ -434,46 +443,31 @@ export function injectFallbackConditions( }; } -/** - * A simple comparison fallback condition (e.g., column > value) - */ -export interface SimpleComparisonFallback { - /** The comparison operator */ - op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte"; - /** The value to compare against */ - value: Date | string | number; -} - -/** - * A between fallback condition (e.g., column BETWEEN low AND high) - */ -export interface BetweenFallback { - /** The between operator */ - op: "between"; - /** The low bound of the range */ - low: Date | string | number; - /** The high bound of the range */ - high: Date | string | number; -} - -/** - * A WHERE clause fallback condition. - * Used to apply default filters when the user hasn't specified one for a column. - */ -export type WhereClauseFallback = SimpleComparisonFallback | BetweenFallback; /** * Options for compiling a TSQL query to ClickHouse SQL */ export interface CompileTSQLOptions { - /** The organization ID for tenant isolation (required) */ - organizationId: string; - /** The project ID for tenant isolation (optional - omit to query across all projects) */ - projectId?: string; - /** The environment ID for tenant isolation (optional - omit to query across all environments) */ - environmentId?: string; /** Schema definitions for allowed tables and columns */ tableSchema: TableSchema[]; + /** + * REQUIRED: Conditions always applied at the table level. + * Must include tenant columns (e.g., organization_id) for multi-tenant tables. + * Applied to every table reference including subqueries, CTEs, and JOINs. + * + * @example + * ```typescript + * { + * // Tenant isolation + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * environment_id: { op: "eq", value: "env_789" }, + * // Plan-based time limit + * triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) } + * } + * ``` + */ + enforcedWhereClause: Record; /** Optional query settings */ settings?: Partial; /** @@ -491,6 +485,7 @@ export interface CompileTSQLOptions { /** * Fallback WHERE conditions to apply when the user hasn't filtered on a column. * Key is the column name, value is the fallback condition. + * These are applied at the AST level (top-level query only). * * @example * ```typescript @@ -505,7 +500,7 @@ export interface CompileTSQLOptions { * } * ``` */ - whereClauseFallback?: Record; + whereClauseFallback?: Record; } /** @@ -514,24 +509,28 @@ export interface CompileTSQLOptions { * This function: * 1. Parses the TSQL query into an AST * 2. Validates tables and columns against the schema - * 3. Injects tenant isolation WHERE clauses - * 4. Generates parameterized ClickHouse SQL + * 3. Injects enforced WHERE clauses (tenant isolation + plan limits) at printer level + * 4. Optionally injects fallback WHERE conditions at AST level + * 5. Generates parameterized ClickHouse SQL * * @param query - The TSQL query string to compile - * @param options - Compilation options including tenant IDs and schema + * @param options - Compilation options including enforcedWhereClause and schema * @returns The compiled SQL and parameters * @throws TSQLSyntaxError if the query is invalid - * @throws QueryError if tables/columns are not allowed + * @throws QueryError if tables/columns are not allowed or required tenant columns are missing * * @example * ```typescript * const { sql, params } = compileTSQL( * "SELECT * FROM task_runs WHERE status = 'completed' LIMIT 100", * { - * organizationId: "org_123", - * projectId: "proj_456", - * environmentId: "env_789", * tableSchema: [taskRunsSchema], + * enforcedWhereClause: { + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * environment_id: { op: "eq", value: "env_789" }, + * triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }, + * }, * } * ); * ``` @@ -540,7 +539,7 @@ export function compileTSQL(query: string, options: CompileTSQLOptions): PrintRe // 1. Parse the TSQL query let ast = parseTSQLSelect(query); - // 2. Inject fallback WHERE conditions if provided + // 2. Inject fallback WHERE conditions if provided (applied at AST level - top-level query only) if (options.whereClauseFallback && Object.keys(options.whereClauseFallback).length > 0) { ast = injectFallbackConditions(ast, options.whereClauseFallback); } @@ -548,16 +547,20 @@ export function compileTSQL(query: string, options: CompileTSQLOptions): PrintRe // 3. Create schema registry from table schemas const schemaRegistry = createSchemaRegistry(options.tableSchema); - // 4. Create printer context with tenant IDs and field mappings + + // 4. Strip undefined values from enforcedWhereClause + const enforcedWhereClause = Object.fromEntries( + Object.entries(options.enforcedWhereClause).filter(([_, value]) => value !== undefined) + ) as Record; + + // 5. Create printer context with enforced WHERE clause and field mappings const context = createPrinterContext({ - organizationId: options.organizationId, - projectId: options.projectId, - environmentId: options.environmentId, schema: schemaRegistry, settings: options.settings, fieldMappings: options.fieldMappings, + enforcedWhereClause, }); - // 5. Print the AST to ClickHouse SQL + // 6. Print the AST to ClickHouse SQL (enforced conditions applied at printer level) return printToClickHouse(ast, context); } diff --git a/internal-packages/tsql/src/query/printer.test.ts b/internal-packages/tsql/src/query/printer.test.ts index dcbb79b2d84..6c5cda6e626 100644 --- a/internal-packages/tsql/src/query/printer.test.ts +++ b/internal-packages/tsql/src/query/printer.test.ts @@ -85,10 +85,12 @@ function createTestContext( ): PrinterContext { const schema = createSchemaRegistry([taskRunsSchema, taskEventsSchema]); return createPrinterContext({ - organizationId: "org_test123", - projectId: "proj_test456", - environmentId: "env_test789", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test123" }, + project_id: { op: "eq", value: "proj_test456" }, + environment_id: { op: "eq", value: "env_test789" }, + }, ...overrides, }); } @@ -153,10 +155,12 @@ describe("ClickHousePrinter", () => { it("should expand SELECT * with column name mapping", () => { const schema = createSchemaRegistry([runsSchema]); const ctx = createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); const { sql, columns } = printQuery("SELECT * FROM runs", ctx); @@ -216,10 +220,12 @@ describe("ClickHousePrinter", () => { const schema = createSchemaRegistry([schemaWithVirtual]); const ctx = createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); const { sql, columns } = printQuery("SELECT * FROM runs", ctx); @@ -241,15 +247,17 @@ describe("ClickHousePrinter", () => { describe("Table and column name mapping", () => { function createMappedContext() { const schema = createSchemaRegistry([runsSchema]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); + } - it("should map user-friendly table name to ClickHouse name", () => { + it("should map user-friendly table name to ClickHouse name", () => { const ctx = createMappedContext(); const { sql } = printQuery("SELECT * FROM runs", ctx); @@ -472,15 +480,17 @@ describe("ClickHousePrinter", () => { function createJsonContext() { const schema = createSchemaRegistry([jsonSchema]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); + } - it("should transform IS NULL to equals empty object for JSON columns with nullValue", () => { + it("should transform IS NULL to equals empty object for JSON columns with nullValue", () => { const ctx = createJsonContext(); const { sql } = printQuery("SELECT * FROM runs WHERE error IS NULL", ctx); @@ -597,6 +607,501 @@ describe("ClickHousePrinter", () => { expect(sql).toContain("GROUP BY status"); expect(sql).not.toContain(".:String"); }); + + it("should NOT add .:String type hint for JSON subfield in WHERE comparison", () => { + const ctx = createJsonContext(); + const { sql } = printQuery( + "SELECT id FROM runs WHERE error.data.name = 'test'", + ctx + ); + + // WHERE clause should NOT have .:String type hint (it breaks the query) + expect(sql).toContain("equals(error.data.name,"); + expect(sql).not.toContain("error.data.name.:String"); + }); + + it("should NOT add .:String for JSON subfield in WHERE with LIKE", () => { + const ctx = createJsonContext(); + const { sql } = printQuery( + "SELECT id FROM runs WHERE error.message LIKE '%error%'", + ctx + ); + + // WHERE clause should NOT have .:String type hint + expect(sql).toContain("like(error.message,"); + expect(sql).not.toContain("error.message.:String"); + }); + + it("should NOT add .:String in SELECT or WHERE when no GROUP BY", () => { + const ctx = createJsonContext(); + const { sql } = printQuery( + "SELECT error.data.name FROM runs WHERE error.data.name = 'test'", + ctx + ); + + // SELECT should NOT have .:String (no GROUP BY, so no need for type hint) + expect(sql).toContain("error.data.name AS error_data_name"); + expect(sql).not.toContain(".:String"); + // WHERE should NOT have .:String + expect(sql).toContain("equals(error.data.name,"); + }); + + it("should add .:String in GROUP BY but not in WHERE for same query", () => { + const ctx = createJsonContext(); + const { sql } = printQuery( + "SELECT error.data.name, count() AS cnt FROM runs WHERE error.data.name = 'test' GROUP BY error.data.name", + ctx + ); + + // SELECT should have .:String + expect(sql).toContain("error.data.name.:String AS error_data_name"); + // GROUP BY should have .:String + expect(sql).toContain("GROUP BY error.data.name.:String"); + // WHERE should NOT have .:String + expect(sql).toContain("equals(error.data.name,"); + expect(sql).not.toMatch(/equals\(error\.data\.name\.:String/); + }); + }); + + describe("textColumn optimization for JSON columns", () => { + // Create a schema with JSON columns that have textColumn set + const textColumnSchema: TableSchema = { + name: "runs", + clickhouseName: "trigger_dev.task_runs_v2", + columns: { + id: { name: "id", ...column("String") }, + output: { + name: "output", + ...column("JSON"), + nullValue: "'{}'", + textColumn: "output_text", + }, + error: { + name: "error", + ...column("JSON"), + nullValue: "'{}'", + textColumn: "error_text", + }, + status: { name: "status", ...column("String") }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + function createTextColumnContext() { + const schema = createSchemaRegistry([textColumnSchema]); + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); + } + + describe("SELECT clause", () => { + it("should use text column when selecting bare JSON column", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT output FROM runs", ctx); + + // Should use the text column with an alias to preserve the column name + expect(sql).toContain("output_text AS output"); + }); + + it("should use text column for multiple JSON columns", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT output, error FROM runs", ctx); + + expect(sql).toContain("output_text AS output"); + expect(sql).toContain("error_text AS error"); + }); + + it("should use JSON column for subfield access without .:String when no GROUP BY", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT output.data.name FROM runs", ctx); + + // Should use the original JSON column without .:String (no GROUP BY) + expect(sql).toContain("output.data.name AS output_data_name"); + expect(sql).not.toContain("output_text"); + expect(sql).not.toContain(".:String"); + }); + }); + + describe("SELECT * expansion", () => { + it("should use text columns when expanding SELECT *", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT * FROM runs", ctx); + + // Should use text columns for JSON columns + expect(sql).toContain("output_text AS output"); + expect(sql).toContain("error_text AS error"); + }); + }); + + describe("WHERE clause", () => { + it("should use text column for exact equality comparison", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE output = '{}'", ctx); + + expect(sql).toContain("equals(output_text,"); + expect(sql).not.toMatch(/equals\(output,/); + }); + + it("should use text column for inequality comparison", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE output != '{}'", ctx); + + expect(sql).toContain("notEquals(output_text,"); + }); + + it("should use text column for LIKE comparison", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE output LIKE '%error%'", ctx); + + expect(sql).toContain("like(output_text,"); + expect(sql).not.toMatch(/like\(output,/); + }); + + it("should use text column for ILIKE comparison", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE error ILIKE '%failed%'", ctx); + + expect(sql).toContain("ilike(error_text,"); + }); + + it("should use text column for NOT LIKE comparison", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE output NOT LIKE '%test%'", ctx); + + expect(sql).toContain("notLike(output_text,"); + }); + + it("should use JSON column for subfield comparison without .:String", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery( + "SELECT id FROM runs WHERE output.data.name = 'test'", + ctx + ); + + // Should use the original JSON column, not the text column + // And should NOT have .:String in WHERE (breaks the query) + expect(sql).toContain("equals(output.data.name,"); + expect(sql).not.toContain("output_text"); + expect(sql).not.toContain("output.data.name.:String"); + }); + + it("should still use nullValue transformation for IS NULL", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE output IS NULL", ctx); + + // NULL check should use the text column with nullValue + expect(sql).toContain("equals(output_text, '{}')"); + }); + + it("should still use nullValue transformation for IS NOT NULL", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE error IS NOT NULL", ctx); + + expect(sql).toContain("notEquals(error_text, '{}')"); + }); + }); + + describe("edge cases", () => { + it("should work with columns without textColumn defined", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT status FROM runs WHERE status = 'completed'", ctx); + + // Regular column should work as before + expect(sql).toContain("status"); + expect(sql).not.toContain("status_text"); + }); + + it("should use text column for aliased JSON columns in SELECT", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT output AS result FROM runs", ctx); + + // Should use text column with user's alias + expect(sql).toContain("output_text AS result"); + }); + + it("should use text column for table-qualified JSON columns in SELECT", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT runs.output FROM runs", ctx); + + // Should use text column + expect(sql).toContain("output_text AS output"); + }); + + it("should use text column in both SELECT and WHERE for same query", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery( + "SELECT output FROM runs WHERE output LIKE '%test%'", + ctx + ); + + // SELECT should use text column + expect(sql).toContain("output_text AS output"); + // WHERE should use text column + expect(sql).toContain("like(output_text,"); + }); + }); + + describe("JOINs with textColumn", () => { + // Create a second schema with the same JSON column names to test JOIN ambiguity + const runsSchemaWithTextColumn: TableSchema = { + name: "runs", + clickhouseName: "trigger_dev.task_runs_v2", + columns: { + id: { name: "id", ...column("String") }, + output: { + name: "output", + ...column("JSON"), + nullValue: "'{}'", + textColumn: "output_text", + }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + const eventsSchemaWithTextColumn: TableSchema = { + name: "events", + clickhouseName: "trigger_dev.task_events_v2", + columns: { + id: { name: "id", ...column("String") }, + run_id: { name: "run_id", ...column("String") }, + output: { + name: "output", + ...column("JSON"), + nullValue: "'{}'", + textColumn: "output_text", + }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + function createJoinTextColumnContext() { + const schema = createSchemaRegistry([runsSchemaWithTextColumn, eventsSchemaWithTextColumn]); + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); + } + + it("should qualify text column with table alias in JOIN WHERE clause to avoid ambiguity", () => { + const ctx = createJoinTextColumnContext(); + const { sql } = printQuery( + `SELECT r.id FROM runs r JOIN events e ON r.id = e.run_id WHERE r.output = '{}'`, + ctx + ); + + // The text column should be table-qualified to avoid ambiguity + // since both tables have an output_text column + expect(sql).toContain("equals(r.output_text,"); + // Should NOT have unqualified output_text in the comparison + expect(sql).not.toMatch(/equals\(output_text,/); + }); + + it("should qualify text column with table alias for LIKE in JOIN", () => { + const ctx = createJoinTextColumnContext(); + const { sql } = printQuery( + `SELECT r.id FROM runs r JOIN events e ON r.id = e.run_id WHERE e.output LIKE '%error%'`, + ctx + ); + + // Should use table-qualified text column + expect(sql).toContain("like(e.output_text,"); + expect(sql).not.toMatch(/like\(output_text,/); + }); + + it("should handle multiple qualified text column comparisons in JOIN", () => { + const ctx = createJoinTextColumnContext(); + const { sql } = printQuery( + `SELECT r.id FROM runs r JOIN events e ON r.id = e.run_id WHERE r.output = '{}' AND e.output != '{}'`, + ctx + ); + + // Both comparisons should be table-qualified + expect(sql).toContain("equals(r.output_text,"); + expect(sql).toContain("notEquals(e.output_text,"); + }); + }); + }); + + describe("dataPrefix for JSON columns", () => { + // Create a schema with JSON columns that have dataPrefix set + const dataPrefixSchema: TableSchema = { + name: "runs", + clickhouseName: "trigger_dev.task_runs_v2", + columns: { + id: { name: "id", ...column("String") }, + output: { + name: "output", + ...column("JSON"), + nullValue: "'{}'", + dataPrefix: "data", + }, + error: { + name: "error", + ...column("JSON"), + nullValue: "'{}'", + dataPrefix: "data", + }, + status: { name: "status", ...column("String") }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + function createDataPrefixContext() { + const schema = createSchemaRegistry([dataPrefixSchema]); + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); + } + + describe("SELECT clause", () => { + it("should inject dataPrefix into JSON subfield path", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT output.message FROM runs", ctx); + + // Should transform output.message to output.data.message + expect(sql).toContain("output.data.message"); + }); + + it("should generate clean alias without dataPrefix", () => { + const ctx = createDataPrefixContext(); + const { sql, columns } = printQuery("SELECT output.message FROM runs", ctx); + + // Alias should be output_message, not output_data_message + expect(sql).toContain("AS output_message"); + expect(sql).not.toContain("AS output_data_message"); + expect(columns).toContainEqual( + expect.objectContaining({ name: "output_message" }) + ); + }); + + it("should handle nested paths with dataPrefix", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT output.user.name FROM runs", ctx); + + // Should transform output.user.name to output.data.user.name + expect(sql).toContain("output.data.user.name"); + // Alias should be output_user_name + expect(sql).toContain("AS output_user_name"); + }); + + it("should work with multiple JSON columns with dataPrefix", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT output.msg, error.code FROM runs", ctx); + + expect(sql).toContain("output.data.msg"); + expect(sql).toContain("error.data.code"); + expect(sql).toContain("AS output_msg"); + expect(sql).toContain("AS error_code"); + }); + + it("should not affect bare JSON column selection", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT output FROM runs", ctx); + + // Bare column should not have dataPrefix injected + expect(sql).not.toContain("output.data"); + expect(sql).toMatch(/SELECT\s+output[\s,]/); + }); + }); + + describe("WHERE clause", () => { + it("should inject dataPrefix into WHERE comparison", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery( + "SELECT id FROM runs WHERE output.status = 'success'", + ctx + ); + + // Should transform output.status to output.data.status + expect(sql).toContain("output.data.status"); + }); + + it("should inject dataPrefix into LIKE comparison", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery( + "SELECT id FROM runs WHERE error.message LIKE '%failed%'", + ctx + ); + + expect(sql).toContain("error.data.message"); + }); + }); + + describe("GROUP BY clause", () => { + it("should inject dataPrefix into GROUP BY", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery( + "SELECT output.type, count() AS cnt FROM runs GROUP BY output.type", + ctx + ); + + // Should inject dataPrefix in both SELECT and GROUP BY + expect(sql).toContain("output.data.type"); + expect(sql).toContain("GROUP BY output.data.type"); + }); + }); + + describe("edge cases", () => { + it("should not affect columns without dataPrefix", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT status FROM runs", ctx); + + // Regular column should not be affected + expect(sql).toContain("status"); + expect(sql).not.toContain("status.data"); + }); + + it("should work with explicit alias on JSON subfield", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT output.message AS msg FROM runs", ctx); + + // Should inject dataPrefix but use user's alias + expect(sql).toContain("output.data.message"); + expect(sql).toContain("AS msg"); + }); + }); }); describe("ORDER BY clauses", () => { @@ -742,9 +1247,11 @@ describe("ClickHousePrinter", () => { describe("Tenant isolation", () => { it("should inject tenant guards for single table", () => { const context = createTestContext({ - organizationId: "org_abc", - projectId: "proj_def", - environmentId: "env_ghi", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_abc" }, + project_id: { op: "eq", value: "proj_def" }, + environment_id: { op: "eq", value: "env_ghi" }, + }, }); const { sql, params } = printQuery("SELECT * FROM task_runs", context); @@ -1057,15 +1564,17 @@ describe("Value mapping (valueMap)", () => { function createValueMapContext() { const schema = createSchemaRegistry([statusMappedSchema]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); +} - it("should transform user-friendly value to internal value in equality comparison", () => { +it("should transform user-friendly value to internal value in equality comparison", () => { const ctx = createValueMapContext(); const { sql, params } = printQuery("SELECT * FROM runs WHERE status = 'Completed'", ctx); @@ -1173,15 +1682,17 @@ describe("WHERE transform (whereTransform)", () => { function createPrefixedContext() { const schema = createSchemaRegistry([prefixedIdSchema]); - return createPrinterContext({ - organizationId: "org_test123", - projectId: "proj_test456", - environmentId: "env_test789", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test123" }, + project_id: { op: "eq", value: "proj_test456" }, + environment_id: { op: "eq", value: "env_test789" }, + }, + }); +} - it("should strip prefix from value in equality comparison", () => { +it("should strip prefix from value in equality comparison", () => { const ctx = createPrefixedContext(); const { params } = printQuery("SELECT * FROM runs WHERE batch_id = 'batch_abc123'", ctx); @@ -1397,16 +1908,18 @@ describe("Virtual columns", () => { function createVirtualColumnContext() { const schema = createSchemaRegistry([virtualColumnSchema]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); +} - describe("SELECT clause", () => { - it("should expand bare virtual column to expression with alias", () => { +describe("SELECT clause", () => { + it("should expand bare virtual column to expression with alias", () => { const ctx = createVirtualColumnContext(); const { sql } = printQuery("SELECT execution_duration FROM runs", ctx); @@ -1638,16 +2151,18 @@ describe("Expression columns with division (cost/invocation_cost pattern)", () = function createCostExpressionContext() { const schema = createSchemaRegistry([costExpressionSchema]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); +} - describe("WHERE clause with division expression columns", () => { - it("should expand invocation_cost > 100 to (base_cost_in_cents / 100.0) > 100", () => { +describe("WHERE clause with division expression columns", () => { + it("should expand invocation_cost > 100 to (base_cost_in_cents / 100.0) > 100", () => { const ctx = createCostExpressionContext(); const { sql } = printQuery("SELECT * FROM runs WHERE invocation_cost > 100", ctx); @@ -1782,16 +2297,18 @@ describe("Column metadata", () => { function createMetadataTestContext() { const schema = createSchemaRegistry([schemaWithRenderTypes]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); +} - describe("Basic column metadata", () => { - it("should return column metadata for simple field references", () => { +describe("Basic column metadata", () => { + it("should return column metadata for simple field references", () => { const ctx = createMetadataTestContext(); const { columns } = printQuery("SELECT run_id, created_at FROM runs", ctx); @@ -2193,10 +2710,12 @@ describe("Unknown column blocking", () => { // Using the internal name directly should be blocked const schema = createSchemaRegistry([runsSchema]); const ctx = createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); // 'created_at' is not in runsSchema - only 'created' which maps to 'created_at' @@ -2210,10 +2729,12 @@ describe("Unknown column blocking", () => { // When user types 'created_at', we should suggest 'created' const schema = createSchemaRegistry([runsSchema]); const ctx = createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); expect(() => { @@ -2346,9 +2867,6 @@ describe("Field Mapping Value Transformation", () => { function createFieldMappingContext(): PrinterContext { const schemaRegistry = createSchemaRegistry([fieldMappingSchema]); return new PrinterContext( - "org_123", - "proj_456", - "env_789", schemaRegistry, {}, { @@ -2356,6 +2874,11 @@ describe("Field Mapping Value Transformation", () => { proj_tenant1: "my-project-ref", proj_other: "other-project", }, + }, + { + organization_id: { op: "eq", value: "org_123" }, + project_id: { op: "eq", value: "proj_456" }, + environment_id: { op: "eq", value: "env_789" }, } ); } @@ -2474,20 +2997,24 @@ describe("Internal-only column blocking", () => { function createHiddenTenantContext(): PrinterContext { const schema = createSchemaRegistry([hiddenTenantSchema]); return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); } function createHiddenFilterContext(): PrinterContext { const schema = createSchemaRegistry([hiddenFilterSchema]); return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); } @@ -2655,10 +3182,12 @@ describe("Required Filters", () => { function createRequiredFiltersContext(): PrinterContext { const schemaRegistry = createSchemaRegistry([schemaWithRequiredFilters]); return createPrinterContext({ - organizationId: "org_test123", - projectId: "proj_test456", - environmentId: "env_test789", schema: schemaRegistry, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test123" }, + project_id: { op: "eq", value: "proj_test456" }, + environment_id: { op: "eq", value: "env_test789" }, + }, }); } diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts index 75fcd566288..789f1836ec1 100644 --- a/internal-packages/tsql/src/query/printer.ts +++ b/internal-packages/tsql/src/query/printer.ts @@ -46,7 +46,7 @@ import { findTSQLFunction, validateFunctionArgs, } from "./functions"; -import { PrinterContext } from "./printer_context"; +import { PrinterContext, WhereClauseCondition } from "./printer_context"; import { findTable, validateTable, @@ -114,6 +114,8 @@ export class ClickHousePrinter { private outputColumns: OutputColumnMetadata[] = []; /** Whether we're currently processing GROUP BY expressions */ private inGroupByContext = false; + /** Whether the current query has a GROUP BY clause (used for JSON subfield type hints) */ + private queryHasGroupBy = false; /** Columns hidden when SELECT * is expanded to core columns only */ private hiddenColumns: string[] = []; /** @@ -392,6 +394,11 @@ export class ClickHousePrinter { } } + // Track if query has GROUP BY for JSON subfield type hint decisions + // (ClickHouse requires .:String for Dynamic types in GROUP BY, and SELECT must match) + const savedQueryHasGroupBy = this.queryHasGroupBy; + this.queryHasGroupBy = !!node.group_by; + // Process SELECT columns and collect metadata // Using flatMap because asterisk expansion can return multiple columns // Set inProjectionContext to block internal-only columns in user projections @@ -543,6 +550,7 @@ export class ClickHousePrinter { // Restore saved contexts (for nested queries) this.selectAliases = savedAliases; + this.queryHasGroupBy = savedQueryHasGroupBy; this.tableContexts = savedTableContexts; this.allowedInternalColumns = savedInternalColumns; this.internalOnlyColumns = savedInternalOnlyColumns; @@ -627,37 +635,47 @@ export class ClickHousePrinter { let sqlResult: string; if ((col as Field).expression_type === "field") { const field = col as Field; - const virtualColumnName = this.getVirtualColumnNameForField(field.chain); - if (virtualColumnName !== null) { - // Visit the field (which will return the expression) - const visited = this.visit(col); - // Add the alias to preserve the column name - sqlResult = `${visited} AS ${this.printIdentifier(virtualColumnName)}`; + // Check if this is a bare JSON field that should use a text column + const textColumn = this.getTextColumnForField(field.chain); + if (textColumn !== null && outputName) { + // Use the text column instead of the JSON column, with alias to preserve name + sqlResult = `${this.printIdentifier(textColumn)} AS ${this.printIdentifier(outputName)}`; } else { - // Visit the field to get the ClickHouse SQL - const visited = this.visit(col); - - // Check if this is a JSON subfield access (will have .:String type hint) - // If so, add an alias to preserve the nice column name (dots → underscores) - const isJsonSubfield = this.isJsonSubfieldAccess(field.chain); - if (isJsonSubfield) { - // Build the alias using underscores (e.g., "error_data_name") - const aliasName = field.chain.filter((p): p is string => typeof p === "string").join("_"); - sqlResult = `${visited} AS ${this.printIdentifier(aliasName)}`; - // Override output name for metadata - effectiveOutputName = aliasName; - } - // Check if the column has a different clickhouseName - if so, add an alias - // to ensure results come back with the user-facing name - else if ( - outputName && - sourceColumn?.clickhouseName && - sourceColumn.clickhouseName !== outputName - ) { - sqlResult = `${visited} AS ${this.printIdentifier(outputName)}`; + const virtualColumnName = this.getVirtualColumnNameForField(field.chain); + + if (virtualColumnName !== null) { + // Visit the field (which will return the expression) + const visited = this.visit(col); + // Add the alias to preserve the column name + sqlResult = `${visited} AS ${this.printIdentifier(virtualColumnName)}`; } else { - sqlResult = visited; + // Visit the field to get the ClickHouse SQL + const visited = this.visit(col); + + // Check if this is a JSON subfield access (will have .:String type hint) + // If so, add an alias to preserve the nice column name (dots → underscores) + const isJsonSubfield = this.isJsonSubfieldAccess(field.chain); + if (isJsonSubfield) { + // Build the alias using underscores, excluding any dataPrefix + // e.g., output.message -> "output_message" (not "output_data_message") + const dataPrefix = this.getDataPrefixForField(field.chain); + const aliasName = this.buildAliasWithoutDataPrefix(field.chain, dataPrefix); + sqlResult = `${visited} AS ${this.printIdentifier(aliasName)}`; + // Override output name for metadata + effectiveOutputName = aliasName; + } + // Check if the column has a different clickhouseName - if so, add an alias + // to ensure results come back with the user-facing name + else if ( + outputName && + sourceColumn?.clickhouseName && + sourceColumn.clickhouseName !== outputName + ) { + sqlResult = `${visited} AS ${this.printIdentifier(outputName)}`; + } else { + sqlResult = visited; + } } } } else if ( @@ -675,8 +693,23 @@ export class ClickHousePrinter { } else { sqlResult = visited; } + } else if ((col as Alias).expression_type === "alias") { + // Handle Alias expressions - check if inner expression is a bare JSON field with textColumn + const alias = col as Alias; + if ((alias.expr as Field).expression_type === "field") { + const innerField = alias.expr as Field; + const textColumn = this.getTextColumnForField(innerField.chain); + if (textColumn !== null) { + // Use the text column with the user's explicit alias + sqlResult = `${this.printIdentifier(textColumn)} AS ${this.printIdentifier(alias.alias)}`; + } else { + sqlResult = this.visit(col); + } + } else { + sqlResult = this.visit(col); + } } else { - // For Alias expressions or other types, visit normally + // For other types, visit normally sqlResult = this.visit(col); } @@ -817,6 +850,11 @@ export class ClickHousePrinter { if (isVirtualColumn(columnSchema)) { // Virtual column: use the expression with an alias sqlResult = `(${columnSchema.expression}) AS ${this.printIdentifier(columnName)}`; + } else if (columnSchema.textColumn) { + // JSON column with text column optimization: use the text column with alias + sqlResult = `${this.printIdentifier(columnSchema.textColumn)} AS ${this.printIdentifier( + columnName + )}`; } else { // Regular column: use the actual ClickHouse column name const clickhouseName = columnSchema.clickhouseName ?? columnName; @@ -1438,6 +1476,9 @@ export class ClickHousePrinter { // Look up table schema and get ClickHouse table name const tableSchema = this.lookupTable(tableName); + // Validate that required tenant columns are present in enforcedWhereClause + this.validateRequiredTenantColumns(tableSchema); + // Always add the TSQL table name as an alias if no explicit alias is provided // This ensures table-qualified column references work in WHERE clauses // (needed to avoid alias conflicts when columns have expressions) @@ -1486,8 +1527,8 @@ export class ClickHousePrinter { } } - // Add tenant isolation guard - extraWhere = this.createTenantGuard(tableSchema, effectiveAlias); + // Add enforced WHERE clause guard (tenant isolation + plan limits) + extraWhere = this.createEnforcedGuard(tableSchema, effectiveAlias); } else if ( (tableExpr as SelectQuery).expression_type === "select_query" || (tableExpr as SelectSetQuery).expression_type === "select_set_query" @@ -1534,77 +1575,202 @@ export class ClickHousePrinter { } // ============================================================ - // Tenant Isolation + // Enforced WHERE Clause // ============================================================ /** - * Create a WHERE clause expression for tenant isolation and required filters - * Note: We use just the column name without table prefix since ClickHouse - * requires the actual table name (task_runs_v2), not the TSQL alias (task_runs) + * Validate that required tenant columns are present in enforcedWhereClause. * - * Organization ID is always required. Project ID and Environment ID are optional - - * if not provided, the query will return results across all projects/environments. + * If a table defines `tenantColumns.organizationId`, the `enforcedWhereClause` + * MUST include that column to ensure tenant isolation. This prevents accidental + * data leaks when the caller forgets to include tenant isolation conditions. * - * Required filters from the table schema are also always included. + * @throws QueryError if a required tenant column is missing */ - private createTenantGuard(tableSchema: TableSchema, _tableAlias: string): And | CompareOperation { - const { tenantColumns, requiredFilters } = tableSchema; + private validateRequiredTenantColumns(tableSchema: TableSchema): void { + const { tenantColumns } = tableSchema; + if (!tenantColumns) return; + + // Organization ID is always required if the table defines it + if (tenantColumns.organizationId) { + const orgColumn = tenantColumns.organizationId; + if (!this.context.enforcedWhereClause[orgColumn]) { + throw new QueryError( + `Table '${tableSchema.name}' requires '${orgColumn}' in enforcedWhereClause for tenant isolation` + ); + } + } + // Note: projectId and environmentId are optional - no validation needed + } - // Organization guard is always required - const orgGuard: CompareOperation = { - expression_type: "compare_operation", - op: CompareOperationOp.Eq, - left: { expression_type: "field", chain: [tenantColumns.organizationId] } as Field, - right: { expression_type: "constant", value: this.context.organizationId } as Constant, - }; + /** + * Format a Date as a ClickHouse-compatible DateTime64 string. + * ClickHouse expects format: 'YYYY-MM-DD HH:MM:SS.mmm' (in UTC) + */ + private formatDateForClickHouse(date: Date): string { + const year = date.getUTCFullYear(); + const month = String(date.getUTCMonth() + 1).padStart(2, "0"); + const day = String(date.getUTCDate()).padStart(2, "0"); + const hours = String(date.getUTCHours()).padStart(2, "0"); + const minutes = String(date.getUTCMinutes()).padStart(2, "0"); + const seconds = String(date.getUTCSeconds()).padStart(2, "0"); + const ms = String(date.getUTCMilliseconds()).padStart(3, "0"); + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${ms}`; + } - // Collect all guards - org is always included - const guards: CompareOperation[] = [orgGuard]; + /** + * Create an AST expression for a value. + * Date values are wrapped in toDateTime64() for ClickHouse compatibility. + */ + private createValueExpression(value: Date | string | number): Expression { + if (value instanceof Date) { + // Wrap Date in toDateTime64(formatted_string, 3) for ClickHouse DateTime64(3) columns + return { + expression_type: "call", + name: "toDateTime64", + args: [ + { expression_type: "constant", value: this.formatDateForClickHouse(value) } as Constant, + { expression_type: "constant", value: 3 } as Constant, + ], + } as Call; + } + return { expression_type: "constant", value } as Constant; + } - // Only add project guard if projectId is provided - if (this.context.projectId !== undefined) { - const projectGuard: CompareOperation = { - expression_type: "compare_operation", - op: CompareOperationOp.Eq, - left: { expression_type: "field", chain: [tenantColumns.projectId] } as Field, - right: { expression_type: "constant", value: this.context.projectId } as Constant, - }; - guards.push(projectGuard); + /** + * Map condition operator to CompareOperationOp + */ + private mapConditionOpToCompareOp( + op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" + ): CompareOperationOp { + switch (op) { + case "eq": + return CompareOperationOp.Eq; + case "neq": + return CompareOperationOp.NotEq; + case "gt": + return CompareOperationOp.Gt; + case "gte": + return CompareOperationOp.GtEq; + case "lt": + return CompareOperationOp.Lt; + case "lte": + return CompareOperationOp.LtEq; } + } - // Only add environment guard if environmentId is provided - if (this.context.environmentId !== undefined) { - const envGuard: CompareOperation = { - expression_type: "compare_operation", - op: CompareOperationOp.Eq, - left: { expression_type: "field", chain: [tenantColumns.environmentId] } as Field, - right: { expression_type: "constant", value: this.context.environmentId } as Constant, + /** + * Create an AST expression from a WhereClauseCondition + * + * @param column - The column name + * @param condition - The condition to apply + * @param tableAlias - Optional table alias to qualify the column reference. + * When provided, constructs the field chain as [tableAlias, column] + * so resolveFieldChain will resolve to the correct table in multi-join queries. + * @returns The AST expression for the condition + */ + private createConditionExpression( + column: string, + condition: WhereClauseCondition, + tableAlias?: string + ): Expression { + // When tableAlias is provided, qualify the field chain to ensure it binds + // to the correct table in multi-join queries + const fieldExpr: Field = { + expression_type: "field", + chain: tableAlias ? [tableAlias, column] : [column], + }; + + if (condition.op === "between") { + const betweenExpr: BetweenExpr = { + expression_type: "between_expr", + expr: fieldExpr, + low: this.createValueExpression(condition.low), + high: this.createValueExpression(condition.high), }; - guards.push(envGuard); + return betweenExpr; } - // Add required filters from the table schema + // Simple comparison + const compareExpr: CompareOperation = { + expression_type: "compare_operation", + left: fieldExpr, + right: this.createValueExpression(condition.value), + op: this.mapConditionOpToCompareOp(condition.op), + }; + return compareExpr; + } + + /** + * Create a WHERE clause expression for enforced conditions and required filters. + * + * This method applies: + * 1. All conditions from enforcedWhereClause (tenant isolation + plan limits) + * 2. Required filters from the table schema (e.g., engine = 'V2') + * + * Conditions are applied if the column exists in either: + * - The exposed columns (tableSchema.columns) + * - The tenant columns (tableSchema.tenantColumns) + * + * This ensures the same enforcedWhereClause can be used across different tables. + * + * All guard expressions are qualified with the table alias to ensure they bind + * to the correct table in multi-join queries, preventing potential security + * issues where an unqualified column reference could bind to the wrong table. + */ + private createEnforcedGuard(tableSchema: TableSchema, tableAlias: string): Expression | null { + const { requiredFilters, tenantColumns } = tableSchema; + const guards: Expression[] = []; + + // Build a set of valid columns for this table (exposed + tenant columns) + const validColumns = new Set(Object.keys(tableSchema.columns)); + if (tenantColumns) { + if (tenantColumns.organizationId) validColumns.add(tenantColumns.organizationId); + if (tenantColumns.projectId) validColumns.add(tenantColumns.projectId); + if (tenantColumns.environmentId) validColumns.add(tenantColumns.environmentId); + } + + // Apply all enforced conditions for columns that exist in this table + // Pass tableAlias to ensure guards are qualified and bind to the correct table + for (const [column, condition] of Object.entries(this.context.enforcedWhereClause)) { + // Skip undefined/null conditions (allows conditional inclusion like project_id?: condition) + if (condition === undefined || condition === null) { + continue; + } + // Only apply if column exists in this table's schema or is a tenant column + if (validColumns.has(column)) { + guards.push(this.createConditionExpression(column, condition, tableAlias)); + } + } + + // Add required filters from the table schema (e.g., engine = 'V2') + // Also qualified with table alias to ensure correct binding in multi-join queries if (requiredFilters && requiredFilters.length > 0) { for (const filter of requiredFilters) { const filterGuard: CompareOperation = { expression_type: "compare_operation", op: CompareOperationOp.Eq, - left: { expression_type: "field", chain: [filter.column] } as Field, + left: { expression_type: "field", chain: [tableAlias, filter.column] } as Field, right: { expression_type: "constant", value: filter.value } as Constant, }; guards.push(filterGuard); } } - // If only org guard, return it directly (no need for AND wrapper) + // Return null if no guards (empty enforcedWhereClause and no requiredFilters) + if (guards.length === 0) { + return null; + } + + // If only one guard, return it directly (no need for AND wrapper) if (guards.length === 1) { - return orgGuard; + return guards[0]; } return { expression_type: "and", exprs: guards, - }; + } as And; } // ============================================================ @@ -1711,7 +1877,39 @@ export class ClickHousePrinter { // Transform the right side if it contains user-friendly values const transformedRight = this.transformValueMapExpression(node.right, columnSchema); - const left = this.visit(node.left); + // Check if we should use a text column for bare JSON field comparisons + // This applies to: Eq, NotEq, Like, ILike, NotLike, NotILike + const textColumnOps = [ + CompareOperationOp.Eq, + CompareOperationOp.NotEq, + CompareOperationOp.Like, + CompareOperationOp.ILike, + CompareOperationOp.NotLike, + CompareOperationOp.NotILike, + ]; + const useTextColumn = textColumnOps.includes(node.op); + const leftTextColumn = useTextColumn ? this.getTextColumnForExpression(node.left) : null; + + // Build the left side, qualifying the text column with table alias if present + let left: string; + if (leftTextColumn) { + // Check if the field is qualified with a table alias (e.g., r.output) + // and prepend that alias to the text column to avoid ambiguity in JOINs + const fieldNode = node.left as Field; + if (fieldNode.expression_type === "field" && fieldNode.chain.length >= 2) { + const firstPart = fieldNode.chain[0]; + if (typeof firstPart === "string" && this.tableContexts.has(firstPart)) { + // The field is qualified with a table alias, prepend it to the text column + left = this.printIdentifier(firstPart) + "." + this.printIdentifier(leftTextColumn); + } else { + left = this.printIdentifier(leftTextColumn); + } + } else { + left = this.printIdentifier(leftTextColumn); + } + } else { + left = this.visit(node.left); + } const right = this.visit(transformedRight); switch (node.op) { @@ -2074,19 +2272,31 @@ export class ClickHousePrinter { return `(${virtualExpression})`; } + // Inject dataPrefix for JSON columns if needed (e.g., output.message -> output.data.message) + const chainWithPrefix = this.injectDataPrefix(node.chain); + // Try to resolve column names through table context - const resolvedChain = this.resolveFieldChain(node.chain); + const resolvedChain = this.resolveFieldChain(chainWithPrefix); // Print each chain element let result = resolvedChain.map((part) => this.printIdentifierOrIndex(part)).join("."); // For JSON column subfield access (e.g., error.data.name), add .:String type hint - // This is required because ClickHouse's Dynamic/Variant types are not allowed in - // GROUP BY without type casting, and SELECT/GROUP BY expressions must match + // This is ONLY required when the query has GROUP BY, because: + // 1. ClickHouse's Dynamic/Variant types are not allowed in GROUP BY without type casting + // 2. SELECT/GROUP BY expressions must match + // For queries without GROUP BY, the .:String type hint actually breaks the query + // (returns NULL instead of the actual value) + // We also skip this in WHERE comparisons where it breaks the query if (resolvedChain.length > 1) { // Check if the root column (first part) is a JSON column const rootColumnSchema = this.resolveFieldToColumnSchema([node.chain[0]]); - if (rootColumnSchema?.type === "JSON") { + // Add .:String ONLY for GROUP BY queries, and NOT in WHERE comparisons + if ( + rootColumnSchema?.type === "JSON" && + this.queryHasGroupBy && + !this.isInWhereComparisonContext() + ) { // Add .:String type hint for JSON subfield access result = `${result}.:String`; } @@ -2114,6 +2324,20 @@ export class ClickHousePrinter { return false; } + /** + * Check if we're inside a WHERE/HAVING comparison operation. + * Unlike isInComparisonContext(), this does NOT include GROUP BY context. + * Used to skip .:String type hints in WHERE clauses where they break queries. + */ + private isInWhereComparisonContext(): boolean { + for (const node of this.stack) { + if ((node as CompareOperation).expression_type === "compare_operation") { + return true; + } + } + return false; + } + /** * Resolve field chain with table alias prefix to avoid alias conflicts. * This is used in WHERE clauses when a column has whereTransform to ensure @@ -2155,6 +2379,125 @@ export class ClickHousePrinter { return rootColumnSchema?.type === "JSON"; } + /** + * Check if a field should use a text column instead of the JSON column. + * Returns the text column name if the field is a bare JSON field with textColumn defined, + * or null if the original column should be used. + * + * A "bare" JSON field means selecting the entire column (e.g., SELECT output) + * rather than accessing a subfield (e.g., SELECT output.data.name). + */ + private getTextColumnForField(chain: Array): string | null { + if (chain.length === 0) return null; + + const firstPart = chain[0]; + if (typeof firstPart !== "string") return null; + + let columnSchema: ColumnSchema | null = null; + + if (chain.length === 1) { + // Unqualified: just column name + columnSchema = this.resolveFieldToColumnSchema(chain); + } else if (chain.length === 2) { + // Could be table.column (qualified) - check if first part is a table alias + const tableSchema = this.tableContexts.get(firstPart); + if (tableSchema) { + const columnName = chain[1]; + if (typeof columnName === "string") { + columnSchema = tableSchema.columns[columnName] || null; + } + } + // If not a table alias, it's JSON path access (e.g., output.data) - return null + } + // chain.length > 2 means JSON path access - return null + + return columnSchema?.textColumn ?? null; + } + + /** + * Get the text column for an expression if it's a bare JSON field. + * Returns null if the expression is not a field or doesn't have a textColumn. + */ + private getTextColumnForExpression(expr: Expression): string | null { + if ((expr as Field).expression_type !== "field") return null; + return this.getTextColumnForField((expr as Field).chain); + } + + /** + * Get the dataPrefix for a field chain if the root column has one defined. + * Returns null if the column doesn't have a dataPrefix or if this isn't a subfield access. + */ + private getDataPrefixForField(chain: Array): string | null { + if (chain.length < 2) return null; // Need at least column.subfield + + const firstPart = chain[0]; + if (typeof firstPart !== "string") return null; + + // Check if first part is a table alias (table.column.subfield) + const tableSchema = this.tableContexts.get(firstPart); + if (tableSchema) { + // Qualified: table.column.subfield - need at least 3 parts + if (chain.length < 3) return null; + const columnName = chain[1]; + if (typeof columnName !== "string") return null; + const columnSchema = tableSchema.columns[columnName]; + return columnSchema?.dataPrefix ?? null; + } + + // Unqualified: column.subfield + const columnSchema = this.resolveFieldToColumnSchema([firstPart]); + return columnSchema?.dataPrefix ?? null; + } + + /** + * Inject dataPrefix into a field chain if the root column has one defined. + * e.g., [output, message] -> [output, data, message] when dataPrefix is "data" + * Returns the original chain if no dataPrefix applies. + */ + private injectDataPrefix(chain: Array): Array { + const dataPrefix = this.getDataPrefixForField(chain); + if (!dataPrefix) return chain; + + const firstPart = chain[0]; + if (typeof firstPart !== "string") return chain; + + // Check if first part is a table alias + const tableSchema = this.tableContexts.get(firstPart); + if (tableSchema) { + // Qualified: table.column.subfield -> table.column.dataPrefix.subfield + // [table, column, subfield] -> [table, column, dataPrefix, subfield] + return [chain[0], chain[1], dataPrefix, ...chain.slice(2)]; + } + + // Unqualified: column.subfield -> column.dataPrefix.subfield + // [column, subfield] -> [column, dataPrefix, subfield] + return [chain[0], dataPrefix, ...chain.slice(1)]; + } + + /** + * Build an alias name for a field chain, excluding the dataPrefix if present. + * e.g., [output, message] with dataPrefix "data" -> "output_message" + * This gives users clean column names without the internal data wrapper. + */ + private buildAliasWithoutDataPrefix( + chain: Array, + dataPrefix: string | null + ): string { + // Filter to just string parts and join with underscores + const parts = chain.filter((p): p is string => typeof p === "string"); + + if (dataPrefix) { + // Remove the dataPrefix from the parts (it's an implementation detail) + const prefixIndex = parts.indexOf(dataPrefix); + if (prefixIndex > 0) { + // Only remove if it's not the first element (column name) + parts.splice(prefixIndex, 1); + } + } + + return parts.join("_"); + } + /** * Resolve a field chain to its column schema (if it references a known column) */ @@ -2380,6 +2723,31 @@ export class ClickHousePrinter { return columnSchema.clickhouseName || columnSchema.name; } + // Check if this is a tenant column that's not exposed in the schema's columns + // These are internal columns used for tenant isolation guards + const { tenantColumns, requiredFilters } = tableSchema; + if (tenantColumns) { + if ( + columnName === tenantColumns.organizationId || + columnName === tenantColumns.projectId || + columnName === tenantColumns.environmentId + ) { + // Tenant columns are already ClickHouse column names, return as-is + return columnName; + } + } + + // Check if this is a required filter column (e.g., engine = 'V2') + // These are internal columns used for enforced filters + if (requiredFilters) { + for (const filter of requiredFilters) { + if (columnName === filter.column) { + // Required filter columns are already ClickHouse column names, return as-is + return columnName; + } + } + } + // Column not in schema - this is a security issue, block access // Check if the user typed a ClickHouse column name instead of the TSQL name for (const [tsqlName, colSchema] of Object.entries(tableSchema.columns)) { diff --git a/internal-packages/tsql/src/query/printer_context.ts b/internal-packages/tsql/src/query/printer_context.ts index 15089e01f7c..2956e660d0c 100644 --- a/internal-packages/tsql/src/query/printer_context.ts +++ b/internal-packages/tsql/src/query/printer_context.ts @@ -18,6 +18,34 @@ export interface QuerySettings { timeoutSeconds?: number; } +/** + * A simple comparison condition (e.g., column > value) + */ +export interface SimpleComparisonCondition { + /** The comparison operator */ + op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte"; + /** The value to compare against */ + value: Date | string | number; +} + +/** + * A between condition (e.g., column BETWEEN low AND high) + */ +export interface BetweenCondition { + /** The between operator */ + op: "between"; + /** The low bound of the range */ + low: Date | string | number; + /** The high bound of the range */ + high: Date | string | number; +} + +/** + * A WHERE clause condition that can be either a simple comparison or a BETWEEN. + * Used for both enforcedWhereClause (always applied) and whereClauseFallback (default when user doesn't filter). + */ +export type WhereClauseCondition = SimpleComparisonCondition | BetweenCondition; + /** * Default query settings */ @@ -42,7 +70,7 @@ export interface QueryNotice { * Context for the TSQL to ClickHouse printer * * Holds: - * - Tenant IDs for automatic WHERE clause injection + * - Enforced WHERE conditions for tenant isolation and plan limits * - Schema registry for table/column validation * - Parameter accumulator for SQL injection safety * - Query settings and execution options @@ -64,23 +92,30 @@ export class PrinterContext { /** Runtime field mappings for dynamic value translation */ readonly fieldMappings: FieldMappings; + /** + * Enforced WHERE conditions that are ALWAYS applied at the table level. + * Used for tenant isolation (org_id, project_id, env_id) and plan-based limits. + * Applied to every table reference including subqueries, CTEs, and JOINs. + */ + readonly enforcedWhereClause: Record; + constructor( - /** The organization ID for tenant isolation (required) */ - public readonly organizationId: string, - /** The project ID for tenant isolation (optional - omit to query across all projects) */ - public readonly projectId: string | undefined, - /** The environment ID for tenant isolation (optional - omit to query across all environments) */ - public readonly environmentId: string | undefined, /** Schema registry containing allowed tables and columns */ public readonly schema: SchemaRegistry, /** Query execution settings */ public readonly settings: QuerySettings = {}, /** Runtime field mappings for dynamic value translation */ - fieldMappings: FieldMappings = {} + fieldMappings: FieldMappings = {}, + /** + * Enforced WHERE conditions that are ALWAYS applied at the table level. + * Must include tenant columns (e.g., organization_id) for multi-tenant tables. + */ + enforcedWhereClause: Record = {} ) { // Initialize with default settings this.settings = { ...DEFAULT_QUERY_SETTINGS, ...settings }; this.fieldMappings = fieldMappings; + this.enforcedWhereClause = enforcedWhereClause; } /** @@ -157,12 +192,10 @@ export class PrinterContext { */ createChildContext(): PrinterContext { const child = new PrinterContext( - this.organizationId, - this.projectId, - this.environmentId, this.schema, this.settings, - this.fieldMappings + this.fieldMappings, + this.enforcedWhereClause ); // Share the same values map so parameters are unified child.values = this.values; @@ -184,19 +217,30 @@ export class PrinterContext { * Options for creating a printer context */ export interface PrinterContextOptions { - /** The organization ID for tenant isolation (required) */ - organizationId: string; - /** The project ID for tenant isolation (optional - omit to query across all projects) */ - projectId?: string; - /** The environment ID for tenant isolation (optional - omit to query across all environments) */ - environmentId?: string; + /** Schema registry containing allowed tables and columns */ schema: SchemaRegistry; + /** Query execution settings */ settings?: QuerySettings; /** * Runtime field mappings for dynamic value translation. * Maps internal ClickHouse values to external user-facing values. */ fieldMappings?: FieldMappings; + /** + * REQUIRED: Conditions always applied at the table level. + * Must include tenant columns (e.g., organization_id) for multi-tenant tables. + * Applied to every table reference including subqueries, CTEs, and JOINs. + * + * @example + * ```typescript + * { + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) } + * } + * ``` + */ + enforcedWhereClause: Record; } /** @@ -204,12 +248,10 @@ export interface PrinterContextOptions { */ export function createPrinterContext(options: PrinterContextOptions): PrinterContext { return new PrinterContext( - options.organizationId, - options.projectId, - options.environmentId, options.schema, options.settings, - options.fieldMappings + options.fieldMappings, + options.enforcedWhereClause ); } diff --git a/internal-packages/tsql/src/query/schema.ts b/internal-packages/tsql/src/query/schema.ts index a5153597add..21bc9ec955f 100644 --- a/internal-packages/tsql/src/query/schema.ts +++ b/internal-packages/tsql/src/query/schema.ts @@ -214,6 +214,42 @@ export interface ColumnSchema { * ``` */ nullValue?: string; + /** + * Alternative text column to use when selecting or comparing the full JSON value. + * + * For JSON columns, this allows using a pre-materialized string column + * which is more efficient than reading from the JSON column directly. + * + * @example + * ```typescript + * { + * name: "output", + * type: "JSON", + * textColumn: "output_text", + * } + * ``` + */ + textColumn?: string; + /** + * Prefix path for JSON column data access. + * + * When set, user paths like `output.message` are automatically transformed + * to `output.data.message` in the actual query, and result aliases exclude + * the prefix (e.g., `output_message` instead of `output_data_message`). + * + * This is useful when JSON data is stored wrapped in a container object + * (e.g., `{"data": actualData}`) to handle arrays and primitives. + * + * @example + * ```typescript + * { + * name: "output", + * type: "JSON", + * dataPrefix: "data", // output.message → output.data.message + * } + * ``` + */ + dataPrefix?: string; } /** diff --git a/internal-packages/tsql/src/query/security.test.ts b/internal-packages/tsql/src/query/security.test.ts index f576c5e93ef..2fcca477770 100644 --- a/internal-packages/tsql/src/query/security.test.ts +++ b/internal-packages/tsql/src/query/security.test.ts @@ -53,10 +53,12 @@ const taskEventsSchema: TableSchema = { }; const defaultOptions: CompileTSQLOptions = { - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", tableSchema: [taskRunsSchema, taskEventsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, }; function compile(query: string, options: Partial = {}) { @@ -412,8 +414,10 @@ describe("Optional Tenant Filters", () => { describe("Organization ID is always required", () => { it("should always inject organization guard even with optional project/env", () => { const { sql, params } = compile("SELECT * FROM task_runs", { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, }); const whereClause = getWhereClause(sql); @@ -431,8 +435,11 @@ describe("Optional Tenant Filters", () => { describe("Project ID is optional", () => { it("should inject org and project guards when project is provided", () => { const { sql, params } = compile("SELECT * FROM task_runs", { - projectId: "proj_tenant1", - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + // environment_id omitted + }, }); const whereClause = getWhereClause(sql); @@ -449,8 +456,10 @@ describe("Optional Tenant Filters", () => { it("should allow querying across all projects when projectId is omitted", () => { const { sql, params } = compile("SELECT * FROM task_runs", { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, }); const whereClause = getWhereClause(sql); @@ -479,8 +488,11 @@ describe("Optional Tenant Filters", () => { it("should allow querying across all environments when environmentId is omitted", () => { const { sql, params } = compile("SELECT * FROM task_runs", { - projectId: "proj_tenant1", - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + // environment_id omitted + }, }); const whereClause = getWhereClause(sql); @@ -501,8 +513,10 @@ describe("Optional Tenant Filters", () => { const { sql, params } = compile( "SELECT * FROM task_runs WHERE organization_id = 'org_other'", { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, } ); @@ -518,8 +532,10 @@ describe("Optional Tenant Filters", () => { JOIN task_events e ON r.id = e.run_id `, { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, } ); @@ -540,8 +556,10 @@ describe("Optional Tenant Filters", () => { SELECT id, status FROM task_runs WHERE status = 'failed' `, { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, } ); @@ -557,8 +575,10 @@ describe("Optional Tenant Filters", () => { WHERE id IN (SELECT run_id FROM task_events) `, { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, } ); @@ -570,6 +590,95 @@ describe("Optional Tenant Filters", () => { }); }); +describe("Multi-join Tenant Guard Qualification", () => { + /** + * Security Test: Verifies that tenant guards are properly table-qualified in multi-join queries. + * + * The bug: createEnforcedGuard was building unqualified guard expressions like: + * organization_id = 'org_tenant1' + * + * In a multi-table join where both tables have the same column (organization_id), + * an unqualified reference could potentially bind to the wrong table during resolution, + * or be ambiguous. The guards should be qualified like: + * r.organization_id = 'org_tenant1' AND e.organization_id = 'org_tenant1' + * + * This ensures each table's guard binds to the correct table, not just any matching column. + */ + it("should qualify tenant guards with table alias in JOIN queries", () => { + const { sql } = compile(` + SELECT r.id, e.event_type + FROM task_runs r + JOIN task_events e ON r.id = e.run_id + `); + + // The guards should be table-qualified to prevent binding to the wrong table + // Look for pattern like: r.organization_id and e.organization_id (with table alias prefix) + // The exact format in ClickHouse SQL is just "alias.column" after resolution + + // Count qualified organization_id references (should have table prefixes) + // In the WHERE clause, we should see both r.organization_id and e.organization_id + const whereClause = sql.substring(sql.indexOf("WHERE")); + + // Both tables should have their own qualified tenant guards + // The pattern should be: table_alias.organization_id for each table + expect(whereClause).toMatch(/\br\b[^,]*organization_id/); + expect(whereClause).toMatch(/\be\b[^,]*organization_id/); + }); + + it("should qualify tenant guards with table alias in LEFT JOIN queries", () => { + const { sql } = compile(` + SELECT r.id, e.event_type + FROM task_runs r + LEFT JOIN task_events e ON r.id = e.run_id + `); + + const whereClause = sql.substring(sql.indexOf("WHERE")); + + // Both tables should have qualified guards + expect(whereClause).toMatch(/\br\b[^,]*organization_id/); + expect(whereClause).toMatch(/\be\b[^,]*organization_id/); + }); + + it("should qualify tenant guards in multi-way JOIN queries", () => { + const { sql } = compile(` + SELECT r.id, e1.event_type, e2.event_type + FROM task_runs r + JOIN task_events e1 ON r.id = e1.run_id + JOIN task_events e2 ON r.id = e2.run_id + `); + + const whereClause = sql.substring(sql.indexOf("WHERE")); + + // All three table aliases should have qualified guards + expect(whereClause).toMatch(/\br\b[^,]*organization_id/); + expect(whereClause).toMatch(/\be1\b[^,]*organization_id/); + expect(whereClause).toMatch(/\be2\b[^,]*organization_id/); + }); + + it("should ensure guards cannot bind to wrong table by verifying separate qualifications", () => { + const { sql, params } = compile(` + SELECT r.id, e.event_type + FROM task_runs r + JOIN task_events e ON r.id = e.run_id + WHERE r.status = 'completed' + `); + + // Count organization_id occurrences with different table prefixes + // This ensures each table gets its own guard, not shared/ambiguous references + const orgIdPattern = /(\w+)\.organization_id/g; + const matches = [...sql.matchAll(orgIdPattern)]; + const tableAliases = matches.map(m => m[1]); + + // Should have at least 2 different table aliases for organization_id + // (one for task_runs alias 'r' and one for task_events alias 'e') + expect(tableAliases).toContain("r"); + expect(tableAliases).toContain("e"); + + // Both should use the same tenant value (parameterized) + expect(Object.values(params)).toContain("org_tenant1"); + }); +}); + describe("Edge Cases", () => { it("should handle empty string values", () => { const { params } = compile("SELECT * FROM task_runs WHERE status = ''"); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d73bef99fe8..525beac4790 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -497,8 +497,8 @@ importers: specifier: workspace:* version: link:../../internal-packages/otlp-importer '@trigger.dev/platform': - specifier: 1.0.21 - version: 1.0.21 + specifier: 1.0.22 + version: 1.0.22 '@trigger.dev/redis-worker': specifier: workspace:* version: link:../../packages/redis-worker @@ -10301,8 +10301,8 @@ packages: react: ^18.2.0 react-dom: 18.2.0 - '@trigger.dev/platform@1.0.21': - resolution: {integrity: sha512-D1p+Y5pj21Un8hhN7oS/X7c+mhHKL58w1nwI9XYxbKUK1cNIIVhEMNZ0IyYmYuLelSARUXYePlKSl0v4hlusZg==} + '@trigger.dev/platform@1.0.22': + resolution: {integrity: sha512-tvPf40wqEDcQCZsHt/9A+WoQ08z+uObSWQ+oahqCgp3dSgKOUH8NdzZ/2ISSRiCkN2jURixNiUyDJmgsZipExg==} '@types/acorn@4.0.6': resolution: {integrity: sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ==} @@ -30255,7 +30255,7 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@trigger.dev/platform@1.0.21': + '@trigger.dev/platform@1.0.22': dependencies: zod: 3.23.8 From 9e087127495bbc3b5fa4730f89f5889ae1a26ba5 Mon Sep 17 00:00:00 2001 From: DKP <8297864+D-K-P@users.noreply.github.com> Date: Thu, 29 Jan 2026 13:55:43 +0000 Subject: [PATCH 009/400] Add building with ai/skills pages and updated intro (#2962) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes # ## ✅ Checklist - [ ] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [ ] The PR title follows the convention. - [ ] I ran and tested the code works --- ## Testing _[Describe the steps you took to test this change]_ --- ## Changelog _[Short description of what has changed]_ --- ## Screenshots _[Screenshots]_ 💯 ---

--- docs/building-with-ai.mdx | 24 ++ docs/docs.json | 15 +- docs/images/intro-ai.jpg | Bin 0 -> 5495 bytes docs/introduction.mdx | 4 +- docs/mcp-agent-rules.mdx | 1 - docs/mcp-introduction.mdx | 399 +++++++++++++++++------- docs/mcp-tools.mdx | 532 +++++--------------------------- docs/quick-start.mdx | 28 +- docs/skills.mdx | 83 +++++ docs/snippets/step-cli-init.mdx | 15 +- 10 files changed, 498 insertions(+), 603 deletions(-) create mode 100644 docs/building-with-ai.mdx create mode 100644 docs/images/intro-ai.jpg create mode 100644 docs/skills.mdx diff --git a/docs/building-with-ai.mdx b/docs/building-with-ai.mdx new file mode 100644 index 00000000000..ba8cd5bb47c --- /dev/null +++ b/docs/building-with-ai.mdx @@ -0,0 +1,24 @@ +--- +title: "Overview" +sidebarTitle: "Overview" +description: "Tools and resources for building Trigger.dev projects with AI coding assistants." +--- + +We provide tools to help you build Trigger.dev projects with AI coding assistants. We recommend using them for the best developer experience. + + + + Give your AI assistant direct access to Trigger.dev tools - search docs, trigger tasks, deploy projects, and monitor runs. + + ```bash + npx trigger.dev@latest install-mcp + ``` + + + Portable instruction sets that teach any AI coding assistant Trigger.dev best practices for writing tasks, configs, and more. + + ```bash + npx skills add triggerdotdev/skills + ``` + + diff --git a/docs/docs.json b/docs/docs.json index cff62e6d095..405f4eb0a2c 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -43,6 +43,17 @@ "apikeys" ] }, + { + "group": "Building with AI", + "pages": [ + "building-with-ai", + { + "group": "MCP Server", + "pages": ["mcp-introduction", "mcp-tools", "mcp-agent-rules"] + }, + "skills" + ] + }, { "group": "Writing tasks", "pages": [ @@ -166,10 +177,6 @@ } ] }, - { - "group": "MCP Server", - "pages": ["mcp-introduction", "mcp-tools", "mcp-agent-rules"] - }, { "group": "Using the Dashboard", "pages": ["run-tests", "troubleshooting-alerts", "replaying", "bulk-actions"] diff --git a/docs/images/intro-ai.jpg b/docs/images/intro-ai.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9905922a22b3b586e618754f9d975831b7858f57 GIT binary patch literal 5495 zcmbuDbwCtRx4>tY1(s%kMY=;uT1q7Z0TG0SC8ZlA1?i9y7NtQ_y1N@`6c9xbBvlZQ z5@8Vug*R)w?|py2voW*xo;mm2bI+Xp-8p}M{t=+Oc|-jM0D%Ai1bhMK^FRau`y=@H zFan$q!3hcBL}bJ`A)_IqAOqhtRJ2qG@WV!b;Q~DyH!BM(w-6tnkj!Nn4GkN}|JT5I zKY)Nk=pY~QARGV`0l`B+&IbWW&`k&c#RJ{^ClDwBAs!5l4|YHRJlrh)2^4|{!zUm- zp94t23!s#Elwi-%l&9Js1xVg&iADv04;fveVnzDzBJfBeSyid+I;Aban4gEIKmd>u z@Qd?v>2r&-^@6?7L}eWYdI!4=lPvGs)Xoc+rx|t0nxp2DHEr|Lvf3SNOYH218Li3c zA0}Co6-Uixyw%>joW_`u;rOTw!vR_##&x?yHM$|9WSUy5bb#awwRPyMtw^5spzaNmVB5_(z%E+${l6*BxUhQb;b zFI2Div~!2`*$SvTTa9|k7x_PIeWtR1Vo)7$n%869E;m-#T~qxF^~G%Z^n+9qg-Pgg zx4{Y)%aVQ=6!PjS-IXPUNDZQ{O*%xXAX_&16*$boLHK&I@yiBUl9Ihni-{o)aotO-;rU#HPIrEFb z`=2|lYxjnYFwNS8kudpL(lu<7y#74Lhaa9)(^zR?D5M?`xJ3TyfOFrk)Vk*!c=_|g zo2zj^7v8&1U3=3#37^bS?!wt$y;_3g_7;yCBPg9^SH`1zBd2}nR}lFl5=umbEVMTS z+rpET4vU+FwwW@Qr0&wbHqmjj682*i-+WR&bP&RQEY%oXSh`TSd2+qBV`kKsSI75E zd~$WcWN61XCtg3tScy2HKn#vEOF{Ze*Z9;d&M{Vj2-(cmGhE%Z&-p+ znMGsL*hsVjTRpuc9UW~82_a8aQ>@^Uly^!Crij+CSi@LI-fbf!S5fav*x#Hd%1BZpjldw^T z5>yE&9SD^R@n9=0O!ZKsMQFA>r$RJCnM}y*XFNedKbR?A9%FOd(o(^0{P)L<3>P+v*>x=z3K@mpx0F-_TGSuq2KW6@T*;X#Qe4_JeD-7upwSv~lzLALOUd12UCgRK5> zvoN*8_{$gLO=)^62yid}aID($;V(kw=M~?mAbzZe#8;pLM5suSGE%V@#la!|NJ9c7 zZ;_0otXhHKiwH^pVTkFYD;p;jEc&PL=SUC=0KNa32*H|kg%W^3@n8gSd>91w+Zy;2 zfFbay*zyS|MMP1agzRDpx|R>>sW}vNJZO{>C&iJR+67i&{a|T=gB1%R5A3F3ROFW@ zWK|?*&>qtQ$hc}fOh_N|x~b$$moBh-b4LJM(R~_-A^?|XiLczWIQYH<#(5JnH^}PMWJBEjjDE z7al_I#ST1>8T!=xYr3FmX6X{*8^Z{8uxg%%Co3 zWcTo|Q{tF7NVc9k>2hI&J(7t7oRh(GmB0x9yBm=~%~)oyU9aB9>)(Q!XWDcbPMz0e zE9;AN(URWIMJPU+IZnUTYl~|M&-R7WSHNa(18X5ZN!c$>#-=0L#jcl=3R7-r+`1`S zQbOnPgy2AkoBxc|0L_sFbGuS;IHH;GzP6>uIx4dgId|a0mf=kDog4adzQT03dN3=` z#=0m2)@?hc6+yDU3oAD_iJdX^8jPDuFbGE@aNBwtwD!$U4jYj0hu zYnZRDZhpL>%@%(>tW9SQ4Bgb_0&@P3_rT!Ql`vB$oCDf#NF)m~^T`9z`>j~}%A$6P za7XnCF7I1n+BJ5s4!jTCV??bwTEc>5f)uf*&-Pd!1@Um){&h{!sq>^ZUfF~8i<`fG zdLZ$Hd=Vlt=EBiVme3lJ!fC<< z*0Px*V-xnmkC*ezGqo!W)r$!iRljfq+&+l9TPA_^ER(;HAn>!IUQ$X@cH_RC?_&%q(p4#nOMvX~W&#jIvfv(&y$8?EV zuKrs3jy(fA+{+qw3yJ-N>A1DnDQsdA>}))3wks>n*QBI=rwS&9Mvr!sJU?0F z`6e@>PTd#FVx}jsmJz5I^!kI*M0y<8{doTF&h$Zc+_G^(xmQbgmadC!3#}t(|KuWh zeD!e$-)G5-;miCLXViuXU*^WFHnKDQZSU|EI*gV|C)96u3Zk@B(lq&eDbmmM{3rO@ z`P2`4Xr>z73L_nf=?BjN)ily~XyLtL8egQJe19eNFUEONz0q6Z61jQj00VY&f_HmN zY1~~(=HeEyTy0$8k=!G-Io(6+;#rwiIwKlxLSHPaVRmkIlbP_#@wU^H;?>O8K@Z1hK)t5_SNhsn)JHJUJur!C31X|bQ;fAm%9MJ zYJW8ConuezbHaL8)kze7ujSc`AvH=Obr=JJiovLe)*DYhxl-X*99Sgfcc_?oIxc3M zO7fppN9vwcoGp*N*&?1snb_UEyxA)8rLFMjgh{`++vfd?s_kV)9VA(NhW%^hb3nb} zMOOx+j4NxwlD66zzEy~pM4W%&_h+Swzp6+wzE?_nubUaCZN+-n`=_Y$PHoAt6Ya3* z?nG}B3~W>>;p$~i=I~XDw^HeTS+Z|@t5dV8uQg7G9}r)WmUyL+Y8}qWm47l+gsd%# zN(wm<*{Zp=yx=_~#L%sb6zksbqt!e+X$s8h?7>yzRF-$xmLhp zJL{wW_9b5{r9;t0?eI174i%4daMW6tw%PW1b5YE;Y(eIMU7%6RPL_x!B@5h~{|t5S z-5YSFMw(YwA;-2xNLJhSY0r{M=!j{%lPK4dYYb?#kXQ9oY1L3pzVTIiQ|%;ba*Czn zXKCB{d^f46QbK_|yfU(T)95V6g6+@Aw}UXrSc<+S&YMjPHq;*b?|;sG70r*q^3Vmvq$O-ouuo6%r{+a+ zoxx`u9l2v|QhA($n%W4DlCiZ%h++n>SYypudOq)#v4}ubrs7pDhS!izq@;l{u4*r3 zgGF(x%KNS}5*KpA*Yh4*Jkw7c+CntNKDvAk;;+f8)iic?>9jWdT-Vu1O>(4JxtZ8~ zld>2|RZOKW$+gRCc?Z73f)4OO#C3pZ!EqeNFfxz698jq0a^EG3U*Galu~yWZCvPZ> z_3_E9Z|_7^3mBEqc}b5K5IZGn-_faPp27E!>e{cIqwq6#2^?Xz>evWtQ{-q8)zphC zt1_?%$KtHZuHZWD4XgA8n2kFME74ZFykXscgO z;Jwz$k5iRD#77Sf@)xomfsh4G1#=9qO^DZQ_7-lceer%;AS&Izd0L=B3rC!(a|UYy zAIhuceo^@AV{xG%^p~5E{kVy-DydKr(HE#=-Sgi40MalU0^Djt!5#Yl?!+Mg0!9VJ z#{*fIP2o53LLm?+@Ux=Q)VreM9Kc8Gcf3QRovYEop-40uUDJ$4tHm4v_yChrMMXuW z!DBQB?>I;66M2w%80Fc zk6$4VDV_#idzByfC|EQe?6ZU`vBH%oJx*O)XL-!@Z#VdP(hAUxtk4aM;H*(9EBoKg zbr^kSFmT8ui3aoLk@Yv_Fb)n*pRMkmVDnl(RnAEcP6Prd!! zR;N~{b3lpnzGr9W+^%zHYtkGT4l7Qd0GF0|@NK$SQM=;vR~kO-`1rKU;9Adtw6DA0 z!B~qS04TWEz{e-V|NGbjw-G@AREqf2?4k;k9F`u5G@LppEBAacaV5_O^?##}Cl8!b z8CYX>_Lf`+vpU_M`&U#Ky@AsN)RE9$W^$jDD~uHr)n|o@r~^>-As3tSu9)ShANH2k zZh!})lBK}cQJb-Yx#=Nqn)tK}X#zh?@sE5TjL~u-tfyS>-~SdTF5~vioo>NJMvaLZ zs(AKI+B8SMvB_u{)iPzSgJJjqk3(I5!DRS!l$cPov+=o10ZC%y95P8y(4?-H#CbsX z?W-mpB8KeN-5!ic!gU%o!LoeSgaL!zc*?GNvw=0jTcCF+_+_?OMAQv^)O9`>BA>41 zxn$_0VbX!qM7Pth!6aApcHR=>(dTTAVua4L!{cFt{kHP-Waj`=U1FJ@3YsaDmbs}} zz8fkZ!xZG9LMDM7+@V(yO35n!Hkj4DwAWyE?CKJ7sn7v6@1u9fIuvJl+?%NqRgENJ z6pS4`yP7cBi@i0ivw6GMssW|UZfUOnUAnx&%}+2X2g3E;uF_stM#@4=D)Za0f0<+7 zCk8vtP1j+ywF)n`?o{FZID0Y;vwc|8J)e|~AXT)caw_H2SfV<4)ZpG{Rnh=cTjdkE zq@Fubmz4^Mb6%IdS`_naeQ-qt4x_x0HYBS&oL`98dO8~-J^RwgpHj?*z9IIK8+l#B zE4AH|fW0uUDbfJ5$y78}g~CJ7*If579?TUM1OS2k*Ml=W2o%f}1xhv%?caF;-m*~O zpeD84{C<~!3)kI*@;)2G&t5eflUKMq1Rja0_Ki@UTuLNfzaeH-Koa@(7Uv?LK@_D} zW#p9ATxOwfbMb?rwZY{TTwf z!;ec{JSdC++$#SO2owR}u~R8fit1QWbBZatdnlo-9_0UZ4NTc1-idE3)YKgsI%U{A z1-6ghT0#d1uHw_@`emx(v9Ub7CuRK|S@C=aBc)v?Y9JKDp{mor+Q+7_=`gR*DCgbO zaQk(^xFMD&vV^@`NFnrw3Y2Xo_tQBbjCFOHw}V=s*M1gm#)#e8^EzlpdgP!(#jG4| z^*-&4z5Np35kDb$KKzbFrGA>E27@x4c4zm6J0Vv2^c_#B(2&(+#hnK-cU&_+>Fe+& pR?|-U6Zfc1%39BdQ%Q^;R_0$B$XK$yLq6POO!wl&QT+Mr{{UzMte*e? literal 0 HcmV?d00001 diff --git a/docs/introduction.mdx b/docs/introduction.mdx index fa57a458608..383040de145 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -16,8 +16,8 @@ mode: "center" > Browse our wide range of guides, frameworks and example projects - - Learn how to install and configure the Trigger.dev MCP Server + + Learn how to build Trigger.dev projects using AI coding assistants Watch an end-to-end demo of Trigger.dev in 10 minutes diff --git a/docs/mcp-agent-rules.mdx b/docs/mcp-agent-rules.mdx index 664d8bcf299..321f312a842 100644 --- a/docs/mcp-agent-rules.mdx +++ b/docs/mcp-agent-rules.mdx @@ -2,7 +2,6 @@ title: "Agent rules" sidebarTitle: "Agent rules" description: "Learn how to use the Trigger.dev agent rules with the MCP server" -tag: "new" --- ## What are Trigger.dev agent rules? diff --git a/docs/mcp-introduction.mdx b/docs/mcp-introduction.mdx index d9dc3474e17..257522d5792 100644 --- a/docs/mcp-introduction.mdx +++ b/docs/mcp-introduction.mdx @@ -2,7 +2,6 @@ title: "MCP Introduction" sidebarTitle: "Introduction" description: "Learn how to install and configure the Trigger.dev MCP Server" -tag: "new" --- ## What is the Trigger.dev MCP Server? @@ -18,44 +17,306 @@ The Trigger.dev MCP (Model Context Protocol) Server enables AI assistants to int ## Installation -### Automatic Installation (Recommended) - -The easiest way to install the Trigger.dev MCP Server is using the interactive installation wizard: +The quickest way to get set up is the interactive installer: ```bash npx trigger.dev@latest install-mcp ``` -This command will guide you through: +It will detect your installed clients and configure them automatically. You can also copy-paste the config for your client below. -1. Selecting which MCP clients to configure -2. Choosing installation scope (user, project, or local) -3. Automatically configuring the selected clients +## Client Configuration -## Command Line Options +Each client has a slightly different config format. Copy the snippet for your client into the appropriate file. -The `install-mcp` command supports the following options: + + + Install using the command line: -### Core Options + ```bash + npx trigger.dev@latest install-mcp --client claude-code + ``` -- `-p, --project-ref ` - Scope the MCP server to a specific Trigger.dev project by providing its project ref -- `-t, --tag ` - The version of the trigger.dev CLI package to use for the MCP server (default: latest or v4-beta) -- `--dev-only` - Restrict the MCP server to the dev environment only -- `--yolo` - Install the MCP server into all supported clients automatically -- `--scope ` - Choose the scope of the MCP server: `user`, `project`, or `local` -- `--client ` - Choose specific client(s) to install into + Or add this configuration to `~/.claude.json` (user) or `.mcp.json` (project): -### Configuration Options + ```json + { + "mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Claude Code MCP docs ↗](https://code.claude.com/docs/en/mcp) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client cursor + ``` + + Or add this configuration to `~/.cursor/mcp.json` (user) or `.cursor/mcp.json` (project): + + ```json + { + "mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Cursor MCP docs ↗](https://cursor.com/docs/context/mcp) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client windsurf + ``` + + Or add this configuration to `~/.codeium/windsurf/mcp_config.json`: + + ```json + { + "mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Windsurf MCP docs ↗](https://docs.windsurf.com/windsurf/cascade/mcp) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client vscode + ``` + + Or add this configuration to `.vscode/mcp.json` (project) or `~/Library/Application Support/Code/User/mcp.json` (user, macOS): + + ```json + { + "servers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + VS Code uses `servers` instead of `mcpServers`. + + [View VS Code MCP docs ↗](https://code.visualstudio.com/docs/copilot/chat/mcp-servers) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client zed + ``` + + Or add this configuration to `~/.config/zed/settings.json`: + + ```json + { + "context_servers": { + "trigger": { + "source": "custom", + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Zed context servers docs ↗](https://zed.dev/docs/ai/mcp) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client cline + ``` + + Or add this configuration to `~/Library/Application Support/Code/User/globalStorage/saoudrizwan.claude-dev/settings/cline_mcp_settings.json`: + + ```json + { + "mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Cline MCP docs ↗](https://docs.cline.bot/mcp/configuring-mcp-servers) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client gemini-cli + ``` + + Or add this configuration to `~/.gemini/settings.json` (user) or `.gemini/settings.json` (project): + + ```json + { + "mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client amp + ``` + + Or add this configuration to `~/.config/amp/settings.json`: + + ```json + { + "amp.mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Sourcegraph AMP MCP docs ↗](https://ampcode.com/manual#mcp) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client openai-codex + ``` + + Or add this configuration to `~/.codex/config.toml`: + + ```toml + [mcp_servers.trigger] + command = "npx" + args = ["trigger.dev@latest", "mcp"] + ``` + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client crush + ``` + + Or add this configuration to `.crush.json` (project), `crush.json`, or `~/.config/crush/crush.json` (user). Files are loaded in priority order: `.crush.json` → `crush.json` → `$HOME/.config/crush/crush.json`. + + ```json + { + "mcp": { + "trigger": { + "type": "stdio", + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Charm MCP docs ↗](https://github.com/charmbracelet/crush) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client opencode + ``` + + Or add this configuration to `~/.config/opencode/opencode.json` (user) or `./opencode.json` (project): + + ```json + { + "mcp": { + "trigger": { + "type": "local", + "command": ["npx", "trigger.dev@latest", "mcp"], + "enabled": true + } + } + } + ``` + + [View opencode MCP docs ↗](https://opencode.ai/docs/mcp-servers/) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client ruler + ``` + + Or add this configuration to `.ruler/mcp.json`: + + ```json + { + "mcpServers": { + "trigger": { + "type": "stdio", + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + -- `--log-file ` - Configure the MCP server to write logs to a file -- `-a, --api-url ` - Configure a custom Trigger.dev API URL -- `-l, --log-level ` - Set CLI log level (debug, info, log, warn, error, none) +After adding the config, restart your client. You should see a server named **trigger** connect automatically. ## Authentication -You can use the MCP server without authentication with the `search_docs` tool, but for any other tool call you will need to authenticate the MCP server via the same method as the [Trigger.dev CLI](/cli-login-commands).The first time you attempt to use a tool that requires authentication, you will be prompted to authenticate the MCP server via the MCP client. +The `search_docs` tool works without authentication. All other tools require you to be logged in via the [Trigger.dev CLI](/cli-login-commands). The first time you use an authenticated tool, your MCP client will prompt you to log in. + + -### Examples +The `install-mcp` command supports these options: + +**Core Options** + +- `-p, --project-ref ` — Scope the MCP server to a specific project +- `-t, --tag ` — CLI package version to use (default: latest) +- `--dev-only` — Restrict to the dev environment only +- `--yolo` — Install into all supported clients automatically +- `--scope ` — `user`, `project`, or `local` +- `--client ` — Install into specific client(s) + +**Configuration Options** + +- `--log-file ` — Write logs to a file +- `-a, --api-url ` — Custom Trigger.dev API URL +- `-l, --log-level ` — Log level (debug, info, log, warn, error, none) + +**Examples** Install for all supported clients: @@ -69,105 +330,21 @@ Install for specific clients: npx trigger.dev@latest install-mcp --client claude-code cursor --scope user ``` -Install with development environment restriction: +Restrict to dev environment for a specific project: ```bash npx trigger.dev@latest install-mcp --dev-only --project-ref proj_abc123 ``` -## Supported MCP Clients - -The Trigger.dev MCP Server supports the following clients: - -| Client | Scope Options | Configuration File | Documentation | -| -------------------- | -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| **Claude Code** | user, project, local | `~/.claude.json` or `./.mcp.json` (project/local scope) | [Claude Code MCP Docs](https://docs.anthropic.com/en/docs/claude-code/mcp) | -| **Cursor** | user, project | `~/.cursor/mcp.json` (user) or `./.cursor/mcp.json` (project) | [Cursor MCP Docs](https://docs.cursor.com/features/mcp) | -| **VSCode** | user, project | `~/Library/Application Support/Code/User/mcp.json` (user) or `./.vscode/mcp.json` (project) | [VSCode MCP Docs](https://code.visualstudio.com/docs/copilot/chat/mcp-servers) | -| **Zed** | user | `~/.config/zed/settings.json` | [Zed Context Servers Docs](https://zed.dev/docs/context-servers) | -| **Windsurf** | user | `~/.codeium/windsurf/mcp_config.json` | [Windsurf MCP Docs](https://docs.codeium.com/windsurf/mcp) | -| **Gemini CLI** | user, project | `~/.gemini/settings.json` (user) or `./.gemini/settings.json` (project) | [Gemini CLI MCP Tutorial](https://medium.com/@joe.njenga/gemini-cli-mcp-tutorial-setup-commands-practical-use-step-by-step-example-b57f55db5f4a) | -| **Charm Crush** | user, project, local | `~/.config/crush/crush.json` (user), `./crush.json` (project), or `./.crush.json` (local) | [Charm MCP Docs](https://github.com/charmbracelet/mcp) | -| **Cline** | user | `~/Library/Application Support/Code/User/globalStorage/saoudrizwan.claude-dev/settings/cline_mcp_settings.json` | [Cline MCP Docs](https://github.com/saoudrizwan/claude-dev#mcp) | -| **OpenAI Codex CLI** | user | `~/.codex/config.toml` | See OpenAI Codex CLI documentation for MCP configuration | -| **Sourcegraph AMP** | user | `~/.config/amp/settings.json` | [Sourcegraph AMP MCP Docs](https://docs.sourcegraph.com/amp/mcp) | -| **opencode** | user, project | `~/.config/opencode/opencode.json` (user) or `./opencode.json` (project) | [opencode MCP Docs](https://opencode.ai/docs/mcp-servers/) | - -## Manual Configuration - -If your client isn't directly supported by the installer, you can configure it manually. The MCP server uses the following configuration: - -**Server Name:** `trigger` - -**Command:** `npx` - -**Arguments:** `["trigger.dev@latest", "mcp"]` - -### Example JSON Configuration +To add these options to a manual config, append them to the `args` array: ```json { - "mcpServers": { - "trigger": { - "command": "npx", - "args": ["trigger.dev@latest", "mcp"] - } - } + "args": ["trigger.dev@latest", "mcp", "--dev-only", "--project-ref", "proj_abc123"] } ``` -### Example TOML Configuration (for Codex CLI) - -```toml -[mcp_servers.trigger] -command = "npx" -args = ["trigger.dev@latest", "mcp"] -``` - -### Additional Options - -You can add these optional arguments to customize the server behavior: - -- `--log-file ` - Log to a specific file -- `--api-url ` - Use a custom Trigger.dev API URL -- `--dev-only` - Restrict to dev environment only -- `--project-ref ` - Scope to a specific project - -## Environment-Specific Configuration - -### Development Only - -To restrict the MCP server to only work with the development environment: - -```json -{ - "mcpServers": { - "trigger": { - "command": "npx", - "args": ["trigger.dev@latest", "mcp", "--dev-only"] - } - } -} -``` - -### Project-Scoped - -To scope the server to a specific project: - -```json -{ - "mcpServers": { - "trigger": { - "command": "npx", - "args": ["trigger.dev@latest", "mcp", "--project-ref", "proj_your_project_ref"] - } - } -} -``` - -## Verification - -After installation, restart your MCP client and look for a server named "trigger". The server should connect automatically and provide access to all Trigger.dev tools. + ## Getting Started diff --git a/docs/mcp-tools.mdx b/docs/mcp-tools.mdx index 0163de97a47..058a3671ab7 100644 --- a/docs/mcp-tools.mdx +++ b/docs/mcp-tools.mdx @@ -1,527 +1,133 @@ --- title: "MCP Tools" sidebarTitle: "Tools" -description: "Learn about the tools available in the Trigger.dev MCP Server" -tag: "new" +description: "Learn about how to use the tools available in the Trigger.dev MCP Server" --- -The Trigger.dev MCP Server provides a comprehensive set of tools that enable AI assistants to interact with your Trigger.dev projects. These tools cover everything from project management to task execution and monitoring. - ## Documentation and Search Tools ### search_docs -Search across the Trigger.dev documentation to find relevant information, code examples, API references, and guides. +Search the Trigger.dev documentation for guides, examples, and API references. - - The search query to find information in the Trigger.dev documentation - +**Example usage:** +- _"How do I create a scheduled task?"_ +- _"Show me webhook examples"_ +- _"What are the deployment options?"_ -**Usage Examples:** +## Project Management Tools -- "How do I create a scheduled task?" -- "webhook examples" -- "deployment configuration" -- "error handling patterns" +### list_orgs - -```json Example Usage -{ - "tool": "search_docs", - "arguments": { - "query": "webhook examples" - } -} -``` - +List all organizations you have access to. -## Project Management Tools +**Example usage:** +- _"What organizations do I have?"_ +- _"Show me my orgs"_ ### list_projects List all projects in your Trigger.dev account. -**No parameters required** - - - Array of project objects containing project details, IDs, and metadata - - - -```json Example Response -{ - "projects": [ - { - "id": "proj_abc123", - "name": "My App", - "slug": "my-app", - "organizationId": "org_xyz789" - } - ] -} -``` - - -### list_orgs - -List all organizations you have access to. - -**No parameters required** - - - Array of organization objects containing organization details and metadata - +**Example usage:** +- _"What projects do I have?"_ +- _"List my Trigger.dev projects"_ ### create_project_in_org Create a new project in an organization. - - The organization to create the project in, can either be the organization slug or the ID. Use the - `list_orgs` tool to get a list of organizations and ask the user to select one. - - - - The name of the project to create - - - -```json Example Usage -{ - "tool": "create_project_in_org", - "arguments": { - "orgParam": "my-org", - "name": "New Project" - } -} -``` - +**Example usage:** +- _"Create a new project called 'my-app'"_ +- _"Set up a new Trigger.dev project"_ ### initialize_project Initialize Trigger.dev in your project with automatic setup and configuration. - - The organization to create the project in, can either be the organization slug or the ID. Use the - `list_orgs` tool to get a list of organizations and ask the user to select one. - - - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - +**Example usage:** +- _"Set up Trigger.dev in this project"_ +- _"Add Trigger.dev to my app"_ - - The name of the project to create. If projectRef is not provided, we will use this name to create - a new project in the organization you select. - +## Task Management Tools - - The current working directory of the project - +### get_current_worker -## Task Management Tools +Get the current worker for a project, including the worker version, SDK version, and registered tasks with their payload schemas. -### get_tasks - -Get all tasks in a project. - - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). If not provided, we will try to find the config file in the - current working directory. - - - - The environment to get tasks for. Options: `dev`, `staging`, `prod`, `preview` - - - - The branch to get tasks for, only used for preview environments - - - -```json Example Usage -{ - "tool": "get_tasks", - "arguments": { - "projectRef": "proj_abc123", - "environment": "dev" - } -} -``` - +**Example usage:** +- _"What tasks are available?"_ +- _"Show me the tasks in dev"_ ### trigger_task -Trigger a task to run. - - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - - - - The environment to trigger the task in. Options: `dev`, `staging`, `prod`, `preview` - - - - The branch to trigger the task in, only used for preview environments - - - - The ID/slug of the task to trigger. Use the `get_tasks` tool to get a list of tasks and ask the - user to select one if it's not clear which one to use. - - - - The payload to trigger the task with, must be a valid JSON string - - - - Additional options for the task run - - - The name of the queue to trigger the task in, by default will use the queue configured in the - task - - - The delay before the task run is executed - - - The idempotency key to use for the task run - - - The machine preset to use for the task run. Options: `micro`, `small-1x`, `small-2x`, - `medium-1x`, `medium-2x`, `large-1x`, `large-2x` - - - The maximum number of attempts to retry the task run - - - The maximum duration in seconds of the task run - - - Tags to add to the task run. Must be less than 128 characters and cannot have more than 5 - - - The time to live of the task run. If the run doesn't start executing within this time, it will - be automatically cancelled. - - - - - -```json Example Usage -{ - "tool": "trigger_task", - "arguments": { - "projectRef": "proj_abc123", - "taskId": "email-notification", - "payload": "{\"email\": \"user@example.com\", \"subject\": \"Hello World\"}", - "options": { - "tags": ["urgent"], - "maxAttempts": 3 - } - } -} -``` - +Trigger a task to run with a specific payload. You can add a delay, set tags, configure retries, choose a machine size, set a TTL, or use an idempotency key. + +**Example usage:** +- _"Run the email-notification task"_ +- _"Trigger my-task with userId 123"_ +- _"Execute the sync task in production"_ ## Run Monitoring Tools ### get_run_details -Get the details of a specific task run. +Get detailed information about a specific task run, including logs and status. Enable debug mode to get the full trace with all logs and spans. - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - +**Example usage:** +- _"Show me details for run run_abc123"_ +- _"Why did this run fail?"_ - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - +### list_runs + +List runs for a project. Filter by status, task, tags, version, machine size, or time period. - - The environment to get the run details from. Options: `dev`, `staging`, `prod`, `preview` - +**Example usage:** +- _"Show me recent runs"_ +- _"List failed runs from the last 7 days"_ +- _"What runs are currently executing?"_ - - The branch to get the run details from, only used for preview environments - +### wait_for_run_to_complete - - The ID of the run to get the details of, starts with `run_` - +Wait for a specific run to finish and return the result. - - Enable debug mode to get more detailed information about the run, including the entire trace (all logs and spans for the run and any child run). Set this to true if prompted to debug a run. - +**Example usage:** +- _"Wait for run run_abc123 to complete"_ ### cancel_run -Cancel a running task. - - - The ID of the run to cancel, starts with `run_` - - - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - - - - The environment to cancel the run in. Options: `dev`, `staging`, `prod`, `preview` - - - - The branch to cancel the run in, only used for preview environments - - - -```json Example Usage -{ - "tool": "cancel_run", - "arguments": { - "runId": "run_abc123", - "projectRef": "proj_abc123" - } -} -``` - +Cancel a running or queued run. -### list_runs - -List all runs for a project with comprehensive filtering options. - - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - - - - The environment to list runs from. Options: `dev`, `staging`, `prod`, `preview` - - - - The branch to list runs from, only used for preview environments - - - - The cursor to use for pagination, starts with `run_` - - - - The number of runs to list in a single page. Up to 100 - - - - Filter for runs with this run status. Options: `PENDING_VERSION`, `QUEUED`, `DEQUEUED`, - `EXECUTING`, `WAITING`, `COMPLETED`, `CANCELED`, `FAILED`, `CRASHED`, `SYSTEM_FAILURE`, `DELAYED`, - `EXPIRED`, `TIMED_OUT` - - - - Filter for runs that match this task identifier - - - - Filter for runs that match this version, e.g. `20250808.3` - - - - Filter for runs that include this tag - - - - Filter for runs created after this ISO 8601 timestamp - - - - Filter for runs created before this ISO 8601 timestamp - - - - Filter for runs created in the last N time period. Examples: `7d`, `30d`, `365d` - - - - Filter for runs that match this machine preset. Options: `micro`, `small-1x`, `small-2x`, - `medium-1x`, `medium-2x`, `large-1x`, `large-2x` - - - -```json Example Usage -{ - "tool": "list_runs", - "arguments": { - "projectRef": "proj_abc123", - "status": "COMPLETED", - "limit": 10, - "period": "7d" - } -} -``` - +**Example usage:** +- _"Cancel run run_abc123"_ +- _"Stop that task"_ ## Deployment Tools ### deploy -Deploy a project to staging or production environments. - - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - - - - The environment to deploy to. Options: `staging`, `prod`, `preview` - - - - The branch to deploy, only used for preview environments - - - - Skip promoting the deployment to the current deployment for the environment - - - - Skip syncing environment variables when using the syncEnvVars extension - - - - Skip checking for @trigger.dev package updates - - - -```json Example Usage -{ - "tool": "deploy", - "arguments": { - "projectRef": "proj_abc123", - "environment": "prod", - "skipUpdateCheck": true - } -} -``` - - -### list_deployments - -List deployments for a project with comprehensive filtering options. - - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - - - - The environment to list deployments for. Options: `staging`, `prod`, `preview` - - - - The branch to list deployments from, only used for preview environments - - - - The deployment ID to start the search from, to get the next page - - - - The number of deployments to return, defaults to 20 (max 100) - - - - Filter deployments that are in this status. Options: `PENDING`, `BUILDING`, `DEPLOYING`, `DEPLOYED`, `FAILED`, `CANCELED`, `TIMED_OUT` - - - - The date to start the search from, in ISO 8601 format - - - - The date to end the search, in ISO 8601 format - - - - The period to search within. Examples: `1d`, `7d`, `3h` - - - -```json Example Usage -{ - "tool": "list_deployments", - "arguments": { - "projectRef": "proj_abc123", - "environment": "prod", - "status": "DEPLOYED", - "limit": 10 - } -} -``` - +Deploy your project to staging or production. + +**Example usage:** +- _"Deploy to production"_ +- _"Deploy to staging"_ + +### list_deploys + +List deployments for a project. Filter by status or time period. + +**Example usage:** +- _"Show me recent deployments"_ +- _"What's deployed to production?"_ ### list_preview_branches List all preview branches in the project. - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). If not provided, we will try to find the config file in the - current working directory. - - - -```json Example Usage -{ - "tool": "list_preview_branches", - "arguments": { - "projectRef": "proj_abc123" - } -} -``` - +**Example usage:** +- _"What preview branches exist?"_ +- _"Show me preview deployments"_ - The deploy tool and list_preview_branches tool are not available when the MCP server is running with the `--dev-only` flag. + The deploy and list_preview_branches tools are not available when the MCP server is running with the `--dev-only` flag. diff --git a/docs/quick-start.mdx b/docs/quick-start.mdx index d6253bc2697..375d225b670 100644 --- a/docs/quick-start.mdx +++ b/docs/quick-start.mdx @@ -8,29 +8,14 @@ import CliDevStep from '/snippets/step-cli-dev.mdx'; import CliRunTestStep from '/snippets/step-run-test.mdx'; import CliViewRunStep from '/snippets/step-view-run.mdx'; -In this guide we will: -1. Create a `trigger.config.ts` file and a `/trigger` directory with an example task. -2. Get you to run the task using the CLI. -3. Show you how to view the run logs for that task. + -You can either: - -- Use the [Trigger.dev Cloud](https://cloud.trigger.dev). -- Or [self-host](/open-source-self-hosting) the service. - - - - - -Once you've created an account, follow the steps in the app to: - -1. Complete your account details. -2. Create your first Organization and Project. +Sign up at [Trigger.dev Cloud](https://cloud.trigger.dev) (or [self-host](/open-source-self-hosting)). The onboarding flow will guide you through creating your first organization and project. @@ -43,11 +28,18 @@ Once you've created an account, follow the steps in the app to: ## Next steps - + + + Learn how to build Trigger.dev projects using AI coding assistants + Learn how to trigger tasks from your code. Tasks are the core of Trigger.dev. Learn what they are and how to write them. + + Guides and examples for triggering tasks from your code. + + diff --git a/docs/skills.mdx b/docs/skills.mdx new file mode 100644 index 00000000000..eb4add47952 --- /dev/null +++ b/docs/skills.mdx @@ -0,0 +1,83 @@ +--- +title: "Skills" +description: "Install Trigger.dev skills to teach any AI coding assistant best practices for writing tasks, agents, and workflows." +sidebarTitle: "Skills" +tag: "new" +--- + +## What are agent skills? + +Skills are portable instruction sets that teach AI coding assistants how to use Trigger.dev effectively. Unlike vendor-specific config files (`.cursor/rules`, `CLAUDE.md`), skills use an open standard that works across all major AI assistants. For example, Cursor users and Claude Code users can get the same knowledge from a single install. + +Skills are installed as directories containing a `SKILL.md` file. Each `SKILL.md` includes YAML frontmatter (name, description) and markdown instructions with patterns, examples, and best practices that AI assistants automatically discover and follow. + +## Installation + +When you run `npx skills add triggerdotdev/skills`, the CLI detects your installed AI tools and copies the appropriate files to each tool's expected location. For example, `.claude/skills/`, `.cursor/skills/`, `.github/skills/`, etc. + +```bash +npx skills add triggerdotdev/skills +``` + +`skills` is an open-source CLI by Vercel. Learn more at [skills.sh](https://skills.sh). + +The result: your AI assistant understands Trigger.dev's specific patterns for exports, schema validation, error handling, retries, and more. + + +## Available skills + +Install all skills at once, or pick the ones relevant to your current work: + +```bash +# Install all Trigger.dev skills +npx skills add triggerdotdev/skills + +# Or install individual skills +npx skills add triggerdotdev/skills --skill trigger-tasks +npx skills add triggerdotdev/skills --skill trigger-agents +npx skills add triggerdotdev/skills --skill trigger-config +npx skills add triggerdotdev/skills --skill trigger-realtime +npx skills add triggerdotdev/skills --skill trigger-setup +``` + +| Skill | Use for | Covers | +|-------|---------|--------| +| `trigger-setup` | First time setup, new projects | SDK install, `npx trigger init`, project structure | +| `trigger-tasks` | Writing background tasks, async workflows, scheduled tasks | Triggering, waits, queues, retries, cron, metadata | +| `trigger-agents` | LLM workflows, orchestration, multi-step AI agents | Prompt chaining, routing, parallelization, human-in-the-loop | +| `trigger-realtime` | Live updates, progress indicators, streaming | React hooks, progress bars, streaming AI responses | +| `trigger-config` | Project setup, build configuration | `trigger.config.ts`, extensions (Prisma, FFmpeg, Playwright) | + +Not sure which skill to install? Install `trigger-tasks`; it covers the most common patterns for writing Trigger.dev tasks. + + +## Supported AI assistants + +Skills work with any AI coding assistant that supports the [Agent Skills standard](https://agentskills.io), including: + +- [Claude Code](https://docs.anthropic.com/en/docs/claude-code) +- [Cursor](https://cursor.com) +- [Windsurf](https://codeium.com/windsurf) +- [GitHub Copilot](https://github.com/features/copilot) +- [Cline](https://github.com/cline/cline) +- [Codex CLI](https://github.com/openai/codex) +- [Gemini CLI](https://github.com/google-gemini/gemini-cli) +- [OpenCode](https://opencode.ai) +- [View all →](https://skills.sh) + +## Next steps + + + + Give your AI assistant direct access to Trigger.dev tools and APIs. + + + Learn the task patterns that skills teach your AI assistant. + + + Build durable AI workflows with prompt chaining and human-in-the-loop. + + + Browse the full Agent Skills ecosystem. + + \ No newline at end of file diff --git a/docs/snippets/step-cli-init.mdx b/docs/snippets/step-cli-init.mdx index 265f3d9c894..84bb340dcfe 100644 --- a/docs/snippets/step-cli-init.mdx +++ b/docs/snippets/step-cli-init.mdx @@ -20,12 +20,19 @@ yarn dlx trigger.dev@latest init + It will do a few things: -1. Log you into the CLI if you're not already logged in. -2. Create a `trigger.config.ts` file in the root of your project. -3. Ask where you'd like to create the `/trigger` directory. -4. Create the `/trigger` directory with an example task, `/trigger/example.[ts/js]`. + + Our [Trigger.dev MCP server](/mcp-introduction) gives your AI assistant direct access to Trigger.dev tools; search docs, trigger tasks, deploy projects, and monitor runs. We recommend installing it for the best developer experience. + + +1. Ask if you want to install the [Trigger.dev MCP server](/mcp-introduction) for your AI assistant. +2. Log you into the CLI if you're not already logged in. +3. Ask you to select your project. +4. Install the required SDK packages. +5. Ask where you'd like to create the `/trigger` directory and create it with an example task. +6. Create a `trigger.config.ts` file in the root of your project. Install the "Hello World" example task when prompted. We'll use this task to test the setup. From 0674d74bbbe17ff4ab5b5583e7b7cddf4314df23 Mon Sep 17 00:00:00 2001 From: DKP <8297864+D-K-P@users.noreply.github.com> Date: Thu, 29 Jan 2026 13:59:27 +0000 Subject: [PATCH 010/400] Added Trigger.dark theme to the docs (#2967) --- --- docs/docs.json | 5 +++++ docs/style.css | 30 +++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/docs.json b/docs/docs.json index 405f4eb0a2c..dcf637aea2f 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -446,6 +446,11 @@ "display": "simple" } }, + "styling": { + "codeblocks": { + "theme": "css-variables" + } + }, "appearance": { "default": "dark", "strict": true diff --git a/docs/style.css b/docs/style.css index b209952ad23..94aea582db3 100644 --- a/docs/style.css +++ b/docs/style.css @@ -1,3 +1,31 @@ button~.absolute.peer-hover\:opacity-100 { color: #000 -} \ No newline at end of file +} + +:root { + /* Code block colors - Trigger.dark theme */ + --mint-color-background: #121317; + --mint-color-text: #D4D4D4; + --mint-token-constant: #9B99FF; + --mint-token-string: #AFEC73; + --mint-token-comment: #5F6570; + --mint-token-keyword: #E888F8; + --mint-token-parameter: #CCCBFF; + --mint-token-function: #D9F07C; + --mint-token-string-expression: #AFEC73; + --mint-token-punctuation: #878C99; + --mint-token-link: #826DFF; + + /* Shiki css-variables fallbacks */ + --shiki-foreground: #D4D4D4; + --shiki-background: #121317; + --shiki-token-constant: #9B99FF; + --shiki-token-string: #AFEC73; + --shiki-token-comment: #5F6570; + --shiki-token-keyword: #E888F8; + --shiki-token-parameter: #CCCBFF; + --shiki-token-function: #D9F07C; + --shiki-token-string-expression: #AFEC73; + --shiki-token-punctuation: #878C99; + --shiki-token-link: #826DFF; +} From 72c357125b3035ad515212f3f6456533ff4323e2 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 29 Jan 2026 15:30:56 +0000 Subject: [PATCH 011/400] Query enabled via feature flag (#2968) - Renamed FeatureFlag functions to be singular where it makes sense. - Added function to handle multiple feature flags - canAccessQuery now checks the global feature flag and environment variable as well --- --- apps/webapp/app/env.server.ts | 3 + .../OrganizationsPresenter.server.ts | 16 +++-- .../presenters/v3/RegionsPresenter.server.ts | 4 +- .../route.tsx | 60 ++++------------ .../runsRepository/runsRepository.server.ts | 4 +- apps/webapp/app/v3/canAccessQuery.server.ts | 47 +++++++++++++ .../app/v3/eventRepository/index.server.ts | 24 +++---- apps/webapp/app/v3/featureFlags.server.ts | 69 ++++++++++++++++--- .../worker/workerGroupService.server.ts | 8 +-- 9 files changed, 154 insertions(+), 81 deletions(-) create mode 100644 apps/webapp/app/v3/canAccessQuery.server.ts diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 98c04b6f953..dcbcac079a0 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1190,6 +1190,9 @@ const EnvironmentSchema = z CLICKHOUSE_LOGS_DETAIL_MAX_THREADS: z.coerce.number().int().default(2), CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME: z.coerce.number().int().default(60), + // Query feature flag + QUERY_FEATURE_ENABLED: z.string().default("1"), + // Query page ClickHouse limits (for TSQL queries) QUERY_CLICKHOUSE_MAX_EXECUTION_TIME: z.coerce.number().int().default(10), QUERY_CLICKHOUSE_MAX_MEMORY_USAGE: z.coerce.number().int().default(1_073_741_824), // 1GB in bytes diff --git a/apps/webapp/app/presenters/OrganizationsPresenter.server.ts b/apps/webapp/app/presenters/OrganizationsPresenter.server.ts index b52e69db9c9..c229a0d7f45 100644 --- a/apps/webapp/app/presenters/OrganizationsPresenter.server.ts +++ b/apps/webapp/app/presenters/OrganizationsPresenter.server.ts @@ -1,4 +1,4 @@ -import { RuntimeEnvironment, type PrismaClient } from "@trigger.dev/database"; +import type { RuntimeEnvironment, PrismaClient } from "@trigger.dev/database"; import { redirect } from "remix-typedjson"; import { prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; @@ -10,7 +10,7 @@ import { } from "./SelectBestEnvironmentPresenter.server"; import { sortEnvironments } from "~/utils/environmentSort"; import { defaultAvatar, parseAvatar } from "~/components/primitives/Avatar"; -import { validatePartialFeatureFlags } from "~/v3/featureFlags.server"; +import { flags, validatePartialFeatureFlags } from "~/v3/featureFlags.server"; export class OrganizationsPresenter { #prismaClient: PrismaClient; @@ -153,18 +153,24 @@ export class OrganizationsPresenter { }, }); + // Get global feature flags (no overrides or defaults) + const globalFlags = await flags(); + return orgs.map((org) => { - const flagsResult = org.featureFlags + const orgFlagsResult = org.featureFlags ? validatePartialFeatureFlags(org.featureFlags as Record) : ({ success: false } as const); - const flags = flagsResult.success ? flagsResult.data : {}; + const orgFlags = orgFlagsResult.success ? orgFlagsResult.data : {}; + + // Combine global flags with org flags (org flags win) + const combinedFlags = { ...globalFlags, ...orgFlags }; return { id: org.id, slug: org.slug, title: org.title, avatar: parseAvatar(org.avatar, defaultAvatar), - featureFlags: flags, + featureFlags: combinedFlags, projects: org.projects.map((project) => ({ id: project.id, slug: project.slug, diff --git a/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts b/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts index c304597bb1f..7a35fb6fb9b 100644 --- a/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts @@ -1,6 +1,6 @@ import { type Project } from "~/models/project.server"; import { type User } from "~/models/user.server"; -import { FEATURE_FLAG, makeFlags } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; import { BasePresenter } from "./basePresenter.server"; import { getCurrentPlan } from "~/services/platform.v3.server"; @@ -48,7 +48,7 @@ export class RegionsPresenter extends BasePresenter { throw new Error("Project not found"); } - const getFlag = makeFlags(this._replica); + const getFlag = makeFlag(this._replica); const defaultWorkerInstanceGroupId = await getFlag({ key: FEATURE_FLAG.defaultWorkerInstanceGroupId, }); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx index 996149a4697..72020d8adf9 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx @@ -75,7 +75,7 @@ import { executeQuery, type QueryScope } from "~/services/queryService.server"; import { requireUser } from "~/services/session.server"; import { downloadFile, rowsToCSV, rowsToJSON } from "~/utils/dataExport"; import { EnvironmentParamSchema, organizationBillingPath } from "~/utils/pathBuilder"; -import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags.server"; +import { canAccessQuery } from "~/v3/canAccessQuery.server"; import { querySchemas } from "~/v3/querySchemas"; import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; import { QueryHelpSidebar } from "./QueryHelpSidebar"; @@ -91,40 +91,6 @@ function toISOString(value: Date | string): string { return value.toISOString(); } -async function hasQueryAccess( - userId: string, - isAdmin: boolean, - isImpersonating: boolean, - organizationSlug: string -): Promise { - if (isAdmin || isImpersonating) { - return true; - } - - // Check organization feature flags - const organization = await prisma.organization.findFirst({ - where: { - slug: organizationSlug, - members: { some: { userId } }, - }, - select: { - featureFlags: true, - }, - }); - - if (!organization?.featureFlags) { - return false; - } - - const flags = organization.featureFlags as Record; - const hasQueryAccessResult = validateFeatureFlagValue( - FEATURE_FLAG.hasQueryAccess, - flags.hasQueryAccess - ); - - return hasQueryAccessResult.success && hasQueryAccessResult.data === true; -} - const scopeOptions = [ { value: "environment", label: "Environment" }, { value: "project", label: "Project" }, @@ -135,12 +101,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await requireUser(request); const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params); - const canAccess = await hasQueryAccess( - user.id, - user.admin, - user.isImpersonating, - organizationSlug - ); + const canAccess = await canAccessQuery({ + userId: user.id, + isAdmin: user.admin, + isImpersonating: user.isImpersonating, + organizationSlug, + }); if (!canAccess) { throw redirect("/"); } @@ -200,12 +166,12 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const user = await requireUser(request); const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params); - const canAccess = await hasQueryAccess( - user.id, - user.admin, - user.isImpersonating, - organizationSlug - ); + const canAccess = await canAccessQuery({ + userId: user.id, + isAdmin: user.admin, + isImpersonating: user.isImpersonating, + organizationSlug, + }); if (!canAccess) { return typedjson( { diff --git a/apps/webapp/app/services/runsRepository/runsRepository.server.ts b/apps/webapp/app/services/runsRepository/runsRepository.server.ts index 895c8b5fe5c..90b58b8a980 100644 --- a/apps/webapp/app/services/runsRepository/runsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/runsRepository.server.ts @@ -8,7 +8,7 @@ import parseDuration from "parse-duration"; import { z } from "zod"; import { timeFilters } from "~/components/runs/v3/SharedFilters"; import { type PrismaClient, type PrismaClientOrTransaction } from "~/db.server"; -import { FEATURE_FLAG, makeFlags } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; import { startActiveSpan } from "~/v3/tracer.server"; import { logger } from "../logger.server"; import { ClickHouseRunsRepository } from "./clickhouseRunsRepository.server"; @@ -163,7 +163,7 @@ export class RunsRepository implements IRunsRepository { async #getRepository(): Promise { return startActiveSpan("runsRepository.getRepository", async (span) => { - const getFlag = makeFlags(this.options.prisma); + const getFlag = makeFlag(this.options.prisma); const runsListRepository = await getFlag({ key: FEATURE_FLAG.runsListRepository, defaultValue: this.defaultRepository, diff --git a/apps/webapp/app/v3/canAccessQuery.server.ts b/apps/webapp/app/v3/canAccessQuery.server.ts new file mode 100644 index 00000000000..87a248725b5 --- /dev/null +++ b/apps/webapp/app/v3/canAccessQuery.server.ts @@ -0,0 +1,47 @@ +import { prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; + +export async function canAccessQuery(options: { + userId: string; + isAdmin: boolean; + isImpersonating: boolean; + organizationSlug: string; +}): Promise { + const { userId, isAdmin, isImpersonating, organizationSlug } = options; + + // 1. If it's on then we have access + const globallyEnabled = env.QUERY_FEATURE_ENABLED === "1"; + if (globallyEnabled) { + return true; + } + + // 2. Admins always have access + if (isAdmin || isImpersonating) { + return true; + } + + // 3. Check if org/global feature flag is on + const org = await prisma.organization.findFirst({ + where: { + slug: organizationSlug, + members: { some: { userId } }, + }, + select: { + featureFlags: true, + }, + }); + + const flag = makeFlag(); + const flagResult = await flag({ + key: FEATURE_FLAG.hasQueryAccess, + defaultValue: false, + overrides: (org?.featureFlags as Record) ?? {}, + }); + if (flagResult) { + return true; + } + + // 4. Not enabled anywhere + return false; +} diff --git a/apps/webapp/app/v3/eventRepository/index.server.ts b/apps/webapp/app/v3/eventRepository/index.server.ts index cb211e2b02f..2f457e23593 100644 --- a/apps/webapp/app/v3/eventRepository/index.server.ts +++ b/apps/webapp/app/v3/eventRepository/index.server.ts @@ -5,9 +5,9 @@ import { clickhouseEventRepositoryV2, } from "./clickhouseEventRepositoryInstance.server"; import { IEventRepository, TraceEventOptions } from "./eventRepository.types"; -import { prisma } from "~/db.server"; +import { prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; -import { FEATURE_FLAG, flags } from "../featureFlags.server"; +import { FEATURE_FLAG, flag } from "../featureFlags.server"; import { getTaskEventStore } from "../taskEventStore.server"; export function resolveEventRepositoryForStore(store: string | undefined): IEventRepository { @@ -24,13 +24,13 @@ export function resolveEventRepositoryForStore(store: string | undefined): IEven return eventRepository; } - export const EVENT_STORE_TYPES = { - POSTGRES: "postgres", - CLICKHOUSE: "clickhouse", - CLICKHOUSE_V2: "clickhouse_v2", - } as const; +export const EVENT_STORE_TYPES = { + POSTGRES: "postgres", + CLICKHOUSE: "clickhouse", + CLICKHOUSE_V2: "clickhouse_v2", +} as const; -export type EventStoreType = typeof EVENT_STORE_TYPES[keyof typeof EVENT_STORE_TYPES]; +export type EventStoreType = (typeof EVENT_STORE_TYPES)[keyof typeof EVENT_STORE_TYPES]; export async function getConfiguredEventRepository( organizationId: string @@ -122,21 +122,21 @@ export async function getV3EventRepository( async function resolveTaskEventRepositoryFlag( featureFlags: Record | undefined ): Promise<"clickhouse" | "clickhouse_v2" | "postgres"> { - const flag = await flags({ + const flagResult = await flag({ key: FEATURE_FLAG.taskEventRepository, defaultValue: env.EVENT_REPOSITORY_DEFAULT_STORE, overrides: featureFlags, }); - if (flag === "clickhouse_v2") { + if (flagResult === "clickhouse_v2") { return "clickhouse_v2"; } - if (flag === "clickhouse") { + if (flagResult === "clickhouse") { return "clickhouse"; } - return flag; + return flagResult; } export async function recordRunDebugLog( diff --git a/apps/webapp/app/v3/featureFlags.server.ts b/apps/webapp/app/v3/featureFlags.server.ts index 605c11defc3..e889b2123d2 100644 --- a/apps/webapp/app/v3/featureFlags.server.ts +++ b/apps/webapp/app/v3/featureFlags.server.ts @@ -25,14 +25,14 @@ export type FlagsOptions = { overrides?: Record; }; -export function makeFlags(_prisma: PrismaClientOrTransaction = prisma) { - function flags( +export function makeFlag(_prisma: PrismaClientOrTransaction = prisma) { + function flag( opts: FlagsOptions & { defaultValue: z.infer<(typeof FeatureFlagCatalog)[T]> } ): Promise>; - function flags( + function flag( opts: FlagsOptions ): Promise | undefined>; - async function flags( + async function flag( opts: FlagsOptions ): Promise | undefined> { const value = await _prisma.featureFlag.findUnique({ @@ -60,11 +60,11 @@ export function makeFlags(_prisma: PrismaClientOrTransaction = prisma) { return parsed.data; } - return flags; + return flag; } -export function makeSetFlags(_prisma: PrismaClientOrTransaction = prisma) { - return async function setFlags( +export function makeSetFlag(_prisma: PrismaClientOrTransaction = prisma) { + return async function setFlag( opts: FlagsOptions & { value: z.infer<(typeof FeatureFlagCatalog)[T]> } ): Promise { await _prisma.featureFlag.upsert({ @@ -82,8 +82,59 @@ export function makeSetFlags(_prisma: PrismaClientOrTransaction = prisma) { }; } +export type AllFlagsOptions = { + defaultValues?: Partial; + overrides?: Record; +}; + +export function makeFlags(_prisma: PrismaClientOrTransaction = prisma) { + return async function flags(options?: AllFlagsOptions): Promise> { + const rows = await _prisma.featureFlag.findMany(); + + // Build a map of key -> value from database + const dbValues = new Map(); + for (const row of rows) { + dbValues.set(row.key, row.value); + } + + const result: Partial = {}; + + // Process each flag in the catalog + for (const key of Object.keys(FeatureFlagCatalog) as FeatureFlagKey[]) { + const schema = FeatureFlagCatalog[key]; + + // Priority: overrides > database > defaultValues + if (options?.overrides?.[key] !== undefined) { + const parsed = schema.safeParse(options.overrides[key]); + if (parsed.success) { + (result as any)[key] = parsed.data; + continue; + } + } + + if (dbValues.has(key)) { + const parsed = schema.safeParse(dbValues.get(key)); + if (parsed.success) { + (result as any)[key] = parsed.data; + continue; + } + } + + if (options?.defaultValues?.[key] !== undefined) { + const parsed = schema.safeParse(options.defaultValues[key]); + if (parsed.success) { + (result as any)[key] = parsed.data; + } + } + } + + return result; + }; +} + +export const flag = makeFlag(); export const flags = makeFlags(); -export const setFlags = makeSetFlags(); +export const setFlag = makeSetFlag(); // Create a Zod schema from the existing catalog export const FeatureFlagCatalogSchema = z.object(FeatureFlagCatalog); @@ -112,7 +163,7 @@ export function makeSetMultipleFlags(_prisma: PrismaClientOrTransaction = prisma return async function setMultipleFlags( flags: Partial> ): Promise<{ key: string; value: any }[]> { - const setFlag = makeSetFlags(_prisma); + const setFlag = makeSetFlag(_prisma); const updatedFlags: { key: string; value: any }[] = []; for (const [key, value] of Object.entries(flags)) { diff --git a/apps/webapp/app/v3/services/worker/workerGroupService.server.ts b/apps/webapp/app/v3/services/worker/workerGroupService.server.ts index f05c8783ecd..936f8bbd48e 100644 --- a/apps/webapp/app/v3/services/worker/workerGroupService.server.ts +++ b/apps/webapp/app/v3/services/worker/workerGroupService.server.ts @@ -2,7 +2,7 @@ import { WorkerInstanceGroup, WorkerInstanceGroupType } from "@trigger.dev/datab import { WithRunEngine } from "../baseService.server"; import { WorkerGroupTokenService } from "./workerGroupTokenService.server"; import { logger } from "~/services/logger.server"; -import { FEATURE_FLAG, makeFlags, makeSetFlags } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG, makeFlag, makeSetFlag } from "~/v3/featureFlags.server"; export class WorkerGroupService extends WithRunEngine { private readonly defaultNamePrefix = "worker_group"; @@ -47,14 +47,14 @@ export class WorkerGroupService extends WithRunEngine { }, }); - const getFlag = makeFlags(this._prisma); + const getFlag = makeFlag(this._prisma); const defaultWorkerInstanceGroupId = await getFlag({ key: FEATURE_FLAG.defaultWorkerInstanceGroupId, }); // If there's no global default yet we should set it to the new worker group if (!defaultWorkerInstanceGroupId) { - const setFlag = makeSetFlags(this._prisma); + const setFlag = makeSetFlag(this._prisma); await setFlag({ key: FEATURE_FLAG.defaultWorkerInstanceGroupId, value: workerGroup.id, @@ -166,7 +166,7 @@ export class WorkerGroupService extends WithRunEngine { } async getGlobalDefaultWorkerGroup() { - const flags = makeFlags(this._prisma); + const flags = makeFlag(this._prisma); const defaultWorkerInstanceGroupId = await flags({ key: FEATURE_FLAG.defaultWorkerInstanceGroupId, From 5e049cde3a73e4a95ac648d60e2f8d6f3f6a87e3 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Thu, 29 Jan 2026 19:03:54 +0000 Subject: [PATCH 012/400] fix(run-engine): avoid NAPI string overflow in getExecutionSnapshotsSince by only fetching waitpoints for latest snapshot (#2972) --- .../engine/systems/executionSnapshotSystem.ts | 116 ++- .../engine/tests/getSnapshotsSince.test.ts | 670 ++++++++++++++++++ .../tests/helpers/executionStateMachine.ts | 257 +++++++ .../tests/helpers/snapshotTestHelpers.ts | 322 +++++++++ 4 files changed, 1354 insertions(+), 11 deletions(-) create mode 100644 internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts create mode 100644 internal-packages/run-engine/src/engine/tests/helpers/executionStateMachine.ts create mode 100644 internal-packages/run-engine/src/engine/tests/helpers/snapshotTestHelpers.ts diff --git a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts index b6f31bcffc3..a224e5a86b0 100644 --- a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts @@ -8,10 +8,14 @@ import { TaskRunExecutionSnapshot, TaskRunExecutionStatus, TaskRunStatus, + Waitpoint, } from "@trigger.dev/database"; import { HeartbeatTimeouts } from "../types.js"; import { SystemResources } from "./systems.js"; +/** Chunk size for fetching waitpoints to avoid NAPI string conversion limits */ +const WAITPOINT_CHUNK_SIZE = 100; + export type ExecutionSnapshotSystemOptions = { resources: SystemResources; heartbeatTimeouts: HeartbeatTimeouts; @@ -31,19 +35,41 @@ type ExecutionSnapshotWithCheckAndWaitpoints = Prisma.TaskRunExecutionSnapshotGe }; }>; +type ExecutionSnapshotWithCheckpoint = Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { + checkpoint: true; + }; +}>; + function enhanceExecutionSnapshot( snapshot: ExecutionSnapshotWithCheckAndWaitpoints +): EnhancedExecutionSnapshot { + return enhanceExecutionSnapshotWithWaitpoints( + snapshot, + snapshot.completedWaitpoints, + snapshot.completedWaitpointOrder + ); +} + +/** + * Transforms a snapshot (with checkpoint but without waitpoints) into an EnhancedExecutionSnapshot + * by combining it with pre-fetched waitpoints. + */ +function enhanceExecutionSnapshotWithWaitpoints( + snapshot: ExecutionSnapshotWithCheckpoint, + waitpoints: Waitpoint[], + completedWaitpointOrder: string[] ): EnhancedExecutionSnapshot { return { ...snapshot, friendlyId: SnapshotId.toFriendlyId(snapshot.id), runFriendlyId: RunId.toFriendlyId(snapshot.runId), - completedWaitpoints: snapshot.completedWaitpoints.flatMap((w) => { - //get all indexes of the waitpoint in the completedWaitpointOrder - //we do this because the same run can be in a batch multiple times (i.e. same idempotencyKey) + completedWaitpoints: waitpoints.flatMap((w) => { + // Get all indexes of the waitpoint in the completedWaitpointOrder + // We do this because the same run can be in a batch multiple times (i.e. same idempotencyKey) let indexes: (number | undefined)[] = []; - for (let i = 0; i < snapshot.completedWaitpointOrder.length; i++) { - if (snapshot.completedWaitpointOrder[i] === w.id) { + for (let i = 0; i < completedWaitpointOrder.length; i++) { + if (completedWaitpointOrder[i] === w.id) { indexes.push(i); } } @@ -60,9 +86,7 @@ function enhanceExecutionSnapshot( type: w.type, completedAt: w.completedAt ?? new Date(), idempotencyKey: - w.userProvidedIdempotencyKey && !w.inactiveIdempotencyKey - ? w.idempotencyKey - : undefined, + w.userProvidedIdempotencyKey && !w.inactiveIdempotencyKey ? w.idempotencyKey : undefined, completedByTaskRun: w.completedByTaskRunId ? { id: w.completedByTaskRunId, @@ -91,6 +115,42 @@ function enhanceExecutionSnapshot( }; } +/** + * Gets the waitpoint IDs linked to a snapshot via the _completedWaitpoints join table. + * Uses raw SQL to avoid fetching full waitpoint data. + */ +async function getSnapshotWaitpointIds( + prisma: PrismaClientOrTransaction, + snapshotId: string +): Promise { + const result = await prisma.$queryRaw<{ B: string }[]>` + SELECT "B" FROM "_completedWaitpoints" WHERE "A" = ${snapshotId} + `; + return result.map((r) => r.B); +} + +/** + * Fetches waitpoints in chunks to avoid NAPI string conversion limits. + * This is necessary because waitpoints can have large outputs (100KB+), + * and fetching many at once can exceed Node.js string limits. + */ +async function fetchWaitpointsInChunks( + prisma: PrismaClientOrTransaction, + waitpointIds: string[] +): Promise { + if (waitpointIds.length === 0) return []; + + const allWaitpoints: Waitpoint[] = []; + for (let i = 0; i < waitpointIds.length; i += WAITPOINT_CHUNK_SIZE) { + const chunk = waitpointIds.slice(i, i + WAITPOINT_CHUNK_SIZE); + const waitpoints = await prisma.waitpoint.findMany({ + where: { id: { in: chunk } }, + }); + allWaitpoints.push(...waitpoints); + } + return allWaitpoints; +} + /* Gets the most recent valid snapshot for a run */ export async function getLatestExecutionSnapshot( prisma: PrismaClientOrTransaction, @@ -191,12 +251,27 @@ export function executionDataFromSnapshot(snapshot: EnhancedExecutionSnapshot): }; } +/** + * Gets execution snapshots created after the specified snapshot. + * + * IMPORTANT: This function is optimized to avoid N×M data explosion when runs have many + * completed waitpoints. Due to the many-to-many relation, once waitpoints complete, + * all subsequent snapshots have the same waitpoints linked. For a run with 24 snapshots + * and 236 waitpoints with 100KB outputs each, fetching all waitpoints for all snapshots + * would result in ~570MB of data, causing "Failed to convert rust String into napi string" errors. + * + * Solution: Only the LATEST snapshot's waitpoints are fetched and included. The runner's + * SnapshotManager only processes completedWaitpoints from the latest snapshot anyway - + * intermediate snapshots' waitpoints are ignored. This reduces data from N×M to just M. + * + * Waitpoints are fetched in chunks (100 at a time) to handle batches up to 1000 items. + */ export async function getExecutionSnapshotsSince( prisma: PrismaClientOrTransaction, runId: string, sinceSnapshotId: string ): Promise { - // Find the createdAt of the sinceSnapshotId + // Step 1: Find the createdAt of the sinceSnapshotId const sinceSnapshot = await prisma.taskRunExecutionSnapshot.findFirst({ where: { id: sinceSnapshotId }, select: { createdAt: true }, @@ -206,6 +281,7 @@ export async function getExecutionSnapshotsSince( throw new Error(`No execution snapshot found for id ${sinceSnapshotId}`); } + // Step 2: Fetch snapshots WITHOUT waitpoints to avoid N×M data explosion const snapshots = await prisma.taskRunExecutionSnapshot.findMany({ where: { runId, @@ -213,14 +289,32 @@ export async function getExecutionSnapshotsSince( createdAt: { gt: sinceSnapshot.createdAt }, }, include: { - completedWaitpoints: true, checkpoint: true, + // DO NOT include completedWaitpoints here - this causes the N×M explosion }, orderBy: { createdAt: "desc" }, take: 50, }); - return snapshots.reverse().map(enhanceExecutionSnapshot); + if (snapshots.length === 0) return []; + + // Step 3: Get waitpoint IDs for the LATEST snapshot only (first in desc order) + const latestSnapshot = snapshots[0]; + const waitpointIds = await getSnapshotWaitpointIds(prisma, latestSnapshot.id); + + // Step 4: Fetch waitpoints in chunks to avoid NAPI string conversion limits + const waitpoints = await fetchWaitpointsInChunks(prisma, waitpointIds); + + // Step 5: Build enhanced snapshots - only latest gets waitpoints, others get empty arrays + // The runner only uses completedWaitpoints from the latest snapshot anyway + return snapshots.reverse().map((snapshot) => { + const isLatest = snapshot.id === latestSnapshot.id; + return enhanceExecutionSnapshotWithWaitpoints( + snapshot, + isLatest ? waitpoints : [], + latestSnapshot.completedWaitpointOrder + ); + }); } export class ExecutionSnapshotSystem { diff --git a/internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts b/internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts new file mode 100644 index 00000000000..4352e726866 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts @@ -0,0 +1,670 @@ +import { containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { expect, describe } from "vitest"; +import { RunEngine } from "../index.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; +import { setTimeout } from "node:timers/promises"; +import { + generateTestScenarios, + type SnapshotTestScenario, +} from "./helpers/executionStateMachine.js"; +import { + createWaitpointsWithOutput, + setupTestScenario, + generateLargeOutput, +} from "./helpers/snapshotTestHelpers.js"; +import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic"; + +vi.setConfig({ testTimeout: 120_000 }); + +describe("RunEngine getSnapshotsSince", () => { + containerTest( + "returns empty array when querying from latest snapshot", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const runFriendlyId = generateFriendlyId("run"); + const run = await engine.trigger( + { + number: 1, + friendlyId: runFriendlyId, + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_empty", + spanId: "s_empty", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_empty", + workerQueue: "main", + }); + + // Get all snapshots + const allSnapshots = await prisma.taskRunExecutionSnapshot.findMany({ + where: { runId: run.id, isValid: true }, + orderBy: { createdAt: "asc" }, + }); + + expect(allSnapshots.length).toBeGreaterThan(0); + + // Query from the last snapshot + const lastSnapshot = allSnapshots[allSnapshots.length - 1]; + const result = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: lastSnapshot.id, + }); + + expect(result).not.toBeNull(); + expect(result!.length).toBe(0); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "returns snapshots after the specified one with waitpoints only on latest", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const runFriendlyId = generateFriendlyId("run"); + const run = await engine.trigger( + { + number: 1, + friendlyId: runFriendlyId, + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_wp", + spanId: "s_wp", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_wp", + workerQueue: "main", + }); + + // Start attempt + await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + // Create and block with a waitpoint + const { waitpoint } = await engine.createDateTimeWaitpoint({ + projectId: authenticatedEnvironment.project.id, + environmentId: authenticatedEnvironment.id, + completedAfter: new Date(Date.now() + 50), + }); + + await engine.blockRunWithWaitpoint({ + runId: run.id, + waitpoints: [waitpoint.id], + projectId: authenticatedEnvironment.project.id, + organizationId: authenticatedEnvironment.organization.id, + }); + + // Wait for waitpoint completion + await setTimeout(200); + + // Get all snapshots + const allSnapshots = await prisma.taskRunExecutionSnapshot.findMany({ + where: { runId: run.id, isValid: true }, + orderBy: { createdAt: "asc" }, + }); + + expect(allSnapshots.length).toBeGreaterThanOrEqual(3); + + // Query from the first snapshot + const result = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: allSnapshots[0].id, + }); + + expect(result).not.toBeNull(); + expect(result!.length).toBeGreaterThanOrEqual(2); + + // The latest snapshot should have completedWaitpoints + const latest = result![result!.length - 1]; + expect(latest.completedWaitpoints.length).toBeGreaterThan(0); + + // Earlier snapshots should have empty waitpoints (optimization) + for (let i = 0; i < result!.length - 1; i++) { + expect(result![i].completedWaitpoints.length).toBe(0); + } + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "handles multiple waitpoints correctly - only latest has them", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const runFriendlyId = generateFriendlyId("run"); + const run = await engine.trigger( + { + number: 1, + friendlyId: runFriendlyId, + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_mwp", + spanId: "s_mwp", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_mwp", + workerQueue: "main", + }); + + await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + // Create multiple waitpoints + const waitpointCount = 5; + const waitpointPromises = Array.from({ length: waitpointCount }).map(() => + engine.createManualWaitpoint({ + environmentId: authenticatedEnvironment.id, + projectId: authenticatedEnvironment.projectId, + }) + ); + const waitpoints = await Promise.all(waitpointPromises); + + // Block the run with all waitpoints + for (const { waitpoint } of waitpoints) { + await engine.blockRunWithWaitpoint({ + runId: run.id, + waitpoints: waitpoint.id, + projectId: authenticatedEnvironment.projectId, + organizationId: authenticatedEnvironment.organizationId, + }); + } + + // Complete all waitpoints + for (const { waitpoint } of waitpoints) { + await engine.completeWaitpoint({ id: waitpoint.id }); + } + + await setTimeout(500); + + // Get all snapshots + const allSnapshots = await prisma.taskRunExecutionSnapshot.findMany({ + where: { runId: run.id, isValid: true }, + orderBy: { createdAt: "asc" }, + }); + + // Query from early in the sequence + const result = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: allSnapshots[0].id, + }); + + expect(result).not.toBeNull(); + expect(result!.length).toBeGreaterThan(0); + + // Only the latest should have waitpoints + const latest = result![result!.length - 1]; + + // Earlier snapshots must have empty completedWaitpoints + for (let i = 0; i < result!.length - 1; i++) { + expect(result![i].completedWaitpoints.length).toBe(0); + } + } finally { + await engine.quit(); + } + } + ); + + containerTest("returns null for invalid snapshot ID", async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const runFriendlyId = generateFriendlyId("run"); + const run = await engine.trigger( + { + number: 1, + friendlyId: runFriendlyId, + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_invalid", + spanId: "s_invalid", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + // Query with invalid snapshot ID + const result = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: "invalid-snapshot-id", + }); + + // Should return null (caught by getSnapshotsSince error handler) + expect(result).toBeNull(); + } finally { + await engine.quit(); + } + }); + + // Direct database tests for the core function + containerTest( + "direct test: large waitpoint scenario - 100 waitpoints with 10KB outputs", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + // Create scenario directly in database + const scenario = await setupTestScenario(prisma, authenticatedEnvironment, { + totalWaitpoints: 100, + outputSizeKB: 10, + snapshotConfigs: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 50 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "EXECUTING", completedWaitpointCount: 100 }, + { status: "FINISHED", completedWaitpointCount: 100 }, + ], + }); + + // Query from early snapshot + const result = await engine.getSnapshotsSince({ + runId: scenario.run.id, + snapshotId: scenario.snapshots[2].id, // After PENDING_EXECUTING + }); + + expect(result).not.toBeNull(); + expect(result!.length).toBe(6); // EXECUTING through FINISHED + + // Latest should have all 100 waitpoints + const latest = result![result!.length - 1]; + expect(latest.completedWaitpoints.length).toBe(100); + + // Verify all earlier snapshots have empty waitpoints + for (let i = 0; i < result!.length - 1; i++) { + expect(result![i].completedWaitpoints.length).toBe(0); + } + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "direct test: zombie run scenario - 236 waitpoints with 100KB outputs, 24 snapshots", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + // This scenario matches the exact conditions that caused the NAPI error + // 24 snapshots × 236 waitpoints × 100KB = ~570MB if not optimized + const scenario = await setupTestScenario(prisma, authenticatedEnvironment, { + totalWaitpoints: 236, + outputSizeKB: 100, + snapshotConfigs: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 200 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + ], + }); + + expect(scenario.snapshots.length).toBe(24); + expect(scenario.waitpoints.length).toBe(236); + + // Query from the 6th snapshot (after waitpoints start completing) + const queryFromIndex = 5; + const result = await engine.getSnapshotsSince({ + runId: scenario.run.id, + snapshotId: scenario.snapshots[queryFromIndex].id, + }); + + expect(result).not.toBeNull(); + // Should return snapshots after index 5, which is 24 - 6 = 18 snapshots + expect(result!.length).toBe(24 - queryFromIndex - 1); + + // Latest should have all 236 waitpoints + const latest = result![result!.length - 1]; + expect(latest.completedWaitpoints.length).toBe(236); + + // All other snapshots should have 0 waitpoints (optimization) + for (let i = 0; i < result!.length - 1; i++) { + expect(result![i].completedWaitpoints.length).toBe(0); + } + + // Verify the outputs are present and correct size + for (const wp of latest.completedWaitpoints) { + expect(wp.output).toBeDefined(); + // ~100KB output as JSON string + expect(typeof wp.output).toBe("string"); + } + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "direct test: verifies chunked fetching works with 500+ waitpoints", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + // 500 waitpoints requires 5 chunks (100 per chunk) + const scenario = await setupTestScenario(prisma, authenticatedEnvironment, { + totalWaitpoints: 500, + outputSizeKB: 10, // Smaller outputs for faster test + snapshotConfigs: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 500 }, + { status: "EXECUTING", completedWaitpointCount: 500 }, + ], + }); + + const result = await engine.getSnapshotsSince({ + runId: scenario.run.id, + snapshotId: scenario.snapshots[0].id, + }); + + expect(result).not.toBeNull(); + expect(result!.length).toBe(4); + + const latest = result![result!.length - 1]; + expect(latest.completedWaitpoints.length).toBe(500); + + // All other snapshots should be empty + for (let i = 0; i < result!.length - 1; i++) { + expect(result![i].completedWaitpoints.length).toBe(0); + } + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/helpers/executionStateMachine.ts b/internal-packages/run-engine/src/engine/tests/helpers/executionStateMachine.ts new file mode 100644 index 00000000000..4dd92cdbd41 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/executionStateMachine.ts @@ -0,0 +1,257 @@ +import { TaskRunExecutionStatus } from "@trigger.dev/database"; + +/** + * Defines valid execution status transitions for the Run Engine 2.0. + * This is a model of the state machine that governs run execution. + */ +export const EXECUTION_STATUS_TRANSITIONS: Record< + TaskRunExecutionStatus, + TaskRunExecutionStatus[] +> = { + RUN_CREATED: ["QUEUED", "DELAYED"], + DELAYED: ["QUEUED"], + QUEUED: ["PENDING_EXECUTING", "QUEUED_EXECUTING"], + QUEUED_EXECUTING: ["PENDING_EXECUTING", "QUEUED"], + PENDING_EXECUTING: ["EXECUTING", "PENDING_CANCEL", "FINISHED", "QUEUED"], + EXECUTING: ["EXECUTING_WITH_WAITPOINTS", "FINISHED", "PENDING_CANCEL", "QUEUED"], + EXECUTING_WITH_WAITPOINTS: ["EXECUTING", "SUSPENDED", "FINISHED", "PENDING_CANCEL"], + SUSPENDED: ["QUEUED", "PENDING_CANCEL", "FINISHED"], + PENDING_CANCEL: ["FINISHED"], + FINISHED: ["QUEUED"], // Retry case +}; + +/** + * Validates if a transition from one status to another is valid. + */ +export function isValidTransition( + from: TaskRunExecutionStatus, + to: TaskRunExecutionStatus +): boolean { + return EXECUTION_STATUS_TRANSITIONS[from]?.includes(to) ?? false; +} + +/** + * Configuration for a snapshot in a test scenario. + */ +export interface SnapshotConfig { + /** The execution status for this snapshot */ + status: TaskRunExecutionStatus; + /** Number of waitpoints completed at this snapshot (cumulative) */ + completedWaitpointCount: number; + /** Whether this snapshot has a checkpoint */ + hasCheckpoint?: boolean; + /** Description for the snapshot */ + description?: string; +} + +/** + * A test scenario for getSnapshotsSince testing. + */ +export interface SnapshotTestScenario { + /** Unique name for the scenario */ + name: string; + /** Description of what this scenario tests */ + description: string; + /** Total number of waitpoints to create */ + totalWaitpoints: number; + /** Size of each waitpoint's output in KB */ + outputSizeKB: number; + /** Configuration for each snapshot to create */ + snapshots: SnapshotConfig[]; + /** Which snapshot index to query "since" (0-based) */ + queryFromIndex: number; + /** Expected number of waitpoints on the latest snapshot returned */ + expectedWaitpointsOnLatest: number; +} + +/** + * Generates test scenarios for comprehensive getSnapshotsSince testing. + * These scenarios cover various edge cases and stress tests. + */ +export function generateTestScenarios(): SnapshotTestScenario[] { + return [ + { + name: "simple_no_waitpoints", + description: "Basic run without any waitpoints", + totalWaitpoints: 0, + outputSizeKB: 0, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "FINISHED", completedWaitpointCount: 0 }, + ], + queryFromIndex: 0, + expectedWaitpointsOnLatest: 0, + }, + { + name: "single_small_waitpoint", + description: "Single waitpoint with small output", + totalWaitpoints: 1, + outputSizeKB: 1, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 1 }, + ], + queryFromIndex: 2, + expectedWaitpointsOnLatest: 1, + }, + { + name: "batch_100_medium", + description: "Medium batch with 100 waitpoints and medium outputs", + totalWaitpoints: 100, + outputSizeKB: 10, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 50 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "SUSPENDED", completedWaitpointCount: 100, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 100 }, + { status: "EXECUTING", completedWaitpointCount: 100 }, + { status: "FINISHED", completedWaitpointCount: 100 }, + ], + queryFromIndex: 3, + expectedWaitpointsOnLatest: 100, + }, + { + name: "batch_236_large_zombie_scenario", + description: + "Matches the zombie run scenario: 24 snapshots, 236 waitpoints, 100KB outputs each", + totalWaitpoints: 236, + outputSizeKB: 100, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 50 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 150 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 200 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + ], + queryFromIndex: 6, + expectedWaitpointsOnLatest: 236, + }, + { + name: "batch_500_large", + description: "Large batch requiring chunked fetching", + totalWaitpoints: 500, + outputSizeKB: 50, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 250 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 400 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 500 }, + { status: "SUSPENDED", completedWaitpointCount: 500, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 500 }, + { status: "EXECUTING", completedWaitpointCount: 500 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 500 }, + { status: "SUSPENDED", completedWaitpointCount: 500, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 500 }, + { status: "EXECUTING", completedWaitpointCount: 500 }, + ], + queryFromIndex: 5, + expectedWaitpointsOnLatest: 500, + }, + { + name: "system_failure_finished", + description: "Latest snapshot is FINISHED status with completed waitpoints", + totalWaitpoints: 100, + outputSizeKB: 50, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 50 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "EXECUTING", completedWaitpointCount: 100 }, + { status: "FINISHED", completedWaitpointCount: 100 }, + ], + queryFromIndex: 3, + expectedWaitpointsOnLatest: 100, + }, + { + name: "query_from_latest", + description: "Querying from the latest snapshot should return empty array", + totalWaitpoints: 10, + outputSizeKB: 10, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 10 }, + ], + queryFromIndex: 4, // The last snapshot + expectedWaitpointsOnLatest: 0, // No snapshots returned, so no waitpoints + }, + { + name: "requeue_loop", + description: "Multiple QUEUED->PENDING_EXECUTING cycles with waitpoints", + totalWaitpoints: 236, + outputSizeKB: 100, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "QUEUED", completedWaitpointCount: 236 }, // Requeued + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "QUEUED", completedWaitpointCount: 236 }, // Requeued again + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "QUEUED", completedWaitpointCount: 236 }, // Requeued + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "QUEUED", completedWaitpointCount: 236 }, // Requeued again + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "QUEUED", completedWaitpointCount: 236 }, // Requeued + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + ], + queryFromIndex: 7, + expectedWaitpointsOnLatest: 236, + }, + ]; +} diff --git a/internal-packages/run-engine/src/engine/tests/helpers/snapshotTestHelpers.ts b/internal-packages/run-engine/src/engine/tests/helpers/snapshotTestHelpers.ts new file mode 100644 index 00000000000..f981f35145f --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/snapshotTestHelpers.ts @@ -0,0 +1,322 @@ +import { generateFriendlyId, WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { + PrismaClient, + TaskRunExecutionSnapshot, + TaskRunExecutionStatus, + Waitpoint, + WaitpointStatus, +} from "@trigger.dev/database"; +import type { AuthenticatedEnvironment } from "../setup.js"; + +/** + * Generates a large output string of the specified size in KB. + * The output is a valid JSON string to simulate realistic waitpoint output. + */ +export function generateLargeOutput(sizeKB: number): string { + if (sizeKB <= 0) return JSON.stringify({ data: "" }); + + // Create a string that's approximately the target size + // Account for JSON wrapper overhead + const targetBytes = sizeKB * 1024; + const overhead = JSON.stringify({ data: "" }).length; + const payloadSize = Math.max(0, targetBytes - overhead); + + // Generate a payload of repeating 'x' characters + const payload = "x".repeat(payloadSize); + return JSON.stringify({ data: payload }); +} + +/** + * Creates waitpoints with specified output sizes for testing. + */ +export async function createWaitpointsWithOutput( + prisma: PrismaClient, + count: number, + outputSizeKB: number, + environmentId: string, + projectId: string +): Promise { + if (count === 0) return []; + + const output = generateLargeOutput(outputSizeKB); + const waitpoints: Waitpoint[] = []; + + // Create waitpoints in batches to avoid overwhelming the database + const batchSize = 50; + for (let i = 0; i < count; i += batchSize) { + const batchCount = Math.min(batchSize, count - i); + const batch = await Promise.all( + Array.from({ length: batchCount }).map(async (_, j) => { + const waitpointIds = WaitpointId.generate(); + return prisma.waitpoint.create({ + data: { + id: waitpointIds.id, + friendlyId: waitpointIds.friendlyId, + type: "MANUAL", + status: "COMPLETED" as WaitpointStatus, + idempotencyKey: `test-idempotency-${waitpointIds.id}`, + userProvidedIdempotencyKey: false, + completedAt: new Date(), + output, + outputType: "application/json", + outputIsError: false, + environmentId, + projectId, + }, + }); + }) + ); + waitpoints.push(...batch); + } + + return waitpoints; +} + +/** + * Creates a snapshot directly in the database for testing purposes. + * This bypasses the normal engine flow to allow creating specific test scenarios. + */ +export async function createTestSnapshot( + prisma: PrismaClient, + { + runId, + status, + environmentId, + environmentType, + projectId, + organizationId, + completedWaitpointIds, + checkpointId, + previousSnapshotId, + batchId, + workerId, + runnerId, + attemptNumber, + }: { + runId: string; + status: TaskRunExecutionStatus; + environmentId: string; + environmentType: "PRODUCTION" | "STAGING" | "DEVELOPMENT" | "PREVIEW"; + projectId: string; + organizationId: string; + completedWaitpointIds?: string[]; + checkpointId?: string; + previousSnapshotId?: string; + batchId?: string; + workerId?: string; + runnerId?: string; + attemptNumber?: number; + } +): Promise { + // Determine run status based on execution status + const runStatus = getRunStatusFromExecutionStatus(status); + + const snapshot = await prisma.taskRunExecutionSnapshot.create({ + data: { + engine: "V2", + executionStatus: status, + description: `Test snapshot: ${status}`, + previousSnapshotId, + runId, + runStatus, + attemptNumber, + batchId, + environmentId, + environmentType, + projectId, + organizationId, + checkpointId, + workerId, + runnerId, + isValid: true, + completedWaitpoints: completedWaitpointIds + ? { + connect: completedWaitpointIds.map((id) => ({ id })), + } + : undefined, + completedWaitpointOrder: completedWaitpointIds ?? [], + }, + }); + + // Small delay to ensure different createdAt timestamps + await new Promise((resolve) => setTimeout(resolve, 5)); + + return snapshot; +} + +/** + * Maps execution status to run status for test snapshot creation. + */ +function getRunStatusFromExecutionStatus( + status: TaskRunExecutionStatus +): "PENDING" | "EXECUTING" | "WAITING_FOR_DEPLOY" | "COMPLETED_SUCCESSFULLY" | "SYSTEM_FAILURE" { + switch (status) { + case "RUN_CREATED": + case "QUEUED": + case "QUEUED_EXECUTING": + case "PENDING_EXECUTING": + case "DELAYED": + return "PENDING"; + case "EXECUTING": + case "EXECUTING_WITH_WAITPOINTS": + case "SUSPENDED": + case "PENDING_CANCEL": + return "EXECUTING"; + case "FINISHED": + return "COMPLETED_SUCCESSFULLY"; + default: + return "PENDING"; + } +} + +/** + * Creates a checkpoint for testing suspended snapshots. + */ +export async function createTestCheckpoint( + prisma: PrismaClient, + { + runId, + environmentId, + projectId, + }: { + runId: string; + environmentId: string; + projectId: string; + } +) { + return prisma.taskRunCheckpoint.create({ + data: { + friendlyId: generateFriendlyId("checkpoint"), + type: "DOCKER", + location: `s3://test-bucket/checkpoints/${runId}`, + imageRef: `test-image:${runId}`, + reason: "WAIT_FOR_DURATION", + runtimeEnvironment: { + connect: { id: environmentId }, + }, + project: { + connect: { id: projectId }, + }, + }, + }); +} + +/** + * Interface for a complete test scenario setup result. + */ +export interface TestScenarioResult { + run: { + id: string; + friendlyId: string; + }; + snapshots: TaskRunExecutionSnapshot[]; + waitpoints: Waitpoint[]; + checkpoints: Array<{ id: string }>; +} + +/** + * Sets up a complete test scenario with run, snapshots, waitpoints, and checkpoints. + * This creates the full database state needed for testing getSnapshotsSince. + */ +export async function setupTestScenario( + prisma: PrismaClient, + environment: AuthenticatedEnvironment, + { + totalWaitpoints, + outputSizeKB, + snapshotConfigs, + }: { + totalWaitpoints: number; + outputSizeKB: number; + snapshotConfigs: Array<{ + status: TaskRunExecutionStatus; + completedWaitpointCount: number; + hasCheckpoint?: boolean; + }>; + } +): Promise { + // Create waitpoints first + const waitpoints = await createWaitpointsWithOutput( + prisma, + totalWaitpoints, + outputSizeKB, + environment.id, + environment.project.id + ); + + // Create the run + const runFriendlyId = generateFriendlyId("run"); + const run = await prisma.taskRun.create({ + data: { + friendlyId: runFriendlyId, + engine: "V2", + status: "PENDING", + runtimeEnvironmentId: environment.id, + environmentType: environment.type, + organizationId: environment.organization.id, + projectId: environment.project.id, + taskIdentifier: "test-task", + payload: "{}", + payloadType: "application/json", + traceId: `trace_${runFriendlyId}`, + spanId: `span_${runFriendlyId}`, + context: {}, + traceContext: {}, + isTest: false, + queue: "task/test-task", + workerQueue: "main", + }, + }); + + // Create snapshots in order + const snapshots: TaskRunExecutionSnapshot[] = []; + const checkpoints: Array<{ id: string }> = []; + let previousSnapshotId: string | undefined; + let attemptNumber = 0; + + for (const config of snapshotConfigs) { + // Create checkpoint if needed + let checkpointId: string | undefined; + if (config.hasCheckpoint) { + const checkpoint = await createTestCheckpoint(prisma, { + runId: run.id, + environmentId: environment.id, + projectId: environment.project.id, + }); + checkpointId = checkpoint.id; + checkpoints.push({ id: checkpoint.id }); + } + + // Increment attempt number when entering a new execution attempt + // PENDING_EXECUTING is the entry point - EXECUTING follows within the same attempt + if (config.status === "PENDING_EXECUTING") { + attemptNumber++; + } + + // Get the waitpoint IDs that should be "completed" at this snapshot + const completedWaitpointIds = waitpoints.slice(0, config.completedWaitpointCount).map((w) => w.id); + + const snapshot = await createTestSnapshot(prisma, { + runId: run.id, + status: config.status, + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.project.id, + organizationId: environment.organization.id, + completedWaitpointIds, + checkpointId, + previousSnapshotId, + attemptNumber, + }); + + snapshots.push(snapshot); + previousSnapshotId = snapshot.id; + } + + return { + run: { id: run.id, friendlyId: runFriendlyId }, + snapshots, + waitpoints, + checkpoints, + }; +} From 01208fde2750701016afe7170a4bf9197d1f4d58 Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Fri, 30 Jan 2026 08:43:23 +0000 Subject: [PATCH 013/400] chore(docs): Add playwright workaround (#2973) Adds a workaround to playwright to browser download failures based on this GH issue: https://github.com/triggerdotdev/trigger.dev/issues/2440 --- --- docs/config/extensions/playwright.mdx | 26 +++ docs/docs.json | 8 + docs/management/deployments/get-latest.mdx | 4 + docs/management/deployments/promote.mdx | 4 + docs/management/deployments/retrieve.mdx | 4 + docs/v3-openapi.yaml | 228 +++++++++++++++++++++ 6 files changed, 274 insertions(+) create mode 100644 docs/management/deployments/get-latest.mdx create mode 100644 docs/management/deployments/promote.mdx create mode 100644 docs/management/deployments/retrieve.mdx diff --git a/docs/config/extensions/playwright.mdx b/docs/config/extensions/playwright.mdx index db9ad342573..ea8461301e6 100644 --- a/docs/config/extensions/playwright.mdx +++ b/docs/config/extensions/playwright.mdx @@ -91,6 +91,32 @@ The extension sets the following environment variables during the build: - `PLAYWRIGHT_SKIP_BROWSER_VALIDATION`: Set to `1` to skip browser validation at runtime - `DISPLAY`: Set to `:99` if `headless: false` (for Xvfb) +## Troubleshooting + +### Browser download failures + +If you encounter errors during the build process related to browser downloads (e.g., "failed to solve: process did not complete successfully: exit code: 9"), this is a known issue with certain Playwright versions. + +**Workaround:** Revert Playwright to version `1.40.0` in your project dependencies. You can specify this version explicitly in your config: + +```ts +import { defineConfig } from "@trigger.dev/sdk"; +import { playwright } from "@trigger.dev/build/extensions/playwright"; + +export default defineConfig({ + project: "", + build: { + extensions: [ + playwright({ + version: "1.40.0", + }), + ], + }, +}); +``` + +For more details, see [GitHub issue #2440](https://github.com/triggerdotdev/trigger.dev/issues/2440#issuecomment-3815104376). + ## Managing browser instances To prevent issues with waits and resumes, you can use middleware and locals to manage the browser instance. This will ensure the browser is available for the whole run, and is properly cleaned up on waits, resumes, and after the run completes. diff --git a/docs/docs.json b/docs/docs.json index dcf637aea2f..c1f5d273804 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -276,6 +276,14 @@ "management/envvars/update", "management/envvars/delete" ] + }, + { + "group": "Deployments API", + "pages": [ + "management/deployments/retrieve", + "management/deployments/get-latest", + "management/deployments/promote" + ] } ] }, diff --git a/docs/management/deployments/get-latest.mdx b/docs/management/deployments/get-latest.mdx new file mode 100644 index 00000000000..78be92be293 --- /dev/null +++ b/docs/management/deployments/get-latest.mdx @@ -0,0 +1,4 @@ +--- +title: "Get latest deployment" +openapi: "v3-openapi GET /api/v1/deployments/latest" +--- diff --git a/docs/management/deployments/promote.mdx b/docs/management/deployments/promote.mdx new file mode 100644 index 00000000000..e1e885c0d75 --- /dev/null +++ b/docs/management/deployments/promote.mdx @@ -0,0 +1,4 @@ +--- +title: "Promote deployment" +openapi: "v3-openapi POST /api/v1/deployments/{version}/promote" +--- diff --git a/docs/management/deployments/retrieve.mdx b/docs/management/deployments/retrieve.mdx new file mode 100644 index 00000000000..8a7eb215572 --- /dev/null +++ b/docs/management/deployments/retrieve.mdx @@ -0,0 +1,4 @@ +--- +title: "Get deployment" +openapi: "v3-openapi GET /api/v1/deployments/{deploymentId}" +--- diff --git a/docs/v3-openapi.yaml b/docs/v3-openapi.yaml index d406ce6c93b..2fdcd0afd9d 100644 --- a/docs/v3-openapi.yaml +++ b/docs/v3-openapi.yaml @@ -505,6 +505,234 @@ paths: await runs.cancel("run_1234"); + "/api/v1/deployments/{deploymentId}": + parameters: + - in: path + name: deploymentId + required: true + schema: + type: string + description: The deployment ID. + get: + operationId: get_deployment_v1 + summary: Get deployment + description: Retrieve information about a specific deployment by its ID. + responses: + "200": + description: Successful request + content: + application/json: + schema: + type: object + properties: + id: + type: string + description: The deployment ID + status: + type: string + enum: ["PENDING", "INSTALLING", "BUILDING", "DEPLOYING", "DEPLOYED", "FAILED", "CANCELED", "TIMED_OUT"] + description: The current status of the deployment + contentHash: + type: string + description: Hash of the deployment content + shortCode: + type: string + description: The short code for the deployment + version: + type: string + description: The deployment version (e.g., "20250228.1") + imageReference: + type: string + nullable: true + description: Reference to the deployment image + imagePlatform: + type: string + description: Platform of the deployment image + externalBuildData: + type: object + nullable: true + description: External build data if applicable + errorData: + type: object + nullable: true + description: Error data if the deployment failed + worker: + type: object + nullable: true + description: Worker information if available + properties: + id: + type: string + version: + type: string + tasks: + type: array + items: + type: object + properties: + id: + type: string + slug: + type: string + filePath: + type: string + exportName: + type: string + "401": + description: Unauthorized - Access token is missing or invalid + "404": + description: Deployment not found + tags: + - deployments + security: + - secretKey: [] + x-codeSamples: + - lang: typescript + source: |- + const response = await fetch( + `https://api.trigger.dev/api/v1/deployments/${deploymentId}`, + { + method: "GET", + headers: { + "Authorization": `Bearer ${secretKey}`, + }, + } + ); + const deployment = await response.json(); + - lang: curl + source: |- + curl -X GET "https://api.trigger.dev/api/v1/deployments/deployment_1234" \ + -H "Authorization: Bearer tr_dev_1234" + + "/api/v1/deployments/latest": + get: + operationId: get_latest_deployment_v1 + summary: Get latest deployment + description: Retrieve information about the latest unmanaged deployment for the authenticated project. + responses: + "200": + description: Successful request + content: + application/json: + schema: + type: object + properties: + id: + type: string + description: The deployment ID + status: + type: string + enum: ["PENDING", "INSTALLING", "BUILDING", "DEPLOYING", "DEPLOYED", "FAILED", "CANCELED", "TIMED_OUT"] + description: The current status of the deployment + contentHash: + type: string + description: Hash of the deployment content + shortCode: + type: string + description: The short code for the deployment + version: + type: string + description: The deployment version (e.g., "20250228.1") + imageReference: + type: string + nullable: true + description: Reference to the deployment image + errorData: + type: object + nullable: true + description: Error data if the deployment failed + "401": + description: Unauthorized - API key is missing or invalid + "404": + description: No deployment found + tags: + - deployments + security: + - secretKey: [] + x-codeSamples: + - lang: typescript + source: |- + const response = await fetch( + "https://api.trigger.dev/api/v1/deployments/latest", + { + method: "GET", + headers: { + "Authorization": `Bearer ${secretKey}`, + }, + } + ); + const deployment = await response.json(); + - lang: curl + source: |- + curl -X GET "https://api.trigger.dev/api/v1/deployments/latest" \ + -H "Authorization: Bearer tr_dev_1234" + + "/api/v1/deployments/{version}/promote": + parameters: + - in: path + name: version + required: true + schema: + type: string + description: The deployment version to promote (e.g., "20250228.1"). + post: + operationId: promote_deployment_v1 + summary: Promote deployment + description: Promote a previously deployed version to be the current version for the environment. This makes the specified version active for new task runs. + responses: + "200": + description: Deployment promoted successfully + content: + application/json: + schema: + type: object + properties: + id: + type: string + description: The deployment ID + version: + type: string + description: The deployment version (e.g., "20250228.1") + shortCode: + type: string + description: The short code for the deployment + "400": + description: Invalid request + content: + application/json: + schema: + type: object + properties: + error: + type: string + "401": + description: Unauthorized - API key is missing or invalid + "404": + description: Deployment not found + tags: + - deployments + security: + - secretKey: [] + x-codeSamples: + - lang: typescript + source: |- + const response = await fetch( + `https://api.trigger.dev/api/v1/deployments/${version}/promote`, + { + method: "POST", + headers: { + "Authorization": `Bearer ${secretKey}`, + "Content-Type": "application/json", + }, + } + ); + const result = await response.json(); + - lang: curl + source: |- + curl -X POST "https://api.trigger.dev/api/v1/deployments/20250228.1/promote" \ + -H "Authorization: Bearer tr_dev_1234" \ + -H "Content-Type: application/json" + "/api/v1/runs/{runId}/reschedule": parameters: - $ref: "#/components/parameters/runId" From 3925f8cc49c0316b7ffdcf255433c208987107b4 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 30 Jan 2026 09:15:02 +0000 Subject: [PATCH 014/400] fix(core): vendor superjson to fix ESM/CJS compatibility (#2949) Bundle superjson and its dependency (copy-anything) during build to avoid ERR_REQUIRE_ESM errors on Node.js versions that don't support require(ESM) by default (< 22.12.0) and AWS Lambda which intentionally disables it. - Add scripts/bundle-superjson.mjs to bundle superjson with esbuild - Update build script to bundle vendor files before tshy compilation - Move superjson from dependencies to devDependencies - Update imports to use vendored bundles Fixes #2937 --- --------- Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: Eric Allam --- .changeset/vendor-superjson-esm-fix.md | 7 + .github/workflows/pr_checks.yml | 4 + .github/workflows/sdk-compat.yml | 178 +++++++ .gitignore | 3 + .../sdk-compat-tests/package.json | 20 + .../src/fixtures/bun/package.json | 5 + .../sdk-compat-tests/src/fixtures/bun/test.ts | 62 +++ .../src/fixtures/cjs-require/package.json | 4 + .../src/fixtures/cjs-require/test.cjs | 57 +++ .../fixtures/cloudflare-worker/package.json | 11 + .../fixtures/cloudflare-worker/src/index.ts | 41 ++ .../fixtures/cloudflare-worker/wrangler.toml | 4 + .../src/fixtures/deno/deno.json | 6 + .../src/fixtures/deno/test.ts | 65 +++ .../src/fixtures/esm-import/package.json | 5 + .../fixtures/esm-import/superjson-test.mjs | 59 +++ .../src/fixtures/esm-import/test.mjs | 54 ++ .../src/fixtures/typescript/package.json | 5 + .../src/fixtures/typescript/test.ts | 74 +++ .../src/fixtures/typescript/tsconfig.json | 12 + .../src/tests/bundler.test.ts | 117 +++++ .../sdk-compat-tests/src/tests/import.test.ts | 84 ++++ .../sdk-compat-tests/tsconfig.json | 16 + .../sdk-compat-tests/vitest.config.ts | 11 + packages/core/package.json | 12 +- packages/core/scripts/bundle-superjson.mjs | 93 ++++ .../core/src/v3/imports/superjson-cjs.cts | 8 +- packages/core/src/v3/imports/superjson.ts | 10 +- pnpm-lock.yaml | 464 +++++++++++++++++- 29 files changed, 1463 insertions(+), 28 deletions(-) create mode 100644 .changeset/vendor-superjson-esm-fix.md create mode 100644 .github/workflows/sdk-compat.yml create mode 100644 internal-packages/sdk-compat-tests/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/bun/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/bun/test.ts create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/cjs-require/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/cjs-require/test.cjs create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/src/index.ts create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/wrangler.toml create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/deno/deno.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/deno/test.ts create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/esm-import/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/esm-import/superjson-test.mjs create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/esm-import/test.mjs create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/typescript/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/typescript/test.ts create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json create mode 100644 internal-packages/sdk-compat-tests/src/tests/bundler.test.ts create mode 100644 internal-packages/sdk-compat-tests/src/tests/import.test.ts create mode 100644 internal-packages/sdk-compat-tests/tsconfig.json create mode 100644 internal-packages/sdk-compat-tests/vitest.config.ts create mode 100644 packages/core/scripts/bundle-superjson.mjs diff --git a/.changeset/vendor-superjson-esm-fix.md b/.changeset/vendor-superjson-esm-fix.md new file mode 100644 index 00000000000..ef04201d2c9 --- /dev/null +++ b/.changeset/vendor-superjson-esm-fix.md @@ -0,0 +1,7 @@ +--- +"@trigger.dev/core": patch +--- + +fix: vendor superjson to fix ESM/CJS compatibility + +Bundle superjson during build to avoid `ERR_REQUIRE_ESM` errors on Node.js versions that don't support `require(ESM)` by default (< 22.12.0) and AWS Lambda which intentionally disables it. diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index b6be1eddfa1..dab18223e35 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -29,3 +29,7 @@ jobs: with: package: cli-v3 secrets: inherit + + sdk-compat: + uses: ./.github/workflows/sdk-compat.yml + secrets: inherit diff --git a/.github/workflows/sdk-compat.yml b/.github/workflows/sdk-compat.yml new file mode 100644 index 00000000000..eb347c0f771 --- /dev/null +++ b/.github/workflows/sdk-compat.yml @@ -0,0 +1,178 @@ +name: "🔌 SDK Compatibility Tests" + +permissions: + contents: read + +on: + workflow_call: + +jobs: + node-compat: + name: "Node.js ${{ matrix.node }} (${{ matrix.os }})" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + node: ["20.20", "22.12"] + + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.23.0 + + - name: ⎔ Setup node + uses: buildjet/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: "pnpm" + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 📀 Generate Prisma Client + run: pnpm run generate + + - name: 🔨 Build SDK dependencies + shell: bash + run: pnpm run build --filter '@trigger.dev/sdk^...' + + - name: 🔨 Build SDK + shell: bash + run: pnpm run build --filter '@trigger.dev/sdk' + + - name: 🧪 Run SDK Compatibility Tests + shell: bash + run: pnpm --filter @internal/sdk-compat-tests test + + bun-compat: + name: "Bun Runtime" + runs-on: ubuntu-latest + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.23.0 + + - name: ⎔ Setup node + uses: buildjet/setup-node@v4 + with: + node-version: 20.20.0 + cache: "pnpm" + + - name: 🥟 Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 📀 Generate Prisma Client + run: pnpm run generate + + - name: 🔨 Build SDK dependencies + run: pnpm run build --filter @trigger.dev/sdk^... + + - name: 🔨 Build SDK + run: pnpm run build --filter @trigger.dev/sdk + + - name: 🧪 Run Bun Compatibility Test + working-directory: internal-packages/sdk-compat-tests/src/fixtures/bun + run: bun run test.ts + + deno-compat: + name: "Deno Runtime" + runs-on: ubuntu-latest + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.23.0 + + - name: ⎔ Setup node + uses: buildjet/setup-node@v4 + with: + node-version: 20.20.0 + cache: "pnpm" + + - name: 🦕 Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 📀 Generate Prisma Client + run: pnpm run generate + + - name: 🔨 Build SDK dependencies + run: pnpm run build --filter @trigger.dev/sdk^... + + - name: 🔨 Build SDK + run: pnpm run build --filter @trigger.dev/sdk + + - name: 🔗 Link node_modules for Deno fixture + working-directory: internal-packages/sdk-compat-tests/src/fixtures/deno + run: ln -s ../../../../../node_modules node_modules + + - name: 🧪 Run Deno Compatibility Test + working-directory: internal-packages/sdk-compat-tests/src/fixtures/deno + run: deno run --allow-read --allow-env --allow-sys test.ts + + cloudflare-compat: + name: "Cloudflare Workers" + runs-on: ubuntu-latest + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.23.0 + + - name: ⎔ Setup node + uses: buildjet/setup-node@v4 + with: + node-version: 20.20.0 + cache: "pnpm" + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 📀 Generate Prisma Client + run: pnpm run generate + + - name: 🔨 Build SDK dependencies + run: pnpm run build --filter @trigger.dev/sdk^... + + - name: 🔨 Build SDK + run: pnpm run build --filter @trigger.dev/sdk + + - name: 📥 Install Cloudflare fixture deps + working-directory: internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker + run: pnpm install + + - name: 🧪 Run Cloudflare Workers Compatibility Test (dry-run) + working-directory: internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker + run: npx wrangler deploy --dry-run --outdir dist diff --git a/.gitignore b/.gitignore index d0dfea89c5a..071b9b59035 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ out/ dist packages/**/dist +# vendored bundles (generated during build) +packages/**/src/**/vendor + # Tailwind apps/**/styles/tailwind.css packages/**/styles/tailwind.css diff --git a/internal-packages/sdk-compat-tests/package.json b/internal-packages/sdk-compat-tests/package.json new file mode 100644 index 00000000000..e903e69f3f1 --- /dev/null +++ b/internal-packages/sdk-compat-tests/package.json @@ -0,0 +1,20 @@ +{ + "name": "@internal/sdk-compat-tests", + "private": true, + "version": "0.0.1", + "type": "module", + "scripts": { + "test": "vitest", + "test:watch": "vitest", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@trigger.dev/sdk": "workspace:*" + }, + "devDependencies": { + "esbuild": "^0.24.0", + "execa": "^9.3.0", + "typescript": "^5.5.0", + "vitest": "3.1.4" + } +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/bun/package.json b/internal-packages/sdk-compat-tests/src/fixtures/bun/package.json new file mode 100644 index 00000000000..c69e2dd23e3 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/bun/package.json @@ -0,0 +1,5 @@ +{ + "name": "bun-fixture", + "private": true, + "type": "module" +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/bun/test.ts b/internal-packages/sdk-compat-tests/src/fixtures/bun/test.ts new file mode 100644 index 00000000000..853869304f4 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/bun/test.ts @@ -0,0 +1,62 @@ +/** + * Bun Import Test Fixture + * + * Tests that the SDK works correctly with Bun runtime. + * Bun has high Node.js compatibility but uses its own module resolver. + */ + +import { task, logger, schedules, runs, configure, queue, retry, wait } from "@trigger.dev/sdk"; + +// Validate exports exist +const checks: [string, boolean][] = [ + ["task", typeof task === "function"], + ["logger", typeof logger === "object" && typeof logger.info === "function"], + ["schedules", typeof schedules === "object"], + ["runs", typeof runs === "object"], + ["configure", typeof configure === "function"], + ["queue", typeof queue === "function"], + ["retry", typeof retry === "object"], + ["wait", typeof wait === "object"], +]; + +let failed = false; +for (const [name, passed] of checks) { + if (!passed) { + console.error(`FAIL: ${name} export check failed`); + failed = true; + } +} + +// Test task definition with types +interface Payload { + message: string; +} + +const myTask = task({ + id: "bun-test-task", + run: async (payload: Payload) => { + return { received: payload.message }; + }, +}); + +if (myTask.id !== "bun-test-task") { + console.error(`FAIL: task.id mismatch`); + failed = true; +} + +// Test queue definition +const myQueue = queue({ + name: "bun-test-queue", + concurrencyLimit: 5, +}); + +if (!myQueue) { + console.error(`FAIL: queue creation failed`); + failed = true; +} + +if (failed) { + process.exit(1); +} + +console.log("SUCCESS: Bun imports validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/package.json b/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/package.json new file mode 100644 index 00000000000..953ed7d2db7 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/package.json @@ -0,0 +1,4 @@ +{ + "name": "cjs-require-fixture", + "private": true +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/test.cjs b/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/test.cjs new file mode 100644 index 00000000000..447d03970aa --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/test.cjs @@ -0,0 +1,57 @@ +/** + * CJS Require Test Fixture + * + * This file validates that the SDK can be required using CommonJS syntax. + * This is critical for: + * - Node.js < 22.12.0 (where require(ESM) is not enabled by default) + * - AWS Lambda (intentionally disables require(ESM)) + * - Legacy Node.js applications + */ + +// Test main export +const sdk = require("@trigger.dev/sdk"); + +// Test /v3 subpath +const sdkV3 = require("@trigger.dev/sdk/v3"); + +// Validate exports exist +const checks = [ + ["task", typeof sdk.task === "function"], + ["taskV3", typeof sdkV3.task === "function"], + ["logger", typeof sdk.logger === "object" && typeof sdk.logger.info === "function"], + ["schedules", typeof sdk.schedules === "object"], + ["runs", typeof sdk.runs === "object"], + ["configure", typeof sdk.configure === "function"], + ["queue", typeof sdk.queue === "function"], + ["retry", typeof sdk.retry === "object"], + ["wait", typeof sdk.wait === "object"], + ["metadata", typeof sdk.metadata === "object"], + ["tags", typeof sdk.tags === "object"], +]; + +let failed = false; +for (const [name, passed] of checks) { + if (!passed) { + console.error(`FAIL: ${name} export check failed`); + failed = true; + } +} + +// Test task definition works +const myTask = sdk.task({ + id: "cjs-test-task", + run: async (payload) => { + return { received: payload }; + }, +}); + +if (myTask.id !== "cjs-test-task") { + console.error(`FAIL: task.id mismatch: expected "cjs-test-task", got "${myTask.id}"`); + failed = true; +} + +if (failed) { + process.exit(1); +} + +console.log("SUCCESS: All CJS requires validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/package.json b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/package.json new file mode 100644 index 00000000000..d9fca987c33 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/package.json @@ -0,0 +1,11 @@ +{ + "name": "cloudflare-worker-fixture", + "private": true, + "type": "module", + "scripts": { + "build": "wrangler deploy --dry-run --outdir dist" + }, + "devDependencies": { + "wrangler": "^3.0.0" + } +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/src/index.ts b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/src/index.ts new file mode 100644 index 00000000000..30b5fcc79ef --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/src/index.ts @@ -0,0 +1,41 @@ +/** + * Cloudflare Worker Test Fixture + * + * Tests that the SDK can be bundled for Cloudflare Workers (workerd runtime). + * This validates the bundling process works - actual execution would require + * a Trigger.dev API connection. + */ + +import { task, runs, configure } from "@trigger.dev/sdk"; + +// Define a task (won't execute in worker, but validates import) +const myTask = task({ + id: "cloudflare-test-task", + run: async (payload: { message: string }) => { + return { received: payload.message }; + }, +}); + +export default { + async fetch(request: Request, env: unknown, ctx: ExecutionContext): Promise { + // Validate SDK imports work + const checks = { + taskDefined: typeof task === "function", + runsDefined: typeof runs === "object", + configureDefined: typeof configure === "function", + taskIdCorrect: myTask.id === "cloudflare-test-task", + }; + + const allPassed = Object.values(checks).every((v) => v === true); + + return new Response( + JSON.stringify({ + success: allPassed, + checks, + }), + { + headers: { "Content-Type": "application/json" }, + } + ); + }, +}; diff --git a/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/wrangler.toml b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/wrangler.toml new file mode 100644 index 00000000000..f038e47bb6f --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/wrangler.toml @@ -0,0 +1,4 @@ +name = "sdk-compat-test" +main = "src/index.ts" +compatibility_date = "2024-01-01" +compatibility_flags = ["nodejs_compat"] diff --git a/internal-packages/sdk-compat-tests/src/fixtures/deno/deno.json b/internal-packages/sdk-compat-tests/src/fixtures/deno/deno.json new file mode 100644 index 00000000000..4525b34d3e2 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/deno/deno.json @@ -0,0 +1,6 @@ +{ + "tasks": { + "test": "deno run --allow-read --allow-env --allow-sys test.ts" + }, + "nodeModulesDir": "manual" +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/deno/test.ts b/internal-packages/sdk-compat-tests/src/fixtures/deno/test.ts new file mode 100644 index 00000000000..6894606fd0a --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/deno/test.ts @@ -0,0 +1,65 @@ +/** + * Deno Import Test Fixture + * + * Tests that the SDK can be imported in Deno using Node.js compatibility. + * The CI workflow installs the SDK into node_modules via npm for local resolution. + */ + +// Use bare specifier - resolved via node_modules when nodeModulesDir is enabled +import { task, logger, schedules, runs, configure, queue, retry, wait, metadata, tags } from "@trigger.dev/sdk"; + +// Validate exports exist +const checks: [string, boolean][] = [ + ["task", typeof task === "function"], + ["logger", typeof logger === "object" && typeof logger.info === "function"], + ["schedules", typeof schedules === "object"], + ["runs", typeof runs === "object"], + ["configure", typeof configure === "function"], + ["queue", typeof queue === "function"], + ["retry", typeof retry === "object"], + ["wait", typeof wait === "object"], + ["metadata", typeof metadata === "object"], + ["tags", typeof tags === "object"], +]; + +let failed = false; +for (const [name, passed] of checks) { + if (!passed) { + console.error(`FAIL: ${name} export check failed`); + failed = true; + } +} + +// Test task definition with types +interface Payload { + message: string; +} + +const myTask = task({ + id: "deno-test-task", + run: async (payload: Payload) => { + return { received: payload.message }; + }, +}); + +if (myTask.id !== "deno-test-task") { + console.error(`FAIL: task.id mismatch`); + failed = true; +} + +// Test queue definition +const myQueue = queue({ + name: "deno-test-queue", + concurrencyLimit: 5, +}); + +if (!myQueue) { + console.error(`FAIL: queue creation failed`); + failed = true; +} + +if (failed) { + Deno.exit(1); +} + +console.log("SUCCESS: Deno imports validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/esm-import/package.json b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/package.json new file mode 100644 index 00000000000..c0d56cd02fa --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/package.json @@ -0,0 +1,5 @@ +{ + "name": "esm-import-fixture", + "private": true, + "type": "module" +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/esm-import/superjson-test.mjs b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/superjson-test.mjs new file mode 100644 index 00000000000..fc034de19e0 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/superjson-test.mjs @@ -0,0 +1,59 @@ +/** + * SuperJSON Serialization Test + * + * This validates the fix for #2937 - ESM/CJS compatibility with superjson. + * Tests that complex types (Date, Set, Map, BigInt) serialize correctly. + */ + +import { task, logger } from "@trigger.dev/sdk"; + +// The SDK uses superjson internally for serialization +// This test ensures the vendored superjson works correctly + +const complexData = { + date: new Date("2024-01-15T12:00:00Z"), + set: new Set([1, 2, 3]), + map: new Map([ + ["key1", "value1"], + ["key2", "value2"], + ]), + bigint: BigInt("9007199254740991"), + nested: { + innerDate: new Date("2024-06-01"), + innerSet: new Set(["a", "b"]), + }, +}; + +// Create a task that uses complex types +const complexTask = task({ + id: "superjson-test-task", + run: async (payload) => { + // Just verify the payload structure matches expectations + return { + hasDate: payload.date instanceof Date, + hasSet: payload.set instanceof Set, + hasMap: payload.map instanceof Map, + hasBigInt: typeof payload.bigint === "bigint", + hasNestedDate: payload.nested?.innerDate instanceof Date, + }; + }, +}); + +// Verify task was created successfully +if (!complexTask.id) { + console.error("FAIL: Task creation failed"); + process.exit(1); +} + +// Test that logger works (it uses superjson for structured logging) +try { + logger.info("Testing superjson serialization", { + complexData, + timestamp: new Date(), + }); +} catch (error) { + console.error("FAIL: Logger with complex data failed:", error); + process.exit(1); +} + +console.log("SUCCESS: SuperJSON serialization validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/esm-import/test.mjs b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/test.mjs new file mode 100644 index 00000000000..70b055aaa2f --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/test.mjs @@ -0,0 +1,54 @@ +/** + * ESM Import Test Fixture + * + * This file validates that the SDK can be imported using ESM syntax. + * It tests all major export paths and verifies runtime functionality. + */ + +// Test main export +import { task, logger, schedules, runs, configure, queue, retry, wait, metadata, tags } from "@trigger.dev/sdk"; + +// Test /v3 subpath (legacy, but should still work) +import { task as taskV3 } from "@trigger.dev/sdk/v3"; + +// Validate exports are functions/objects +const checks = [ + ["task", typeof task === "function"], + ["taskV3", typeof taskV3 === "function"], + ["logger", typeof logger === "object" && typeof logger.info === "function"], + ["schedules", typeof schedules === "object"], + ["runs", typeof runs === "object"], + ["configure", typeof configure === "function"], + ["queue", typeof queue === "function"], + ["retry", typeof retry === "object"], + ["wait", typeof wait === "object"], + ["metadata", typeof metadata === "object"], + ["tags", typeof tags === "object"], +]; + +let failed = false; +for (const [name, passed] of checks) { + if (!passed) { + console.error(`FAIL: ${name} export check failed`); + failed = true; + } +} + +// Test task definition works +const myTask = task({ + id: "esm-test-task", + run: async (payload) => { + return { received: payload }; + }, +}); + +if (myTask.id !== "esm-test-task") { + console.error(`FAIL: task.id mismatch: expected "esm-test-task", got "${myTask.id}"`); + failed = true; +} + +if (failed) { + process.exit(1); +} + +console.log("SUCCESS: All ESM imports validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/typescript/package.json b/internal-packages/sdk-compat-tests/src/fixtures/typescript/package.json new file mode 100644 index 00000000000..7663bc7aca3 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/typescript/package.json @@ -0,0 +1,5 @@ +{ + "name": "typescript-fixture", + "private": true, + "type": "module" +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/typescript/test.ts b/internal-packages/sdk-compat-tests/src/fixtures/typescript/test.ts new file mode 100644 index 00000000000..bfcb4892abe --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/typescript/test.ts @@ -0,0 +1,74 @@ +/** + * TypeScript Import Test Fixture + * + * This file validates that the SDK types work correctly with TypeScript. + * It tests type inference, generics, and type-only imports. + */ + +import { + task, + logger, + schedules, + runs, + configure, + queue, + retry, + wait, + metadata, + tags, + type Context, + type RetryOptions, +} from "@trigger.dev/sdk"; + +// Type-only import test +import type { ApiClientConfiguration } from "@trigger.dev/sdk"; + +// Test typed task with payload +interface MyPayload { + message: string; + count: number; +} + +interface MyOutput { + processed: boolean; + result: string; +} + +const typedTask = task({ + id: "typescript-test-task", + run: async (payload: MyPayload, { ctx }): Promise => { + // Verify context type + const runId: string = ctx.run.id; + + return { + processed: true, + result: `Processed ${payload.message} with count ${payload.count}`, + }; + }, +}); + +// Verify task type inference +type TaskPayload = Parameters[0]; +type _PayloadCheck = TaskPayload extends MyPayload ? true : never; + +// Test queue definition +const myQueue = queue({ + name: "test-queue", + concurrencyLimit: 10, +}); + +// Test retry options type +const retryOpts: RetryOptions = { + maxAttempts: 3, + factor: 2, + minTimeoutInMs: 1000, + maxTimeoutInMs: 30000, +}; + +// Validate runtime +if (typedTask.id !== "typescript-test-task") { + console.error(`FAIL: task.id mismatch`); + process.exit(1); +} + +console.log("SUCCESS: TypeScript types validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json b/internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json new file mode 100644 index 00000000000..432fff32ade --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["test.ts"] +} diff --git a/internal-packages/sdk-compat-tests/src/tests/bundler.test.ts b/internal-packages/sdk-compat-tests/src/tests/bundler.test.ts new file mode 100644 index 00000000000..e3e18c49f37 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/tests/bundler.test.ts @@ -0,0 +1,117 @@ +/** + * Bundler Compatibility Tests + * + * These tests validate that the SDK can be bundled correctly using + * common bundlers like esbuild. + */ + +import { describe, it, expect } from "vitest"; +import * as esbuild from "esbuild"; +import { resolve, dirname } from "path"; +import { fileURLToPath } from "url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const fixturesDir = resolve(__dirname, "../fixtures"); + +describe("esbuild Bundling Tests", () => { + it("should bundle ESM entrypoint without errors", async () => { + const result = await esbuild.build({ + entryPoints: [resolve(fixturesDir, "esm-import/test.mjs")], + bundle: true, + format: "esm", + platform: "node", + target: "node18", + write: false, + external: ["@trigger.dev/sdk", "@trigger.dev/sdk/*"], + logLevel: "silent", + }); + + expect(result.errors).toHaveLength(0); + expect(result.outputFiles).toHaveLength(1); + }); + + it("should bundle CJS entrypoint without errors", async () => { + const result = await esbuild.build({ + entryPoints: [resolve(fixturesDir, "cjs-require/test.cjs")], + bundle: true, + format: "cjs", + platform: "node", + target: "node18", + write: false, + external: ["@trigger.dev/sdk", "@trigger.dev/sdk/*"], + logLevel: "silent", + }); + + expect(result.errors).toHaveLength(0); + expect(result.outputFiles).toHaveLength(1); + }); + + it("should bundle SDK inline (simulating production build)", async () => { + // This simulates what happens when a user bundles their app with the SDK included + const entryContent = ` + import { task, logger } from "@trigger.dev/sdk"; + + export const myTask = task({ + id: "bundled-task", + run: async (payload) => { + logger.info("Processing", { payload }); + return { success: true }; + }, + }); + `; + + const result = await esbuild.build({ + stdin: { + contents: entryContent, + loader: "ts", + resolveDir: resolve(__dirname, "../../"), + }, + bundle: true, + format: "esm", + platform: "node", + target: "node18", + write: false, + // Don't externalize SDK - bundle it inline + logLevel: "silent", + metafile: true, + }); + + expect(result.errors).toHaveLength(0); + expect(result.outputFiles).toHaveLength(1); + + // Verify the bundle contains the SDK code + const bundleContent = result.outputFiles[0].text; + expect(bundleContent).toBeTruthy(); + expect(bundleContent.length).toBeGreaterThan(1000); // Should be substantial + }); + + it("should handle tree-shaking correctly", async () => { + // Import only specific functions to test tree-shaking + const entryContent = ` + import { task } from "@trigger.dev/sdk"; + + export const myTask = task({ + id: "tree-shake-task", + run: async () => ({ done: true }), + }); + `; + + const result = await esbuild.build({ + stdin: { + contents: entryContent, + loader: "ts", + resolveDir: resolve(__dirname, "../../"), + }, + bundle: true, + format: "esm", + platform: "node", + target: "node18", + write: false, + treeShaking: true, + logLevel: "silent", + }); + + expect(result.errors).toHaveLength(0); + expect(result.outputFiles).toHaveLength(1); + }); +}); diff --git a/internal-packages/sdk-compat-tests/src/tests/import.test.ts b/internal-packages/sdk-compat-tests/src/tests/import.test.ts new file mode 100644 index 00000000000..7d81ccce259 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/tests/import.test.ts @@ -0,0 +1,84 @@ +/** + * Import Validation Tests + * + * These tests validate that the SDK can be imported correctly across + * different module systems (ESM and CJS). + */ + +import { describe, it, expect, beforeAll } from "vitest"; +import { execa, type Options as ExecaOptions } from "execa"; +import { resolve, dirname } from "path"; +import { fileURLToPath } from "url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const fixturesDir = resolve(__dirname, "../fixtures"); + +// Find the SDK package in the monorepo +const sdkDir = resolve(__dirname, "../../../../packages/trigger-sdk"); + +// Common execa options +const execaOpts: ExecaOptions = { + env: { + ...process.env, + // Ensure Node.js can resolve workspace packages + NODE_PATH: resolve(__dirname, "../../../../node_modules"), + }, + timeout: 30_000, +}; + +describe("ESM Import Tests", () => { + it("should import SDK using ESM syntax", async () => { + const result = await execa("node", ["test.mjs"], { + ...execaOpts, + cwd: resolve(fixturesDir, "esm-import"), + }); + + expect(result.stdout).toContain("SUCCESS"); + expect(result.exitCode).toBe(0); + }); + + it("should validate superjson serialization in ESM", async () => { + const result = await execa("node", ["superjson-test.mjs"], { + ...execaOpts, + cwd: resolve(fixturesDir, "esm-import"), + }); + + expect(result.stdout).toContain("SUCCESS"); + expect(result.exitCode).toBe(0); + }); +}); + +describe("CJS Require Tests", () => { + it("should require SDK using CommonJS syntax", async () => { + const result = await execa("node", ["test.cjs"], { + ...execaOpts, + cwd: resolve(fixturesDir, "cjs-require"), + }); + + expect(result.stdout).toContain("SUCCESS"); + expect(result.exitCode).toBe(0); + }); + + it("should work with --experimental-require-module flag on older Node", async () => { + // This flag is needed for Node < 22.12.0 to require ESM modules + // On newer Node.js, it's a no-op + const result = await execa("node", ["--experimental-require-module", "test.cjs"], { + ...execaOpts, + cwd: resolve(fixturesDir, "cjs-require"), + }); + + expect(result.stdout).toContain("SUCCESS"); + expect(result.exitCode).toBe(0); + }); +}); + +describe("TypeScript Compilation Tests", () => { + it("should typecheck SDK imports successfully", async () => { + const result = await execa("npx", ["tsc", "--noEmit"], { + ...execaOpts, + cwd: resolve(fixturesDir, "typescript"), + }); + + expect(result.exitCode).toBe(0); + }); +}); diff --git a/internal-packages/sdk-compat-tests/tsconfig.json b/internal-packages/sdk-compat-tests/tsconfig.json new file mode 100644 index 00000000000..05afb6f355a --- /dev/null +++ b/internal-packages/sdk-compat-tests/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "declaration": false, + "outDir": "dist", + "rootDir": "src", + "types": ["vitest/globals"] + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "src/fixtures"] +} diff --git a/internal-packages/sdk-compat-tests/vitest.config.ts b/internal-packages/sdk-compat-tests/vitest.config.ts new file mode 100644 index 00000000000..2617dd10185 --- /dev/null +++ b/internal-packages/sdk-compat-tests/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/tests/**/*.test.ts"], + globals: true, + isolate: true, + testTimeout: 120_000, // Some framework builds can take time + hookTimeout: 60_000, + }, +}); diff --git a/packages/core/package.json b/packages/core/package.json index 989a707eae8..d73b425f7d4 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -157,11 +157,13 @@ }, "sideEffects": false, "scripts": { - "clean": "rimraf dist .tshy .tshy-build .turbo", + "clean": "rimraf dist .tshy .tshy-build .turbo src/v3/vendor", "update-version": "tsx ../../scripts/updateVersion.ts", - "build": "tshy && pnpm run update-version", - "dev": "tshy --watch", - "typecheck": "tsc --noEmit -p tsconfig.src.json", + "bundle-vendor": "node scripts/bundle-superjson.mjs", + "build": "pnpm run bundle-vendor && tshy && node scripts/bundle-superjson.mjs --copy && pnpm run update-version", + "dev": "pnpm run bundle-vendor && tshy --watch", + "typecheck": "pnpm run bundle-vendor && tsc --noEmit -p tsconfig.src.json", + "pretest": "pnpm run bundle-vendor", "test": "vitest", "check-exports": "attw --pack ." }, @@ -193,7 +195,6 @@ "socket.io": "4.7.4", "socket.io-client": "4.7.5", "std-env": "^3.8.1", - "superjson": "^2.2.1", "tinyexec": "^0.3.2", "uncrypto": "^0.1.3", "zod": "3.25.76", @@ -212,6 +213,7 @@ "defu": "^6.1.4", "esbuild": "^0.23.0", "rimraf": "^3.0.2", + "superjson": "^2.2.1", "ts-essentials": "10.0.1", "tshy": "^3.0.2", "tsx": "4.17.0" diff --git a/packages/core/scripts/bundle-superjson.mjs b/packages/core/scripts/bundle-superjson.mjs new file mode 100644 index 00000000000..c4e9a7b0018 --- /dev/null +++ b/packages/core/scripts/bundle-superjson.mjs @@ -0,0 +1,93 @@ +#!/usr/bin/env node + +/** + * This script bundles superjson and its dependency (copy-anything) into + * vendored CJS and ESM bundles to avoid the ERR_REQUIRE_ESM error. + * + * superjson v2.x is ESM-only, which causes issues on: + * - Node.js versions before 22.12.0 (require(ESM) not enabled by default) + * - AWS Lambda (intentionally disables require(ESM)) + * + * The output files are gitignored and regenerated during each build. + * This script runs automatically as part of `pnpm run build`. + * + * Usage: + * node scripts/bundle-superjson.mjs # Bundle to src/v3/vendor + * node scripts/bundle-superjson.mjs --copy # Also copy to dist directories + */ + +import * as esbuild from "esbuild"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { readFileSync, mkdirSync, copyFileSync } from "node:fs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const packageRoot = join(__dirname, ".."); +const vendorDir = join(packageRoot, "src", "v3", "vendor"); + +// Get the installed superjson version for the banner +const superjsonPkg = JSON.parse( + readFileSync(join(packageRoot, "node_modules", "superjson", "package.json"), "utf-8") +); +const banner = `/** + * Bundled superjson v${superjsonPkg.version} + * + * This file is auto-generated by scripts/bundle-superjson.mjs + * Do not edit directly - run the script to regenerate. + * + * Original package: https://github.com/flightcontrolhq/superjson + * License: MIT + */`; + +async function bundle() { + // Ensure vendor directory exists + mkdirSync(vendorDir, { recursive: true }); + + // Bundle for CommonJS + await esbuild.build({ + entryPoints: [join(packageRoot, "node_modules", "superjson", "dist", "index.js")], + bundle: true, + format: "cjs", + platform: "node", + target: "node18", + outfile: join(vendorDir, "superjson.cjs"), + banner: { js: banner }, + // Don't minify to keep it debuggable + minify: false, + }); + + // Bundle for ESM + await esbuild.build({ + entryPoints: [join(packageRoot, "node_modules", "superjson", "dist", "index.js")], + bundle: true, + format: "esm", + platform: "node", + target: "node18", + outfile: join(vendorDir, "superjson.mjs"), + banner: { js: banner }, + minify: false, + }); + + console.log("Bundled superjson v" + superjsonPkg.version); + console.log(" -> src/v3/vendor/superjson.cjs (CommonJS)"); + console.log(" -> src/v3/vendor/superjson.mjs (ESM)"); + + // Copy to dist directories if --copy flag is passed + if (process.argv.includes("--copy")) { + const distCommonjsVendor = join(packageRoot, "dist", "commonjs", "v3", "vendor"); + const distEsmVendor = join(packageRoot, "dist", "esm", "v3", "vendor"); + + mkdirSync(distCommonjsVendor, { recursive: true }); + mkdirSync(distEsmVendor, { recursive: true }); + + copyFileSync(join(vendorDir, "superjson.cjs"), join(distCommonjsVendor, "superjson.cjs")); + copyFileSync(join(vendorDir, "superjson.mjs"), join(distEsmVendor, "superjson.mjs")); + + console.log("Copied to dist directories"); + } +} + +bundle().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/packages/core/src/v3/imports/superjson-cjs.cts b/packages/core/src/v3/imports/superjson-cjs.cts index a7f1466e7cd..89ed3d2458d 100644 --- a/packages/core/src/v3/imports/superjson-cjs.cts +++ b/packages/core/src/v3/imports/superjson-cjs.cts @@ -1,8 +1,10 @@ +// Use vendored superjson bundle to avoid ESM/CJS compatibility issues +// See: https://github.com/triggerdotdev/trigger.dev/issues/2937 // @ts-ignore -const { default: superjson } = require("superjson"); +const superjson = require("../vendor/superjson.cjs"); // @ts-ignore -superjson.registerCustom( +superjson.default.registerCustom( { isApplicable: (v: unknown): v is Buffer => typeof Buffer === "function" && Buffer.isBuffer(v), serialize: (v: Buffer) => [...v], @@ -12,4 +14,4 @@ superjson.registerCustom( ); // @ts-ignore -module.exports.default = superjson; +module.exports.default = superjson.default; diff --git a/packages/core/src/v3/imports/superjson.ts b/packages/core/src/v3/imports/superjson.ts index aa29250523a..1545c083e04 100644 --- a/packages/core/src/v3/imports/superjson.ts +++ b/packages/core/src/v3/imports/superjson.ts @@ -1,11 +1,13 @@ +// Use vendored superjson bundle to avoid ESM/CJS compatibility issues +// See: https://github.com/triggerdotdev/trigger.dev/issues/2937 // @ts-ignore -import superjson from "superjson"; +import superjson from "../vendor/superjson.mjs"; superjson.registerCustom( { - isApplicable: (v): v is Buffer => typeof Buffer === "function" && Buffer.isBuffer(v), - serialize: (v) => [...v], - deserialize: (v) => Buffer.from(v), + isApplicable: (v: unknown): v is Buffer => typeof Buffer === "function" && Buffer.isBuffer(v), + serialize: (v: Buffer) => [...v], + deserialize: (v: number[]) => Buffer.from(v), }, "buffer" ); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 525beac4790..b02c6cd7340 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1251,6 +1251,25 @@ importers: specifier: 6.0.1 version: 6.0.1 + internal-packages/sdk-compat-tests: + dependencies: + '@trigger.dev/sdk': + specifier: workspace:* + version: link:../../packages/trigger-sdk + devDependencies: + esbuild: + specifier: ^0.24.0 + version: 0.24.2 + execa: + specifier: ^9.3.0 + version: 9.6.1 + typescript: + specifier: 5.5.4 + version: 5.5.4 + vitest: + specifier: 3.1.4 + version: 3.1.4(@types/debug@4.1.12)(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + internal-packages/testcontainers: dependencies: '@clickhouse/client': @@ -1726,9 +1745,6 @@ importers: std-env: specifier: ^3.8.1 version: 3.8.1 - superjson: - specifier: ^2.2.1 - version: 2.2.1 tinyexec: specifier: ^0.3.2 version: 0.3.2 @@ -1778,6 +1794,9 @@ importers: rimraf: specifier: ^3.0.2 version: 3.0.2 + superjson: + specifier: ^2.2.1 + version: 2.2.1 ts-essentials: specifier: 10.0.1 version: 10.0.1(typescript@5.5.4) @@ -4230,6 +4249,12 @@ packages: cpu: [ppc64] os: [aix] + '@esbuild/aix-ppc64@0.24.2': + resolution: {integrity: sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + '@esbuild/aix-ppc64@0.25.1': resolution: {integrity: sha512-kfYGy8IdzTGy+z0vFGvExZtxkFlA4zAxgKEahG9KE1ScBjpQnFsNOX8KTU5ojNru5ed5CVoJYXFtoxaq5nFbjQ==} engines: {node: '>=18'} @@ -4266,6 +4291,12 @@ packages: cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.24.2': + resolution: {integrity: sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm64@0.25.1': resolution: {integrity: sha512-50tM0zCJW5kGqgG7fQ7IHvQOcAn9TKiVRuQ/lN0xR+T2lzEFvAi1ZcS8DiksFcEpf1t/GYOeOfCAgDHFpkiSmA==} engines: {node: '>=18'} @@ -4308,6 +4339,12 @@ packages: cpu: [arm] os: [android] + '@esbuild/android-arm@0.24.2': + resolution: {integrity: sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + '@esbuild/android-arm@0.25.1': resolution: {integrity: sha512-dp+MshLYux6j/JjdqVLnMglQlFu+MuVeNrmT5nk6q07wNhCdSnB7QZj+7G8VMUGh1q+vj2Bq8kRsuyA00I/k+Q==} engines: {node: '>=18'} @@ -4344,6 +4381,12 @@ packages: cpu: [x64] os: [android] + '@esbuild/android-x64@0.24.2': + resolution: {integrity: sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + '@esbuild/android-x64@0.25.1': resolution: {integrity: sha512-GCj6WfUtNldqUzYkN/ITtlhwQqGWu9S45vUXs7EIYf+7rCiiqH9bCloatO9VhxsL0Pji+PF4Lz2XXCES+Q8hDw==} engines: {node: '>=18'} @@ -4380,6 +4423,12 @@ packages: cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.24.2': + resolution: {integrity: sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-arm64@0.25.1': resolution: {integrity: sha512-5hEZKPf+nQjYoSr/elb62U19/l1mZDdqidGfmFutVUjjUZrOazAtwK+Kr+3y0C/oeJfLlxo9fXb1w7L+P7E4FQ==} engines: {node: '>=18'} @@ -4416,6 +4465,12 @@ packages: cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.24.2': + resolution: {integrity: sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + '@esbuild/darwin-x64@0.25.1': resolution: {integrity: sha512-hxVnwL2Dqs3fM1IWq8Iezh0cX7ZGdVhbTfnOy5uURtao5OIVCEyj9xIzemDi7sRvKsuSdtCAhMKarxqtlyVyfA==} engines: {node: '>=18'} @@ -4452,6 +4507,12 @@ packages: cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.24.2': + resolution: {integrity: sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-arm64@0.25.1': resolution: {integrity: sha512-1MrCZs0fZa2g8E+FUo2ipw6jw5qqQiH+tERoS5fAfKnRx6NXH31tXBKI3VpmLijLH6yriMZsxJtaXUyFt/8Y4A==} engines: {node: '>=18'} @@ -4488,6 +4549,12 @@ packages: cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.24.2': + resolution: {integrity: sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + '@esbuild/freebsd-x64@0.25.1': resolution: {integrity: sha512-0IZWLiTyz7nm0xuIs0q1Y3QWJC52R8aSXxe40VUxm6BB1RNmkODtW6LHvWRrGiICulcX7ZvyH6h5fqdLu4gkww==} engines: {node: '>=18'} @@ -4524,6 +4591,12 @@ packages: cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.24.2': + resolution: {integrity: sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm64@0.25.1': resolution: {integrity: sha512-jaN3dHi0/DDPelk0nLcXRm1q7DNJpjXy7yWaWvbfkPvI+7XNSc/lDOnCLN7gzsyzgu6qSAmgSvP9oXAhP973uQ==} engines: {node: '>=18'} @@ -4560,6 +4633,12 @@ packages: cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.24.2': + resolution: {integrity: sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + '@esbuild/linux-arm@0.25.1': resolution: {integrity: sha512-NdKOhS4u7JhDKw9G3cY6sWqFcnLITn6SqivVArbzIaf3cemShqfLGHYMx8Xlm/lBit3/5d7kXvriTUGa5YViuQ==} engines: {node: '>=18'} @@ -4596,6 +4675,12 @@ packages: cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.24.2': + resolution: {integrity: sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-ia32@0.25.1': resolution: {integrity: sha512-OJykPaF4v8JidKNGz8c/q1lBO44sQNUQtq1KktJXdBLn1hPod5rE/Hko5ugKKZd+D2+o1a9MFGUEIUwO2YfgkQ==} engines: {node: '>=18'} @@ -4638,6 +4723,12 @@ packages: cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.24.2': + resolution: {integrity: sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-loong64@0.25.1': resolution: {integrity: sha512-nGfornQj4dzcq5Vp835oM/o21UMlXzn79KobKlcs3Wz9smwiifknLy4xDCLUU0BWp7b/houtdrgUz7nOGnfIYg==} engines: {node: '>=18'} @@ -4674,6 +4765,12 @@ packages: cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.24.2': + resolution: {integrity: sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-mips64el@0.25.1': resolution: {integrity: sha512-1osBbPEFYwIE5IVB/0g2X6i1qInZa1aIoj1TdL4AaAb55xIIgbg8Doq6a5BzYWgr+tEcDzYH67XVnTmUzL+nXg==} engines: {node: '>=18'} @@ -4710,6 +4807,12 @@ packages: cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.24.2': + resolution: {integrity: sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-ppc64@0.25.1': resolution: {integrity: sha512-/6VBJOwUf3TdTvJZ82qF3tbLuWsscd7/1w+D9LH0W/SqUgM5/JJD0lrJ1fVIfZsqB6RFmLCe0Xz3fmZc3WtyVg==} engines: {node: '>=18'} @@ -4746,6 +4849,12 @@ packages: cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.24.2': + resolution: {integrity: sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-riscv64@0.25.1': resolution: {integrity: sha512-nSut/Mx5gnilhcq2yIMLMe3Wl4FK5wx/o0QuuCLMtmJn+WeWYoEGDN1ipcN72g1WHsnIbxGXd4i/MF0gTcuAjQ==} engines: {node: '>=18'} @@ -4782,6 +4891,12 @@ packages: cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.24.2': + resolution: {integrity: sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-s390x@0.25.1': resolution: {integrity: sha512-cEECeLlJNfT8kZHqLarDBQso9a27o2Zd2AQ8USAEoGtejOrCYHNtKP8XQhMDJMtthdF4GBmjR2au3x1udADQQQ==} engines: {node: '>=18'} @@ -4818,12 +4933,24 @@ packages: cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.24.2': + resolution: {integrity: sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + '@esbuild/linux-x64@0.25.1': resolution: {integrity: sha512-xbfUhu/gnvSEg+EGovRc+kjBAkrvtk38RlerAzQxvMzlB4fXpCFCeUAYzJvrnhFtdeyVCDANSjJvOvGYoeKzFA==} engines: {node: '>=18'} cpu: [x64] os: [linux] + '@esbuild/netbsd-arm64@0.24.2': + resolution: {integrity: sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + '@esbuild/netbsd-arm64@0.25.1': resolution: {integrity: sha512-O96poM2XGhLtpTh+s4+nP7YCCAfb4tJNRVZHfIE7dgmax+yMP2WgMd2OecBuaATHKTHsLWHQeuaxMRnCsH8+5g==} engines: {node: '>=18'} @@ -4860,6 +4987,12 @@ packages: cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.24.2': + resolution: {integrity: sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + '@esbuild/netbsd-x64@0.25.1': resolution: {integrity: sha512-X53z6uXip6KFXBQ+Krbx25XHV/NCbzryM6ehOAeAil7X7oa4XIq+394PWGnwaSQ2WRA0KI6PUO6hTO5zeF5ijA==} engines: {node: '>=18'} @@ -4872,6 +5005,12 @@ packages: cpu: [arm64] os: [openbsd] + '@esbuild/openbsd-arm64@0.24.2': + resolution: {integrity: sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + '@esbuild/openbsd-arm64@0.25.1': resolution: {integrity: sha512-Na9T3szbXezdzM/Kfs3GcRQNjHzM6GzFBeU1/6IV/npKP5ORtp9zbQjvkDJ47s6BCgaAZnnnu/cY1x342+MvZg==} engines: {node: '>=18'} @@ -4908,6 +5047,12 @@ packages: cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.24.2': + resolution: {integrity: sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/openbsd-x64@0.25.1': resolution: {integrity: sha512-T3H78X2h1tszfRSf+txbt5aOp/e7TAz3ptVKu9Oyir3IAOFPGV6O9c2naym5TOriy1l0nNf6a4X5UXRZSGX/dw==} engines: {node: '>=18'} @@ -4944,6 +5089,12 @@ packages: cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.24.2': + resolution: {integrity: sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + '@esbuild/sunos-x64@0.25.1': resolution: {integrity: sha512-2H3RUvcmULO7dIE5EWJH8eubZAI4xw54H1ilJnRNZdeo8dTADEZ21w6J22XBkXqGJbe0+wnNJtw3UXRoLJnFEg==} engines: {node: '>=18'} @@ -4980,6 +5131,12 @@ packages: cpu: [arm64] os: [win32] + '@esbuild/win32-arm64@0.24.2': + resolution: {integrity: sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-arm64@0.25.1': resolution: {integrity: sha512-GE7XvrdOzrb+yVKB9KsRMq+7a2U/K5Cf/8grVFRAGJmfADr/e/ODQ134RK2/eeHqYV5eQRFxb1hY7Nr15fv1NQ==} engines: {node: '>=18'} @@ -5016,6 +5173,12 @@ packages: cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.24.2': + resolution: {integrity: sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + '@esbuild/win32-ia32@0.25.1': resolution: {integrity: sha512-uOxSJCIcavSiT6UnBhBzE8wy3n0hOkJsBOzy7HDAuTDE++1DJMRRVCPGisULScHL+a/ZwdXPpXD3IyFKjA7K8A==} engines: {node: '>=18'} @@ -5052,6 +5215,12 @@ packages: cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.24.2': + resolution: {integrity: sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@esbuild/win32-x64@0.25.1': resolution: {integrity: sha512-Y1EQdcfwMSeQN/ujR5VayLOJ1BHaK+ssyk0AEzPjC+t1lITgsnccPqFjb6V+LsTp/9Iov4ysfjxLaGJ9RPtkVg==} engines: {node: '>=18'} @@ -9387,6 +9556,10 @@ packages: resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} engines: {node: '>=10'} + '@sindresorhus/merge-streams@4.0.0': + resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} + engines: {node: '>=18'} + '@slack/logger@4.0.0': resolution: {integrity: sha512-Wz7QYfPAlG/DR+DfABddUZeNgoeY7d1J39OCR2jR+v7VBsB8ezulDK5szTnDDPDwLH5IWhLvXIHlCFZV7MSKgA==} engines: {node: '>= 18', npm: '>= 8.6.0'} @@ -10492,9 +10665,6 @@ packages: '@types/estree@1.0.6': resolution: {integrity: sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==} - '@types/estree@1.0.7': - resolution: {integrity: sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==} - '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} @@ -13195,6 +13365,11 @@ packages: engines: {node: '>=18'} hasBin: true + esbuild@0.24.2: + resolution: {integrity: sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA==} + engines: {node: '>=18'} + hasBin: true + esbuild@0.25.1: resolution: {integrity: sha512-BGO5LtrGC7vxnqucAe/rmvKdJllfGaYWdyABvyMoXQlfYMb2bbRuReWR5tEGE//4LcNJj9XrkovTqNYRFZHAMQ==} engines: {node: '>=18'} @@ -13537,6 +13712,10 @@ packages: resolution: {integrity: sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==} engines: {node: '>=16.17'} + execa@9.6.1: + resolution: {integrity: sha512-9Be3ZoN4LmYR90tUoVu2te2BsbzHfhJyfEiAVfz7N5/zv+jduIfLrV2xdQXOHbaD6KgpGdO9PRPM1Y4Q9QkPkA==} + engines: {node: ^18.19.0 || >=20.5.0} + exit-hook@2.2.1: resolution: {integrity: sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==} engines: {node: '>=6'} @@ -13707,6 +13886,10 @@ packages: fft.js@4.0.4: resolution: {integrity: sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==} + figures@6.1.0: + resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} + engines: {node: '>=18'} + file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -14277,6 +14460,10 @@ packages: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} + human-signals@8.0.1: + resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==} + engines: {node: '>=18.18.0'} + humanize-duration@3.27.3: resolution: {integrity: sha512-iimHkHPfIAQ8zCDQLgn08pRqSVioyWvnGfaQ8gond2wf7Jq2jJ+24ykmnRyiz3fIldcn4oUuQXpjqKLhSVR7lw==} @@ -14622,6 +14809,10 @@ packages: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} + is-unicode-supported@2.1.0: + resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} + engines: {node: '>=18'} + is-weakref@1.0.2: resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} @@ -16168,6 +16359,10 @@ packages: resolution: {integrity: sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + npm-run-path@6.0.0: + resolution: {integrity: sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==} + engines: {node: '>=18'} + num2fraction@1.2.2: resolution: {integrity: sha512-Y1wZESM7VUThYY+4W+X4ySH2maqcA+p7UR+w8VWNWVAd6lwuXXWz/w/Cz43J/dI2I+PS6wD5N+bJUF+gjWvIqg==} @@ -16508,6 +16703,10 @@ packages: resolution: {integrity: sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==} engines: {node: '>=6'} + parse-ms@4.0.0: + resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==} + engines: {node: '>=18'} + parse5-htmlparser2-tree-adapter@6.0.1: resolution: {integrity: sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==} @@ -17094,6 +17293,10 @@ packages: resolution: {integrity: sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==} engines: {node: '>=10'} + pretty-ms@9.3.0: + resolution: {integrity: sha512-gjVS5hOP+M3wMm5nmNOucbIrqudzs9v/57bWRHQWLYklXqoXKrVfYW2W9+glfGsqtPgpiz5WwyEEB+ksXIx3gQ==} + engines: {node: '>=18'} + prism-react-renderer@2.1.0: resolution: {integrity: sha512-I5cvXHjA1PVGbGm1MsWCpvBCRrYyxEri0MC7/JbfIfYfcXAxHyO5PaUjs3A8H5GW6kJcLhTHxxMaOZZpRZD2iQ==} peerDependencies: @@ -18422,6 +18625,10 @@ packages: resolution: {integrity: sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==} engines: {node: '>=12'} + strip-final-newline@4.0.0: + resolution: {integrity: sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==} + engines: {node: '>=18'} + strip-indent@3.0.0: resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==} engines: {node: '>=8'} @@ -19194,6 +19401,10 @@ packages: resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==} engines: {node: '>=18'} + unicorn-magic@0.3.0: + resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} + engines: {node: '>=18'} + unified@10.1.2: resolution: {integrity: sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==} @@ -19853,6 +20064,10 @@ packages: resolution: {integrity: sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==} engines: {node: '>=12.20'} + yoctocolors@2.1.2: + resolution: {integrity: sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==} + engines: {node: '>=18'} + yup@1.6.1: resolution: {integrity: sha512-JED8pB50qbA4FOkDol0bYF/p60qSEDQqBD0/qeIrUCG1KbPBIQ776fCUNb9ldbPcSTxA69g/47XTo4TqWiuXOA==} @@ -22840,6 +23055,9 @@ snapshots: '@esbuild/aix-ppc64@0.23.0': optional: true + '@esbuild/aix-ppc64@0.24.2': + optional: true + '@esbuild/aix-ppc64@0.25.1': optional: true @@ -22858,6 +23076,9 @@ snapshots: '@esbuild/android-arm64@0.23.0': optional: true + '@esbuild/android-arm64@0.24.2': + optional: true + '@esbuild/android-arm64@0.25.1': optional: true @@ -22879,6 +23100,9 @@ snapshots: '@esbuild/android-arm@0.23.0': optional: true + '@esbuild/android-arm@0.24.2': + optional: true + '@esbuild/android-arm@0.25.1': optional: true @@ -22897,6 +23121,9 @@ snapshots: '@esbuild/android-x64@0.23.0': optional: true + '@esbuild/android-x64@0.24.2': + optional: true + '@esbuild/android-x64@0.25.1': optional: true @@ -22915,6 +23142,9 @@ snapshots: '@esbuild/darwin-arm64@0.23.0': optional: true + '@esbuild/darwin-arm64@0.24.2': + optional: true + '@esbuild/darwin-arm64@0.25.1': optional: true @@ -22933,6 +23163,9 @@ snapshots: '@esbuild/darwin-x64@0.23.0': optional: true + '@esbuild/darwin-x64@0.24.2': + optional: true + '@esbuild/darwin-x64@0.25.1': optional: true @@ -22951,6 +23184,9 @@ snapshots: '@esbuild/freebsd-arm64@0.23.0': optional: true + '@esbuild/freebsd-arm64@0.24.2': + optional: true + '@esbuild/freebsd-arm64@0.25.1': optional: true @@ -22969,6 +23205,9 @@ snapshots: '@esbuild/freebsd-x64@0.23.0': optional: true + '@esbuild/freebsd-x64@0.24.2': + optional: true + '@esbuild/freebsd-x64@0.25.1': optional: true @@ -22987,6 +23226,9 @@ snapshots: '@esbuild/linux-arm64@0.23.0': optional: true + '@esbuild/linux-arm64@0.24.2': + optional: true + '@esbuild/linux-arm64@0.25.1': optional: true @@ -23005,6 +23247,9 @@ snapshots: '@esbuild/linux-arm@0.23.0': optional: true + '@esbuild/linux-arm@0.24.2': + optional: true + '@esbuild/linux-arm@0.25.1': optional: true @@ -23023,6 +23268,9 @@ snapshots: '@esbuild/linux-ia32@0.23.0': optional: true + '@esbuild/linux-ia32@0.24.2': + optional: true + '@esbuild/linux-ia32@0.25.1': optional: true @@ -23044,6 +23292,9 @@ snapshots: '@esbuild/linux-loong64@0.23.0': optional: true + '@esbuild/linux-loong64@0.24.2': + optional: true + '@esbuild/linux-loong64@0.25.1': optional: true @@ -23062,6 +23313,9 @@ snapshots: '@esbuild/linux-mips64el@0.23.0': optional: true + '@esbuild/linux-mips64el@0.24.2': + optional: true + '@esbuild/linux-mips64el@0.25.1': optional: true @@ -23080,6 +23334,9 @@ snapshots: '@esbuild/linux-ppc64@0.23.0': optional: true + '@esbuild/linux-ppc64@0.24.2': + optional: true + '@esbuild/linux-ppc64@0.25.1': optional: true @@ -23098,6 +23355,9 @@ snapshots: '@esbuild/linux-riscv64@0.23.0': optional: true + '@esbuild/linux-riscv64@0.24.2': + optional: true + '@esbuild/linux-riscv64@0.25.1': optional: true @@ -23116,6 +23376,9 @@ snapshots: '@esbuild/linux-s390x@0.23.0': optional: true + '@esbuild/linux-s390x@0.24.2': + optional: true + '@esbuild/linux-s390x@0.25.1': optional: true @@ -23134,9 +23397,15 @@ snapshots: '@esbuild/linux-x64@0.23.0': optional: true + '@esbuild/linux-x64@0.24.2': + optional: true + '@esbuild/linux-x64@0.25.1': optional: true + '@esbuild/netbsd-arm64@0.24.2': + optional: true + '@esbuild/netbsd-arm64@0.25.1': optional: true @@ -23155,12 +23424,18 @@ snapshots: '@esbuild/netbsd-x64@0.23.0': optional: true + '@esbuild/netbsd-x64@0.24.2': + optional: true + '@esbuild/netbsd-x64@0.25.1': optional: true '@esbuild/openbsd-arm64@0.23.0': optional: true + '@esbuild/openbsd-arm64@0.24.2': + optional: true + '@esbuild/openbsd-arm64@0.25.1': optional: true @@ -23179,6 +23454,9 @@ snapshots: '@esbuild/openbsd-x64@0.23.0': optional: true + '@esbuild/openbsd-x64@0.24.2': + optional: true + '@esbuild/openbsd-x64@0.25.1': optional: true @@ -23197,6 +23475,9 @@ snapshots: '@esbuild/sunos-x64@0.23.0': optional: true + '@esbuild/sunos-x64@0.24.2': + optional: true + '@esbuild/sunos-x64@0.25.1': optional: true @@ -23215,6 +23496,9 @@ snapshots: '@esbuild/win32-arm64@0.23.0': optional: true + '@esbuild/win32-arm64@0.24.2': + optional: true + '@esbuild/win32-arm64@0.25.1': optional: true @@ -23233,6 +23517,9 @@ snapshots: '@esbuild/win32-ia32@0.23.0': optional: true + '@esbuild/win32-ia32@0.24.2': + optional: true + '@esbuild/win32-ia32@0.25.1': optional: true @@ -23251,6 +23538,9 @@ snapshots: '@esbuild/win32-x64@0.23.0': optional: true + '@esbuild/win32-x64@0.24.2': + optional: true + '@esbuild/win32-x64@0.25.1': optional: true @@ -29012,6 +29302,8 @@ snapshots: '@sindresorhus/is@4.6.0': {} + '@sindresorhus/merge-streams@4.0.0': {} + '@slack/logger@4.0.0': dependencies: '@types/node': 20.14.14 @@ -30480,14 +30772,12 @@ snapshots: '@types/estree-jsx@1.0.0': dependencies: - '@types/estree': 1.0.7 + '@types/estree': 1.0.8 '@types/estree@1.0.0': {} '@types/estree@1.0.6': {} - '@types/estree@1.0.7': {} - '@types/estree@1.0.8': {} '@types/eventsource@1.1.15': {} @@ -31108,6 +31398,14 @@ snapshots: optionalDependencies: vite: 5.4.21(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) + '@vitest/mocker@3.1.4(vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1))': + dependencies: + '@vitest/spy': 3.1.4 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + '@vitest/pretty-format@2.1.9': dependencies: tinyrainbow: 1.2.0 @@ -33637,6 +33935,34 @@ snapshots: '@esbuild/win32-ia32': 0.23.0 '@esbuild/win32-x64': 0.23.0 + esbuild@0.24.2: + optionalDependencies: + '@esbuild/aix-ppc64': 0.24.2 + '@esbuild/android-arm': 0.24.2 + '@esbuild/android-arm64': 0.24.2 + '@esbuild/android-x64': 0.24.2 + '@esbuild/darwin-arm64': 0.24.2 + '@esbuild/darwin-x64': 0.24.2 + '@esbuild/freebsd-arm64': 0.24.2 + '@esbuild/freebsd-x64': 0.24.2 + '@esbuild/linux-arm': 0.24.2 + '@esbuild/linux-arm64': 0.24.2 + '@esbuild/linux-ia32': 0.24.2 + '@esbuild/linux-loong64': 0.24.2 + '@esbuild/linux-mips64el': 0.24.2 + '@esbuild/linux-ppc64': 0.24.2 + '@esbuild/linux-riscv64': 0.24.2 + '@esbuild/linux-s390x': 0.24.2 + '@esbuild/linux-x64': 0.24.2 + '@esbuild/netbsd-arm64': 0.24.2 + '@esbuild/netbsd-x64': 0.24.2 + '@esbuild/openbsd-arm64': 0.24.2 + '@esbuild/openbsd-x64': 0.24.2 + '@esbuild/sunos-x64': 0.24.2 + '@esbuild/win32-arm64': 0.24.2 + '@esbuild/win32-ia32': 0.24.2 + '@esbuild/win32-x64': 0.24.2 + esbuild@0.25.1: optionalDependencies: '@esbuild/aix-ppc64': 0.25.1 @@ -34085,6 +34411,21 @@ snapshots: signal-exit: 4.1.0 strip-final-newline: 3.0.0 + execa@9.6.1: + dependencies: + '@sindresorhus/merge-streams': 4.0.0 + cross-spawn: 7.0.6 + figures: 6.1.0 + get-stream: 9.0.1 + human-signals: 8.0.1 + is-plain-obj: 4.1.0 + is-stream: 4.0.1 + npm-run-path: 6.0.0 + pretty-ms: 9.3.0 + signal-exit: 4.1.0 + strip-final-newline: 4.0.0 + yoctocolors: 2.1.2 + exit-hook@2.2.1: {} expand-template@2.0.3: {} @@ -34315,6 +34656,10 @@ snapshots: fft.js@4.0.4: {} + figures@6.1.0: + dependencies: + is-unicode-supported: 2.1.0 + file-entry-cache@6.0.1: dependencies: flat-cache: 3.0.4 @@ -34908,7 +35253,7 @@ snapshots: hast-util-to-estree@2.1.0: dependencies: - '@types/estree': 1.0.7 + '@types/estree': 1.0.8 '@types/estree-jsx': 1.0.0 '@types/hast': 2.3.4 '@types/unist': 2.0.6 @@ -34942,7 +35287,7 @@ snapshots: hast-util-to-jsx-runtime@2.3.6: dependencies: - '@types/estree': 1.0.7 + '@types/estree': 1.0.8 '@types/hast': 3.0.4 '@types/unist': 3.0.3 comma-separated-tokens: 2.0.3 @@ -35073,6 +35418,8 @@ snapshots: human-signals@5.0.0: {} + human-signals@8.0.1: {} + humanize-duration@3.27.3: {} humanize-ms@1.2.1: @@ -35373,6 +35720,8 @@ snapshots: is-unicode-supported@0.1.0: {} + is-unicode-supported@2.1.0: {} + is-weakref@1.0.2: dependencies: call-bind: 1.0.8 @@ -36961,7 +37310,7 @@ snapshots: nano-css@5.6.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 css-tree: 1.1.3 csstype: 3.2.0 fastest-stable-stringify: 2.0.2 @@ -37254,6 +37603,11 @@ snapshots: dependencies: path-key: 4.0.0 + npm-run-path@6.0.0: + dependencies: + path-key: 4.0.0 + unicorn-magic: 0.3.0 + num2fraction@1.2.2: {} nypm@0.3.9: @@ -37673,6 +38027,8 @@ snapshots: parse-ms@2.1.0: {} + parse-ms@4.0.0: {} + parse5-htmlparser2-tree-adapter@6.0.1: dependencies: parse5: 6.0.1 @@ -37758,7 +38114,7 @@ snapshots: periscopic@3.1.0: dependencies: - '@types/estree': 1.0.7 + '@types/estree': 1.0.8 estree-walker: 3.0.3 is-reference: 3.0.3 @@ -38228,6 +38584,10 @@ snapshots: dependencies: parse-ms: 2.1.0 + pretty-ms@9.3.0: + dependencies: + parse-ms: 4.0.0 + prism-react-renderer@2.1.0(react@18.3.1): dependencies: '@types/prismjs': 1.26.0 @@ -40152,6 +40512,8 @@ snapshots: strip-final-newline@3.0.0: {} + strip-final-newline@4.0.0: {} + strip-indent@3.0.0: dependencies: min-indent: 1.0.1 @@ -41024,6 +41386,8 @@ snapshots: unicorn-magic@0.1.0: {} + unicorn-magic@0.3.0: {} + unified@10.1.2: dependencies: '@types/unist': 2.0.6 @@ -41371,6 +41735,24 @@ snapshots: - supports-color - terser + vite-node@3.1.4(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): + dependencies: + cac: 6.7.14 + debug: 4.4.1(supports-color@10.0.0) + es-module-lexer: 1.7.0 + pathe: 2.0.3 + vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + transitivePeerDependencies: + - '@types/node' + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vite-tsconfig-paths@4.0.5(typescript@5.5.4): dependencies: debug: 4.3.7(supports-color@10.0.0) @@ -41402,6 +41784,17 @@ snapshots: lightningcss: 1.29.2 terser: 5.44.1 + vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): + dependencies: + esbuild: 0.21.5 + postcss: 8.5.6 + rollup: 4.36.0 + optionalDependencies: + '@types/node': 22.13.9 + fsevents: 2.3.3 + lightningcss: 1.29.2 + terser: 5.44.1 + vitest@3.1.4(@types/debug@4.1.12)(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1): dependencies: '@vitest/expect': 3.1.4 @@ -41412,9 +41805,9 @@ snapshots: '@vitest/spy': 3.1.4 '@vitest/utils': 3.1.4 chai: 5.2.0 - debug: 4.4.0 + debug: 4.4.1(supports-color@10.0.0) expect-type: 1.2.1 - magic-string: 0.30.17 + magic-string: 0.30.21 pathe: 2.0.3 std-env: 3.9.0 tinybench: 2.9.0 @@ -41439,6 +41832,43 @@ snapshots: - supports-color - terser + vitest@3.1.4(@types/debug@4.1.12)(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): + dependencies: + '@vitest/expect': 3.1.4 + '@vitest/mocker': 3.1.4(vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1)) + '@vitest/pretty-format': 3.1.4 + '@vitest/runner': 3.1.4 + '@vitest/snapshot': 3.1.4 + '@vitest/spy': 3.1.4 + '@vitest/utils': 3.1.4 + chai: 5.2.0 + debug: 4.4.1(supports-color@10.0.0) + expect-type: 1.2.1 + magic-string: 0.30.21 + pathe: 2.0.3 + std-env: 3.9.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.13 + tinypool: 1.0.2 + tinyrainbow: 2.0.0 + vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + vite-node: 3.1.4(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/debug': 4.1.12 + '@types/node': 22.13.9 + transitivePeerDependencies: + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vscode-jsonrpc@8.2.0: {} vscode-languageserver-protocol@3.17.5: @@ -41552,7 +41982,7 @@ snapshots: webpack@5.88.2(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.19.11): dependencies: '@types/eslint-scope': 3.7.4 - '@types/estree': 1.0.7 + '@types/estree': 1.0.8 '@webassemblyjs/ast': 1.11.5 '@webassemblyjs/wasm-edit': 1.11.5 '@webassemblyjs/wasm-parser': 1.11.5 @@ -41761,6 +42191,8 @@ snapshots: yocto-queue@1.1.1: {} + yoctocolors@2.1.2: {} + yup@1.6.1: dependencies: property-expr: 2.0.6 From 9937823a7f174b316d567d7e8b2c77ee27e2fbbe Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 30 Jan 2026 09:15:35 +0000 Subject: [PATCH 015/400] Standardize @types/node to version 20.14.14 across monorepo (#2970) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## ✅ Checklist - [ ] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [ ] The PR title follows the convention. - [ ] I ran and tested the code works --- ## Description This PR standardizes the `@types/node` dependency across the entire monorepo to version `20.14.14`. Previously, different packages were using different versions (ranging from 12.20.55 to 22.13.9), which could cause type conflicts and inconsistencies. ### Changes Made 1. **tsconfig.json** - Added `"node"` to the `types` array in `apps/webapp/tsconfig.json` to ensure Node.js types are properly recognized 2. **package.json overrides** - Added `@types/node` version override to `20.14.14` in the root `package.json` 3. **pnpm-lock.yaml** - Updated lock file to reflect the standardized version across all packages and their dependencies 4. **Fixture package.json** - Updated `packages/cli-v3/e2e/fixtures/emit-decorator-metadata/package.json` to use the standardized version This ensures consistent type definitions across the monorepo and prevents version mismatches that could lead to type errors or unexpected behavior. --- ## Testing - Verified that all package references to `@types/node` now point to version `20.14.14` - Confirmed that the lock file properly reflects the override across all transitive dependencies - Ensured TypeScript configuration includes Node.js types for proper type checking --- ## Changelog - Standardized `@types/node` to version `20.14.14` across all packages in the monorepo - Added `"node"` to TypeScript compiler types in webapp configuration - Updated all package dependencies to use the consistent version through pnpm overrides 💯 https://claude.ai/code/session_018eqp2LvvErkFSN9oK5xBh1 --- --------- Co-authored-by: Claude Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: Eric Allam --- apps/webapp/tsconfig.json | 2 +- package.json | 1 + pnpm-lock.yaml | 125 ++++++++++++++------------------------ 3 files changed, 46 insertions(+), 82 deletions(-) diff --git a/apps/webapp/tsconfig.json b/apps/webapp/tsconfig.json index a10eda99cfb..36944e395b6 100644 --- a/apps/webapp/tsconfig.json +++ b/apps/webapp/tsconfig.json @@ -2,7 +2,7 @@ "exclude": ["./cypress", "./cypress.config.ts"], "include": ["remix.env.d.ts", "global.d.ts", "**/*.ts", "**/*.tsx"], "compilerOptions": { - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "lib": ["DOM", "DOM.Iterable", "DOM.AsyncIterable", "ES2020"], "isolatedModules": true, "esModuleInterop": true, diff --git a/package.json b/package.json index 61ec8c56fbf..6420ec29935 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ }, "overrides": { "typescript": "5.5.4", + "@types/node": "20.14.14", "express@^4>body-parser": "1.20.3", "@remix-run/dev@2.1.0>tar-fs": "2.1.3", "testcontainers@10.28.0>tar-fs": "3.0.9", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b02c6cd7340..ba4facc3fd5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,6 +6,7 @@ settings: overrides: typescript: 5.5.4 + '@types/node': 20.14.14 express@^4>body-parser: 1.20.3 '@remix-run/dev@2.1.0>tar-fs': 2.1.3 testcontainers@10.28.0>tar-fs: 3.0.9 @@ -814,7 +815,7 @@ importers: version: link:../../internal-packages/testcontainers '@remix-run/dev': specifier: 2.1.0 - version: 2.1.0(@remix-run/serve@2.1.0(typescript@5.5.4))(@types/node@22.13.9)(bufferutil@4.0.9)(encoding@0.1.13)(lightningcss@1.29.2)(terser@5.44.1)(typescript@5.5.4) + version: 2.1.0(@remix-run/serve@2.1.0(typescript@5.5.4))(@types/node@20.14.14)(bufferutil@4.0.9)(encoding@0.1.13)(lightningcss@1.29.2)(terser@5.44.1)(typescript@5.5.4) '@remix-run/eslint-config': specifier: 2.1.0 version: 2.1.0(eslint@8.31.0)(react@18.2.0)(typescript@5.5.4) @@ -1091,7 +1092,7 @@ importers: version: 18.3.1 react-email: specifier: ^2.1.1 - version: 2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(eslint@8.31.0) + version: 2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(bufferutil@4.0.9)(eslint@8.31.0) resend: specifier: ^3.2.0 version: 3.2.0 @@ -1119,7 +1120,7 @@ importers: version: 7.3.2 devDependencies: '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 rimraf: specifier: ^3.0.2 @@ -1949,7 +1950,7 @@ importers: specifier: workspace:^4.3.3 version: link:../build '@types/node': - specifier: ^20.14.14 + specifier: 20.14.14 version: 20.14.14 '@types/react': specifier: '*' @@ -2211,7 +2212,7 @@ importers: specifier: ^4.0.3 version: 4.0.8 '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 '@types/react': specifier: ^19 @@ -2293,7 +2294,7 @@ importers: specifier: workspace:* version: link:../../packages/build '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 '@types/react': specifier: ^19 @@ -2674,7 +2675,7 @@ importers: specifier: ^4 version: 4.0.17 '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 '@types/react': specifier: ^19 @@ -2729,7 +2730,7 @@ importers: specifier: ^4 version: 4.0.17 '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 '@types/react': specifier: ^19 @@ -2791,7 +2792,7 @@ importers: version: link:../../packages/trigger-sdk devDependencies: '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 trigger.dev: specifier: workspace:* @@ -10782,24 +10783,9 @@ packages: '@types/node-fetch@2.6.4': resolution: {integrity: sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==} - '@types/node@12.20.55': - resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} - - '@types/node@18.19.20': - resolution: {integrity: sha512-SKXZvI375jkpvAj8o+5U2518XQv76mAsixqfXiVyWyXZbVWQK25RurFovYpVIxVzul0rZoH58V/3SkEnm7s3qA==} - - '@types/node@20.11.22': - resolution: {integrity: sha512-/G+IxWxma6V3E+pqK1tSl2Fo1kl41pK1yeCyDsgkF9WlVAme4j5ISYM2zR11bgLFJGLN5sVK40T4RJNuiZbEjA==} - - '@types/node@20.12.14': - resolution: {integrity: sha512-scnD59RpYD91xngrQQLGkE+6UrHUPzeKZWhhjBSa3HSkwjbQc38+q3RoIVEwxQGRw3M+j5hpNAM+lgV3cVormg==} - '@types/node@20.14.14': resolution: {integrity: sha512-d64f00982fS9YoOgJkAMolK7MN8Iq3TDdVjchbYHdEmjth/DHowx82GnoA+tVUAN+7vxfYUgAzi+JXbKNd2SDQ==} - '@types/node@22.13.9': - resolution: {integrity: sha512-acBjXdRJ3A6Pb3tqnw9HZmyR3Fiol3aGxRCK1x3d+6CDAMjl7I649wpSd+yNURCjbOUGu9tqtLKnTGxmK6CyGw==} - '@types/nodemailer@7.0.4': resolution: {integrity: sha512-ee8fxWqOchH+Hv6MDDNNy028kwvVnLplrStm4Zf/3uHWw5zzo8FoYYeffpJtGs2wWysEumMH0ZIdMGMY1eMAow==} @@ -19386,9 +19372,6 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@6.20.0: - resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} - undici@5.29.0: resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} engines: {node: '>=14.0'} @@ -19679,7 +19662,7 @@ packages: engines: {node: ^14.18.0 || >=16.0.0} hasBin: true peerDependencies: - '@types/node': '>= 14' + '@types/node': 20.14.14 less: '*' lightningcss: ^1.21.0 sass: '*' @@ -19707,7 +19690,7 @@ packages: engines: {node: ^18.0.0 || >=20.0.0} hasBin: true peerDependencies: - '@types/node': ^18.0.0 || >=20.0.0 + '@types/node': 20.14.14 less: '*' lightningcss: ^1.21.0 sass: '*' @@ -19740,7 +19723,7 @@ packages: peerDependencies: '@edge-runtime/vm': '*' '@types/debug': ^4.1.12 - '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + '@types/node': 20.14.14 '@vitest/browser': 3.1.4 '@vitest/ui': 3.1.4 happy-dom: '*' @@ -24104,7 +24087,7 @@ snapshots: '@kubernetes/client-node@0.20.0(bufferutil@4.0.9)': dependencies: '@types/js-yaml': 4.0.9 - '@types/node': 20.11.22 + '@types/node': 20.14.14 '@types/request': 2.48.12 '@types/ws': 8.5.10 byline: 5.0.0 @@ -24126,7 +24109,7 @@ snapshots: '@kubernetes/client-node@1.0.0(patch_hash=ba1a06f46256cdb8d6faf7167246692c0de2e7cd846a9dc0f13be0137e1c3745)(bufferutil@4.0.9)(encoding@0.1.13)': dependencies: '@types/js-yaml': 4.0.9 - '@types/node': 22.13.9 + '@types/node': 20.14.14 '@types/node-fetch': 2.6.12 '@types/stream-buffers': 3.0.7 '@types/tar': 6.1.4 @@ -24196,7 +24179,7 @@ snapshots: '@manypkg/find-root@1.1.0': dependencies: '@babel/runtime': 7.28.4 - '@types/node': 12.20.55 + '@types/node': 20.14.14 find-up: 4.1.0 fs-extra: 8.1.0 @@ -28814,7 +28797,7 @@ snapshots: transitivePeerDependencies: - encoding - '@remix-run/dev@2.1.0(@remix-run/serve@2.1.0(typescript@5.5.4))(@types/node@22.13.9)(bufferutil@4.0.9)(encoding@0.1.13)(lightningcss@1.29.2)(terser@5.44.1)(typescript@5.5.4)': + '@remix-run/dev@2.1.0(@remix-run/serve@2.1.0(typescript@5.5.4))(@types/node@20.14.14)(bufferutil@4.0.9)(encoding@0.1.13)(lightningcss@1.29.2)(terser@5.44.1)(typescript@5.5.4)': dependencies: '@babel/core': 7.22.17 '@babel/generator': 7.24.7 @@ -28827,7 +28810,7 @@ snapshots: '@npmcli/package-json': 4.0.1 '@remix-run/server-runtime': 2.1.0(typescript@5.5.4) '@types/mdx': 2.0.5 - '@vanilla-extract/integration': 6.2.1(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + '@vanilla-extract/integration': 6.2.1(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) arg: 5.0.2 cacache: 17.1.4 chalk: 4.1.2 @@ -30878,7 +30861,7 @@ snapshots: '@types/morgan@1.9.4': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/ms@0.7.31': {} @@ -30893,7 +30876,7 @@ snapshots: '@types/node-fetch@2.6.2': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 form-data: 3.0.4 '@types/node-fetch@2.6.4': @@ -30901,28 +30884,10 @@ snapshots: '@types/node': 20.14.14 form-data: 3.0.4 - '@types/node@12.20.55': {} - - '@types/node@18.19.20': - dependencies: - undici-types: 5.26.5 - - '@types/node@20.11.22': - dependencies: - undici-types: 5.26.5 - - '@types/node@20.12.14': - dependencies: - undici-types: 5.26.5 - '@types/node@20.14.14': dependencies: undici-types: 5.26.5 - '@types/node@22.13.9': - dependencies: - undici-types: 6.20.0 - '@types/nodemailer@7.0.4': dependencies: '@aws-sdk/client-sesv2': 3.940.0 @@ -30958,7 +30923,7 @@ snapshots: '@types/pg@8.6.6': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 pg-protocol: 1.6.1 pg-types: 2.2.0 @@ -31012,7 +30977,7 @@ snapshots: '@types/readable-stream@4.0.14': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 safe-buffer: 5.1.2 '@types/regression@2.0.6': {} @@ -31072,7 +31037,7 @@ snapshots: '@types/ssh2@1.15.1': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/stream-buffers@3.0.7': dependencies: @@ -31092,7 +31057,7 @@ snapshots: '@types/tar@6.1.4': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 minipass: 4.0.0 '@types/tedious@4.0.14': @@ -31137,7 +31102,7 @@ snapshots: '@types/ws@8.5.4': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/yauzl@2.10.3': dependencies: @@ -31316,7 +31281,7 @@ snapshots: media-query-parser: 2.0.2 outdent: 0.8.0 - '@vanilla-extract/integration@6.2.1(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1)': + '@vanilla-extract/integration@6.2.1(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1)': dependencies: '@babel/core': 7.22.17 '@babel/plugin-syntax-typescript': 7.21.4(@babel/core@7.22.17) @@ -31329,8 +31294,8 @@ snapshots: lodash: 4.17.23 mlly: 1.7.4 outdent: 0.8.0 - vite: 4.4.9(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) - vite-node: 0.28.5(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + vite: 4.4.9(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) + vite-node: 0.28.5(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) transitivePeerDependencies: - '@types/node' - less @@ -32297,7 +32262,7 @@ snapshots: bun-types@1.1.17: dependencies: - '@types/node': 20.12.14 + '@types/node': 20.14.14 '@types/ws': 8.5.10 bundle-name@4.1.0: @@ -33537,7 +33502,7 @@ snapshots: dependencies: '@types/cookie': 0.4.1 '@types/cors': 2.8.17 - '@types/node': 18.19.20 + '@types/node': 20.14.14 accepts: 1.3.8 base64id: 2.0.0 cookie: 0.4.2 @@ -37769,7 +37734,7 @@ snapshots: openai@4.33.1(encoding@0.1.13): dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/node-fetch': 2.6.4 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -37782,7 +37747,7 @@ snapshots: openai@4.68.4(encoding@0.1.13)(zod@3.25.76): dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/node-fetch': 2.6.4 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -37796,7 +37761,7 @@ snapshots: openai@4.97.0(encoding@0.1.13)(ws@8.12.0(bufferutil@4.0.9))(zod@3.25.76): dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/node-fetch': 2.6.12 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -37811,7 +37776,7 @@ snapshots: openai@4.97.0(encoding@0.1.13)(ws@8.18.3(bufferutil@4.0.9))(zod@3.25.76): dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/node-fetch': 2.6.12 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -38715,7 +38680,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 18.19.20 + '@types/node': 20.14.14 long: 5.2.3 proxy-addr@2.0.7: @@ -38967,7 +38932,7 @@ snapshots: react: 19.1.0 scheduler: 0.26.0 - react-email@2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(eslint@8.31.0): + react-email@2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(bufferutil@4.0.9)(eslint@8.31.0): dependencies: '@babel/parser': 7.24.1 '@radix-ui/colors': 1.0.1 @@ -39004,8 +38969,8 @@ snapshots: react: 18.3.1 react-dom: 18.2.0(react@18.3.1) shelljs: 0.8.5 - socket.io: 4.7.3 - socket.io-client: 4.7.3 + socket.io: 4.7.3(bufferutil@4.0.9) + socket.io-client: 4.7.3(bufferutil@4.0.9) sonner: 1.3.1(react-dom@18.2.0(react@18.3.1))(react@18.3.1) source-map-js: 1.0.2 stacktrace-parser: 0.1.10 @@ -40152,7 +40117,7 @@ snapshots: - supports-color - utf-8-validate - socket.io-client@4.7.3: + socket.io-client@4.7.3(bufferutil@4.0.9): dependencies: '@socket.io/component-emitter': 3.1.0 debug: 4.3.7(supports-color@10.0.0) @@ -40181,7 +40146,7 @@ snapshots: transitivePeerDependencies: - supports-color - socket.io@4.7.3: + socket.io@4.7.3(bufferutil@4.0.9): dependencies: accepts: 1.3.8 base64id: 2.0.0 @@ -41376,8 +41341,6 @@ snapshots: undici-types@5.26.5: {} - undici-types@6.20.0: {} - undici@5.29.0: dependencies: '@fastify/busboy': 2.1.1 @@ -41697,7 +41660,7 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 - vite-node@0.28.5(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): + vite-node@0.28.5(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1): dependencies: cac: 6.7.14 debug: 4.4.1(supports-color@10.0.0) @@ -41706,7 +41669,7 @@ snapshots: picocolors: 1.1.1 source-map: 0.6.1 source-map-support: 0.5.21 - vite: 4.4.9(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + vite: 4.4.9(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) transitivePeerDependencies: - '@types/node' - less @@ -41762,13 +41725,13 @@ snapshots: - supports-color - typescript - vite@4.4.9(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): + vite@4.4.9(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1): dependencies: esbuild: 0.18.11 postcss: 8.5.6 rollup: 3.29.1 optionalDependencies: - '@types/node': 22.13.9 + '@types/node': 20.14.14 fsevents: 2.3.3 lightningcss: 1.29.2 terser: 5.44.1 From bc7ce781030423e570c132c1004608a93d0f23e5 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 30 Jan 2026 09:17:37 +0000 Subject: [PATCH 016/400] fix(sdk): export AnyOnStartAttemptHookFunction type (#2966) Export AnyOnStartAttemptHookFunction type to allow defining onStartAttempt hooks for individual tasks. https://claude.ai/code/session_018jgSVcFtKVyv65ktGNQFFq --- Co-authored-by: Claude --- .changeset/export-start-attempt-hook-type.md | 5 +++++ packages/trigger-sdk/src/v3/hooks.ts | 1 + 2 files changed, 6 insertions(+) create mode 100644 .changeset/export-start-attempt-hook-type.md diff --git a/.changeset/export-start-attempt-hook-type.md b/.changeset/export-start-attempt-hook-type.md new file mode 100644 index 00000000000..bad7c5258b7 --- /dev/null +++ b/.changeset/export-start-attempt-hook-type.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/sdk": patch +--- + +Export `AnyOnStartAttemptHookFunction` type to allow defining `onStartAttempt` hooks for individual tasks. diff --git a/packages/trigger-sdk/src/v3/hooks.ts b/packages/trigger-sdk/src/v3/hooks.ts index c6811ca6e9b..9c4bd8eb691 100644 --- a/packages/trigger-sdk/src/v3/hooks.ts +++ b/packages/trigger-sdk/src/v3/hooks.ts @@ -17,6 +17,7 @@ import { export type { AnyOnStartHookFunction, + AnyOnStartAttemptHookFunction, TaskStartHookParams, OnStartHookFunction, AnyOnFailureHookFunction, From e6861f4fe4761b18d3f70d7bfb4cbdebe989c377 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 30 Jan 2026 09:47:56 +0000 Subject: [PATCH 017/400] Fix: run page logs keep refreshing when a run finishes (#2971) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #2798 When a run finished the logs UI could get stuck and so be pending and never update again. If you did a hard reload it would be correct. This happened because when we insert a log/span we ping Redis which causes a reload of the UI. However there was a race condition – the insert into ClickHouse can take a while so we were refreshing the UI too early. Then never refreshing it again. Changes - Send refresh pings every 5s to keep run page logs live - Throttle updates so the run UI is never updated more than once per second - Stop auto-reloading when a run has been completed for >= 30s - Add type inference improvements for the throttle function --- --- .../v3/RunStreamPresenter.server.ts | 59 ++++++++++--------- .../route.tsx | 25 +++++++- apps/webapp/app/utils/throttle.ts | 8 +-- 3 files changed, 58 insertions(+), 34 deletions(-) diff --git a/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts b/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts index 9d54020ad2b..1dd4edc6233 100644 --- a/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts @@ -1,12 +1,12 @@ -import { PrismaClient, prisma } from "~/db.server"; +import { type PrismaClient, prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; import { singleton } from "~/utils/singleton"; -import { createSSELoader } from "~/utils/sse"; +import { createSSELoader, SendFunction } from "~/utils/sse"; import { throttle } from "~/utils/throttle"; import { tracePubSub } from "~/v3/services/tracePubSub.server"; -const PING_INTERVAL = 1000; -const STREAM_TIMEOUT = 30 * 1000; // 30 seconds +const PING_INTERVAL = 5_000; +const STREAM_TIMEOUT = 30_000; export class RunStreamPresenter { #prismaClient: PrismaClient; @@ -49,36 +49,40 @@ export class RunStreamPresenter { // Subscribe to trace updates const { unsubscribe, eventEmitter } = await tracePubSub.subscribeToTrace(run.traceId); - // Store throttled send function and message listener for cleanup - let throttledSend: ReturnType | undefined; + // Only send max every 1 second + const throttledSend = throttle( + (args: { send: SendFunction; event?: string; data: string }) => { + try { + args.send({ event: args.event, data: args.data }); + } catch (error) { + if (error instanceof Error) { + if (error.name !== "TypeError") { + logger.debug("Error sending SSE in RunStreamPresenter", { + error: { + name: error.name, + message: error.message, + stack: error.stack, + }, + }); + } + } + // Abort the stream on send error + context.controller.abort("Send error"); + } + }, + 1000 + ); + let messageListener: ((event: string) => void) | undefined; return { initStream: ({ send }) => { // Create throttled send function - throttledSend = throttle((args: { event?: string; data: string }) => { - try { - send(args); - } catch (error) { - if (error instanceof Error) { - if (error.name !== "TypeError") { - logger.debug("Error sending SSE in RunStreamPresenter", { - error: { - name: error.name, - message: error.message, - stack: error.stack, - }, - }); - } - } - // Abort the stream on send error - context.controller.abort("Send error"); - } - }, 1000); + throttledSend({ send, event: "message", data: new Date().toISOString() }); // Set up message listener for pub/sub events messageListener = (event: string) => { - throttledSend?.({ data: event }); + throttledSend({ send, event: "message", data: event }); }; eventEmitter.addListener("message", messageListener); @@ -88,7 +92,8 @@ export class RunStreamPresenter { iterator: ({ send }) => { // Send ping to keep connection alive try { - send({ event: "ping", data: new Date().toISOString() }); + // Send an actual message so the client refreshes + throttledSend({ send, event: "message", data: new Date().toISOString() }); } catch (error) { // If we can't send a ping, the connection is likely dead return false; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx index 899306eb816..1ffd128b308 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx @@ -436,6 +436,24 @@ export default function Page() { ); } +function shouldLiveReload({ + events, + maximumLiveReloadingSetting, + run, +}: { + events: TraceEvent[]; + maximumLiveReloadingSetting: number; + run: { completedAt: string | null }; +}): boolean { + // We don't live reload if there are a ton of spans/logs + if (events.length > maximumLiveReloadingSetting) return false; + + // If the run was completed a while ago, we don't need to live reload anymore + if (run.completedAt && new Date(run.completedAt).getTime() < Date.now() - 30_000) return false; + + return true; +} + function TraceView({ run, trace, @@ -453,18 +471,19 @@ function TraceView({ const { events, duration, rootSpanStatus, rootStartedAt, queuedDuration, overridesBySpanId } = trace; - const shouldLiveReload = events.length <= maximumLiveReloadingSetting; const changeToSpan = useDebounce((selectedSpan: string) => { replaceSearchParam("span", selectedSpan, { replace: true }); }, 250); + const isLiveReloading = shouldLiveReload({ events, maximumLiveReloadingSetting, run }); + const revalidator = useRevalidator(); const streamedEvents = useEventSource( v3RunStreamingPath(organization, project, environment, run), { event: "message", - disabled: !shouldLiveReload, + disabled: !isLiveReloading, } ); useEffect(() => { @@ -511,7 +530,7 @@ function TraceView({ rootStartedAt={rootStartedAt ? new Date(rootStartedAt) : undefined} queuedDuration={queuedDuration} environmentType={run.environment.type} - shouldLiveReload={shouldLiveReload} + shouldLiveReload={isLiveReloading} maximumLiveReloadingSetting={maximumLiveReloadingSetting} rootRun={run.rootTaskRun} parentRun={run.parentTaskRun} diff --git a/apps/webapp/app/utils/throttle.ts b/apps/webapp/app/utils/throttle.ts index a6c1a77a32d..5b264ebd862 100644 --- a/apps/webapp/app/utils/throttle.ts +++ b/apps/webapp/app/utils/throttle.ts @@ -1,13 +1,13 @@ //From: https://kettanaito.com/blog/debounce-vs-throttle /** A very simple throttle. Will execute the function at the end of each period and discard any other calls during that period. */ -export function throttle( - func: (...args: any[]) => void, +export function throttle( + func: (...args: TArgs) => void, durationMs: number -): (...args: any[]) => void { +): (...args: TArgs) => void { let isPrimedToFire = false; - return (...args: any[]) => { + return (...args: TArgs) => { if (!isPrimedToFire) { isPrimedToFire = true; From b221719c09a9c48ec937bfaa68db5e49d31dbd8c Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 30 Jan 2026 10:02:30 +0000 Subject: [PATCH 018/400] chore(repo): fixed missing dependency in pnpm lockfile (#2976) --- --- pnpm-lock.yaml | 88 ++++---------------------------------------------- 1 file changed, 7 insertions(+), 81 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ba4facc3fd5..99024a016bb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1092,7 +1092,7 @@ importers: version: 18.3.1 react-email: specifier: ^2.1.1 - version: 2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(bufferutil@4.0.9)(eslint@8.31.0) + version: 2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(eslint@8.31.0) resend: specifier: ^3.2.0 version: 3.2.0 @@ -1269,7 +1269,7 @@ importers: version: 5.5.4 vitest: specifier: 3.1.4 - version: 3.1.4(@types/debug@4.1.12)(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + version: 3.1.4(@types/debug@4.1.12)(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) internal-packages/testcontainers: dependencies: @@ -31363,14 +31363,6 @@ snapshots: optionalDependencies: vite: 5.4.21(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) - '@vitest/mocker@3.1.4(vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1))': - dependencies: - '@vitest/spy': 3.1.4 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) - '@vitest/pretty-format@2.1.9': dependencies: tinyrainbow: 1.2.0 @@ -38932,7 +38924,7 @@ snapshots: react: 19.1.0 scheduler: 0.26.0 - react-email@2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(bufferutil@4.0.9)(eslint@8.31.0): + react-email@2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(eslint@8.31.0): dependencies: '@babel/parser': 7.24.1 '@radix-ui/colors': 1.0.1 @@ -38969,8 +38961,8 @@ snapshots: react: 18.3.1 react-dom: 18.2.0(react@18.3.1) shelljs: 0.8.5 - socket.io: 4.7.3(bufferutil@4.0.9) - socket.io-client: 4.7.3(bufferutil@4.0.9) + socket.io: 4.7.3 + socket.io-client: 4.7.3 sonner: 1.3.1(react-dom@18.2.0(react@18.3.1))(react@18.3.1) source-map-js: 1.0.2 stacktrace-parser: 0.1.10 @@ -40117,7 +40109,7 @@ snapshots: - supports-color - utf-8-validate - socket.io-client@4.7.3(bufferutil@4.0.9): + socket.io-client@4.7.3: dependencies: '@socket.io/component-emitter': 3.1.0 debug: 4.3.7(supports-color@10.0.0) @@ -40146,7 +40138,7 @@ snapshots: transitivePeerDependencies: - supports-color - socket.io@4.7.3(bufferutil@4.0.9): + socket.io@4.7.3: dependencies: accepts: 1.3.8 base64id: 2.0.0 @@ -41698,24 +41690,6 @@ snapshots: - supports-color - terser - vite-node@3.1.4(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): - dependencies: - cac: 6.7.14 - debug: 4.4.1(supports-color@10.0.0) - es-module-lexer: 1.7.0 - pathe: 2.0.3 - vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) - transitivePeerDependencies: - - '@types/node' - - less - - lightningcss - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - vite-tsconfig-paths@4.0.5(typescript@5.5.4): dependencies: debug: 4.3.7(supports-color@10.0.0) @@ -41747,17 +41721,6 @@ snapshots: lightningcss: 1.29.2 terser: 5.44.1 - vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): - dependencies: - esbuild: 0.21.5 - postcss: 8.5.6 - rollup: 4.36.0 - optionalDependencies: - '@types/node': 22.13.9 - fsevents: 2.3.3 - lightningcss: 1.29.2 - terser: 5.44.1 - vitest@3.1.4(@types/debug@4.1.12)(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1): dependencies: '@vitest/expect': 3.1.4 @@ -41795,43 +41758,6 @@ snapshots: - supports-color - terser - vitest@3.1.4(@types/debug@4.1.12)(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): - dependencies: - '@vitest/expect': 3.1.4 - '@vitest/mocker': 3.1.4(vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1)) - '@vitest/pretty-format': 3.1.4 - '@vitest/runner': 3.1.4 - '@vitest/snapshot': 3.1.4 - '@vitest/spy': 3.1.4 - '@vitest/utils': 3.1.4 - chai: 5.2.0 - debug: 4.4.1(supports-color@10.0.0) - expect-type: 1.2.1 - magic-string: 0.30.21 - pathe: 2.0.3 - std-env: 3.9.0 - tinybench: 2.9.0 - tinyexec: 0.3.2 - tinyglobby: 0.2.13 - tinypool: 1.0.2 - tinyrainbow: 2.0.0 - vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) - vite-node: 3.1.4(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) - why-is-node-running: 2.3.0 - optionalDependencies: - '@types/debug': 4.1.12 - '@types/node': 22.13.9 - transitivePeerDependencies: - - less - - lightningcss - - msw - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - vscode-jsonrpc@8.2.0: {} vscode-languageserver-protocol@3.17.5: From 279102c17c4f21f1875b5a2bfcbd52baa5553641 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 30 Jan 2026 16:44:46 +0000 Subject: [PATCH 019/400] fix(cli): reject execute() immediately when child process is dead (#2978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - When a child process crashes and a retry (`RETRY_IMMEDIATELY`) is attempted on the same `TaskRunProcess`, `execute()` hangs forever because the IPC send is silently skipped and the attempt promise can never resolve - This caused runner pods to stay up indefinitely with no heartbeats or polls - Fix: reject the attempt promise immediately when the child is not connected, so the controller can proceed to warm start or exit ## Test plan - [x] Added `taskRunProcess.test.ts` — verifies `execute()` rejects promptly instead of hanging when the child process is dead - [x] Deploy and verify no more stuck runner pods accumulate over time --- .changeset/fix-dead-process-execute-hang.md | 5 + .../src/executions/taskRunProcess.test.ts | 121 ++++++++++++++++++ .../cli-v3/src/executions/taskRunProcess.ts | 13 ++ 3 files changed, 139 insertions(+) create mode 100644 .changeset/fix-dead-process-execute-hang.md create mode 100644 packages/cli-v3/src/executions/taskRunProcess.test.ts diff --git a/.changeset/fix-dead-process-execute-hang.md b/.changeset/fix-dead-process-execute-hang.md new file mode 100644 index 00000000000..fa96e9c88c3 --- /dev/null +++ b/.changeset/fix-dead-process-execute-hang.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Fix runner getting stuck indefinitely when `execute()` is called on a dead child process. diff --git a/packages/cli-v3/src/executions/taskRunProcess.test.ts b/packages/cli-v3/src/executions/taskRunProcess.test.ts new file mode 100644 index 00000000000..82ab19639b2 --- /dev/null +++ b/packages/cli-v3/src/executions/taskRunProcess.test.ts @@ -0,0 +1,121 @@ +import { TaskRunProcess, type TaskRunProcessOptions } from "./taskRunProcess.js"; +import { describe, it, expect, vi } from "vitest"; +import { UnexpectedExitError } from "@trigger.dev/core/v3/errors"; +import type { + TaskRunExecution, + TaskRunExecutionPayload, + WorkerManifest, + ServerBackgroundWorker, + MachinePresetResources, +} from "@trigger.dev/core/v3"; + +function createTaskRunProcessOptions( + overrides: Partial = {} +): TaskRunProcessOptions { + return { + workerManifest: { + runtime: "node", + workerEntryPoint: "/dev/null", + configEntryPoint: "/dev/null", + otelImportHook: {}, + } as unknown as WorkerManifest, + serverWorker: {} as unknown as ServerBackgroundWorker, + env: {}, + machineResources: { cpu: 1, memory: 1 } as MachinePresetResources, + ...overrides, + }; +} + +function createExecution(runId: string, attemptNumber: number): TaskRunExecution { + return { + run: { + id: runId, + payload: "{}", + payloadType: "application/json", + tags: [], + isTest: false, + createdAt: new Date(), + startedAt: new Date(), + maxAttempts: 3, + version: "1", + durationMs: 0, + costInCents: 0, + baseCostInCents: 0, + }, + attempt: { + number: attemptNumber, + startedAt: new Date(), + id: "deprecated", + backgroundWorkerId: "deprecated", + backgroundWorkerTaskId: "deprecated", + status: "deprecated" as any, + }, + task: { id: "test-task", filePath: "test.ts" }, + queue: { id: "queue-1", name: "test-queue" }, + environment: { id: "env-1", slug: "test", type: "DEVELOPMENT" }, + organization: { id: "org-1", slug: "test-org", name: "Test Org" }, + project: { id: "proj-1", ref: "proj_test", slug: "test", name: "Test" }, + machine: { name: "small-1x", cpu: 0.5, memory: 0.5, centsPerMs: 0 }, + } as unknown as TaskRunExecution; +} + +describe("TaskRunProcess", () => { + describe("execute() on a dead child process", () => { + it("should reject when child process has already exited and IPC send is skipped", async () => { + const proc = new TaskRunProcess(createTaskRunProcessOptions()); + + // Simulate a child process that has exited: _child exists but is not connected + const fakeChild = { + connected: false, + killed: false, + pid: 12345, + kill: vi.fn(), + on: vi.fn(), + stdout: { on: vi.fn() }, + stderr: { on: vi.fn() }, + }; + + // Set internal state to mimic a process whose child has crashed + (proc as any)._child = fakeChild; + (proc as any)._childPid = 12345; + (proc as any)._isBeingKilled = false; + + const execution = createExecution("run-1", 2); + + // This should NOT hang forever - it should reject promptly. + // + // BUG: Currently execute() creates a promise, skips the IPC send because + // _child.connected is false, then awaits the promise which will never + // resolve because the child is dead and #handleExit already ran. + // + // The Promise.race with a timeout detects the hang. + const result = await Promise.race([ + proc + .execute( + { + payload: { execution, traceContext: {}, metrics: [] }, + messageId: "run_run-1", + env: {}, + }, + true + ) + .then( + (v) => ({ type: "resolved" as const, value: v }), + (e) => ({ type: "rejected" as const, error: e }) + ), + new Promise<{ type: "hung" }>((resolve) => + setTimeout(() => resolve({ type: "hung" as const }), 2000) + ), + ]); + + // The test fails (proving the bug) if execute() hangs + expect(result.type).not.toBe("hung"); + expect(result.type).toBe("rejected"); + + if (result.type === "rejected") { + expect(result.error).toBeInstanceOf(UnexpectedExitError); + expect(result.error.stderr).toContain("not connected"); + } + }); + }); +}); diff --git a/packages/cli-v3/src/executions/taskRunProcess.ts b/packages/cli-v3/src/executions/taskRunProcess.ts index 098b0f261c4..1e274ba02f0 100644 --- a/packages/cli-v3/src/executions/taskRunProcess.ts +++ b/packages/cli-v3/src/executions/taskRunProcess.ts @@ -297,6 +297,19 @@ export class TaskRunProcess { env: params.env, isWarmStart: isWarmStart ?? this.options.isWarmStart, }); + } else { + // Child process is dead or disconnected — the IPC send was skipped so the attempt + // promise would hang forever. Reject it immediately to let the caller handle it. + this._attemptStatuses.set(key, "REJECTED"); + + // @ts-expect-error - rejecter is assigned in the promise constructor above + rejecter( + new UnexpectedExitError( + -1, + null, + "Child process is not connected, cannot execute task run" + ) + ); } const result = await promise; From 1ccb8c186fa50743d6acf4c002645a788a894e90 Mon Sep 17 00:00:00 2001 From: Mihai Popescu Date: Mon, 2 Feb 2026 02:43:39 +0200 Subject: [PATCH 020/400] changed schema id (#2983) Fixed duplicate schema id for clickhouse --- ..._serch_indexes.sql => 015_add_task_runs_v2_search_indexes.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal-packages/clickhouse/schema/{014_add_task_runs_v2_serch_indexes.sql => 015_add_task_runs_v2_search_indexes.sql} (100%) diff --git a/internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql b/internal-packages/clickhouse/schema/015_add_task_runs_v2_search_indexes.sql similarity index 100% rename from internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql rename to internal-packages/clickhouse/schema/015_add_task_runs_v2_search_indexes.sql From b72cacc6710296c5095e9e14c2829aa82b714964 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 2 Feb 2026 20:15:06 +0000 Subject: [PATCH 021/400] feat(debounce): add maxDelay option to limit total debounce time (#2984) --- .changeset/add-debounce-maxdelay.md | 16 + .../src/engine/systems/debounceSystem.ts | 30 +- .../src/engine/tests/debounce.test.ts | 327 ++++++++++++++++++ .../run-engine/src/engine/types.ts | 1 + packages/core/src/v3/isomorphic/duration.ts | 23 +- packages/core/src/v3/schemas/api.ts | 2 + packages/core/src/v3/types/tasks.ts | 16 + .../hello-world/src/trigger/debounce.ts | 156 +++++++++ 8 files changed, 562 insertions(+), 9 deletions(-) create mode 100644 .changeset/add-debounce-maxdelay.md diff --git a/.changeset/add-debounce-maxdelay.md b/.changeset/add-debounce-maxdelay.md new file mode 100644 index 00000000000..a70b95d4765 --- /dev/null +++ b/.changeset/add-debounce-maxdelay.md @@ -0,0 +1,16 @@ +--- +"@trigger.dev/core": patch +"@trigger.dev/sdk": patch +--- + +Add `maxDelay` option to debounce feature. This allows setting a maximum time limit for how long a debounced run can be delayed, ensuring execution happens within a specified window even with continuous triggers. + +```typescript +await myTask.trigger(payload, { + debounce: { + key: "my-key", + delay: "5s", + maxDelay: "30m", // Execute within 30 minutes regardless of continuous triggers + }, +}); +``` diff --git a/internal-packages/run-engine/src/engine/systems/debounceSystem.ts b/internal-packages/run-engine/src/engine/systems/debounceSystem.ts index af25a31552f..8cd06d07732 100644 --- a/internal-packages/run-engine/src/engine/systems/debounceSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/debounceSystem.ts @@ -6,7 +6,10 @@ import { type Result, } from "@internal/redis"; import { startSpan } from "@internal/tracing"; -import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/isomorphic"; +import { + parseNaturalLanguageDuration, + parseNaturalLanguageDurationInMs, +} from "@trigger.dev/core/v3/isomorphic"; import { PrismaClientOrTransaction, TaskRun, Waitpoint } from "@trigger.dev/database"; import { nanoid } from "nanoid"; import { SystemResources } from "./systems.js"; @@ -17,6 +20,12 @@ export type DebounceOptions = { key: string; delay: string; mode?: "leading" | "trailing"; + /** + * Maximum total delay before the run must execute, regardless of subsequent triggers. + * This prevents indefinite delays when continuous triggers keep pushing the execution time. + * If not specified, falls back to the server's maxDebounceDurationMs config. + */ + maxDelay?: string; /** When mode: "trailing", these fields will be used to update the existing run */ updateData?: { payload: string; @@ -521,8 +530,22 @@ return 0 } // Check if max debounce duration would be exceeded + // Use per-trigger maxDelay if provided, otherwise use global config + let maxDurationMs = this.maxDebounceDurationMs; + if (debounce.maxDelay) { + const parsedMaxDelay = parseNaturalLanguageDurationInMs(debounce.maxDelay); + if (parsedMaxDelay !== undefined) { + maxDurationMs = parsedMaxDelay; + } else { + this.$.logger.warn("handleExistingRun: invalid maxDelay duration, using global config", { + maxDelay: debounce.maxDelay, + fallbackMs: this.maxDebounceDurationMs, + }); + } + } + const runCreatedAt = existingRun.createdAt; - const maxDelayUntil = new Date(runCreatedAt.getTime() + this.maxDebounceDurationMs); + const maxDelayUntil = new Date(runCreatedAt.getTime() + maxDurationMs); if (newDelayUntil > maxDelayUntil) { this.$.logger.debug("handleExistingRun: max debounce duration would be exceeded", { @@ -531,7 +554,8 @@ return 0 runCreatedAt, newDelayUntil, maxDelayUntil, - maxDebounceDurationMs: this.maxDebounceDurationMs, + maxDurationMs, + maxDelayProvided: debounce.maxDelay, }); // Clean up Redis key since this debounce window is closed await this.redis.del(redisKey); diff --git a/internal-packages/run-engine/src/engine/tests/debounce.test.ts b/internal-packages/run-engine/src/engine/tests/debounce.test.ts index 0c3d09d887c..1c201c4b4c4 100644 --- a/internal-packages/run-engine/src/engine/tests/debounce.test.ts +++ b/internal-packages/run-engine/src/engine/tests/debounce.test.ts @@ -2170,5 +2170,332 @@ describe("RunEngine debounce", () => { } } ); + + containerTest( + "Debounce: per-trigger maxDelay overrides global maxDebounceDuration", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + // Set a long global max debounce duration (1 minute) + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + debounce: { + maxDebounceDurationMs: 60_000, // 1 minute global max + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + // First trigger with a very short per-trigger maxDelay (1 second) + const run1 = await engine.trigger( + { + number: 1, + friendlyId: "run_maxwait1", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "first"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 5000), + debounce: { + key: "maxwait-key", + delay: "5s", + maxDelay: "1s", // Very short per-trigger maxDelay (1 second) + }, + }, + prisma + ); + + expect(run1.friendlyId).toBe("run_maxwait1"); + + // Wait for the per-trigger maxDelay to be exceeded (1.5s > 1s) + await setTimeout(1500); + + // Second trigger should create a new run because per-trigger maxDelay exceeded + // (even though global maxDebounceDurationMs is 60 seconds) + const run2 = await engine.trigger( + { + number: 2, + friendlyId: "run_maxwait2", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "second"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12346", + spanId: "s12346", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 5000), + debounce: { + key: "maxwait-key", + delay: "5s", + maxDelay: "1s", + }, + }, + prisma + ); + + // Should be a different run because per-trigger maxDelay was exceeded + expect(run2.id).not.toBe(run1.id); + expect(run2.friendlyId).toBe("run_maxwait2"); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "Debounce: falls back to global config when maxDelay not specified", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + // Set a very short global max debounce duration (1 second) + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + debounce: { + maxDebounceDurationMs: 1000, // 1 second global max + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + // First trigger without maxDelay - should use global config + const run1 = await engine.trigger( + { + number: 1, + friendlyId: "run_noglobal1", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "first"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 5000), + debounce: { + key: "global-fallback-key", + delay: "5s", + // No maxDelay specified - should use global maxDebounceDurationMs + }, + }, + prisma + ); + + // Wait for global maxDebounceDurationMs to be exceeded (1.5s > 1s) + await setTimeout(1500); + + // Second trigger should create a new run because global max exceeded + const run2 = await engine.trigger( + { + number: 2, + friendlyId: "run_noglobal2", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "second"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12346", + spanId: "s12346", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 5000), + debounce: { + key: "global-fallback-key", + delay: "5s", + }, + }, + prisma + ); + + // Should be a different run because global max exceeded + expect(run2.id).not.toBe(run1.id); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "Debounce: long maxDelay allows more debounce time than global config", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + // Set a short global max debounce duration (1 second) + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + debounce: { + maxDebounceDurationMs: 1000, // 1 second global max + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + // First trigger with long maxDelay that overrides the short global config + const run1 = await engine.trigger( + { + number: 1, + friendlyId: "run_longmax1", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "first"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 2000), + debounce: { + key: "long-maxwait-key", + delay: "2s", + maxDelay: "60s", // Long per-trigger maxDelay overrides short global config + }, + }, + prisma + ); + + // Wait past the global maxDebounceDurationMs (1s) but within our per-trigger maxDelay (60s) + await setTimeout(1500); + + // Second trigger should return SAME run because per-trigger maxDelay is 60s + const run2 = await engine.trigger( + { + number: 2, + friendlyId: "run_longmax2", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "second"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12346", + spanId: "s12346", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 2000), + debounce: { + key: "long-maxwait-key", + delay: "2s", + maxDelay: "60s", + }, + }, + prisma + ); + + // Should be the SAME run because per-trigger maxDelay allows it + expect(run2.id).toBe(run1.id); + } finally { + await engine.quit(); + } + } + ); }); diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts index ee5176c2fa1..2adc63415fb 100644 --- a/internal-packages/run-engine/src/engine/types.ts +++ b/internal-packages/run-engine/src/engine/types.ts @@ -180,6 +180,7 @@ export type TriggerParams = { key: string; delay: string; mode?: "leading" | "trailing"; + maxDelay?: string; }; /** * Called when a run is debounced (existing delayed run found with triggerAndWait). diff --git a/packages/core/src/v3/isomorphic/duration.ts b/packages/core/src/v3/isomorphic/duration.ts index b4c5cd20d3b..9315f79683d 100644 --- a/packages/core/src/v3/isomorphic/duration.ts +++ b/packages/core/src/v3/isomorphic/duration.ts @@ -1,5 +1,15 @@ -export function parseNaturalLanguageDuration(duration: string): Date | undefined { - // Handle Code scanning alert #44 (https://github.com/triggerdotdev/trigger.dev/security/code-scanning/44) by limiting the length of the input string +/** + * Parses a natural language duration string into milliseconds. + * + * @param duration - Duration string like "1s", "5m", "2h", "1d", "1w" + * @returns The duration in milliseconds, or undefined if invalid + * + * @example + * parseNaturalLanguageDurationInMs("30m") // 1800000 + * parseNaturalLanguageDurationInMs("2h") // 7200000 + */ +export function parseNaturalLanguageDurationInMs(duration: string): number | undefined { + // Handle Code scanning alert #44 by limiting the length of the input string if (duration.length > 100) { return undefined; } @@ -60,11 +70,12 @@ export function parseNaturalLanguageDuration(duration: string): Date | undefined } } - if (hasMatch) { - return new Date(Date.now() + totalMilliseconds); - } + return hasMatch ? totalMilliseconds : undefined; +} - return undefined; +export function parseNaturalLanguageDuration(duration: string): Date | undefined { + const ms = parseNaturalLanguageDurationInMs(duration); + return ms !== undefined ? new Date(Date.now() + ms) : undefined; } export function safeParseNaturalLanguageDuration(duration: string): Date | undefined { diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index 9080a7f596c..0291d2a05c2 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -218,6 +218,7 @@ export const TriggerTaskRequestBody = z.object({ key: z.string().max(512), delay: z.string(), mode: z.enum(["leading", "trailing"]).optional(), + maxDelay: z.string().optional(), }) .optional(), }) @@ -275,6 +276,7 @@ export const BatchTriggerTaskItem = z.object({ key: z.string().max(512), delay: z.string(), mode: z.enum(["leading", "trailing"]).optional(), + maxDelay: z.string().optional(), }) .optional(), }) diff --git a/packages/core/src/v3/types/tasks.ts b/packages/core/src/v3/types/tasks.ts index f463b20f49f..3b8b2e9ecdd 100644 --- a/packages/core/src/v3/types/tasks.ts +++ b/packages/core/src/v3/types/tasks.ts @@ -945,6 +945,22 @@ export type TriggerOptions = { * @default "leading" */ mode?: "leading" | "trailing"; + /** + * Maximum total delay before the run must execute, regardless of subsequent triggers. + * This prevents indefinite delays when continuous triggers keep pushing the execution time. + * + * When specified, if a new trigger would push the execution time beyond this limit + * (measured from the first trigger), the current debounced run will be allowed to execute + * and a new run will be created for subsequent triggers. + * + * If not specified, falls back to the server's default maximum (typically 1 hour). + * + * Supported formats: `{number}s` (seconds), `{number}m` (minutes), `{number}h` (hours), + * `{number}d` (days), `{number}w` (weeks). + * + * @example "30m", "2h", "1d" + */ + maxDelay?: string; }; }; diff --git a/references/hello-world/src/trigger/debounce.ts b/references/hello-world/src/trigger/debounce.ts index e396714eb8e..f0a7e8b6389 100644 --- a/references/hello-world/src/trigger/debounce.ts +++ b/references/hello-world/src/trigger/debounce.ts @@ -1056,3 +1056,159 @@ export const demonstrateTrailingWithMetadata = task({ }; }, }); + +/** + * Example 12: Debounce with maxDelay + * + * The maxDelay option limits how long a debounced run can be delayed. + * Even if triggers keep coming, the run will eventually execute after maxDelay + * from the first trigger. This is useful for scenarios where you want to + * debounce but also guarantee execution within a certain time window. + * + * Use case: Summarizing AI conversation threads that need to stay relatively + * up to date. You want to debounce as messages come in, but also guarantee + * the summary runs at least every 30 minutes. + */ +export const processConversationSummary = task({ + id: "process-conversation-summary", + run: async (payload: { conversationId: string; messageCount: number }) => { + logger.info("Generating conversation summary", { payload }); + + // Simulate AI summarization work + await wait.for({ seconds: 2 }); + + logger.info("Conversation summary generated", { + conversationId: payload.conversationId, + messageCount: payload.messageCount, + }); + + return { + summarized: true, + conversationId: payload.conversationId, + messageCount: payload.messageCount, + summarizedAt: new Date().toISOString(), + }; + }, +}); + +/** + * Demonstrates maxDelay in action. + * + * This simulates a chat application where messages come in continuously. + * With just debounce, the summary task would keep getting delayed forever. + * With maxDelay: "30s", the summary will run at most 30 seconds after the first trigger, + * even if messages keep coming. + * + * Run this task and observe: + * - Messages trigger the summary task with debounce + * - Each trigger extends the delay by 5s + * - But maxDelay ensures execution happens within 30s of the first trigger + */ +export const simulateChatWithMaxWait = task({ + id: "simulate-chat-with-max-wait", + run: async (payload: { conversationId?: string; simulateDelay?: number }) => { + const conversationId = payload.conversationId ?? "conv-123"; + const delayBetweenMessages = payload.simulateDelay ?? 3000; // 3 seconds + + logger.info("Starting chat simulation with maxDelay", { + conversationId, + delayBetweenMessages, + }); + + logger.info( + "Debounce delay is 5s, maxDelay is 30s. Messages arrive every 3s, so debounce would normally keep extending. But maxDelay ensures execution within 30s." + ); + + const handles: string[] = []; + + // Simulate 15 messages over ~45 seconds + // Without maxDelay, the task would never run because each trigger resets the 5s delay + // With maxDelay: "30s", the task will run after 30 seconds from the first trigger + for (let i = 1; i <= 15; i++) { + logger.info(`Message ${i}/15 received`, { messageNumber: i }); + + const handle = await processConversationSummary.trigger( + { + conversationId, + messageCount: i, + }, + { + debounce: { + key: `conversation-${conversationId}`, + delay: "5s", + mode: "trailing", // Use latest message count + maxDelay: "30s", // Ensure execution within 30s of first trigger + }, + } + ); + + handles.push(handle.id); + logger.info(`Message ${i} triggered, run ID: ${handle.id}`, { + messageNumber: i, + runId: handle.id, + }); + + // Wait between messages (simulating real chat) + if (i < 15) { + await new Promise((resolve) => setTimeout(resolve, delayBetweenMessages)); + } + } + + const uniqueHandles = [...new Set(handles)]; + + logger.info("Chat simulation complete", { + totalMessages: 15, + uniqueRuns: uniqueHandles.length, + note: + "With maxDelay, runs should have been created periodically despite continuous triggering", + }); + + return { + conversationId, + totalMessages: 15, + uniqueRunsCreated: uniqueHandles.length, + runIds: uniqueHandles, + message: + "Due to maxDelay: '30s', the summary task runs periodically even with continuous triggers", + }; + }, +}); + +/** + * A simpler maxDelay example showing the basic usage pattern. + * + * This is the recommended pattern for using maxDelay: + * - delay: How long to wait after each trigger before executing + * - maxDelay: Maximum total wait time from the first trigger + */ +export const onNewMessage = task({ + id: "on-new-message", + run: async (payload: { conversationId: string; message: string }) => { + logger.info("New message received", { + conversationId: payload.conversationId, + messagePreview: payload.message.substring(0, 50), + }); + + // Trigger summarization with debounce and maxDelay + const handle = await processConversationSummary.trigger( + { + conversationId: payload.conversationId, + messageCount: 1, // In real code, you'd track actual count + }, + { + debounce: { + key: `summary-${payload.conversationId}`, + delay: "10s", // Wait 10s after last message before summarizing + mode: "trailing", // Use latest state + maxDelay: "5m", // But always summarize within 5 minutes + }, + } + ); + + logger.info("Summary task triggered (debounced with maxDelay)", { + runId: handle.id, + }); + + return { summaryRunId: handle.id }; + }, +}); From 8e0034484cf42b2f6dfd43cceb621b162b5f16e4 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 4 Feb 2026 14:39:47 +0100 Subject: [PATCH 022/400] feat(supervisor): project-based scheduling affinity for image cache locality (#2995) Adds optional pod affinity so pods from the same project prefer scheduling on the same node. This can help improve image cache hit rates; subsequent pods benefit from already-pulled image layers, reducing startup time. Complements the built-in ImageLocality scheduler plugin by helping during burst scheduling scenarios. Pod affinity sees scheduled pods immediately, while ImageLocality only sees images after they're fully pulled. Configuration: - `KUBERNETES_PROJECT_AFFINITY_ENABLED` - Enable/disable (default: false) - `KUBERNETES_PROJECT_AFFINITY_WEIGHT` - Scheduler weight 1-100 (default: 50) - `KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY` - Topology key (default: kubernetes.io/hostname) Uses soft (preferred) affinity so pods always schedule even if preferred node is full. --- --- apps/supervisor/src/env.ts | 5 ++ .../src/workloadManager/kubernetes.ts | 88 +++++++++++++------ 2 files changed, 67 insertions(+), 26 deletions(-) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 9ef0cff2537..faf34bcd025 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -112,6 +112,11 @@ const Env = z.object({ KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods KUBERNETES_LARGE_MACHINE_POOL_LABEL: z.string().optional(), // if set, large-* presets affinity for machinepool= + // Project affinity settings - pods from the same project prefer the same node + KUBERNETES_PROJECT_AFFINITY_ENABLED: BoolEnv.default(false), + KUBERNETES_PROJECT_AFFINITY_WEIGHT: z.coerce.number().int().min(1).max(100).default(50), + KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY: z.string().trim().min(1).default("kubernetes.io/hostname"), + // Placement tags settings PLACEMENT_TAGS_ENABLED: BoolEnv.default(false), PLACEMENT_TAGS_PREFIX: z.string().default("node.cluster.x-k8s.io"), diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index a725971a845..16c5eff9da1 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -120,7 +120,7 @@ export class KubernetesWorkloadManager implements WorkloadManager { }, spec: { ...this.addPlacementTags(this.#defaultPodSpec, opts.placementTags), - affinity: this.#getNodeAffinity(opts.machine), + affinity: this.#getAffinity(opts.machine, opts.projectId), terminationGracePeriodSeconds: 60 * 60, containers: [ { @@ -390,7 +390,21 @@ export class KubernetesWorkloadManager implements WorkloadManager { return preset.name.startsWith("large-"); } - #getNodeAffinity(preset: MachinePreset): k8s.V1Affinity | undefined { + #getAffinity(preset: MachinePreset, projectId: string): k8s.V1Affinity | undefined { + const nodeAffinity = this.#getNodeAffinityRules(preset); + const podAffinity = this.#getProjectPodAffinity(projectId); + + if (!nodeAffinity && !podAffinity) { + return undefined; + } + + return { + ...(nodeAffinity && { nodeAffinity }), + ...(podAffinity && { podAffinity }), + }; + } + + #getNodeAffinityRules(preset: MachinePreset): k8s.V1NodeAffinity | undefined { if (!env.KUBERNETES_LARGE_MACHINE_POOL_LABEL) { return undefined; } @@ -398,42 +412,64 @@ export class KubernetesWorkloadManager implements WorkloadManager { if (this.#isLargeMachine(preset)) { // soft preference for the large-machine pool, falls back to standard if unavailable return { - nodeAffinity: { - preferredDuringSchedulingIgnoredDuringExecution: [ - { - weight: 100, - preference: { - matchExpressions: [ - { - key: "node.cluster.x-k8s.io/machinepool", - operator: "In", - values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], - }, - ], - }, + preferredDuringSchedulingIgnoredDuringExecution: [ + { + weight: 100, + preference: { + matchExpressions: [ + { + key: "node.cluster.x-k8s.io/machinepool", + operator: "In", + values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], + }, + ], }, - ], - }, + }, + ], }; } // not schedulable in the large-machine pool return { - nodeAffinity: { - requiredDuringSchedulingIgnoredDuringExecution: { - nodeSelectorTerms: [ - { + requiredDuringSchedulingIgnoredDuringExecution: { + nodeSelectorTerms: [ + { + matchExpressions: [ + { + key: "node.cluster.x-k8s.io/machinepool", + operator: "NotIn", + values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], + }, + ], + }, + ], + }, + }; + } + + #getProjectPodAffinity(projectId: string): k8s.V1PodAffinity | undefined { + if (!env.KUBERNETES_PROJECT_AFFINITY_ENABLED) { + return undefined; + } + + return { + preferredDuringSchedulingIgnoredDuringExecution: [ + { + weight: env.KUBERNETES_PROJECT_AFFINITY_WEIGHT, + podAffinityTerm: { + labelSelector: { matchExpressions: [ { - key: "node.cluster.x-k8s.io/machinepool", - operator: "NotIn", - values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], + key: "project", + operator: "In", + values: [projectId], }, ], }, - ], + topologyKey: env.KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY, + }, }, - }, + ], }; } } From 7781e2aad1e629b1df31b93f709716fc2bdef20f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 4 Feb 2026 06:05:59 -0800 Subject: [PATCH 023/400] docs: usage function examples were missing the imports (#2830) Closes #2828 --- docs/run-usage.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/run-usage.mdx b/docs/run-usage.mdx index 0b9163db1ba..4c00423310f 100644 --- a/docs/run-usage.mdx +++ b/docs/run-usage.mdx @@ -8,6 +8,8 @@ description: "Get compute duration and cost from inside a run, or for a specific You can get the cost and duration of the current including retries of the same run. ```ts +import { task, usage, wait } from "@trigger.dev/sdk"; + export const heavyTask = task({ id: "heavy-task", machine: { @@ -87,6 +89,8 @@ console.log("Total cost", totalCost); You can also wrap code with `usage.measure` to get the cost and duration of that block of code: ```ts +import { usage, logger } from "@trigger.dev/sdk"; + // Inside a task run function, or inside a function that's called from there. const { result, compute } = await usage.measure(async () => { //...Do something for 1 second From e0179130214801dfc20c6be7dc03229f1e46bf03 Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Wed, 4 Feb 2026 09:29:01 -0500 Subject: [PATCH 024/400] docs(self-hosting): added graphile worker troubleshooting to docs (#2883) Add troubleshooting documentation for graphile worker schema migration failures and PostgreSQL SSL certificate issues that prevent worker initialization. --- docs/self-hosting/docker.mdx | 2 ++ docs/self-hosting/kubernetes.mdx | 1 + 2 files changed, 3 insertions(+) diff --git a/docs/self-hosting/docker.mdx b/docs/self-hosting/docker.mdx index dbbc3578e80..add4e06ba2e 100644 --- a/docs/self-hosting/docker.mdx +++ b/docs/self-hosting/docker.mdx @@ -354,6 +354,8 @@ TRIGGER_IMAGE_TAG=v4.0.0 docker compose logs -f webapp ``` +- **Deploy fails with `ERROR: schema "graphile_worker" does not exist`.** This error occurs when Graphile Worker migrations fail to run during webapp startup. Check the webapp logs for certificate-related errors like `self-signed certificate in certificate chain`. This is often caused by PostgreSQL SSL certificate issues when using an external PostgreSQL instance with SSL enabled. Ensure that both the webapp and supervisor containers have access to the same CA certificate used by your PostgreSQL instance. You can configure this by mounting the certificate file and setting the `NODE_EXTRA_CA_CERTS` environment variable to point to the certificate path. Once the certificate issue is resolved, the migrations will complete and create the required `graphile_worker` schema. + ## CLI usage This section highlights some of the CLI commands and options that are useful when self-hosting. Please check the [CLI reference](/cli-introduction) for more in-depth documentation. diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx index 4506d6da9b8..eba66f4ed07 100644 --- a/docs/self-hosting/kubernetes.mdx +++ b/docs/self-hosting/kubernetes.mdx @@ -555,6 +555,7 @@ kubectl delete namespace trigger - **Deploy fails**: Verify registry access and authentication - **Pods stuck pending**: Describe the pod and check the events - **Worker token issues**: Check webapp and supervisor logs for errors +- **Deploy fails with `ERROR: schema "graphile_worker" does not exist`**: See the [Docker troubleshooting](/self-hosting/docker#troubleshooting) section for details on resolving PostgreSQL SSL certificate issues that prevent Graphile Worker migrations. See the [Docker troubleshooting](/self-hosting/docker#troubleshooting) section for more information. From 104f720f6ff0b27403a0c01c3cbbf85fc4e21f20 Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Wed, 4 Feb 2026 09:30:51 -0500 Subject: [PATCH 025/400] docs(troubleshooting): add COULD_NOT_FIND_EXECUTOR error and IPv4 support (#2950) Document COULD_NOT_FIND_EXECUTOR error with dynamic imports and IPv4 database connection limitation in troubleshooting guide. --- docs/troubleshooting.mdx | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index b6254f62d93..c5040e59217 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -151,6 +151,10 @@ Your code is deployed separately from the rest of your app(s) so you need to mak Prisma uses code generation to create the client from your schema file. This means you need to add a bit of config so we can generate this file before your tasks run: [Read the guide](/config/extensions/prismaExtension). +### Database connection requires IPv4 + +Trigger.dev currently only supports IPv4 database connections. If your database provider only provides an IPv6 connection string, you'll need to use an IPv4 address instead. [Upvote IPv6 support](https://triggerdev.featurebase.app/p/support-ipv6-database-connections). + ### `Parallel waits are not supported` In the current version, you can't perform more that one "wait" in parallel. @@ -171,6 +175,36 @@ The most common situation this happens is if you're using `Promise.all` around s Make sure that you always use `await` when you call `trigger`, `triggerAndWait`, `batchTrigger`, and `batchTriggerAndWait`. If you don't then it's likely the task(s) won't be triggered because the calling function process can be terminated before the networks calls are sent. +### `COULD_NOT_FIND_EXECUTOR` + +If you see a `COULD_NOT_FIND_EXECUTOR` error when triggering a task, it may be caused by dynamically importing the child task. When tasks are dynamically imported, the executor may not be properly registered. + +Use a top-level import instead: + +```ts +import { myChildTask } from "~/trigger/my-child-task"; + +export const myTask = task({ + id: "my-task", + run: async (payload: string) => { + await myChildTask.trigger({ payload: "data" }); + }, +}); +``` + +Alternatively, use `tasks.trigger()` or `batch.triggerAndWait()` without importing the task: + +```ts +import { batch } from "@trigger.dev/sdk"; + +export const myTask = task({ + id: "my-task", + run: async (payload: string) => { + await batch.triggerAndWait([{ id: "my-child-task", payload: "data" }]); + }, +}); +``` + ### Rate limit exceeded From 6a45f5623b89ed107176c5f8e461a4c2f2dde54d Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Wed, 4 Feb 2026 09:33:05 -0500 Subject: [PATCH 026/400] docs: multi-tenant applications and concurrency limits (#2961) Adds an example of multi-tenant applications as alternative to project/limit increase, and more info about queue times and concurrency --- docs/deploy-environment-variables.mdx | 53 ++++++++++++++++++++++++++- docs/limits.mdx | 8 ++++ docs/troubleshooting.mdx | 10 +++++ 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/docs/deploy-environment-variables.mdx b/docs/deploy-environment-variables.mdx index e123f341705..e7b64964035 100644 --- a/docs/deploy-environment-variables.mdx +++ b/docs/deploy-environment-variables.mdx @@ -360,4 +360,55 @@ This will read your .env.production file using dotenvx and sync the variables to - Trigger.dev does not automatically detect .env.production or dotenvx files - You can paste them manually into the dashboard -- Or sync them automatically using a build extension \ No newline at end of file +- Or sync them automatically using a build extension + +## Multi-tenant applications + +If you're building a multi-tenant application where each tenant needs different environment variables (like tenant-specific API keys or database credentials), you don't need a separate project for each tenant. Instead, use a single project and load tenant-specific secrets at runtime. + + + This is different from [syncing environment variables at deploy time](#sync-env-vars-from-another-service). + Here, secrets are loaded dynamically during task execution, not synced to Trigger.dev's environment variables. + + +### Recommended approach + +Use a secrets service (Infisical, AWS Secrets Manager, HashiCorp Vault, etc.) to store tenant-specific secrets, then retrieve them at the start of each task run based on the tenant identifier in your payload or context. + +**Important:** Never pass secrets in the task payload, as payloads are logged and visible in the dashboard. + +### Example implementation + +```ts +import { task } from "@trigger.dev/sdk"; +import { SecretsManagerClient, GetSecretValueCommand } from "@aws-sdk/client-secrets-manager"; + +export const processTenantData = task({ + id: "process-tenant-data", + run: async (payload: { tenantId: string; data: unknown }) => { + // Retrieve tenant-specific secret at runtime + const client = new SecretsManagerClient({ region: "us-east-1" }); + const response = await client.send( + new GetSecretValueCommand({ + SecretId: `tenants/${payload.tenantId}/supabase-key`, + }) + ); + + const supabaseKey = JSON.parse(response.SecretString!).SUPABASE_SERVICE_KEY; + + // Your task logic using the tenant-specific secret + // ... + }, +}); +``` + +You can use any secrets service - see the [sync env vars section](#sync-env-vars-from-another-service) for an example with Infisical. + +### Benefits + +- **Single codebase** - Deploy once, works for all tenants +- **Secure** - Secrets never appear in payloads or logs +- **Scalable** - No project limit constraints +- **Flexible** - Easy to add new tenants without redeploying + +This approach allows you to support unlimited tenants with a single Trigger.dev project, avoiding the [project limit](/limits#projects) while maintaining security and separation of tenant data. \ No newline at end of file diff --git a/docs/limits.mdx b/docs/limits.mdx index b4df2001ad5..45da4e89aba 100644 --- a/docs/limits.mdx +++ b/docs/limits.mdx @@ -55,6 +55,14 @@ If you add them [dynamically using code](/management/schedules/create) make sure If you're creating schedules for your user you will definitely need to request more schedules from us. +## Projects + +| Pricing tier | Limit | +| :----------- | :----------------- | +| All tiers | 10 per organization | + +Each project receives its own concurrency allocation. If you need to support multiple tenants with the same codebase but different environment variables, see the [Multi-tenant applications](/deploy-environment-variables#multi-tenant-applications) section for a recommended workaround. + ## Preview branches | Pricing tier | Limit | diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index c5040e59217..7a003194fa7 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -211,6 +211,16 @@ export const myTask = task({ View the [rate limits](/limits) page for more information. +### Runs waiting in queue due to concurrency limits + +If runs are staying in the `QUEUED` state for extended periods, check your concurrency usage in the dashboard. Review how many runs are `EXECUTING` or `DEQUEUED` (these count against limits) and check if any runs are stuck in `EXECUTING` state, as they may be blocking new runs. + +**Solutions:** + +- **Increase concurrency limits** - If you're on a paid plan, increase your environment concurrency limit via the dashboard +- **Review queue concurrency limits** - Check if individual queues have restrictive `concurrencyLimit` settings +- **Check for stuck runs** - See if stalled runs are blocking new executions + ### `Crypto is not defined` This can happen in different situations, for example when using plain strings as idempotency keys. Support for `Crypto` without a special flag was added in Node `v19.0.0`. You will have to upgrade Node - we recommend even-numbered major releases, e.g. `v20` or `v22`. Alternatively, you can switch from plain strings to the `idempotencyKeys.create` SDK function. [Read the guide](/idempotency). From c0595700f8506cf92b91816338f3f174297eb1d2 Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Wed, 4 Feb 2026 09:37:35 -0500 Subject: [PATCH 027/400] docs: clarify .env.local loading and idempotency key reset scope (#2996) Document that .env.local is automatically loaded during dev, and clarify that backend-triggered idempotency keys should be reset with global scope --- docs/deploy-environment-variables.mdx | 12 ++++++++++++ docs/idempotency.mdx | 8 ++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/deploy-environment-variables.mdx b/docs/deploy-environment-variables.mdx index e7b64964035..a4c3b75daf4 100644 --- a/docs/deploy-environment-variables.mdx +++ b/docs/deploy-environment-variables.mdx @@ -66,6 +66,18 @@ You can edit an environment variable's values. You cannot edit the key name, you +## Local development + +When running `npx trigger.dev dev`, the CLI automatically loads environment variables from these files in order (later files override any duplicate keys from earlier ones): + +- `.env` +- `.env.development` +- `.env.local` +- `.env.development.local` +- `dev.vars` + +These variables are available to your tasks via `process.env`. You don't need to use the `--env-file` flag for this automatic loading. + ## In your code You can use our SDK to get and manipulate environment variables. You can also easily sync environment variables from another service into Trigger.dev. diff --git a/docs/idempotency.mdx b/docs/idempotency.mdx index 9ed3933d567..034246eafbf 100644 --- a/docs/idempotency.mdx +++ b/docs/idempotency.mdx @@ -428,18 +428,22 @@ export const parentTask = task({ }); ``` -When resetting from outside a task (e.g., from your backend code), you must provide the `parentRunId`: +When resetting from outside a task, you must provide the `parentRunId` if the key was created within a task context: ```ts import { idempotencyKeys } from "@trigger.dev/sdk"; -// From your backend code - you need to know the parent run ID +// If the key was created within a task, you need the parent run ID await idempotencyKeys.reset("my-task", "my-key", { scope: "run", parentRunId: "run_abc123" }); ``` + +If you triggered the task from backend code, all scopes behave as global (see [Triggering from backend code](#triggering-from-backend-code)). Use `scope: "global"` when resetting. + + ### Resetting attempt-scoped keys Keys created with `"attempt"` scope include both the parent run ID and attempt number. When resetting from outside a task, you must provide both: From db4fb9eeefae32ef0509a698f208d6263bc92ad4 Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Wed, 4 Feb 2026 19:50:59 -0500 Subject: [PATCH 028/400] docs: Add Hookdeck example (#3005) Add documentation for integrating Hookdeck with Trigger.dev to receive webhooks and forward them to Trigger.dev tasks. --- --- docs/docs.json | 3 +- docs/guides/examples/hookdeck-webhook.mdx | 71 +++++++++++++++++++ .../frameworks/webhooks-guides-overview.mdx | 4 ++ docs/guides/introduction.mdx | 1 + 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 docs/guides/examples/hookdeck-webhook.mdx diff --git a/docs/docs.json b/docs/docs.json index c1f5d273804..324052905ac 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -352,7 +352,8 @@ "guides/frameworks/webhooks-guides-overview", "guides/frameworks/nextjs-webhooks", "guides/frameworks/remix-webhooks", - "guides/examples/stripe-webhook" + "guides/examples/stripe-webhook", + "guides/examples/hookdeck-webhook" ] } ] diff --git a/docs/guides/examples/hookdeck-webhook.mdx b/docs/guides/examples/hookdeck-webhook.mdx new file mode 100644 index 00000000000..a28949fca94 --- /dev/null +++ b/docs/guides/examples/hookdeck-webhook.mdx @@ -0,0 +1,71 @@ +--- +title: "Trigger tasks from Hookdeck webhooks" +sidebarTitle: "Hookdeck webhooks" +description: "This example demonstrates how to use Hookdeck to receive webhooks and trigger Trigger.dev tasks." +--- + +## Overview + +This example shows how to use [Hookdeck](https://hookdeck.com) as your webhook infrastructure to trigger Trigger.dev tasks. Hookdeck receives webhooks from external services, and forwards them directly to the Trigger.dev API. This gives you the best of both worlds: Hookdeck's webhook management, logging, and replay capabilities, combined with Trigger.dev's reliable task execution. + +## Key features + +- Use Hookdeck as your webhook endpoint for external services +- Hookdeck forwards webhooks directly to Trigger.dev tasks via the API +- All webhooks are logged and replayable in Hookdeck + +## Setting up Hookdeck + +You'll configure everything in the [Hookdeck dashboard](https://dashboard.hookdeck.com). No code changes needed in your app. + +### 1. Create a destination + +In Hookdeck, create a new [destination](https://hookdeck.com/docs/destinations) with the following settings: + +- **URL**: `https://api.trigger.dev/api/v1/tasks//trigger` (replace `` with your task ID) +- **Method**: POST +- **Authentication**: Bearer token (use your `TRIGGER_SECRET_KEY` from Trigger.dev) + +### 2. Add a transformation + +Create a [transformation](https://hookdeck.com/docs/transformations) to wrap the webhook body in the `payload` field that Trigger.dev expects: + +```javascript +addHandler("transform", (request, context) => { + request.body = { payload: { ...request.body } }; + return request; +}); +``` + +### 3. Create a connection + +Create a [connection](https://hookdeck.com/docs/connections) that links your source (where webhooks come from) to the destination and transformation you created above. + +## Task code + +This task will be triggered when Hookdeck forwards a webhook to the Trigger.dev API. + +```ts trigger/webhook-handler.ts +import { task } from "@trigger.dev/sdk"; + +export const webhookHandler = task({ + id: "webhook-handler", + run: async (payload: Record) => { + // The payload contains the original webhook data from the external service + console.log("Received webhook:", payload); + + // Add your custom logic here + }, +}); +``` + +## Testing your setup + +To test everything is working: + +1. Set up your destination, transformation, and connection in [Hookdeck](https://dashboard.hookdeck.com) +2. Send a test webhook to your Hookdeck source URL (use the Hookdeck Console or cURL) +3. Check the Hookdeck dashboard to verify the webhook was received and forwarded +4. Check the [Trigger.dev dashboard](https://cloud.trigger.dev) to see the successful run of your task + +For more information on setting up Hookdeck, refer to the [Hookdeck Documentation](https://hookdeck.com/docs). diff --git a/docs/guides/frameworks/webhooks-guides-overview.mdx b/docs/guides/frameworks/webhooks-guides-overview.mdx index 5e9703b53ef..4c0a4404276 100644 --- a/docs/guides/frameworks/webhooks-guides-overview.mdx +++ b/docs/guides/frameworks/webhooks-guides-overview.mdx @@ -31,6 +31,10 @@ A webhook handler is code that executes in response to an event. They can be end How to create a Stripe webhook handler and trigger a task when a 'checkout session completed' event is received. + + Use Hookdeck to receive webhooks and forward them to Trigger.dev tasks with logging and replay + capabilities. + Date: Wed, 4 Feb 2026 17:17:18 -0800 Subject: [PATCH 029/400] fix(webapp): ask ai button missing tooltip (#2964) Fixes - the tooltip not displaying on the AskAI button in the side menu - incorrect AskAI button heights - Small UI tweaks --- --------- Co-authored-by: Mihai Popescu --- apps/webapp/app/components/AskAI.tsx | 21 ++++++++++--------- apps/webapp/app/components/Shortcuts.tsx | 3 ++- .../app/components/navigation/SideMenu.tsx | 5 ++++- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/apps/webapp/app/components/AskAI.tsx b/apps/webapp/app/components/AskAI.tsx index bc55469b84a..814d4649c8f 100644 --- a/apps/webapp/app/components/AskAI.tsx +++ b/apps/webapp/app/components/AskAI.tsx @@ -118,30 +118,31 @@ function AskAIProvider({ websiteId, isCollapsed = false }: AskAIProviderProps) { -
- + + - -
+ + Ask AI - + diff --git a/apps/webapp/app/components/Shortcuts.tsx b/apps/webapp/app/components/Shortcuts.tsx index e3e4d6fe957..a3fcd074988 100644 --- a/apps/webapp/app/components/Shortcuts.tsx +++ b/apps/webapp/app/components/Shortcuts.tsx @@ -76,7 +76,8 @@ function ShortcutContent() { - + + diff --git a/apps/webapp/app/components/navigation/SideMenu.tsx b/apps/webapp/app/components/navigation/SideMenu.tsx index 1e7d48cd57c..95282f15f5c 100644 --- a/apps/webapp/app/components/navigation/SideMenu.tsx +++ b/apps/webapp/app/components/navigation/SideMenu.tsx @@ -993,7 +993,10 @@ function CollapseToggle({ return (
{/* Vertical line to mask the side menu border */} -
+
From 283f88b20382ee993d32f9fea48490898526b078 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Thu, 5 Feb 2026 15:24:23 +0100 Subject: [PATCH 030/400] feat(webapp): add triggered via field to deployment details page (#2850) Display the deployment trigger source (CLI, CI/CD, Dashboard, GitHub Integration) with appropriate icons on the deployment details page. The triggeredVia field was already in the database but not displayed. Co-authored-by: Claude --- .../v3/DeploymentPresenter.server.ts | 2 + .../route.tsx | 110 +++++++++++++++++- 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts b/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts index ea59c657228..bc494c118aa 100644 --- a/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts @@ -156,6 +156,7 @@ export class DeploymentPresenter { }, }, buildServerMetadata: true, + triggeredVia: true, }, }); @@ -225,6 +226,7 @@ export class DeploymentPresenter { isBuilt: !!deployment.builtAt, type: deployment.type, git: gitMetadata, + triggeredVia: deployment.triggeredVia, }, }; } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx index aebc934ba38..9d32d89fd56 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx @@ -3,7 +3,16 @@ import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { useEffect, useState, useRef, useCallback } from "react"; import { S2, S2Error } from "@s2-dev/streamstore"; -import { Clipboard, ClipboardCheck, ChevronDown, ChevronUp } from "lucide-react"; +import { + Clipboard, + ClipboardCheck, + ChevronDown, + ChevronUp, + TerminalSquareIcon, + LayoutDashboardIcon, + GitBranchIcon, + ServerIcon, +} from "lucide-react"; import { ExitIcon } from "~/assets/icons/ExitIcon"; import { GitMetadata } from "~/components/GitMetadata"; import { RuntimeIcon } from "~/components/RuntimeIcon"; @@ -73,6 +82,90 @@ type LogEntry = { level: "info" | "error" | "warn" | "debug"; }; +function getTriggeredViaDisplay(triggeredVia: string | null | undefined): { + icon: React.ReactNode; + label: string; +} | null { + if (!triggeredVia) return null; + + const iconClass = "size-4 text-text-dimmed"; + + switch (triggeredVia) { + case "cli:manual": + return { + icon: , + label: "CLI (Manual)", + }; + case "cli:github_actions": + return { + icon: , + label: "CLI (GitHub Actions)", + }; + case "cli:gitlab_ci": + return { + icon: , + label: "CLI (GitLab CI)", + }; + case "cli:circleci": + return { + icon: , + label: "CLI (CircleCI)", + }; + case "cli:jenkins": + return { + icon: , + label: "CLI (Jenkins)", + }; + case "cli:azure_pipelines": + return { + icon: , + label: "CLI (Azure Pipelines)", + }; + case "cli:bitbucket_pipelines": + return { + icon: , + label: "CLI (Bitbucket Pipelines)", + }; + case "cli:travis_ci": + return { + icon: , + label: "CLI (Travis CI)", + }; + case "cli:buildkite": + return { + icon: , + label: "CLI (Buildkite)", + }; + case "cli:ci_other": + return { + icon: , + label: "CLI (CI)", + }; + case "git_integration:github": + return { + icon: , + label: "GitHub Integration", + }; + case "dashboard": + return { + icon: , + label: "Dashboard", + }; + default: + // Handle any unknown values gracefully + if (triggeredVia.startsWith("cli:")) { + return { + icon: , + label: `CLI (${triggeredVia.replace("cli:", "")})`, + }; + } + return { + icon: , + label: triggeredVia, + }; + } +} + export default function Page() { const { deployment, eventStream } = useTypedLoaderData(); const organization = useOrganization(); @@ -408,6 +501,21 @@ export default function Page() { )} + + Triggered via + + {(() => { + const display = getTriggeredViaDisplay(deployment.triggeredVia); + if (!display) return "–"; + return ( + + {display.icon} + {display.label} + + ); + })()} + +
From 3bb9aac01405b259003f4a223f1bc87b4c63338e Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Thu, 5 Feb 2026 07:45:40 -0800 Subject: [PATCH 031/400] Fix(webapp): Prevent big numbers on Queue page from jumping around when animating (#3007) --- .../route.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 3a8a7544c5b..3ea70e1e18a 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -364,14 +364,14 @@ export default function Page() { />
} - valueClassName={env.paused ? "text-warning" : undefined} + valueClassName={cn(env.paused ? "text-warning" : undefined, "tabular-nums")} compactThreshold={1000000} /> From b96a0b70d49b04d3a29b803b720a3dac6be7e8c1 Mon Sep 17 00:00:00 2001 From: DKP <8297864+D-K-P@users.noreply.github.com> Date: Thu, 5 Feb 2026 23:22:44 +0000 Subject: [PATCH 032/400] Docs: Clarify AI tool compatibility and expand context snippet (#3000) This pull request overhauls the "Building with AI" documentation section. It includes a comprehensive restructuring of the main building-with-ai page with new setup guides and troubleshooting sections, reorganizes the navigation hierarchy to elevate mcp-agent-rules as a top-level page, and updates multiple documentation pages to clarify the relationships between three AI tools: Skills, Agent Rules, and MCP Server. Changes also include formatting improvements, such as replacing italicized text with inline code formatting, and consistent additions of explanatory Note blocks and CardGroup components across related pages. --- docs/building-with-ai.mdx | 275 ++++++++++++++++++++++++++++++++++++-- docs/docs.json | 5 +- docs/mcp-agent-rules.mdx | 23 +++- docs/mcp-introduction.mdx | 12 +- docs/mcp-tools.mdx | 60 ++++----- docs/skills.mdx | 12 +- 6 files changed, 334 insertions(+), 53 deletions(-) diff --git a/docs/building-with-ai.mdx b/docs/building-with-ai.mdx index ba8cd5bb47c..e551324cadf 100644 --- a/docs/building-with-ai.mdx +++ b/docs/building-with-ai.mdx @@ -4,21 +4,272 @@ sidebarTitle: "Overview" description: "Tools and resources for building Trigger.dev projects with AI coding assistants." --- -We provide tools to help you build Trigger.dev projects with AI coding assistants. We recommend using them for the best developer experience. +## Quick setup - - - Give your AI assistant direct access to Trigger.dev tools - search docs, trigger tasks, deploy projects, and monitor runs. +We provide multiple tools to help AI coding assistants write correct Trigger.dev code. Use one or all of them for the best developer experience. + + + + + + Give your AI assistant direct access to Trigger.dev tools — search docs, trigger tasks, deploy projects, and monitor runs. Works with Claude Code, Cursor, Windsurf, VS Code (Copilot), and Zed. + + ```bash + npx trigger.dev@latest install-mcp + ``` + + [Learn more →](/mcp-introduction) + + + + Portable instruction sets that teach any AI coding assistant Trigger.dev best practices. Works with Claude Code, Cursor, Windsurf, VS Code (Copilot), and any tool that supports the [Agent Skills standard](https://agentskills.io). + + ```bash + npx skills add triggerdotdev/skills + ``` + + [Learn more →](/skills) + + + + Comprehensive rule sets installed directly into your AI client's config files. Works with Cursor, Claude Code, VS Code (Copilot), Windsurf, Gemini CLI, Cline, and more. Claude Code also gets a dedicated subagent for hands-on help. + + ```bash + npx trigger.dev@latest install-rules + ``` + + [Learn more →](/mcp-agent-rules) + + + + +## Skills vs Agent Rules vs MCP + +Not sure which tool to use? Here's how they compare: + +| | **Skills** | **Agent Rules** | **MCP Server** | +|:--|:-----------|:----------------|:---------------| +| **What it does** | Drops skill files into your project | Installs rule sets into client config | Runs a live server your AI connects to | +| **Installs to** | `.claude/skills/`, `.cursor/skills/`, etc. | `.cursor/rules/`, `CLAUDE.md`, `AGENTS.md`, etc. | `mcp.json`, `~/.claude.json`, etc. | +| **Updates** | Re-run `npx skills add` | Re-run `npx trigger.dev@latest install-rules` or auto-prompted on `trigger dev` | Always latest (uses `@latest`) | +| **Best for** | Teaching patterns and best practices | Comprehensive code generation guidance | Live project interaction (deploy, trigger, monitor) | +| **Works offline** | Yes | Yes | No (calls Trigger.dev API) | + +**Our recommendation:** Install all three. Skills and Agent Rules teach your AI *how* to write code. The MCP Server lets it *do things* in your project. + +## Project-level context snippet + +If you prefer a lightweight/passive approach, paste the snippet below into a context file at the root of your project. Different AI tools read different files: + +| File | Read by | +|:-----|:--------| +| `CLAUDE.md` | Claude Code | +| `AGENTS.md` | OpenAI Codex, Jules, OpenCode | +| `.cursor/rules/*.md` | Cursor | +| `.github/copilot-instructions.md` | GitHub Copilot | +| `CONVENTIONS.md` | Windsurf, Cline, and others | + +Create the file that matches your AI tool (or multiple files if your team uses different tools) and paste the snippet below. This gives the AI essential Trigger.dev context without installing anything. + + + +````markdown +# Trigger.dev rules + +## Imports + +Always import from `@trigger.dev/sdk` — never from `@trigger.dev/sdk/v3` or use the deprecated `client.defineJob` pattern. + +## Task pattern + +Every task must be exported. Use `task()` from `@trigger.dev/sdk`: + +```ts +import { task } from "@trigger.dev/sdk"; + +export const myTask = task({ + id: "my-task", + retry: { + maxAttempts: 3, + factor: 1.8, + minTimeoutInMs: 500, + maxTimeoutInMs: 30_000, + }, + run: async (payload: { url: string }) => { + // No timeouts — runs can take as long as needed + return { success: true }; + }, +}); +``` + +## Triggering tasks + +From your backend (Next.js route, Express handler, etc.): + +```ts +import type { myTask } from "./trigger/my-task"; +import { tasks } from "@trigger.dev/sdk"; + +// Fire and forget +const handle = await tasks.trigger("my-task", { url: "https://example.com" }); + +// Batch trigger (up to 1,000 items) +const batchHandle = await tasks.batchTrigger("my-task", [ + { payload: { url: "https://example.com/1" } }, + { payload: { url: "https://example.com/2" } }, +]); +``` + +### From inside other tasks + +```ts +export const parentTask = task({ + id: "parent-task", + run: async (payload) => { + // Fire and forget + await childTask.trigger({ data: "value" }); + + // Wait for result — returns a Result object, NOT the output directly + const result = await childTask.triggerAndWait({ data: "value" }); + if (result.ok) { + console.log(result.output); // The actual return value + } else { + console.error(result.error); + } - ```bash - npx trigger.dev@latest install-mcp - ``` + // Or use .unwrap() to get output directly (throws on failure) + const output = await childTask.triggerAndWait({ data: "value" }).unwrap(); + }, +}); +``` + +> Never wrap `triggerAndWait` or `batchTriggerAndWait` in `Promise.all` — this is not supported. + +## Error handling + +```ts +import { task, retry, AbortTaskRunError } from "@trigger.dev/sdk"; + +export const resilientTask = task({ + id: "resilient-task", + retry: { maxAttempts: 5 }, + run: async (payload) => { + // Permanent error — skip retrying + if (!payload.isValid) { + throw new AbortTaskRunError("Invalid payload, will not retry"); + } + + // Retry a specific block (not the whole task) + const data = await retry.onThrow( + async () => await fetchExternalApi(payload), + { maxAttempts: 3 } + ); + + return data; + }, +}); +``` + +## Schema validation + +Use `schemaTask` with Zod for payload validation: + +```ts +import { schemaTask } from "@trigger.dev/sdk"; +import { z } from "zod"; + +export const processVideo = schemaTask({ + id: "process-video", + schema: z.object({ videoUrl: z.string().url() }), + run: async (payload) => { + // payload is typed and validated + }, +}); +``` + +## Waits + +Use `wait.for` for delays, `wait.until` for dates, and `wait.forToken` for external callbacks: + +```ts +import { wait } from "@trigger.dev/sdk"; +await wait.for({ seconds: 30 }); +await wait.until({ date: new Date("2025-01-01") }); +``` + +## Configuration + +`trigger.config.ts` lives at the project root: + +```ts +import { defineConfig } from "@trigger.dev/sdk/build"; + +export default defineConfig({ + project: "", + dirs: ["./trigger"], +}); +``` + +## Common mistakes + +1. **Forgetting to export tasks** — every task must be a named export +2. **Importing from `@trigger.dev/sdk/v3`** — this is the old v3 path; always use `@trigger.dev/sdk` +3. **Using `client.defineJob()`** — this is the deprecated v2 API +4. **Calling `task.trigger()` directly** — use `tasks.trigger("task-id", payload)` from your backend +5. **Using `triggerAndWait` result as output** — it returns a `Result` object; check `result.ok` then access `result.output`, or use `.unwrap()` +6. **Wrapping waits/triggerAndWait in `Promise.all`** — not supported in Trigger.dev tasks +7. **Adding timeouts to tasks** — tasks have no built-in timeout; use `maxDuration` in config if needed +```` + + + +## llms.txt + +We also publish machine-readable documentation for LLM consumption: + +- [trigger.dev/docs/llms.txt](https://trigger.dev/docs/llms.txt) — concise overview +- [trigger.dev/docs/llms-full.txt](https://trigger.dev/docs/llms-full.txt) — full documentation + +These follow the [llms.txt standard](https://llmstxt.org) and can be fed directly into any LLM context window. + + +## Troubleshooting + + + + + Install [Agent Rules](/mcp-agent-rules) or [Skills](/skills) — they override the outdated patterns in the AI's training data. The [context snippet](#project-level-context-snippet) above is a quick alternative. + + + + 1. Make sure you've restarted your AI client after adding the config + 2. Run `npx trigger.dev@latest install-mcp` again — it will detect and fix common issues + 3. Check that `npx trigger.dev@latest mcp` runs without errors in your terminal + 4. See the [MCP introduction](/mcp-introduction) for client-specific config details + + + + All three if possible. If you can only pick one: + - **Agent Rules** if you want the broadest code generation improvement + - **Skills** if you use multiple AI tools and want a single install + - **MCP Server** if you need to trigger tasks, deploy, and search docs from your AI + + + + +## Next steps + + + + Install and configure the MCP Server for live project interaction. - Portable instruction sets that teach any AI coding assistant Trigger.dev best practices for writing tasks, configs, and more. - - ```bash - npx skills add triggerdotdev/skills - ``` + Portable instruction sets for any AI coding assistant. + + + Comprehensive rule sets installed into your AI client. + + + Learn the task patterns your AI assistant will follow. diff --git a/docs/docs.json b/docs/docs.json index 324052905ac..4ec2fafc0eb 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -49,9 +49,10 @@ "building-with-ai", { "group": "MCP Server", - "pages": ["mcp-introduction", "mcp-tools", "mcp-agent-rules"] + "pages": ["mcp-introduction", "mcp-tools"] }, - "skills" + "skills", + "mcp-agent-rules" ] }, { diff --git a/docs/mcp-agent-rules.mdx b/docs/mcp-agent-rules.mdx index 321f312a842..d9ba021b891 100644 --- a/docs/mcp-agent-rules.mdx +++ b/docs/mcp-agent-rules.mdx @@ -1,13 +1,17 @@ --- title: "Agent rules" sidebarTitle: "Agent rules" -description: "Learn how to use the Trigger.dev agent rules with the MCP server" +description: "Install Trigger.dev agent rules to guide AI assistants toward correct, up-to-date code patterns." --- ## What are Trigger.dev agent rules? Trigger.dev agent rules are comprehensive instruction sets that guide AI assistants to write optimal Trigger.dev code. These rules ensure your AI assistant understands best practices, current APIs, and recommended patterns when working with Trigger.dev projects. + + Agent Rules are one of three AI tools we provide. You can also install [Skills](/skills) for portable cross-editor instruction sets or the [MCP Server](/mcp-introduction) for live project interaction. See the [comparison table](/building-with-ai#skills-vs-agent-rules-vs-mcp) for details. + + ## Installation Install the agent rules with the following command: @@ -112,6 +116,17 @@ npx trigger.dev@latest install-rules ## Next steps -- [Install the MCP server](/mcp-introduction) for complete Trigger.dev integration -- [Explore MCP tools](/mcp-tools) for project management and task execution - + + + Portable instruction sets that work across all AI coding assistants. + + + Give your AI assistant direct access to Trigger.dev tools and APIs. + + + See all AI tools and how they compare. + + + Learn the task patterns that agent rules teach your AI assistant. + + diff --git a/docs/mcp-introduction.mdx b/docs/mcp-introduction.mdx index 257522d5792..a00f3dda896 100644 --- a/docs/mcp-introduction.mdx +++ b/docs/mcp-introduction.mdx @@ -361,4 +361,14 @@ Once installed, you can start using the MCP server by asking your AI assistant q ## Next Steps -- [Explore available MCP tools](/mcp-tools) + + + Explore all available MCP tools for managing your projects. + + + Portable instruction sets that teach AI assistants Trigger.dev patterns. + + + Install comprehensive rule sets directly into your AI client. + + diff --git a/docs/mcp-tools.mdx b/docs/mcp-tools.mdx index 058a3671ab7..fdcdb56f3d6 100644 --- a/docs/mcp-tools.mdx +++ b/docs/mcp-tools.mdx @@ -11,9 +11,9 @@ description: "Learn about how to use the tools available in the Trigger.dev MCP Search the Trigger.dev documentation for guides, examples, and API references. **Example usage:** -- _"How do I create a scheduled task?"_ -- _"Show me webhook examples"_ -- _"What are the deployment options?"_ +- `"How do I create a scheduled task?"` +- `"Show me webhook examples"` +- `"What are the deployment options?"` ## Project Management Tools @@ -22,32 +22,32 @@ Search the Trigger.dev documentation for guides, examples, and API references. List all organizations you have access to. **Example usage:** -- _"What organizations do I have?"_ -- _"Show me my orgs"_ +- `"What organizations do I have?"` +- `"Show me my orgs"` ### list_projects List all projects in your Trigger.dev account. **Example usage:** -- _"What projects do I have?"_ -- _"List my Trigger.dev projects"_ +- `"What projects do I have?"` +- `"List my Trigger.dev projects"` ### create_project_in_org Create a new project in an organization. **Example usage:** -- _"Create a new project called 'my-app'"_ -- _"Set up a new Trigger.dev project"_ +- `"Create a new project called 'my-app'"` +- `"Set up a new Trigger.dev project"` ### initialize_project Initialize Trigger.dev in your project with automatic setup and configuration. **Example usage:** -- _"Set up Trigger.dev in this project"_ -- _"Add Trigger.dev to my app"_ +- `"Set up Trigger.dev in this project"` +- `"Add Trigger.dev to my app"` ## Task Management Tools @@ -56,17 +56,17 @@ Initialize Trigger.dev in your project with automatic setup and configuration. Get the current worker for a project, including the worker version, SDK version, and registered tasks with their payload schemas. **Example usage:** -- _"What tasks are available?"_ -- _"Show me the tasks in dev"_ +- `"What tasks are available?"` +- `"Show me the tasks in dev"` ### trigger_task Trigger a task to run with a specific payload. You can add a delay, set tags, configure retries, choose a machine size, set a TTL, or use an idempotency key. **Example usage:** -- _"Run the email-notification task"_ -- _"Trigger my-task with userId 123"_ -- _"Execute the sync task in production"_ +- `"Run the email-notification task"` +- `"Trigger my-task with userId 123"` +- `"Execute the sync task in production"` ## Run Monitoring Tools @@ -75,32 +75,32 @@ Trigger a task to run with a specific payload. You can add a delay, set tags, co Get detailed information about a specific task run, including logs and status. Enable debug mode to get the full trace with all logs and spans. **Example usage:** -- _"Show me details for run run_abc123"_ -- _"Why did this run fail?"_ +- `"Show me details for run run_abc123"` +- `"Why did this run fail?"` ### list_runs List runs for a project. Filter by status, task, tags, version, machine size, or time period. **Example usage:** -- _"Show me recent runs"_ -- _"List failed runs from the last 7 days"_ -- _"What runs are currently executing?"_ +- `"Show me recent runs"` +- `"List failed runs from the last 7 days"` +- `"What runs are currently executing?"` ### wait_for_run_to_complete Wait for a specific run to finish and return the result. **Example usage:** -- _"Wait for run run_abc123 to complete"_ +- `"Wait for run run_abc123 to complete"` ### cancel_run Cancel a running or queued run. **Example usage:** -- _"Cancel run run_abc123"_ -- _"Stop that task"_ +- `"Cancel run run_abc123"` +- `"Stop that task"` ## Deployment Tools @@ -109,24 +109,24 @@ Cancel a running or queued run. Deploy your project to staging or production. **Example usage:** -- _"Deploy to production"_ -- _"Deploy to staging"_ +- `"Deploy to production"` +- `"Deploy to staging"` ### list_deploys List deployments for a project. Filter by status or time period. **Example usage:** -- _"Show me recent deployments"_ -- _"What's deployed to production?"_ +- `"Show me recent deployments"` +- `"What's deployed to production?"` ### list_preview_branches List all preview branches in the project. **Example usage:** -- _"What preview branches exist?"_ -- _"Show me preview deployments"_ +- `"What preview branches exist?"` +- `"Show me preview deployments"` The deploy and list_preview_branches tools are not available when the MCP server is running with the `--dev-only` flag. diff --git a/docs/skills.mdx b/docs/skills.mdx index eb4add47952..f12c5c36eb6 100644 --- a/docs/skills.mdx +++ b/docs/skills.mdx @@ -7,7 +7,11 @@ tag: "new" ## What are agent skills? -Skills are portable instruction sets that teach AI coding assistants how to use Trigger.dev effectively. Unlike vendor-specific config files (`.cursor/rules`, `CLAUDE.md`), skills use an open standard that works across all major AI assistants. For example, Cursor users and Claude Code users can get the same knowledge from a single install. +Skills are portable instruction sets that teach AI coding assistants how to use Trigger.dev effectively. Unlike vendor-specific config files (`.cursor/rules`, `CLAUDE.md`), skills use an open standard that works across all major AI assistants. For example, Cursor users and Claude Code users can get the same knowledge from a single install. + + + Skills are one of three AI tools we provide. You can also install [Agent Rules](/mcp-agent-rules) for client-specific rule sets or the [MCP Server](/mcp-introduction) for live project interaction. See the [comparison table](/building-with-ai#skills-vs-agent-rules-vs-mcp) for details. + Skills are installed as directories containing a `SKILL.md` file. Each `SKILL.md` includes YAML frontmatter (name, description) and markdown instructions with patterns, examples, and best practices that AI assistants automatically discover and follow. @@ -68,15 +72,15 @@ Skills work with any AI coding assistant that supports the [Agent Skills standar ## Next steps + + Install comprehensive rule sets directly into your AI client. + Give your AI assistant direct access to Trigger.dev tools and APIs. Learn the task patterns that skills teach your AI assistant. - - Build durable AI workflows with prompt chaining and human-in-the-loop. - Browse the full Agent Skills ecosystem. From e536d35b1717033a638b4245a8e7c17c5e5ba6ab Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 6 Feb 2026 13:56:39 +0000 Subject: [PATCH 033/400] fix(ci): fix docker image publishing and worker builds (#3013) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - **Fix Docker publish automation**: The `v.docker.*` tags pushed by the release workflow using `GITHUB_TOKEN` don't trigger the publish workflow (GitHub Actions limitation to prevent infinite loops). Added a `workflow_call` to `publish.yml` directly from the release job so Docker images are built automatically after npm publish. Tags are still pushed for reference. - **Fix worker Containerfiles**: The coordinator, docker-provider, and kubernetes-provider builds have been failing since the superjson vendoring change in `@trigger.dev/core` (#2949). The Containerfiles now run `bundle-vendor` before `build:bundle` to generate the vendor files that esbuild needs. ### Context - Docker images on GHCR have been stuck at v4.3.0 — v4.3.1, v4.3.2, v4.3.3 tags existed on GitHub but never triggered publish runs - The worker builds (publish-worker) have been failing on every push to main since Jan 30 ## Test plan - [x] Verified kubernetes-provider Containerfile builds locally with the fix - [x] Manually dispatched publish workflow for v4.3.1 — all jobs succeeded --- --- .github/workflows/release.yml | 12 +++++++++++- apps/coordinator/Containerfile | 2 +- apps/docker-provider/Containerfile | 2 +- apps/kubernetes-provider/Containerfile | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ca0f0ebf16b..3b4135ec099 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -122,7 +122,6 @@ jobs: package_version=$(echo '${{ steps.changesets.outputs.publishedPackages }}' | jq -r '.[0].version') echo "package_version=${package_version}" >> "$GITHUB_OUTPUT" - # this triggers the publish workflow for the docker images - name: Create and push Docker tag if: steps.changesets.outputs.published == 'true' run: | @@ -130,6 +129,17 @@ jobs: git tag "v.docker.${{ steps.get_version.outputs.package_version }}" git push origin "v.docker.${{ steps.get_version.outputs.package_version }}" + # Trigger Docker builds directly via workflow_call since tags pushed with + # GITHUB_TOKEN don't trigger other workflows (GitHub Actions limitation). + publish-docker: + name: 🐳 Publish Docker images + needs: release + if: needs.release.outputs.published == 'true' + uses: ./.github/workflows/publish.yml + secrets: inherit + with: + image_tag: v${{ needs.release.outputs.published_package_version }} + # The prerelease job needs to be on the same workflow file due to a limitation related to how npm verifies OIDC claims. prerelease: name: 🧪 Prerelease diff --git a/apps/coordinator/Containerfile b/apps/coordinator/Containerfile index 4e7b89e0af1..9e973675ab9 100644 --- a/apps/coordinator/Containerfile +++ b/apps/coordinator/Containerfile @@ -35,7 +35,7 @@ COPY --from=pruner --chown=node:node /app/out/full/ . COPY --from=dev-deps --chown=node:node /app/ . COPY --chown=node:node turbo.json turbo.json -RUN pnpm run -r --filter coordinator build:bundle +RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter coordinator build:bundle FROM alpine AS cri-tools diff --git a/apps/docker-provider/Containerfile b/apps/docker-provider/Containerfile index bea730bda80..42a7ac23092 100644 --- a/apps/docker-provider/Containerfile +++ b/apps/docker-provider/Containerfile @@ -31,7 +31,7 @@ COPY --from=pruner --chown=node:node /app/out/full/ . COPY --from=dev-deps --chown=node:node /app/ . COPY --chown=node:node turbo.json turbo.json -RUN pnpm run -r --filter docker-provider build:bundle +RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter docker-provider build:bundle FROM base AS runner diff --git a/apps/kubernetes-provider/Containerfile b/apps/kubernetes-provider/Containerfile index fb96304c26b..b46b9943275 100644 --- a/apps/kubernetes-provider/Containerfile +++ b/apps/kubernetes-provider/Containerfile @@ -31,7 +31,7 @@ COPY --from=pruner --chown=node:node /app/out/full/ . COPY --from=dev-deps --chown=node:node /app/ . COPY --chown=node:node turbo.json turbo.json -RUN pnpm run -r --filter kubernetes-provider build:bundle +RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter kubernetes-provider build:bundle FROM base AS runner From 9b21f8d322c5d802ddd8cd848002c4d0b9afbb33 Mon Sep 17 00:00:00 2001 From: Oskar Otwinowski Date: Tue, 10 Feb 2026 10:37:09 +0100 Subject: [PATCH 034/400] feat(webapp): Vercel integration (#2994) Vercel integration Desc + Vid coming soon For human reviewer: - check the db schema - check if posthog user attribution call is correct (telemetry.server.ts & `referralSource`) --- --- .changeset/vercel-integration.md | 5 + .vscode/settings.json | 3 +- .../app/components/GitHubLoginButton.tsx | 2 - .../environments/RegenerateApiKeyModal.tsx | 30 +- .../integrations/VercelBuildSettings.tsx | 176 ++ .../components/integrations/VercelLogo.tsx | 12 + .../integrations/VercelOnboardingModal.tsx | 1085 +++++++++++ .../OrganizationSettingsSideMenu.tsx | 9 + apps/webapp/app/env.server.ts | 5 + .../app/models/orgIntegration.server.ts | 24 + .../app/models/vercelIntegration.server.ts | 1659 +++++++++++++++++ .../presenters/v3/ApiKeysPresenter.server.ts | 16 + .../v3/DeploymentListPresenter.server.ts | 59 +- .../EnvironmentVariablesPresenter.server.ts | 92 +- .../v3/VercelSettingsPresenter.server.ts | 585 ++++++ .../route.tsx | 7 +- .../route.tsx | 28 +- .../route.tsx | 8 +- .../route.tsx | 230 ++- .../route.tsx | 162 +- ...ationSlug.settings.integrations.vercel.tsx | 375 ++++ .../route.tsx | 45 + .../webapp/app/routes/_app.orgs.new/route.tsx | 21 + .../api.v1.deployments.$deploymentId.ts | 12 + ....projects.$projectParam.vercel.projects.ts | 147 ++ ...ojects.$projectRef.envvars.$slug.import.ts | 4 + .../app/routes/auth.github.callback.tsx | 5 +- .../app/routes/auth.google.callback.tsx | 5 +- .../app/routes/confirm-basic-details.tsx | 20 +- apps/webapp/app/routes/login._index/route.tsx | 2 +- apps/webapp/app/routes/login.magic/route.tsx | 15 +- apps/webapp/app/routes/login.mfa/route.tsx | 13 +- apps/webapp/app/routes/magic.tsx | 3 + ...ents.$environmentId.regenerate-api-key.tsx | 38 + ...cts.$projectParam.env.$envParam.github.tsx | 36 +- ...cts.$projectParam.env.$envParam.vercel.tsx | 926 +++++++++ apps/webapp/app/routes/vercel.callback.ts | 78 + apps/webapp/app/routes/vercel.configure.tsx | 52 + apps/webapp/app/routes/vercel.connect.tsx | 170 ++ apps/webapp/app/routes/vercel.install.tsx | 73 + apps/webapp/app/routes/vercel.onboarding.tsx | 465 +++++ apps/webapp/app/services/org.server.ts | 20 + apps/webapp/app/services/postAuth.server.ts | 5 +- .../app/services/referralSource.server.ts | 53 + apps/webapp/app/services/telemetry.server.ts | 32 +- .../app/services/vercelIntegration.server.ts | 656 +++++++ apps/webapp/app/utils/pathBuilder.ts | 24 + .../environmentVariablesRepository.server.ts | 147 +- .../app/v3/environmentVariables/repository.ts | 18 + .../v3/services/alerts/deliverAlert.server.ts | 3 +- .../services/initializeDeployment.server.ts | 1 + apps/webapp/app/v3/vercel/index.ts | 17 + .../app/v3/vercel/vercelOAuthState.server.ts | 40 + .../vercel/vercelProjectIntegrationSchema.ts | 225 +++ .../webapp/app/v3/vercel/vercelUrls.server.ts | 26 + apps/webapp/package.json | 1 + apps/webapp/test/vercelUrls.test.ts | 56 + .../migration.sql | 3 + .../migration.sql | 29 + .../migration.sql | 22 + .../migration.sql | 9 + .../migration.sql | 3 + .../migration.sql | 3 + .../database/prisma/schema.prisma | 113 +- packages/core/src/v3/schemas/api.ts | 18 + pnpm-lock.yaml | 335 +++- 66 files changed, 8388 insertions(+), 173 deletions(-) create mode 100644 .changeset/vercel-integration.md create mode 100644 apps/webapp/app/components/integrations/VercelBuildSettings.tsx create mode 100644 apps/webapp/app/components/integrations/VercelLogo.tsx create mode 100644 apps/webapp/app/components/integrations/VercelOnboardingModal.tsx create mode 100644 apps/webapp/app/models/vercelIntegration.server.ts create mode 100644 apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts create mode 100644 apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.integrations.vercel.tsx create mode 100644 apps/webapp/app/routes/api.v1.orgs.$organizationSlug.projects.$projectParam.vercel.projects.ts create mode 100644 apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.vercel.tsx create mode 100644 apps/webapp/app/routes/vercel.callback.ts create mode 100644 apps/webapp/app/routes/vercel.configure.tsx create mode 100644 apps/webapp/app/routes/vercel.connect.tsx create mode 100644 apps/webapp/app/routes/vercel.install.tsx create mode 100644 apps/webapp/app/routes/vercel.onboarding.tsx create mode 100644 apps/webapp/app/services/org.server.ts create mode 100644 apps/webapp/app/services/referralSource.server.ts create mode 100644 apps/webapp/app/services/vercelIntegration.server.ts create mode 100644 apps/webapp/app/v3/vercel/index.ts create mode 100644 apps/webapp/app/v3/vercel/vercelOAuthState.server.ts create mode 100644 apps/webapp/app/v3/vercel/vercelProjectIntegrationSchema.ts create mode 100644 apps/webapp/app/v3/vercel/vercelUrls.server.ts create mode 100644 apps/webapp/test/vercelUrls.test.ts create mode 100644 internal-packages/database/prisma/migrations/20260126175159_add_environment_variable_versioning/migration.sql create mode 100644 internal-packages/database/prisma/migrations/20260129162621_add_organization_project_integration/migration.sql create mode 100644 internal-packages/database/prisma/migrations/20260129162810_add_integration_deployment/migration.sql create mode 100644 internal-packages/database/prisma/migrations/20260129162946_alter_tables_for_integrations_data/migration.sql create mode 100644 internal-packages/database/prisma/migrations/20260129165555_add_organization_integration_idx/migration.sql create mode 100644 internal-packages/database/prisma/migrations/20260129165809_add_worker_deployment_idx/migration.sql diff --git a/.changeset/vercel-integration.md b/.changeset/vercel-integration.md new file mode 100644 index 00000000000..8b638e36431 --- /dev/null +++ b/.changeset/vercel-integration.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Add Vercel integration support to API schemas: `commitSHA` and `integrationDeployments` on deployment responses, and `source` field for environment variable imports. diff --git a/.vscode/settings.json b/.vscode/settings.json index 12aefeb358f..382a5ae6201 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,5 +7,6 @@ "packages/cli-v3/e2e": true }, "vitest.disableWorkspaceWarning": true, - "typescript.experimental.useTsgo": false + "typescript.experimental.useTsgo": true, + "chat.agent.maxRequests": 10000 } diff --git a/apps/webapp/app/components/GitHubLoginButton.tsx b/apps/webapp/app/components/GitHubLoginButton.tsx index 87238db087e..76a494927cd 100644 --- a/apps/webapp/app/components/GitHubLoginButton.tsx +++ b/apps/webapp/app/components/GitHubLoginButton.tsx @@ -32,8 +32,6 @@ export function OctoKitty({ className }: { className?: string }) { baseProfile="tiny" id="Layer_1" xmlns="http://www.w3.org/2000/svg" - x="0px" - y="0px" viewBox="0 0 2350 2314.8" xmlSpace="preserve" fill="currentColor" diff --git a/apps/webapp/app/components/environments/RegenerateApiKeyModal.tsx b/apps/webapp/app/components/environments/RegenerateApiKeyModal.tsx index fb0c77ca7c7..439fd892f91 100644 --- a/apps/webapp/app/components/environments/RegenerateApiKeyModal.tsx +++ b/apps/webapp/app/components/environments/RegenerateApiKeyModal.tsx @@ -10,11 +10,14 @@ import { FormButtons } from "../primitives/FormButtons"; import { Input } from "../primitives/Input"; import { InputGroup } from "../primitives/InputGroup"; import { Paragraph } from "../primitives/Paragraph"; +import { CheckboxWithLabel } from "../primitives/Checkbox"; import { Spinner } from "../primitives/Spinner"; type ModalProps = { id: string; title: string; + hasVercelIntegration: boolean; + isDevelopment: boolean; }; type ModalContentProps = ModalProps & { @@ -22,7 +25,12 @@ type ModalContentProps = ModalProps & { closeModal: () => void; }; -export function RegenerateApiKeyModal({ id, title }: ModalProps) { +export function RegenerateApiKeyModal({ + id, + title, + hasVercelIntegration, + isDevelopment, +}: ModalProps) { const randomWord = generateTwoRandomWords(); const [open, setOpen] = useState(false); return ( @@ -37,6 +45,8 @@ export function RegenerateApiKeyModal({ id, title }: ModalProps) { setOpen(false)} /> @@ -45,7 +55,14 @@ export function RegenerateApiKeyModal({ id, title }: ModalProps) { ); } -const RegenerateApiKeyModalContent = ({ id, randomWord, title, closeModal }: ModalContentProps) => { +const RegenerateApiKeyModalContent = ({ + id, + randomWord, + title, + hasVercelIntegration, + isDevelopment, + closeModal, +}: ModalContentProps) => { const [confirmationText, setConfirmationText] = useState(""); const fetcher = useFetcher(); const isSubmitting = fetcher.state === "submitting"; @@ -83,6 +100,15 @@ const RegenerateApiKeyModalContent = ({ id, randomWord, title, closeModal }: Mod onChange={(e) => setConfirmationText(e.target.value)} /> + {hasVercelIntegration && !isDevelopment && ( + + )} void; + discoverEnvVars: EnvSlug[]; + onDiscoverEnvVarsChange: (slugs: EnvSlug[]) => void; + atomicBuilds: EnvSlug[]; + onAtomicBuildsChange: (slugs: EnvSlug[]) => void; + envVarsConfigLink?: string; +}; + +export function BuildSettingsFields({ + availableEnvSlugs, + pullEnvVarsBeforeBuild, + onPullEnvVarsChange, + discoverEnvVars, + onDiscoverEnvVarsChange, + atomicBuilds, + onAtomicBuildsChange, + envVarsConfigLink, +}: BuildSettingsFieldsProps) { + return ( + <> + {/* Pull env vars before build */} +
+
+
+ Pull env vars before build + + Select which environments should pull environment variables from Vercel before each + build.{" "} + {envVarsConfigLink && ( + <> + Configure which variables to pull. + + )} + +
+ {availableEnvSlugs.length > 1 && ( + 0 && + availableEnvSlugs.every((s) => pullEnvVarsBeforeBuild.includes(s)) + } + onCheckedChange={(checked) => { + onPullEnvVarsChange(checked ? [...availableEnvSlugs] : []); + }} + /> + )} +
+
+ {availableEnvSlugs.map((slug) => { + const envType = envSlugToType(slug); + return ( +
+
+ + + {environmentFullTitle({ type: envType })} + +
+ { + onPullEnvVarsChange( + checked + ? [...pullEnvVarsBeforeBuild, slug] + : pullEnvVarsBeforeBuild.filter((s) => s !== slug) + ); + }} + /> +
+ ); + })} +
+
+ + {/* Discover new env vars */} +
+
+
+ Discover new env vars + + Select which environments should automatically discover and create new environment + variables from Vercel during builds. + +
+ {availableEnvSlugs.length > 1 && ( + 0 && + availableEnvSlugs.every( + (s) => discoverEnvVars.includes(s) || !pullEnvVarsBeforeBuild.includes(s) + ) && + availableEnvSlugs.some((s) => discoverEnvVars.includes(s)) + } + disabled={!availableEnvSlugs.some((s) => pullEnvVarsBeforeBuild.includes(s))} + onCheckedChange={(checked) => { + onDiscoverEnvVarsChange( + checked + ? availableEnvSlugs.filter((s) => pullEnvVarsBeforeBuild.includes(s)) + : [] + ); + }} + /> + )} +
+
+ {availableEnvSlugs.map((slug) => { + const envType = envSlugToType(slug); + const isPullDisabled = !pullEnvVarsBeforeBuild.includes(slug); + return ( +
+
+ + + {environmentFullTitle({ type: envType })} + +
+ { + onDiscoverEnvVarsChange( + checked + ? [...discoverEnvVars, slug] + : discoverEnvVars.filter((s) => s !== slug) + ); + }} + /> +
+ ); + })} +
+
+ + {/* Atomic deployments */} +
+
+
+ Atomic deployments + + When enabled, production deployments wait for Vercel deployment to complete before + promoting the Trigger.dev deployment. + +
+ { + onAtomicBuildsChange(checked ? ["prod"] : []); + }} + /> +
+
+ + ); +} diff --git a/apps/webapp/app/components/integrations/VercelLogo.tsx b/apps/webapp/app/components/integrations/VercelLogo.tsx new file mode 100644 index 00000000000..7ddf039abfd --- /dev/null +++ b/apps/webapp/app/components/integrations/VercelLogo.tsx @@ -0,0 +1,12 @@ +export function VercelLogo({ className }: { className?: string }) { + return ( + + ); +} diff --git a/apps/webapp/app/components/integrations/VercelOnboardingModal.tsx b/apps/webapp/app/components/integrations/VercelOnboardingModal.tsx new file mode 100644 index 00000000000..c2a5bfec43a --- /dev/null +++ b/apps/webapp/app/components/integrations/VercelOnboardingModal.tsx @@ -0,0 +1,1085 @@ +import { + CheckCircleIcon, + ExclamationTriangleIcon, + ChevronDownIcon, + ChevronUpIcon, +} from "@heroicons/react/20/solid"; +import { + useFetcher, + useNavigation, + useSearchParams, +} from "@remix-run/react"; +import { useTypedFetcher } from "remix-typedjson"; +import { Dialog, DialogContent, DialogHeader } from "~/components/primitives/Dialog"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { Callout } from "~/components/primitives/Callout"; +import { FormButtons } from "~/components/primitives/FormButtons"; +import { FormError } from "~/components/primitives/FormError"; +import { Header3 } from "~/components/primitives/Headers"; +import { Hint } from "~/components/primitives/Hint"; +import { Label } from "~/components/primitives/Label"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { Select, SelectItem } from "~/components/primitives/Select"; +import { SpinnerWhite } from "~/components/primitives/Spinner"; +import { Switch } from "~/components/primitives/Switch"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, + TooltipProvider, +} from "~/components/primitives/Tooltip"; +import { VercelLogo } from "~/components/integrations/VercelLogo"; +import { BuildSettingsFields } from "~/components/integrations/VercelBuildSettings"; +import { OctoKitty } from "~/components/GitHubLoginButton"; +import { + ConnectGitHubRepoModal, +} from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github"; +import { + type SyncEnvVarsMapping, + type EnvSlug, + ALL_ENV_SLUGS, + shouldSyncEnvVarForAnyEnvironment, + getAvailableEnvSlugs, + getAvailableEnvSlugsForBuildSettings, +} from "~/v3/vercel/vercelProjectIntegrationSchema"; +import { type VercelCustomEnvironment } from "~/models/vercelIntegration.server"; +import { type VercelOnboardingData } from "~/presenters/v3/VercelSettingsPresenter.server"; +import { vercelAppInstallPath, v3ProjectSettingsPath, githubAppInstallPath, vercelResourcePath } from "~/utils/pathBuilder"; +import type { loader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.vercel"; +import { useEffect, useState, useCallback, useRef } from "react"; + +function safeRedirectUrl(url: string): string | null { + try { + const parsed = new URL(url, window.location.origin); + if (parsed.origin === window.location.origin) { + return parsed.toString(); + } + if (parsed.protocol === "https:" && /^([a-z0-9-]+\.)*vercel\.com$/i.test(parsed.hostname)) { + return parsed.toString(); + } + } catch { + // Invalid URL + } + return null; +} + +function formatVercelTargets(targets: string[]): string { + const targetLabels: Record = { + production: "Production", + preview: "Preview", + development: "Development", + staging: "Staging", + }; + + return targets + .map((t) => targetLabels[t.toLowerCase()] || t) + .join(", "); +} + +type OnboardingState = + | "idle" + | "installing" + | "loading-projects" + | "project-selection" + | "loading-env-mapping" + | "env-mapping" + | "loading-env-vars" + | "env-var-sync" + | "build-settings" + | "github-connection" + | "completed"; + +export function VercelOnboardingModal({ + isOpen, + onClose, + onboardingData, + organizationSlug, + projectSlug, + environmentSlug, + hasStagingEnvironment, + hasPreviewEnvironment, + hasOrgIntegration, + nextUrl, + onDataReload, +}: { + isOpen: boolean; + onClose: () => void; + onboardingData: VercelOnboardingData | null; + organizationSlug: string; + projectSlug: string; + environmentSlug: string; + hasStagingEnvironment: boolean; + hasPreviewEnvironment: boolean; + hasOrgIntegration: boolean; + nextUrl?: string; + onDataReload?: (vercelStagingEnvironment?: string) => void; +}) { + const navigation = useNavigation(); + const fetcher = useTypedFetcher(); + const envMappingFetcher = useFetcher(); + const completeOnboardingFetcher = useFetcher(); + const { Form: CompleteOnboardingForm } = completeOnboardingFetcher; + const [searchParams] = useSearchParams(); + const fromMarketplaceContext = searchParams.get("origin") === "marketplace"; + + const availableProjects = onboardingData?.availableProjects || []; + const hasProjectSelected = onboardingData?.hasProjectSelected ?? false; + const customEnvironments = onboardingData?.customEnvironments || []; + const envVars = onboardingData?.environmentVariables || []; + const existingVars = onboardingData?.existingVariables || {}; + const hasCustomEnvs = customEnvironments.length > 0 && hasStagingEnvironment; + + const computeInitialState = useCallback((): OnboardingState => { + if (!hasOrgIntegration || onboardingData?.authInvalid) { + return "idle"; + } + const projectSelected = onboardingData?.hasProjectSelected ?? false; + if (!projectSelected) { + if (!onboardingData?.availableProjects || onboardingData.availableProjects.length === 0) { + return "loading-projects"; + } + return "project-selection"; + } + // For marketplace origin, skip env-mapping step and go directly to env-var-sync + if (!fromMarketplaceContext) { + const customEnvs = (onboardingData?.customEnvironments?.length ?? 0) > 0 && hasStagingEnvironment; + if (customEnvs) { + return "env-mapping"; + } + } + if (!onboardingData?.environmentVariables || onboardingData.environmentVariables.length === 0) { + return "loading-env-vars"; + } + return "env-var-sync"; + }, [hasOrgIntegration, onboardingData, hasStagingEnvironment, fromMarketplaceContext]); + + const [state, setState] = useState(() => { + if (!isOpen) return "idle"; + return computeInitialState(); + }); + + const prevIsOpenRef = useRef(isOpen); + const hasSyncedStagingRef = useRef(false); + const hasSyncedPreviewRef = useRef(false); + useEffect(() => { + if (isOpen && !prevIsOpenRef.current) { + setState(computeInitialState()); + hasSyncedStagingRef.current = false; + hasSyncedPreviewRef.current = false; + } else if (isOpen && state === "idle") { + setState(computeInitialState()); + } + prevIsOpenRef.current = isOpen; + }, [isOpen, state, computeInitialState]); + + const [selectedVercelProject, setSelectedVercelProject] = useState<{ + id: string; + name: string; + } | null>(null); + const [vercelStagingEnvironment, setVercelStagingEnvironment] = useState<{ + environmentId: string; + displayName: string; + } | null>(null); + const availableEnvSlugsForOnboarding = getAvailableEnvSlugs(hasStagingEnvironment, hasPreviewEnvironment); + const availableEnvSlugsForOnboardingBuildSettings = getAvailableEnvSlugsForBuildSettings(hasStagingEnvironment, hasPreviewEnvironment); + const [pullEnvVarsBeforeBuild, setPullEnvVarsBeforeBuild] = useState( + () => availableEnvSlugsForOnboardingBuildSettings + ); + const [atomicBuilds, setAtomicBuilds] = useState( + () => ["prod"] + ); + const [discoverEnvVars, setDiscoverEnvVars] = useState( + () => availableEnvSlugsForOnboardingBuildSettings + ); + + // Sync pullEnvVarsBeforeBuild and discoverEnvVars when hasStagingEnvironment becomes true (once) + useEffect(() => { + if (hasStagingEnvironment && !hasSyncedStagingRef.current) { + hasSyncedStagingRef.current = true; + setPullEnvVarsBeforeBuild((prev) => { + if (!prev.includes("stg")) { + return [...prev, "stg"]; + } + return prev; + }); + setDiscoverEnvVars((prev) => { + if (!prev.includes("stg")) { + return [...prev, "stg"]; + } + return prev; + }); + } + }, [hasStagingEnvironment]); + + // Sync pullEnvVarsBeforeBuild and discoverEnvVars when hasPreviewEnvironment becomes true (once) + useEffect(() => { + if (hasPreviewEnvironment && !hasSyncedPreviewRef.current) { + hasSyncedPreviewRef.current = true; + setPullEnvVarsBeforeBuild((prev) => { + if (!prev.includes("preview")) { + return [...prev, "preview"]; + } + return prev; + }); + setDiscoverEnvVars((prev) => { + if (!prev.includes("preview")) { + return [...prev, "preview"]; + } + return prev; + }); + } + }, [hasPreviewEnvironment]); + const [syncEnvVarsMapping, setSyncEnvVarsMapping] = useState({}); + const [expandedEnvVars, setExpandedEnvVars] = useState(false); + const [expandedSecretEnvVars, setExpandedSecretEnvVars] = useState(false); + const [projectSelectionError, setProjectSelectionError] = useState(null); + const [isRedirecting, setIsRedirecting] = useState(false); + + const gitHubAppInstallations = onboardingData?.gitHubAppInstallations ?? []; + const isGitHubConnectedForOnboarding = onboardingData?.isGitHubConnected ?? false; + const isOnboardingComplete = onboardingData?.isOnboardingComplete ?? false; + + const hasTriggeredMarketplaceRedirectRef = useRef(false); + + // Auto-redirect for marketplace flow when returning from GitHub with everything complete + useEffect(() => { + if (hasTriggeredMarketplaceRedirectRef.current) { + return; + } + + if ( + isOpen && + fromMarketplaceContext && + nextUrl && + isOnboardingComplete && + isGitHubConnectedForOnboarding + ) { + hasTriggeredMarketplaceRedirectRef.current = true; + const validUrl = safeRedirectUrl(nextUrl); + if (validUrl) { + setTimeout(() => { + window.location.href = validUrl; + }, 100); + } + } + }, [isOpen, fromMarketplaceContext, nextUrl, isOnboardingComplete, isGitHubConnectedForOnboarding]); + + useEffect(() => { + if (!isOpen) { + hasTriggeredMarketplaceRedirectRef.current = false; + setIsRedirecting(false); + } + }, [isOpen]); + + const loadingStateRef = useRef(null); + + useEffect(() => { + if (!isOpen || state === "idle") { + loadingStateRef.current = null; + return; + } + + if (onboardingData?.authInvalid) { + onClose(); + return; + } + + if (loadingStateRef.current === state) { + return; + } + + switch (state) { + + case "loading-projects": + loadingStateRef.current = state; + if (onDataReload) { + onDataReload(); + } + break; + + case "loading-env-mapping": + loadingStateRef.current = state; + if (onDataReload) { + onDataReload(); + } + break; + + case "loading-env-vars": + loadingStateRef.current = state; + if (onDataReload) { + onDataReload(vercelStagingEnvironment?.environmentId || undefined); + } + break; + + case "installing": + case "project-selection": + case "env-mapping": + case "env-var-sync": + case "completed": + case "build-settings": + case "github-connection": + loadingStateRef.current = null; + break; + } + }, [isOpen, state, onboardingData?.authInvalid, vercelStagingEnvironment, onDataReload, onClose]); + + useEffect(() => { + if (!onboardingData?.authInvalid && state === "loading-projects" && onboardingData?.availableProjects !== undefined) { + setState("project-selection"); + } + }, [state, onboardingData?.availableProjects, onboardingData?.authInvalid]); + + useEffect(() => { + if (!onboardingData?.authInvalid && state === "loading-env-vars" && onboardingData?.environmentVariables) { + setState("env-var-sync"); + } + }, [state, onboardingData?.environmentVariables, onboardingData?.authInvalid]); + + useEffect(() => { + if (state === "project-selection" && fetcher.data && "success" in fetcher.data && fetcher.data.success && fetcher.state === "idle") { + setState("loading-env-mapping"); + if (onDataReload) { + onDataReload(); + } + } else if (fetcher.data && "error" in fetcher.data && typeof fetcher.data.error === "string") { + setProjectSelectionError(fetcher.data.error); + } + }, [state, fetcher.data, fetcher.state, onDataReload]); + + // For marketplace origin, skip env-mapping step + useEffect(() => { + if (state === "loading-env-mapping" && onboardingData) { + const hasCustomEnvs = (onboardingData.customEnvironments?.length ?? 0) > 0 && hasStagingEnvironment; + if (hasCustomEnvs && !fromMarketplaceContext) { + setState("env-mapping"); + } else { + setState("loading-env-vars"); + } + } + }, [state, onboardingData, hasStagingEnvironment, fromMarketplaceContext]); + + const secretEnvVars = envVars.filter((v) => v.isSecret); + const syncableEnvVars = envVars.filter((v) => !v.isSecret); + const enabledEnvVars = syncableEnvVars.filter( + (v) => shouldSyncEnvVarForAnyEnvironment(syncEnvVarsMapping, v.key) + ); + + const overlappingEnvVarsCount = enabledEnvVars.filter((v) => existingVars[v.key]).length; + + const isSubmitting = + navigation.state === "submitting" || navigation.state === "loading"; + + const actionUrl = vercelResourcePath(organizationSlug, projectSlug, environmentSlug); + + const handleToggleEnvVar = useCallback((key: string, enabled: boolean) => { + setSyncEnvVarsMapping((prev) => { + const newMapping = { ...prev }; + + if (enabled) { + for (const envSlug of ALL_ENV_SLUGS) { + if (newMapping[envSlug]) { + const { [key]: _, ...rest } = newMapping[envSlug]; + if (Object.keys(rest).length === 0) { + delete newMapping[envSlug]; + } else { + newMapping[envSlug] = rest; + } + } + } + } else { + for (const envSlug of ALL_ENV_SLUGS) { + newMapping[envSlug] = { + ...(newMapping[envSlug] || {}), + [key]: false, + }; + } + } + + return newMapping; + }); + }, []); + + const handleToggleAllEnvVars = useCallback( + (enabled: boolean, syncableVars: Array<{ key: string }>) => { + if (enabled) { + setSyncEnvVarsMapping({}); + } else { + const newMapping: SyncEnvVarsMapping = {}; + for (const envSlug of ALL_ENV_SLUGS) { + newMapping[envSlug] = {}; + for (const v of syncableVars) { + newMapping[envSlug][v.key] = false; + } + } + setSyncEnvVarsMapping(newMapping); + } + }, + [] + ); + + const handleProjectSelection = useCallback(async () => { + if (!selectedVercelProject) { + setProjectSelectionError("Please select a Vercel project"); + return; + } + + setProjectSelectionError(null); + + const formData = new FormData(); + formData.append("action", "select-vercel-project"); + formData.append("vercelProjectId", selectedVercelProject.id); + formData.append("vercelProjectName", selectedVercelProject.name); + + fetcher.submit(formData, { + method: "post", + action: actionUrl, + }); + }, [selectedVercelProject, fetcher, actionUrl]); + + const handleSkipOnboarding = useCallback(() => { + onClose(); + + if (fromMarketplaceContext) { + return window.close(); + } + + const formData = new FormData(); + formData.append("action", "skip-onboarding"); + fetcher.submit(formData, { + method: "post", + action: actionUrl, + }); + }, [actionUrl, fetcher, onClose, nextUrl, fromMarketplaceContext]); + + const handleSkipEnvMapping = useCallback(() => { + setVercelStagingEnvironment(null); + setState("loading-env-vars"); + }, []); + + const handleUpdateEnvMapping = useCallback(() => { + if (!vercelStagingEnvironment) { + setState("loading-env-vars"); + return; + } + + const formData = new FormData(); + formData.append("action", "update-env-mapping"); + formData.append("vercelStagingEnvironment", JSON.stringify(vercelStagingEnvironment)); + + envMappingFetcher.submit(formData, { + method: "post", + action: actionUrl, + }); + + }, [vercelStagingEnvironment, envMappingFetcher, actionUrl]); + + const handleBuildSettingsNext = useCallback(() => { + if (nextUrl && fromMarketplaceContext && isGitHubConnectedForOnboarding) { + setIsRedirecting(true); + } + + const formData = new FormData(); + formData.append("action", "complete-onboarding"); + formData.append("vercelStagingEnvironment", vercelStagingEnvironment ? JSON.stringify(vercelStagingEnvironment) : ""); + formData.append("pullEnvVarsBeforeBuild", JSON.stringify(pullEnvVarsBeforeBuild)); + formData.append("atomicBuilds", JSON.stringify(atomicBuilds)); + formData.append("discoverEnvVars", JSON.stringify(discoverEnvVars)); + formData.append("syncEnvVarsMapping", JSON.stringify(syncEnvVarsMapping)); + if (nextUrl && fromMarketplaceContext && isGitHubConnectedForOnboarding) { + formData.append("next", nextUrl); + } + + if (!isGitHubConnectedForOnboarding) { + formData.append("skipRedirect", "true"); + } + + completeOnboardingFetcher.submit(formData, { + method: "post", + action: actionUrl, + }); + + if (!isGitHubConnectedForOnboarding) { + setState("github-connection"); + } + }, [vercelStagingEnvironment, pullEnvVarsBeforeBuild, atomicBuilds, discoverEnvVars, syncEnvVarsMapping, nextUrl, fromMarketplaceContext, isGitHubConnectedForOnboarding, completeOnboardingFetcher, actionUrl]); + + const handleFinishOnboarding = useCallback((e: React.FormEvent) => { + e.preventDefault(); + const form = e.currentTarget; + const formData = new FormData(form); + completeOnboardingFetcher.submit(formData, { + method: "post", + action: actionUrl, + }); + }, [completeOnboardingFetcher, actionUrl]); + + useEffect(() => { + if (completeOnboardingFetcher.data && typeof completeOnboardingFetcher.data === "object" && "success" in completeOnboardingFetcher.data && completeOnboardingFetcher.data.success && completeOnboardingFetcher.state === "idle") { + if (state === "github-connection") { + return; + } + if ("redirectTo" in completeOnboardingFetcher.data && typeof completeOnboardingFetcher.data.redirectTo === "string") { + const validRedirect = safeRedirectUrl(completeOnboardingFetcher.data.redirectTo); + if (validRedirect) { + window.location.href = validRedirect; + } + return; + } + setState("completed"); + } + }, [completeOnboardingFetcher.data, completeOnboardingFetcher.state, state]); + + useEffect(() => { + if (state === "completed") { + onClose(); + } + }, [state, onClose]); + + useEffect(() => { + if (state === "installing") { + const installUrl = vercelAppInstallPath(organizationSlug, projectSlug); + window.location.href = installUrl; + } + }, [state, organizationSlug, projectSlug]); + + useEffect(() => { + if (envMappingFetcher.data && typeof envMappingFetcher.data === "object" && "success" in envMappingFetcher.data && envMappingFetcher.data.success && envMappingFetcher.state === "idle") { + setState("loading-env-vars"); + } + }, [envMappingFetcher.data, envMappingFetcher.state]); + + useEffect(() => { + if (state === "env-mapping" && customEnvironments.length > 0 && !vercelStagingEnvironment) { + let selectedEnv: VercelCustomEnvironment; + + if (customEnvironments.length === 1) { + selectedEnv = customEnvironments[0]; + } else { + const stagingEnv = customEnvironments.find( + (env) => env.slug.toLowerCase() === "staging" + ); + selectedEnv = stagingEnv ?? customEnvironments[0]; + } + + setVercelStagingEnvironment({ environmentId: selectedEnv.id, displayName: selectedEnv.slug }); + } + }, [state, customEnvironments, vercelStagingEnvironment]); + + if (!isOpen || onboardingData?.authInvalid) { + return null; + } + + const isLoadingState = + state === "loading-projects" || + state === "loading-env-mapping" || + state === "loading-env-vars" || + state === "installing" || + (state === "idle" && !onboardingData); + + if (isLoadingState) { + return ( + !open && !fromMarketplaceContext && onClose()}> + + +
+ + Set up Vercel Integration +
+ +
+ +
+ + + ); + } + + const showProjectSelection = state === "project-selection"; + const showEnvMapping = state === "env-mapping"; + const showEnvVarSync = state === "env-var-sync"; + const showBuildSettings = state === "build-settings"; + const showGitHubConnection = state === "github-connection"; + + return ( + !open && !fromMarketplaceContext && onClose()}> + + +
+ + Set up Vercel Integration +
+ + +
+ {showProjectSelection && ( +
+ Select Vercel Project + + Choose which Vercel project to connect with this Trigger.dev project. + Your API keys will be automatically synced to Vercel. + + + {availableProjects.length === 0 ? ( + + No Vercel projects found. Please create a project in Vercel first. + + ) : ( + + )} + + {projectSelectionError && ( + {projectSelectionError} + )} + + + Once connected, your TRIGGER_SECRET_KEY will be + automatically synced to Vercel for each environment. + + + + {fetcher.state !== "idle" ? "Connecting..." : "Connect Project"} + + } + cancelButton={ + + } + /> +
+ )} + + {showEnvMapping && ( +
+ Map Vercel Environment to Staging + + Select which custom Vercel environment should map to Trigger.dev's Staging + environment. Production and Preview environments are mapped automatically. + + + + +
+ +
+ {!fromMarketplaceContext && ( + + )} + +
+
+
+ )} + + {showEnvVarSync && ( +
+ Pull Environment Variables + + Select which environment variables to pull from Vercel now. This is a one-time pull. + + +
+
+ {syncableEnvVars.length} + can be pulled +
+ {secretEnvVars.length > 0 && ( +
+ {secretEnvVars.length} + secret (cannot pull) +
+ )} +
+ +
+
+ Pull all environment variables now + Select all variables to pull from Vercel. +
+ handleToggleAllEnvVars(checked, syncableEnvVars)} + /> +
+ + {syncableEnvVars.length > 0 && ( +
+ + + {expandedEnvVars && ( +
+ {syncableEnvVars.map((envVar) => ( +
+
+ {existingVars[envVar.key] ? ( + + + +
+ {envVar.key} +
+ + + {`This variable is going to be replaced in: ${existingVars[ + envVar.key + ].environments.join(", ")}`} + + + + ) : ( + {envVar.key} + )} + {envVar.target && envVar.target.length > 0 && ( + + {formatVercelTargets(envVar.target)} + {envVar.isShared && " · Shared"} + + )} +
+ + handleToggleEnvVar(envVar.key, checked) + } + /> +
+ ))} +
+ )} +
+ )} + + {secretEnvVars.length > 0 && ( +
+ + + {expandedSecretEnvVars && ( +
+ {secretEnvVars.map((envVar) => ( +
+
+ {envVar.key} + {envVar.target && envVar.target.length > 0 && ( + + {formatVercelTargets(envVar.target)} + {envVar.isShared && " · Shared"} + + )} +
+ Secret +
+ ))} +
+ )} +
+ )} + + {overlappingEnvVarsCount > 0 && enabledEnvVars.length > 0 && ( +
+ + + {overlappingEnvVarsCount} env vars are going to be updated (marked with{" "} + + underline + + ) + +
+ )} + + { + if (fromMarketplaceContext) { + handleBuildSettingsNext(); + } else { + setState("build-settings"); + } + }} + disabled={fromMarketplaceContext && (completeOnboardingFetcher.state !== "idle" || isRedirecting)} + LeadingIcon={fromMarketplaceContext && (completeOnboardingFetcher.state !== "idle" || isRedirecting) ? SpinnerWhite : undefined} + > + {fromMarketplaceContext ? (isGitHubConnectedForOnboarding ? "Finish" : "Next") : "Next"} + + } + cancelButton={ + hasCustomEnvs && !fromMarketplaceContext ? ( + + ) : ( + + ) + } + /> +
+ )} + + {showBuildSettings && ( +
+ Build Settings + + Configure how environment variables are pulled during builds and atomic deployments. + + + + + + {isGitHubConnectedForOnboarding ? "Finish" : "Next"} + + } + cancelButton={ + + } + /> +
+ )} + + {showGitHubConnection && ( +
+ Connect GitHub Repository + + To fully integrate with Vercel, Trigger.dev needs access to your source code. + This allows automatic deployments and build synchronization. + + + +
+ Connecting your GitHub repository enables Trigger.dev to read your source code + and automatically create deployments when you push changes to Vercel. +
+ + + {(() => { + const baseSettingsPath = v3ProjectSettingsPath( + { slug: organizationSlug }, + { slug: projectSlug }, + { slug: environmentSlug } + ); + const redirectParams = new URLSearchParams(); + redirectParams.set("vercelOnboarding", "true"); + if (fromMarketplaceContext) { + redirectParams.set("origin", "marketplace"); + } + if (nextUrl) { + redirectParams.set("next", nextUrl); + } + const redirectUrlWithContext = `${baseSettingsPath}?${redirectParams.toString()}`; + + return gitHubAppInstallations.length === 0 ? ( +
+ + Install GitHub app + +
+ ) : ( +
+
+ + + GitHub app is installed + +
+
+ ); + })()} + + { + setState("completed"); + const validUrl = safeRedirectUrl(nextUrl); + if (validUrl) { + window.location.href = validUrl; + } + }} + > + Complete + + ) : ( + + ) + } + cancelButton={ + isGitHubConnectedForOnboarding && fromMarketplaceContext && nextUrl ? ( + + ) : undefined + } + /> +
+ )} +
+ + + ); +} diff --git a/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx b/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx index e42928cdd6a..8758e181ff8 100644 --- a/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx +++ b/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx @@ -3,6 +3,7 @@ import { ChartBarIcon, Cog8ToothIcon, CreditCardIcon, + PuzzlePieceIcon, UserGroupIcon, } from "@heroicons/react/20/solid"; import { ArrowLeftIcon } from "@heroicons/react/24/solid"; @@ -12,6 +13,7 @@ import { cn } from "~/utils/cn"; import { organizationSettingsPath, organizationTeamPath, + organizationVercelIntegrationPath, rootPath, v3BillingAlertsPath, v3BillingPath, @@ -113,6 +115,13 @@ export function OrganizationSettingsSideMenu({ to={organizationSettingsPath(organization)} data-action="settings" /> +

diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index dcbcac079a0..6733af0addb 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -425,6 +425,11 @@ const EnvironmentSchema = z ORG_SLACK_INTEGRATION_CLIENT_ID: z.string().optional(), ORG_SLACK_INTEGRATION_CLIENT_SECRET: z.string().optional(), + /** Vercel integration OAuth credentials */ + VERCEL_INTEGRATION_CLIENT_ID: z.string().optional(), + VERCEL_INTEGRATION_CLIENT_SECRET: z.string().optional(), + VERCEL_INTEGRATION_APP_SLUG: z.string().optional(), + /** These enable the alerts feature in v3 */ ALERT_EMAIL_TRANSPORT: z.enum(["resend", "smtp", "aws-ses"]).optional(), ALERT_FROM_EMAIL: z.string().optional(), diff --git a/apps/webapp/app/models/orgIntegration.server.ts b/apps/webapp/app/models/orgIntegration.server.ts index 343da2701d9..a69aa917671 100644 --- a/apps/webapp/app/models/orgIntegration.server.ts +++ b/apps/webapp/app/models/orgIntegration.server.ts @@ -47,6 +47,13 @@ export type AuthenticatableIntegration = OrganizationIntegration & { tokenReference: SecretReference; }; +export function isIntegrationForService( + integration: AuthenticatableIntegration, + service: TService +): integration is OrganizationIntegrationForService { + return (integration.service satisfies IntegrationService) === service; +} + export class OrgIntegrationRepository { static async getAuthenticatedClientForIntegration( integration: OrganizationIntegrationForService, @@ -89,6 +96,23 @@ export class OrgIntegrationRepository { static isSlackSupported = !!env.ORG_SLACK_INTEGRATION_CLIENT_ID && !!env.ORG_SLACK_INTEGRATION_CLIENT_SECRET; + static isVercelSupported = + !!env.VERCEL_INTEGRATION_CLIENT_ID && !!env.VERCEL_INTEGRATION_CLIENT_SECRET && !!env.VERCEL_INTEGRATION_APP_SLUG; + + /** + * Generate the URL to install the Vercel integration. + * Users are redirected to Vercel's marketplace to complete the installation. + * + * @param state - Base64-encoded state containing org/project info for the callback + */ + static vercelInstallUrl(state: string): string { + // The user goes to Vercel's marketplace to install the integration + // After installation, Vercel redirects to our callback with the authorization code + const redirectUri = encodeURIComponent(`${env.APP_ORIGIN}/vercel/callback`); + const encodedState = encodeURIComponent(state); + return `https://vercel.com/integrations/${env.VERCEL_INTEGRATION_APP_SLUG}/new?state=${encodedState}&redirect_uri=${redirectUri}`; + } + static slackAuthorizationUrl( state: string, scopes: string[] = [ diff --git a/apps/webapp/app/models/vercelIntegration.server.ts b/apps/webapp/app/models/vercelIntegration.server.ts new file mode 100644 index 00000000000..c31b8bde27a --- /dev/null +++ b/apps/webapp/app/models/vercelIntegration.server.ts @@ -0,0 +1,1659 @@ +import pLimit from "p-limit"; +import { Vercel } from "@vercel/sdk"; +import type { + ResponseBodyEnvs, + FilterProjectEnvsResponseBody, +} from "@vercel/sdk/models/filterprojectenvsop"; +import type { + GetV9ProjectsIdOrNameCustomEnvironmentsEnvironments, +} from "@vercel/sdk/models/getv9projectsidornamecustomenvironmentsop"; +import type { ResponseBodyProjects } from "@vercel/sdk/models/getprojectsop"; +import { + Organization, + OrganizationIntegration, + SecretReference, +} from "@trigger.dev/database"; +import { z } from "zod"; +import { ResultAsync, errAsync, okAsync } from "neverthrow"; +import { $transaction, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { getSecretStore } from "~/services/secrets/secretStore.server"; +import { generateFriendlyId } from "~/v3/friendlyIdentifiers"; +import { + SyncEnvVarsMapping, + shouldSyncEnvVar, + TriggerEnvironmentType, + envTypeToVercelTarget, +} from "~/v3/vercel/vercelProjectIntegrationSchema"; +import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; + +// --------------------------------------------------------------------------- +// Pure helpers +// --------------------------------------------------------------------------- + +function normalizeTarget(target: string[] | string | undefined): string[] { + if (Array.isArray(target)) return target.filter(Boolean); + if (typeof target === 'string') return [target]; + return []; +} + +function extractVercelEnvs( + response: FilterProjectEnvsResponseBody +): ResponseBodyEnvs[] { + if ("envs" in response && Array.isArray(response.envs)) { + return response.envs; + } + return []; +} + +function isVercelSecretType(type: string): boolean { + return type === "secret" || type === "sensitive"; +} + +// --------------------------------------------------------------------------- +// Error handling +// --------------------------------------------------------------------------- + +export type VercelApiError = { + message: string; + authInvalid: boolean; +}; + +const VercelErrorSchema = z.union([ + z.object({ status: z.number() }), + z.object({ response: z.object({ status: z.number() }) }), + z.object({ statusCode: z.number() }), +]); + +function extractVercelErrorStatus(error: unknown): number | null { + if (error && typeof error === 'object' && 'status' in error) { + const parsed = VercelErrorSchema.safeParse(error); + if (parsed.success && 'status' in parsed.data) { + return parsed.data.status; + } + } + + if (error && typeof error === 'object' && 'response' in error) { + const parsed = VercelErrorSchema.safeParse(error); + if (parsed.success && 'response' in parsed.data) { + return parsed.data.response.status; + } + } + + if (error && typeof error === 'object' && 'statusCode' in error) { + const parsed = VercelErrorSchema.safeParse(error); + if (parsed.success && 'statusCode' in parsed.data) { + return parsed.data.statusCode; + } + } + + if (typeof error === 'string') { + if (error.includes('401')) return 401; + if (error.includes('403')) return 403; + } + + return null; +} + +function isVercelAuthError(error: unknown): boolean { + const status = extractVercelErrorStatus(error); + return status === 401 || status === 403; +} + +function toVercelApiError(error: unknown): VercelApiError { + if (isVercelApiErrorShape(error)) return error; + return { + message: error instanceof Error ? error.message : "Unknown error", + authInvalid: isVercelAuthError(error), + }; +} + +function isVercelApiErrorShape(error: unknown): error is VercelApiError { + return ( + error !== null && + typeof error === "object" && + "message" in error && + "authInvalid" in error && + typeof (error as VercelApiError).message === "string" && + typeof (error as VercelApiError).authInvalid === "boolean" + ); +} + +/** + * Wrap a Vercel SDK call in ResultAsync with structured error logging. + */ +function wrapVercelCall( + promise: Promise, + message: string, + context: Record +): ResultAsync { + return ResultAsync.fromPromise(promise, (error) => { + const apiError = toVercelApiError(error); + logger.error(message, { ...context, error, authInvalid: apiError.authInvalid }); + return apiError; + }); +} + +// --------------------------------------------------------------------------- +// Schemas & token types +// --------------------------------------------------------------------------- + +export const VercelSecretSchema = z.object({ + accessToken: z.string(), + tokenType: z.string().optional(), + teamId: z.string().nullable().optional(), + userId: z.string().optional(), + installationId: z.string().optional(), + raw: z.record(z.any()).optional(), +}); + +export type VercelSecret = z.infer; + +export type TokenResponse = { + accessToken: string; + tokenType: string; + teamId?: string; + userId?: string; + raw: Record; +}; + +// --------------------------------------------------------------------------- +// Domain types narrowed from Vercel SDK response types. +// +// Using Pick and indexed-access types ties these definitions to the SDK so +// that any upstream type change surfaces as a compile error here rather than +// silently breaking at runtime. +// --------------------------------------------------------------------------- + +/** Narrowed env-var type from the SDK's FilterProjectEnvs response. */ +export type VercelEnvironmentVariable = { + id: string; // narrowed from ResponseBodyEnvs["id"] (string | undefined) + key: ResponseBodyEnvs["key"]; + type: ResponseBodyEnvs["type"]; + isSecret: boolean; + target: string[]; + isShared?: boolean; + customEnvironmentIds: string[]; +}; + +/** Narrowed custom-environment type – only the fields we consume. */ +export type VercelCustomEnvironment = Pick< + GetV9ProjectsIdOrNameCustomEnvironmentsEnvironments, + "id" | "slug" | "description" | "branchMatcher" +>; + +/** Narrowed env-var-with-value type from the SDK's FilterProjectEnvs response. */ +export type VercelEnvironmentVariableValue = { + key: ResponseBodyEnvs["key"]; + value: string; // narrowed from ResponseBodyEnvs["value"] – only present after null-check + target: string[]; + type: ResponseBodyEnvs["type"]; + isSecret: boolean; +}; + +/** Narrowed Vercel project type – only id and name. */ +export type VercelProject = Pick; + +// --------------------------------------------------------------------------- +// Mapper functions – narrow wide SDK responses into our domain types. +// --------------------------------------------------------------------------- + +function toVercelEnvironmentVariable( + env: ResponseBodyEnvs +): VercelEnvironmentVariable { + return { + id: env.id ?? "", + key: env.key, + type: env.type, + isSecret: isVercelSecretType(env.type), + target: normalizeTarget(env.target), + customEnvironmentIds: env.customEnvironmentIds ?? [], + }; +} + +function toVercelCustomEnvironment({ + id, + slug, + description, + branchMatcher, +}: GetV9ProjectsIdOrNameCustomEnvironmentsEnvironments): VercelCustomEnvironment { + return { id, slug, description, branchMatcher }; +} + +function toVercelEnvironmentVariableValue( + env: ResponseBodyEnvs +): VercelEnvironmentVariableValue | null { + if (!env.value) return null; + return { + key: env.key, + value: env.value, + target: normalizeTarget(env.target), + type: env.type, + isSecret: isVercelSecretType(env.type), + }; +} + +// --------------------------------------------------------------------------- +// Repository +// --------------------------------------------------------------------------- + +export class VercelIntegrationRepository { + static exchangeCodeForToken(code: string): ResultAsync { + const clientId = env.VERCEL_INTEGRATION_CLIENT_ID; + const clientSecret = env.VERCEL_INTEGRATION_CLIENT_SECRET; + const redirectUri = `${env.APP_ORIGIN}/vercel/callback`; + + if (!clientId || !clientSecret) { + logger.error("Vercel integration not configured"); + return errAsync({ message: "Vercel integration not configured", authInvalid: false }); + } + + return ResultAsync.fromPromise( + fetch("https://api.vercel.com/v2/oauth/access_token", { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ + client_id: clientId, + client_secret: clientSecret, + code, + redirect_uri: redirectUri, + }), + }).then(async (response) => { + if (!response.ok) { + const errorText = await response.text(); + logger.error("Failed to exchange Vercel OAuth code", { + status: response.status, + error: errorText, + }); + throw new Error(`HTTP ${response.status}: ${errorText}`); + } + return response.json() as Promise<{ + access_token: string; + token_type: string; + team_id?: string; + user_id?: string; + }>; + }), + (error) => { + logger.error("Error exchanging Vercel OAuth code", { error }); + return toVercelApiError(error); + } + ).map((data): TokenResponse => ({ + accessToken: data.access_token, + tokenType: data.token_type, + teamId: data.team_id, + userId: data.user_id, + raw: data as Record, + })); + } + + static getVercelClient( + integration: OrganizationIntegration & { tokenReference: SecretReference } + ): ResultAsync { + return ResultAsync.fromPromise( + (async () => { + const secretStore = getSecretStore(integration.tokenReference.provider); + const secret = await secretStore.getSecret( + VercelSecretSchema, + integration.tokenReference.key + ); + if (!secret) { + throw new Error("Failed to get Vercel access token"); + } + return new Vercel({ bearerToken: secret.accessToken }); + })(), + (error) => toVercelApiError(error) + ); + } + + static getTeamSlug( + client: Vercel, + teamId: string | null + ): ResultAsync { + if (teamId) { + return wrapVercelCall( + client.teams.getTeam({ teamId }), + "Failed to fetch Vercel team", + { teamId } + ).map((response) => response.slug); + } + + return wrapVercelCall( + client.user.getAuthUser(), + "Failed to fetch Vercel user", + {} + ).map((response) => response?.user.username ?? "unknown"); + } + + static validateVercelToken( + integration: OrganizationIntegration & { tokenReference: SecretReference } + ): ResultAsync<{ isValid: boolean }, VercelApiError> { + return this.getVercelClient(integration) + .andThen((client) => + ResultAsync.fromPromise( + client.user.getAuthUser(), + toVercelApiError + ) + ) + .map(() => ({ isValid: true })) + .orElse((error) => + error.authInvalid + ? okAsync({ isValid: false }) + : errAsync(error) + ); + } + + static async getTeamIdFromIntegration( + integration: OrganizationIntegration & { tokenReference: SecretReference } + ): Promise { + const secretStore = getSecretStore(integration.tokenReference.provider); + + const secret = await secretStore.getSecret( + VercelSecretSchema, + integration.tokenReference.key + ); + + if (!secret) { + return null; + } + + return secret.teamId ?? null; + } + + static getVercelIntegrationConfiguration( + accessToken: string, + configurationId: string, + teamId?: string | null + ): ResultAsync<{ + id: string; + teamId: string | null; + projects: string[]; + }, VercelApiError> { + return ResultAsync.fromPromise( + fetch( + `https://api.vercel.com/v1/integrations/configuration/${configurationId}${teamId ? `?teamId=${teamId}` : ""}`, + { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + } + ).then(async (response) => { + if (!response.ok) { + const errorText = await response.text(); + logger.error("Failed to fetch Vercel integration configuration", { + status: response.status, + error: errorText, + configurationId, + teamId, + }); + throw new Error(`HTTP ${response.status}: ${errorText}`); + } + return response.json() as Promise<{ + id: string; + teamId?: string | null; + projects?: string[]; + [key: string]: any; + }>; + }), + (error) => { + logger.error("Error fetching Vercel integration configuration", { + configurationId, + teamId, + error, + }); + return toVercelApiError(error); + } + ).map((data) => ({ + id: data.id, + teamId: data.teamId ?? null, + projects: data.projects || [], + })); + } + + static getVercelCustomEnvironments( + client: Vercel, + projectId: string, + teamId?: string | null + ): ResultAsync { + return wrapVercelCall( + client.environment.getV9ProjectsIdOrNameCustomEnvironments({ + idOrName: projectId, + ...(teamId && { teamId }), + }), + "Failed to fetch Vercel custom environments", + { projectId, teamId } + ).map((response) => (response.environments || []).map(toVercelCustomEnvironment)); + } + + static getVercelEnvironmentVariables( + client: Vercel, + projectId: string, + teamId?: string | null, + ): ResultAsync { + return wrapVercelCall( + client.projects.filterProjectEnvs({ + idOrName: projectId, + ...(teamId && { teamId }), + }), + "Failed to fetch Vercel environment variables", + { projectId, teamId } + ).map((response) => { + // Warn if response is paginated (more data exists that we're not fetching) + if ( + "pagination" in response && + response.pagination && + "next" in response.pagination && + response.pagination.next !== null + ) { + logger.warn( + "Vercel filterProjectEnvs returned paginated response - some env vars may be missing", + { projectId, count: response.pagination.count } + ); + } + return extractVercelEnvs(response).map(toVercelEnvironmentVariable); + }); + } + + static getVercelEnvironmentVariableValues( + client: Vercel, + projectId: string, + teamId?: string | null, + target?: string, + /** If provided, only include keys that pass this filter */ + shouldIncludeKey?: (key: string) => boolean + ): ResultAsync { + return wrapVercelCall( + client.projects.filterProjectEnvs({ + idOrName: projectId, + ...(teamId && { teamId }), + }), + "Failed to fetch Vercel environment variable values", + { projectId, teamId, target } + ).andThen((response) => { + // Apply all filters BEFORE decryption to avoid unnecessary API calls + const filteredEnvs = extractVercelEnvs(response).filter((env) => { + if (target && !normalizeTarget(env.target).includes(target)) return false; + if (shouldIncludeKey && !shouldIncludeKey(env.key)) return false; + if (isVercelSecretType(env.type)) return false; + return true; + }); + + // Fetch decrypted values for encrypted vars, use list values for others + const concurrencyLimit = pLimit(5); + return ResultAsync.fromPromise( + Promise.all( + filteredEnvs.map((env) => + concurrencyLimit(() => this.#resolveEnvVarValue(client, projectId, teamId, env)) + ) + ), + (error) => toVercelApiError(error) + ).map((results) => results.filter((v): v is VercelEnvironmentVariableValue => v !== null)); + }); + } + + static async #resolveEnvVarValue( + client: Vercel, + projectId: string, + teamId: string | null | undefined, + env: ResponseBodyEnvs + ): Promise { + // Non-encrypted vars: use value from list response if present + if (env.type !== "encrypted" || !env.id) { + if (env.value === undefined || env.value === null) return null; + return toVercelEnvironmentVariableValue(env); + } + + // Encrypted vars: fetch decrypted value via individual endpoint + // (list endpoint's decrypt param is deprecated) + const result = await ResultAsync.fromPromise( + client.projects.getProjectEnv({ + idOrName: projectId, + id: env.id, + ...(teamId && { teamId }), + }), + (error) => error + ); + + if (result.isErr()) { + logger.warn("Failed to decrypt Vercel env var", { + projectId, + envVarKey: env.key, + error: result.error instanceof Error ? result.error.message : String(result.error), + }); + return null; + } + + // API returns union: ResponseBody1 has no value, ResponseBody2/3 have value + const decryptedValue = (result.value as { value?: string }).value; + if (typeof decryptedValue !== "string") return null; + + return { + key: env.key, + value: decryptedValue, + target: normalizeTarget(env.target), + type: env.type, + isSecret: false, + }; + } + + static getVercelSharedEnvironmentVariables( + client: Vercel, + teamId: string, + projectId?: string // Optional: filter by project + ): ResultAsync, VercelApiError> { + return wrapVercelCall( + client.environment.listSharedEnvVariable({ + teamId, + ...(projectId && { projectId }), + }), + "Failed to fetch Vercel shared environment variables", + { teamId, projectId } + ).map((response) => { + const envVars = response.data || []; + return envVars + .filter((env): env is typeof env & { id: string; key: string } => + typeof env.id === "string" && typeof env.key === "string" + ) + .map((env) => { + const type = env.type || "plain"; + return { + id: env.id, + key: env.key, + type, + isSecret: isVercelSecretType(type), + target: normalizeTarget(env.target), + }; + }); + }); + } + + static getVercelSharedEnvironmentVariableValues( + client: Vercel, + teamId: string, + projectId?: string // Optional: filter by project + ): ResultAsync< + Array<{ + key: string; + value: string; + target: string[]; + type: string; + isSecret: boolean; + applyToAllCustomEnvironments?: boolean; + }>, + VercelApiError + > { + return wrapVercelCall( + client.environment.listSharedEnvVariable({ + teamId, + ...(projectId && { projectId }), + }), + "Failed to fetch Vercel shared environment variable values", + { teamId, projectId } + ).andThen((listResponse) => { + const envVars = listResponse.data || []; + if (envVars.length === 0) { + return okAsync([]); + } + + const concurrencyLimit = pLimit(5); + return ResultAsync.fromPromise( + Promise.all( + envVars.map((env) => + concurrencyLimit(async () => { + if (!env.id || !env.key) return null; + + const envId = env.id; + const envKey = env.key; + const type = env.type || "plain"; + const isSecret = isVercelSecretType(type); + + if (isSecret) return null; + + const listValue = (env as any).value as string | undefined; + const applyToAllCustomEnvs = (env as any).applyToAllCustomEnvironments as boolean | undefined; + + if (listValue) { + return { + key: envKey, + value: listValue, + target: normalizeTarget(env.target), + type, + isSecret, + applyToAllCustomEnvironments: applyToAllCustomEnvs, + }; + } + + // Try to get the decrypted value for this shared env var + const getResult = await ResultAsync.fromPromise( + client.environment.getSharedEnvVar({ + id: envId, + teamId, + }), + (error) => error + ); + + if (getResult.isOk()) { + if (!getResult.value.value) return null; + return { + key: envKey, + value: getResult.value.value, + target: normalizeTarget(env.target), + type, + isSecret, + applyToAllCustomEnvironments: applyToAllCustomEnvs, + }; + } + + // Workaround: Vercel SDK may throw ResponseValidationError even when the API response + // is valid (e.g., deletedAt: null vs expected number). Extract value from rawValue. + const error = getResult.error; + let errorValue: string | undefined; + if (error && typeof error === "object" && "rawValue" in error) { + const rawValue = (error as any).rawValue; + if (rawValue && typeof rawValue === "object" && "value" in rawValue) { + errorValue = rawValue.value as string | undefined; + } + } + + const fallbackValue = errorValue || listValue; + + if (fallbackValue) { + logger.warn("getSharedEnvVar failed validation, using value from error.rawValue or list response", { + teamId, + envId, + envKey, + error: error instanceof Error ? error.message : String(error), + hasErrorRawValue: !!errorValue, + hasListValue: !!listValue, + valueLength: fallbackValue.length, + }); + return { + key: envKey, + value: fallbackValue, + target: normalizeTarget(env.target), + type, + isSecret, + applyToAllCustomEnvironments: applyToAllCustomEnvs, + }; + } + + logger.warn("Failed to get decrypted value for shared env var, no fallback available", { + teamId, + projectId, + envId, + envKey, + error: error instanceof Error ? error.message : String(error), + errorStack: error instanceof Error ? error.stack : undefined, + hasRawValue: error && typeof error === "object" && "rawValue" in error, + }); + return null; + }) + ) + ), + (error) => { + logger.error("Failed to process shared environment variable values", { + teamId, + projectId, + error: error instanceof Error ? error.message : String(error), + }); + return toVercelApiError(error); + } + ).map((results) => results.filter((r): r is NonNullable => r !== null)); + }); + } + + static getVercelProjects( + client: Vercel, + teamId?: string | null + ): ResultAsync { + return ResultAsync.fromPromise( + (async () => { + const allProjects: VercelProject[] = []; + let from: string | undefined; + + do { + const response = await client.projects.getProjects({ + ...(teamId && { teamId }), + limit: "100", + ...(from && { from }), + }); + + const projects = Array.isArray(response) + ? response + : "projects" in response + ? response.projects + : []; + allProjects.push(...projects.map(({ id, name }): VercelProject => ({ id, name }))); + + // Get pagination token for next page + const pagination = + !Array.isArray(response) && "pagination" in response + ? response.pagination + : undefined; + from = + pagination && "next" in pagination && pagination.next !== null + ? String(pagination.next) + : undefined; + } while (from); + + return allProjects; + })(), + (error) => { + logger.error("Failed to fetch Vercel projects", { teamId, error }); + return toVercelApiError(error); + } + ); + } + + static async updateVercelOrgIntegrationToken(params: { + integrationId: string; + accessToken: string; + tokenType?: string; + teamId: string | null; + userId?: string; + installationId?: string; + raw?: Record; + }): Promise { + await $transaction(prisma, async (tx) => { + const integration = await tx.organizationIntegration.findUnique({ + where: { id: params.integrationId }, + include: { tokenReference: true }, + }); + + if (!integration) { + throw new Error("Vercel integration not found"); + } + + const secretStore = getSecretStore(integration.tokenReference.provider, { + prismaClient: tx, + }); + + const secretValue: VercelSecret = { + accessToken: params.accessToken, + tokenType: params.tokenType, + teamId: params.teamId, + userId: params.userId, + installationId: params.installationId, + raw: params.raw, + }; + + await secretStore.setSecret(integration.tokenReference.key, secretValue); + + await tx.organizationIntegration.update({ + where: { id: params.integrationId }, + data: { + integrationData: { + teamId: params.teamId, + userId: params.userId, + installationId: params.installationId, + } as any, + }, + }); + }); + } + + static async createVercelOrgIntegration(params: { + accessToken: string; + tokenType?: string; + teamId: string | null; + userId?: string; + installationId?: string; + organization: Pick; + raw?: Record; + origin: 'marketplace' | 'dashboard'; + }): Promise { + const result = await $transaction(prisma, async (tx) => { + const secretStore = getSecretStore("DATABASE", { + prismaClient: tx, + }); + + const integrationFriendlyId = generateFriendlyId("org_integration"); + + const secretValue: VercelSecret = { + accessToken: params.accessToken, + tokenType: params.tokenType, + teamId: params.teamId, + userId: params.userId, + installationId: params.installationId, + raw: params.raw, + }; + + await secretStore.setSecret(integrationFriendlyId, secretValue); + + const reference = await tx.secretReference.create({ + data: { + provider: "DATABASE", + key: integrationFriendlyId, + }, + }); + + return await tx.organizationIntegration.create({ + data: { + friendlyId: integrationFriendlyId, + organizationId: params.organization.id, + service: "VERCEL", + externalOrganizationId: params.teamId, + tokenReferenceId: reference.id, + integrationData: { + teamId: params.teamId, + userId: params.userId, + installationId: params.installationId, + origin: params.origin, + } as any, + }, + }); + }); + + if (!result) { + throw new Error("Failed to create Vercel organization integration"); + } + + return result; + } + + static async findVercelOrgIntegrationByTeamId( + organizationId: string, + teamId: string | null + ): Promise<(OrganizationIntegration & { tokenReference: SecretReference }) | null> { + return prisma.organizationIntegration.findFirst({ + where: { + organizationId, + service: "VERCEL", + externalOrganizationId: teamId, + deletedAt: null, + }, + include: { + tokenReference: true, + }, + }); + } + + static async findVercelOrgIntegrationForProject( + projectId: string + ): Promise<(OrganizationIntegration & { tokenReference: SecretReference }) | null> { + const projectIntegration = await prisma.organizationProjectIntegration.findFirst({ + where: { + projectId, + deletedAt: null, + organizationIntegration: { + service: "VERCEL", + deletedAt: null, + }, + }, + include: { + organizationIntegration: { + include: { + tokenReference: true, + }, + }, + }, + }); + + return projectIntegration?.organizationIntegration ?? null; + } + + static async findVercelOrgIntegrationByOrganization( + organizationId: string + ): Promise<(OrganizationIntegration & { tokenReference: SecretReference }) | null> { + return prisma.organizationIntegration.findFirst({ + where: { + organizationId, + service: "VERCEL", + deletedAt: null, + }, + include: { + tokenReference: true, + }, + }); + } + + static syncApiKeysToVercel(params: { + projectId: string; + vercelProjectId: string; + teamId: string | null; + vercelStagingEnvironment?: { environmentId: string; displayName: string } | null; + orgIntegration: OrganizationIntegration & { tokenReference: SecretReference }; + }): ResultAsync<{ created: number; updated: number; errors: string[] }, VercelApiError> { + return this.getVercelClient(params.orgIntegration).andThen((client) => + ResultAsync.fromPromise( + (async () => { + // Get all environments for the project (exclude DEVELOPMENT — we don't push keys to Vercel's development target) + const environments = await prisma.runtimeEnvironment.findMany({ + where: { + projectId: params.projectId, + type: { + in: ["PRODUCTION", "STAGING", "PREVIEW"], + }, + }, + select: { + id: true, + type: true, + apiKey: true, + }, + }); + + // Build the list of env vars to sync + const envVarsToSync: Array<{ + key: string; + value: string; + target: string[]; + type: "sensitive" | "encrypted" | "plain"; + environmentType: string; + }> = []; + + for (const runtimeEnv of environments) { + const vercelTarget = envTypeToVercelTarget( + runtimeEnv.type as TriggerEnvironmentType, + params.vercelStagingEnvironment?.environmentId + ); + + if (!vercelTarget) { + continue; + } + + envVarsToSync.push({ + key: "TRIGGER_SECRET_KEY", + value: runtimeEnv.apiKey, + target: vercelTarget, + type: "encrypted", + environmentType: runtimeEnv.type, + }); + } + + if (envVarsToSync.length === 0) { + return { created: 0, updated: 0, errors: [] as string[] }; + } + + const result = await this.batchUpsertVercelEnvVars({ + client, + vercelProjectId: params.vercelProjectId, + teamId: params.teamId, + envVars: envVarsToSync, + }); + + logger.info("Synced API keys to Vercel", { + projectId: params.projectId, + vercelProjectId: params.vercelProjectId, + syncedCount: result.created + result.updated, + created: result.created, + updated: result.updated, + errors: result.errors, + }); + + return result; + })(), + (error) => { + logger.error("Failed to sync API keys to Vercel", { + projectId: params.projectId, + vercelProjectId: params.vercelProjectId, + error, + }); + return toVercelApiError(error); + } + ) + ); + } + + static syncSingleApiKeyToVercel(params: { + projectId: string; + environmentType: "PRODUCTION" | "STAGING" | "PREVIEW" | "DEVELOPMENT"; + apiKey: string; + }): ResultAsync { + return ResultAsync.fromPromise( + (async () => { + const projectIntegration = await prisma.organizationProjectIntegration.findFirst({ + where: { + projectId: params.projectId, + deletedAt: null, + organizationIntegration: { + service: "VERCEL", + deletedAt: null, + }, + }, + include: { + organizationIntegration: { + include: { + tokenReference: true, + }, + }, + }, + }); + + if (!projectIntegration) { + return; // No integration, nothing to sync + } + + const orgIntegration = projectIntegration.organizationIntegration; + const clientResult = await this.getVercelClient(orgIntegration); + if (clientResult.isErr()) throw clientResult.error; + const client = clientResult.value; + + const teamId = await this.getTeamIdFromIntegration(orgIntegration); + + const integrationData = projectIntegration.integrationData as any; + const vercelStagingEnvironment = integrationData?.config?.vercelStagingEnvironment; + + const vercelTarget = envTypeToVercelTarget( + params.environmentType, + vercelStagingEnvironment?.environmentId + ); + + if (!vercelTarget) { + return; + } + + await this.upsertVercelEnvVar({ + client, + vercelProjectId: projectIntegration.externalEntityId, + teamId, + key: "TRIGGER_SECRET_KEY", + value: params.apiKey, + target: vercelTarget, + type: "encrypted", + }); + + logger.info("Synced regenerated API key to Vercel", { + projectId: params.projectId, + vercelProjectId: projectIntegration.externalEntityId, + environmentType: params.environmentType, + target: vercelTarget, + }); + })(), + (error) => { + logger.error("Failed to sync API key to Vercel", { + projectId: params.projectId, + environmentType: params.environmentType, + error, + }); + return toVercelApiError(error); + } + ); + } + + static pullEnvVarsFromVercel(params: { + projectId: string; + vercelProjectId: string; + teamId: string | null; + vercelStagingEnvironment?: { environmentId: string; displayName: string } | null; + syncEnvVarsMapping: SyncEnvVarsMapping; + orgIntegration: OrganizationIntegration & { tokenReference: SecretReference }; + }): ResultAsync<{ syncedCount: number; errors: string[] }, VercelApiError> { + logger.info("pullEnvVarsFromVercel: Starting", { + projectId: params.projectId, + vercelProjectId: params.vercelProjectId, + teamId: params.teamId, + vercelStagingEnvironment: params.vercelStagingEnvironment, + syncEnvVarsMappingKeys: Object.keys(params.syncEnvVarsMapping), + }); + + return this.getVercelClient(params.orgIntegration).andThen((client) => + ResultAsync.fromPromise( + (async () => { + const errors: string[] = []; + let syncedCount = 0; + + // Get all runtime environments for the project + const runtimeEnvironments = await prisma.runtimeEnvironment.findMany({ + where: { + projectId: params.projectId, + type: { + in: ["PRODUCTION", "STAGING", "PREVIEW", "DEVELOPMENT"], + }, + }, + select: { + id: true, + type: true, + }, + }); + + const envMapping: Array<{ + triggerEnvType: "PRODUCTION" | "STAGING" | "PREVIEW" | "DEVELOPMENT"; + vercelTarget: string; + runtimeEnvironmentId: string; + }> = []; + + for (const runtimeEnv of runtimeEnvironments) { + const vercelTarget = envTypeToVercelTarget( + runtimeEnv.type as TriggerEnvironmentType, + params.vercelStagingEnvironment?.environmentId + ); + + if (!vercelTarget) { + continue; + } + + envMapping.push({ + triggerEnvType: runtimeEnv.type as "PRODUCTION" | "STAGING" | "PREVIEW" | "DEVELOPMENT", + vercelTarget: vercelTarget[0], + runtimeEnvironmentId: runtimeEnv.id, + }); + } + + if (envMapping.length === 0) { + logger.warn("No environments to sync for Vercel integration", { + projectId: params.projectId, + vercelProjectId: params.vercelProjectId, + }); + return { syncedCount: 0, errors: [] as string[] }; + } + + const envVarRepository = new EnvironmentVariablesRepository(); + + // Fetch shared env vars once (they apply across all targets) + let sharedEnvVars: Array<{ + key: string; + value: string; + target: string[]; + type: string; + isSecret: boolean; + applyToAllCustomEnvironments?: boolean; + }> = []; + + if (params.teamId) { + const sharedResult = await this.getVercelSharedEnvironmentVariableValues( + client, + params.teamId, + params.vercelProjectId + ); + sharedEnvVars = sharedResult.unwrapOr([]); + } + + // Process each environment mapping + for (const mapping of envMapping) { + const iterResult = await ResultAsync.fromPromise( + (async () => { + // Build filter to avoid decrypting vars that will be filtered out anyway + const excludeKeys = new Set(["TRIGGER_SECRET_KEY", "TRIGGER_VERSION"]); + const shouldIncludeKey = (key: string) => + !excludeKeys.has(key) && + shouldSyncEnvVar(params.syncEnvVarsMapping, key, mapping.triggerEnvType as TriggerEnvironmentType); + + const envVarsResult = await this.getVercelEnvironmentVariableValues( + client, + params.vercelProjectId, + params.teamId, + mapping.vercelTarget, + shouldIncludeKey + ); + + if (envVarsResult.isErr()) { + logger.error("pullEnvVarsFromVercel: Failed to get env vars", { + triggerEnvType: mapping.triggerEnvType, + vercelTarget: mapping.vercelTarget, + error: envVarsResult.error.message, + }); + errors.push(`Failed to get env vars for ${mapping.triggerEnvType}: ${envVarsResult.error.message}`); + return; + } + + const projectEnvVars = envVarsResult.value; + const standardTargets = ["production", "preview", "development"]; + const isCustomEnvironment = !standardTargets.includes(mapping.vercelTarget); + + const filteredSharedEnvVars = sharedEnvVars.filter((envVar) => { + const matchesTarget = envVar.target.includes(mapping.vercelTarget); + const matchesCustomEnv = isCustomEnvironment && envVar.applyToAllCustomEnvironments === true; + return matchesTarget || matchesCustomEnv; + }); + + const projectEnvVarKeys = new Set(projectEnvVars.map((v) => v.key)); + const sharedEnvVarsToAdd = filteredSharedEnvVars.filter((v) => !projectEnvVarKeys.has(v.key)); + const mergedEnvVars = [ + ...projectEnvVars, + ...sharedEnvVarsToAdd, + ]; + + if (mergedEnvVars.length === 0) { + return; + } + + const varsToSync = mergedEnvVars.filter((envVar) => { + if (envVar.isSecret) { + return false; + } + if (envVar.key === "TRIGGER_SECRET_KEY" || envVar.key === "TRIGGER_VERSION") { + return false; + } + return shouldSyncEnvVar( + params.syncEnvVarsMapping, + envVar.key, + mapping.triggerEnvType as TriggerEnvironmentType + ); + }); + + if (varsToSync.length === 0) { + return; + } + + const existingSecretKeys = new Set(); + const existingValues = new Map(); + + const existingVarValues = await prisma.environmentVariableValue.findMany({ + where: { + environmentId: mapping.runtimeEnvironmentId, + variable: { + projectId: params.projectId, + key: { + in: varsToSync.map((v) => v.key), + }, + }, + }, + select: { + isSecret: true, + valueReference: { + select: { + key: true, + }, + }, + variable: { + select: { + key: true, + }, + }, + }, + }); + + if (existingVarValues.length > 0) { + const secretStore = getSecretStore("DATABASE", { prismaClient: prisma }); + const SecretValue = z.object({ secret: z.string() }); + + for (const varValue of existingVarValues) { + if (varValue.isSecret) { + existingSecretKeys.add(varValue.variable.key); + } + + if (varValue.valueReference?.key) { + const existingSecret = await ResultAsync.fromPromise( + secretStore.getSecret(SecretValue, varValue.valueReference.key), + () => null + ).unwrapOr(null); + if (existingSecret) { + existingValues.set(varValue.variable.key, existingSecret.secret); + } + } + } + } + + const changedVars = varsToSync.filter((v) => { + const existingValue = existingValues.get(v.key); + return existingValue === undefined || existingValue !== v.value; + }); + + if (changedVars.length === 0) { + return; + } + + const secretVars = changedVars.filter((v) => existingSecretKeys.has(v.key)); + const nonSecretVars = changedVars.filter((v) => !existingSecretKeys.has(v.key)); + + if (nonSecretVars.length > 0) { + const result = await envVarRepository.create(params.projectId, { + override: true, + environmentIds: [mapping.runtimeEnvironmentId], + isSecret: false, + variables: nonSecretVars.map((v) => ({ + key: v.key, + value: v.value, + })), + lastUpdatedBy: { + type: "integration", + integration: "vercel", + }, + }); + + if (result.success) { + syncedCount += nonSecretVars.length; + } else { + const errorMsg = `Failed to sync env vars for ${mapping.triggerEnvType}: ${result.error}`; + errors.push(errorMsg); + logger.error(errorMsg, { + projectId: params.projectId, + vercelProjectId: params.vercelProjectId, + vercelTarget: mapping.vercelTarget, + error: result.error, + variableErrors: result.variableErrors, + attemptedKeys: nonSecretVars.map((v) => v.key), + }); + } + } + + if (secretVars.length > 0) { + const result = await envVarRepository.create(params.projectId, { + override: true, + environmentIds: [mapping.runtimeEnvironmentId], + isSecret: true, + variables: secretVars.map((v) => ({ + key: v.key, + value: v.value, + })), + lastUpdatedBy: { + type: "integration", + integration: "vercel", + }, + }); + + if (result.success) { + syncedCount += secretVars.length; + } else { + const errorMsg = `Failed to sync secret env vars for ${mapping.triggerEnvType}: ${result.error}`; + errors.push(errorMsg); + logger.error(errorMsg, { + projectId: params.projectId, + vercelProjectId: params.vercelProjectId, + vercelTarget: mapping.vercelTarget, + error: result.error, + variableErrors: result.variableErrors, + attemptedKeys: secretVars.map((v) => v.key), + }); + } + } + })(), + (error) => error + ); + + if (iterResult.isErr()) { + const errorMsg = `Failed to process env vars for ${mapping.triggerEnvType}: ${iterResult.error instanceof Error ? iterResult.error.message : "Unknown error"}`; + errors.push(errorMsg); + logger.error(errorMsg, { + projectId: params.projectId, + vercelProjectId: params.vercelProjectId, + vercelTarget: mapping.vercelTarget, + error: iterResult.error, + }); + } + } + + return { syncedCount, errors }; + })(), + (error) => { + logger.error("Failed to pull env vars from Vercel", { + projectId: params.projectId, + vercelProjectId: params.vercelProjectId, + error, + }); + return toVercelApiError(error); + } + ) + ); + } + + static async batchUpsertVercelEnvVars(params: { + client: Vercel; + vercelProjectId: string; + teamId: string | null; + envVars: Array<{ + key: string; + value: string; + target: string[]; + type: "sensitive" | "encrypted" | "plain"; + environmentType?: string; // For logging purposes + }>; + }): Promise<{ created: number; updated: number; errors: string[] }> { + const { client, vercelProjectId, teamId, envVars } = params; + const errors: string[] = []; + let created = 0; + let updated = 0; + + if (envVars.length === 0) { + return { created: 0, updated: 0, errors: [] }; + } + + const existingEnvs = await client.projects.filterProjectEnvs({ + idOrName: vercelProjectId, + ...(teamId && { teamId }), + }); + + const existingEnvsList = extractVercelEnvs(existingEnvs); + + const toCreate: Array<{ + key: string; + value: string; + target: string[]; + type: "sensitive" | "encrypted" | "plain"; + }> = []; + + const toUpdate: Array<{ + id: string; + key: string; + value: string; + target: string[]; + type: "sensitive" | "encrypted" | "plain"; + environmentType?: string; + }> = []; + + for (const envVar of envVars) { + const existingEnv = existingEnvsList.find((existing) => { + if (existing.key !== envVar.key) { + return false; + } + const envTargets = normalizeTarget(existing.target); + return ( + envVar.target.length === envTargets.length && + envVar.target.every((t) => envTargets.includes(t)) + ); + }); + + if (existingEnv && existingEnv.id) { + toUpdate.push({ + id: existingEnv.id, + key: envVar.key, + value: envVar.value, + target: envVar.target, + type: envVar.type, + environmentType: envVar.environmentType, + }); + } else { + toCreate.push({ + key: envVar.key, + value: envVar.value, + target: envVar.target, + type: envVar.type, + }); + } + } + + if (toCreate.length > 0) { + const createResult = await ResultAsync.fromPromise( + client.projects.createProjectEnv({ + idOrName: vercelProjectId, + ...(teamId && { teamId }), + requestBody: toCreate.map((item) => ({ + key: item.key, + value: item.value, + target: item.target as any, + type: item.type, + })) as any, + }), + (error) => error + ); + + if (createResult.isOk()) { + created = toCreate.length; + } else { + const errorMsg = `Failed to batch create env vars: ${createResult.error instanceof Error ? createResult.error.message : "Unknown error"}`; + errors.push(errorMsg); + logger.error(errorMsg, { + vercelProjectId, + teamId, + count: toCreate.length, + error: createResult.error, + }); + } + } + + // Update existing env vars (Vercel doesn't support batch updates) + for (const envVar of toUpdate) { + const updateResult = await ResultAsync.fromPromise( + client.projects.editProjectEnv({ + idOrName: vercelProjectId, + id: envVar.id, + ...(teamId && { teamId }), + requestBody: { + value: envVar.value, + target: envVar.target as any, + type: envVar.type, + }, + }), + (error) => error + ); + + if (updateResult.isOk()) { + updated++; + } else { + const errorMsg = `Failed to update ${envVar.environmentType || envVar.key} env var: ${updateResult.error instanceof Error ? updateResult.error.message : "Unknown error"}`; + errors.push(errorMsg); + logger.error(errorMsg, { + vercelProjectId, + teamId, + envVarId: envVar.id, + key: envVar.key, + error: updateResult.error, + }); + } + } + + return { created, updated, errors }; + } + + private static async upsertVercelEnvVar(params: { + client: Vercel; + vercelProjectId: string; + teamId: string | null; + key: string; + value: string; + target: string[]; + type: "sensitive" | "encrypted" | "plain"; + }): Promise { + const { client, vercelProjectId, teamId, key, value, target, type } = params; + + const existingEnvs = await client.projects.filterProjectEnvs({ + idOrName: vercelProjectId, + ...(teamId && { teamId }), + }); + + const envs = extractVercelEnvs(existingEnvs); + + // Vercel can have multiple env vars with the same key but different targets + const existingEnv = envs.find((existing) => { + if (existing.key !== key) { + return false; + } + const envTargets = normalizeTarget(existing.target); + return target.length === envTargets.length && target.every((t) => envTargets.includes(t)); + }); + + if (existingEnv && existingEnv.id) { + await client.projects.editProjectEnv({ + idOrName: vercelProjectId, + id: existingEnv.id, + ...(teamId && { teamId }), + requestBody: { + value, + target: target as any, + type, + }, + }); + } else { + await client.projects.createProjectEnv({ + idOrName: vercelProjectId, + ...(teamId && { teamId }), + requestBody: { + key, + value, + target: target as any, + type, + }, + }); + } + } + + static getAutoAssignCustomDomains( + client: Vercel, + vercelProjectId: string, + teamId?: string | null + ): ResultAsync { + // Vercel SDK lacks a getProject method — updateProject with empty body reads without modifying. + return wrapVercelCall( + client.projects.updateProject({ + idOrName: vercelProjectId, + ...(teamId && { teamId }), + requestBody: {}, + }), + "Failed to get Vercel project autoAssignCustomDomains", + { vercelProjectId, teamId } + ).map((project) => project.autoAssignCustomDomains ?? null); + } + + /** Disable autoAssignCustomDomains — required for atomic deployments. */ + static disableAutoAssignCustomDomains( + client: Vercel, + vercelProjectId: string, + teamId?: string | null + ): ResultAsync { + return wrapVercelCall( + client.projects.updateProject({ + idOrName: vercelProjectId, + ...(teamId && { teamId }), + requestBody: { + autoAssignCustomDomains: false, + }, + }), + "Failed to disable autoAssignCustomDomains", + { vercelProjectId, teamId } + ).map(() => undefined); + } + + static uninstallVercelIntegration( + integration: OrganizationIntegration & { tokenReference: SecretReference } + ): ResultAsync<{ authInvalid: boolean }, VercelApiError> { + return this.getVercelClient(integration).andThen((client) => + ResultAsync.fromPromise( + (async () => { + const secret = await getSecretStore(integration.tokenReference.provider).getSecret( + VercelSecretSchema, + integration.tokenReference.key + ); + + if (!secret?.installationId) { + throw new Error("Installation ID not found in Vercel integration"); + } + + return secret.installationId; + })(), + toVercelApiError + ).andThen((installationId) => + ResultAsync.fromPromise( + client.integrations.deleteConfiguration({ + id: installationId, + }), + (error) => error + ) + .map(() => ({ authInvalid: false })) + .orElse((error) => { + const isAuthError = isVercelAuthError(error); + logger.error("Failed to uninstall Vercel integration", { + installationId, + error: error instanceof Error ? error.message : "Unknown error", + isAuthError, + }); + // Auth errors (401/403): still clean up on our side, return flag for caller + if (isAuthError) { + return okAsync({ authInvalid: true }); + } + return errAsync(toVercelApiError(error)); + }) + ) + ); + } +} diff --git a/apps/webapp/app/presenters/v3/ApiKeysPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiKeysPresenter.server.ts index 1ff560586a9..78e64403e74 100644 --- a/apps/webapp/app/presenters/v3/ApiKeysPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiKeysPresenter.server.ts @@ -38,6 +38,11 @@ export class ApiKeysPresenter { apiKey: true, }, }, + project: { + select: { + id: true, + }, + }, }, where: { project: { @@ -64,11 +69,22 @@ export class ApiKeysPresenter { throw new Error("Environment not found"); } + const vercelIntegration = + await this.#prismaClient.organizationProjectIntegration.findFirst({ + where: { + projectId: environment.project.id, + deletedAt: null, + organizationIntegration: { service: "VERCEL", deletedAt: null }, + }, + select: { id: true }, + }); + return { environment: { ...environment, apiKey: environment?.parentEnvironment?.apiKey ?? environment?.apiKey, }, + hasVercelIntegration: vercelIntegration !== null, }; } } diff --git a/apps/webapp/app/presenters/v3/DeploymentListPresenter.server.ts b/apps/webapp/app/presenters/v3/DeploymentListPresenter.server.ts index 0b920e29421..1f5996f9967 100644 --- a/apps/webapp/app/presenters/v3/DeploymentListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/DeploymentListPresenter.server.ts @@ -1,5 +1,5 @@ import { - type Prisma, + Prisma, type WorkerDeploymentStatus, type WorkerInstanceGroupType, } from "@trigger.dev/database"; @@ -10,6 +10,7 @@ import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { type User } from "~/models/user.server"; import { processGitMetadata } from "./BranchesPresenter.server"; import { BranchTrackingConfigSchema, getTrackedBranchForEnvironment } from "~/v3/github"; +import { VercelProjectIntegrationDataSchema } from "~/v3/vercel/vercelProjectIntegrationSchema"; const pageSize = 20; @@ -105,6 +106,51 @@ export class DeploymentListPresenter { }, }); + // Check for Vercel integration before the main query so we can conditionally LEFT JOIN + let hasVercelIntegration = false; + let vercelTeamSlug: string | undefined; + let vercelProjectName: string | undefined; + + const vercelProjectIntegration = + await this.#prismaClient.organizationProjectIntegration.findFirst({ + where: { + projectId: project.id, + deletedAt: null, + organizationIntegration: { + service: "VERCEL", + deletedAt: null, + }, + }, + select: { + integrationData: true, + }, + }); + + if (vercelProjectIntegration) { + const parsed = VercelProjectIntegrationDataSchema.safeParse( + vercelProjectIntegration.integrationData + ); + + if (parsed.success && parsed.data.vercelTeamSlug) { + hasVercelIntegration = true; + vercelTeamSlug = parsed.data.vercelTeamSlug; + vercelProjectName = parsed.data.vercelProjectName; + } + } + + const vercelSelect = hasVercelIntegration + ? Prisma.sql`, id_dep."integrationDeploymentId"` + : Prisma.sql``; + const vercelJoin = hasVercelIntegration + ? Prisma.sql`LEFT JOIN LATERAL ( + SELECT id_inner."integrationDeploymentId" + FROM ${sqlDatabaseSchema}."IntegrationDeployment" as id_inner + WHERE id_inner."deploymentId" = wd."id" AND id_inner."integrationName" = 'vercel' + ORDER BY id_inner."createdAt" DESC + LIMIT 1 + ) id_dep ON true` + : Prisma.sql``; + const deployments = await this.#prismaClient.$queryRaw< { id: string; @@ -123,6 +169,7 @@ export class DeploymentListPresenter { userAvatarUrl: string | null; type: WorkerInstanceGroupType; git: Prisma.JsonValue | null; + integrationDeploymentId: string | null; }[] >` SELECT @@ -142,10 +189,12 @@ export class DeploymentListPresenter { wd."deployedAt", wd."type", wd."git" + ${vercelSelect} FROM ${sqlDatabaseSchema}."WorkerDeployment" as wd LEFT JOIN ${sqlDatabaseSchema}."User" as u ON wd."triggeredById" = u."id" +${vercelJoin} WHERE wd."projectId" = ${project.id} AND wd."environmentId" = ${environment.id} @@ -173,6 +222,7 @@ LIMIT ${pageSize} OFFSET ${pageSize * (page - 1)};`; return { currentPage: page, totalPages: Math.ceil(totalCount / pageSize), + hasVercelIntegration, connectedGithubRepository: project.connectedGithubRepository ?? undefined, environmentGitHubBranch, deployments: deployments.map((deployment, index) => { @@ -180,6 +230,12 @@ LIMIT ${pageSize} OFFSET ${pageSize * (page - 1)};`; (labeledDeployment) => labeledDeployment.deploymentId === deployment.id ); + let vercelDeploymentUrl: string | null = null; + if (hasVercelIntegration && deployment.integrationDeploymentId && vercelTeamSlug && vercelProjectName) { + const vercelId = deployment.integrationDeploymentId.replace(/^dpl_/, ""); + vercelDeploymentUrl = `https://vercel.com/${vercelTeamSlug}/${vercelProjectName}/${vercelId}`; + } + return { id: deployment.id, shortCode: deployment.shortCode, @@ -210,6 +266,7 @@ LIMIT ${pageSize} OFFSET ${pageSize * (page - 1)};`; } : undefined, git: processGitMetadata(deployment.git), + vercelDeploymentUrl, }; }), }; diff --git a/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts b/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts index 730591f4ebc..aff55263fec 100644 --- a/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts @@ -1,9 +1,14 @@ -import { flipCauseOption } from "effect/Cause"; import { PrismaClient, prisma } from "~/db.server"; import { Project } from "~/models/project.server"; import { User } from "~/models/user.server"; import { filterOrphanedEnvironments, sortEnvironments } from "~/utils/environmentSort"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; +import type { EnvironmentVariableUpdater } from "~/v3/environmentVariables/repository"; +import { + SyncEnvVarsMapping, + EnvSlug, +} from "~/v3/vercel/vercelProjectIntegrationSchema"; +import { VercelIntegrationService } from "~/services/vercelIntegration.server"; type Result = Awaited>; export type EnvironmentVariableWithSetValues = Result["environmentVariables"][number]; @@ -44,6 +49,9 @@ export class EnvironmentVariablesPresenter { select: { id: true, environmentId: true, + version: true, + lastUpdatedBy: true, + updatedAt: true, valueReference: { select: { key: true, @@ -67,6 +75,42 @@ export class EnvironmentVariablesPresenter { }, }); + const userIds = new Set( + environmentVariables + .flatMap((envVar) => envVar.values) + .map((value) => value.lastUpdatedBy) + .filter( + (lastUpdatedBy): lastUpdatedBy is { type: "user"; userId: string } => + lastUpdatedBy !== null && + typeof lastUpdatedBy === "object" && + "type" in lastUpdatedBy && + lastUpdatedBy.type === "user" && + "userId" in lastUpdatedBy && + typeof lastUpdatedBy.userId === "string" + ) + .map((lastUpdatedBy) => lastUpdatedBy.userId) + ); + + const users = + userIds.size > 0 + ? await this.#prismaClient.user.findMany({ + where: { + id: { + in: Array.from(userIds), + }, + }, + select: { + id: true, + name: true, + displayName: true, + avatarUrl: true, + }, + }) + : []; + + const usersRecord: Record = + Object.fromEntries(users.map((u) => [u.id, u])); + const environments = await this.#prismaClient.runtimeEnvironment.findMany({ select: { id: true, @@ -94,6 +138,18 @@ export class EnvironmentVariablesPresenter { const repository = new EnvironmentVariablesRepository(this.#prismaClient); const variables = await repository.getProject(project.id); + // Get Vercel integration data if it exists + const vercelService = new VercelIntegrationService(this.#prismaClient); + const vercelIntegration = await vercelService.getVercelProjectIntegration(project.id); + + let vercelSyncEnvVarsMapping: SyncEnvVarsMapping = {}; + let vercelPullEnvVarsBeforeBuild: EnvSlug[] | null = null; + + if (vercelIntegration) { + vercelSyncEnvVarsMapping = vercelIntegration.parsedIntegrationData.syncEnvVarsMapping; + vercelPullEnvVarsBeforeBuild = vercelIntegration.parsedIntegrationData.config.pullEnvVarsBeforeBuild ?? null; + } + return { environmentVariables: environmentVariables .flatMap((environmentVariable) => { @@ -101,13 +157,29 @@ export class EnvironmentVariablesPresenter { return sortedEnvironments.flatMap((env) => { const val = variable?.values.find((v) => v.environment.id === env.id); - const isSecret = - environmentVariable.values.find((v) => v.environmentId === env.id)?.isSecret ?? false; + const valueRecord = environmentVariable.values.find((v) => v.environmentId === env.id); + const isSecret = valueRecord?.isSecret ?? false; - if (!val) { + if (!val || !valueRecord) { return []; } + const lastUpdatedBy = valueRecord.lastUpdatedBy as EnvironmentVariableUpdater | null; + + const updatedByUser = + lastUpdatedBy?.type === "user" + ? (() => { + const user = usersRecord[lastUpdatedBy.userId]; + return user + ? { + id: user.id, + name: user.displayName || user.name || "Unknown", + avatarUrl: user.avatarUrl, + } + : null; + })() + : null; + return [ { id: environmentVariable.id, @@ -115,6 +187,10 @@ export class EnvironmentVariablesPresenter { environment: { type: env.type, id: env.id, branchName: env.branchName }, value: isSecret ? "" : val.value, isSecret, + version: valueRecord.version, + lastUpdatedBy, + updatedByUser, + updatedAt: valueRecord.updatedAt, }, ]; }); @@ -127,6 +203,14 @@ export class EnvironmentVariablesPresenter { branchName: environment.branchName, })), hasStaging: environments.some((environment) => environment.type === "STAGING"), + // Vercel integration data + vercelIntegration: vercelIntegration + ? { + enabled: true, + pullEnvVarsBeforeBuild: vercelPullEnvVarsBeforeBuild, + syncEnvVarsMapping: vercelSyncEnvVarsMapping, + } + : null, }; } } diff --git a/apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts b/apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts new file mode 100644 index 00000000000..d92fdbf7f7a --- /dev/null +++ b/apps/webapp/app/presenters/v3/VercelSettingsPresenter.server.ts @@ -0,0 +1,585 @@ +import { type PrismaClient } from "@trigger.dev/database"; +import { type Result, fromPromise, ok, okAsync, ResultAsync } from "neverthrow"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { OrgIntegrationRepository } from "~/models/orgIntegration.server"; +import { + VercelIntegrationRepository, + VercelCustomEnvironment, + VercelEnvironmentVariable, +} from "~/models/vercelIntegration.server"; +import { type GitHubAppInstallation } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github"; +import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; +import { + VercelProjectIntegrationDataSchema, + VercelProjectIntegrationData, +} from "~/v3/vercel/vercelProjectIntegrationSchema"; +import { BasePresenter } from "./basePresenter.server"; + +type VercelSettingsOptions = { + projectId: string; + organizationId: string; +}; + +export type VercelSettingsResult = { + enabled: boolean; + hasOrgIntegration: boolean; + authInvalid?: boolean; + connectedProject?: { + id: string; + vercelProjectId: string; + vercelProjectName: string; + vercelTeamId: string | null; + integrationData: VercelProjectIntegrationData; + createdAt: Date; + }; + isGitHubConnected: boolean; + hasStagingEnvironment: boolean; + hasPreviewEnvironment: boolean; + customEnvironments: VercelCustomEnvironment[]; + /** Whether autoAssignCustomDomains is enabled on the Vercel project. null if unknown. */ + autoAssignCustomDomains?: boolean | null; +}; + +export type VercelAvailableProject = { + id: string; + name: string; +}; + +export type VercelOnboardingData = { + customEnvironments: VercelCustomEnvironment[]; + environmentVariables: VercelEnvironmentVariable[]; + availableProjects: VercelAvailableProject[]; + hasProjectSelected: boolean; + authInvalid?: boolean; + existingVariables: Record; // Environment slugs (non-archived only) + gitHubAppInstallations: GitHubAppInstallation[]; + isGitHubConnected: boolean; + isOnboardingComplete: boolean; +}; + +export class VercelSettingsPresenter extends BasePresenter { + /** + * Get Vercel integration settings for the settings page + */ + public async call({ projectId, organizationId }: VercelSettingsOptions): Promise> { + const vercelIntegrationEnabled = OrgIntegrationRepository.isVercelSupported; + + if (!vercelIntegrationEnabled) { + return ok({ + enabled: false, + hasOrgIntegration: false, + authInvalid: false, + connectedProject: undefined, + isGitHubConnected: false, + hasStagingEnvironment: false, + hasPreviewEnvironment: false, + customEnvironments: [], + } as VercelSettingsResult); + } + + const orgIntegrationResult = await fromPromise( + (this._replica as PrismaClient).organizationIntegration.findFirst({ + where: { + organizationId, + service: "VERCEL", + deletedAt: null, + }, + include: { + tokenReference: true, + }, + }), + (error) => error + ); + + if (orgIntegrationResult.isErr()) { + logger.error("Unexpected error in VercelSettingsPresenter.call", { error: orgIntegrationResult.error }); + return ok({ + enabled: true, + hasOrgIntegration: false, + authInvalid: true, + connectedProject: undefined, + isGitHubConnected: false, + hasStagingEnvironment: false, + hasPreviewEnvironment: false, + customEnvironments: [], + } as VercelSettingsResult); + } + + const orgIntegration = orgIntegrationResult.value; + const hasOrgIntegration = orgIntegration !== null; + + if (hasOrgIntegration) { + const tokenResult = await VercelIntegrationRepository.validateVercelToken(orgIntegration); + if (tokenResult.isErr() || !tokenResult.value.isValid) { + return ok({ + enabled: true, + hasOrgIntegration: true, + authInvalid: true, + connectedProject: undefined, + isGitHubConnected: false, + hasStagingEnvironment: false, + hasPreviewEnvironment: false, + customEnvironments: [], + } as VercelSettingsResult); + } + } + + const checkOrgIntegration = () => fromPromise( + Promise.resolve(hasOrgIntegration), + (error) => ({ + type: "other" as const, + cause: error, + }) + ); + + const checkGitHubConnection = () => + fromPromise( + (this._replica as PrismaClient).connectedGithubRepository.findFirst({ + where: { + projectId, + repository: { + installation: { + deletedAt: null, + suspendedAt: null, + }, + }, + }, + select: { + id: true, + }, + }), + (error) => ({ + type: "other" as const, + cause: error, + }) + ).map((repo) => repo !== null); + + const checkStagingEnvironment = () => + fromPromise( + (this._replica as PrismaClient).runtimeEnvironment.findFirst({ + select: { + id: true, + }, + where: { + projectId, + type: "STAGING", + }, + }), + (error) => ({ + type: "other" as const, + cause: error, + }) + ).map((env) => env !== null); + + const checkPreviewEnvironment = () => + fromPromise( + (this._replica as PrismaClient).runtimeEnvironment.findFirst({ + select: { + id: true, + }, + where: { + projectId, + type: "PREVIEW", + }, + }), + (error) => ({ + type: "other" as const, + cause: error, + }) + ).map((env) => env !== null); + + const getVercelProjectIntegration = () => + fromPromise( + (this._replica as PrismaClient).organizationProjectIntegration.findFirst({ + where: { + projectId, + deletedAt: null, + organizationIntegration: { + service: "VERCEL", + deletedAt: null, + }, + }, + include: { + organizationIntegration: true, + }, + }), + (error) => ({ + type: "other" as const, + cause: error, + }) + ).map((integration) => { + if (!integration) { + return undefined; + } + + const parsedData = VercelProjectIntegrationDataSchema.safeParse( + integration.integrationData + ); + + if (!parsedData.success) { + return undefined; + } + + return { + id: integration.id, + vercelProjectId: integration.externalEntityId, + vercelProjectName: parsedData.data.vercelProjectName, + vercelTeamId: parsedData.data.vercelTeamId, + integrationData: parsedData.data, + createdAt: integration.createdAt, + }; + }); + + return ResultAsync.combine([ + checkOrgIntegration(), + checkGitHubConnection(), + checkStagingEnvironment(), + checkPreviewEnvironment(), + getVercelProjectIntegration(), + ]).andThen(([hasOrgIntegration, isGitHubConnected, hasStagingEnvironment, hasPreviewEnvironment, connectedProject]) => { + const fetchCustomEnvsAndProjectSettings = async (): Promise<{ + customEnvironments: VercelCustomEnvironment[]; + autoAssignCustomDomains: boolean | null; + }> => { + if (!connectedProject || !orgIntegration) { + return { customEnvironments: [], autoAssignCustomDomains: null }; + } + const clientResult = await VercelIntegrationRepository.getVercelClient(orgIntegration); + if (clientResult.isErr()) { + return { customEnvironments: [], autoAssignCustomDomains: null }; + } + const client = clientResult.value; + const teamId = await VercelIntegrationRepository.getTeamIdFromIntegration(orgIntegration); + const [customEnvsResult, autoAssignResult] = await Promise.all([ + VercelIntegrationRepository.getVercelCustomEnvironments( + client, + connectedProject.vercelProjectId, + teamId + ), + VercelIntegrationRepository.getAutoAssignCustomDomains( + client, + connectedProject.vercelProjectId, + teamId + ), + ]); + return { + customEnvironments: customEnvsResult.isOk() ? customEnvsResult.value : [], + autoAssignCustomDomains: autoAssignResult.isOk() ? autoAssignResult.value : null, + }; + }; + + return fromPromise( + fetchCustomEnvsAndProjectSettings(), + (error) => ({ type: "other" as const, cause: error }) + ).map(({ customEnvironments, autoAssignCustomDomains }) => ({ + enabled: true, + hasOrgIntegration, + authInvalid: false, + connectedProject, + isGitHubConnected, + hasStagingEnvironment, + hasPreviewEnvironment, + customEnvironments, + autoAssignCustomDomains, + } as VercelSettingsResult)); + }).mapErr((error) => { + // Log the error and return a safe fallback + logger.error("Error in VercelSettingsPresenter.call", { error }); + return error; + }); + } + + /** + * Get data needed for the onboarding modal (custom environments and env vars) + */ + public async getOnboardingData( + projectId: string, + organizationId: string, + vercelEnvironmentId?: string + ): Promise { + const result = await ResultAsync.fromPromise( + (async (): Promise => { + const [gitHubInstallations, connectedGitHubRepo] = await Promise.all([ + (this._replica as PrismaClient).githubAppInstallation.findMany({ + where: { + organizationId, + deletedAt: null, + suspendedAt: null, + }, + select: { + id: true, + accountHandle: true, + targetType: true, + appInstallationId: true, + repositories: { + select: { + id: true, + name: true, + fullName: true, + htmlUrl: true, + private: true, + }, + take: 200, + }, + }, + take: 20, + orderBy: { + createdAt: "desc", + }, + }), + (this._replica as PrismaClient).connectedGithubRepository.findFirst({ + where: { + projectId, + repository: { + installation: { + deletedAt: null, + suspendedAt: null, + }, + }, + }, + select: { + id: true, + }, + }), + ]); + + const isGitHubConnected = connectedGitHubRepo !== null; + const gitHubAppInstallations: GitHubAppInstallation[] = gitHubInstallations.map((installation) => ({ + id: installation.id, + appInstallationId: installation.appInstallationId, + targetType: installation.targetType, + accountHandle: installation.accountHandle, + repositories: installation.repositories.map((repo) => ({ + id: repo.id, + name: repo.name, + fullName: repo.fullName, + private: repo.private, + htmlUrl: repo.htmlUrl, + })), + })); + + const orgIntegration = await (this._replica as PrismaClient).organizationIntegration.findFirst({ + where: { + organizationId, + service: "VERCEL", + deletedAt: null, + }, + include: { + tokenReference: true, + }, + }); + + if (!orgIntegration) { + return null; + } + + const tokenResult = await VercelIntegrationRepository.validateVercelToken(orgIntegration); + if (tokenResult.isErr() || !tokenResult.value.isValid) { + return { + customEnvironments: [], + environmentVariables: [], + availableProjects: [], + hasProjectSelected: false, + authInvalid: true, + existingVariables: {}, + gitHubAppInstallations, + isGitHubConnected, + isOnboardingComplete: false, + }; + } + + const clientResult = await VercelIntegrationRepository.getVercelClient(orgIntegration); + if (clientResult.isErr()) { + return { + customEnvironments: [], + environmentVariables: [], + availableProjects: [], + hasProjectSelected: false, + authInvalid: clientResult.error.authInvalid, + existingVariables: {}, + gitHubAppInstallations, + isGitHubConnected, + isOnboardingComplete: false, + }; + } + const client = clientResult.value; + const teamId = await VercelIntegrationRepository.getTeamIdFromIntegration(orgIntegration); + + const projectIntegration = await (this._replica as PrismaClient).organizationProjectIntegration.findFirst({ + where: { + projectId, + deletedAt: null, + organizationIntegration: { + service: "VERCEL", + deletedAt: null, + }, + }, + }); + + const availableProjectsResult = await VercelIntegrationRepository.getVercelProjects(client, teamId); + + if (availableProjectsResult.isErr()) { + return { + customEnvironments: [], + environmentVariables: [], + availableProjects: [], + hasProjectSelected: false, + authInvalid: availableProjectsResult.error.authInvalid, + existingVariables: {}, + gitHubAppInstallations, + isGitHubConnected, + isOnboardingComplete: false, + }; + } + + if (!projectIntegration) { + return { + customEnvironments: [], + environmentVariables: [], + availableProjects: availableProjectsResult.value, + hasProjectSelected: false, + existingVariables: {}, + gitHubAppInstallations, + isGitHubConnected, + isOnboardingComplete: false, + }; + } + + const [customEnvironmentsResult, projectEnvVarsResult, sharedEnvVarsResult] = await Promise.all([ + VercelIntegrationRepository.getVercelCustomEnvironments( + client, + projectIntegration.externalEntityId, + teamId + ), + VercelIntegrationRepository.getVercelEnvironmentVariables( + client, + projectIntegration.externalEntityId, + teamId + ), + // Only fetch shared env vars if teamId is available + teamId + ? VercelIntegrationRepository.getVercelSharedEnvironmentVariables( + client, + teamId, + projectIntegration.externalEntityId + ) + : okAsync([] as Array<{ id: string; key: string; type: string; isSecret: boolean; target: string[] }>), + ]); + const authInvalid = + (customEnvironmentsResult.isErr() && customEnvironmentsResult.error.authInvalid) || + (projectEnvVarsResult.isErr() && projectEnvVarsResult.error.authInvalid) || + (sharedEnvVarsResult.isErr() && sharedEnvVarsResult.error.authInvalid); + + if (authInvalid) { + return { + customEnvironments: [], + environmentVariables: [], + availableProjects: availableProjectsResult.value, + hasProjectSelected: true, + authInvalid: true, + existingVariables: {}, + gitHubAppInstallations, + isGitHubConnected, + isOnboardingComplete: false, + }; + } + + const customEnvironments = customEnvironmentsResult.isOk() ? customEnvironmentsResult.value : []; + const projectEnvVars = projectEnvVarsResult.isOk() ? projectEnvVarsResult.value : []; + const sharedEnvVars = sharedEnvVarsResult.isOk() ? sharedEnvVarsResult.value : []; + + // Filter out TRIGGER_SECRET_KEY and TRIGGER_VERSION (managed by Trigger.dev) and merge project + shared env vars + const excludedKeys = new Set(["TRIGGER_SECRET_KEY", "TRIGGER_VERSION"]); + const projectEnvVarKeys = new Set(projectEnvVars.map((v) => v.key)); + const mergedEnvVars: VercelEnvironmentVariable[] = [ + ...projectEnvVars + .filter((v) => !excludedKeys.has(v.key)) + .map((v) => { + const envVar = { ...v }; + if (vercelEnvironmentId && (v as any).customEnvironmentIds?.includes(vercelEnvironmentId)) { + envVar.target = [...v.target, 'staging']; + } + return envVar; + }), + ...sharedEnvVars + .filter((v) => !projectEnvVarKeys.has(v.key) && !excludedKeys.has(v.key)) + .map((v) => { + const envVar = { + id: v.id, + key: v.key, + type: v.type as VercelEnvironmentVariable["type"], + isSecret: v.isSecret, + target: v.target, + isShared: true, + customEnvironmentIds: [] as string[], + }; + if (vercelEnvironmentId && (v as any).customEnvironmentIds?.includes(vercelEnvironmentId)) { + envVar.target = [...v.target, 'staging']; + } + return envVar; + }), + ]; + + const sortedEnvVars = [...mergedEnvVars].sort((a, b) => + a.key.localeCompare(b.key) + ); + + const projectEnvs = await (this._replica as PrismaClient).runtimeEnvironment.findMany({ + where: { + projectId, + archivedAt: null, // Filter out archived environments + }, + select: { + id: true, + slug: true, + type: true, + }, + }); + const envIdToSlug = new Map(projectEnvs.map((e) => [e.id, e.slug])); + const activeEnvIds = new Set(projectEnvs.map((e) => e.id)); + + const envVarRepository = new EnvironmentVariablesRepository(this._replica as PrismaClient); + const existingVariables = await envVarRepository.getProject(projectId); + const existingVariablesRecord: Record = {}; + for (const v of existingVariables) { + // Filter out archived environments and map to slugs + const activeEnvSlugs = v.values + .filter((val) => activeEnvIds.has(val.environment.id)) + .map((val) => envIdToSlug.get(val.environment.id) || val.environment.type.toLowerCase()); + if (activeEnvSlugs.length > 0) { + existingVariablesRecord[v.key] = { + environments: activeEnvSlugs, + }; + } + } + + const parsedIntegrationData = VercelProjectIntegrationDataSchema.safeParse( + projectIntegration.integrationData + ); + + return { + customEnvironments, + environmentVariables: sortedEnvVars, + availableProjects: availableProjectsResult.value, + hasProjectSelected: true, + existingVariables: existingVariablesRecord, + gitHubAppInstallations, + isGitHubConnected, + isOnboardingComplete: parsedIntegrationData.success + ? (parsedIntegrationData.data.onboardingCompleted ?? false) + : false, + }; + })(), + (error) => error + ); + + if (result.isErr()) { + logger.error("Error in getOnboardingData", { error: result.error }); + return null; + } + + return result.value; + } + +} \ No newline at end of file diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.apikeys/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.apikeys/route.tsx index acbf29c4f3f..897687f4ec9 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.apikeys/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.apikeys/route.tsx @@ -51,7 +51,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { try { const presenter = new ApiKeysPresenter(); - const { environment } = await presenter.call({ + const { environment, hasVercelIntegration } = await presenter.call({ userId, projectSlug: projectParam, environmentSlug: envParam, @@ -59,6 +59,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { return typedjson({ environment, + hasVercelIntegration, }); } catch (error) { console.error(error); @@ -70,7 +71,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }; export default function Page() { - const { environment } = useTypedLoaderData(); + const { environment, hasVercelIntegration } = useTypedLoaderData(); const organization = useOrganization(); if (!environment) { @@ -132,6 +133,8 @@ export default function Page() {
(); const hasDeployments = totalPages > 0; @@ -234,6 +237,7 @@ export default function Page() { Deployed at Deployed by Git + {hasVercelIntegration && Linked} Go to page @@ -307,6 +311,28 @@ export default function Page() {

+ {hasVercelIntegration && ( + + {deployment.vercelDeploymentUrl ? ( + e.stopPropagation()} + > + + + } + content="View on Vercel" + /> + ) : ( + "–" + )} + + )} + No deploys match your filters diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx index c52942a8acb..86bd5bbc95d 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx @@ -151,7 +151,13 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { } const repository = new EnvironmentVariablesRepository(prisma); - const result = await repository.create(project.id, submission.value); + const result = await repository.create(project.id, { + ...submission.value, + lastUpdatedBy: { + type: "user", + userId, + }, + }); if (!result.success) { if (result.variableErrors) { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx index 80976d41fcc..2670f0188df 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx @@ -9,7 +9,7 @@ import { PlusIcon, TrashIcon, } from "@heroicons/react/20/solid"; -import { Form, type MetaFunction, Outlet, useActionData, useFetcher, useNavigation } from "@remix-run/react"; +import { Form, type MetaFunction, Outlet, useActionData, useFetcher, useNavigation, useRevalidator } from "@remix-run/react"; import { type ActionFunctionArgs, type LoaderFunctionArgs, @@ -19,10 +19,12 @@ import { useEffect, useMemo, useState } from "react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { EnvironmentCombo } from "~/components/environments/EnvironmentLabel"; +import { VercelLogo } from "~/components/integrations/VercelLogo"; import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { ClipboardField } from "~/components/primitives/ClipboardField"; import { CopyableText } from "~/components/primitives/CopyableText"; +import { DateTime } from "~/components/primitives/DateTime"; import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; import { Fieldset } from "~/components/primitives/Fieldset"; import { FormButtons } from "~/components/primitives/FormButtons"; @@ -70,6 +72,11 @@ import { EditEnvironmentVariableValue, EnvironmentVariable, } from "~/v3/environmentVariables/repository"; +import { UserAvatar } from "~/components/UserProfilePhoto"; +import { VercelIntegrationService } from "~/services/vercelIntegration.server"; +import { fromPromise } from "neverthrow"; +import { logger } from "~/services/logger.server"; +import { shouldSyncEnvVar, isPullEnvVarsEnabledForEnvironment, type TriggerEnvironmentType } from "~/v3/vercel/vercelProjectIntegrationSchema"; export const meta: MetaFunction = () => { return [ @@ -85,7 +92,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { try { const presenter = new EnvironmentVariablesPresenter(); - const { environmentVariables, environments, hasStaging } = await presenter.call({ + const { environmentVariables, environments, hasStaging, vercelIntegration } = await presenter.call({ userId, projectSlug: projectParam, }); @@ -94,6 +101,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { environmentVariables, environments, hasStaging, + vercelIntegration, }); } catch (error) { console.error(error); @@ -111,6 +119,12 @@ const schema = z.discriminatedUnion("action", [ key: z.string(), ...DeleteEnvironmentVariableValue.shape, }), + z.object({ + action: z.literal("update-vercel-sync"), + key: z.string(), + environmentType: z.enum(["PRODUCTION", "STAGING", "PREVIEW", "DEVELOPMENT"]), + syncEnabled: z.union([z.literal("true"), z.literal("false")]).transform((val) => val === "true"), + }), ]); export const action = async ({ request, params }: ActionFunctionArgs) => { @@ -151,7 +165,13 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { switch (submission.value.action) { case "edit": { const repository = new EnvironmentVariablesRepository(prisma); - const result = await repository.editValue(project.id, submission.value); + const result = await repository.editValue(project.id, { + ...submission.value, + lastUpdatedBy: { + type: "user", + userId, + }, + }); if (!result.success) { submission.error.key = [result.error]; @@ -169,6 +189,32 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return json(submission); } + // Clean up syncEnvVarsMapping if Vercel integration exists (best-effort) + const { environmentId, key } = submission.value; + const vercelService = new VercelIntegrationService(); + await fromPromise( + (async () => { + const integration = await vercelService.getVercelProjectIntegration(project.id); + if (integration) { + const runtimeEnv = await prisma.runtimeEnvironment.findUnique({ + where: { id: environmentId }, + select: { type: true }, + }); + if (runtimeEnv) { + await vercelService.removeSyncEnvVarForEnvironment( + project.id, + key, + runtimeEnv.type as TriggerEnvironmentType + ); + } + } + })(), + (error) => error + ).mapErr((error) => { + logger.error("Failed to remove Vercel sync mapping", { error }); + return error; + }); + return redirectWithSuccessMessage( v3EnvironmentVariablesPath( { slug: organizationSlug }, @@ -179,12 +225,31 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { `Deleted ${submission.value.key} environment variable` ); } + case "update-vercel-sync": { + const vercelService = new VercelIntegrationService(); + const integration = await vercelService.getVercelProjectIntegration(project.id); + + if (!integration) { + submission.error.key = ["Vercel integration not found"]; + return json(submission); + } + + // Update the sync mapping for the specific env var and environment + await vercelService.updateSyncEnvVarForEnvironment( + project.id, + submission.value.key, + submission.value.environmentType, + submission.value.syncEnabled + ); + + return json({ success: true }); + } } }; export default function Page() { const [revealAll, setRevealAll] = useState(false); - const { environmentVariables, environments } = useTypedLoaderData(); + const { environmentVariables, environments, vercelIntegration } = useTypedLoaderData(); const organization = useOrganization(); const project = useProject(); const environment = useEnvironment(); @@ -279,10 +344,32 @@ export default function Page() { - Key - Value - Environment - + + Key + + + Value + + + Environment + + {vercelIntegration?.enabled && ( + + + Sync + + + } + content="When enabled, this variable will be pulled from Vercel during builds. Requires 'Pull env vars before build' to be enabled in settings." + /> + + )} + + Updated + + Actions @@ -341,9 +428,54 @@ export default function Page() { + {vercelIntegration?.enabled && ( + + {variable.environment.type !== "DEVELOPMENT" && ( + + )} + + )} + +
+ {variable.updatedByUser ? ( +
+ + {variable.updatedByUser.name} +
+ ) : (variable.lastUpdatedBy?.type === "integration" && variable.lastUpdatedBy?.integration === 'vercel' ) ? ( +
+ + + {variable.lastUpdatedBy.integration} + +
+ ) : null} + {variable.updatedAt ? ( + + + + ) : null} +
+ - + {environmentVariables.length === 0 ? (
You haven't set any environment variables yet. @@ -430,7 +562,7 @@ function EditEnvironmentVariablePanel({ @@ -526,8 +658,82 @@ function DeleteEnvironmentVariableButton({ leadingIconClassName="text-rose-500 group-hover/button:text-text-bright transition-colors" className="ml-0.5 transition-colors group-hover/button:bg-error" > - {isLoading ? "Deleting" : "Delete"} + {isLoading ? "Deleting" : ""} ); } + +/** + * Toggle component for controlling whether an environment variable is pulled from Vercel. + * + * When enabled, the variable will be pulled from Vercel during builds. + * By default, all variables are pulled unless explicitly disabled. + * + * Note: If the env slug is missing from syncEnvVarsMapping, all vars are pulled by default. + * Only when syncEnvVarsMapping[envSlug][envVarName] = false, the env var is skipped during builds. + */ +function VercelSyncCheckbox({ + envVarKey, + environmentType, + syncEnabled, + pullEnvVarsEnabledForEnv, +}: { + envVarKey: string; + environmentType: "PRODUCTION" | "STAGING" | "PREVIEW" | "DEVELOPMENT"; + syncEnabled: boolean; + pullEnvVarsEnabledForEnv: boolean; +}) { + const fetcher = useFetcher(); + const revalidator = useRevalidator(); + + const isLoading = fetcher.state !== "idle"; + + // Revalidate loader data after successful submission (without full page reload) + useEffect(() => { + if (fetcher.state === "idle" && fetcher.data) { + const data = fetcher.data as { success?: boolean }; + if (data.success) { + revalidator.revalidate(); + } + } + }, [fetcher.state, fetcher.data, revalidator]); + + const handleChange = (checked: boolean) => { + fetcher.submit( + { + action: "update-vercel-sync", + key: envVarKey, + environmentType, + syncEnabled: checked.toString(), + }, + { method: "post" } + ); + }; + + // If pull env vars is disabled for this environment, show disabled state + if (!pullEnvVarsEnabledForEnv) { + return ( + {}} + /> + } + content="Enable 'Pull env vars before build' for this environment in Vercel settings." + /> + ); + } + + return ( + + ); +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.settings/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.settings/route.tsx index 66ea64cb36f..a5a70c39af6 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.settings/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.settings/route.tsx @@ -38,12 +38,20 @@ import { import { ProjectSettingsService } from "~/services/projectSettings.server"; import { logger } from "~/services/logger.server"; import { requireUserId } from "~/services/session.server"; -import { organizationPath, v3ProjectPath, EnvironmentParamSchema, v3BillingPath } from "~/utils/pathBuilder"; -import React, { useEffect, useState } from "react"; +import { organizationPath, v3ProjectPath, EnvironmentParamSchema, v3BillingPath, vercelResourcePath } from "~/utils/pathBuilder"; +import React, { useEffect, useState, useCallback, useRef } from "react"; +import { useSearchParams } from "@remix-run/react"; import { useEnvironment } from "~/hooks/useEnvironment"; import { ProjectSettingsPresenter } from "~/services/projectSettingsPresenter.server"; import { type BuildSettings } from "~/v3/buildSettings"; import { GitHubSettingsPanel } from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.github"; +import { + VercelSettingsPanel, + VercelOnboardingModal, +} from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.vercel"; +import type { loader as vercelLoader } from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.vercel"; +import { OrgIntegrationRepository } from "~/models/orgIntegration.server"; +import { useTypedFetcher } from "remix-typedjson"; export const meta: MetaFunction = () => { return [ @@ -92,6 +100,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { return typedjson({ githubAppEnabled: gitHubApp.enabled, buildSettings, + vercelIntegrationEnabled: OrgIntegrationRepository.isVercelSupported, }); }; @@ -290,12 +299,121 @@ export const action: ActionFunction = async ({ request, params }) => { }; export default function Page() { - const { githubAppEnabled, buildSettings } = useTypedLoaderData(); + const { githubAppEnabled, buildSettings, vercelIntegrationEnabled } = + useTypedLoaderData(); const project = useProject(); const organization = useOrganization(); const environment = useEnvironment(); const lastSubmission = useActionData(); const navigation = useNavigation(); + const [searchParams, setSearchParams] = useSearchParams(); + + // Vercel onboarding modal state + const hasQueryParam = searchParams.get("vercelOnboarding") === "true"; + const nextUrl = searchParams.get("next"); + const [isModalOpen, setIsModalOpen] = useState(false); + const vercelFetcher = useTypedFetcher(); + + // Helper to open modal and ensure query param is present + const openVercelOnboarding = useCallback(() => { + setIsModalOpen(true); + // Ensure query param is present to maintain state during form submissions + if (!hasQueryParam) { + setSearchParams((prev) => { + prev.set("vercelOnboarding", "true"); + return prev; + }); + } + }, [hasQueryParam, setSearchParams]); + + const closeVercelOnboarding = useCallback(() => { + // Remove query param if present + if (hasQueryParam) { + setSearchParams((prev) => { + prev.delete("vercelOnboarding"); + return prev; + }); + } + // Close modal + setIsModalOpen(false); + }, [hasQueryParam, setSearchParams]); + + // When query param is present, handle modal opening + // Note: We don't close the modal based on data state during onboarding - only when explicitly closed + useEffect(() => { + if (hasQueryParam && vercelIntegrationEnabled) { + // Ensure query param is present and modal is open + if (vercelFetcher.data?.onboardingData && vercelFetcher.state === "idle") { + // Data is loaded, ensure modal is open (query param takes precedence) + if (!isModalOpen) { + openVercelOnboarding(); + } + } else if (vercelFetcher.state === "idle" && vercelFetcher.data === undefined) { + // Load onboarding data + vercelFetcher.load( + `${vercelResourcePath(organization.slug, project.slug, environment.slug)}?vercelOnboarding=true` + ); + } + } else if (!hasQueryParam && isModalOpen) { + // Query param removed but modal is open, close modal + setIsModalOpen(false); + } + }, [hasQueryParam, vercelIntegrationEnabled, organization.slug, project.slug, environment.slug, vercelFetcher.data, vercelFetcher.state, isModalOpen, openVercelOnboarding]); + + // Ensure modal stays open when query param is present (even after data reloads) + // This is a safeguard to prevent the modal from closing during form submissions + useEffect(() => { + if (hasQueryParam && !isModalOpen) { + // Query param is present but modal is closed, open it + // This ensures the modal stays open during the onboarding flow + openVercelOnboarding(); + } + }, [hasQueryParam, isModalOpen, openVercelOnboarding]); + + // When data finishes loading (from query param), ensure modal is open + useEffect(() => { + if (hasQueryParam && vercelFetcher.data?.onboardingData && vercelFetcher.state === "idle") { + // Data loaded and query param is present, ensure modal is open + if (!isModalOpen) { + openVercelOnboarding(); + } + } + }, [hasQueryParam, vercelFetcher.data, vercelFetcher.state, isModalOpen, openVercelOnboarding]); + + + // Track if we're waiting for data from button click (not query param) + const waitingForButtonClickRef = useRef(false); + + // Handle opening modal from button click (without query param) + const handleOpenVercelModal = useCallback(() => { + // Add query param to maintain state during form submissions + if (!hasQueryParam) { + setSearchParams((prev) => { + prev.set("vercelOnboarding", "true"); + return prev; + }); + } + + if (vercelFetcher.data && vercelFetcher.data.onboardingData) { + // Data already loaded, open modal immediately + openVercelOnboarding(); + } else { + // Need to load data first, mark that we're waiting for button click + waitingForButtonClickRef.current = true; + vercelFetcher.load( + `${vercelResourcePath(organization.slug, project.slug, environment.slug)}?vercelOnboarding=true` + ); + } + }, [organization.slug, project.slug, environment.slug, vercelFetcher, setSearchParams, hasQueryParam, openVercelOnboarding]); + + // When data loads from button click, open modal + useEffect(() => { + if (waitingForButtonClickRef.current && vercelFetcher.data?.onboardingData && vercelFetcher.state === "idle") { + // Data loaded from button click, open modal and ensure query param is present + waitingForButtonClickRef.current = false; + openVercelOnboarding(); + } + }, [vercelFetcher.data, vercelFetcher.state, openVercelOnboarding]); const [hasRenameFormChanges, setHasRenameFormChanges] = useState(false); @@ -425,6 +543,21 @@ export default function Page() {
+ {vercelIntegrationEnabled && ( +
+ Vercel integration +
+ +
+
+ )} +
Build settings
@@ -477,6 +610,29 @@ export default function Page() {
+ + {/* Vercel Onboarding Modal */} + {vercelIntegrationEnabled && ( + { + vercelFetcher.load( + `${vercelResourcePath(organization.slug, project.slug, environment.slug)}?vercelOnboarding=true${ + vercelEnvironmentId ? `&vercelEnvironmentId=${vercelEnvironmentId}` : "" + }` + ); + }} + /> + )} ); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.integrations.vercel.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.integrations.vercel.tsx new file mode 100644 index 00000000000..10b3f2283ce --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.integrations.vercel.tsx @@ -0,0 +1,375 @@ +import type { + ActionFunctionArgs, + LoaderFunctionArgs, +} from "@remix-run/node"; +import { json, redirect } from "@remix-run/node"; +import { fromPromise } from "neverthrow"; +import { Form, useActionData, useNavigation } from "@remix-run/react"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { DialogClose } from "@radix-ui/react-dialog"; +import { Button } from "~/components/primitives/Buttons"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "~/components/primitives/Dialog"; +import { FormButtons } from "~/components/primitives/FormButtons"; +import { Header1 } from "~/components/primitives/Headers"; +import { PageBody, PageContainer } from "~/components/layout/AppLayout"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { Table, TableBody, TableCell, TableHeader, TableHeaderCell, TableRow } from "~/components/primitives/Table"; +import { VercelIntegrationRepository } from "~/models/vercelIntegration.server"; +import { $transaction, prisma } from "~/db.server"; +import { requireOrganization } from "~/services/org.server"; +import { OrganizationParamsSchema } from "~/utils/pathBuilder"; +import { logger } from "~/services/logger.server"; +import { TrashIcon } from "@heroicons/react/20/solid"; +import { v3ProjectSettingsPath } from "~/utils/pathBuilder"; +import { LinkButton } from "~/components/primitives/Buttons"; + +function formatDate(date: Date): string { + return new Intl.DateTimeFormat("en-US", { + month: "short", + day: "numeric", + year: "numeric", + hour: "numeric", + minute: "2-digit", + second: "2-digit", + hour12: true, + }).format(date); +} + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const { organizationSlug } = OrganizationParamsSchema.parse(params); + const url = new URL(request.url); + const configurationId = url.searchParams.get("configurationId") ?? undefined; + const { organization } = await requireOrganization(request, organizationSlug); + + // Find Vercel integration for this organization + let vercelIntegration = await prisma.organizationIntegration.findFirst({ + where: { + organizationId: organization.id, + service: "VERCEL", + deletedAt: null, + // If configurationId is provided, filter by it in integrationData + ...(configurationId && { + integrationData: { + path: ["installationId"], + equals: configurationId, + }, + }), + }, + include: { + tokenReference: true, + }, + }); + + if (!vercelIntegration) { + return typedjson({ + organization, + vercelIntegration: null, + connectedProjects: [], + teamId: null, + installationId: null, + }); + } + + // Get team ID from integrationData + const integrationData = vercelIntegration.integrationData as any; + const teamId = integrationData?.teamId ?? null; + const installationId = integrationData?.installationId ?? null; + + // Get all connected projects for this integration + const connectedProjects = await prisma.organizationProjectIntegration.findMany({ + where: { + organizationIntegrationId: vercelIntegration.id, + deletedAt: null, + }, + include: { + project: { + select: { + id: true, + slug: true, + name: true, + }, + }, + }, + orderBy: { + createdAt: "desc", + }, + }); + + return typedjson({ + organization, + vercelIntegration, + connectedProjects, + teamId, + installationId, + }); +}; + +const ActionSchema = z.object({ + intent: z.literal("uninstall"), +}); + +export const action = async ({ request, params }: ActionFunctionArgs) => { + const { organizationSlug } = OrganizationParamsSchema.parse(params); + const { organization, userId } = await requireOrganization(request, organizationSlug); + + const formData = await request.formData(); + const result = ActionSchema.safeParse({ intent: formData.get("intent") }); + if (!result.success) { + return json({ error: "Invalid action" }, { status: 400 }); + } + + // Find Vercel integration + const vercelIntegration = await prisma.organizationIntegration.findFirst({ + where: { + organizationId: organization.id, + service: "VERCEL", + deletedAt: null, + }, + include: { + tokenReference: true, + }, + }); + + if (!vercelIntegration) { + return json({ error: "Vercel integration not found" }, { status: 404 }); + } + + // Uninstall from Vercel side + const uninstallResult = await VercelIntegrationRepository.uninstallVercelIntegration(vercelIntegration); + + if (uninstallResult.isErr()) { + logger.error("Failed to uninstall Vercel integration", { + organizationId: organization.id, + organizationSlug, + userId, + integrationId: vercelIntegration.id, + error: uninstallResult.error.message, + }); + + return json( + { error: "Failed to uninstall Vercel integration. Please try again." }, + { status: 500 } + ); + } + + // Soft-delete the integration and all connected projects in a transaction + const txResult = await fromPromise( + $transaction(prisma, async (tx) => { + await tx.organizationProjectIntegration.updateMany({ + where: { + organizationIntegrationId: vercelIntegration.id, + deletedAt: null, + }, + data: { deletedAt: new Date() }, + }); + + await tx.organizationIntegration.update({ + where: { id: vercelIntegration.id }, + data: { deletedAt: new Date() }, + }); + }), + (error) => error + ); + + if (txResult.isErr()) { + logger.error("Failed to soft-delete Vercel integration records", { + organizationId: organization.id, + organizationSlug, + userId, + integrationId: vercelIntegration.id, + error: txResult.error instanceof Error ? txResult.error.message : String(txResult.error), + }); + + return json( + { error: "Failed to uninstall Vercel integration. Please try again." }, + { status: 500 } + ); + } + + if (uninstallResult.value.authInvalid) { + logger.warn("Vercel integration uninstalled with auth error - token invalid", { + organizationId: organization.id, + organizationSlug, + userId, + integrationId: vercelIntegration.id, + }); + } else { + logger.info("Vercel integration uninstalled successfully", { + organizationId: organization.id, + organizationSlug, + userId, + integrationId: vercelIntegration.id, + }); + } + + // Redirect back to organization settings + return redirect(`/orgs/${organizationSlug}/settings`); +}; + +export default function VercelIntegrationPage() { + const { organization, vercelIntegration, connectedProjects, teamId, installationId } = + useTypedLoaderData(); + const actionData = useActionData(); + const navigation = useNavigation(); + const isUninstalling = navigation.state === "submitting" && + navigation.formData?.get("intent") === "uninstall"; + + if (!vercelIntegration) { + return ( + + +
+ No Vercel Integration Found + + This organization doesn't have a Vercel integration configured. + +
+ + + ); + } + + return ( + + +
+ Vercel Integration + + Manage your organization's Vercel integration and connected projects. + +
+ + {/* Integration Info Section */} +
+
+
+
Integration Details
+
+ {teamId && ( +
+ Vercel Team ID: {teamId} +
+ )} + {installationId && ( +
+ Installation ID: {installationId} +
+ )} +
+ Installed:{" "} + {formatDate(new Date(vercelIntegration.createdAt))} +
+
+
+
+ + + + + + + Remove Vercel Integration + + + This will permanently remove the Vercel integration and disconnect all projects. + This action cannot be undone. + + + + + + } + cancelButton={ + + + + } + /> + + + {actionData?.error && ( + + {actionData.error} + + )} +
+
+
+ + {/* Connected Projects Section */} +
+
+ Connected Projects ({connectedProjects.length}) +
+ + {connectedProjects.length === 0 ? ( +
+ + No projects are currently connected to this Vercel integration. + +
+ ) : ( +

+ + + Project Name + Vercel Project ID + Connected + Actions + + + + {connectedProjects.map((projectIntegration) => ( + + {projectIntegration.project.name} + + {projectIntegration.externalEntityId} + + + {formatDate(new Date(projectIntegration.createdAt))} + + + + Configure + + + + ))} + +
+ )} +

{title}

+ {visiblePart} + {fadingPart && ( + + {fadingPart} + + )} +

{title}

{title}

Integration Details

+ Connected Projects ({connectedProjects.length}) +