From 2a8cde6ed563e0e869ef3d83372a3a0e6223eac0 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 4 Mar 2024 18:09:52 +0000 Subject: [PATCH 01/11] Initial work creating the Redis publishing and subscribing --- apps/webapp/app/v3/eventRepository.server.ts | 46 +++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/v3/eventRepository.server.ts b/apps/webapp/app/v3/eventRepository.server.ts index 070e1b44a12..5afc008db3c 100644 --- a/apps/webapp/app/v3/eventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository.server.ts @@ -21,6 +21,8 @@ import { createHash } from "node:crypto"; import { PrismaClient, prisma } from "~/db.server"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server"; +import Redis, { RedisOptions } from "ioredis"; +import { env } from "~/env.server"; export type CreatableEvent = Omit< Prisma.TaskEventCreateInput, @@ -77,6 +79,7 @@ export type EventBuilder = { export type EventRepoConfig = { batchSize: number; batchInterval: number; + redis: RedisOptions; }; export type QueryOptions = Prisma.TaskEventWhereInput; @@ -119,7 +122,6 @@ export type UpdateEventOptions = { export class EventRepository { private readonly _flushScheduler: DynamicFlushScheduler; - private _randomIdGenerator = new RandomIdGenerator(); constructor(private db: PrismaClient = prisma, private readonly _config: EventRepoConfig) { @@ -138,6 +140,8 @@ export class EventRepository { await this.db.taskEvent.create({ data: event as Prisma.TaskEventCreateInput, }); + + this.#publishToRedis([event.traceId]); } async insertMany(events: CreatableEvent[]) { @@ -576,12 +580,44 @@ export class EventRepository { return result; } + async subscribeToTrace(traceId: string, updateHandler: (message: string) => void) { + const redis = new Redis(this._config.redis); + + const channel = `events:${traceId}`; + + // Subscribe to the channel. + await redis.subscribe(channel); + + // Define the message handler. + redis.on("message", (channelReceived, message) => { + if (channelReceived === channel) { + updateHandler(message); + } + }); + + // Return a function that can be used to unsubscribe. + return async function unsubscribe() { + await redis.unsubscribe(channel); + console.log(`Unsubscribed from ${channel}`); + }; + } + async #flushBatch(batch: CreatableEvent[]) { const events = excludePartialEventsWithCorrespondingFullEvent(batch); await this.db.taskEvent.createMany({ data: events as Prisma.TaskEventCreateManyInput[], }); + + this.#publishToRedis(events.map((event) => event.traceId)); + } + + #publishToRedis(traceIds: string[]) { + const redis = new Redis(this._config.redis); + const uniqueTraceIds = new Set(traceIds); + for (const traceId of uniqueTraceIds) { + redis.publish(`events:${traceId}`, new Date().toISOString()); + } } public generateTraceId() { @@ -614,6 +650,14 @@ export class EventRepository { export const eventRepository = new EventRepository(prisma, { batchSize: 100, batchInterval: 5000, + redis: { + port: env.REDIS_PORT, + host: env.REDIS_HOST, + username: env.REDIS_USERNAME, + password: env.REDIS_PASSWORD, + enableAutoPipelining: true, + ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), + }, }); export function stripAttributePrefix(attributes: Attributes, prefix: string) { From 6776d7545b52c14bc6e07df195520444f29560b2 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 5 Mar 2024 13:29:47 +0000 Subject: [PATCH 02/11] Live view is working --- .../v3/RunStreamPresenter.server.ts | 142 +++++++++--------- apps/webapp/app/v3/eventRepository.server.ts | 45 +++--- 2 files changed, 98 insertions(+), 89 deletions(-) diff --git a/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts b/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts index 46547104685..d254a341225 100644 --- a/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts @@ -1,7 +1,8 @@ -import { JobRun, TaskRun, TaskRunAttempt } from "@trigger.dev/database"; +import { TaskRun, TaskRunAttempt } from "@trigger.dev/database"; +import { eventStream } from "remix-utils/sse/server"; import { PrismaClient, prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; -import { sse } from "~/utils/sse.server"; +import { eventRepository } from "~/v3/eventRepository.server"; type RunWithAttempts = { updatedAt: Date; @@ -11,6 +12,8 @@ type RunWithAttempts = { }[]; }; +const pingInterval = 1000; + export class RunStreamPresenter { #prismaClient: PrismaClient; @@ -25,94 +28,89 @@ export class RunStreamPresenter { request: Request; runFriendlyId: TaskRun["friendlyId"]; }) { - const run = await this.#runForUpdates(runFriendlyId); + const run = await this.#prismaClient.taskRun.findUnique({ + where: { + friendlyId: runFriendlyId, + }, + select: { + traceId: true, + }, + }); if (!run) { return new Response("Not found", { status: 404 }); } - let lastUpdatedAt = this.#getLatestUpdatedAt(run); - logger.info("RunStreamPresenter.call", { runFriendlyId, - lastUpdatedAt, + traceId: run.traceId, }); - return sse({ - request, - run: async (send, stop) => { - const result = await this.#runForUpdates(runFriendlyId); - if (!result) { - return stop(); + let pinger: NodeJS.Timer | undefined = undefined; + + const { unsubscribe, eventEmitter } = await eventRepository.subscribeToTrace(run.traceId); + + return eventStream(request.signal, (send, close) => { + const safeSend = (args: { event?: string; data: string }) => { + try { + send(args); + } catch (error) { + if (error instanceof Error) { + if (error.name !== "TypeError") { + logger.debug("Error sending SSE, aborting", { + error: { + name: error.name, + message: error.message, + stack: error.stack, + }, + args, + }); + } + } else { + logger.debug("Unknown error sending SSE, aborting", { + error, + args, + }); + } + + close(); } + }; - if (this.#isRunCompleted(result)) { - logger.info("RunStreamPresenter.call completed", { - runFriendlyId, - lastUpdatedAt, - completed: true, - }); - send({ data: new Date().toISOString() }); - return stop(); - } + eventEmitter.addListener("message", (event) => { + safeSend({ data: event }); + }); - const newUpdatedAt = this.#getLatestUpdatedAt(result); - if (lastUpdatedAt !== newUpdatedAt) { - logger.info("RunStreamPresenter.call updated", { - runFriendlyId, - lastUpdatedAt, - newUpdatedAt, - }); - send({ data: result.updatedAt.toISOString() }); + pinger = setInterval(() => { + if (request.signal.aborted) { + return close(); } - logger.info("RunStreamPresenter.call waiting", { + safeSend({ event: "ping", data: new Date().toISOString() }); + }, pingInterval); + + return function clear() { + logger.info("RunStreamPresenter.abort", { runFriendlyId, - lastUpdatedAt, - newUpdatedAt, + traceId: run.traceId, }); - lastUpdatedAt = newUpdatedAt; - }, - }); - } - - #runForUpdates(friendlyId: string) { - return this.#prismaClient.taskRun.findUnique({ - where: { - friendlyId, - }, - select: { - updatedAt: true, - attempts: { - select: { - status: true, - updatedAt: true, - }, - orderBy: { - createdAt: "desc", - }, - take: 1, - }, - }, - }); - } + clearInterval(pinger); - #getLatestUpdatedAt(run: RunWithAttempts) { - const lastAttempt = run.attempts[0]; - if (lastAttempt) { - return lastAttempt.updatedAt.getTime(); - } - - return run.updatedAt.getTime(); - } + eventEmitter.removeAllListeners(); - #isRunCompleted(run: RunWithAttempts) { - return run.attempts.some( - (attempt) => - attempt.status === "FAILED" || - attempt.status === "CANCELED" || - attempt.status === "COMPLETED" - ); + unsubscribe().catch((error) => { + logger.error("RunStreamPresenter.abort.unsubscribe", { + runFriendlyId, + traceId: run.traceId, + error: { + name: error.name, + message: error.message, + stack: error.stack, + }, + }); + }); + }; + }); } } diff --git a/apps/webapp/app/v3/eventRepository.server.ts b/apps/webapp/app/v3/eventRepository.server.ts index 5afc008db3c..0d11e8216ea 100644 --- a/apps/webapp/app/v3/eventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository.server.ts @@ -12,7 +12,6 @@ import { flattenAndNormalizeAttributes, flattenAttributes, isExceptionSpanEvent, - logger, omit, unflattenAttributes, } from "@trigger.dev/core/v3"; @@ -23,6 +22,8 @@ import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server"; import Redis, { RedisOptions } from "ioredis"; import { env } from "~/env.server"; +import { EventEmitter } from "node:stream"; +import { logger } from "~/services/logger.server"; export type CreatableEvent = Omit< Prisma.TaskEventCreateInput, @@ -123,6 +124,7 @@ export type UpdateEventOptions = { export class EventRepository { private readonly _flushScheduler: DynamicFlushScheduler; private _randomIdGenerator = new RandomIdGenerator(); + private _redisPublishClient: Redis; constructor(private db: PrismaClient = prisma, private readonly _config: EventRepoConfig) { this._flushScheduler = new DynamicFlushScheduler({ @@ -130,6 +132,8 @@ export class EventRepository { flushInterval: _config.batchInterval, callback: this.#flushBatch.bind(this), }); + + this._redisPublishClient = new Redis(this._config.redis); } async insert(event: CreatableEvent) { @@ -141,7 +145,7 @@ export class EventRepository { data: event as Prisma.TaskEventCreateInput, }); - this.#publishToRedis([event.traceId]); + this.#publishToRedis([event]); } async insertMany(events: CreatableEvent[]) { @@ -580,25 +584,32 @@ export class EventRepository { return result; } - async subscribeToTrace(traceId: string, updateHandler: (message: string) => void) { + async subscribeToTrace(traceId: string) { const redis = new Redis(this._config.redis); - const channel = `events:${traceId}`; + const channel = `events:${traceId}:*`; // Subscribe to the channel. - await redis.subscribe(channel); + await redis.psubscribe(channel); + + const eventEmitter = new EventEmitter(); // Define the message handler. - redis.on("message", (channelReceived, message) => { - if (channelReceived === channel) { - updateHandler(message); + redis.on("pmessage", (pattern, channelReceived, message) => { + if (channelReceived.startsWith(`events:${traceId}:`)) { + eventEmitter.emit("message", message); } }); // Return a function that can be used to unsubscribe. - return async function unsubscribe() { - await redis.unsubscribe(channel); - console.log(`Unsubscribed from ${channel}`); + const unsubscribe = async () => { + logger.debug(`subscribeToTrace unsubscribe ${channel}`); + await redis.punsubscribe(channel); + }; + + return { + unsubscribe, + eventEmitter, }; } @@ -609,14 +620,14 @@ export class EventRepository { data: events as Prisma.TaskEventCreateManyInput[], }); - this.#publishToRedis(events.map((event) => event.traceId)); + this.#publishToRedis(events); } - #publishToRedis(traceIds: string[]) { - const redis = new Redis(this._config.redis); - const uniqueTraceIds = new Set(traceIds); - for (const traceId of uniqueTraceIds) { - redis.publish(`events:${traceId}`, new Date().toISOString()); + async #publishToRedis(events: CreatableEvent[]) { + if (events.length === 0) return; + const uniqueTraceSpans = new Set(events.map((e) => `events:${e.traceId}:${e.spanId}`)); + for (const id of uniqueTraceSpans) { + await this._redisPublishClient.publish(id, new Date().toISOString()); } } From 8aec352ce65738aca7ec976b5955e11d46ed5d97 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 5 Mar 2024 15:58:26 +0000 Subject: [PATCH 03/11] Show the live reloading status on the page --- .../app/components/runs/v3/LiveTimer.tsx | 31 ++++++++++++ .../app/presenters/v3/RunPresenter.server.ts | 1 + .../route.tsx | 50 ++++++++++++++++--- apps/webapp/app/v3/eventRepository.server.ts | 1 - 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/LiveTimer.tsx b/apps/webapp/app/components/runs/v3/LiveTimer.tsx index f6e5032e5d5..02926fff328 100644 --- a/apps/webapp/app/components/runs/v3/LiveTimer.tsx +++ b/apps/webapp/app/components/runs/v3/LiveTimer.tsx @@ -39,3 +39,34 @@ export function LiveTimer({ ); } + +export function LiveCountUp({ + lastUpdated, + updateInterval = 250, + className, +}: { + lastUpdated: Date; + updateInterval?: number; + className?: string; +}) { + const [now, setNow] = useState(); + + useEffect(() => { + const interval = setInterval(() => { + const date = new Date(); + setNow(date); + }, updateInterval); + + return () => clearInterval(interval); + }, [lastUpdated]); + + return ( + <> + {formatDuration(lastUpdated, now, { + style: "short", + maxDecimalPoints: 0, + units: ["m", "s"], + })} + + ); +} diff --git a/apps/webapp/app/presenters/v3/RunPresenter.server.ts b/apps/webapp/app/presenters/v3/RunPresenter.server.ts index 419d5087b01..277daad916c 100644 --- a/apps/webapp/app/presenters/v3/RunPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RunPresenter.server.ts @@ -109,6 +109,7 @@ export class RunPresenter { userName: getUsername(run.runtimeEnvironment.orgMember?.user), }, }, + rootSpanCompleted: events.at(0) ? !events.at(0)!.data.isPartial : false, events: events, parentRunFriendlyId: tree?.id === traceSummary.rootSpan.id ? undefined : traceSummary.rootSpan.runId, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx index 10038839f6d..78c3c8c7ef5 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx @@ -26,6 +26,7 @@ import { Slider } from "~/components/primitives/Slider"; import { Switch } from "~/components/primitives/Switch"; import * as Timeline from "~/components/primitives/Timeline"; import { TreeView, useTree } from "~/components/primitives/TreeView/TreeView"; +import { LiveCountUp, LiveTimer } from "~/components/runs/v3/LiveTimer"; import { RunIcon } from "~/components/runs/v3/RunIcon"; import { SpanTitle, eventBackgroundClassName } from "~/components/runs/v3/SpanTitle"; import { useDebounce } from "~/hooks/useDebounce"; @@ -51,7 +52,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const { projectParam, organizationSlug, runParam } = v3RunParamsSchema.parse(params); const presenter = new RunPresenter(); - const { run, events, parentRunFriendlyId, duration } = await presenter.call({ + const { run, events, parentRunFriendlyId, duration, rootSpanCompleted } = await presenter.call({ userId, organizationSlug, projectSlug: projectParam, @@ -67,6 +68,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { parentRunFriendlyId, resizeSettings, duration, + rootSpanCompleted, }); }; @@ -77,7 +79,7 @@ function getSpanId(path: string): string | undefined { } export default function Page() { - const { run, events, parentRunFriendlyId, resizeSettings, duration } = + const { run, events, parentRunFriendlyId, resizeSettings, duration, rootSpanCompleted } = useTypedLoaderData(); const navigate = useNavigate(); const organization = useOrganization(); @@ -136,6 +138,7 @@ export default function Page() { changeToSpan(selectedSpan); }} totalDuration={duration} + rootSpanCompleted={rootSpanCompleted} /> ) : ( @@ -184,12 +188,14 @@ function TasksTreeView({ parentRunFriendlyId, onSelectedIdChanged, totalDuration, + rootSpanCompleted, }: { events: RunEvent[]; selectedId?: string; parentRunFriendlyId?: string; onSelectedIdChanged: (selectedId: string | undefined) => void; totalDuration: number; + rootSpanCompleted: boolean; }) { const [filterText, setFilterText] = useState(""); const [errorsOnly, setErrorsOnly] = useState(false); @@ -239,6 +245,7 @@ function TasksTreeView({ onChange={(e) => setFilterText(e.target.value)} />
+
-
- {parentRunFriendlyId && } +
+ {parentRunFriendlyId ? ( + + ) : ( + + This is the root task + + )}
(
@@ -479,7 +491,7 @@ function TasksTreeView({ @@ -550,6 +562,30 @@ function ShowParentLink({ runFriendlyId }: { runFriendlyId: string }) { ); } +function LiveReloadingStatus({ rootSpanCompleted }: { rootSpanCompleted: boolean }) { + if (rootSpanCompleted) return null; + + return ( +
+ + + Live reloading + +
+ ); +} + +function PulsingDot() { + return ( + + + + + ); +} + function SpanWithDuration({ showDuration, node, diff --git a/apps/webapp/app/v3/eventRepository.server.ts b/apps/webapp/app/v3/eventRepository.server.ts index 0d11e8216ea..c7fd2cfd4ea 100644 --- a/apps/webapp/app/v3/eventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository.server.ts @@ -603,7 +603,6 @@ export class EventRepository { // Return a function that can be used to unsubscribe. const unsubscribe = async () => { - logger.debug(`subscribeToTrace unsubscribe ${channel}`); await redis.punsubscribe(channel); }; From 59d63c47649bd15d927717e71150109cf84a10a5 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 5 Mar 2024 16:34:30 +0000 Subject: [PATCH 04/11] Fixed some style issues on the run page with James --- apps/webapp/app/components/VersionLabel.tsx | 31 --------- .../app/components/runs/v3/SpanTitle.tsx | 26 +++---- .../app/components/runs/v3/TaskRunStatus.tsx | 6 +- .../route.tsx | 68 +++++++++++-------- .../route.tsx | 39 ++++++++--- 5 files changed, 88 insertions(+), 82 deletions(-) delete mode 100644 apps/webapp/app/components/VersionLabel.tsx diff --git a/apps/webapp/app/components/VersionLabel.tsx b/apps/webapp/app/components/VersionLabel.tsx deleted file mode 100644 index e2470454a86..00000000000 --- a/apps/webapp/app/components/VersionLabel.tsx +++ /dev/null @@ -1,31 +0,0 @@ -import { RuntimeEnvironment } from "~/models/runtimeEnvironment.server"; -import { cn } from "~/utils/cn"; -import { environmentTextClassName, environmentTitle } from "./environments/EnvironmentLabel"; - -type Environment = Pick; - -type VersionLabelProps = { - environment: Environment; - userName?: string; - version: string; -}; - -export function VersionLabel({ environment, userName, version }: VersionLabelProps) { - return ( -
-
v{version}
-
- {environmentTitle(environment, userName)} -
-
- ); -} diff --git a/apps/webapp/app/components/runs/v3/SpanTitle.tsx b/apps/webapp/app/components/runs/v3/SpanTitle.tsx index de034be0a74..158218527a5 100644 --- a/apps/webapp/app/components/runs/v3/SpanTitle.tsx +++ b/apps/webapp/app/components/runs/v3/SpanTitle.tsx @@ -9,6 +9,7 @@ type SpanTitleProps = { isError: boolean; style: TaskEventStyle; level: TaskEventLevel; + isPartial: boolean; size: "small" | "large"; }; @@ -90,10 +91,6 @@ export function SpanCodePathAccessory({ } function eventTextClassName(event: Pick) { - if (event.isError) { - return "text-rose-500"; - } - switch (event.level) { case "TRACE": { return textClassNameForVariant(event.style.variant); @@ -107,7 +104,7 @@ function eventTextClassName(event: Pick + event: Pick ) { if (event.isError) { - return "bg-rose-500"; + return "bg-error"; } switch (event.level) { case "TRACE": { - return backgroundClassNameForVariant(event.style.variant); + return backgroundClassNameForVariant(event.style.variant, event.isPartial); } case "LOG": case "INFO": case "DEBUG": { - return backgroundClassNameForVariant(event.style.variant); + return backgroundClassNameForVariant(event.style.variant, event.isPartial); } case "WARN": { return "bg-amber-400"; } case "ERROR": { - return "bg-rose-500"; + return "bg-error"; } default: { - return backgroundClassNameForVariant(event.style.variant); + return backgroundClassNameForVariant(event.style.variant, event.isPartial); } } } @@ -154,10 +151,13 @@ function textClassNameForVariant(variant: TaskEventStyle["variant"]) { } } -function backgroundClassNameForVariant(variant: TaskEventStyle["variant"]) { +function backgroundClassNameForVariant(variant: TaskEventStyle["variant"], isPartial: boolean) { switch (variant) { case "primary": { - return "bg-blue-500"; + if (isPartial) { + return "bg-blue-500"; + } + return "bg-success"; } default: { return "bg-charcoal-500"; diff --git a/apps/webapp/app/components/runs/v3/TaskRunStatus.tsx b/apps/webapp/app/components/runs/v3/TaskRunStatus.tsx index 5ec0a2c200f..606b3677fc5 100644 --- a/apps/webapp/app/components/runs/v3/TaskRunStatus.tsx +++ b/apps/webapp/app/components/runs/v3/TaskRunStatus.tsx @@ -74,15 +74,15 @@ export function runStatusClassNameColor(status: ExtendedTaskAttemptStatus | null case "PENDING": return "text-charcoal-500"; case "EXECUTING": - return "text-blue-500"; + return "text-pending"; case "PAUSED": return "text-amber-300"; case "FAILED": - return "text-rose-500"; + return "text-error"; case "CANCELED": return "text-charcoal-500"; case "COMPLETED": - return "text-green-500"; + return "text-success"; default: { const _exhaustiveCheck: never = status; throw new Error(`Non-exhaustive match for value: ${status}`); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx index ba0cc336bf8..ce9ae2d27ea 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx @@ -1,29 +1,27 @@ +import { useParams } from "@remix-run/react"; import { LoaderFunctionArgs } from "@remix-run/server-runtime"; import { formatDurationNanoseconds, nanosecondsToMilliseconds } from "@trigger.dev/core/v3"; import { ReactNode } from "react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; -import { VersionLabel } from "~/components/VersionLabel"; +import { ExitIcon } from "~/assets/icons/ExitIcon"; import { CodeBlock } from "~/components/code/CodeBlock"; -import { InlineCode } from "~/components/code/InlineCode"; -import { DateTime, DateTimeAccurate } from "~/components/primitives/DateTime"; +import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel"; +import { LinkButton } from "~/components/primitives/Buttons"; +import { DateTimeAccurate } from "~/components/primitives/DateTime"; import { Header2 } from "~/components/primitives/Headers"; import { Paragraph } from "~/components/primitives/Paragraph"; -import { ShortcutKey } from "~/components/primitives/ShortcutKey"; -import { SpanTitle } from "~/components/runs/v3/SpanTitle"; import { LiveTimer } from "~/components/runs/v3/LiveTimer"; import { RunIcon } from "~/components/runs/v3/RunIcon"; +import { SpanEvents } from "~/components/runs/v3/SpanEvents"; +import { SpanTitle } from "~/components/runs/v3/SpanTitle"; +import { TaskPath } from "~/components/runs/v3/TaskPath"; +import { TaskRunStatus } from "~/components/runs/v3/TaskRunStatus"; +import { useOrganization } from "~/hooks/useOrganizations"; +import { useProject } from "~/hooks/useProject"; import { SpanPresenter } from "~/presenters/v3/SpanPresenter.server"; import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { v3RunPath, v3SpanParamsSchema } from "~/utils/pathBuilder"; -import { TaskPath } from "~/components/runs/v3/TaskPath"; -import { SpanEvents } from "~/components/runs/v3/SpanEvents"; -import { LinkButton } from "~/components/primitives/Buttons"; -import { AlignRightIcon } from "lucide-react"; -import { useOrganization } from "~/hooks/useOrganizations"; -import { useProject } from "~/hooks/useProject"; -import { useParams } from "@remix-run/react"; -import { ExitIcon } from "~/assets/icons/ExitIcon"; export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); @@ -49,7 +47,7 @@ export default function Page() { const { runParam } = useParams(); return ( -
+
@@ -66,7 +64,7 @@ export default function Page() { /> )}
-
+
{event.level === "TRACE" ? ( @@ -85,6 +83,22 @@ export default function Page() { )} + {event.style.variant === "primary" && ( + + + + )} {event.message} {event.taskSlug} {event.taskPath && event.taskExportName && ( @@ -100,10 +114,10 @@ export default function Page() { {event.queueName && {event.queueName}} {event.workerVersion && ( - +
+ {event.workerVersion} + +
)} @@ -206,7 +220,7 @@ function TimelineBar({ ) : state === "complete" ? (
- + {formatDurationNanoseconds(duration, { style: "short" })}
@@ -231,10 +245,10 @@ function VerticalBar({ state }: { state: TimelineState }) { function DottedLine() { return (
-
-
-
-
+
+
+
+
); } @@ -242,13 +256,13 @@ function DottedLine() { function classNameForState(state: TimelineState) { switch (state) { case "pending": { - return "bg-blue-500"; + return "bg-pending"; } case "complete": { - return "bg-green-500"; + return "bg-success"; } case "error": { - return "bg-rose-500"; + return "bg-error"; } } } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx index 78c3c8c7ef5..e3f0f928106 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx @@ -3,6 +3,7 @@ import { ChevronRightIcon, MagnifyingGlassMinusIcon, MagnifyingGlassPlusIcon, + NoSymbolIcon, } from "@heroicons/react/20/solid"; import { Link, Outlet, useNavigate, useRevalidator } from "@remix-run/react"; import { LoaderFunctionArgs } from "@remix-run/server-runtime"; @@ -29,6 +30,7 @@ import { TreeView, useTree } from "~/components/primitives/TreeView/TreeView"; import { LiveCountUp, LiveTimer } from "~/components/runs/v3/LiveTimer"; import { RunIcon } from "~/components/runs/v3/RunIcon"; import { SpanTitle, eventBackgroundClassName } from "~/components/runs/v3/SpanTitle"; +import { TaskRunStatusIcon, runStatusClassNameColor } from "~/components/runs/v3/TaskRunStatus"; import { useDebounce } from "~/hooks/useDebounce"; import { useEventSource } from "~/hooks/useEventSource"; import { useOrganization } from "~/hooks/useOrganizations"; @@ -302,7 +304,7 @@ function TasksTreeView({ renderNode={({ node, state }) => (
-
+
{node.data.isRoot && Root}
-
- {node.data.isCancelled ? ( - - Cancelled - - ) : null} +
+
@@ -524,6 +522,31 @@ function NodeText({ node }: { node: RunEvent }) { ); } +function NodeStatusIcon({ node }: { node: RunEvent }) { + if (node.data.level !== "TRACE") return null; + if (node.data.style.variant !== "primary") return null; + + if (node.data.isCancelled) { + return ( + <> + + Canceled + + + + ); + } + + if (node.data.isError) { + return ; + } + + if (node.data.isPartial) { + return ; + } + + return ; +} function TaskLine({ isError, isSelected }: { isError: boolean; isSelected: boolean }) { return ( From 0bde11042a40bde68c62aba6e9a3eccfc89bdd5a Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 5 Mar 2024 16:59:57 +0000 Subject: [PATCH 05/11] Fix for invalid JSX attributes in the ExitIcon svg --- apps/webapp/app/assets/icons/ExitIcon.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/assets/icons/ExitIcon.tsx b/apps/webapp/app/assets/icons/ExitIcon.tsx index 3bbd24cfc9f..29d52609cdd 100644 --- a/apps/webapp/app/assets/icons/ExitIcon.tsx +++ b/apps/webapp/app/assets/icons/ExitIcon.tsx @@ -1,13 +1,13 @@ export function ExitIcon({ className }: { className?: string }) { return ( - - + + ); From 7250189e9baa0dcb0fdd4b29fcc30ad0cc0dc7d6 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 5 Mar 2024 18:16:07 +0000 Subject: [PATCH 06/11] Some sexy shit --- .../app/components/primitives/Timeline.tsx | 17 +- apps/webapp/app/hooks/useInitialDimensions.ts | 13 ++ .../route.tsx | 158 ++++++++++++------ apps/webapp/app/utils/lerp.ts | 15 ++ 4 files changed, 138 insertions(+), 65 deletions(-) create mode 100644 apps/webapp/app/hooks/useInitialDimensions.ts create mode 100644 apps/webapp/app/utils/lerp.ts diff --git a/apps/webapp/app/components/primitives/Timeline.tsx b/apps/webapp/app/components/primitives/Timeline.tsx index 23ea46d208f..dfb32537501 100644 --- a/apps/webapp/app/components/primitives/Timeline.tsx +++ b/apps/webapp/app/components/primitives/Timeline.tsx @@ -9,6 +9,7 @@ import { useRef, useState, } from "react"; +import { inverseLerp, lerp } from "~/utils/lerp"; interface MousePosition { x: number; @@ -225,19 +226,3 @@ export function FollowCursor({ children }: FollowCursorProps) { function calculatePixelWidth(minWidth: number, maxWidth: number, scale: number) { return lerp(minWidth, maxWidth, scale); } - -/** Linearly interpolates between the min/max values, using t. - * It can't go outside the range */ -function lerp(min: number, max: number, t: number) { - return min + (max - min) * clamp(t, 0, 1); -} - -/** Inverse lerp */ -function inverseLerp(min: number, max: number, value: number) { - return (value - min) / (max - min); -} - -/** Clamps a value between a min and max */ -function clamp(value: number, min: number, max: number) { - return Math.min(max, Math.max(min, value)); -} diff --git a/apps/webapp/app/hooks/useInitialDimensions.ts b/apps/webapp/app/hooks/useInitialDimensions.ts new file mode 100644 index 00000000000..4e230f87f02 --- /dev/null +++ b/apps/webapp/app/hooks/useInitialDimensions.ts @@ -0,0 +1,13 @@ +import { useEffect, useLayoutEffect, useState } from "react"; + +export function useInitialDimensions(ref: React.RefObject) { + const [dimensions, setDimensions] = useState(null); + + useEffect(() => { + if (ref.current) { + setDimensions(ref.current.getBoundingClientRect()); + } + }, [ref]); + + return dimensions; +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx index e3f0f928106..7327716dfde 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx @@ -33,6 +33,7 @@ import { SpanTitle, eventBackgroundClassName } from "~/components/runs/v3/SpanTi import { TaskRunStatusIcon, runStatusClassNameColor } from "~/components/runs/v3/TaskRunStatus"; import { useDebounce } from "~/hooks/useDebounce"; import { useEventSource } from "~/hooks/useEventSource"; +import { useInitialDimensions } from "~/hooks/useInitialDimensions"; import { useOrganization } from "~/hooks/useOrganizations"; import { usePathName } from "~/hooks/usePathName"; import { useProject } from "~/hooks/useProject"; @@ -41,6 +42,7 @@ import { RunEvent, RunPresenter } from "~/presenters/v3/RunPresenter.server"; import { getResizableRunSettings, setResizableRunSettings } from "~/services/resizablePanel"; import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; +import { lerp } from "~/utils/lerp"; import { v3RunParamsSchema, v3RunPath, @@ -202,10 +204,12 @@ function TasksTreeView({ const [filterText, setFilterText] = useState(""); const [errorsOnly, setErrorsOnly] = useState(false); const [showDurations, setShowDurations] = useState(false); - const [scale, setScale] = useState(0.25); + const [scale, setScale] = useState(0); const parentRef = useRef(null); const treeScrollRef = useRef(null); const timelineScrollRef = useRef(null); + const timelineContainerRef = useRef(null); + const initialTimelineDimensions = useInitialDimensions(timelineContainerRef); const { nodes, @@ -373,66 +377,73 @@ function TasksTreeView({ {/* Timeline */} -
+
{/* Follows the cursor */} - - {(ms) => ( -
-
-
- {formatDurationMilliseconds(ms, { - style: "short", - maxDecimalPoints: ms < 1000 ? 0 : 1, - })} -
-
-
-
- )} - + {/* The duration labels */} - + - {(ms: number, index: number) => ( - - {(ms) => ( -
- {formatDurationMilliseconds(ms, { - style: "short", - maxDecimalPoints: ms < 1000 ? 0 : 1, - })} -
- )} - - )} + {(ms: number, index: number) => { + if (index === tickCount - 1) return null; + return ( + + {(ms) => ( +
+ {formatDurationMilliseconds(ms, { + style: "short", + maxDecimalPoints: ms < 1000 ? 0 : 1, + })} +
+ )} + + ); + }} + {rootSpanCompleted && ( + + {(ms) => ( +
+ {formatDurationMilliseconds(ms, { + style: "short", + maxDecimalPoints: ms < 1000 ? 0 : 1, + })} +
+ )} + + )} - + {(ms: number, index: number) => { - if (index === 0) return null; + if (index === 0 || index === tickCount - 1) return null; return ( + {/* Main timeline body */} @@ -454,6 +469,13 @@ function TasksTreeView({ ); }} + {/* The completed line */} + {rootSpanCompleted && ( + + )} @@ -618,7 +640,7 @@ function SpanWithDuration({
@@ -645,3 +667,41 @@ function SpanWithDuration({ ); } + +const edgeBoundary = 0.05; + +function CurrentTimeIndicator({ totalDuration }: { totalDuration: number }) { + return ( + + {(ms) => { + const ratio = ms / nanosecondsToMilliseconds(totalDuration); + let offset = 0.5; + if (ratio < edgeBoundary) { + offset = lerp(0, 0.5, ratio / edgeBoundary); + } else if (ratio > 1 - edgeBoundary) { + offset = lerp(0.5, 1, (ratio - (1 - edgeBoundary)) / edgeBoundary); + } + + return ( +
+
+
+ {formatDurationMilliseconds(ms, { + style: "short", + maxDecimalPoints: ms < 1000 ? 0 : 1, + })} +
+
+
+
+ ); + }} + + ); +} diff --git a/apps/webapp/app/utils/lerp.ts b/apps/webapp/app/utils/lerp.ts new file mode 100644 index 00000000000..c2df9a5d2d2 --- /dev/null +++ b/apps/webapp/app/utils/lerp.ts @@ -0,0 +1,15 @@ +/** Linearly interpolates between the min/max values, using t. + * It can't go outside the range */ +export function lerp(min: number, max: number, t: number) { + return min + (max - min) * clamp(t, 0, 1); +} + +/** Inverse lerp */ +export function inverseLerp(min: number, max: number, value: number) { + return (value - min) / (max - min); +} + +/** Clamps a value between a min and max */ +export function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} From e6701fb52a93fc3355124344d5e3e65621cc5817 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 5 Mar 2024 18:32:55 +0000 Subject: [PATCH 07/11] The end line is now correctly coloured --- .../app/presenters/v3/RunPresenter.server.ts | 11 +++++- .../route.tsx | 35 ++++++++++++------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/apps/webapp/app/presenters/v3/RunPresenter.server.ts b/apps/webapp/app/presenters/v3/RunPresenter.server.ts index 277daad916c..a7524834ca1 100644 --- a/apps/webapp/app/presenters/v3/RunPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RunPresenter.server.ts @@ -98,6 +98,15 @@ export class RunPresenter { } } + let rootSpanStatus: "executing" | "completed" | "failed" = "executing"; + if (events[0]) { + if (events[0].data.isError) { + rootSpanStatus = "failed"; + } else if (!events[0].data.isPartial) { + rootSpanStatus = "completed"; + } + } + return { run: { number: run.number, @@ -109,7 +118,7 @@ export class RunPresenter { userName: getUsername(run.runtimeEnvironment.orgMember?.user), }, }, - rootSpanCompleted: events.at(0) ? !events.at(0)!.data.isPartial : false, + rootSpanStatus, events: events, parentRunFriendlyId: tree?.id === traceSummary.rootSpan.id ? undefined : traceSummary.rootSpan.runId, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx index 7327716dfde..3a1aab99a9e 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx @@ -56,7 +56,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const { projectParam, organizationSlug, runParam } = v3RunParamsSchema.parse(params); const presenter = new RunPresenter(); - const { run, events, parentRunFriendlyId, duration, rootSpanCompleted } = await presenter.call({ + const { run, events, parentRunFriendlyId, duration, rootSpanStatus } = await presenter.call({ userId, organizationSlug, projectSlug: projectParam, @@ -72,7 +72,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { parentRunFriendlyId, resizeSettings, duration, - rootSpanCompleted, + rootSpanStatus, }); }; @@ -83,7 +83,7 @@ function getSpanId(path: string): string | undefined { } export default function Page() { - const { run, events, parentRunFriendlyId, resizeSettings, duration, rootSpanCompleted } = + const { run, events, parentRunFriendlyId, resizeSettings, duration, rootSpanStatus } = useTypedLoaderData(); const navigate = useNavigate(); const organization = useOrganization(); @@ -142,7 +142,7 @@ export default function Page() { changeToSpan(selectedSpan); }} totalDuration={duration} - rootSpanCompleted={rootSpanCompleted} + rootSpanStatus={rootSpanStatus} /> ) : ( @@ -192,14 +192,14 @@ function TasksTreeView({ parentRunFriendlyId, onSelectedIdChanged, totalDuration, - rootSpanCompleted, + rootSpanStatus, }: { events: RunEvent[]; selectedId?: string; parentRunFriendlyId?: string; onSelectedIdChanged: (selectedId: string | undefined) => void; totalDuration: number; - rootSpanCompleted: boolean; + rootSpanStatus: "executing" | "completed" | "failed"; }) { const [filterText, setFilterText] = useState(""); const [errorsOnly, setErrorsOnly] = useState(false); @@ -251,7 +251,7 @@ function TasksTreeView({ onChange={(e) => setFilterText(e.target.value)} />
- + - {rootSpanCompleted && ( + {rootSpanStatus !== "executing" && ( {(ms) => (
@@ -454,7 +457,10 @@ function TasksTreeView({ @@ -470,10 +476,13 @@ function TasksTreeView({ }} {/* The completed line */} - {rootSpanCompleted && ( + {rootSpanStatus !== "executing" && ( )} Date: Tue, 5 Mar 2024 19:17:02 +0000 Subject: [PATCH 08/11] millisecondsToNanoseconds exported from core/v3 and imported correctly --- apps/webapp/app/presenters/v3/RunPresenter.server.ts | 2 +- packages/core/src/v3/index.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/presenters/v3/RunPresenter.server.ts b/apps/webapp/app/presenters/v3/RunPresenter.server.ts index a7524834ca1..c55ddc4b20c 100644 --- a/apps/webapp/app/presenters/v3/RunPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RunPresenter.server.ts @@ -1,4 +1,4 @@ -import { millisecondsToNanoseconds } from "@trigger.dev/core/v3/utils/durations"; +import { millisecondsToNanoseconds } from "@trigger.dev/core/v3"; import { createTreeFromFlatItems, flattenTree } from "~/components/primitives/TreeView/TreeView"; import { PrismaClient, prisma } from "~/db.server"; import { getUsername } from "~/utils/username"; diff --git a/packages/core/src/v3/index.ts b/packages/core/src/v3/index.ts index d17ca66e2bc..d16422d1b83 100644 --- a/packages/core/src/v3/index.ts +++ b/packages/core/src/v3/index.ts @@ -16,6 +16,7 @@ export { formatDurationNanoseconds, formatDurationInDays, nanosecondsToMilliseconds, + millisecondsToNanoseconds, } from "./utils/durations"; export { getEnvVar } from "./utils/getEnv"; From a0810d39e589b75aed60b2a80ea77005c38b5618 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 6 Mar 2024 09:21:25 +0000 Subject: [PATCH 09/11] Fix for e2e tests, we changed the h1 to an h2 --- tests/e2e/e2e.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/e2e.spec.ts b/tests/e2e/e2e.spec.ts index 684425c7dd4..8b765de98d9 100644 --- a/tests/e2e/e2e.spec.ts +++ b/tests/e2e/e2e.spec.ts @@ -31,13 +31,13 @@ test("Verify jobs from the test nextjs project", async ({ page }) => { await page.locator("a").filter({ hasText: "Test Project" }).click(); await page.getByRole("link", { name: "Environments & API Keys" }).click(); - await expect(page.locator("h1").filter({ hasText: "Environments & API Keys" })).toBeVisible(); + await expect(page.locator("h2").filter({ hasText: "Environments & API Keys" })).toBeVisible(); await expect( page.locator("h3").filter({ hasText: "nextjs-test" }) // Set the timeout high to allow the cli to register jobs ).toBeVisible({ timeout: 15000 }); await page.getByRole("link", { name: "Jobs" }).click(); - await expect(page.locator("h1").filter({ hasText: /^Jobs$/ })).toBeVisible(); + await expect(page.locator("h2").filter({ hasText: /^Jobs$/ })).toBeVisible(); await expect(page.getByRole("link", { name: /Test Job One/ })).toBeVisible(); }); From 2ee06137a6c382ca94824d3b1952d339da03e105 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 6 Mar 2024 09:21:32 +0000 Subject: [PATCH 10/11] Tidied imports --- apps/webapp/app/components/primitives/Icon.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/components/primitives/Icon.tsx b/apps/webapp/app/components/primitives/Icon.tsx index 00200a18843..5d49770c4d0 100644 --- a/apps/webapp/app/components/primitives/Icon.tsx +++ b/apps/webapp/app/components/primitives/Icon.tsx @@ -1,7 +1,6 @@ -import React, { FunctionComponent, ReactElement, createElement } from "react"; -import { IconNamesOrString, NamedIcon } from "./NamedIcon"; +import React, { FunctionComponent, createElement } from "react"; import { cn } from "~/utils/cn"; -import { render } from "react-dom"; +import { IconNamesOrString, NamedIcon } from "./NamedIcon"; export type RenderIcon = | IconNamesOrString From 97a936acd88263c1c6a7ba401d12303f286eeb4f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 6 Mar 2024 09:21:36 +0000 Subject: [PATCH 11/11] Removed unused hook --- apps/webapp/app/hooks/useIsOrgChildPage.ts | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 apps/webapp/app/hooks/useIsOrgChildPage.ts diff --git a/apps/webapp/app/hooks/useIsOrgChildPage.ts b/apps/webapp/app/hooks/useIsOrgChildPage.ts deleted file mode 100644 index 685630ccc17..00000000000 --- a/apps/webapp/app/hooks/useIsOrgChildPage.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { UIMatch, useMatches } from "@remix-run/react"; - -export function useIsOrgChildPage(matches?: UIMatch[]) { - if (!matches) { - matches = useMatches(); - } - - return matches.some((matchData) => { - return matchData.id.startsWith("routes/_app.orgs.$organizationSlug"); - }); -}