Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .server-changes/trace-view-large-runs-subtree.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
area: webapp
type: fix
---

Fix empty trace views for child and nested runs in very large traces. The dashboard and retrieve-trace API now return the requested run's span subtree.
57 changes: 52 additions & 5 deletions apps/webapp/app/presenters/v3/RunPresenter.server.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { millisecondsToNanoseconds, RunAnnotations } from "@trigger.dev/core/v3";
import { createTreeFromFlatItems, flattenTree } from "~/components/primitives/TreeView/TreeView";
import { prisma, type PrismaClient } from "~/db.server";
import { logger } from "~/services/logger.server";
import { createTimelineSpanEventsFromSpanEvents } from "~/utils/timelineSpanEvents";
import { getUsername } from "~/utils/username";
import { SpanSummary } from "~/v3/eventRepository/eventRepository.types";
Expand Down Expand Up @@ -179,16 +180,49 @@ export class RunPresenter {
run.runtimeEnvironment.organizationId
);

// get the events
const traceTimeBounds = {
startCreatedAt: run.rootTaskRun?.createdAt ?? run.createdAt,
endCreatedAt: run.completedAt ?? undefined,
};

// Fast path: full trace summary. Slow path: subtree fetch when the anchor
// span fell past the row cap (large traces ordered by start_time ASC).
let traceSummary = await repository.getTraceSummary(
getTaskEventStoreTableForRun(run),
run.runtimeEnvironment.id,
run.traceId,
run.rootTaskRun?.createdAt ?? run.createdAt,
run.completedAt ?? undefined,
traceTimeBounds.startCreatedAt,
traceTimeBounds.endCreatedAt,
{ includeDebugLogs: showDebug }
);

let isTruncated = traceSummary?.isTruncated ?? false;
const hasAnchorSpan = traceSummary?.spans.some((span) => span.id === run.spanId) ?? false;

if (traceSummary && !hasAnchorSpan) {
logger.warn("Trace summary missing anchor span, falling back to subtree fetch", {
runId: run.friendlyId,
spanId: run.spanId,
traceId: run.traceId,
spanCount: traceSummary.spans.length,
});

const subtreeSummary = await repository.getTraceSubtreeSummary(
getTaskEventStoreTableForRun(run),
run.runtimeEnvironment.id,
run.traceId,
run.spanId,
traceTimeBounds.startCreatedAt,
traceTimeBounds.endCreatedAt,
{ includeDebugLogs: showDebug }
);

if (subtreeSummary) {
traceSummary = subtreeSummary;
isTruncated = subtreeSummary.isTruncated ?? false;
}
}

if (!traceSummary) {
const spanSummary: SpanSummary = {
id: run.spanId,
Expand Down Expand Up @@ -237,11 +271,22 @@ export class RunPresenter {

// Resolve agent-kind once so the tree renderer can swap icon/colour for
// the current run's spans without doing per-row lookups.
const isAgentRun =
RunAnnotations.safeParse(run.annotations).data?.taskKind === "AGENT";
const isAgentRun = RunAnnotations.safeParse(run.annotations).data?.taskKind === "AGENT";

//this tree starts at the passed in span (hides parent elements if there are any)
const tree = createTreeFromFlatItems(traceSummary.spans, run.spanId);
const missingAnchor = !traceSummary.spans.some((span) => span.id === run.spanId) || !tree;

if (missingAnchor) {
logger.warn("Trace view anchor span not found in trace summary", {
runId: run.friendlyId,
spanId: run.spanId,
traceId: run.traceId,
spanCount: traceSummary.spans.length,
});

isTruncated = true;
}

//we need the start offset for each item, and the total duration of the entire tree
const treeRootStartTimeMs = tree ? tree?.data.startTime.getTime() : 0;
Expand Down Expand Up @@ -313,6 +358,8 @@ export class RunPresenter {
: undefined,
overridesBySpanId: traceSummary.overridesBySpanId,
linkedRunIdBySpanId,
isTruncated,
missingAnchor,
},
maximumLiveReloadingSetting: repository.maximumLiveReloadingSetting,
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
import { PageBody } from "~/components/layout/AppLayout";
import { Badge } from "~/components/primitives/Badge";
import { Button, LinkButton } from "~/components/primitives/Buttons";
import { Callout } from "~/components/primitives/Callout";
import { CopyableText } from "~/components/primitives/CopyableText";
import { DateTimeShort } from "~/components/primitives/DateTime";
import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
Expand Down Expand Up @@ -599,8 +600,16 @@ function TraceView({
return <></>;
}

const { events, duration, rootSpanStatus, rootStartedAt, queuedDuration, overridesBySpanId } =
trace;
const {
events,
duration,
rootSpanStatus,
rootStartedAt,
queuedDuration,
overridesBySpanId,
isTruncated = false,
missingAnchor = false,
} = trace;

const changeToSpan = useDebounce((selectedSpan: string) => {
replaceSearchParam("span", selectedSpan, { replace: true });
Expand Down Expand Up @@ -647,31 +656,44 @@ function TraceView({
id={resizableSettings.parent.main.id}
min={resizableSettings.parent.main.min}
>
<TasksTreeView
selectedId={selectedSpanId}
key={events[0]?.id ?? "-"}
events={events}
onSelectedIdChanged={(selectedSpan) => {
//instantly close the panel if no span is selected
if (!selectedSpan) {
replaceSearchParam("span");
return;
}

changeToSpan(selectedSpan);
}}
totalDuration={duration}
rootSpanStatus={rootSpanStatus}
rootStartedAt={rootStartedAt ? new Date(rootStartedAt) : undefined}
queuedDuration={queuedDuration}
environmentType={run.environment.type}
shouldLiveReload={isLiveReloading}
maximumLiveReloadingSetting={maximumLiveReloadingSetting}
rootRun={run.rootTaskRun}
parentRun={run.parentTaskRun}
isCompleted={run.completedAt !== null}
treeSnapshot={resizable.tree as ResizableSnapshot}
/>
<div className="flex h-full flex-col overflow-hidden">
{isTruncated && (
<div className="shrink-0 border-b border-charcoal-700 px-3 py-2">
<Callout variant="warning" className="text-sm">
{missingAnchor
? "Trace too large to display completely."
: "This run's trace is partially displayed because it exceeds the view limit."}
</Callout>
</div>
)}
<div className="min-h-0 flex-1">
<TasksTreeView
selectedId={selectedSpanId}
key={events[0]?.id ?? "-"}
events={events}
onSelectedIdChanged={(selectedSpan) => {
//instantly close the panel if no span is selected
if (!selectedSpan) {
replaceSearchParam("span");
return;
}

changeToSpan(selectedSpan);
}}
totalDuration={duration}
rootSpanStatus={rootSpanStatus}
rootStartedAt={rootStartedAt ? new Date(rootStartedAt) : undefined}
queuedDuration={queuedDuration}
environmentType={run.environment.type}
shouldLiveReload={isLiveReloading}
maximumLiveReloadingSetting={maximumLiveReloadingSetting}
rootRun={run.rootTaskRun}
parentRun={run.parentTaskRun}
isCompleted={run.completedAt !== null}
treeSnapshot={resizable.tree as ResizableSnapshot}
/>
</div>
</div>
</ResizablePanel>
<ResizableHandle
id={resizableSettings.parent.handleId}
Expand Down
18 changes: 8 additions & 10 deletions apps/webapp/app/routes/api.v1.runs.$runId.trace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@ import { json } from "@remix-run/server-runtime";
import { BatchId } from "@trigger.dev/core/v3/isomorphic";
import { z } from "zod";
import { $replica } from "~/db.server";
import {
anyResource,
createLoaderApiRoute,
} from "~/services/routeBuilders/apiBuilder.server";
import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server";
import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
import { findRunByIdWithMollifierFallback } from "~/v3/mollifier/readFallback.server";
Expand All @@ -24,13 +21,13 @@ const ParamsSchema = z.object({
// pass-through control case in scripts/mollifier-api-parity.sh).
type ResolvedRun =
| { source: "pg"; run: Awaited<ReturnType<typeof findPgRun>> & {} }
| { source: "buffer"; run: NonNullable<Awaited<ReturnType<typeof findRunByIdWithMollifierFallback>>> };
| {
source: "buffer";
run: NonNullable<Awaited<ReturnType<typeof findRunByIdWithMollifierFallback>>>;
};

async function findPgRun(runId: string, environmentId: string) {
return runStore.findRun(
{ friendlyId: runId, runtimeEnvironmentId: environmentId },
$replica
);
return runStore.findRun({ friendlyId: runId, runtimeEnvironmentId: environmentId }, $replica);
}

export const loader = createLoaderApiRoute(
Expand Down Expand Up @@ -96,10 +93,11 @@ export const loader = createLoaderApiRoute(
authentication.environment.organization.id
);

const traceSummary = await eventRepository.getTraceDetailedSummary(
const traceSummary = await eventRepository.getTraceDetailedSubtreeSummary(
getTaskEventStoreTableForRun(run),
authentication.environment.id,
run.traceId,
run.spanId,
run.createdAt,
run.completedAt ?? undefined
);
Comment thread
kathiekiwi marked this conversation as resolved.
Expand Down
Loading