Skip to content

Commit 7a9bd18

Browse files
nicktrnnicktrn
authored
v3: stop swallowing deployment errors and display them better (#1020)
* exit worker with code 111 for handled errors * task monitor will ignore previously handled errors * fix logging of index errors * deployment error component * changeset
1 parent 03af44e commit 7a9bd18

6 files changed

Lines changed: 83 additions & 36 deletions

File tree

‎.changeset/tender-oranges-rhyme.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"trigger.dev": patch
3+
---
4+
5+
Stop swallowing deployment errors and display them better

‎apps/kubernetes-provider/src/taskMonitor.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,13 @@ export class TaskMonitor {
136136

137137
const podStatus = this.#getPodStatusSummary(pod.status);
138138
const containerState = this.#getContainerStateSummary(containerStatus.state);
139+
const exitCode = containerState.exitCode ?? -1;
140+
141+
// We use this special exit code to signal any errors were already handled elsewhere
142+
if (exitCode === 111) {
143+
return;
144+
}
145+
139146
const rawLogs = await this.#getLogTail(podName);
140147

141148
this.#logger.log(`${podName} failed with:`, {
@@ -144,7 +151,6 @@ export class TaskMonitor {
144151
rawLogs,
145152
});
146153

147-
const exitCode = containerState.exitCode ?? -1;
148154
const rawReason = podStatus.reason ?? containerState.reason ?? "";
149155
const message = podStatus.message ?? containerState.message ?? "";
150156

‎apps/webapp/app/components/runs/v3/DeploymentError.tsx‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import { CodeBlock } from "~/components/code/CodeBlock";
2+
import { Callout } from "~/components/primitives/Callout";
3+
import { Header2 } from "~/components/primitives/Headers";
4+
import type { ErrorData } from "~/presenters/v3/DeploymentPresenter.server";
5+
6+
type DeploymentErrorProps = {
7+
errorData: ErrorData;
8+
};
9+
10+
export function DeploymentError({ errorData }: DeploymentErrorProps) {
11+
return (
12+
<div className="flex flex-col gap-2 rounded-sm border border-rose-500/50 p-3">
13+
<DeploymentErrorHeader title={errorData.name ?? "Error"} titleClassName="text-rose-500" />
14+
{errorData.message && <Callout variant="error">{errorData.message}</Callout>}
15+
{errorData.stack && (
16+
<CodeBlock
17+
showCopyButton={false}
18+
showLineNumbers={false}
19+
code={errorData.stack}
20+
maxLines={20}
21+
/>
22+
)}
23+
</div>
24+
);
25+
}
26+
27+
function DeploymentErrorHeader({
28+
title,
29+
titleClassName,
30+
}: {
31+
title: string;
32+
titleClassName?: string;
33+
}) {
34+
return (
35+
<div className="flex items-center justify-between">
36+
<Header2 className={titleClassName}>{title}</Header2>
37+
</div>
38+
);
39+
}

‎apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,12 @@ import { User } from "~/models/user.server";
1212
import { safeJsonParse } from "~/utils/json";
1313
import { getUsername } from "~/utils/username";
1414

15+
export type ErrorData = {
16+
name: string;
17+
message: string;
18+
stack?: string;
19+
};
20+
1521
export class DeploymentPresenter {
1622
#prismaClient: PrismaClient;
1723

@@ -133,7 +139,7 @@ export class DeploymentPresenter {
133139
};
134140
}
135141

136-
#prepareErrorData(errorData: WorkerDeployment["errorData"]) {
142+
#prepareErrorData(errorData: WorkerDeployment["errorData"]): ErrorData | undefined {
137143
if (!errorData) {
138144
return;
139145
}

‎apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments.$deploymentParam/route.tsx‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import {
1919
TableHeaderCell,
2020
TableRow,
2121
} from "~/components/primitives/Table";
22+
import { DeploymentError } from "~/components/runs/v3/DeploymentError";
2223
import { DeploymentStatus } from "~/components/runs/v3/DeploymentStatus";
2324
import { TaskFunctionName } from "~/components/runs/v3/TaskPath";
2425
import { useOrganization } from "~/hooks/useOrganizations";
@@ -158,25 +159,7 @@ export default function Page() {
158159
</Table>
159160
</div>
160161
) : deployment.errorData ? (
161-
<div className="flex flex-col">
162-
{deployment.errorData.stack ? (
163-
<CodeBlock
164-
language="markdown"
165-
rowTitle={deployment.errorData.message}
166-
code={deployment.errorData.stack}
167-
maxLines={20}
168-
/>
169-
) : (
170-
<div className="flex flex-col">
171-
<Paragraph
172-
variant="base/bright"
173-
className="w-full border-b border-grid-dimmed py-2.5"
174-
>
175-
{deployment.errorData.message}
176-
</Paragraph>
177-
</div>
178-
)}
179-
</div>
162+
<DeploymentError errorData={deployment.errorData} />
180163
) : null}
181164
</div>
182165
</div>

‎packages/cli-v3/src/workers/prod/entry-point.ts‎

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,10 @@ class ProdWorker {
417417
}
418418
} catch (e) {
419419
if (e instanceof TaskMetadataParseError) {
420-
logger.error("tasks metadata parse error", { message: e.zodIssues, tasks: e.tasks });
420+
logger.error("tasks metadata parse error", {
421+
zodIssues: e.zodIssues,
422+
tasks: e.tasks,
423+
});
421424

422425
socket.emit("INDEXING_FAILED", {
423426
version: "v1",
@@ -429,31 +432,35 @@ class ProdWorker {
429432
},
430433
});
431434
} else if (e instanceof UncaughtExceptionError) {
432-
logger.error("uncaught exception", { message: e.originalError.message });
435+
const error = {
436+
name: e.originalError.name,
437+
message: e.originalError.message,
438+
stack: e.originalError.stack,
439+
};
440+
441+
logger.error("uncaught exception", { originalError: error });
433442

434443
socket.emit("INDEXING_FAILED", {
435444
version: "v1",
436445
deploymentId: this.deploymentId,
437-
error: {
438-
name: e.originalError.name,
439-
message: e.originalError.message,
440-
stack: e.originalError.stack,
441-
},
446+
error,
442447
});
443448
} else if (e instanceof Error) {
444-
logger.error("error", { message: e.message });
449+
const error = {
450+
name: e.name,
451+
message: e.message,
452+
stack: e.stack,
453+
};
454+
455+
logger.error("error", { error });
445456

446457
socket.emit("INDEXING_FAILED", {
447458
version: "v1",
448459
deploymentId: this.deploymentId,
449-
error: {
450-
name: e.name,
451-
message: e.message,
452-
stack: e.stack,
453-
},
460+
error,
454461
});
455462
} else if (typeof e === "string") {
456-
logger.error("string error", { message: e });
463+
logger.error("string error", { error: { message: e } });
457464

458465
socket.emit("INDEXING_FAILED", {
459466
version: "v1",
@@ -477,7 +484,8 @@ class ProdWorker {
477484
}
478485

479486
await setTimeout(200);
480-
process.exit(1);
487+
// Use exit code 111 so we can ignore those failures in the task monitor
488+
process.exit(111);
481489
}
482490
}
483491

0 commit comments

Comments
 (0)