Skip to content

Commit 35d0c2a

Browse files
ericallamericallam
committed
Implement the task output redacting to prevent redacted values from showing in the logs
1 parent 813ec74 commit 35d0c2a

7 files changed

Lines changed: 267 additions & 12 deletions

File tree

‎apps/webapp/app/presenters/TaskDetailsPresenter.server.ts‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
import { RedactSchema } from "@trigger.dev/core";
12
import { StyleSchema } from "@trigger.dev/core";
23
import { PrismaClient, prisma } from "~/db.server";
34
import { mergeProperties } from "~/utils/mergeProperties.server";
5+
import { Redactor } from "~/utils/redactor";
46

57
type DetailsProps = {
68
id: string;
@@ -61,6 +63,7 @@ export class TaskDetailsPresenter {
6163
completedAt: true,
6264
style: true,
6365
parentId: true,
66+
redact: true,
6467
attempts: {
6568
select: {
6669
number: true,
@@ -85,11 +88,32 @@ export class TaskDetailsPresenter {
8588

8689
return {
8790
...task,
88-
output: task.output ? JSON.stringify(task.output, null, 2) : undefined,
91+
redact: undefined,
92+
output: task.output
93+
? JSON.stringify(this.#stringifyOutputWithRedactions(task.output, task.redact), null, 2)
94+
: undefined,
8995
connection: task.runConnection,
9096
params: task.params as Record<string, any>,
9197
properties: mergeProperties(task.properties, task.outputProperties),
9298
style: task.style ? StyleSchema.parse(task.style) : undefined,
9399
};
94100
}
101+
102+
#stringifyOutputWithRedactions(output: any, redact: unknown): any {
103+
if (!output) {
104+
return;
105+
}
106+
107+
const parsedRedact = RedactSchema.safeParse(redact);
108+
109+
if (!parsedRedact.success) {
110+
return output;
111+
}
112+
113+
const paths = parsedRedact.data.paths;
114+
115+
const redactor = new Redactor(paths);
116+
117+
return redactor.redact(output);
118+
}
95119
}

‎apps/webapp/app/utils/redactor.ts‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Redacts the given object based on the given paths
2+
// Example:
3+
// const redactor = new Redactor(["data.object.balance_transaction"]);
4+
// redactor.redact({
5+
// data: {
6+
// object: {
7+
// balance_transaction: "txn_1NYWgTI0XSgju2urW3aXpinM",
8+
// },
9+
// },
10+
// });
11+
// Returns:
12+
// {
13+
// data: {
14+
// object: {
15+
// balance_transaction: "[REDACTED]",
16+
// },
17+
// },
18+
// }
19+
// Does not currenly support arrays
20+
export class Redactor {
21+
constructor(private paths: string[]) {}
22+
23+
public redact(subject: unknown): unknown {
24+
if (!Array.isArray(this.paths)) {
25+
return subject;
26+
}
27+
28+
if (this.paths.length === 0) {
29+
return subject;
30+
}
31+
32+
const clonedSubject = JSON.parse(JSON.stringify(subject));
33+
34+
return this.redactPathsRecursive(clonedSubject, this.paths);
35+
}
36+
37+
private redactPathsRecursive(subject: any, paths: string[]): any {
38+
for (let path of paths) {
39+
let parts = path.split(".");
40+
41+
let curSubject = subject;
42+
43+
// Make sure curSubject is an object
44+
if (typeof curSubject !== "object") {
45+
break;
46+
}
47+
48+
for (let i = 0; i < parts.length; i++) {
49+
const part = parts[i];
50+
51+
if (Object.prototype.hasOwnProperty.call(curSubject, part) === false) {
52+
// Path is not found in object
53+
break;
54+
}
55+
56+
if (i === parts.length - 1) {
57+
// We're at the end of our path and have a string, redact it
58+
curSubject[part] = "[REDACTED]";
59+
} else if (part in curSubject && typeof curSubject[part] === "object") {
60+
// More paths to follow, continue down the path
61+
curSubject = curSubject[part];
62+
} else {
63+
// Path is not found in object or doesn't point to a string
64+
break;
65+
}
66+
}
67+
}
68+
69+
return subject;
70+
}
71+
}

‎apps/webapp/package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,6 @@
7373
"cuid": "^2.1.8",
7474
"emails": "workspace:*",
7575
"express": "^4.18.1",
76-
"fast-redact": "^3.1.2",
7776
"framer-motion": "^10.12.11",
7877
"graphile-worker": "^0.13.0",
7978
"highlight.run": "^7.3.4",

‎docs/sdk/io/runtask.mdx‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,12 @@ A Task is a resumable unit of a Run that can be retried, resumed and is logged.
8282
</Expandable>
8383
</ResponseField>
8484

85-
<ResponseField name="params" type="any">
86-
The input params to the Task, will be displayed in the logs.
87-
</ResponseField>
85+
{" "}
86+
87+
<ResponseField name="params" type="any">
88+
The input params to the Task, will be displayed in the logs.
89+
</ResponseField>
90+
8891
<ResponseField name="style" type="object">
8992
The style of the log entry.
9093

@@ -98,6 +101,17 @@ A Task is a resumable unit of a Run that can be retried, resumed and is logged.
98101
</Expandable>
99102

100103
</ResponseField>
104+
105+
<ResponseField name="redact" type="RedactOptions">
106+
An optional object that specifies which fields to redact from the logs. This is useful for sensitive data like API keys.
107+
108+
<Expandable title="redact" defaultOpen>
109+
<ResponseField name="paths" type="string[]">
110+
An array of paths to redact. A path is a dot separated string, e.g. `user.email`. Currently does not support wildcards.
111+
</ResponseField>
112+
</Expandable>
113+
114+
</ResponseField>
101115
</Expandable>
102116
</ResponseField>
103117

‎pnpm-lock.yaml‎

Lines changed: 0 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎references/job-catalog/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
"linear": "nodemon --watch src/linear.ts -r tsconfig-paths/register -r dotenv/config src/linear.ts",
2525
"status": "nodemon --watch src/status.ts -r tsconfig-paths/register -r dotenv/config src/status.ts",
2626
"byo-auth": "nodemon --watch src/byo-auth.ts -r tsconfig-paths/register -r dotenv/config src/byo-auth.ts",
27+
"redacted": "nodemon --watch src/redacted.ts -r tsconfig-paths/register -r dotenv/config src/redacted.ts",
2728
"dev:trigger": "trigger-cli dev --port 8080"
2829
},
2930
"dependencies": {

‎references/job-catalog/src/redacted.ts‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
import { createExpressServer } from "@trigger.dev/express";
2+
import { TriggerClient, eventTrigger } from "@trigger.dev/sdk";
3+
import { z } from "zod";
4+
5+
export const client = new TriggerClient({
6+
id: "job-catalog",
7+
apiKey: process.env["TRIGGER_API_KEY"],
8+
apiUrl: process.env["TRIGGER_API_URL"],
9+
verbose: false,
10+
ioLogLocalEnabled: true,
11+
});
12+
13+
client.defineJob({
14+
id: "redaction-example-1",
15+
name: "Redaction Example 1",
16+
version: "1.0.0",
17+
enabled: true,
18+
trigger: eventTrigger({
19+
name: "redaction.example",
20+
}),
21+
run: async (payload, io, ctx) => {
22+
const result = await io.runTask(
23+
"task-example-1",
24+
async () => {
25+
return {
26+
id: "evt_3NYWgVI0XSgju2ur0PN22Hsu",
27+
object: "event",
28+
api_version: "2022-11-15",
29+
created: 1690473903,
30+
data: {
31+
object: {
32+
id: "ch_3NYWgVI0XSgju2ur0C2UzeKC",
33+
object: "charge",
34+
amount: 1500,
35+
amount_captured: 1500,
36+
amount_refunded: 0,
37+
application: null,
38+
application_fee: null,
39+
application_fee_amount: null,
40+
balance_transaction: "txn_3NYWgVI0XSgju2ur0qujz4Kc",
41+
billing_details: {
42+
address: {
43+
city: null,
44+
country: null,
45+
line1: null,
46+
line2: null,
47+
postal_code: null,
48+
state: null,
49+
},
50+
email: null,
51+
name: null,
52+
phone: null,
53+
},
54+
calculated_statement_descriptor: "WWW.TRIGGER.DEV",
55+
captured: true,
56+
created: 1690473903,
57+
currency: "usd",
58+
customer: "cus_OLD6IR3D8CJasG",
59+
description: "Subscription creation",
60+
destination: null,
61+
dispute: null,
62+
disputed: false,
63+
failure_balance_transaction: null,
64+
failure_code: null,
65+
failure_message: null,
66+
fraud_details: {},
67+
invoice: "in_1NYWgUI0XSgju2urV5ZTEyIn",
68+
livemode: false,
69+
metadata: {},
70+
on_behalf_of: null,
71+
order: null,
72+
outcome: {
73+
network_status: "approved_by_network",
74+
reason: null,
75+
risk_level: "normal",
76+
risk_score: 61,
77+
seller_message: "Payment complete.",
78+
type: "authorized",
79+
},
80+
paid: true,
81+
payment_intent: "pi_3NYWgVI0XSgju2ur0fWNLexG",
82+
payment_method: "pm_1NYWgTI0XSgju2urW3aXpinM",
83+
payment_method_details: {
84+
card: {
85+
brand: "visa",
86+
checks: {
87+
address_line1_check: null,
88+
address_postal_code_check: null,
89+
cvc_check: null,
90+
},
91+
country: "US",
92+
exp_month: 7,
93+
exp_year: 2024,
94+
fingerprint: "w6qgKDLO5EbIJ5VZ",
95+
funding: "credit",
96+
installments: null,
97+
last4: "4242",
98+
mandate: null,
99+
network: "visa",
100+
network_token: {
101+
used: false,
102+
},
103+
three_d_secure: null,
104+
wallet: null,
105+
},
106+
type: "card",
107+
},
108+
receipt_email: null,
109+
receipt_number: null,
110+
receipt_url:
111+
"https://pay.stripe.com/receipts/invoices/CAcaFwoVYWNjdF8xTVJtRzRJMFhTZ2p1MnVyKLCriqYGMga_ozxgMkA6LBbrKccthI_hGdug_gXtuu_piRAvzyNVaH_aMq9mUTOl3VdNbfcH7nhFjK08?s=ap",
112+
refunded: false,
113+
review: null,
114+
shipping: null,
115+
source: null,
116+
source_transfer: null,
117+
statement_descriptor: null,
118+
statement_descriptor_suffix: null,
119+
status: "succeeded",
120+
transfer_data: null,
121+
transfer_group: null,
122+
},
123+
},
124+
livemode: false,
125+
pending_webhooks: 2,
126+
request: {
127+
id: "req_vtwGrzB2O98Pnc",
128+
idempotency_key: "215856c0-4f06-48eb-94c6-7ed4e839d7bc",
129+
},
130+
type: "charge.succeeded",
131+
};
132+
},
133+
{
134+
redact: {
135+
paths: [
136+
"data.object.balance_transaction",
137+
"data.object.billing_details",
138+
"data.object.this_does_not_exist",
139+
"data.object.$$$$hello",
140+
],
141+
},
142+
}
143+
);
144+
145+
await io.logger.info("Log.1", { ctx, result });
146+
147+
await io.wait("wait-1", 1);
148+
149+
await io.logger.info("Log.2", { ctx, result });
150+
},
151+
});
152+
153+
createExpressServer(client);

0 commit comments

Comments
 (0)