diff --git a/.server-changes/supervisor-outbound-request-metrics.md b/.server-changes/supervisor-outbound-request-metrics.md new file mode 100644 index 0000000000..c66a499b81 --- /dev/null +++ b/.server-changes/supervisor-outbound-request-metrics.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: improvement +--- + +Improved supervisor observability: it now reports metrics for its outbound requests, making failed calls to upstream services easier to monitor. diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index cf73be90ee..f30203df54 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -22,7 +22,7 @@ import { isKubernetesEnvironment, } from "@trigger.dev/core/v3/serverOnly"; import { createK8sApi, createApiserverMetricsFetcher } from "./clients/kubernetes.js"; -import { collectDefaultMetrics, Gauge, Histogram } from "prom-client"; +import { collectDefaultMetrics, Counter, Gauge, Histogram } from "prom-client"; import { register } from "./metrics.js"; import { PodCleaner } from "./services/podCleaner.js"; import { FailedPodHandler } from "./services/failedPodHandler.js"; @@ -60,6 +60,21 @@ const workloadCreateDuration = new Histogram({ registers: [register], }); +const outboundRequestsTotal = new Counter({ + name: "supervisor_outbound_request_total", + help: "Count of outbound HTTP requests from the supervisor, by target name, method, response status, and outcome (ok, http_error, invalid_response, network_error).", + labelNames: ["name", "method", "status", "outcome"], + registers: [register], +}); + +const outboundRequestDuration = new Histogram({ + name: "supervisor_outbound_request_duration_seconds", + help: "Duration of outbound HTTP requests from the supervisor, by target name and outcome. Includes the HTTP client's internal retries and backoff.", + labelNames: ["name", "outcome"], + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 11, 12.5, 15, 20, 30, 60], + registers: [register], +}); + class ManagedSupervisor { private readonly workerSession: SupervisorSession; private readonly metricsServer?: HttpServer; @@ -322,6 +337,10 @@ class ManagedSupervisor { runNotificationsEnabled: env.TRIGGER_WORKLOAD_API_ENABLED, heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS, sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS, + onHttpRequestComplete: ({ name, method, status, outcome, durationMs }) => { + outboundRequestsTotal.inc({ name, method, status, outcome }); + outboundRequestDuration.observe({ name, outcome }, durationMs / 1000); + }, preDequeue: async () => { // Synchronous, hot-path-safe cached read; false when no monitors are active. const skipForBackpressure = this.backpressureMonitors.some((m) => m.shouldSkipDequeue()); @@ -692,6 +711,18 @@ class ManagedSupervisor { headers.traceparent = traceparent; } + const requestStart = performance.now(); + const record = ( + status: string, + outcome: "ok" | "http_error" | "invalid_response" | "network_error" + ) => { + outboundRequestsTotal.inc({ name: "warm_start", method: "POST", status, outcome }); + outboundRequestDuration.observe( + { name: "warm_start", outcome }, + (performance.now() - requestStart) / 1000 + ); + }; + try { const res = await fetch(warmStartUrlWithPath.href, { method: "POST", @@ -700,8 +731,10 @@ class ManagedSupervisor { }); if (!res.ok) { + record(String(res.status), "http_error"); this.logger.error("Warm start failed", { runId: dequeuedMessage.run.id, + statusCode: res.status, }); return false; } @@ -710,6 +743,7 @@ class ManagedSupervisor { const parsedData = z.object({ didWarmStart: z.boolean() }).safeParse(data); if (!parsedData.success) { + record(String(res.status), "invalid_response"); this.logger.error("Warm start response invalid", { runId: dequeuedMessage.run.id, data, @@ -717,8 +751,11 @@ class ManagedSupervisor { return false; } + record(String(res.status), "ok"); + return parsedData.data.didWarmStart; } catch (error) { + record("none", "network_error"); this.logger.error("Warm start error", { runId: dequeuedMessage.run.id, error, diff --git a/packages/core/src/v3/runEngineWorker/supervisor/http.ts b/packages/core/src/v3/runEngineWorker/supervisor/http.ts index d6d51b1c2b..450be43257 100644 --- a/packages/core/src/v3/runEngineWorker/supervisor/http.ts +++ b/packages/core/src/v3/runEngineWorker/supervisor/http.ts @@ -21,9 +21,9 @@ import { WorkerApiSuspendRunResponseBody, WorkerApiRunSnapshotsSinceResponseBody, } from "./schemas.js"; -import type { SupervisorClientCommonOptions } from "./types.js"; +import type { SupervisorClientCommonOptions, SupervisorHttpRequestMetric } from "./types.js"; import { getDefaultWorkerHeaders } from "./util.js"; -import { wrapZodFetch } from "../../zodfetch.js"; +import { wrapZodFetch, type ApiResult, type ZodFetchOptions } from "../../zodfetch.js"; import { createHeaders } from "../util.js"; import { WORKER_HEADERS } from "../consts.js"; import { SimpleStructuredLogger } from "../../utils/structuredLogger.js"; @@ -36,6 +36,7 @@ export class SupervisorHttpClient { private readonly instanceName: string; private readonly defaultHeaders: Record; private readonly sendRunDebugLogs: boolean; + private readonly onHttpRequestComplete?: (metric: SupervisorHttpRequestMetric) => void; private readonly logger = new SimpleStructuredLogger("supervisor-http-client"); @@ -45,6 +46,7 @@ export class SupervisorHttpClient { this.instanceName = opts.instanceName; this.defaultHeaders = getDefaultWorkerHeaders(opts); this.sendRunDebugLogs = opts.sendRunDebugLogs ?? false; + this.onHttpRequestComplete = opts.onHttpRequestComplete; if (!this.apiUrl) { throw new Error("apiURL is required and needs to be a non-empty string"); @@ -59,8 +61,55 @@ export class SupervisorHttpClient { } } + private async request( + name: string, + schema: T, + url: string, + requestInit?: RequestInit, + options?: ZodFetchOptions> + ): Promise>> { + const start = performance.now(); + const result = await wrapZodFetch(schema, url, requestInit, options); + + if (this.onHttpRequestComplete) { + const durationMs = performance.now() - start; + const method = requestInit?.method ?? "GET"; + + if (result.success) { + this.onHttpRequestComplete({ name, method, status: "2xx", outcome: "ok", durationMs }); + } else if (result.statusCode === 200) { + this.onHttpRequestComplete({ + name, + method, + status: "200", + outcome: "invalid_response", + durationMs, + }); + } else if (typeof result.statusCode === "number") { + this.onHttpRequestComplete({ + name, + method, + status: String(result.statusCode), + outcome: "http_error", + durationMs, + }); + } else { + this.onHttpRequestComplete({ + name, + method, + status: "none", + outcome: "network_error", + durationMs, + }); + } + } + + return result; + } + async connect(body: WorkerApiConnectRequestBody) { - return wrapZodFetch( + return this.request( + "connect", WorkerApiConnectResponseBody, `${this.apiUrl}/engine/v1/worker-actions/connect`, { @@ -75,7 +124,8 @@ export class SupervisorHttpClient { } async dequeue(body: WorkerApiDequeueRequestBody) { - return wrapZodFetch( + return this.request( + "dequeue", WorkerApiDequeueResponseBody, `${this.apiUrl}/engine/v1/worker-actions/dequeue`, { @@ -91,7 +141,8 @@ export class SupervisorHttpClient { /** @deprecated Not currently used */ async dequeueFromVersion(deploymentId: string, maxRunCount = 1, runnerId?: string) { - return wrapZodFetch( + return this.request( + "dequeue_from_version", WorkerApiDequeueResponseBody, `${this.apiUrl}/engine/v1/worker-actions/deployments/${deploymentId}/dequeue?maxRunCount=${maxRunCount}`, { @@ -104,7 +155,8 @@ export class SupervisorHttpClient { } async heartbeatWorker(body: WorkerApiHeartbeatRequestBody) { - return wrapZodFetch( + return this.request( + "heartbeat_worker", WorkerApiHeartbeatResponseBody, `${this.apiUrl}/engine/v1/worker-actions/heartbeat`, { @@ -124,7 +176,8 @@ export class SupervisorHttpClient { body: WorkerApiRunHeartbeatRequestBody, runnerId?: string ) { - return wrapZodFetch( + return this.request( + "heartbeat_run", WorkerApiRunHeartbeatResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/heartbeat`, { @@ -146,7 +199,8 @@ export class SupervisorHttpClient { runnerId?: string, environmentId?: string ) { - return wrapZodFetch( + return this.request( + "start_run_attempt", WorkerApiRunAttemptStartResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/attempts/start`, { @@ -168,7 +222,8 @@ export class SupervisorHttpClient { runnerId?: string, environmentId?: string ) { - return wrapZodFetch( + return this.request( + "complete_run_attempt", WorkerApiRunAttemptCompleteResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/attempts/complete`, { @@ -184,7 +239,8 @@ export class SupervisorHttpClient { } async getLatestSnapshot(runId: string, runnerId?: string, environmentId?: string) { - return wrapZodFetch( + return this.request( + "get_latest_snapshot", WorkerApiRunLatestSnapshotResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/latest`, { @@ -204,7 +260,8 @@ export class SupervisorHttpClient { runnerId?: string, environmentId?: string ) { - return wrapZodFetch( + return this.request( + "get_snapshots_since", WorkerApiRunSnapshotsSinceResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/since/${snapshotId}`, { @@ -224,7 +281,8 @@ export class SupervisorHttpClient { } try { - const res = await wrapZodFetch( + const res = await this.request( + "send_debug_log", z.unknown(), `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/logs/debug`, { @@ -252,7 +310,8 @@ export class SupervisorHttpClient { runnerId?: string, environmentId?: string ) { - return wrapZodFetch( + return this.request( + "continue_run_execution", WorkerApiContinueRunExecutionRequestBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/continue`, { @@ -292,7 +351,8 @@ export class SupervisorHttpClient { runnerId?: string; body: WorkerApiSuspendRunRequestBody; }) { - return wrapZodFetch( + return this.request( + "submit_suspend_completion", WorkerApiSuspendRunResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/suspend`, { diff --git a/packages/core/src/v3/runEngineWorker/supervisor/types.ts b/packages/core/src/v3/runEngineWorker/supervisor/types.ts index 4d0479cbca..aff6ffa7d2 100644 --- a/packages/core/src/v3/runEngineWorker/supervisor/types.ts +++ b/packages/core/src/v3/runEngineWorker/supervisor/types.ts @@ -1,5 +1,13 @@ import type { MachineResources } from "../../schemas/runEngine.js"; +export type SupervisorHttpRequestMetric = { + name: string; + method: string; + status: string; + outcome: "ok" | "http_error" | "invalid_response" | "network_error"; + durationMs: number; +}; + export type SupervisorClientCommonOptions = { apiUrl: string; workerToken: string; @@ -7,6 +15,7 @@ export type SupervisorClientCommonOptions = { deploymentId?: string; managedWorkerSecret?: string; sendRunDebugLogs?: boolean; + onHttpRequestComplete?: (metric: SupervisorHttpRequestMetric) => void; }; export type PreDequeueFn = () => Promise<{