fix(core): jittered retry on the resume hops for transient DB outages

matt-aitken · matt-aitken · commit 988f53d69f8a · 2026-07-05T19:45:45.000+01:00
Restore the extended, jittered retry on the workload and supervisor
continueRunExecution calls so the resume can ride out a transient database
outage. Recorded via .server-changes; no package changeset.
diff --git a/packages/core/src/v3/runEngineWorker/supervisor/http.ts b/packages/core/src/v3/runEngineWorker/supervisor/http.ts
@@ -245,6 +245,21 @@ export class SupervisorHttpClient {
           ...this.defaultHeaders,
           ...this.runnerIdHeader(runnerId),
         },
+      },
+      {
+        // This is the hop that reaches the engine, so it's where a transient
+        // database outage during resume surfaces (as a retryable 5xx). Resuming
+        // is idempotent server-side (guarded by the snapshot id), so retry
+        // generously to ride out the outage rather than aborting the run.
+        // `randomize` jitters the delay so a fleet of runs resuming at once
+        // doesn't stampede the DB the moment it recovers.
+        retry: {
+          minTimeoutInMs: 500,
+          maxTimeoutInMs: 10_000,
+          maxAttempts: 8,
+          factor: 2,
+          randomize: true,
+        },
       }
     );
   }
diff --git a/packages/core/src/v3/runEngineWorker/workload/http.ts b/packages/core/src/v3/runEngineWorker/workload/http.ts
@@ -132,6 +132,20 @@ export class WorkloadHttpClient {
           headers: {
             ...this.defaultHeaders(),
           },
+        },
+        {
+          // This hop only reaches the supervisor's workload server, so retry
+          // generously with jittered backoff to ride out a transient blip
+          // talking to the supervisor (e.g. a restart) rather than aborting the
+          // run. Database outages surface one hop further in, on the
+          // supervisor-to-engine call, which carries its own retry for them.
+          retry: {
+            minTimeoutInMs: 500,
+            maxTimeoutInMs: 10_000,
+            maxAttempts: 8,
+            factor: 2,
+            randomize: true,
+          },
         }
       )
     );
