triggerdotdev · d-cs · Jul 7, 2026 · Jul 7, 2026 · Jul 7, 2026 · Jul 7, 2026
diff --git a/.server-changes/fix-batch-item-mint-anchor.md b/.server-changes/fix-batch-item-mint-anchor.md
@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Anchor batch item run-ops residency on the batch's own friendlyId (not a fresh per-org flag read) in the run-engine batch trigger service and the BatchQueue item callback, on both the success path and the pre-failed-run failure path, so a mid-batch mint-flag flip can no longer mint an item (or a pre-failed item) into a different physical store than its BatchTaskRun row.
diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts
@@ -290,6 +290,7 @@ export const runOpsSplitReadEnabled: boolean = computeRunOpsSplitReadEnabled({
   controlPlaneReplica: $replica,
   hasNewUrl: !!env.RUN_OPS_DATABASE_URL,
   hasLegacyUrl: !!env.RUN_OPS_LEGACY_DATABASE_URL,
+  logger,
 });
 
 // Boot-time interlock: if the flag is on but the distinct-DB sentinel does not

diff --git a/apps/webapp/app/runEngine/services/batchTrigger.server.ts b/apps/webapp/app/runEngine/services/batchTrigger.server.ts
@@ -18,6 +18,7 @@ import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { batchTriggerWorker } from "~/v3/batchTriggerWorker.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
+import { mintAnchoredRunFriendlyId } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
 import { mintBatchFriendlyId } from "~/v3/runOpsMigration/mintBatchFriendlyId.server";
 import {
   downloadPacketFromObjectStore,
@@ -588,6 +589,9 @@ export class RunEngineBatchTriggerService extends WithRunEngine {
           parentRunId,
           resumeParentOnCompletion,
           batch: { id: batch.id, index: workingIndex },
+          // Anchor the pre-failed run on the BATCH's residency (same as the happy path) so it
+          // co-resides with its BatchTaskRun row regardless of a mid-batch mint-flag flip.
+          runFriendlyId: mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region),
           options: item.options as Record<string, unknown>,
           traceContext: options?.traceContext as Record<string, unknown> | undefined,
           spanParentAsLink: options?.spanParentAsLink,
@@ -677,6 +681,12 @@ export class RunEngineBatchTriggerService extends WithRunEngine {
 
     const triggerTaskService = new TriggerTaskService();
 
+    // Anchor the item's mint on the BATCH's own friendlyId (not a fresh per-org flag read) so
+    // an org's mint flag flipping between batch creation and this (possibly much later,
+    // worker-processed) item never splits the item from its BatchTaskRun row. See
+    // mintAnchoredRunFriendlyId.server.ts.
+    const runFriendlyId = mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region);
+
     const result = await triggerTaskService.call(
       item.task,
       environment,
@@ -695,6 +705,7 @@ export class RunEngineBatchTriggerService extends WithRunEngine {
         spanParentAsLink: options?.spanParentAsLink,
         batchId: batch.id,
         batchIndex: currentIndex,
+        runFriendlyId,
         realtimeStreamsVersion: options?.realtimeStreamsVersion,
         triggerSource: options?.triggerSource ?? "api",
         triggerAction: options?.triggerAction ?? "trigger",

diff --git a/apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts b/apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts
@@ -0,0 +1,42 @@
+import { describe, expect, it, vi } from "vitest";
+
+// Empty singletons satisfy the module-level wiring imports; the mint method under test is driven
+// directly via (service as any) and never touches the DB (same boundary as triggerTask.server.test.ts).
+vi.mock("~/db.server", () => ({
+  prisma: {},
+  $replica: {},
+  runOpsNewPrisma: {},
+  runOpsLegacyPrisma: {},
+  runOpsNewReplica: {},
+  runOpsLegacyReplica: {},
+}));
+vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false }));
+
+import { classifyKind, generateRunOpsId, RunId } from "@trigger.dev/core/v3/isomorphic";
+import { TriggerFailedTaskService } from "./triggerFailedTask.server";
+
+function buildService() {
+  return new TriggerFailedTaskService({ prisma: {} as any, engine: {} as any });
+}
+
+describe("TriggerFailedTaskService.mintFailedRunFriendlyId", () => {
+  it("returns the caller-supplied runFriendlyId verbatim (override wins over any mint)", async () => {
+    const override = RunId.toFriendlyId(generateRunOpsId());
+    const minted = await (buildService() as any).mintFailedRunFriendlyId({
+      organizationId: "org_1",
+      environmentId: "env_1",
+      runFriendlyId: override,
+    });
+    expect(minted).toBe(override);
+  });
+
+  it("without an override, still inherits a run-ops (NEW) parent by id-shape", async () => {
+    const parentRunFriendlyId = RunId.toFriendlyId(generateRunOpsId());
+    const minted = await (buildService() as any).mintFailedRunFriendlyId({
+      organizationId: "org_1",
+      environmentId: "env_1",
+      parentRunFriendlyId,
+    });
+    expect(classifyKind(minted)).toBe("runOpsId");
+  });
+});
diff --git a/apps/webapp/app/runEngine/services/triggerFailedTask.server.ts b/apps/webapp/app/runEngine/services/triggerFailedTask.server.ts
@@ -43,6 +43,9 @@ export type TriggerFailedTaskRequest = {
   spanParentAsLink?: boolean;
 
   errorCode?: TaskRunErrorCodes;
+
+  /** Pre-minted friendlyId; when set it wins over the mint. Batch callers pass a batch-anchored id. */
+  runFriendlyId?: string;
 };
 
 /**
@@ -86,14 +89,20 @@ export class TriggerFailedTaskService {
 
   // Mint a failed run's friendlyId. The id-kind decides which store the run is
   // born in (cuid → legacy store, run-ops id → new store); the whole subgraph of a
-  // run must agree. Root failed runs mint by the environment's setting; child
-  // failed runs inherit the parent's current store so they never split.
+  // run must agree. A caller-supplied runFriendlyId (batch-anchored id) wins verbatim;
+  // otherwise root failed runs mint by the environment's setting and child failed runs
+  // inherit the parent's current store so they never split.
   private async mintFailedRunFriendlyId(args: {
     organizationId: string;
     environmentId: string;
     orgFeatureFlags?: unknown;
     parentRunFriendlyId?: string;
+    runFriendlyId?: string;
   }): Promise<string> {
+    if (args.runFriendlyId) {
+      return args.runFriendlyId;
+    }
+
     const mintKind = args.parentRunFriendlyId
       ? resolveInheritedMintKind(args.parentRunFriendlyId)
       : await resolveRunIdMintKind({
@@ -125,6 +134,7 @@ export class TriggerFailedTaskService {
         environmentId: request.environment.id,
         orgFeatureFlags: request.environment.organization.featureFlags,
         parentRunFriendlyId: request.parentRunId,
+        runFriendlyId: request.runFriendlyId,
       });
       mintedFriendlyId = failedRunFriendlyId;
 
@@ -321,6 +331,8 @@ export class TriggerFailedTaskService {
     resumeParentOnCompletion?: boolean;
     batch?: { id: string; index: number };
     errorCode?: TaskRunErrorCodes;
+    /** Pre-minted friendlyId; when set it wins over the mint. Batch callers pass a batch-anchored id. */
+    runFriendlyId?: string;
   }): Promise<string | null> {
     // Held for the catch's log line; the in-try `const` is what consumers use.
     let mintedFriendlyId: string | undefined;
@@ -335,6 +347,7 @@ export class TriggerFailedTaskService {
         // single replica lookup by organizationId only when there is no parent.
         orgFeatureFlags: undefined,
         parentRunFriendlyId: opts.parentRunId,
+        runFriendlyId: opts.runFriendlyId,
       });
       mintedFriendlyId = failedRunFriendlyId;
 

diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.ts b/apps/webapp/app/runEngine/services/triggerTask.server.ts
@@ -14,7 +14,6 @@ import {
   TriggerTraceContext,
 } from "@trigger.dev/core/v3";
 import {
-  generateRunOpsId,
   parseTraceparent,
   RunId,
   serializeTraceparent,
@@ -28,6 +27,7 @@ import { handleMetadataPacket } from "~/utils/packets";
 import { startSpan } from "~/v3/tracing.server";
 import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
 import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server";
+import { mintFriendlyIdForKind } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
 import type {
   TriggerTaskServiceOptions,
   TriggerTaskServiceResult,
@@ -153,9 +153,7 @@ export class RunEngineTriggerTaskService {
           orgFeatureFlags: environment.organization.featureFlags,
         });
 
-    return mintKind === "runOpsId"
-      ? RunId.toFriendlyId(generateRunOpsId(region))
-      : RunId.generate().friendlyId;
+    return mintFriendlyIdForKind(mintKind, region);
   }
 
   public async call({

diff --git a/apps/webapp/app/v3/runEngineHandlers.server.ts b/apps/webapp/app/v3/runEngineHandlers.server.ts
@@ -28,6 +28,7 @@ import { getEventRepositoryForStore, recordRunDebugLog } from "./eventRepository
 import { roomFromFriendlyRunId, socketIo } from "./handleSocketIo.server";
 import { engine } from "./runEngine.server";
 import { runStore } from "./runStore.server";
+import { mintAnchoredRunFriendlyId } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
 import { isSplitEnabled } from "~/v3/runOpsMigration/splitMode.server";
 import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server";
 import {
@@ -808,6 +809,14 @@ export function setupBatchQueueCallbacks() {
           },
         },
         async (span) => {
+          // Anchor every item mint on the BATCH's friendlyId so a mid-batch mint-flag flip
+          // can't split an item (or pre-failed item) from its BatchTaskRun row.
+          const mintItemRunFriendlyId = () =>
+            mintAnchoredRunFriendlyId(
+              friendlyId,
+              (item.options as { region?: string } | undefined)?.region
+            );
+
           const triggerFailedTaskService = new TriggerFailedTaskService({
             prisma,
             engine,
@@ -837,6 +846,7 @@ export function setupBatchQueueCallbacks() {
                 parentRunId: meta.parentRunId,
                 resumeParentOnCompletion: meta.resumeParentOnCompletion,
                 batch: { id: batchId, index: itemIndex },
+                runFriendlyId: mintItemRunFriendlyId(),
                 traceContext: meta.traceContext as Record<string, unknown> | undefined,
                 spanParentAsLink: meta.spanParentAsLink,
               });
@@ -874,6 +884,8 @@ export function setupBatchQueueCallbacks() {
             // Normalize payload - for application/store (R2 paths), this passes through as-is
             const payload = normalizePayload(item.payload, item.payloadType);
 
+            const runFriendlyId = mintItemRunFriendlyId();
+
             const result = await triggerTaskService.call(
               item.task,
               environment,
@@ -893,6 +905,7 @@ export function setupBatchQueueCallbacks() {
                 spanParentAsLink: meta.spanParentAsLink,
                 batchId,
                 batchIndex: itemIndex,
+                runFriendlyId,
                 realtimeStreamsVersion: meta.realtimeStreamsVersion,
                 planType: meta.planType,
                 triggerSource: meta.parentRunId ? "sdk" : (meta.triggerSource ?? "api"),
@@ -929,6 +942,7 @@ export function setupBatchQueueCallbacks() {
                   parentRunId: meta.parentRunId,
                   resumeParentOnCompletion: meta.resumeParentOnCompletion,
                   batch: { id: batchId, index: itemIndex },
+                  runFriendlyId: mintItemRunFriendlyId(),
                   options: item.options as Record<string, unknown>,
                   traceContext: meta.traceContext as Record<string, unknown> | undefined,
                   spanParentAsLink: meta.spanParentAsLink,
@@ -1009,6 +1023,7 @@ export function setupBatchQueueCallbacks() {
                 parentRunId: meta.parentRunId,
                 resumeParentOnCompletion: meta.resumeParentOnCompletion,
                 batch: { id: batchId, index: itemIndex },
+                runFriendlyId: mintItemRunFriendlyId(),
                 options: item.options as Record<string, unknown>,
                 traceContext: meta.traceContext as Record<string, unknown> | undefined,
                 spanParentAsLink: meta.spanParentAsLink,

diff --git a/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts b/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts
@@ -0,0 +1,31 @@
+import {
+  BatchId,
+  classifyKind,
+  generateRunOpsId,
+  parseRunId,
+  REGION_CODES,
+} from "@trigger.dev/core/v3/isomorphic";
+import { describe, expect, it } from "vitest";
+import { mintAnchoredRunFriendlyId } from "./mintAnchoredRunFriendlyId.server";
+
+describe("mintAnchoredRunFriendlyId", () => {
+  it("a run-ops (NEW) batch anchor yields a run-ops (NEW) item friendlyId", () => {
+    const batchFriendlyId = BatchId.toFriendlyId(generateRunOpsId());
+    const itemFriendlyId = mintAnchoredRunFriendlyId(batchFriendlyId);
+    expect(classifyKind(itemFriendlyId)).toBe("runOpsId");
+  });
+
+  it("a cuid (LEGACY) batch anchor yields a cuid (LEGACY) item friendlyId", () => {
+    const batchFriendlyId = BatchId.generate().friendlyId;
+    const itemFriendlyId = mintAnchoredRunFriendlyId(batchFriendlyId);
+    expect(classifyKind(itemFriendlyId)).toBe("cuid");
+  });
+
+  it("stamps the requested region char into a run-ops id", () => {
+    const batchFriendlyId = BatchId.toFriendlyId(generateRunOpsId());
+    const itemFriendlyId = mintAnchoredRunFriendlyId(batchFriendlyId, "us-east-1");
+    const parsed = parseRunId(itemFriendlyId);
+    expect(parsed.format).toBe("b32hex");
+    expect(parsed.format === "b32hex" && parsed.region).toBe(REGION_CODES["us-east-1"]);
+  });
+});
diff --git a/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.ts b/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.ts
@@ -0,0 +1,15 @@
+import { generateRunOpsId, RunId, type ResidencyKind } from "@trigger.dev/core/v3/isomorphic";
+import { resolveInheritedMintKind } from "./resolveInheritedMintKind.server";
+
+// Shared id-generation branch for every run-mint path: "runOpsId" -> NEW store, "cuid" -> LEGACY.
+export function mintFriendlyIdForKind(mintKind: ResidencyKind, region?: string): string {
+  return mintKind === "runOpsId"
+    ? RunId.toFriendlyId(generateRunOpsId(region))
+    : RunId.generate().friendlyId;
+}
+
+// Anchor a batch item's mint on the BATCH's friendlyId (id-shape, zero I/O), never the per-org
+// flag, so the item and its BatchTaskRun stay co-resident across a mid-batch flag flip.
+export function mintAnchoredRunFriendlyId(batchFriendlyId: string, region?: string): string {
+  return mintFriendlyIdForKind(resolveInheritedMintKind(batchFriendlyId), region);
+}
diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsSplitReadGate.ts b/apps/webapp/app/v3/runOpsMigration/runOpsSplitReadGate.ts
@@ -6,9 +6,21 @@ export function computeRunOpsSplitReadEnabled(args: {
   controlPlaneReplica: unknown;
   hasNewUrl: boolean;
   hasLegacyUrl: boolean;
+  logger?: { warn: (msg: string, meta?: Record<string, unknown>) => void };
 }): boolean {
   const newIsDistinctDedicatedClient =
     args.newReplica !== args.controlPlaneWriter && args.newReplica !== args.controlPlaneReplica;
 
-  return newIsDistinctDedicatedClient && args.hasNewUrl && args.hasLegacyUrl;
+  const enabled = newIsDistinctDedicatedClient && args.hasNewUrl && args.hasLegacyUrl;
+
+  // Configured for split but the identity check failed: fan-out is being silently disabled.
+  if (!newIsDistinctDedicatedClient && args.hasNewUrl && args.hasLegacyUrl) {
+    args.logger?.warn(
+      "run-ops split read fan-out is configured (RUN_OPS_DATABASE_URL and " +
+        "RUN_OPS_LEGACY_DATABASE_URL are both set) but the NEW client is not a distinct " +
+        "instance from the control-plane client; read fan-out is silently disabled."
+    );
+  }
+
+  return enabled;
 }
diff --git a/apps/webapp/app/v3/services/batchTriggerV3.server.ts b/apps/webapp/app/v3/services/batchTriggerV3.server.ts
@@ -12,7 +12,6 @@ import {
   Prisma,
 } from "@trigger.dev/database";
 import type { RunStore } from "@internal/run-store";
-import { generateRunOpsId, RunId } from "@trigger.dev/core/v3/isomorphic";
 import { z } from "zod";
 import type { PrismaClientOrTransaction } from "~/db.server";
 import { prisma } from "~/db.server";
@@ -26,6 +25,7 @@ import { getEntitlement } from "~/services/platform.v3.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
 import { resolveRunIdMintKind, type RunIdMintKind } from "~/v3/engineVersion.server";
 import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server";
+import { mintFriendlyIdForKind } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
 import { mintBatchFriendlyId } from "~/v3/runOpsMigration/mintBatchFriendlyId.server";
 import { batchTriggerWorker } from "../batchTriggerWorker.server";
 import { legacyRunEngineWorker } from "../legacyRunEngineWorker.server";
@@ -361,9 +361,7 @@ export class BatchTriggerV3Service extends BaseService {
           orgFeatureFlags: environment.organization.featureFlags,
         });
 
-    return mintKind === "runOpsId"
-      ? RunId.toFriendlyId(generateRunOpsId(region))
-      : RunId.generate().friendlyId;
+    return mintFriendlyIdForKind(mintKind, region);
   }
 
   async #prepareRunData(