area: webapp

type: fix

Close the PG+buffer idempotency-key race during the mollifier gate-transition window. Without this, two simultaneous same-key triggers arriving as the gate trips could each become race-winners (one PG, one buffer) — the customer would receive two distinct runIds for the same idempotency key, and operations on the buffered "loser" would silently vanish on drain. Design: `_plans/2026-05-21-mollifier-idempotency-claim.md`.

`IdempotencyKeyConcern.handleTriggerRequest` now does a pre-gate Redis `SETNX` claim after the existing PG + buffer cache checks miss. All same-key triggers serialise through this claim before the gate decides PG-passthrough vs mollify; losers poll until the winner publishes its runId, then return that runId with `isCached:true`. Skipped for `resumeParentOnCompletion` (triggerAndWait bypasses the gate via F4 and is PG-canonical).

`RunEngineTriggerTaskService.callV2` wraps the trigger pipeline in a try/catch around the claim: on success, the winning runId is published to the claim key so waiters resolve; on any pipeline error, the claim is released so the next claimant can retry. Failure to publish/release is logged but non-fatal — the claim TTL (default 30s) is the safety net.

Verified by `scripts/mollifier-challenge/04-idempotency-collision.sh`: 30 cold-gate same-key triggers (no pre-warm) now converge on one runId, one `isCached:false` response, 29 `isCached:true`. Before this fix the same test produced 2 unique runIds and 2 `isCached:false` responses.

import { ServiceValidationError } from "~/v3/services/common.server";

import { claimOrAwait } from "~/v3/mollifier/idempotencyClaim.server";

export type ClaimedIdempotency = {

envId: string;

taskIdentifier: string;

idempotencyKey: string;

| {

isCached: false;

idempotencyKey?: string;

idempotencyKeyExpiresAt?: Date;

// Set when this trigger holds a pre-gate claim. The caller's

// trigger pipeline MUST resolve the claim by either publishing

// the runId on success or releasing on failure. Undefined when

// the request has no idempotency key, when the buffer is

// unavailable, or when the request is a triggerAndWait (claim

// path skipped per plan doc).

claim?: ClaimedIdempotency;

};

// Pre-gate claim — closes the PG+buffer race during gate transition

// (see _plans/2026-05-21-mollifier-idempotency-claim.md). All

// same-key triggers serialise here before evaluateGate decides

// PG-pass-through vs mollify. Skipped for triggerAndWait

// (resumeParentOnCompletion) — that path bypasses the gate via F4

// and its existing PG-side dedup is sufficient.

if (!request.body.options?.resumeParentOnCompletion) {

const ttlSeconds = Math.max(

1,

Math.min(

30,

Math.ceil((idempotencyKeyExpiresAt.getTime() - Date.now()) / 1000),

),

);

const outcome = await claimOrAwait({

envId: request.environment.id,

taskIdentifier: request.taskId,

idempotencyKey,

ttlSeconds,

});

if (outcome.kind === "resolved") {

// Another concurrent trigger committed first. Re-resolve via the

// existing checks: writer-side PG findFirst first (defeats

// replica lag), then buffer fallback for the buffered case.

const writerRun = await this.prisma.taskRun.findFirst({

where: {

runtimeEnvironmentId: request.environment.id,

idempotencyKey,

taskIdentifier: request.taskId,

},

include: { associatedWaitpoint: true },

});

if (writerRun) {

return { isCached: true, run: writerRun };

}

const buffered = await this.findBufferedRunWithIdempotency(

request.environment.id,

request.environment.organizationId,

request.taskId,

idempotencyKey,

);

if (buffered) {

return { isCached: true, run: buffered };

}

// Claim resolved to a runId nothing can find — likely the

// claimant errored after publish, or the row TTL'd out. Log

// and fall through to a fresh trigger.

logger.warn("idempotency claim resolved but runId not findable", {

envId: request.environment.id,

taskIdentifier: request.taskId,

claimedRunId: outcome.runId,

});

}

if (outcome.kind === "timed_out") {

throw new ServiceValidationError(

"Idempotency claim resolution timed out",

503,

);

}

if (outcome.kind === "claimed") {

// Caller MUST publish/release. Signalled via the result's

// `claim` field.

return {

isCached: false,

idempotencyKey,

idempotencyKeyExpiresAt,

claim: {

envId: request.environment.id,

taskIdentifier: request.taskId,

idempotencyKey,

},

};

}

}

import {

IdempotencyKeyConcern,

type ClaimedIdempotency,

} from "../concerns/idempotencyKeys.server";

import {

publishClaim as publishMollifierClaim,

releaseClaim as releaseMollifierClaim,

} from "~/v3/mollifier/idempotencyClaim.server";

// Pre-gate idempotency-claim ownership. Set inside the span when

// `IdempotencyKeyConcern.handleTriggerRequest` returns `claim:

// {...}`. The try/catch below resolves it once the span finishes.

let idempotencyClaim: ClaimedIdempotency | undefined;

try {

const result = await startSpan(

this.tracer,

"RunEngineTriggerTaskService.call()",

async (span) => {

const { idempotencyKey, idempotencyKeyExpiresAt, claim: claimResult } =

idempotencyKeyConcernResult;

// If we own an idempotency claim, the trigger pipeline below MUST

// resolve it — publish on success so waiters see our runId,

// release on error so the next claimant can retry. Stored in an

// outer scope so the try/catch at the bottom of `callV2` can act

// on whichever return path or throw the pipeline takes. Plan doc:

// _plans/2026-05-21-mollifier-idempotency-claim.md

idempotencyClaim = claimResult;

},

);

// Pipeline returned successfully — publish the claim if we held

// one. Waiters polling for our key resolve to this runId.

if (idempotencyClaim && result?.run?.friendlyId) {

await publishMollifierClaim({

envId: idempotencyClaim.envId,

taskIdentifier: idempotencyClaim.taskIdentifier,

idempotencyKey: idempotencyClaim.idempotencyKey,

runId: result.run.friendlyId,

});

}

return result;

} catch (err) {

// Pipeline threw — release the claim so the next claimant can

// retry. Re-throw so the caller sees the original error.

if (idempotencyClaim) {

await releaseMollifierClaim(idempotencyClaim);

}

throw err;

}