diff --git a/docs/superpowers/plans/2026-07-23-fair-queueing-ck-spike.md b/docs/superpowers/plans/2026-07-23-fair-queueing-ck-spike.md new file mode 100644 index 0000000000..72bbe9d125 --- /dev/null +++ b/docs/superpowers/plans/2026-07-23-fair-queueing-ck-spike.md @@ -0,0 +1,69 @@ +# Per-concurrency-key fairness spike Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development or executing-plans. Steps use checkbox syntax. + +**Goal:** Rank baseline (age-order) vs SFQ/DRR/stride/CoDel on per-concurrency-key fairness by driving the real CK-dequeue Lua and controlling `ckIndex` scores. + +**Architecture:** Enqueue runs across many concurrency keys under one base queue via the real `RunQueue`. For candidates, rewrite the `ckIndex` ZSET scores before each dequeue round so the unmodified `dequeueMessagesFromCkQueueTracked` Lua serves keys in the discipline's order; baseline leaves scores untouched (production). Reuse the base-queue spike's workload and metrics. + +**Tech Stack:** TypeScript, vitest, `@internal/testcontainers`, `@internal/run-engine` (RunQueue, keyProducer), `@internal/redis`. + +## Global Constraints + +- All code under `internal-packages/run-engine/src/run-queue/fairness-spike-ck/`. Reuse `../fairness-spike/harness/{workload,metrics}.ts` and disciplines where they transfer. Throwaway; delete before merge. +- Fairness group = concurrency key. One org/project/env, one base queue, many CKs. +- No production Lua edits. The `ckIndex` rescore sweep is the only affordance; discipline state advances via an `onServiced(concurrencyKey)` hook. +- Determinism: seeded workload; logical clock owned by the driver. +- Verify: `cd internal-packages/run-engine && pnpm run test ./src/run-queue/fairness-spike-ck/ --run`. +- Commit with `but` on `chore/fair-queueing-ck-spike` (stacked on `chore/fair-queueing-spike`). + +--- + +### Task 1: ckReader — read ckIndex members + head scores + +**Files:** Create `fairness-spike-ck/ckReader.ts`; Test `fairness-spike-ck/tests/ckReader.test.ts`. + +**Produces:** `type ActiveCk = { ckQueue: string; concurrencyKey: string; headScore: number }`; `class CkReader { constructor(redis, keys); readActiveCks(baseQueue): Promise }`. Reads `ZRANGE ckIndexKey WITHSCORES` for the base queue; member is the prefixless CK-queue name; `concurrencyKey` from `descriptorFromQueue(keyPrefix+member)`. + +- [ ] Write `redisTest`: enqueue 2 runs with concurrencyKeys "ck1","ck2" under one base queue via real RunQueue; assert readActiveCks returns both with numeric headScores. First discover the exact ckIndex member format by dumping `ZRANGE ckIndexKey 0 -1` (grep enqueueMessageCkTracked Lua for the `ZADD ckIndexKey` member). +- [ ] Implement; run; commit. + +### Task 2: ckRescorer — write discipline order into ckIndex + +**Files:** Create `fairness-spike-ck/ckRescorer.ts`; Test `tests/ckRescorer.test.ts`. + +**Produces:** `async function rescoreCkIndex(redis, keys, baseQueue, order: string[] /* ckQueues, highest priority first */, now: number)`. Assigns scores `now - (order.length - i)` so the highest-priority CK gets the smallest (oldest) score, all `<= now`, order-preserving. ZADD each member. + +- [ ] Write `redisTest`: enqueue CKs, rescore with a chosen order, assert `ZRANGEBYSCORE ckIndexKey -inf now` returns that order. +- [ ] Implement; run; commit. + +### Task 3: CK disciplines + +**Files:** Create `fairness-spike-ck/disciplines.ts`; Test `tests/disciplines.test.ts`. + +**Produces:** a `CkDiscipline` interface `{ name; order(active: ActiveCk[]): string[] /* ckQueues, best first */; onServiced(concurrencyKey): void; reset() }` with `Sfq`, `Drr`, `Stride` keyed by concurrency key, plus `Codel` wrapper and `Baseline` (order = by headScore asc, i.e. no rescore needed). Reuse the monotonic-floor SFQ/stride and DRR logic from the base-queue spike (import or copy the core, keyed by concurrencyKey). + +- [ ] Unit tests mirroring the base-queue spike (under-served key first; 3:1 weight ratio if weights used; CoDel hoist/revert). +- [ ] Implement; run; commit. + +### Task 4: ckDriver — enqueue, rescore, real dequeue, hold, ack + +**Files:** Create `fairness-spike-ck/harness/ckDriver.ts`; Test `tests/ckDriver.smoke.test.ts`; fidelity test `tests/fidelity.test.ts`. + +**Produces:** `runCkScenario(config): Promise` reusing `../fairness-spike/harness/metrics.ts`. Loop per instant: release holds (ack), enqueue arrivals (with concurrencyKey), for candidate: compute discipline order over active CKs and `rescoreCkIndex`; call `testDequeueFromMasterQueue`; record per-key events; `onServiced`; hold. Baseline skips the rescore. + +- [ ] Fidelity test: baseline path per-key dequeue counts equal a direct `testDequeueFromMasterQueue`-only run (no rescore) — confirms the harness matches production age-order. +- [ ] Smoke: balanced CKs drain fully and no key starves under a fair discipline. +- [ ] Implement; run; commit. + +### Task 5: scenarios + bench + FINDINGS + +**Files:** Create `fairness-spike-ck/harness/ckScenarios.ts`, `ckFairnessSpike.bench.test.ts`, `FINDINGS.md`. + +- [ ] Scenarios ckSkew / ckBalanced / ckTrickle, multi-seed (3), five selectors; write results JSON; print table. +- [ ] Run the matrix; sanity-check baseline starves under ckSkew and at least one candidate fixes it (or record if not). +- [ ] Write FINDINGS with proof/disproof per discipline at the CK grain and whether the base-queue ranking carried over. Commit. + +## Self-Review + +Covers the spec: ckReader (T1), rescorer (T2), disciplines incl. baseline (T3), driver + fidelity anchor (T4), scenarios/bench/findings (T5). Grain = concurrency key throughout. Reuses workload/metrics from the base-queue spike. The rescore affordance and onServiced seam are Global Constraints. diff --git a/docs/superpowers/plans/2026-07-23-fair-queueing-spike.md b/docs/superpowers/plans/2026-07-23-fair-queueing-spike.md new file mode 100644 index 0000000000..c0c72bdcbf --- /dev/null +++ b/docs/superpowers/plans/2026-07-23-fair-queueing-spike.md @@ -0,0 +1,326 @@ +# Fair-queueing scheduler spike Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a throwaway bench that ranks four fair-queueing selectors (SFQ, hierarchical DRR, stride, CoDel wrapper) against the current `FairQueueSelectionStrategy` on real Redis, and produce a proof/disproof verdict per mechanism. + +**Architecture:** Each selector implements the real `RunQueueSelectionStrategy` interface and drops into a real `RunQueue` running against a testcontainers Redis. A synchronous driver enqueues synthetic runs across groups, dequeues via `RunQueue.testDequeueFromMasterQueue` (which runs the strategy and does real concurrency gating in Lua), holds a concurrency slot for a sampled duration, then acks. The driver records every dequeue event and feeds serviced descriptors back to stateful selectors via an `onServiced` hook. A metrics module turns recorded events into fairness/latency/cost numbers; a bench test runs the selector-by-scenario matrix and prints a ranking table. + +**Tech Stack:** TypeScript, vitest, `@internal/testcontainers` (redisTest), `@internal/run-engine` internals (`RunQueue`, `RunQueueFullKeyProducer`, `FairQueueSelectionStrategy`), ioredis via `@internal/redis`, `seedrandom`. + +## Discovery (pre-implementation, 2026-07-23) + +Reading the real code before coding turned up the single most important fact for +this whole effort, so it is recorded here rather than buried in FINDINGS. + +The `RunQueueSelectionStrategy` interface cannot express per-concurrency-key +fairness. `FairQueueSelectionStrategy.#allChildQueuesByScore` +(`fairQueueSelectionStrategy.ts:526`) reads the master queue members verbatim, +and for concurrency-keyed runs the enqueue path writes a single CK-*wildcard* +entry per base queue to the master queue (`index.ts:1899`, the #3219 change). The +per-CK pick happens later, inside the `dequeueMessagesFromCkQueueTracked` Lua +(`index.ts:4147`): `ckIndexKey` is a ZSET of CK-queues scored by head-message +timestamp, and the Lua selects them oldest-first +(`ZRANGEBYSCORE ckIndexKey -inf currentTime`). That age-ordering is the #2617 +unfairness, and it sits below the selection-strategy interface, in Lua. + +Decision: the spike stays behind the real strategy interface (the harness choice) +and sets the fairness grain to distinct base queues within one environment (one +base queue per group/tenant, no concurrency key). Non-CK enqueue writes each base +queue straight to the master queue scored by its earliest message +(`index.ts:3037`), so the strategy orders per-tenant base queues directly. The +four disciplines are grain-agnostic, so this is a real proof/disproof of each +mechanism against the real RunQueue with real concurrency gating. The exact +per-CK grain would require spiking the CK-dequeue Lua / `ckIndex` scoring +instead; that is a documented follow-on, and "per-CK fairness lives below the +strategy interface" is itself a headline finding for FINDINGS. + +## Global Constraints + +- All spike code lives under `internal-packages/run-engine/src/run-queue/fairness-spike/`. It is throwaway and ships nothing. +- Fairness grain is the base queue name (the groupId). Each group is one distinct base queue (no concurrency key) in one shared environment. See the Discovery note for why this grain, not the concurrency key. +- No changes to production files. Do not edit `index.ts`, `fairQueueSelectionStrategy.ts`, or any file outside `fairness-spike/`. +- Selectors advance internal state only via the `onServiced` hook, never by editing production Lua. +- Determinism: every random draw goes through a seeded `seedrandom` instance. No `Date.now()` for ordering decisions inside selectors; the driver owns a logical clock and passes timestamps in. +- Single Redis shard (`shardCount: 1`). Multi-shard fairness is out of scope. +- Verify with: `cd internal-packages/run-engine && pnpm run test ./src/run-queue/fairness-spike/ --run` (may need `pnpm run build --filter @internal/run-engine` first). +- Commit with GitButler (`but`) onto branch `chore/fair-queueing-spike`, one commit per task. + +--- + +### Task 1: Spike scaffolding and shared types + +**Files:** +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/types.ts` +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/queueReader.ts` +- Test: `internal-packages/run-engine/src/run-queue/fairness-spike/tests/queueReader.test.ts` + +**Interfaces:** +- Consumes: `RunQueueSelectionStrategy`, `QueueDescriptor`, `EnvQueues`, `RunQueueKeyProducer` from `../types.js`; `Redis` from `@internal/redis`. +- Produces: + - `type GroupId = string` + - `interface SpikeSelectionStrategy extends RunQueueSelectionStrategy { readonly name: string; onServiced(descriptor: QueueDescriptor, now: number): void | Promise; reset?(): void | Promise; }` + - `type ActiveQueue = { queue: string; env: EnvDescriptor; groupId: GroupId; headScore: number | undefined }` + - `type WeightFn = (groupId: GroupId) => number` (default returns 1) + - `class SpikeQueueReader { constructor(redis: Redis, keys: RunQueueKeyProducer); readActiveQueues(parentQueue: string): Promise }` + +`SpikeQueueReader.readActiveQueues` reads the master queue ZSET members (base queue keys) for the parent, and for each queue reads its head via `ZRANGE queue 0 0 WITHSCORES` to get `headScore` (the oldest enqueue timestamp). `groupId` is `keys.descriptorFromQueue(queue).queue` (the base queue name). `env` is built from the descriptor. Queues with no head (empty) are dropped. + +- [ ] **Step 1: Write the failing test** — `queueReader.test.ts` using `redisTest`. Enqueue two messages with different base queue names (`"task/g1"`, `"task/g2"`), no concurrency key, into one prod env via a real `RunQueue` (reuse the construction pattern from `../index.test.ts` lines 74-93, `shardCount: 1`, `masterQueueConsumersDisabled: true`). Then construct `SpikeQueueReader` on a raw `createRedisClient` with the same `keyPrefix`, call `readActiveQueues(keys.masterQueueKeyForShard(0))`, and assert it returns two `ActiveQueue`s with `groupId` `"task/g1"` and `"task/g2"`, each with a numeric `headScore`. + +```ts +const active = await reader.readActiveQueues(testOptions.keys.masterQueueKeyForShard(0)); +expect(active.map((a) => a.groupId).sort()).toEqual(["g1", "g2"]); +expect(active.every((a) => typeof a.headScore === "number")).toBe(true); +``` + +- [ ] **Step 2: Run test, verify it fails** — `pnpm run test ./src/run-queue/fairness-spike/tests/queueReader.test.ts --run`. Expected: FAIL (module not found). +- [ ] **Step 3: Implement `types.ts` and `queueReader.ts`.** `readActiveQueues`: + +```ts +async readActiveQueues(parentQueue: string): Promise { + const queues = await this.redis.zrange(parentQueue, 0, -1); + const out: ActiveQueue[] = []; + for (const queue of queues) { + const head = await this.redis.zrange(queue, 0, 0, "WITHSCORES"); + if (head.length < 2) continue; + const d = this.keys.descriptorFromQueue(queue); + out.push({ + queue, + env: { orgId: d.orgId, projectId: d.projectId, envId: d.envId }, + groupId: d.queue, + headScore: Number(head[1]), + }); + } + return out; +} +``` + +Note: because the grain is base queues (no concurrency key, per the Discovery note), the master queue holds one plain entry per base queue and no CK-wildcard expansion is needed. Skip any queue where `keys.isCkWildcard(queue)` is true (there should be none in the spike workloads); if one appears, it means a workload leaked a concurrency key. + +- [ ] **Step 4: Run test, verify it passes.** +- [ ] **Step 5: Commit** — `but commit chore/fair-queueing-spike -m "test(run-engine): spike queue reader over master queue" --changes ` + +--- + +### Task 2: Seeded workload generator + +**Files:** +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/harness/workload.ts` +- Test: `internal-packages/run-engine/src/run-queue/fairness-spike/tests/workload.test.ts` + +**Interfaces:** +- Consumes: `seedrandom` (already a dep of `fairQueueSelectionStrategy.ts`). +- Produces: + - `type GroupSpec = { groupId: GroupId; runCount: number; weight: number; enqueueAtMs: number[]; holdMs: () => number }` + - `type WorkloadSpec = { seed: string; envConcurrencyLimit: number; groups: GroupSpec[] }` + - `function buildWorkload(config: WorkloadConfig): WorkloadSpec` where `WorkloadConfig = { seed: string; envConcurrencyLimit: number; groups: Array<{ groupId: string; runCount: number; weight?: number; arrival?: "immediate" | "poisson"; ratePerSec?: number; holdMsMean?: number }> }` + - `type EnqueueEvent = { groupId: GroupId; runId: string; enqueueAtMs: number }` + - `function expandEvents(spec: WorkloadSpec): EnqueueEvent[]` (flattened, sorted by `enqueueAtMs`, stable by groupId) + +`holdMs` samples an exponential hold from the seeded rng around `holdMsMean` (default 50). `arrival: "immediate"` sets all `enqueueAtMs` to 0; `"poisson"` spaces them by exponential inter-arrival from `ratePerSec`. `runId` is `${groupId}-${i}`. + +- [ ] **Step 1: Write the failing test.** Assert determinism and shape: two `buildWorkload` calls with the same seed produce identical `expandEvents` output (deep equal); a group with `runCount: 5, arrival: "immediate"` yields 5 events all at `enqueueAtMs === 0`; total event count equals sum of `runCount`. + +```ts +const a = expandEvents(buildWorkload(cfg)); +const b = expandEvents(buildWorkload(cfg)); +expect(a).toEqual(b); +expect(a).toHaveLength(cfg.groups.reduce((n, g) => n + g.runCount, 0)); +``` + +- [ ] **Step 2: Run test, verify it fails.** +- [ ] **Step 3: Implement `workload.ts`** with a single `seedrandom(config.seed)` instance threaded through all draws. Exponential sample: `-Math.log(1 - rng()) * meanMs`. +- [ ] **Step 4: Run test, verify it passes.** +- [ ] **Step 5: Commit.** + +--- + +### Task 3: Metrics module + +**Files:** +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/harness/metrics.ts` +- Test: `internal-packages/run-engine/src/run-queue/fairness-spike/tests/metrics.test.ts` + +**Interfaces:** +- Produces: + - `type DequeueEvent = { groupId: GroupId; runId: string; enqueueAtMs: number; dequeueAtMs: number }` + - `type GroupMetrics = { groupId: GroupId; dequeued: number; weight: number; share: number; shareOverWeight: number; waitP50: number; waitP99: number; waitMax: number }` + - `type RunMetrics = { perGroup: GroupMetrics[]; jainIndex: number; worstShareOverWeight: number; totalDequeued: number; redisOps: number; wallClockMs: number }` + - `function computeMetrics(input: { events: DequeueEvent[]; weights: Record; redisOps: number; wallClockMs: number }): RunMetrics` + +`share` = group dequeued / total dequeued. `shareOverWeight` = share / (weight / sumWeights). Jain index over the `shareOverWeight` vector: `(Σx)² / (n·Σx²)`. `waitP*` are percentiles of `dequeueAtMs - enqueueAtMs` per group. `worstShareOverWeight` = min across groups. + +- [ ] **Step 1: Write the failing test.** Feed a hand-built event set: two groups, equal weight, group A dequeued 8, group B dequeued 2. Assert `share` A = 0.8, B = 0.2; `worstShareOverWeight` ≈ 0.4; Jain index for `[1.6, 0.4]` ≈ `(2.0)² / (2·(2.56+0.16))` = `4 / 5.44` ≈ 0.735. Assert a known percentile from a fixed wait array. +- [ ] **Step 2: Run test, verify it fails.** +- [ ] **Step 3: Implement `metrics.ts`.** Percentile via sorted nearest-rank: `sorted[Math.min(sorted.length - 1, Math.ceil(p/100 * sorted.length) - 1)]`. +- [ ] **Step 4: Run test, verify it passes.** +- [ ] **Step 5: Commit.** + +--- + +### Task 4: Driver + baseline smoke (proves the harness on the real strategy) + +**Files:** +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/harness/driver.ts` +- Test: `internal-packages/run-engine/src/run-queue/fairness-spike/tests/driver.smoke.test.ts` + +**Interfaces:** +- Consumes: `RunQueue`, `RunQueueFullKeyProducer`, `FairQueueSelectionStrategy`, workload + metrics types, `SpikeSelectionStrategy`. +- Produces: + - `type DriverConfig = { redis: { host: string; port: number; keyPrefix: string }; strategy: RunQueueSelectionStrategy & { onServiced?: SpikeSelectionStrategy["onServiced"] }; workload: WorkloadSpec; envConcurrencyLimit: number; maxLogicalMs: number }` + - `async function runScenario(config: DriverConfig): Promise` + +Driver loop (single env, single shard, logical clock `t` in ms advancing in fixed ticks, default 10ms): +1. Construct `RunQueue` with `queueSelectionStrategy: config.strategy`, `shardCount: 1`, `masterQueueConsumersDisabled: true`, `defaultEnvConcurrency: config.envConcurrencyLimit`. +2. Maintain `holding: Array<{ orgId; runId; releaseAtMs }>` and `events: DequeueEvent[]`. +3. On each tick `t`: (a) release any holds with `releaseAtMs <= t` via `acknowledgeMessage(orgId, runId)`; (b) enqueue all workload events with `enqueueAtMs === t` via `enqueueMessage({ env, message: { ...message, queue: groupId, runId }, workerQueue: env.id, skipDequeueProcessing: true })` (the groupId is the base queue name, no concurrency key); (c) call `queue.testDequeueFromMasterQueue(0, env.id, maxCount)`; for each returned message record a `DequeueEvent` keyed by `keys.descriptorFromQueue(msg.message.queue).queue`, call `config.strategy.onServiced?.(keys.descriptorFromQueue(msg.message.queue), t)`, and push a hold with `releaseAtMs = t + sampledHold`. +4. Stop when all runs dequeued and holds drained, or `t > maxLogicalMs`. +5. Count Redis ops by wrapping the raw client with a call counter, or approximate as dequeue+enqueue+ack counts. Return `computeMetrics(...)`. + +Use one fixed org/project/env descriptor (`o-spike`/`p-spike`/`e-spike`), `maximumConcurrencyLimit: config.envConcurrencyLimit`, `concurrencyLimitBurstFactor: new Decimal(1.0)`. + +- [ ] **Step 1: Write the failing test.** `redisTest` "balanced scenario is roughly fair under baseline": 4 groups, equal weight, `runCount: 50` each, `arrival: immediate`, `envConcurrencyLimit: 5`, `holdMsMean: 30`. Run with `FairQueueSelectionStrategy`. Assert `totalDequeued === 200` and `worstShareOverWeight > 0.5` (baseline should not fully starve any group in the balanced case). +- [ ] **Step 2: Run test, verify it fails.** +- [ ] **Step 3: Implement `driver.ts`.** Guard against infinite loops with the `maxLogicalMs` cap; assert all runs dequeued before computing metrics or fail loudly. +- [ ] **Step 4: Run test, verify it passes.** This is the harness proof: real RunQueue, real Redis, real concurrency gating, real numbers. +- [ ] **Step 5: Commit.** + +--- + +### Task 5: SFQ selector (start-time virtual tags + EEVDF eligibility) + +**Files:** +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/strategies/sfqStrategy.ts` +- Test: `internal-packages/run-engine/src/run-queue/fairness-spike/tests/sfq.test.ts` + +**Interfaces:** +- Consumes: `SpikeSelectionStrategy`, `SpikeQueueReader`, `WeightFn`, `EnvQueues`, `QueueDescriptor`. +- Produces: `class SfqStrategy implements SpikeSelectionStrategy` with `constructor(opts: { redis: Redis; keys: RunQueueKeyProducer; weight?: WeightFn; quantum?: number })` and `name = "sfq"`. + +State (in-process maps, keyed by groupId, plus a scalar `systemVirtualTime`): `virtualClock: Map`. Selection: +1. `active = await reader.readActiveQueues(parentQueue)`. +2. For each active queue compute `startTag = max(virtualClock.get(groupId) ?? systemVirtualTime, systemVirtualTime)`. +3. Eligibility (EEVDF-style): a queue is eligible if `startTag <= systemVirtualTime` OR it is the global minimum start tag (guarantees progress). Order eligible queues by `startTag` ascending, tiebreak by `headScore` ascending. +4. Group ordered queues by env, return `EnvQueues[]`. + +`onServiced(descriptor)`: `const g = descriptor.queue; const w = weight(g); const cur = virtualClock.get(g) ?? systemVirtualTime; const next = cur + quantum / w; virtualClock.set(g, next); systemVirtualTime = Math.min(...virtualClock.values());` (system virtual time is the monotonic floor = min over active clocks; this is the CFS `min_vruntime` analogue and the anti-starvation guarantee). Throughout the selectors, `groupId = descriptor.queue`. + +- [ ] **Step 1: Write the failing unit test** (no Redis, drive the maps directly): after servicing group A 10 times and group B 0 times at equal weight, A's virtualClock is far ahead, so given both active the selector orders B (smaller tag) first. Assert order. +- [ ] **Step 2: Run test, verify it fails.** +- [ ] **Step 3: Implement `sfqStrategy.ts`.** +- [ ] **Step 4: Run test, verify it passes.** +- [ ] **Step 5: Commit.** + +--- + +### Task 6: DRR selector (deficit round robin) + +**Files:** +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/strategies/drrStrategy.ts` +- Test: `internal-packages/run-engine/src/run-queue/fairness-spike/tests/drr.test.ts` + +**Interfaces:** +- Produces: `class DrrStrategy implements SpikeSelectionStrategy`, `constructor(opts: { redis; keys; weight?: WeightFn; quantum?: number })`, `name = "drr"`. + +State: `deficit: Map`, `activeOrder: GroupId[]` (round-robin cursor). Selection: +1. Read active queues; ensure every active groupId is in `activeOrder` (append new ones at the back). +2. Walk `activeOrder` from the cursor; for each group add `quantum * weight(group)` to its deficit; a group's queue is emitted (in cursor order) as long as `deficit >= 1` (unit head cost). Emit each active queue at most once per selection pass, ordered by the round-robin walk. +3. Return grouped `EnvQueues[]`. + +`onServiced(descriptor)`: `deficit.set(g, (deficit.get(g) ?? 0) - 1)` and advance the round-robin cursor past `g`. Drop groups from `activeOrder` when they have no active queue (checked on next read). + +- [ ] **Step 1: Write the failing unit test.** Two groups equal weight, quantum 1: over 10 selection+service cycles, dequeues alternate A,B,A,B... Assert the emitted group sequence is balanced within 1. +- [ ] **Step 2: Run test, verify it fails.** +- [ ] **Step 3: Implement `drrStrategy.ts`.** +- [ ] **Step 4: Run test, verify it passes.** +- [ ] **Step 5: Commit.** + +--- + +### Task 7: Stride selector (integer virtual-time baseline) + +**Files:** +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/strategies/strideStrategy.ts` +- Test: `internal-packages/run-engine/src/run-queue/fairness-spike/tests/stride.test.ts` + +**Interfaces:** +- Produces: `class StrideStrategy implements SpikeSelectionStrategy`, `constructor(opts: { redis; keys; weight?: WeightFn; stride1?: number })` (`stride1` default `1_000_000`), `name = "stride"`. + +State: `pass: Map`. `stride(g) = stride1 / weight(g)`. Selection: read active queues, order by `pass.get(g) ?? 0` ascending (tiebreak headScore). `onServiced(g)`: `pass.set(g, (pass.get(g) ?? 0) + stride(g))`. New groups initialise `pass` to the current global min pass (late-arrival guard, same role as the SFQ floor). + +- [ ] **Step 1: Write the failing unit test.** Weights A:B = 3:1 → over many cycles A is serviced ~3x as often as B. Assert ratio within tolerance (e.g. 2.5–3.5). +- [ ] **Step 2: Run test, verify it fails.** +- [ ] **Step 3: Implement `strideStrategy.ts`.** +- [ ] **Step 4: Run test, verify it passes.** +- [ ] **Step 5: Commit.** + +--- + +### Task 8: CoDel staleness wrapper + +**Files:** +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/strategies/codelWrapper.ts` +- Test: `internal-packages/run-engine/src/run-queue/fairness-spike/tests/codel.test.ts` + +**Interfaces:** +- Produces: `class CodelWrapper implements SpikeSelectionStrategy`, `constructor(opts: { base: SpikeSelectionStrategy; targetMs: number; intervalMs: number; now: () => number })`, `name = `codel(${base.name})``. + +Wraps a base selector. Tracks per-group minimum sojourn (`now - headScore`) over a sliding `intervalMs`. When a group's min sojourn stays above `targetMs` for a full interval, it enters "escalate" mode: that group's queues are hoisted to the front of the base ordering (ahead of the base's own order) until its min sojourn drops below target. Delegates `onServiced` to the base. + +- [ ] **Step 1: Write the failing unit test.** Base = a stub that always orders group A before group B. Feed B a headScore old enough that its sojourn exceeds target for longer than interval; assert the wrapper hoists B ahead of A. Then feed B a fresh headScore and assert order reverts to the base's (A before B). +- [ ] **Step 2: Run test, verify it fails.** +- [ ] **Step 3: Implement `codelWrapper.ts`.** Track `firstAboveTargetAt: Map`; escalate when `now - firstAboveTargetAt >= intervalMs`. +- [ ] **Step 4: Run test, verify it passes.** +- [ ] **Step 5: Commit.** + +--- + +### Task 9: Scenario definitions + bench matrix + ranking table + +**Files:** +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/harness/scenarios.ts` +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/fairnessSpike.bench.test.ts` +- Test: the bench file is itself the runnable matrix. + +**Interfaces:** +- Consumes: `buildWorkload`, `runScenario`, all four strategies, `FairQueueSelectionStrategy`. +- Produces: `const SCENARIOS: Record` and a bench test that runs every (selector × scenario) cell and prints a ranking table. + +Scenarios (all `seed: "spike-1"`): +- `balanced`: 4 groups, equal weight, runCount 50, immediate, envLimit 5, hold 30. +- `adversarialSkew`: 1 heavy group runCount 1000 + 9 light groups runCount 10, equal weight, immediate, envLimit 5, hold 30. +- `weighted`: 2 groups weights 3 and 1, runCount 300 each, immediate, envLimit 4, hold 30. +- `burst`: 6 groups, runCount 100, all enqueued at t=0 after 500ms idle, envLimit 6, hold 20. +- `longHold`: 4 groups; 2 with holdMsMean 500, 2 with holdMsMean 20; equal weight, runCount 40, envLimit 4. + +Selectors: `baseline` (FairQueueSelectionStrategy), `sfq`, `drr`, `stride`, `codel(sfq)`. + +- [ ] **Step 1: Write the bench** as a `describe` of `redisTest`s, one per scenario, each looping the 5 selectors, collecting `RunMetrics`, and `console.table`-ing rows `{ selector, scenario, worstShareOverWeight, jainIndex, waitP99, redisOps }`. Also write per-scenario JSON to `fairness-spike/results/.json` for the findings writeup. +- [ ] **Step 2: Run the full matrix** — `pnpm run test ./src/run-queue/fairness-spike/fairnessSpike.bench.test.ts --run`. Capture the printed tables. +- [ ] **Step 3: Sanity-check the numbers** — baseline should show low `worstShareOverWeight` on `adversarialSkew` (that is the #2617 gap reproduced); at least one candidate should improve it. If nothing improves it, that is itself a finding, not a bug to hide. +- [ ] **Step 4: Commit** the scenarios, bench, and results JSON. + +--- + +### Task 10: FINDINGS.md writeup + +**Files:** +- Create: `internal-packages/run-engine/src/run-queue/fairness-spike/FINDINGS.md` + +**Interfaces:** none (documentation). + +- [ ] **Step 1:** Write `FINDINGS.md` from the `results/*.json`: a per-scenario table, then a proof/disproof verdict per mechanism (SFQ, DRR, stride, CoDel) grounded in the numbers, plus the fidelity caveats (selection-only seam, single shard, simulated holds) and a recommendation on what to take past the spike. Use the writing-voice skill. +- [ ] **Step 2: Commit.** + +--- + +## Self-Review + +**Spec coverage:** Every spec component maps to a task — SFQ (T5), DRR (T6), stride (T7), CoDel (T8), baseline (T4), workload (T2), driver (T4), metrics (T3), scenarios+bench (T9), FINDINGS (T10), the selection-only `onServiced` seam (types T1 + used T4-T8), all five scenarios (T9). Ranking-only output: T3 metrics + T9 table. Out-of-scope items (no prod Lua edits, single shard, simulated holds) are Global Constraints. + +**Placeholder scan:** No TBD/TODO. The one open verification (does the master queue store CK-wildcard keys?) is called out in T1 Step 3 with the exact fallback (expand via CK index like `index.ts:1590`) rather than left vague. + +**Type consistency:** `SpikeSelectionStrategy.onServiced(descriptor, now)` is defined in T1 and consumed with the same signature in T4 (driver calls `onServiced(descriptor, t)`) and implemented in T5-T8. `groupId = descriptor.queue` (the base queue name) is consistent across queueReader, all selectors, and metrics. `RunMetrics`/`DequeueEvent` defined in T3, produced by T4, consumed by T9/T10. + +**Grain pivot (2026-07-23):** the plan originally used the concurrency key as the grain; the Discovery note explains why that grain lives below the strategy interface, so the grain is now the base queue name. Tasks 1, 4, 5 were updated; the scenario configs in T9 are grain-agnostic (groupId maps to a base queue name). diff --git a/docs/superpowers/specs/2026-07-23-fair-queueing-ck-spike-design.md b/docs/superpowers/specs/2026-07-23-fair-queueing-ck-spike-design.md new file mode 100644 index 0000000000..94aa32f8c1 --- /dev/null +++ b/docs/superpowers/specs/2026-07-23-fair-queueing-ck-spike-design.md @@ -0,0 +1,90 @@ +# Per-concurrency-key fairness spike: design + +## What this is + +The follow-on to the base-queue-grain fair-queueing spike. That spike found the +#2617 gap lives below the `RunQueueSelectionStrategy` interface, in the +CK-dequeue Lua: `dequeueMessagesFromCkQueueTracked` picks concurrency-key queues +from `ckIndexKey` oldest-head-first, so a concurrency key with a big backlog +starves other keys under the same base queue. This spike proves or disproves +whether the same disciplines (SFQ, DRR, stride, CoDel) fix that at the real seam. + +Throwaway. Ships nothing. Delete before any merge to main. + +## Goal + +Rank five selectors on per-concurrency-key fairness under contention: the +production baseline (age-order) plus SFQ, DRR, stride, and a CoDel wrapper. +Relative ranking only, on the same axes the base-queue spike used: contention +share (arrival-aware) and per-key wait, over multiple seeds. + +## Grain + +The fairness group is the concurrency key. One environment, one base queue, many +concurrency keys. A "heavy" key has a large backlog; "light" keys have little. +The question: does the heavy key starve the light keys, and does each discipline +fix it? + +## Approach: rescore ckIndex, drive the real Lua + +The per-CK pick is `ZRANGEBYSCORE ckIndexKey -inf now` inside +`dequeueMessagesFromCkQueueTracked` (`index.ts`), i.e. lowest score first, and +the score is the CK-queue's head-message timestamp. So the discipline is +expressed by controlling that score: + +- baseline: leave the scores as the Lua maintains them (head timestamp). This is + production behaviour, unmodified. +- candidate: before each dequeue round, rewrite each active CK-queue's `ckIndex` + score to encode the discipline's priority (virtual clock / deficit / pass), + mapped into a `<= now` range preserving order so the Lua treats the + highest-priority key as the oldest and serves it first. + +Enqueue and acknowledge go through the real `RunQueue` (real `ckIndex`, per-CK +queues, per-CK and env concurrency, atomic Lua). The only spike affordance is the +rescore sweep before candidate dequeues, plus the `onServiced` hook that advances +discipline state per served key. In production that advance would live inside the +enqueue/dequeue Lua that maintains `ckIndex`. + +Fidelity anchor: a test asserts the baseline path (no rescore) produces the same +per-key dequeue sequence as calling the real Lua directly, so the harness is a +faithful stand-in for production age-ordering. + +## Components + +Reuse from the base-queue spike (`../fairness-spike/`): `harness/workload.ts` +(tenant becomes concurrency-key generator), `harness/metrics.ts` (contention +share + wait, unchanged), and the selector disciplines' core logic where it +transfers. New, under `internal-packages/run-engine/src/run-queue/fairness-spike-ck/`: + +- `ckReader.ts`: reads `ckIndexKey` members and each key's head score for a base + queue. +- `ckRescorer.ts`: given a discipline priority per concurrency key, rewrites the + `ckIndex` scores into a `<= now` order-preserving range. +- `strategies/`: sfq/drr/stride/codel keyed by concurrency key (thin adapters + over the base-queue spike's disciplines, or shared directly). +- `harness/ckDriver.ts`: enqueues runs across concurrency keys, loops + rescore -> real dequeue -> hold -> ack, records per-key events. +- `ckFairnessSpike.bench.test.ts`: the selector-by-scenario matrix, multi-seed. +- `FINDINGS.md`: the writeup. + +## Scenarios (seeded, multi-seed) + +- ckSkew: one heavy concurrency key (large backlog) vs many light keys, equal + weight. The direct #2617 reproduction. +- ckBalanced: equal keys (sanity). +- ckTrickle: a bulk key plus keys whose runs trickle in (poisson), to exercise + wait and the CoDel wrapper honestly. + +## Out of scope + +- Per-CK concurrency-limit multiplication (the other half of #2617) is a limit + problem, not a dequeue-ordering problem; not addressed here. +- No changes to production Lua. The rescore sweep stands in for the production + ckIndex-scoring change. +- Single shard, single environment, single base queue. + +## Deliverable + +The multi-seed ranking table plus `FINDINGS.md` with a proof/disproof verdict per +discipline at the concurrency-key grain, and whether the base-queue spike's +ranking carried over to the real seam. diff --git a/docs/superpowers/specs/2026-07-23-fair-queueing-spike-design.md b/docs/superpowers/specs/2026-07-23-fair-queueing-spike-design.md new file mode 100644 index 0000000000..e4d3bee85b --- /dev/null +++ b/docs/superpowers/specs/2026-07-23-fair-queueing-spike-design.md @@ -0,0 +1,104 @@ +# Fair-queueing scheduler spike: design + +## What this is + +A throwaway spike to rank fair-queueing methods for the Run Engine 2.0 RunQueue +against each other, so we can decide which one (if any) is worth taking further. +It ships nothing. The output is a ranking table plus a short findings writeup. + +Background: the current RunQueue does environment-level fairness via +`FairQueueSelectionStrategy`, but has no per-tenant/per-group fairness below the +environment (GitHub #2617). A `concurrencyKey` spawns a separate queue and a +separate limit per key, so one tenant firing 1000 tasks can occupy a whole +environment while other tenants wait. The design brief (`compass_artifact`) +proposes four methods: SFQ virtual-time tagging, hierarchical DRR, a CoDel-style +staleness monitor, and stride/lottery as a baseline. This spike puts those four +on trial next to the existing strategy. + +## Goal + +Rank five selectors: the four candidates plus the current +`FairQueueSelectionStrategy` as baseline. Ranking only, no absolute pass/fail +bar. Three axes: + +- fairness: per-group throughput share divided by configured weight, plus a Jain + index and the worst-served group. +- anti-staleness: per-group dequeue wait (dequeue time minus enqueue time), p50, + p99, max. Under skew the light tenant's p99 is the headline number. +- cost: wall-clock and Redis op count per dequeue, rough. + +## Harness (Approach 1: full RunQueue driver) + +Stand up the real `RunQueue` against a testcontainers Redis. Swap +`queueSelectionStrategy` per candidate. A synthetic load generator enqueues runs +across N groups at configurable rates. A pool of fake workers dequeues, holds a +concurrency slot for a sampled duration, then acks. The driver records every +dequeue event and computes the metrics. + +This is faithful because the concurrency consume-then-release feedback loop that +drives `availableCapacityBias` is the real one, not a simulation. + +Fairness grain is the `concurrencyKey`/groupId, expressed as ordering among the +sibling queue keys that come back in `EnvQueues.queues`. + +### The selection-only seam + +The `RunQueueSelectionStrategy` interface is invoked at selection and is never +told which queue actually got dequeued. So a virtual clock, a deficit counter, or +a pass counter has nothing to advance on. In production that state would advance +inside the ack/dequeue Lua. For the spike, the driver sees each dequeued message +(org, queue, concurrencyKey) and feeds it back via an optional +`onServiced(descriptor)` hook on the candidate. This is a spike-only affordance +and a fidelity caveat: it proves the ordering logic, it does not prove the +production Lua wiring. + +## Components + +- `strategies/sfqStrategy.ts`: per-group virtual clock in a Redis hash, start tag + = max(group vclock, system vclock floor), order by smallest eligible tag, + EEVDF-style eligibility guard. Advances the vclock on `onServiced`. +- `strategies/drrStrategy.ts`: per-group deficit and quantum (weight), + round-robin scan of active groups. Advances the deficit on `onServiced`. +- `strategies/strideStrategy.ts`: stride = big/weight, pass counter, pick lowest + pass, advance by stride. Integer virtual-time baseline. +- `strategies/codelWrapper.ts`: wraps a base selector. Tracks per-group minimum + sojourn (now minus head enqueue score) over an interval; when it stays above + target, escalates that group's effective weight/priority. Not a standalone + selector. +- baseline: the existing `FairQueueSelectionStrategy`, imported unchanged. +- `harness/workload.ts`: seeded generator. Group set, arrival rates, service-time + sampler, weights. +- `harness/driver.ts`: runs `RunQueue` plus the fake worker pool, records dequeue + events, calls `onServiced`. +- `harness/metrics.ts`: share-vs-weight, Jain index, wait percentiles, cost. +- `harness/scenarios.ts`: named scenarios. +- `fairnessSpike.bench.test.ts`: runs the matrix (5 selectors x scenarios) and + prints the ranking table. +- `FINDINGS.md`: written up at the end. + +## Scenarios (all seeded, deterministic) + +- balanced: equal groups, equal weight. Sanity check: shares should come out + equal. +- adversarial skew: one heavy group with 1000 runs versus many light groups with + 10 each, equal weight. Does the light group starve? +- weighted: 3:1 configured weights. Does share track weight? +- burst: idle, then a thundering enqueue. +- long-hold: some groups hold concurrency slots far longer than others. Tests + that dequeue fairness stays separate from concurrency occupancy (the brief's + 30-day-run point). + +## Out of scope + +- No changes to the production enqueue/dequeue/ack Lua. Candidates advance their + state via the harness `onServiced` hook. +- Single Redis shard only. Global multi-shard fairness is a known gap, noted + rather than solved. +- Simulated hold durations, not real long-running runs. +- No webapp wiring. + +## Deliverable + +The ranking table from the bench test, plus `FINDINGS.md` with a +proven/disproven-ish verdict per method and a recommendation on what (if +anything) to take past the spike. diff --git a/docs/superpowers/specs/2026-07-23-fairness-caps-vs-scheduling-design.md b/docs/superpowers/specs/2026-07-23-fairness-caps-vs-scheduling-design.md new file mode 100644 index 0000000000..5472c9620a --- /dev/null +++ b/docs/superpowers/specs/2026-07-23-fairness-caps-vs-scheduling-design.md @@ -0,0 +1,121 @@ +# Caps vs scheduling: reconciling the fairness spike with the plan of record + +Throwaway spike design. Ships nothing; delete before any merge to main. + +## Why + +Two prior spikes (base-queue grain, then the real CK-dequeue grain) concluded: +score `ckIndex` by a fair discipline (SFQ/stride virtual time, or DRR) instead of +by head timestamp, to fix per-concurrency-key starvation (#2617). That is a +change to the fair-selection SCORING / serve order. + +The plan of record for "Queue multi-tenant fairness" does something different. It +ships bounded concurrency CAP primitives and deliberately leaves the +fair-selection scoring untouched: + +- Phase 1: a per-base-queue TOTAL concurrency cap (`:groupConcurrency` SET SCARD + gated against a `:totalConcurrency` limit). +- Phase 2: a per-KEY concurrency limit (`:ckLimits` HASH, HGET per CK variant). + +The only scheduler-adjacent change is negative: the fair-selection strategy drops +base queues already at their total cap so it stops picking them. Scoring within +the contended region is still head-timestamp order. + +So the spiked mechanism (scheduling) and the shipped mechanism (caps) are +different knobs. This spike proves or disproves whether the caps deliver the +fairness the scheduling disciplines deliver, and reconciles the two. + +## The thesis to test + +From the research (see `RESEARCH.md`): occupancy caps and fair scheduling are +orthogonal. A cap bounds a tenant's occupancy and (via Little's Law) its +throughput share; it does not bound wait unless the dequeue is +oldest-ELIGIBLE-first AND freed slots exist. A scheduler bounds a starved +tenant's first-serve wait but is work-conserving (bounds no occupancy). Neither +substitutes for the other; production systems layer them (Kubernetes APF: seats + +fair queueing). + +Falsifiable claims: + +1. On a simple skew (one heavy key, light keys), a per-key cap on the heavy key + cuts the light keys' wait about as well as SFQ, BECAUSE Trigger's CK dequeue is + eligibility-aware (a variant at its per-key limit is skipped). Predict: + per-key cap ~= SFQ on light-key wait here. +2. A per-key cap is NOT work-conserving: when the heavy key is alone (siblings + idle), the cap throttles it below the env limit and idles slots, inflating + makespan. SFQ/baseline uses the whole env. Predict: per-key cap makespan >> + SFQ makespan on a heavy-alone workload. +3. A per-key cap fails the sybil split: a heavy tenant spreading its backlog over + many CK variants, each under its per-key cap, still starves a late light key + under oldest-first, because sum of per-key caps is unbounded relative to the + queue. Predict: per-key cap light-key wait stays high on the sybil scenario; + SFQ still protects the light key. +4. A total cap (per-task) does NOT fix cross-key starvation within one task: it + lowers the whole task's ceiling uniformly, and age-order still serves the heavy + backlog first within it. Predict: total-cap-only light-key wait ~= baseline. +5. Layered per-key cap + SFQ ordering (the APF pattern) is at least as good as + either alone on every scenario. Predict: layered ~= SFQ on wait, and inherits + the cap's occupancy bound. + +## Harness + +Reuse the CK harness (`fairness-spike-ck`) that drives the real +`dequeueMessagesFromCkQueueTracked` Lua at `maxCount = 1`, rescoring `ckIndex` to +express a discipline's order. Model the caps as admission gates: + +- Per-key cap: a discipline marks any CK variant whose in-flight >= its per-key + cap as INELIGIBLE. The driver writes ineligible variants a beyond-window + `ckIndex` score so the real Lua's `ZRANGEBYSCORE -inf now` skips them, exactly + as the real per-key gate would. Among eligible variants, keep baseline age + order (or an inner scheduler for the layered discipline). +- Total cap: the driver refuses to dequeue when total in-flight (holding.length, + = group SCARD for one base queue) >= the total cap, modelling the group gate. + +This is faithful to the plan's eligibility-aware semantics; the fidelity gap is +the same `maxCount = 1` one the CK spike already documents (production dequeues in +batches; a real per-key/total gate lives inside the batched Lua). + +New driver state: `inFlightByCk` and `runToCk`, maintained on serve/ack. + +## Disciplines + +- `baseline` (existing): age order, no cap. +- `perKeyCap(capOf)`: eligibility by per-key in-flight cap; age order among + eligible. Heavy key capped low, others uncapped (env-bounded). +- `totalCap(n)`: no per-key differentiation, no rescore ordering change; driver + gate on total in-flight. Models Phase 1 within one task. +- `sfq`, `stride`, `drr` (existing): scheduling. +- `perKeyCap+sfq` (layered): eligibility by per-key cap, SFQ order among eligible. +- (CoDel already disproven in the CK spike; not re-run here.) + +## Scenarios + +Existing: `ckSkew`, `ckBalanced`, `ckTrickle`. New: + +- `ckSybil`: heavy tenant as MANY CK variants (~20), each a modest backlog, all + enqueued early (old heads); one light key arrives later via poisson. Tests the + sybil split. +- `ckHeavyIdle`: one heavy key with a large backlog, siblings absent or arriving + very late and few. Tests work-conservation (makespan / idle slots). + +## Metrics + +Reuse `computeMetrics` (per-key wait = headline, contention share = directional). +Add `makespanMs` (max dequeue logical time) as the work-conservation signal: +a non-work-conserving cap inflates makespan on `ckHeavyIdle`. + +## Success bar + +Relative ranking only (as with the prior spikes), multi-seed, driving the real +Lua. The deliverable is a reconciliation verdict: which knob each mechanism is, +what each fixes and fails, and whether the plan's caps + the spike's scheduling +should layer. + +## Not building + +- A production implementation (spike is throwaway). +- A separate multi-base-queue (cross-task) harness for the total cap's real job + (protecting one task's env budget from another task). The total cap's cross-task + isolation is argued analytically from the research; within-task it is shown not + to fix cross-key starvation. A cross-task harness is noted as future work. +- Re-running CoDel (already disproven at both grains). diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/CAPS_FINDINGS.md b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/CAPS_FINDINGS.md new file mode 100644 index 0000000000..b1daf94ee8 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/CAPS_FINDINGS.md @@ -0,0 +1,198 @@ +# Caps vs scheduling: reconciliation findings + +Throwaway spike. Ships nothing; delete before any merge to main. Relative ranking +on a small simulation, not a statistical or load study: read the verdicts as "what +this harness supports", not proofs. Went through a blind two-model adversarial +review (a third stalled); the review's fixes are folded in below. + +Bottom line: the plan-of-record's concurrency CAPS and the earlier spike's fair +SCHEDULING are different knobs, and the data on the real CK-dequeue Lua lines up +with the queueing theory (see `RESEARCH.md`). A per-key cap cuts a starved key's +wait when ONE key floods, because Trigger's CK dequeue is oldest-eligible-first; +it gives no wait improvement once a tenant shards its backlog across many +concurrency keys, and it is not work-conserving. Fair scheduling (SFQ/DRR) is the +only knob here that improves the starved key on every scenario including the +sharded one, and it stays work-conserving. A total (per-task) cap is a +cross-task knob; inside one task it only lowers the ceiling and is not a fairness +lever at all. The mechanisms are complementary, and production systems that need +fairness under saturation layer them (Kubernetes APF: seats + fair queueing). + +## How the mechanisms were modelled (fidelity) + +- Per-key cap (Phase 2): the REAL Lua gate. `updateQueueConcurrencyLimits` sets + the base queue's concurrencyLimit; the CK-dequeue Lua caps each ck variant's + in-flight at it and skips an at-limit variant (oldest-eligible-first, true age + order, no rescore). Uniform across variants: Phase 2's per-key HGET override + would cap only the heavy key, but a light key never approaches the cap so the + effect is equivalent here. (So "just lower the existing per-queue concurrency + limit" already IS a per-key cap; Phase 2 makes it per-key-specific.) +- Total cap (Phase 1): driver-side. The real Lua has no group gate yet, so the + driver refuses to admit while total in-flight across all variants of the base + queue (= `:groupConcurrency` SCARD in one base queue) is at the cap. +- Ordering disciplines (baseline age order, SFQ, DRR) are unchanged from the CK + scheduling spike, driven through the same real Lua at `maxCount = 1`. +- Two fidelity limits matter for reading the numbers, both driver-independent: + - `maxCount = 1` (same as the CK spike): production dequeues in batches, so a + real per-key/total gate lives inside the batched Lua. + - The `*3` scan window: the real CK Lua reads `ZRANGEBYSCORE ckIndexKey -inf now + LIMIT 0, actualMaxCount*3` (`index.ts:4041/4193`). At `maxCount = 1` that is + the 3 oldest-scored variants per call. So a per-key cap frees the light key + only when the light head lands inside that 3-wide window after the at-cap + variants ahead of it. With one heavy key it does; with many old-headed + attacker variants it never does. This window governs the skew-works / + sharded-fails split, and it scales with `maxCount` in production (batches), + not fixed at 3, so the sharded result's exact severity would differ on the + real batched path (direction not established). + +## Results + +env=4, per-key cap=2, total cap=2, 3 seeds. Columns: `lightWait` = the starved +key's mean wait (logical ms, the headline where it is not confounded); +`worstWait` = the largest per-group mean wait (i.e. the busiest key, which a fair +discipline deliberately makes wait its turn, so higher here is often correct); +`makespan` = logical time of the last dequeue (a work-conservation signal ONLY on +ckHeavyIdle, arrival-confounded elsewhere); `contWorstS/W` = worst contention +share over weight (directional; volume-confounded and, for per-key cap on sybil, +seed-noisy). + +| scenario | treatment | lightWait | worstWait | makespan | contWorstS/W | +| ----------- | ------------------ | --------- | --------- | -------- | ------------ | +| ckSkew | baseline | 1098 | 1261 | 2083 | 0.187 | +| ckSkew | perKeyCap | 20 | 1555 | 3038 | 0.814 | +| ckSkew | totalCap | 2840 | 2974 | 3947 | 0.213 | +| ckSkew | total+perKey | 2840 | 2974 | 3947 | 0.213 | +| ckSkew | sfq | 14 | 1069 | 2083 | 0.723 | +| ckSkew | drr | 17 | 1067 | 2083 | 0.608 | +| ckSkew | perKeyCap+sfq | 7 | 1628 | 3114 | 0.800 | +| ckSkew | total+perKey+sfq | 52 | 2363 | 3939 | 0.803 | +| ckTrickle | baseline | 1107 | 1134 | 1940 | 0.279 | +| ckTrickle | perKeyCap | 19 | 1555 | 3038 | 0.922 | +| ckTrickle | totalCap | 2852 | 2861 | 3905 | 0.279 | +| ckTrickle | total+perKey | 2852 | 2861 | 3905 | 0.279 | +| ckTrickle | sfq | 17 | 1070 | 1942 | 0.909 | +| ckTrickle | drr | 23 | 1069 | 1941 | 0.790 | +| ckTrickle | perKeyCap+sfq | 8 | 1606 | 3097 | 0.658 | +| ckTrickle | total+perKey+sfq | 106 | 2337 | 3911 | 0.883 | +| ckSybil | baseline | 1765 | 1876 | 2070 | 0.000 | +| ckSybil | perKeyCap | 1776 | 1869 | 2128 | 0.403 | +| ckSybil | totalCap | 3793 | 3801 | 4147 | 0.000 | +| ckSybil | total+perKey | 3793 | 3801 | 4147 | 0.000 | +| ckSybil | sfq | 1009 | 1019 | 2065 | 1.000 | +| ckSybil | drr | 1061 | 1068 | 2055 | 0.994 | +| ckSybil | perKeyCap+sfq | 1010 | 1019 | 2072 | 1.000 | +| ckSybil | total+perKey+sfq | 2292 | 2292 | 4146 | 1.000 | +| ckHeavyIdle | baseline | 633 | 633 | 1240 | 1.000 | +| ckHeavyIdle | perKeyCap | 1291 | 1291 | 2507 | 1.000 | +| ckHeavyIdle | totalCap | 1291 | 1291 | 2507 | 1.000 | +| ckHeavyIdle | total+perKey | 1291 | 1291 | 2507 | 1.000 | +| ckHeavyIdle | sfq | 633 | 633 | 1240 | 1.000 | +| ckHeavyIdle | drr | 633 | 633 | 1240 | 1.000 | +| ckHeavyIdle | perKeyCap+sfq | 1291 | 1291 | 2507 | 1.000 | +| ckHeavyIdle | total+perKey+sfq | 1291 | 1291 | 2507 | 1.000 | + +(ckHeavyIdle is a single key, so "lightWait" is the heavy key's own wait and the +contention metric is degenerate at 1.0; makespan is the signal there. ckSybil is +20 attacker keys plus one light key.) + +## What each mechanism does, at the cross-key grain + +- Per-key cap (Phase 2). SUPPORTED for the single-heavy case; NOT a wait fix once + the tenant shards; not work-conserving. + - Single heavy key (ckSkew/ckTrickle): cuts the light key's wait like a + scheduler (1098 to 20, 1107 to 19) because capping the one heavy key frees + slots and the CK Lua serves the light head as the next eligible one. This + depends on the light head being reachable inside the Lua's 3-wide scan window; + with one at-cap variant ahead of it, it is. + - Sharded / sybil (ckSybil, 20 attacker keys): NO wait improvement (baseline + 1765, perKeyCap 1776; the difference is within the per-seed spread, baseline + 1681..1926, perKeyCap 1672..1861). Contention share nudges up (0.000 to a + mean 0.403) but that mean hides a ~10x seed swing (0.07..0.71), so it is not a + dependable improvement. The reason is structural: the sum of per-key caps is + unbounded relative to the queue when keys are client-chosen, and the 3-wide + scan window is always full of older attacker heads, so the light head is never + reached. Concurrency keys are client-chosen, so this is cheap to trigger. + - Not work-conserving: throttles the capped key even with the env idle + (ckHeavyIdle makespan 1240 to 2507, 2x, the cleanest single result in the + spike; ckSkew 2083 to 3038, though that scenario's makespan is partly arrival- + confounded). +- Total cap (Phase 1) at the cross-key grain: NOT a fairness lever, and the + in-task comparison is capacity-confounded. `totalCap=2` caps the whole task's + aggregate at half of env=4, so it simply halves throughput: light's wait rises + (ckSkew 1098 to 2840) for the same reason heavy's does (both now share half the + server), which is Little's-Law throughput loss, not a fairness effect. It is the + wrong knob for cross-key starvation, measured on a lower ceiling; do not read + the "worse" numbers as "total caps harm fairness." +- Total cap (Phase 1) at the cross-TASK grain: this IS its job, and it works. + Measured in a separate multi-base-queue bench (`crossTaskCaps.bench.test.ts`): + two keyless tasks share one env, a heavy task floods it, and capping the heavy + task (its per-queue concurrency limit, the real native gate, which for a keyless + task equals its total cap) cuts the light TASK's wait from 475 to 2 under the + production `FairQueueSelectionStrategy`. So the total cap protects a light task + from a heavy task, the reservation-isolation role the research describes. It is + still not work-conserving (makespan 2039 to 3039), and SFQ at the task grain + protects the light task too (wait 14) while staying work-conserving (2039). The + fidelity note: this models a KEYLESS task, so the per-queue limit is the total; + a task WITH concurrency keys needs the group SET to sum across variants (the + unbuilt Phase-1 gate). +- Combined total + per-key (the shipped Phase-1+2 config): in this toy the total + cap (2) is below a single per-key cap's reach, so it dominates and the per-key + cap is non-binding (`total+perKey` equals `totalCap` to the digit). This toy + therefore does not exercise the combined config's real regime (total >> per-key, + cross-task). What it does show: adding a fair order on top (`total+perKey+sfq`) + restores fair share within the throttled aggregate (contWorstS/W 0.80..1.0) but + still pays the total cap's throughput loss (makespan ~3900+). +- Scheduling (SFQ/DRR): the only knob that improves the starved key on every + scenario, and work-conserving (makespan stays at the baseline optimum + 2083/1240). On the sharded case SFQ takes the light key from fully starved to + its full fair share (contWorstS/W 0.000 to 1.000, seed-stable) and roughly + halves its wait (1765 to 1009); the residual wait is real saturation shared + fairly across 21 keys, not starvation. SFQ and DRR track each other within + noise, as in the CK spike. +- Layered per-key cap + SFQ: best light-key wait on the single-heavy case (7, 8) + and it carries the cap's occupancy bound, at the cap's makespan cost (matches or + slightly exceeds perKeyCap makespan: ckSkew 3038 to 3114). On the sharded case + the cap adds nothing and SFQ does all the work (1010, same as SFQ alone). This + is the Kubernetes-APF shape; APF avoids the work-conservation cost by making the + cap ELASTIC (borrow/lend seats), which a static cap cannot. + +## Reconciliation with the earlier spike and the plan of record + +Measured here: caps and scheduling fix different things and can be layered +(the per-key-cap+SFQ and total+perKey+sfq rows). Fair scheduling is the only +mechanism in this harness that improves the starved key on the sharded case and +stays work-conserving, which is what the earlier spike recommended (score +`ckIndex` by virtual time). + +Interpretation, NOT measured by this benchmark (it measures wait/makespan/share on +a simulation, not engineering cost or rollout risk): shipping the caps first still +reads as defensible. A per-key cap is bounded, operator-controlled, self-healing +(a Redis SET), and it fully fixes the common single-heavy-key case, which is a +smaller engine change than reworking the dequeue scoring. Its limits are real +(no help once a tenant shards its keys, not work-conserving), which is the case +for treating automatic fair scheduling as a later phase rather than never. + +The layered end state matches Kubernetes APF, SQL Server Resource Governor, YARN, +and the Parekh-Gallager result that a worst-case delay bound needs BOTH an +admission regulator AND a scheduler: keep the caps for isolation and entitlements, +add a fair dequeue order for the contended region when saturation and key-sharding +make caps alone insufficient. Not either/or. + +## Caveats + +- Relative ranking on a simulation; single shard, single base queue, single + sequential consumer; simulated holds on a logical clock; 3 seeds; equal weights. + The verdict words ("supported", "not a wait fix") are relative to this harness. +- `maxCount = 1` and the `*3` scan window (see fidelity section); the total cap is + driver-modelled, not the real (unbuilt) group gate. +- Per-key cap is modelled uniformly (real per-queue gate); a per-key-specific + Phase-2 override is equivalent here only because the light key never approaches + the cap. +- makespan is the last dequeue, not completion, and is arrival-confounded on the + poisson scenarios; trust it only on ckHeavyIdle. +- Contention share is volume-confounded for low-volume keys, and for the per-key + cap on the sharded case it is seed-noisy (0.07..0.71); wait is the trustworthy + signal, share is directional. +- Cross-task isolation (the total cap's real purpose) is now measured in + `crossTaskCaps.bench.test.ts` for KEYLESS tasks (per-queue limit = total cap). + A task with concurrency keys needs the unbuilt group-SET gate to sum across + variants; that batched, keyed path is still not exercised. diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/FINDINGS.md b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/FINDINGS.md new file mode 100644 index 0000000000..a0e806fd62 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/FINDINGS.md @@ -0,0 +1,122 @@ +# Per-concurrency-key fairness spike: findings + +Throwaway spike. Ships nothing; delete before any merge to main. + +Bottom line: the base-queue spike's direction carries over to the real seam. At +the concurrency-key grain the production baseline (serve the oldest-head CK +first) starves keys that arrive behind a big backlog, and virtual-time (SFQ) and +stride fix it: they cut the starved key's wait from ~1300ms to ~20ms by making +the backlog key wait its turn. DRR does the same. A CoDel wrapper on the baseline +makes it worse. This was measured by driving the real +`dequeueMessagesFromCkQueueTracked` Lua and only rewriting `ckIndex` scores to +express each discipline. That is enough to say the ordering fix is worth a design +spike, but NOT that a production implementation is proven (see the fidelity +caveat: the spike serves one key per Lua call, and production dequeues in +batches). + +## What was driven, and the two things to know before reading numbers + +Runs enqueue across many concurrency keys under one base queue via the real +`RunQueue`. The per-CK pick is `ZRANGEBYSCORE ckIndexKey -inf now` in the CK Lua +(lowest score first, score = head timestamp). Candidates rewrite those scores +each round to encode discipline order; the baseline leaves them (production age +order). Enqueue, dequeue, concurrency gating and ack all run through the real +code. + +Two caveats a review forced, both load-bearing: + +1. Lead with wait, not contention share. The contention-share metric is + volume-confounded for low-volume keys (a key with 15 runs cannot take a third + of a long window even when served instantly), so on these scenarios it lands + around 0.7 to 0.9 for a discipline that has in fact eliminated the starvation. + The per-key wait is the clean signal. +2. The scenarios must give keys genuinely different head ages. An earlier version + enqueued every run at one timestamp; with tied `ckIndex` scores the real Lua + falls back to a lexicographic member-name tie-break, so the "baseline starves + the heavy key's rivals" result was actually "Redis sorts by name" and the + heavy key only won because "heavy" sorts before "light". Fixed: the backlog + key fires at once (persistently old head) and the other keys arrive via + poisson (distinct, later heads), so the baseline now exercises real age order. + +## Results + +`contWorstS/W` mean over 3 seeds (min..max), and the worst-served key's mean wait +(seed-a, logical ms). Read the wait column as the headline. + +| scenario | discipline | contWorstS/W | worst-key wait | backlog-key wait | +| ---------- | ---------- | ------------------- | -------------- | ---------------- | +| ckSkew | baseline | 0.187 (0.186..0.188)| 1321 | 872 | +| ckSkew | sfq | 0.723 (0.655..0.769)| 16 | 1150 | +| ckSkew | drr | 0.608 (0.556..0.648)| 21 | 1149 | +| ckTrickle | baseline | 0.279 (0.254..0.291)| 1339 | 872 | +| ckTrickle | sfq | 0.909 (0.891..0.918)| 24 | 1237 | +| ckTrickle | drr | 0.790 (0.769..0.818)| 30 | 1236 | + +Full matrix (contWorstS/W mean over 3 seeds): + +| scenario | baseline | sfq | drr | stride | codel(sfq) | codel(baseline) | +| ---------- | ------------------- | ------------------- | ------------------- | ------ | ---------- | ------------------- | +| ckSkew | 0.187 | 0.723 | 0.608 | 0.723 | 0.723 | 0.104 (0.000..0.157)| +| ckBalanced | 0.515 (0.444..0.600)| 0.611 (0.462..0.800)| 0.730 (0.615..0.909)| 0.611 | 0.611 | 0.464 | +| ckTrickle | 0.279 | 0.909 | 0.790 | 0.909 | 0.909 | 0.018 (0.000..0.055)| + +## Verdict per discipline (at the concurrency-key grain) + +- SFQ / stride: fix the starvation. Contention share improves (skew 0.187 to + 0.723, trickle 0.279 to 0.909) and the starved key's wait collapses (skew 1321 + to 16, trickle 1339 to 24) because the backlog key now waits its turn (its wait + rises 872 to ~1150 to 1237). Identical to each other on every scenario. + Recommended discipline for the fix. +- DRR: fixes the wait just as well (skew 21, trickle 30) and its contention share + tracks SFQ within noise (sometimes a little lower, sometimes higher, e.g. + balanced 0.730 vs 0.611). Fine. +- CoDel(sfq): no harm, matches SFQ to the decimal. Adds nothing on top of a fair + base. +- CoDel(baseline): harmful. Hoisting stale keys on top of the age-order baseline + drove ckSkew to 0.104 (below baseline's 0.187) and ckTrickle to 0.018 (below + 0.279). A staleness monitor is not a substitute for a fair base. +- Baseline (production age order): starves keys that queue behind a backlog + (ckSkew 0.187, worst key waits 1321ms; ckTrickle 0.279, 1339ms). It is roughly + fair when keys are symmetric (ckBalanced 0.515, though seed-variant 0.444 to + 0.600). This is the #2617 dynamic at the seam where it lives. + +## Fidelity caveat (the reason this is not "proven for production") + +The driver dequeues one key per Lua call (`maxCount = 1`) and rescores `ckIndex` +before each call. Production dequeues in batches (`maxCount` default 10). Inside a +batched CK-dequeue call the Lua re-scores each served key back to its head +timestamp as it goes, so a once-per-round rescore would only steer the FIRST pick +of a batch; the rest would follow head-timestamp order again. So this spike +demonstrates the ordering fix only in a one-key-per-call regime, which is not how +production dequeues. A real fix has to advance per-key discipline state inside the +Lua on every serve (and hold that state in Redis, not process memory). This spike +does not exercise that batch path, so the correct claim is "the ordering fix is +worth a design spike", not "a production fix is viable". + +## Other caveats + +- Contention share is volume-confounded (see above); the wait column is the + trustworthy signal, and the contention numbers should be read as directional. +- The DRR contention-share gap is NOT the base-queue spike's batch-drain artifact + (that harness batched; this one serves one key per call and advances DRR's + deficit every serve). The cause of DRR's slightly lower share here is not + established; its wait result is as good as SFQ's. +- Per-CK concurrency gating never binds in these runs (no per-CK limit is set, so + it collapses to the env limit), so the spike says nothing about the per-CK + concurrency-limit-multiplication half of #2617, which is out of scope. +- A rescore discipline advances its floor/ring state on the final no-op drain + round of an instant (order() is called before the empty dequeue). It is + self-correcting and does not corrupt the event-based metrics, but it is a minor + infidelity to a production per-serve advance. +- Equal weights only; single shard, single base queue, single sequential + consumer; simulated holds on a logical clock; 3 seeds. + +## Recommended direction + +Score `ckIndex` by a fair discipline (SFQ/stride virtual time, or DRR) instead of +by head timestamp. Both spikes agree on the discipline and this one shows the +ordering fix works through the real dequeue path at `maxCount = 1`. The design +spike past this needs to: advance per-key virtual-time state inside the batched +CK-dequeue Lua (the `maxCount > 1` path this spike did not exercise), hold that +state in Redis for the multi-consumer case, and address the per-CK +concurrency-limit multiplication that is the other half of #2617. diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/RESEARCH.md b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/RESEARCH.md new file mode 100644 index 0000000000..4539c2a412 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/RESEARCH.md @@ -0,0 +1,133 @@ +# Queue-fairness research (grounding for the caps-vs-scheduling reconciliation) + +Throwaway spike notes. Five Fable research passes, distilled. Citations kept so +the findings write-up and report can point at real sources. This grounds the +central claim: occupancy caps and fair scheduling are orthogonal knobs, and the +plan-of-record ships the cap knob while the earlier spike measured the scheduler +knob. + +## The orthogonality result (theory) + +Caps bound occupancy, not wait. A tenant capped at C in-flight with mean service +time S has long-run throughput <= C/S (Little's Law, L = lambda*W). That is an +upper bound on the capped tenant's share; it reserves no lower bound for anyone +else and says nothing about any tenant's waiting time (Little relates averages in +a stable system, not tails, and if the capped tenant's arrival rate exceeds C/S +its queue never stabilises so the law does not even apply to it). + +Scheduling bounds wait, not occupancy. WFQ/PGPS tracks GPS within one max job +(Parekh-Gallager finish-time bound L_max/r); SFQ gives a starved flow's head item +a hard wait bound of "one max-size job from every other active tenant" (Goyal-Vin +SFQ Theorem 2), with no server-rate assumption. But all of family A is +work-conserving: a lone backlogged tenant takes 100% of the server. Nothing in a +scheduler limits how many slots a tenant holds. + +Parekh-Gallager is the canonical joint statement: a worst-case per-flow delay +bound is the product of arrival regulation (leaky/token bucket = the admission +knob) AND a scheduling discipline (GPS/WFQ = the order knob). Neither alone yields +the bound. Cruz network-calculus caveat: plain FIFO does get a delay bound IF every +input is burstiness-constrained (arrival regulator on ingress) and aggregate rho < +C, but a concurrency cap is not an ingress regulator (it bounds in-flight, not +queue admission, and queue depth stays unbounded), so under adversarial arrival +FIFO wait is unbounded and the orthogonality holds without qualification. + +Sources: Little 1961 (Oper. Res. 9:383-387); Parekh & Gallager 1993/1994 (GPS, +IEEE/ACM ToN); Goyal, Vin & Cheng, Start-time Fair Queueing (SIGCOMM'96 / ToN'97); +Shreedhar & Varghese, DRR (SIGCOMM'95); Waldspurger & Weihl, Stride Scheduling +(MIT TM-528, 1995); Cruz, A Calculus for Network Delay (IEEE T-IT 1991); Kingman's +formula; Harchol-Balter, Performance Modeling and Design of Computer Systems (CUP +2013). + +## When a cap alone DOES cut a starved tenant's wait (the load-bearing condition) + +A per-tenant concurrency cap on heavy tenant H cuts light tenant L's wait to +near-zero iff BOTH: + +1. Slot availability: sum of caps of all backlogged tenants other than L is < N + (the binding aggregate limit), so freed slots exist that capped tenants can + never occupy; and +2. Eligibility-aware serve order: the dequeue selects the oldest ELIGIBLE item, + skipping items whose tenant is at cap, so L's head is reachable without + draining H's older items first. + +If (2) fails (single global age-ordered list with a head-blocking consumer), the +cap does NOT help L and with strict head-blocking makes L's wait WORSE: H's +backlog drains at k slots instead of N while the freed N-k slots sit idle +(head-of-line blocking; Parekh-Gallager's FCFS-gives-no-isolation remark). + +Trigger's CK dequeue is oldest-ELIGIBLE-first: the CK Lua gates each variant at +its per-key concurrencyLimit and skips a variant that is at its limit, moving to +the next-oldest eligible. So condition (2) holds structurally. That is WHY per-key +caps can work for wait here, and would not work on a head-blocking FIFO. + +## Where caps fail even with eligibility-aware order (the sybil split) + +Concurrency keys are client-chosen. A heavy tenant spreads its backlog across many +CK variants, each under its own per-key cap, all "eligible". The binding +constraint becomes the base-queue total cap (or env limit); oldest-first then +serves the adversary's older backlog across its many keys before a newcomer's +head. Per-key caps bound nothing in aggregate, because sum of per-key caps is +unbounded relative to the queue cap when keys are dynamic. The light tenant's wait +scales with the adversary's total queued backlog, which no per-key cap regulates. +Within a base queue, the fix under adversarial arrival is an order change +(round-robin / fair queueing across keys), not another cap. + +## Known static-cap failure modes (practice) + +2DFQ (Mace et al., SIGCOMM 2016): "Rate limiters, typically implemented as token +buckets, are not designed to provide fairness at short time intervals ... they can +either underutilize the system or concurrent bursts can overload it without +providing any further fairness guarantees." Their desirable-properties section +requires the scheduler be work-conserving, which "precludes the use of ad-hoc +throttling mechanisms to control misbehaving tenants." Pisces (OSDI 2012) and DRF +(NSDI 2011) both exist because static slot partitioning under/over-utilises under +skewed demand. Netflix concurrency-limits: static limits (Limit = RPS * latency, +i.e. Little's Law) "quickly go out of date"; hence adaptive. + +The price of caps when they DO give order-independent wait bounds: they degenerate +into a static partition (sum of caps <= K), which is non-work-conserving +(utilisation ceiling of sum-of-caps even when one tenant could use all K) and +needs bounded, pre-known tenant cardinality. + +## Production precedent: caps and scheduling are LAYERED, not either/or + +- Kubernetes API Priority & Fairness (the closest analog): total server + concurrency split into per-priority-level "seats" (a cap), THEN shuffle-sharded + fair queueing decides dispatch order within a level. Kubernetes hit exactly the + cap-alone failure with max-inflight before APF, and the fix ADDED fair queueing + on top of the existing cap rather than replacing it. (K8s docs; KEP-1040.) +- SQL Server Resource Governor: pool MIN/MAX/CAP percent (caps + reservation) + + workload-group IMPORTANCE biasing the scheduler's order. Same shape. +- YARN Fair/Capacity scheduler: minShare floor + maxResources cap + weighted + fair-share ordering, with preemption to reclaim the floor. +- Mesos/DRF, Borg: quota/admission caps layered with fair-share or priority order. +- Amazon SQS fair queues: fairness metric is DWELL TIME, fixed by reprioritising + delivery ORDER when a tenant's in-flight share is disproportionate, and + explicitly does NOT rate-limit per tenant. Order is the dwell-time knob; + occupancy limits alone were not it. +- Envoy / gRPC / Postgres connection caps: caps ALONE, and they claim overload + PROTECTION, not fairness. AWS token-bucket quotas claim "fairness" only in the + weaker selective-throttling sense, and rely on an elastic, rarely-saturated + fleet. + +Condition for caps-alone to suffice in practice: sum of caps comfortably below (or +elastically kept below) real capacity, and shed/rejected work acceptable, i.e. the +system never holds a contended backlog it must drain in some order. The moment you +hold a queue of admitted-but-waiting work at saturation, the serve order IS the +fairness policy. + +## CoDel (confirms the prior spike's "CoDel disproven" verdict) + +CoDel is an AQM: it bounds standing-queue delay by DROPPING packets when the +minimum sojourn over a window stays above target (5ms/100ms defaults; RFC 8289). +It is not a scheduler and gives no inter-flow fairness (RFC 7567: queue management +and scheduling are complementary, not substitutes). FQ-CoDel gets all its fairness +from a DRR scheduler; CoDel is only the per-flow AQM inside each sub-queue (RFC +8290). In a durable run queue that can never drop work, CoDel's actuator is gone: +reordering conserves total queue and total work, so hoisting one item's sojourn +down pushes others' up. Hoisting the stalest item to the front of an already +age-ordered queue re-applies the age bias the base already has, so it is a no-op +at best and a dominant-tenant amplifier at worst (a tenant that dumps a big +backlog owns the entire stale set). Facebook's server-side CoDel adaptation ("Fail +at Scale", ACM Queue 2015) keeps the drop (stale requests expire) and reorders +adaptive-LIFO (newest first), the opposite of stalest-first hoisting. diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/capsFairness.bench.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/capsFairness.bench.test.ts new file mode 100644 index 0000000000..0521dc7ded --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/capsFairness.bench.test.ts @@ -0,0 +1,239 @@ +import { redisTest } from "@internal/testcontainers"; +import type { RedisOptions } from "@internal/redis"; +import { describe } from "vitest"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCkScenario } from "./harness/ckDriver.js"; +import { + buildWorkload, + weightsOf, + type WorkloadConfig, +} from "../fairness-spike/harness/workload.js"; +import type { RunMetrics, GroupMetrics } from "../fairness-spike/harness/metrics.js"; +import { BaselineCk, SfqCk, DrrCk, type CkDiscipline } from "./disciplines.js"; + +/** + * Caps vs scheduling. Runs the plan-of-record's concurrency CAPS (Phase-2 per-key + * limit via the real per-queue concurrency gate, Phase-1 total cap via a + * driver-side group gate) head-to-head against the scheduling disciplines + * (SFQ/DRR) on identical scenarios, through the real CK-dequeue Lua at + * maxCount = 1. + * + * A "treatment" is (order discipline, per-key cap?, total cap?). Caps are + * admission settings, not disciplines: the per-key cap is the real Lua's native + * per-variant gate (oldest-eligible-first, true age order), the total cap is the + * driver refusing to admit past the group ceiling. + * + * Headline = the light (starved) key's wait. makespan = work-conservation signal. + * contention share = directional. + */ + +const RESULTS_DIR = join(dirname(fileURLToPath(import.meta.url)), "results"); +const SEEDS = ["seed-a", "seed-b", "seed-c"]; +const ENV_LIMIT = 4; +const PER_KEY_CAP = 2; // heavy key(s) bound to half the env +const TOTAL_CAP = 2; // per-task total ceiling, below env + +type Treatment = { + label: string; + makeDiscipline: () => CkDiscipline; + perKeyCap?: number; + totalCap?: number; +}; + +const TREATMENTS: Treatment[] = [ + { label: "baseline", makeDiscipline: () => new BaselineCk() }, + { label: "perKeyCap", makeDiscipline: () => new BaselineCk(), perKeyCap: PER_KEY_CAP }, + { label: "totalCap", makeDiscipline: () => new BaselineCk(), totalCap: TOTAL_CAP }, + // the plan-of-record's shipped combined config: total cap AND per-key cap + { label: "total+perKey", makeDiscipline: () => new BaselineCk(), perKeyCap: PER_KEY_CAP, totalCap: TOTAL_CAP }, + { label: "sfq", makeDiscipline: () => new SfqCk() }, + { label: "drr", makeDiscipline: () => new DrrCk() }, + { label: "perKeyCap+sfq", makeDiscipline: () => new SfqCk(), perKeyCap: PER_KEY_CAP }, + // both caps plus a fair order (the fully-layered end state) + { label: "total+perKey+sfq", makeDiscipline: () => new SfqCk(), perKeyCap: PER_KEY_CAP, totalCap: TOTAL_CAP }, +]; + +type CapScenario = { + config: Omit; + /** the key whose wait is the headline (a starved light key, or the heavy key for heavy-idle) */ + lightKey: string; +}; + +function sybilHeavy(count: number, runsEach: number) { + return Array.from({ length: count }, (_, i) => ({ + tenantId: `heavy-${i}`, + runCount: runsEach, + holdMsMean: 25, + })); +} + +const SCENARIOS: Record = { + // one heavy key floods (old head), four light keys trickle in later. One heavy + // key => a per-key cap frees slots the light keys can take. + ckSkew: { + lightKey: "light-1", + config: { + envConcurrencyLimit: ENV_LIMIT, + tenants: [ + { tenantId: "heavy", runCount: 240, holdMsMean: 25 }, + { tenantId: "light-1", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, + { tenantId: "light-2", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, + { tenantId: "light-3", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, + { tenantId: "light-4", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, + ], + }, + }, + + // a bulk backlog plus two keys trickling in slowly + ckTrickle: { + lightKey: "trickle-1", + config: { + envConcurrencyLimit: ENV_LIMIT, + tenants: [ + { tenantId: "bulk", runCount: 240, holdMsMean: 25 }, + { tenantId: "trickle-1", runCount: 30, arrival: "poisson", ratePerSec: 25, holdMsMean: 25 }, + { tenantId: "trickle-2", runCount: 30, arrival: "poisson", ratePerSec: 25, holdMsMean: 25 }, + ], + }, + }, + + // sybil split: one attacker spreads its backlog across 20 concurrency keys, each + // with a backlog that stays non-empty through the light key's whole arrival + // window. Each attacker key is under the same per-key cap, but the cap frees no + // aggregate slot (attacker keys fill env, and as one empties the next attacker + // key's old head is served before the newer light key). The real CK Lua also + // only scans the 3 oldest-scored variants per call (ZRANGEBYSCORE ... LIMIT 0, + // maxCount*3), so with 20 attacker heads ahead of it the light head is never in + // the window. Only a fair order rescues the light key. + ckSybil: { + lightKey: "light", + config: { + envConcurrencyLimit: ENV_LIMIT, + tenants: [ + ...sybilHeavy(20, 15), + { tenantId: "light", runCount: 20, arrival: "poisson", ratePerSec: 40, holdMsMean: 25 }, + ], + }, + }, + + // work-conservation: one heavy key alone with a big backlog. A per-key or total + // cap throttles it below the env limit and idles slots, inflating makespan; a + // scheduler uses the whole env. Headline here is makespan, not wait. + ckHeavyIdle: { + lightKey: "heavy", + config: { + envConcurrencyLimit: ENV_LIMIT, + tenants: [{ tenantId: "heavy", runCount: 200, holdMsMean: 25 }], + }, + }, +}; + +function stats(xs: number[]) { + return { + mean: xs.reduce((a, b) => a + b, 0) / xs.length, + min: Math.min(...xs), + max: Math.max(...xs), + }; +} + +function fmt(n: number, d = 0): string { + return Number.isFinite(n) ? n.toFixed(d) : String(n); +} + +function waitOf(metrics: RunMetrics, key: string): number { + return metrics.perGroup.find((g) => g.groupId === key)?.meanWait ?? 0; +} + +function worstWaitOf(metrics: RunMetrics): number { + return metrics.perGroup.length ? Math.max(...metrics.perGroup.map((g) => g.meanWait)) : 0; +} + +describe("caps vs scheduling bench", () => { + mkdirSync(RESULTS_DIR, { recursive: true }); + + for (const [scenarioName, scenario] of Object.entries(SCENARIOS)) { + redisTest( + `caps scenario: ${scenarioName}`, + async ({ redisContainer }) => { + const runs = new Map>(); + + for (const seed of SEEDS) { + const config: WorkloadConfig = { ...scenario.config, seed }; + const workload = buildWorkload(config); + const expectedTotal = workload.tenants.reduce((n, t) => n + t.runCount, 0); + + for (const treatment of TREATMENTS) { + const redis: RedisOptions = { + keyPrefix: `rq:caps:${scenarioName}:${treatment.label}:${seed}:`, + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + const metrics = await runCkScenario({ + redis, + discipline: treatment.makeDiscipline(), + workload, + perKeyCap: treatment.perKeyCap, + totalCap: treatment.totalCap, + }); + if (metrics.totalDequeued !== expectedTotal) { + throw new Error( + `${scenarioName}/${treatment.label}/${seed}: dequeued ${metrics.totalDequeued} of ${expectedTotal}` + ); + } + const arr = runs.get(treatment.label) ?? []; + arr.push({ seed, metrics }); + runs.set(treatment.label, arr); + } + } + + const perTreatment = [...runs.entries()].map(([label, rs]) => ({ + treatment: label, + lightWait: stats(rs.map((r) => waitOf(r.metrics, scenario.lightKey))), + worstWait: stats(rs.map((r) => worstWaitOf(r.metrics))), + makespan: stats(rs.map((r) => r.metrics.makespanMs)), + contentionWorst: stats(rs.map((r) => r.metrics.contentionWorstShareOverWeight)), + detailSeed0: rs[0].metrics.perGroup as GroupMetrics[], + })); + + const firstWorkload = buildWorkload({ ...scenario.config, seed: SEEDS[0] }); + writeFileSync( + join(RESULTS_DIR, `caps-${scenarioName}.json`), + JSON.stringify( + { + scenario: scenarioName, + seeds: SEEDS, + envLimit: ENV_LIMIT, + perKeyCap: PER_KEY_CAP, + totalCap: TOTAL_CAP, + lightKey: scenario.lightKey, + weights: weightsOf(firstWorkload), + perTreatment, + }, + null, + 2 + ) + ); + + const lines = [ + ``, + `### caps: ${scenarioName} (${SEEDS.length} seeds, env=${ENV_LIMIT}, perKeyCap=${PER_KEY_CAP}, totalCap=${TOTAL_CAP}, light=${scenario.lightKey})`, + `treatment lightWait worstWait makespan contWorstS/W`, + ...perTreatment.map( + (r) => + `${r.treatment.padEnd(16)} ${fmt(r.lightWait.mean).padStart(8)} ${fmt( + r.worstWait.mean + ).padStart(8)} ${fmt(r.makespan.mean).padStart(7)} ${fmt( + r.contentionWorst.mean, + 3 + ).padStart(7)}` + ), + ``, + ]; + process.stdout.write(lines.join("\n") + "\n"); + }, + 300_000 + ); + } +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/ckFairnessSpike.bench.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/ckFairnessSpike.bench.test.ts new file mode 100644 index 0000000000..91001d8be1 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/ckFairnessSpike.bench.test.ts @@ -0,0 +1,107 @@ +import { redisTest } from "@internal/testcontainers"; +import type { RedisOptions } from "@internal/redis"; +import { describe } from "vitest"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCkScenario } from "./harness/ckDriver.js"; +import { CK_SCENARIOS } from "./harness/ckScenarios.js"; +import { buildWorkload, weightsOf, type WorkloadConfig } from "../fairness-spike/harness/workload.js"; +import type { RunMetrics, GroupMetrics } from "../fairness-spike/harness/metrics.js"; +import { BaselineCk, SfqCk, DrrCk, StrideCk, CodelCk, type CkDiscipline } from "./disciplines.js"; + +const RESULTS_DIR = join(dirname(fileURLToPath(import.meta.url)), "results"); +const SEEDS = ["seed-a", "seed-b", "seed-c"]; + +function makeDisciplines(): CkDiscipline[] { + return [ + new BaselineCk(), + new SfqCk(), + new DrrCk(), + new StrideCk(), + new CodelCk(new SfqCk(), 200, 100), + new CodelCk(new BaselineCk(), 200, 100), + ]; +} + +function stats(xs: number[]) { + return { + mean: xs.reduce((a, b) => a + b, 0) / xs.length, + min: Math.min(...xs), + max: Math.max(...xs), + }; +} + +function fmt(n: number, d = 3): string { + return Number.isFinite(n) ? n.toFixed(d) : String(n); +} + +describe("ck fairness spike bench", () => { + mkdirSync(RESULTS_DIR, { recursive: true }); + + for (const [scenarioName, baseConfig] of Object.entries(CK_SCENARIOS)) { + redisTest( + `ck scenario: ${scenarioName}`, + async ({ redisContainer }) => { + const runs = new Map>(); + + for (const seed of SEEDS) { + const config: WorkloadConfig = { ...baseConfig, seed }; + const workload = buildWorkload(config); + const expectedTotal = workload.tenants.reduce((n, t) => n + t.runCount, 0); + + for (const discipline of makeDisciplines()) { + const redis: RedisOptions = { + keyPrefix: `rq:ck:${scenarioName}:${discipline.name}:${seed}:`, + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + const metrics = await runCkScenario({ redis, discipline, workload }); + if (metrics.totalDequeued !== expectedTotal) { + throw new Error( + `${scenarioName}/${discipline.name}/${seed}: dequeued ${metrics.totalDequeued} of ${expectedTotal}` + ); + } + const arr = runs.get(discipline.name) ?? []; + arr.push({ seed, metrics }); + runs.set(discipline.name, arr); + } + } + + const perDiscipline = [...runs.entries()].map(([name, rs]) => ({ + selector: name, + contentionWorstShareOverWeight: stats( + rs.map((r) => r.metrics.contentionWorstShareOverWeight) + ), + contentionJain: stats(rs.map((r) => r.metrics.contentionJain)), + detailSeed0: rs[0].metrics.perGroup as GroupMetrics[], + })); + + const firstWorkload = buildWorkload({ ...baseConfig, seed: SEEDS[0] }); + writeFileSync( + join(RESULTS_DIR, `${scenarioName}.json`), + JSON.stringify( + { scenario: scenarioName, seeds: SEEDS, weights: weightsOf(firstWorkload), perDiscipline }, + null, + 2 + ) + ); + + const lines = [ + ``, + `### ${scenarioName} (${SEEDS.length} seeds)`, + `discipline contWorstS/W (min..max) contJain`, + ...perDiscipline.map((r) => { + const c = r.contentionWorstShareOverWeight; + return `${r.selector.padEnd(15)} ${fmt(c.mean).padStart(7)} (${fmt(c.min)}..${fmt( + c.max + )}) ${fmt(r.contentionJain.mean).padStart(6)}`; + }), + ``, + ]; + process.stdout.write(lines.join("\n") + "\n"); + }, + 300_000 + ); + } +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/ckReader.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/ckReader.ts new file mode 100644 index 0000000000..0fd5c39cd4 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/ckReader.ts @@ -0,0 +1,44 @@ +import type { Redis } from "@internal/redis"; +import type { RunQueueKeyProducer } from "../types.js"; + +/** + * Reads the concurrency-key index for a base queue. `ckIndexKey` is a ZSET whose + * members are CK-queue names (the queue key WITHOUT the redis keyPrefix, since + * the dequeue Lua re-prepends the prefix) scored by each CK-queue's head-message + * timestamp. This is the structure the per-CK dequeue picks from. + */ + +export type ActiveCk = { + /** the ckIndex member: CK-queue key without the redis keyPrefix */ + ckQueue: string; + concurrencyKey: string; + headScore: number; +}; + +export class CkReader { + constructor( + private readonly redis: Redis, + private readonly keys: RunQueueKeyProducer, + private readonly keyPrefix: string + ) {} + + async readActiveCks(baseQueue: string): Promise { + const ckIndexKey = this.keys.ckIndexKeyFromQueue(baseQueue); + const raw = await this.redis.zrange(ckIndexKey, 0, -1, "WITHSCORES"); + + const out: ActiveCk[] = []; + for (let i = 0; i + 1 < raw.length; i += 2) { + const ckQueue = raw[i]; + const headScore = Number(raw[i + 1]); + // Members are stored without the keyPrefix; descriptorFromQueue parses the + // structural key, which does not include the prefix. + const descriptor = this.keys.descriptorFromQueue(ckQueue); + out.push({ + ckQueue, + concurrencyKey: descriptor.concurrencyKey ?? "__none__", + headScore, + }); + } + return out; + } +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/ckRescorer.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/ckRescorer.ts new file mode 100644 index 0000000000..d63722a452 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/ckRescorer.ts @@ -0,0 +1,29 @@ +import type { Redis } from "@internal/redis"; +import type { RunQueueKeyProducer } from "../types.js"; + +/** + * Rewrites the ckIndex ZSET scores so the unmodified CK-dequeue Lua serves + * CK-queues in a discipline's order. The Lua picks lowest-score-first among + * members with score <= now, so we assign the highest-priority CK the smallest + * score. All scores are placed just below `now` in descending priority, so they + * stay eligible (<= now) and strictly order-preserving. + * + * `order` is the CK-queue members (ckIndex member form, i.e. no keyPrefix), + * highest priority first. + */ +export async function rescoreCkIndex( + redis: Redis, + keys: RunQueueKeyProducer, + baseQueue: string, + order: string[], + now: number +): Promise { + if (order.length === 0) return; + const ckIndexKey = keys.ckIndexKeyFromQueue(baseQueue); + const args: (string | number)[] = []; + for (let i = 0; i < order.length; i++) { + // smallest score for the best (i=0), all < now + args.push(now - (order.length - i), order[i]); + } + await redis.zadd(ckIndexKey, ...args); +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/crossTaskCaps.bench.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/crossTaskCaps.bench.test.ts new file mode 100644 index 0000000000..fd78732a9d --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/crossTaskCaps.bench.test.ts @@ -0,0 +1,193 @@ +import { redisTest } from "@internal/testcontainers"; +import { describe } from "vitest"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createRedisClient } from "@internal/redis"; +import { RunQueueFullKeyProducer } from "../keyProducer.js"; +import { FairQueueSelectionStrategy } from "../fairQueueSelectionStrategy.js"; +import { runScenario } from "../fairness-spike/harness/driver.js"; +import { SfqStrategy } from "../fairness-spike/strategies/sfqStrategy.js"; +import { GROUP_SEPARATOR } from "../fairness-spike/types.js"; +import { + buildWorkload, + weightsOf, + type WorkloadConfig, +} from "../fairness-spike/harness/workload.js"; +import type { RunMetrics, GroupMetrics } from "../fairness-spike/harness/metrics.js"; + +/** + * The total cap's REAL job: cross-TASK isolation. Two keyless tasks (base queues) + * share one env; a heavy task floods it and starves a light task. This is the + * problem #2617's total cap is for, and it is a DIFFERENT problem from the + * cross-KEY starvation the caps bench showed the total cap does not fix. + * + * The total cap on the heavy task is the real per-queue concurrency gate + * (updateQueueConcurrencyLimits); for a keyless task the per-queue limit is the + * per-task total (one base queue, no ck variants to sum), so this is faithful. + * Compared against the production FairQueueSelectionStrategy (baseline) and the + * spike SFQ selector, both driving the real RunQueue + testDequeueFromMasterQueue. + */ + +const RESULTS_DIR = join(dirname(fileURLToPath(import.meta.url)), "results"); +const SEEDS = ["seed-a", "seed-b", "seed-c"]; +const ENV_LIMIT = 4; +const HEAVY_TOTAL_CAP = 2; + +const keys = new RunQueueFullKeyProducer(); +const q = (tenant: string) => `${tenant}${GROUP_SEPARATOR}0`; + +type CrossScenario = { + config: Omit; + heavy: string; + lightKey: string; +}; + +const SCENARIOS: Record = { + // heavy task floods, two light tasks trickle in. All keyless (queueCount 1). + crossTaskSkew: { + heavy: "heavy", + lightKey: "light-1", + config: { + envConcurrencyLimit: ENV_LIMIT, + tenants: [ + { tenantId: "heavy", runCount: 240, holdMsMean: 25 }, + { tenantId: "light-1", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, + { tenantId: "light-2", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, + ], + }, + }, +}; + +type RedisOpts = { keyPrefix: string; host: string; port: number }; +type Treatment = { + label: string; + // returns the strategy and an optional client to quit afterwards. The real + // FairQueueSelectionStrategy takes RedisOptions and owns its own client; the + // spike SfqStrategy takes a live client. + makeStrategy: (redis: RedisOpts) => { strategy: any; client?: ReturnType }; + capHeavy?: boolean; +}; + +const TREATMENTS: Treatment[] = [ + { + label: "baseline(fairqueue)", + makeStrategy: (redis) => ({ strategy: new FairQueueSelectionStrategy({ redis, keys }) }), + }, + { + label: "heavyTotalCap", + makeStrategy: (redis) => ({ strategy: new FairQueueSelectionStrategy({ redis, keys }) }), + capHeavy: true, + }, + { + label: "sfq", + makeStrategy: (redis) => { + const client = createRedisClient(redis); + return { strategy: new SfqStrategy({ redis: client, keys }), client }; + }, + }, +]; + +function stats(xs: number[]) { + return { + mean: xs.reduce((a, b) => a + b, 0) / xs.length, + min: Math.min(...xs), + max: Math.max(...xs), + }; +} + +function fmt(n: number, d = 0): string { + return Number.isFinite(n) ? n.toFixed(d) : String(n); +} + +function waitOf(metrics: RunMetrics, key: string): number { + return metrics.perGroup.find((g) => g.groupId === key)?.meanWait ?? 0; +} + +describe("cross-task total cap bench", () => { + mkdirSync(RESULTS_DIR, { recursive: true }); + + for (const [scenarioName, scenario] of Object.entries(SCENARIOS)) { + redisTest( + `cross-task scenario: ${scenarioName}`, + async ({ redisContainer }) => { + const runs = new Map>(); + + for (const seed of SEEDS) { + const config: WorkloadConfig = { ...scenario.config, seed }; + const workload = buildWorkload(config); + const expectedTotal = workload.tenants.reduce((n, t) => n + t.runCount, 0); + + for (const treatment of TREATMENTS) { + const redis = { + keyPrefix: `rq:xtask:${scenarioName}:${treatment.label}:${seed}:`, + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + const { strategy, client } = treatment.makeStrategy(redis); + const metrics = await runScenario({ + redis, + strategy, + workload, + perQueueCap: treatment.capHeavy ? { [q(scenario.heavy)]: HEAVY_TOTAL_CAP } : undefined, + }); + await client?.quit(); + if (metrics.totalDequeued !== expectedTotal) { + throw new Error( + `${scenarioName}/${treatment.label}/${seed}: dequeued ${metrics.totalDequeued} of ${expectedTotal}` + ); + } + const arr = runs.get(treatment.label) ?? []; + arr.push({ seed, metrics }); + runs.set(treatment.label, arr); + } + } + + const perTreatment = [...runs.entries()].map(([label, rs]) => ({ + treatment: label, + lightWait: stats(rs.map((r) => waitOf(r.metrics, scenario.lightKey))), + heavyWait: stats(rs.map((r) => waitOf(r.metrics, scenario.heavy))), + makespan: stats(rs.map((r) => r.metrics.makespanMs)), + contentionWorst: stats(rs.map((r) => r.metrics.contentionWorstShareOverWeight)), + detailSeed0: rs[0].metrics.perGroup as GroupMetrics[], + })); + + const firstWorkload = buildWorkload({ ...scenario.config, seed: SEEDS[0] }); + writeFileSync( + join(RESULTS_DIR, `xtask-${scenarioName}.json`), + JSON.stringify( + { + scenario: scenarioName, + seeds: SEEDS, + envLimit: ENV_LIMIT, + heavyTotalCap: HEAVY_TOTAL_CAP, + lightKey: scenario.lightKey, + weights: weightsOf(firstWorkload), + perTreatment, + }, + null, + 2 + ) + ); + + const lines = [ + ``, + `### cross-task: ${scenarioName} (${SEEDS.length} seeds, env=${ENV_LIMIT}, heavyTotalCap=${HEAVY_TOTAL_CAP}, light=${scenario.lightKey})`, + `treatment lightWait heavyWait makespan contWorstS/W`, + ...perTreatment.map( + (r) => + `${r.treatment.padEnd(19)} ${fmt(r.lightWait.mean).padStart(8)} ${fmt( + r.heavyWait.mean + ).padStart(8)} ${fmt(r.makespan.mean).padStart(7)} ${fmt( + r.contentionWorst.mean, + 3 + ).padStart(7)}` + ), + ``, + ]; + process.stdout.write(lines.join("\n") + "\n"); + }, + 300_000 + ); + } +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/disciplines.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/disciplines.ts new file mode 100644 index 0000000000..da917205ae --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/disciplines.ts @@ -0,0 +1,201 @@ +import type { ActiveCk } from "./ckReader.js"; + +/** + * Fair-queueing disciplines keyed by concurrency key. Each returns an order over + * the active CK-queues (best priority first); the driver writes that order into + * ckIndex so the real dequeue Lua follows it. Ports the base-queue spike's vetted + * logic (monotonic virtual-time floor, deficit round robin, stride) with all CKs + * at equal weight, since concurrency keys carry no configured weight in + * production. + */ +export interface CkDiscipline { + readonly name: string; + /** false for baseline: leave ckIndex scores as the Lua maintains them (age order) */ + readonly rescore: boolean; + order(active: ActiveCk[], now: number): string[]; + onServiced(concurrencyKey: string): void; + reset(): void; +} + +function byPriority(active: ActiveCk[], priorityOf: (ck: string) => number): string[] { + return [...active] + .sort( + (a, b) => + priorityOf(a.concurrencyKey) - priorityOf(b.concurrencyKey) || + a.headScore - b.headScore || + (a.ckQueue < b.ckQueue ? -1 : 1) + ) + .map((a) => a.ckQueue); +} + +/** Production behaviour: oldest head first. No rescore. */ +export class BaselineCk implements CkDiscipline { + readonly name = "baseline"; + readonly rescore = false; + order(active: ActiveCk[]): string[] { + return byPriority(active, () => 0); // headScore tiebreak gives age order + } + onServiced(): void {} + reset(): void {} +} + +/** Start-time fair queueing with a monotonic floor (CFS min_vruntime analogue). */ +export class SfqCk implements CkDiscipline { + readonly name = "sfq"; + readonly rescore = true; + private clock = new Map(); + private floor = 0; + private readonly quantum: number; + constructor(quantum = 1) { + this.quantum = quantum; + } + reset(): void { + this.clock = new Map(); + this.floor = 0; + } + private startTag(ck: string): number { + return Math.max(this.clock.get(ck) ?? this.floor, this.floor); + } + order(active: ActiveCk[]): string[] { + let min = Infinity; + for (const a of active) min = Math.min(min, this.clock.get(a.concurrencyKey) ?? this.floor); + if (Number.isFinite(min)) this.floor = Math.max(this.floor, min); + return byPriority(active, (ck) => this.startTag(ck)); + } + onServiced(ck: string): void { + this.clock.set(ck, this.startTag(ck) + this.quantum); + } +} + +/** Deficit round robin. */ +export class DrrCk implements CkDiscipline { + readonly name = "drr"; + readonly rescore = true; + private deficit = new Map(); + private ring: string[] = []; + private cursor = 0; + private readonly quantum: number; + constructor(quantum = 1) { + this.quantum = quantum; + } + reset(): void { + this.deficit = new Map(); + this.ring = []; + this.cursor = 0; + } + order(active: ActiveCk[]): string[] { + const activeKeys = new Set(active.map((a) => a.concurrencyKey)); + for (const k of activeKeys) if (!this.ring.includes(k)) this.ring.push(k); + this.ring = this.ring.filter((k) => activeKeys.has(k)); + if (this.ring.length === 0) return []; + if (this.cursor >= this.ring.length) this.cursor = 0; + + let winner = this.ring[this.cursor]; + for (let steps = 0; steps < this.ring.length; steps++) { + const k = this.ring[this.cursor]; + if ((this.deficit.get(k) ?? 0) < 1) { + this.deficit.set(k, (this.deficit.get(k) ?? 0) + this.quantum); + } + if ((this.deficit.get(k) ?? 0) >= 1) { + winner = k; + break; + } + this.cursor = (this.cursor + 1) % this.ring.length; + } + + const rank = new Map(); + rank.set(winner, -1); + for (let i = 0; i < this.ring.length; i++) { + const k = this.ring[(this.cursor + i) % this.ring.length]; + if (!rank.has(k)) rank.set(k, i); + } + return byPriority(active, (ck) => rank.get(ck) ?? this.ring.length); + } + onServiced(ck: string): void { + this.deficit.set(ck, (this.deficit.get(ck) ?? 0) - 1); + if ((this.deficit.get(ck) ?? 0) < 1) { + const idx = this.ring.indexOf(ck); + if (idx !== -1) this.cursor = (idx + 1) % this.ring.length; + } + } +} + +/** Stride scheduling with a monotonic pass floor. */ +export class StrideCk implements CkDiscipline { + readonly name = "stride"; + readonly rescore = true; + private pass = new Map(); + private floor = 0; + private readonly stride1: number; + constructor(stride1 = 1_000_000) { + this.stride1 = stride1; + } + reset(): void { + this.pass = new Map(); + this.floor = 0; + } + private passOf(ck: string): number { + return Math.max(this.pass.get(ck) ?? this.floor, this.floor); + } + order(active: ActiveCk[]): string[] { + let min = Infinity; + for (const a of active) min = Math.min(min, this.pass.get(a.concurrencyKey) ?? this.floor); + if (Number.isFinite(min)) this.floor = Math.max(this.floor, min); + return byPriority(active, (ck) => this.passOf(ck)); + } + onServiced(ck: string): void { + this.pass.set(ck, this.passOf(ck) + this.stride1); + } +} + +/** CoDel-style staleness wrapper: hoists keys whose sojourn stays above target. */ +export class CodelCk implements CkDiscipline { + readonly name: string; + readonly rescore = true; + private firstAbove = new Map(); + constructor( + private readonly base: CkDiscipline, + private readonly targetMs: number, + private readonly intervalMs: number + ) { + this.name = `codel(${base.name})`; + } + reset(): void { + this.firstAbove = new Map(); + this.base.reset(); + } + order(active: ActiveCk[], now: number): string[] { + const baseOrder = this.base.order(active, now); + const ckToKey = new Map(active.map((a) => [a.ckQueue, a.concurrencyKey])); + const minHead = new Map(); + for (const a of active) { + const cur = minHead.get(a.concurrencyKey); + if (cur === undefined || a.headScore < cur) minHead.set(a.concurrencyKey, a.headScore); + } + const escalating = new Set(); + const seen = new Set(); + for (const [ck, head] of minHead) { + seen.add(ck); + if (now - head > this.targetMs) { + const since = this.firstAbove.get(ck) ?? now; + this.firstAbove.set(ck, since); + if (now - since >= this.intervalMs) escalating.add(ck); + } else { + this.firstAbove.delete(ck); + } + } + for (const k of [...this.firstAbove.keys()]) if (!seen.has(k)) this.firstAbove.delete(k); + if (escalating.size === 0) return baseOrder; + const hot: string[] = []; + const cold: string[] = []; + for (const ckQueue of baseOrder) { + const ck = ckToKey.get(ckQueue); + if (ck !== undefined && escalating.has(ck)) hot.push(ckQueue); + else cold.push(ckQueue); + } + return [...hot, ...cold]; + } + onServiced(ck: string): void { + this.base.onServiced(ck); + } +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/harness/ckDriver.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/harness/ckDriver.ts new file mode 100644 index 0000000000..cc89f62b8b --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/harness/ckDriver.ts @@ -0,0 +1,203 @@ +import { trace } from "@internal/tracing"; +import { Logger } from "@trigger.dev/core/logger"; +import { createRedisClient, type RedisOptions } from "@internal/redis"; +import { Decimal } from "@trigger.dev/database"; +import { RunQueue } from "../../index.js"; +import { FairQueueSelectionStrategy } from "../../fairQueueSelectionStrategy.js"; +import { RunQueueFullKeyProducer } from "../../keyProducer.js"; +import type { InputPayload } from "../../types.js"; +import { + computeMetrics, + type DequeueEvent, + type RunMetrics, +} from "../../fairness-spike/harness/metrics.js"; +import { + expandEvents, + totalsOf, + weightsOf, + type WorkloadSpec, +} from "../../fairness-spike/harness/workload.js"; +import { CkReader } from "../ckReader.js"; +import { rescoreCkIndex } from "../ckRescorer.js"; +import type { CkDiscipline } from "../disciplines.js"; + +const keys = new RunQueueFullKeyProducer(); +const ORG = "o-ck"; +const PROJECT = "p-ck"; +const ENV = "e-ck"; +const BASE_QUEUE = "task/base"; + +export type CkDriverConfig = { + redis: RedisOptions; + discipline: CkDiscipline; + workload: WorkloadSpec; + maxLogicalMs?: number; + /** + * Phase-2 per-key limit, modelled with the REAL Lua gate: sets the base queue's + * concurrencyLimit so the CK-dequeue Lua caps EACH ck variant's in-flight at + * this and skips an at-limit variant (oldest-eligible-first). Uniform across + * variants (Phase 2's per-key HGET override would cap only the heavy key, but + * since a light key never approaches the cap the effect is equivalent here). + * Undefined = no per-key cap (env limit is the only per-variant ceiling). + */ + perKeyCap?: number; + /** + * Phase-1 total cap, modelled at the driver: do not admit while total in-flight + * across all CK variants of the base queue (= :groupConcurrency SCARD) is at or + * above this. The real Lua has no group gate yet, so this one is driver-side. + */ + totalCap?: number; +}; + +function authenticatedEnv(limit: number) { + return { + id: ENV, + type: "PRODUCTION" as const, + maximumConcurrencyLimit: limit, + concurrencyLimitBurstFactor: new Decimal(1.0), + project: { id: PROJECT }, + organization: { id: ORG }, + }; +} + +/** + * Drives one workload across many concurrency keys under a single base queue. + * The fairness group is the concurrency key (= workload groupId). For rescore + * disciplines, the ckIndex scores are rewritten each round so the real + * CK-dequeue Lua serves keys in discipline order; the baseline leaves them as + * the Lua maintains them (production age order). + */ +export async function runCkScenario(config: CkDriverConfig): Promise { + const maxLogicalMs = config.maxLogicalMs ?? 600_000; + const limit = config.workload.envConcurrencyLimit; + const env = authenticatedEnv(limit); + + const admin = createRedisClient(config.redis); + await admin.flushdb(); + + const queue = new RunQueue({ + name: "rq-ck", + tracer: trace.getTracer("rq-ck"), + logger: new Logger("RunQueueCk", "error"), + defaultEnvConcurrency: limit, + shardCount: 1, + masterQueueConsumersDisabled: true, + workerOptions: { disabled: true }, + keys, + redis: config.redis, + queueSelectionStrategy: new FairQueueSelectionStrategy({ redis: config.redis, keys }), + }); + + // Phase-2 per-key cap: set the real per-queue concurrency limit the CK Lua + // gates each variant against. Faithful (native gate, true age order, no rescore + // pollution). + if (config.perKeyCap !== undefined) { + await queue.updateQueueConcurrencyLimits(env, BASE_QUEUE, config.perKeyCap); + } + + const reader = new CkReader(admin, keys, config.redis.keyPrefix ?? ""); + config.discipline.reset(); + + const sorted = expandEvents(config.workload); + const total = sorted.length; + const holdByRun = new Map(); + const enqueueByRun = new Map(); + for (const e of sorted) { + holdByRun.set(e.runId, e.holdMs); + enqueueByRun.set(e.runId, e.enqueueAtMs); + } + + const scoreBase = Date.now() - maxLogicalMs - 5_000; + const events: DequeueEvent[] = []; + const holding: Array<{ runId: string; releaseAtMs: number }> = []; + let enqCursor = 0; + let selectionRounds = 0; + let t = 0; + const startedAt = Date.now(); + + try { + while (events.length < total && t <= maxLogicalMs) { + for (let i = holding.length - 1; i >= 0; i--) { + if (holding[i].releaseAtMs <= t) { + const [h] = holding.splice(i, 1); + await queue.acknowledgeMessage(ORG, h.runId, { skipDequeueProcessing: true }); + } + } + + while (enqCursor < sorted.length && sorted[enqCursor].enqueueAtMs <= t) { + const e = sorted[enqCursor++]; + const message: InputPayload = { + runId: e.runId, + orgId: ORG, + projectId: PROJECT, + environmentId: ENV, + environmentType: "PRODUCTION", + queue: BASE_QUEUE, + concurrencyKey: e.groupId, + timestamp: scoreBase + e.enqueueAtMs, + attempt: 0, + }; + await queue.enqueueMessage({ env, message, workerQueue: ENV, skipDequeueProcessing: true }); + } + + const baseQueue = keys.queueKey(env, BASE_QUEUE, "any"); + const totalCap = config.totalCap; + let progressed = true; + while (progressed) { + // Phase-1 total cap: refuse to admit while group in-flight is at the cap + // (models the :groupConcurrency SCARD gate). Wait for a completion. + if (totalCap !== undefined && holding.length >= totalCap) break; + + if (config.discipline.rescore) { + const active = await reader.readActiveCks(baseQueue); + if (active.length > 0) { + // NOTE: order() runs before every dequeue attempt, including the + // terminal iteration whose dequeue returns nothing, so it can advance + // a discipline's state with no matching onServiced. For the shipped + // SFQ/DRR this is idempotent (SFQ floor already at the min clock; DRR's + // winner already has deficit >= 1). A non-idempotent discipline dropped + // in here would need its accounting made robust to that speculative call. + const order = config.discipline.order(active, scoreBase + t); + await rescoreCkIndex(admin, keys, baseQueue, order, Date.now()); + } + } + + const msgs = await queue.testDequeueFromMasterQueue(0, ENV, 1); + selectionRounds++; + progressed = msgs.length > 0; + for (const m of msgs) { + const ck = keys.descriptorFromQueue(m.message.queue).concurrencyKey ?? "__none__"; + const runId = m.messageId; + events.push({ + groupId: ck, + runId, + enqueueAtMs: enqueueByRun.get(runId) ?? 0, + dequeueAtMs: t, + }); + config.discipline.onServiced(ck); + holding.push({ runId, releaseAtMs: t + (holdByRun.get(runId) ?? 0) }); + } + } + + const candidates: number[] = []; + if (enqCursor < sorted.length) candidates.push(sorted[enqCursor].enqueueAtMs); + if (holding.length > 0) candidates.push(Math.min(...holding.map((h) => h.releaseAtMs))); + if (candidates.length === 0) break; + t = Math.max(t + 1, Math.min(...candidates)); + } + + return computeMetrics({ + events, + weights: weightsOf(config.workload), + totals: totalsOf(config.workload), + redisOps: selectionRounds, + wallClockMs: Date.now() - startedAt, + }); + } finally { + for (const h of holding) { + await queue.acknowledgeMessage(ORG, h.runId, { skipDequeueProcessing: true }).catch(() => {}); + } + await admin.quit(); + await queue.quit(); + } +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/harness/ckScenarios.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/harness/ckScenarios.ts new file mode 100644 index 0000000000..a17bca0ef9 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/harness/ckScenarios.ts @@ -0,0 +1,53 @@ +import type { WorkloadConfig } from "../../fairness-spike/harness/workload.js"; + +/** + * Concurrency-key scenarios. Each "tenant" in the workload is a concurrency key + * under one shared base queue. All keys are equal weight (concurrency keys carry + * no configured weight in production). Seed is set per-run by the bench. + * + * Every scenario gives keys genuinely divergent head ages (a bulk key with a + * same-time backlog keeps an old head; other keys arrive via poisson so their + * heads are distinct and later). This is deliberate: if all runs shared one + * enqueue timestamp the ckIndex scores would tie and the real Lua's + * ZRANGEBYSCORE would fall back to a lexicographic member-name tie-break, which + * would make the baseline look starved for reasons that have nothing to do with + * age order. We want the baseline to exercise the real age dynamic #2617 + * describes. + */ +export const CK_SCENARIOS: Record> = { + // #2617 classic: one key fires a big backlog at once (its head stays old), four + // other keys trickle in. Age-order serves the old-headed backlog to exhaustion + // and starves the others. + ckSkew: { + envConcurrencyLimit: 4, + tenants: [ + { tenantId: "heavy", runCount: 240, holdMsMean: 25 }, + { tenantId: "light-1", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, + { tenantId: "light-2", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, + { tenantId: "light-3", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, + { tenantId: "light-4", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, + ], + }, + + // Symmetric keys, all arriving via poisson at the same rate. Baseline should be + // roughly fair here (no key is systematically older), which is the sanity check. + ckBalanced: { + envConcurrencyLimit: 4, + tenants: [ + { tenantId: "k-a", runCount: 60, arrival: "poisson", ratePerSec: 20, holdMsMean: 25 }, + { tenantId: "k-b", runCount: 60, arrival: "poisson", ratePerSec: 20, holdMsMean: 25 }, + { tenantId: "k-c", runCount: 60, arrival: "poisson", ratePerSec: 20, holdMsMean: 25 }, + { tenantId: "k-d", runCount: 60, arrival: "poisson", ratePerSec: 20, holdMsMean: 25 }, + ], + }, + + // A bulk backlog plus two keys trickling in slowly, to exercise wait and CoDel. + ckTrickle: { + envConcurrencyLimit: 4, + tenants: [ + { tenantId: "bulk", runCount: 240, holdMsMean: 25 }, + { tenantId: "trickle-1", runCount: 30, arrival: "poisson", ratePerSec: 25, holdMsMean: 25 }, + { tenantId: "trickle-2", runCount: 30, arrival: "poisson", ratePerSec: 25, holdMsMean: 25 }, + ], + }, +}; diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckHeavyIdle.json b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckHeavyIdle.json new file mode 100644 index 0000000000..a03f6eea06 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckHeavyIdle.json @@ -0,0 +1,305 @@ +{ + "scenario": "ckHeavyIdle", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "envLimit": 4, + "perKeyCap": 2, + "totalCap": 2, + "lightKey": "heavy", + "weights": { + "heavy": 1 + }, + "perTreatment": [ + { + "treatment": "baseline", + "lightWait": { + "mean": 632.8383333333334, + "min": 574.22, + "max": 702.875 + }, + "worstWait": { + "mean": 632.8383333333334, + "min": 574.22, + "max": 702.875 + }, + "makespan": { + "mean": 1240.3333333333333, + "min": 1159, + "max": 1311 + }, + "contentionWorst": { + "mean": 1, + "min": 1, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 200, + "weight": 1, + "share": 1, + "contentionShareOverWeight": 0, + "meanWait": 702.875, + "waitP50": 723, + "waitP99": 1309, + "waitMax": 1311 + } + ] + }, + { + "treatment": "perKeyCap", + "lightWait": { + "mean": 1290.6516666666669, + "min": 1173.15, + "max": 1431.305 + }, + "worstWait": { + "mean": 1290.6516666666669, + "min": 1173.15, + "max": 1431.305 + }, + "makespan": { + "mean": 2507, + "min": 2344, + "max": 2638 + }, + "contentionWorst": { + "mean": 1, + "min": 1, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 200, + "weight": 1, + "share": 1, + "contentionShareOverWeight": 0, + "meanWait": 1431.305, + "waitP50": 1501, + "waitP99": 2623, + "waitMax": 2638 + } + ] + }, + { + "treatment": "totalCap", + "lightWait": { + "mean": 1290.6516666666669, + "min": 1173.15, + "max": 1431.305 + }, + "worstWait": { + "mean": 1290.6516666666669, + "min": 1173.15, + "max": 1431.305 + }, + "makespan": { + "mean": 2507, + "min": 2344, + "max": 2638 + }, + "contentionWorst": { + "mean": 1, + "min": 1, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 200, + "weight": 1, + "share": 1, + "contentionShareOverWeight": 0, + "meanWait": 1431.305, + "waitP50": 1501, + "waitP99": 2623, + "waitMax": 2638 + } + ] + }, + { + "treatment": "total+perKey", + "lightWait": { + "mean": 1290.6516666666669, + "min": 1173.15, + "max": 1431.305 + }, + "worstWait": { + "mean": 1290.6516666666669, + "min": 1173.15, + "max": 1431.305 + }, + "makespan": { + "mean": 2507, + "min": 2344, + "max": 2638 + }, + "contentionWorst": { + "mean": 1, + "min": 1, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 200, + "weight": 1, + "share": 1, + "contentionShareOverWeight": 0, + "meanWait": 1431.305, + "waitP50": 1501, + "waitP99": 2623, + "waitMax": 2638 + } + ] + }, + { + "treatment": "sfq", + "lightWait": { + "mean": 632.8383333333334, + "min": 574.22, + "max": 702.875 + }, + "worstWait": { + "mean": 632.8383333333334, + "min": 574.22, + "max": 702.875 + }, + "makespan": { + "mean": 1240.3333333333333, + "min": 1159, + "max": 1311 + }, + "contentionWorst": { + "mean": 1, + "min": 1, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 200, + "weight": 1, + "share": 1, + "contentionShareOverWeight": 0, + "meanWait": 702.875, + "waitP50": 723, + "waitP99": 1309, + "waitMax": 1311 + } + ] + }, + { + "treatment": "drr", + "lightWait": { + "mean": 632.8383333333334, + "min": 574.22, + "max": 702.875 + }, + "worstWait": { + "mean": 632.8383333333334, + "min": 574.22, + "max": 702.875 + }, + "makespan": { + "mean": 1240.3333333333333, + "min": 1159, + "max": 1311 + }, + "contentionWorst": { + "mean": 1, + "min": 1, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 200, + "weight": 1, + "share": 1, + "contentionShareOverWeight": 0, + "meanWait": 702.875, + "waitP50": 723, + "waitP99": 1309, + "waitMax": 1311 + } + ] + }, + { + "treatment": "perKeyCap+sfq", + "lightWait": { + "mean": 1290.6516666666669, + "min": 1173.15, + "max": 1431.305 + }, + "worstWait": { + "mean": 1290.6516666666669, + "min": 1173.15, + "max": 1431.305 + }, + "makespan": { + "mean": 2507, + "min": 2344, + "max": 2638 + }, + "contentionWorst": { + "mean": 1, + "min": 1, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 200, + "weight": 1, + "share": 1, + "contentionShareOverWeight": 0, + "meanWait": 1431.305, + "waitP50": 1501, + "waitP99": 2623, + "waitMax": 2638 + } + ] + }, + { + "treatment": "total+perKey+sfq", + "lightWait": { + "mean": 1290.6516666666669, + "min": 1173.15, + "max": 1431.305 + }, + "worstWait": { + "mean": 1290.6516666666669, + "min": 1173.15, + "max": 1431.305 + }, + "makespan": { + "mean": 2507, + "min": 2344, + "max": 2638 + }, + "contentionWorst": { + "mean": 1, + "min": 1, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 200, + "weight": 1, + "share": 1, + "contentionShareOverWeight": 0, + "meanWait": 1431.305, + "waitP50": 1501, + "waitP99": 2623, + "waitMax": 2638 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckSkew.json b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckSkew.json new file mode 100644 index 0000000000..310e248fd5 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckSkew.json @@ -0,0 +1,661 @@ +{ + "scenario": "ckSkew", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "envLimit": 4, + "perKeyCap": 2, + "totalCap": 2, + "lightKey": "light-1", + "weights": { + "heavy": 1, + "light-1": 1, + "light-2": 1, + "light-3": 1, + "light-4": 1 + }, + "perTreatment": [ + { + "treatment": "baseline", + "lightWait": { + "mean": 1098.2666666666667, + "min": 857.4, + "max": 1321 + }, + "worstWait": { + "mean": 1261.3777777777777, + "min": 1175.1333333333334, + "max": 1321 + }, + "makespan": { + "mean": 2082.6666666666665, + "min": 1765, + "max": 2285 + }, + "contentionWorst": { + "mean": 0.1866549331190391, + "min": 0.1858108108108108, + "max": 0.1877133105802048 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 4.067796610169491, + "meanWait": 871.9875, + "waitP50": 875, + "waitP99": 1644, + "waitMax": 1646 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.2542372881355932, + "meanWait": 1321, + "waitP50": 1239, + "waitP99": 1653, + "waitMax": 1653 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.23728813559322035, + "meanWait": 981.6666666666666, + "waitP50": 1043, + "waitP99": 1659, + "waitMax": 1659 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.1864406779661017, + "meanWait": 679.1333333333333, + "waitP50": 753, + "waitP99": 1661, + "waitMax": 1661 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.2542372881355932, + "meanWait": 915, + "waitP50": 839, + "waitP99": 1677, + "waitMax": 1677 + } + ] + }, + { + "treatment": "perKeyCap", + "lightWait": { + "mean": 19.622222222222224, + "min": 8.266666666666667, + "max": 38.13333333333333 + }, + "worstWait": { + "mean": 1555.2430555555557, + "min": 1426.3708333333334, + "max": 1770.1916666666666 + }, + "makespan": { + "mean": 3038.3333333333335, + "min": 2863, + "max": 3309 + }, + "contentionWorst": { + "mean": 0.8140259655565085, + "min": 0.7009345794392522, + "max": 0.9259259259259258 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.296296296296296, + "meanWait": 1770.1916666666666, + "waitP50": 1765, + "waitP99": 3293, + "waitMax": 3309 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.9259259259259258, + "meanWait": 8.266666666666667, + "waitP50": 0, + "waitP99": 42, + "waitMax": 42 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.9259259259259258, + "meanWait": 3.933333333333333, + "waitP50": 0, + "waitP99": 55, + "waitMax": 55 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.9259259259259258, + "meanWait": 2.466666666666667, + "waitP50": 0, + "waitP99": 24, + "waitMax": 24 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.9259259259259258, + "meanWait": 8.333333333333334, + "waitP50": 0, + "waitP99": 37, + "waitMax": 37 + } + ] + }, + { + "treatment": "totalCap", + "lightWait": { + "mean": 2840.2000000000003, + "min": 2670.3333333333335, + "max": 3138.4666666666667 + }, + "worstWait": { + "mean": 2973.6222222222223, + "min": 2766.9333333333334, + "max": 3138.4666666666667 + }, + "makespan": { + "mean": 3947.3333333333335, + "min": 3532, + "max": 4290 + }, + "contentionWorst": { + "mean": 0.21268177618484008, + "min": 0.1858108108108108, + "max": 0.23411371237458192 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 4.013377926421405, + "meanWait": 1770.1916666666666, + "waitP50": 1765, + "waitP99": 3293, + "waitMax": 3309 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.2508361204013378, + "meanWait": 3138.4666666666667, + "waitP50": 3102, + "waitP99": 3367, + "waitMax": 3367 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.23411371237458192, + "meanWait": 2885.3333333333335, + "waitP50": 2913, + "waitP99": 3380, + "waitMax": 3380 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.2508361204013378, + "meanWait": 2674, + "waitP50": 2769, + "waitP99": 3374, + "waitMax": 3374 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.2508361204013378, + "meanWait": 2852.0666666666666, + "waitP50": 2841, + "waitP99": 3400, + "waitMax": 3400 + } + ] + }, + { + "treatment": "total+perKey", + "lightWait": { + "mean": 2840.2000000000003, + "min": 2670.3333333333335, + "max": 3138.4666666666667 + }, + "worstWait": { + "mean": 2973.6222222222223, + "min": 2766.9333333333334, + "max": 3138.4666666666667 + }, + "makespan": { + "mean": 3947.3333333333335, + "min": 3532, + "max": 4290 + }, + "contentionWorst": { + "mean": 0.21268177618484008, + "min": 0.1858108108108108, + "max": 0.23411371237458192 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 4.013377926421405, + "meanWait": 1770.1916666666666, + "waitP50": 1765, + "waitP99": 3293, + "waitMax": 3309 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.2508361204013378, + "meanWait": 3138.4666666666667, + "waitP50": 3102, + "waitP99": 3367, + "waitMax": 3367 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.23411371237458192, + "meanWait": 2885.3333333333335, + "waitP50": 2913, + "waitP99": 3380, + "waitMax": 3380 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.2508361204013378, + "meanWait": 2674, + "waitP50": 2769, + "waitP99": 3374, + "waitMax": 3374 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.2508361204013378, + "meanWait": 2852.0666666666666, + "waitP50": 2841, + "waitP99": 3400, + "waitMax": 3400 + } + ] + }, + { + "treatment": "sfq", + "lightWait": { + "mean": 14.311111111111112, + "min": 11.333333333333334, + "max": 16.2 + }, + "worstWait": { + "mean": 1068.6944444444443, + "min": 937.3291666666667, + "max": 1149.6333333333334 + }, + "makespan": { + "mean": 2082.6666666666665, + "min": 1765, + "max": 2285 + }, + "contentionWorst": { + "mean": 0.7228911750188346, + "min": 0.6547619047619048, + "max": 0.7692307692307693 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.8617021276595742, + "meanWait": 1149.6333333333334, + "waitP50": 1193, + "waitP99": 2099, + "waitMax": 2110 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 16.2, + "waitP50": 16, + "waitP99": 40, + "waitMax": 40 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7446808510638298, + "meanWait": 16.066666666666666, + "waitP50": 14, + "waitP99": 48, + "waitMax": 48 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 13, + "waitP50": 10, + "waitP99": 41, + "waitMax": 41 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 9.533333333333333, + "waitP50": 9, + "waitP99": 24, + "waitMax": 24 + } + ] + }, + { + "treatment": "drr", + "lightWait": { + "mean": 16.8, + "min": 14.6, + "max": 20.866666666666667 + }, + "worstWait": { + "mean": 1066.875, + "min": 935.9833333333333, + "max": 1149.1125 + }, + "makespan": { + "mean": 2082.6666666666665, + "min": 1765, + "max": 2285 + }, + "contentionWorst": { + "mean": 0.6077242434174587, + "min": 0.5555555555555555, + "max": 0.648148148148148 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 2.3893805309734515, + "meanWait": 1149.1125, + "waitP50": 1190, + "waitP99": 2099, + "waitMax": 2110 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.6637168141592921, + "meanWait": 20.866666666666667, + "waitP50": 20, + "waitP99": 45, + "waitMax": 45 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.6194690265486725, + "meanWait": 16.866666666666667, + "waitP50": 13, + "waitP99": 35, + "waitMax": 35 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.6637168141592921, + "meanWait": 12.266666666666667, + "waitP50": 11, + "waitP99": 42, + "waitMax": 42 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.6637168141592921, + "meanWait": 16, + "waitP50": 12, + "waitP99": 57, + "waitMax": 57 + } + ] + }, + { + "treatment": "perKeyCap+sfq", + "lightWait": { + "mean": 6.844444444444444, + "min": 6, + "max": 8.133333333333333 + }, + "worstWait": { + "mean": 1627.55, + "min": 1473.6708333333333, + "max": 1797.925 + }, + "makespan": { + "mean": 3114, + "min": 2915, + "max": 3338 + }, + "contentionWorst": { + "mean": 0.8002825914644652, + "min": 0.6521739130434782, + "max": 0.974025974025974 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 0.6521739130434782, + "meanWait": 1797.925, + "waitP50": 1794, + "waitP99": 3322, + "waitMax": 3338 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 1.0869565217391304, + "meanWait": 6, + "waitP50": 0, + "waitP99": 31, + "waitMax": 31 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 1.0869565217391304, + "meanWait": 2.1333333333333333, + "waitP50": 0, + "waitP99": 22, + "waitMax": 22 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 1.0869565217391304, + "meanWait": 0.6, + "waitP50": 0, + "waitP99": 9, + "waitMax": 9 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 1.0869565217391304, + "meanWait": 7.533333333333333, + "waitP50": 0, + "waitP99": 44, + "waitMax": 44 + } + ] + }, + { + "treatment": "total+perKey+sfq", + "lightWait": { + "mean": 51.888888888888886, + "min": 45.06666666666667, + "max": 55.4 + }, + "worstWait": { + "mean": 2363.016666666667, + "min": 2045.6083333333333, + "max": 2573.3708333333334 + }, + "makespan": { + "mean": 3939.3333333333335, + "min": 3564, + "max": 4238 + }, + "contentionWorst": { + "mean": 0.8028035775713794, + "min": 0.588235294117647, + "max": 0.9868421052631579 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 0.588235294117647, + "meanWait": 2573.3708333333334, + "waitP50": 2694, + "waitP99": 4222, + "waitMax": 4238 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 1.102941176470588, + "meanWait": 55.2, + "waitP50": 32, + "waitP99": 149, + "waitMax": 149 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 1.102941176470588, + "meanWait": 23.333333333333332, + "waitP50": 18, + "waitP99": 55, + "waitMax": 55 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 1.102941176470588, + "meanWait": 14.4, + "waitP50": 11, + "waitP99": 44, + "waitMax": 44 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 1.102941176470588, + "meanWait": 21.6, + "waitP50": 17, + "waitP99": 65, + "waitMax": 65 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckSybil.json b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckSybil.json new file mode 100644 index 0000000000..2fb2f04448 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckSybil.json @@ -0,0 +1,2085 @@ +{ + "scenario": "ckSybil", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "envLimit": 4, + "perKeyCap": 2, + "totalCap": 2, + "lightKey": "light", + "weights": { + "heavy-0": 1, + "heavy-1": 1, + "heavy-2": 1, + "heavy-3": 1, + "heavy-4": 1, + "heavy-5": 1, + "heavy-6": 1, + "heavy-7": 1, + "heavy-8": 1, + "heavy-9": 1, + "heavy-10": 1, + "heavy-11": 1, + "heavy-12": 1, + "heavy-13": 1, + "heavy-14": 1, + "heavy-15": 1, + "heavy-16": 1, + "heavy-17": 1, + "heavy-18": 1, + "heavy-19": 1, + "light": 1 + }, + "perTreatment": [ + { + "treatment": "baseline", + "lightWait": { + "mean": 1764.7666666666664, + "min": 1679.1, + "max": 1925.8 + }, + "worstWait": { + "mean": 1876.1333333333332, + "min": 1778.4, + "max": 2046.3333333333333 + }, + "makespan": { + "mean": 2070, + "min": 1893, + "max": 2302 + }, + "contentionWorst": { + "mean": 0, + "min": 0, + "max": 0 + }, + "detailSeed0": [ + { + "groupId": "heavy-0", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 53.93333333333333, + "waitP50": 44, + "waitP99": 128, + "waitMax": 128 + }, + { + "groupId": "heavy-1", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 167.93333333333334, + "waitP50": 163, + "waitP99": 205, + "waitMax": 205 + }, + { + "groupId": "heavy-2", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1408.7333333333333, + "waitP50": 1401, + "waitP99": 1446, + "waitMax": 1446 + }, + { + "groupId": "heavy-3", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1506.2, + "waitP50": 1509, + "waitP99": 1541, + "waitMax": 1541 + }, + { + "groupId": "heavy-4", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1582.4, + "waitP50": 1582, + "waitP99": 1626, + "waitMax": 1626 + }, + { + "groupId": "heavy-5", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1668.6666666666667, + "waitP50": 1668, + "waitP99": 1708, + "waitMax": 1708 + }, + { + "groupId": "heavy-6", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1749.9333333333334, + "waitP50": 1755, + "waitP99": 1786, + "waitMax": 1786 + }, + { + "groupId": "heavy-7", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1847.4666666666667, + "waitP50": 1852, + "waitP99": 1899, + "waitMax": 1899 + }, + { + "groupId": "heavy-8", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1953.3333333333333, + "waitP50": 1947, + "waitP99": 1991, + "waitMax": 1991 + }, + { + "groupId": "heavy-9", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 2046.3333333333333, + "waitP50": 2039, + "waitP99": 2104, + "waitMax": 2104 + }, + { + "groupId": "heavy-10", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 286.4, + "waitP50": 270, + "waitP99": 353, + "waitMax": 353 + }, + { + "groupId": "heavy-11", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 394.53333333333336, + "waitP50": 398, + "waitP99": 437, + "waitMax": 437 + }, + { + "groupId": "heavy-12", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 475.8666666666667, + "waitP50": 478, + "waitP99": 519, + "waitMax": 519 + }, + { + "groupId": "heavy-13", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 597.8666666666667, + "waitP50": 599, + "waitP99": 653, + "waitMax": 653 + }, + { + "groupId": "heavy-14", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 697.4666666666667, + "waitP50": 697, + "waitP99": 749, + "waitMax": 749 + }, + { + "groupId": "heavy-15", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 818.3333333333334, + "waitP50": 799, + "waitP99": 915, + "waitMax": 915 + }, + { + "groupId": "heavy-16", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 952.8666666666667, + "waitP50": 950, + "waitP99": 989, + "waitMax": 989 + }, + { + "groupId": "heavy-17", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1054.4, + "waitP50": 1058, + "waitP99": 1084, + "waitMax": 1084 + }, + { + "groupId": "heavy-18", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1123.2, + "waitP50": 1111, + "waitP99": 1180, + "waitMax": 1180 + }, + { + "groupId": "heavy-19", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1274.3333333333333, + "waitP50": 1248, + "waitP99": 1379, + "waitMax": 1379 + }, + { + "groupId": "light", + "dequeued": 20, + "weight": 1, + "share": 0.0625, + "contentionShareOverWeight": 0, + "meanWait": 1925.8, + "waitP50": 1869, + "waitP99": 2133, + "waitMax": 2133 + } + ] + }, + { + "treatment": "perKeyCap", + "lightWait": { + "mean": 1776.05, + "min": 1691.5, + "max": 1891.35 + }, + "worstWait": { + "mean": 1869.3555555555556, + "min": 1753.2, + "max": 2095.5333333333333 + }, + "makespan": { + "mean": 2128.3333333333335, + "min": 1931, + "max": 2330 + }, + "contentionWorst": { + "mean": 0.40264026402640263, + "min": 0, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy-0", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 135.8, + "waitP50": 124, + "waitP99": 287, + "waitMax": 287 + }, + { + "groupId": "heavy-1", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 70.6, + "waitP50": 57, + "waitP99": 209, + "waitMax": 209 + }, + { + "groupId": "heavy-2", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1381.5333333333333, + "waitP50": 1371, + "waitP99": 1470, + "waitMax": 1470 + }, + { + "groupId": "heavy-3", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1533.1333333333334, + "waitP50": 1529, + "waitP99": 1590, + "waitMax": 1590 + }, + { + "groupId": "heavy-4", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1560.8, + "waitP50": 1551, + "waitP99": 1643, + "waitMax": 1643 + }, + { + "groupId": "heavy-5", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1680.6666666666667, + "waitP50": 1675, + "waitP99": 1752, + "waitMax": 1752 + }, + { + "groupId": "heavy-6", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1745.1333333333334, + "waitP50": 1767, + "waitP99": 1813, + "waitMax": 1813 + }, + { + "groupId": "heavy-7", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1858.8, + "waitP50": 1863, + "waitP99": 1942, + "waitMax": 1942 + }, + { + "groupId": "heavy-8", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1936.7333333333333, + "waitP50": 1929, + "waitP99": 2018, + "waitMax": 2018 + }, + { + "groupId": "heavy-9", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2095.5333333333333, + "waitP50": 2085, + "waitP99": 2257, + "waitMax": 2257 + }, + { + "groupId": "heavy-10", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 328.4, + "waitP50": 316, + "waitP99": 427, + "waitMax": 427 + }, + { + "groupId": "heavy-11", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 371.4, + "waitP50": 371, + "waitP99": 441, + "waitMax": 441 + }, + { + "groupId": "heavy-12", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 530, + "waitP50": 506, + "waitP99": 633, + "waitMax": 633 + }, + { + "groupId": "heavy-13", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 570.8, + "waitP50": 564, + "waitP99": 678, + "waitMax": 678 + }, + { + "groupId": "heavy-14", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 725.6, + "waitP50": 711, + "waitP99": 837, + "waitMax": 837 + }, + { + "groupId": "heavy-15", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 811.6, + "waitP50": 768, + "waitP99": 987, + "waitMax": 987 + }, + { + "groupId": "heavy-16", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 937.2666666666667, + "waitP50": 932, + "waitP99": 1063, + "waitMax": 1063 + }, + { + "groupId": "heavy-17", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1052.8666666666666, + "waitP50": 1065, + "waitP99": 1098, + "waitMax": 1098 + }, + { + "groupId": "heavy-18", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1176.5333333333333, + "waitP50": 1172, + "waitP99": 1315, + "waitMax": 1315 + }, + { + "groupId": "heavy-19", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1239.5333333333333, + "waitP50": 1234, + "waitP99": 1423, + "waitMax": 1423 + }, + { + "groupId": "light", + "dequeued": 20, + "weight": 1, + "share": 0.0625, + "contentionShareOverWeight": 1, + "meanWait": 1891.35, + "waitP50": 1850, + "waitP99": 2031, + "waitMax": 2031 + } + ] + }, + { + "treatment": "totalCap", + "lightWait": { + "mean": 3793.066666666667, + "min": 3551.4, + "max": 4164.25 + }, + "worstWait": { + "mean": 3800.5333333333333, + "min": 3573.8, + "max": 4164.25 + }, + "makespan": { + "mean": 4146.666666666667, + "min": 3790, + "max": 4604 + }, + "contentionWorst": { + "mean": 0, + "min": 0, + "max": 0 + }, + "detailSeed0": [ + { + "groupId": "heavy-0", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 135.8, + "waitP50": 124, + "waitP99": 287, + "waitMax": 287 + }, + { + "groupId": "heavy-1", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 366.6, + "waitP50": 350, + "waitP99": 507, + "waitMax": 507 + }, + { + "groupId": "heavy-2", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 2846.6, + "waitP50": 2832, + "waitP99": 2931, + "waitMax": 2931 + }, + { + "groupId": "heavy-3", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3029, + "waitP50": 3026, + "waitP99": 3087, + "waitMax": 3087 + }, + { + "groupId": "heavy-4", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3188.133333333333, + "waitP50": 3177, + "waitP99": 3270, + "waitMax": 3270 + }, + { + "groupId": "heavy-5", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3355.866666666667, + "waitP50": 3354, + "waitP99": 3427, + "waitMax": 3427 + }, + { + "groupId": "heavy-6", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3520.0666666666666, + "waitP50": 3531, + "waitP99": 3587, + "waitMax": 3587 + }, + { + "groupId": "heavy-7", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3725.133333333333, + "waitP50": 3734, + "waitP99": 3813, + "waitMax": 3813 + }, + { + "groupId": "heavy-8", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3925.6, + "waitP50": 3912, + "waitP99": 4000, + "waitMax": 4000 + }, + { + "groupId": "heavy-9", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 4134.2, + "waitP50": 4129, + "waitP99": 4297, + "waitMax": 4297 + }, + { + "groupId": "heavy-10", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 624.0666666666667, + "waitP50": 614, + "waitP99": 721, + "waitMax": 721 + }, + { + "groupId": "heavy-11", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 803.6, + "waitP50": 803, + "waitP99": 873, + "waitMax": 873 + }, + { + "groupId": "heavy-12", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 983.0666666666667, + "waitP50": 956, + "waitP99": 1085, + "waitMax": 1085 + }, + { + "groupId": "heavy-13", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1222.3333333333333, + "waitP50": 1215, + "waitP99": 1329, + "waitMax": 1329 + }, + { + "groupId": "heavy-14", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1413.8666666666666, + "waitP50": 1394, + "waitP99": 1528, + "waitMax": 1528 + }, + { + "groupId": "heavy-15", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1673, + "waitP50": 1632, + "waitP99": 1848, + "waitMax": 1848 + }, + { + "groupId": "heavy-16", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1935.8, + "waitP50": 1932, + "waitP99": 2060, + "waitMax": 2060 + }, + { + "groupId": "heavy-17", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 2133.266666666667, + "waitP50": 2143, + "waitP99": 2180, + "waitMax": 2180 + }, + { + "groupId": "heavy-18", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 2276.6, + "waitP50": 2273, + "waitP99": 2414, + "waitMax": 2414 + }, + { + "groupId": "heavy-19", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 2580.866666666667, + "waitP50": 2568, + "waitP99": 2759, + "waitMax": 2759 + }, + { + "groupId": "light", + "dequeued": 20, + "weight": 1, + "share": 0.0625, + "contentionShareOverWeight": 0, + "meanWait": 4164.25, + "waitP50": 4123, + "waitP99": 4297, + "waitMax": 4297 + } + ] + }, + { + "treatment": "total+perKey", + "lightWait": { + "mean": 3793.066666666667, + "min": 3551.4, + "max": 4164.25 + }, + "worstWait": { + "mean": 3800.5333333333333, + "min": 3573.8, + "max": 4164.25 + }, + "makespan": { + "mean": 4146.666666666667, + "min": 3790, + "max": 4604 + }, + "contentionWorst": { + "mean": 0, + "min": 0, + "max": 0 + }, + "detailSeed0": [ + { + "groupId": "heavy-0", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 135.8, + "waitP50": 124, + "waitP99": 287, + "waitMax": 287 + }, + { + "groupId": "heavy-1", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 366.6, + "waitP50": 350, + "waitP99": 507, + "waitMax": 507 + }, + { + "groupId": "heavy-2", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 2846.6, + "waitP50": 2832, + "waitP99": 2931, + "waitMax": 2931 + }, + { + "groupId": "heavy-3", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3029, + "waitP50": 3026, + "waitP99": 3087, + "waitMax": 3087 + }, + { + "groupId": "heavy-4", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3188.133333333333, + "waitP50": 3177, + "waitP99": 3270, + "waitMax": 3270 + }, + { + "groupId": "heavy-5", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3355.866666666667, + "waitP50": 3354, + "waitP99": 3427, + "waitMax": 3427 + }, + { + "groupId": "heavy-6", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3520.0666666666666, + "waitP50": 3531, + "waitP99": 3587, + "waitMax": 3587 + }, + { + "groupId": "heavy-7", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3725.133333333333, + "waitP50": 3734, + "waitP99": 3813, + "waitMax": 3813 + }, + { + "groupId": "heavy-8", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 3925.6, + "waitP50": 3912, + "waitP99": 4000, + "waitMax": 4000 + }, + { + "groupId": "heavy-9", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 4134.2, + "waitP50": 4129, + "waitP99": 4297, + "waitMax": 4297 + }, + { + "groupId": "heavy-10", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 624.0666666666667, + "waitP50": 614, + "waitP99": 721, + "waitMax": 721 + }, + { + "groupId": "heavy-11", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 803.6, + "waitP50": 803, + "waitP99": 873, + "waitMax": 873 + }, + { + "groupId": "heavy-12", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 983.0666666666667, + "waitP50": 956, + "waitP99": 1085, + "waitMax": 1085 + }, + { + "groupId": "heavy-13", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1222.3333333333333, + "waitP50": 1215, + "waitP99": 1329, + "waitMax": 1329 + }, + { + "groupId": "heavy-14", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1413.8666666666666, + "waitP50": 1394, + "waitP99": 1528, + "waitMax": 1528 + }, + { + "groupId": "heavy-15", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1673, + "waitP50": 1632, + "waitP99": 1848, + "waitMax": 1848 + }, + { + "groupId": "heavy-16", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 1935.8, + "waitP50": 1932, + "waitP99": 2060, + "waitMax": 2060 + }, + { + "groupId": "heavy-17", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 2133.266666666667, + "waitP50": 2143, + "waitP99": 2180, + "waitMax": 2180 + }, + { + "groupId": "heavy-18", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 2276.6, + "waitP50": 2273, + "waitP99": 2414, + "waitMax": 2414 + }, + { + "groupId": "heavy-19", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1.05, + "meanWait": 2580.866666666667, + "waitP50": 2568, + "waitP99": 2759, + "waitMax": 2759 + }, + { + "groupId": "light", + "dequeued": 20, + "weight": 1, + "share": 0.0625, + "contentionShareOverWeight": 0, + "meanWait": 4164.25, + "waitP50": 4123, + "waitP99": 4297, + "waitMax": 4297 + } + ] + }, + { + "treatment": "sfq", + "lightWait": { + "mean": 1009.25, + "min": 957.95, + "max": 1103.7 + }, + "worstWait": { + "mean": 1018.9888888888889, + "min": 966.1, + "max": 1117.4 + }, + "makespan": { + "mean": 2064.6666666666665, + "min": 1889, + "max": 2297 + }, + "contentionWorst": { + "mean": 1, + "min": 1, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy-0", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1095.7333333333333, + "waitP50": 1009, + "waitP99": 2202, + "waitMax": 2202 + }, + { + "groupId": "heavy-1", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1105.4, + "waitP50": 1110, + "waitP99": 2247, + "waitMax": 2247 + }, + { + "groupId": "heavy-2", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1113.1333333333334, + "waitP50": 1007, + "waitP99": 2145, + "waitMax": 2145 + }, + { + "groupId": "heavy-3", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1114.3333333333333, + "waitP50": 1065, + "waitP99": 2163, + "waitMax": 2163 + }, + { + "groupId": "heavy-4", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1108.4, + "waitP50": 1033, + "waitP99": 2133, + "waitMax": 2133 + }, + { + "groupId": "heavy-5", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1112.6666666666667, + "waitP50": 1106, + "waitP99": 2153, + "waitMax": 2153 + }, + { + "groupId": "heavy-6", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1117.4, + "waitP50": 1030, + "waitP99": 2186, + "waitMax": 2186 + }, + { + "groupId": "heavy-7", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1114.2, + "waitP50": 1009, + "waitP99": 2257, + "waitMax": 2257 + }, + { + "groupId": "heavy-8", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1114.3333333333333, + "waitP50": 1032, + "waitP99": 2220, + "waitMax": 2220 + }, + { + "groupId": "heavy-9", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1112.6666666666667, + "waitP50": 1046, + "waitP99": 2239, + "waitMax": 2239 + }, + { + "groupId": "heavy-10", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1101.4, + "waitP50": 1097, + "waitP99": 2142, + "waitMax": 2142 + }, + { + "groupId": "heavy-11", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1101.4, + "waitP50": 1126, + "waitP99": 2131, + "waitMax": 2131 + }, + { + "groupId": "heavy-12", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1110.4, + "waitP50": 1140, + "waitP99": 2203, + "waitMax": 2203 + }, + { + "groupId": "heavy-13", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1107.4, + "waitP50": 1004, + "waitP99": 2239, + "waitMax": 2239 + }, + { + "groupId": "heavy-14", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1106.6, + "waitP50": 1124, + "waitP99": 2153, + "waitMax": 2153 + }, + { + "groupId": "heavy-15", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1105.6666666666667, + "waitP50": 1096, + "waitP99": 2203, + "waitMax": 2203 + }, + { + "groupId": "heavy-16", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1109.8666666666666, + "waitP50": 1109, + "waitP99": 2144, + "waitMax": 2144 + }, + { + "groupId": "heavy-17", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1116.3333333333333, + "waitP50": 1084, + "waitP99": 2201, + "waitMax": 2201 + }, + { + "groupId": "heavy-18", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1105.4, + "waitP50": 1112, + "waitP99": 2220, + "waitMax": 2220 + }, + { + "groupId": "heavy-19", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1101.5333333333333, + "waitP50": 1035, + "waitP99": 2219, + "waitMax": 2219 + }, + { + "groupId": "light", + "dequeued": 20, + "weight": 1, + "share": 0.0625, + "contentionShareOverWeight": 1, + "meanWait": 1103.7, + "waitP50": 1204, + "waitP99": 1832, + "waitMax": 1832 + } + ] + }, + { + "treatment": "drr", + "lightWait": { + "mean": 1061.3999999999999, + "min": 1008.8, + "max": 1163.15 + }, + "worstWait": { + "mean": 1068.1499999999999, + "min": 1012.25, + "max": 1173.4666666666667 + }, + "makespan": { + "mean": 2055, + "min": 1864, + "max": 2300 + }, + "contentionWorst": { + "mean": 0.9936908517350158, + "min": 0.9936908517350158, + "max": 0.9936908517350158 + }, + "detailSeed0": [ + { + "groupId": "heavy-0", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1037.4, + "waitP50": 990, + "waitP99": 2147, + "waitMax": 2147 + }, + { + "groupId": "heavy-1", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1046.0666666666666, + "waitP50": 991, + "waitP99": 2189, + "waitMax": 2189 + }, + { + "groupId": "heavy-2", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1123, + "waitP50": 1095, + "waitP99": 2162, + "waitMax": 2162 + }, + { + "groupId": "heavy-3", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1133.2666666666667, + "waitP50": 1101, + "waitP99": 2211, + "waitMax": 2211 + }, + { + "groupId": "heavy-4", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1138.1333333333334, + "waitP50": 1106, + "waitP99": 2180, + "waitMax": 2180 + }, + { + "groupId": "heavy-5", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1151.8, + "waitP50": 1121, + "waitP99": 2270, + "waitMax": 2270 + }, + { + "groupId": "heavy-6", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1149.8666666666666, + "waitP50": 1124, + "waitP99": 2184, + "waitMax": 2184 + }, + { + "groupId": "heavy-7", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1158.3333333333333, + "waitP50": 1129, + "waitP99": 2213, + "waitMax": 2213 + }, + { + "groupId": "heavy-8", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1163.2666666666667, + "waitP50": 1133, + "waitP99": 2186, + "waitMax": 2186 + }, + { + "groupId": "heavy-9", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1173.4666666666667, + "waitP50": 1141, + "waitP99": 2242, + "waitMax": 2242 + }, + { + "groupId": "heavy-10", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1048, + "waitP50": 993, + "waitP99": 2148, + "waitMax": 2148 + }, + { + "groupId": "heavy-11", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1063.4666666666667, + "waitP50": 1012, + "waitP99": 2220, + "waitMax": 2220 + }, + { + "groupId": "heavy-12", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1061.8, + "waitP50": 1027, + "waitP99": 2150, + "waitMax": 2150 + }, + { + "groupId": "heavy-13", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1073.7333333333333, + "waitP50": 1042, + "waitP99": 2199, + "waitMax": 2199 + }, + { + "groupId": "heavy-14", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1079.1333333333334, + "waitP50": 1047, + "waitP99": 2151, + "waitMax": 2151 + }, + { + "groupId": "heavy-15", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1092.6, + "waitP50": 1059, + "waitP99": 2253, + "waitMax": 2253 + }, + { + "groupId": "heavy-16", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1091, + "waitP50": 1063, + "waitP99": 2153, + "waitMax": 2153 + }, + { + "groupId": "heavy-17", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1105.3333333333333, + "waitP50": 1069, + "waitP99": 2210, + "waitMax": 2210 + }, + { + "groupId": "heavy-18", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1107.6, + "waitP50": 1087, + "waitP99": 2161, + "waitMax": 2161 + }, + { + "groupId": "heavy-19", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 0.9936908517350158, + "meanWait": 1119.1333333333334, + "waitP50": 1092, + "waitP99": 2237, + "waitMax": 2237 + }, + { + "groupId": "light", + "dequeued": 20, + "weight": 1, + "share": 0.0625, + "contentionShareOverWeight": 1.1261829652996846, + "meanWait": 1163.15, + "waitP50": 1280, + "waitP99": 1826, + "waitMax": 1826 + } + ] + }, + { + "treatment": "perKeyCap+sfq", + "lightWait": { + "mean": 1010.0166666666665, + "min": 957.95, + "max": 1106 + }, + "worstWait": { + "mean": 1018.9888888888889, + "min": 966.1, + "max": 1117.4 + }, + "makespan": { + "mean": 2072, + "min": 1889, + "max": 2319 + }, + "contentionWorst": { + "mean": 1, + "min": 1, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy-0", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1095.7333333333333, + "waitP50": 1009, + "waitP99": 2202, + "waitMax": 2202 + }, + { + "groupId": "heavy-1", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1105.4, + "waitP50": 1110, + "waitP99": 2247, + "waitMax": 2247 + }, + { + "groupId": "heavy-2", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1113.1333333333334, + "waitP50": 1007, + "waitP99": 2145, + "waitMax": 2145 + }, + { + "groupId": "heavy-3", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1114.3333333333333, + "waitP50": 1065, + "waitP99": 2163, + "waitMax": 2163 + }, + { + "groupId": "heavy-4", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1108.4, + "waitP50": 1033, + "waitP99": 2133, + "waitMax": 2133 + }, + { + "groupId": "heavy-5", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1112.6666666666667, + "waitP50": 1106, + "waitP99": 2153, + "waitMax": 2153 + }, + { + "groupId": "heavy-6", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1117.4, + "waitP50": 1030, + "waitP99": 2186, + "waitMax": 2186 + }, + { + "groupId": "heavy-7", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1114.2, + "waitP50": 1009, + "waitP99": 2257, + "waitMax": 2257 + }, + { + "groupId": "heavy-8", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1114.3333333333333, + "waitP50": 1032, + "waitP99": 2220, + "waitMax": 2220 + }, + { + "groupId": "heavy-9", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1112.6666666666667, + "waitP50": 1046, + "waitP99": 2239, + "waitMax": 2239 + }, + { + "groupId": "heavy-10", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1101.4, + "waitP50": 1097, + "waitP99": 2142, + "waitMax": 2142 + }, + { + "groupId": "heavy-11", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1101.4, + "waitP50": 1126, + "waitP99": 2131, + "waitMax": 2131 + }, + { + "groupId": "heavy-12", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1110.4, + "waitP50": 1140, + "waitP99": 2203, + "waitMax": 2203 + }, + { + "groupId": "heavy-13", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1107.4, + "waitP50": 1004, + "waitP99": 2239, + "waitMax": 2239 + }, + { + "groupId": "heavy-14", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1106.6, + "waitP50": 1124, + "waitP99": 2153, + "waitMax": 2153 + }, + { + "groupId": "heavy-15", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1105.6666666666667, + "waitP50": 1096, + "waitP99": 2203, + "waitMax": 2203 + }, + { + "groupId": "heavy-16", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1109.8666666666666, + "waitP50": 1109, + "waitP99": 2144, + "waitMax": 2144 + }, + { + "groupId": "heavy-17", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1116.3333333333333, + "waitP50": 1084, + "waitP99": 2201, + "waitMax": 2201 + }, + { + "groupId": "heavy-18", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1105.4, + "waitP50": 1112, + "waitP99": 2220, + "waitMax": 2220 + }, + { + "groupId": "heavy-19", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 1101.5333333333333, + "waitP50": 1035, + "waitP99": 2219, + "waitMax": 2219 + }, + { + "groupId": "light", + "dequeued": 20, + "weight": 1, + "share": 0.0625, + "contentionShareOverWeight": 1, + "meanWait": 1106, + "waitP50": 1204, + "waitP99": 1832, + "waitMax": 1832 + } + ] + }, + { + "treatment": "total+perKey+sfq", + "lightWait": { + "mean": 2291.7499999999995, + "min": 2143.7, + "max": 2524.85 + }, + "worstWait": { + "mean": 2291.7499999999995, + "min": 2143.7, + "max": 2524.85 + }, + "makespan": { + "mean": 4146.333333333333, + "min": 3792, + "max": 4602 + }, + "contentionWorst": { + "mean": 1, + "min": 1, + "max": 1 + }, + "detailSeed0": [ + { + "groupId": "heavy-0", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2234.6666666666665, + "waitP50": 2154, + "waitP99": 4379, + "waitMax": 4379 + }, + { + "groupId": "heavy-1", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2235.9333333333334, + "waitP50": 2313, + "waitP99": 4380, + "waitMax": 4380 + }, + { + "groupId": "heavy-2", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2247.866666666667, + "waitP50": 2083, + "waitP99": 4403, + "waitMax": 4403 + }, + { + "groupId": "heavy-3", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2254.0666666666666, + "waitP50": 2135, + "waitP99": 4427, + "waitMax": 4427 + }, + { + "groupId": "heavy-4", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2250.6666666666665, + "waitP50": 2081, + "waitP99": 4502, + "waitMax": 4502 + }, + { + "groupId": "heavy-5", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2253.0666666666666, + "waitP50": 2080, + "waitP99": 4511, + "waitMax": 4511 + }, + { + "groupId": "heavy-6", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2255.9333333333334, + "waitP50": 2076, + "waitP99": 4512, + "waitMax": 4512 + }, + { + "groupId": "heavy-7", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2248.266666666667, + "waitP50": 2059, + "waitP99": 4527, + "waitMax": 4527 + }, + { + "groupId": "heavy-8", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2251.266666666667, + "waitP50": 2012, + "waitP99": 4521, + "waitMax": 4521 + }, + { + "groupId": "heavy-9", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2248.6666666666665, + "waitP50": 2040, + "waitP99": 4483, + "waitMax": 4483 + }, + { + "groupId": "heavy-10", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2237.9333333333334, + "waitP50": 2237, + "waitP99": 4287, + "waitMax": 4287 + }, + { + "groupId": "heavy-11", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2236.4666666666667, + "waitP50": 2331, + "waitP99": 4289, + "waitMax": 4289 + }, + { + "groupId": "heavy-12", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2242.133333333333, + "waitP50": 2315, + "waitP99": 4367, + "waitMax": 4367 + }, + { + "groupId": "heavy-13", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2241.3333333333335, + "waitP50": 2167, + "waitP99": 4484, + "waitMax": 4484 + }, + { + "groupId": "heavy-14", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2234.133333333333, + "waitP50": 2221, + "waitP99": 4334, + "waitMax": 4334 + }, + { + "groupId": "heavy-15", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2233, + "waitP50": 2205, + "waitP99": 4362, + "waitMax": 4362 + }, + { + "groupId": "heavy-16", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2236.6, + "waitP50": 2233, + "waitP99": 4325, + "waitMax": 4325 + }, + { + "groupId": "heavy-17", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2248.8, + "waitP50": 2109, + "waitP99": 4483, + "waitMax": 4483 + }, + { + "groupId": "heavy-18", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2246.6666666666665, + "waitP50": 2191, + "waitP99": 4464, + "waitMax": 4464 + }, + { + "groupId": "heavy-19", + "dequeued": 15, + "weight": 1, + "share": 0.046875, + "contentionShareOverWeight": 1, + "meanWait": 2242.9333333333334, + "waitP50": 2172, + "waitP99": 4401, + "waitMax": 4401 + }, + { + "groupId": "light", + "dequeued": 20, + "weight": 1, + "share": 0.0625, + "contentionShareOverWeight": 1, + "meanWait": 2524.85, + "waitP50": 2807, + "waitP99": 4112, + "waitMax": 4112 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckTrickle.json b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckTrickle.json new file mode 100644 index 0000000000..90b0efdbc4 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/caps-ckTrickle.json @@ -0,0 +1,483 @@ +{ + "scenario": "ckTrickle", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "envLimit": 4, + "perKeyCap": 2, + "totalCap": 2, + "lightKey": "trickle-1", + "weights": { + "bulk": 1, + "trickle-1": 1, + "trickle-2": 1 + }, + "perTreatment": [ + { + "treatment": "baseline", + "lightWait": { + "mean": 1107.3222222222223, + "min": 836.3333333333334, + "max": 1338.6666666666667 + }, + "worstWait": { + "mean": 1133.6222222222223, + "min": 872.8, + "max": 1338.6666666666667 + }, + "makespan": { + "mean": 1940.3333333333333, + "min": 1821, + "max": 2142 + }, + "contentionWorst": { + "mean": 0.27872569582223233, + "min": 0.2542372881355932, + "max": 0.29096989966555187 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 2.440677966101695, + "meanWait": 871.9875, + "waitP50": 875, + "waitP99": 1644, + "waitMax": 1646 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.3050847457627119, + "meanWait": 1338.6666666666667, + "waitP50": 1399, + "waitP99": 1705, + "waitMax": 1705 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.2542372881355932, + "meanWait": 1165.8666666666666, + "waitP50": 1162, + "waitP99": 1718, + "waitMax": 1718 + } + ] + }, + { + "treatment": "perKeyCap", + "lightWait": { + "mean": 18.933333333333334, + "min": 8.8, + "max": 26.066666666666666 + }, + "worstWait": { + "mean": 1555.2430555555557, + "min": 1426.3708333333334, + "max": 1770.1916666666666 + }, + "makespan": { + "mean": 3038.3333333333335, + "min": 2863, + "max": 3309 + }, + "contentionWorst": { + "mean": 0.921778350515464, + "min": 0.9, + "max": 0.9375 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.2000000000000002, + "meanWait": 1770.1916666666666, + "waitP50": 1765, + "waitP99": 3293, + "waitMax": 3309 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9, + "meanWait": 26.066666666666666, + "waitP50": 0, + "waitP99": 146, + "waitMax": 146 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9, + "meanWait": 15.766666666666667, + "waitP50": 0, + "waitP99": 159, + "waitMax": 159 + } + ] + }, + { + "treatment": "totalCap", + "lightWait": { + "mean": 2851.788888888889, + "min": 2528.4666666666667, + "max": 3236.766666666667 + }, + "worstWait": { + "mean": 2860.7444444444445, + "min": 2535.3, + "max": 3236.766666666667 + }, + "makespan": { + "mean": 3904.6666666666665, + "min": 3653, + "max": 4342 + }, + "contentionWorst": { + "mean": 0.27872569582223233, + "min": 0.2542372881355932, + "max": 0.29096989966555187 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 2.440677966101695, + "meanWait": 1770.1916666666666, + "waitP50": 1765, + "waitP99": 3293, + "waitMax": 3309 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.3050847457627119, + "meanWait": 3236.766666666667, + "waitP50": 3266, + "waitP99": 3523, + "waitMax": 3523 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.2542372881355932, + "meanWait": 3129.5, + "waitP50": 3132, + "waitP99": 3536, + "waitMax": 3536 + } + ] + }, + { + "treatment": "total+perKey", + "lightWait": { + "mean": 2851.788888888889, + "min": 2528.4666666666667, + "max": 3236.766666666667 + }, + "worstWait": { + "mean": 2860.7444444444445, + "min": 2535.3, + "max": 3236.766666666667 + }, + "makespan": { + "mean": 3904.6666666666665, + "min": 3653, + "max": 4342 + }, + "contentionWorst": { + "mean": 0.27872569582223233, + "min": 0.2542372881355932, + "max": 0.29096989966555187 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 2.440677966101695, + "meanWait": 1770.1916666666666, + "waitP50": 1765, + "waitP99": 3293, + "waitMax": 3309 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.3050847457627119, + "meanWait": 3236.766666666667, + "waitP50": 3266, + "waitP99": 3523, + "waitMax": 3523 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.2542372881355932, + "meanWait": 3129.5, + "waitP50": 3132, + "waitP99": 3536, + "waitMax": 3536 + } + ] + }, + { + "treatment": "sfq", + "lightWait": { + "mean": 17.444444444444443, + "min": 11.633333333333333, + "max": 23.933333333333334 + }, + "worstWait": { + "mean": 1070.3680555555557, + "min": 957.8, + "max": 1237.4333333333334 + }, + "makespan": { + "mean": 1942, + "min": 1818, + "max": 2140 + }, + "contentionWorst": { + "mean": 0.9092746009294808, + "min": 0.8910891089108911, + "max": 0.9183673469387755 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.163265306122449, + "meanWait": 1237.4333333333334, + "waitP50": 1317, + "waitP99": 2139, + "waitMax": 2140 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9183673469387755, + "meanWait": 23.933333333333334, + "waitP50": 13, + "waitP99": 96, + "waitMax": 96 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9183673469387755, + "meanWait": 20.3, + "waitP50": 14, + "waitP99": 62, + "waitMax": 62 + } + ] + }, + { + "treatment": "drr", + "lightWait": { + "mean": 22.755555555555556, + "min": 16.1, + "max": 30.433333333333334 + }, + "worstWait": { + "mean": 1068.9722222222222, + "min": 957.0166666666667, + "max": 1235.5583333333334 + }, + "makespan": { + "mean": 1941.3333333333333, + "min": 1818, + "max": 2140 + }, + "contentionWorst": { + "mean": 0.7900070943549204, + "min": 0.7692307692307692, + "max": 0.8181818181818181 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.4347826086956523, + "meanWait": 1235.5583333333334, + "waitP50": 1315, + "waitP99": 2139, + "waitMax": 2140 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.782608695652174, + "meanWait": 30.433333333333334, + "waitP50": 18, + "waitP99": 128, + "waitMax": 128 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.782608695652174, + "meanWait": 26.333333333333332, + "waitP50": 25, + "waitP99": 61, + "waitMax": 61 + } + ] + }, + { + "treatment": "perKeyCap+sfq", + "lightWait": { + "mean": 8.011111111111111, + "min": 3.6, + "max": 14.333333333333334 + }, + "worstWait": { + "mean": 1606.2347222222222, + "min": 1438.425, + "max": 1844.2416666666666 + }, + "makespan": { + "mean": 3097.3333333333335, + "min": 2876, + "max": 3387 + }, + "contentionWorst": { + "mean": 0.6577540106951871, + "min": 0.42857142857142855, + "max": 0.8823529411764707 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 0.6623376623376623, + "meanWait": 1844.2416666666666, + "waitP50": 1843, + "waitP99": 3371, + "waitMax": 3387 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 1.168831168831169, + "meanWait": 14.333333333333334, + "waitP50": 0, + "waitP99": 96, + "waitMax": 96 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 1.168831168831169, + "meanWait": 5.1, + "waitP50": 0, + "waitP99": 31, + "waitMax": 31 + } + ] + }, + { + "treatment": "total+perKey+sfq", + "lightWait": { + "mean": 105.68888888888888, + "min": 77.96666666666667, + "max": 143.8 + }, + "worstWait": { + "mean": 2337.3555555555554, + "min": 2159.9333333333334, + "max": 2685.2916666666665 + }, + "makespan": { + "mean": 3911.3333333333335, + "min": 3662, + "max": 4298 + }, + "contentionWorst": { + "mean": 0.8826988839201834, + "min": 0.6923076923076924, + "max": 0.9782608695652174 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.0434782608695652, + "meanWait": 2685.2916666666665, + "waitP50": 2754, + "waitP99": 4282, + "waitMax": 4298 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9782608695652174, + "meanWait": 143.8, + "waitP50": 156, + "waitP99": 304, + "waitMax": 304 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9782608695652174, + "meanWait": 63.3, + "waitP50": 58, + "waitP99": 161, + "waitMax": 161 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/ckBalanced.json b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/ckBalanced.json new file mode 100644 index 0000000000..fff0db0f81 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/ckBalanced.json @@ -0,0 +1,370 @@ +{ + "scenario": "ckBalanced", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "weights": { + "k-a": 1, + "k-b": 1, + "k-c": 1, + "k-d": 1 + }, + "perDiscipline": [ + { + "selector": "baseline", + "contentionWorstShareOverWeight": { + "mean": 0.5148148148148147, + "min": 0.4444444444444444, + "max": 0.6 + }, + "contentionJain": { + "mean": 0.8491221317705939, + "min": 0.7422680412371133, + "max": 0.9433962264150942 + }, + "detailSeed0": [ + { + "groupId": "k-a", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 2, + "meanWait": 3.3333333333333335, + "waitP50": 0, + "waitP99": 31, + "waitMax": 31 + }, + { + "groupId": "k-b", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.5, + "meanWait": 0.38333333333333336, + "waitP50": 0, + "waitP99": 18, + "waitMax": 18 + }, + { + "groupId": "k-c", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.8333333333333334, + "meanWait": 3.4, + "waitP50": 0, + "waitP99": 31, + "waitMax": 31 + }, + { + "groupId": "k-d", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.6666666666666666, + "meanWait": 2.4, + "waitP50": 0, + "waitP99": 36, + "waitMax": 36 + } + ] + }, + { + "selector": "sfq", + "contentionWorstShareOverWeight": { + "mean": 0.6109890109890109, + "min": 0.46153846153846156, + "max": 0.8 + }, + "contentionJain": { + "mean": 0.9036239034827617, + "min": 0.8711340206185566, + "max": 0.9433962264150942 + }, + "detailSeed0": [ + { + "groupId": "k-a", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.5384615384615385, + "meanWait": 3.933333333333333, + "waitP50": 0, + "waitP99": 36, + "waitMax": 36 + }, + { + "groupId": "k-b", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.46153846153846156, + "meanWait": 0.38333333333333336, + "waitP50": 0, + "waitP99": 18, + "waitMax": 18 + }, + { + "groupId": "k-c", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0769230769230769, + "meanWait": 2.6166666666666667, + "waitP50": 0, + "waitP99": 28, + "waitMax": 28 + }, + { + "groupId": "k-d", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9230769230769231, + "meanWait": 2.5166666666666666, + "waitP50": 0, + "waitP99": 36, + "waitMax": 36 + } + ] + }, + { + "selector": "drr", + "contentionWorstShareOverWeight": { + "mean": 0.7303807303807304, + "min": 0.6153846153846154, + "max": 0.9090909090909091 + }, + "contentionJain": { + "mean": 0.9085911382775246, + "min": 0.8535353535353534, + "max": 0.9918032786885248 + }, + "detailSeed0": [ + { + "groupId": "k-a", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.6923076923076923, + "meanWait": 3.466666666666667, + "waitP50": 0, + "waitP99": 33, + "waitMax": 33 + }, + { + "groupId": "k-b", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.6153846153846154, + "meanWait": 0.38333333333333336, + "waitP50": 0, + "waitP99": 18, + "waitMax": 18 + }, + { + "groupId": "k-c", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9230769230769231, + "meanWait": 3.216666666666667, + "waitP50": 0, + "waitP99": 28, + "waitMax": 28 + }, + { + "groupId": "k-d", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.7692307692307693, + "meanWait": 2.2333333333333334, + "waitP50": 0, + "waitP99": 36, + "waitMax": 36 + } + ] + }, + { + "selector": "stride", + "contentionWorstShareOverWeight": { + "mean": 0.6109890109890109, + "min": 0.46153846153846156, + "max": 0.8 + }, + "contentionJain": { + "mean": 0.9036239034827617, + "min": 0.8711340206185566, + "max": 0.9433962264150942 + }, + "detailSeed0": [ + { + "groupId": "k-a", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.5384615384615385, + "meanWait": 3.933333333333333, + "waitP50": 0, + "waitP99": 36, + "waitMax": 36 + }, + { + "groupId": "k-b", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.46153846153846156, + "meanWait": 0.38333333333333336, + "waitP50": 0, + "waitP99": 18, + "waitMax": 18 + }, + { + "groupId": "k-c", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0769230769230769, + "meanWait": 2.6166666666666667, + "waitP50": 0, + "waitP99": 28, + "waitMax": 28 + }, + { + "groupId": "k-d", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9230769230769231, + "meanWait": 2.5166666666666666, + "waitP50": 0, + "waitP99": 36, + "waitMax": 36 + } + ] + }, + { + "selector": "codel(sfq)", + "contentionWorstShareOverWeight": { + "mean": 0.6109890109890109, + "min": 0.46153846153846156, + "max": 0.8 + }, + "contentionJain": { + "mean": 0.9036239034827617, + "min": 0.8711340206185566, + "max": 0.9433962264150942 + }, + "detailSeed0": [ + { + "groupId": "k-a", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.5384615384615385, + "meanWait": 3.933333333333333, + "waitP50": 0, + "waitP99": 36, + "waitMax": 36 + }, + { + "groupId": "k-b", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.46153846153846156, + "meanWait": 0.38333333333333336, + "waitP50": 0, + "waitP99": 18, + "waitMax": 18 + }, + { + "groupId": "k-c", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0769230769230769, + "meanWait": 2.6166666666666667, + "waitP50": 0, + "waitP99": 28, + "waitMax": 28 + }, + { + "groupId": "k-d", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9230769230769231, + "meanWait": 2.5166666666666666, + "waitP50": 0, + "waitP99": 36, + "waitMax": 36 + } + ] + }, + { + "selector": "codel(baseline)", + "contentionWorstShareOverWeight": { + "mean": 0.4641604010025063, + "min": 0.4, + "max": 0.5714285714285714 + }, + "contentionJain": { + "mean": 0.7955765336918118, + "min": 0.7112903225806452, + "max": 0.8474576271186439 + }, + "detailSeed0": [ + { + "groupId": "k-a", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 2.0952380952380953, + "meanWait": 3.3, + "waitP50": 0, + "waitP99": 30, + "waitMax": 30 + }, + { + "groupId": "k-b", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.5714285714285714, + "meanWait": 0.38333333333333336, + "waitP50": 0, + "waitP99": 18, + "waitMax": 18 + }, + { + "groupId": "k-c", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.5714285714285714, + "meanWait": 3.4833333333333334, + "waitP50": 0, + "waitP99": 53, + "waitMax": 53 + }, + { + "groupId": "k-d", + "dequeued": 60, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.7619047619047619, + "meanWait": 2.0833333333333335, + "waitP50": 0, + "waitP99": 23, + "waitMax": 23 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/ckSkew.json b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/ckSkew.json new file mode 100644 index 0000000000..255836ca51 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/ckSkew.json @@ -0,0 +1,437 @@ +{ + "scenario": "ckSkew", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "weights": { + "heavy": 1, + "light-1": 1, + "light-2": 1, + "light-3": 1, + "light-4": 1 + }, + "perDiscipline": [ + { + "selector": "baseline", + "contentionWorstShareOverWeight": { + "mean": 0.1866549331190391, + "min": 0.1858108108108108, + "max": 0.1877133105802048 + }, + "contentionJain": { + "mean": 0.2975688789730731, + "min": 0.29443196433164714, + "max": 0.3000753476265495 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 4.067796610169491, + "meanWait": 871.9875, + "waitP50": 875, + "waitP99": 1644, + "waitMax": 1646 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.2542372881355932, + "meanWait": 1321, + "waitP50": 1239, + "waitP99": 1653, + "waitMax": 1653 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.23728813559322035, + "meanWait": 981.6666666666666, + "waitP50": 1043, + "waitP99": 1659, + "waitMax": 1659 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.1864406779661017, + "meanWait": 679.1333333333333, + "waitP50": 753, + "waitP99": 1661, + "waitMax": 1661 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.2542372881355932, + "meanWait": 915, + "waitP50": 839, + "waitP99": 1677, + "waitMax": 1677 + } + ] + }, + { + "selector": "sfq", + "contentionWorstShareOverWeight": { + "mean": 0.7228911750188346, + "min": 0.6547619047619048, + "max": 0.7692307692307693 + }, + "contentionJain": { + "mean": 0.8592005815786168, + "min": 0.8431297709923665, + "max": 0.8739841688654353 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.8617021276595742, + "meanWait": 1149.6333333333334, + "waitP50": 1193, + "waitP99": 2099, + "waitMax": 2110 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 16.2, + "waitP50": 16, + "waitP99": 40, + "waitMax": 40 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7446808510638298, + "meanWait": 16.066666666666666, + "waitP50": 14, + "waitP99": 48, + "waitMax": 48 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 13, + "waitP50": 10, + "waitP99": 41, + "waitMax": 41 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 9.533333333333333, + "waitP50": 9, + "waitP99": 24, + "waitMax": 24 + } + ] + }, + { + "selector": "drr", + "contentionWorstShareOverWeight": { + "mean": 0.6077242434174587, + "min": 0.5555555555555555, + "max": 0.648148148148148 + }, + "contentionJain": { + "mean": 0.6987503929570632, + "min": 0.6743596514391338, + "max": 0.7129584352078239 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 2.3893805309734515, + "meanWait": 1149.1125, + "waitP50": 1190, + "waitP99": 2099, + "waitMax": 2110 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.6637168141592921, + "meanWait": 20.866666666666667, + "waitP50": 20, + "waitP99": 45, + "waitMax": 45 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.6194690265486725, + "meanWait": 16.866666666666667, + "waitP50": 13, + "waitP99": 35, + "waitMax": 35 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.6637168141592921, + "meanWait": 12.266666666666667, + "waitP50": 11, + "waitP99": 42, + "waitMax": 42 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.6637168141592921, + "meanWait": 16, + "waitP50": 12, + "waitP99": 57, + "waitMax": 57 + } + ] + }, + { + "selector": "stride", + "contentionWorstShareOverWeight": { + "mean": 0.7228911750188346, + "min": 0.6547619047619048, + "max": 0.7692307692307693 + }, + "contentionJain": { + "mean": 0.8592005815786168, + "min": 0.8431297709923665, + "max": 0.8739841688654353 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.8617021276595742, + "meanWait": 1149.6333333333334, + "waitP50": 1193, + "waitP99": 2099, + "waitMax": 2110 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 16.2, + "waitP50": 16, + "waitP99": 40, + "waitMax": 40 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7446808510638298, + "meanWait": 16.066666666666666, + "waitP50": 14, + "waitP99": 48, + "waitMax": 48 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 13, + "waitP50": 10, + "waitP99": 41, + "waitMax": 41 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 9.533333333333333, + "waitP50": 9, + "waitP99": 24, + "waitMax": 24 + } + ] + }, + { + "selector": "codel(sfq)", + "contentionWorstShareOverWeight": { + "mean": 0.7228911750188346, + "min": 0.6547619047619048, + "max": 0.7692307692307693 + }, + "contentionJain": { + "mean": 0.8592005815786168, + "min": 0.8431297709923665, + "max": 0.8739841688654353 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.8617021276595742, + "meanWait": 1149.6333333333334, + "waitP50": 1193, + "waitP99": 2099, + "waitMax": 2110 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 16.2, + "waitP50": 16, + "waitP99": 40, + "waitMax": 40 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7446808510638298, + "meanWait": 16.066666666666666, + "waitP50": 14, + "waitP99": 48, + "waitMax": 48 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 13, + "waitP50": 10, + "waitP99": 41, + "waitMax": 41 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.7978723404255319, + "meanWait": 9.533333333333333, + "waitP50": 9, + "waitP99": 24, + "waitMax": 24 + } + ] + }, + { + "selector": "codel(baseline)", + "contentionWorstShareOverWeight": { + "mean": 0.10398894629340381, + "min": 0, + "max": 0.156794425087108 + }, + "contentionJain": { + "mean": 0.28297359307689535, + "min": 0.27694949009374037, + "max": 0.28877519486316655 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 4.137931034482758, + "meanWait": 871.9875, + "waitP50": 875, + "waitP99": 1644, + "waitMax": 1646 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.15517241379310343, + "meanWait": 1602.6, + "waitP50": 1484, + "waitP99": 2022, + "waitMax": 2022 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.24137931034482757, + "meanWait": 1044.3333333333333, + "waitP50": 1116, + "waitP99": 1842, + "waitMax": 1842 + }, + { + "groupId": "light-3", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.20689655172413793, + "meanWait": 516.4, + "waitP50": 477, + "waitP99": 1653, + "waitMax": 1653 + }, + { + "groupId": "light-4", + "dequeued": 15, + "weight": 1, + "share": 0.05, + "contentionShareOverWeight": 0.2586206896551724, + "meanWait": 820.6, + "waitP50": 732, + "waitP99": 1761, + "waitMax": 1761 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/ckTrickle.json b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/ckTrickle.json new file mode 100644 index 0000000000..6272d9960e --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/ckTrickle.json @@ -0,0 +1,303 @@ +{ + "scenario": "ckTrickle", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "weights": { + "bulk": 1, + "trickle-1": 1, + "trickle-2": 1 + }, + "perDiscipline": [ + { + "selector": "baseline", + "contentionWorstShareOverWeight": { + "mean": 0.27872569582223233, + "min": 0.2542372881355932, + "max": 0.29096989966555187 + }, + "contentionJain": { + "mean": 0.4983343471106942, + "min": 0.4906272022551091, + "max": 0.5021879195384868 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 2.440677966101695, + "meanWait": 871.9875, + "waitP50": 875, + "waitP99": 1644, + "waitMax": 1646 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.3050847457627119, + "meanWait": 1338.6666666666667, + "waitP50": 1399, + "waitP99": 1705, + "waitMax": 1705 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.2542372881355932, + "meanWait": 1165.8666666666666, + "waitP50": 1162, + "waitP99": 1718, + "waitMax": 1718 + } + ] + }, + { + "selector": "sfq", + "contentionWorstShareOverWeight": { + "mean": 0.9092746009294808, + "min": 0.8910891089108911, + "max": 0.9183673469387755 + }, + "contentionJain": { + "mean": 0.9835072030385605, + "min": 0.9768265823996936, + "max": 0.9868475133579941 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.163265306122449, + "meanWait": 1237.4333333333334, + "waitP50": 1317, + "waitP99": 2139, + "waitMax": 2140 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9183673469387755, + "meanWait": 23.933333333333334, + "waitP50": 13, + "waitP99": 96, + "waitMax": 96 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9183673469387755, + "meanWait": 20.3, + "waitP50": 14, + "waitP99": 62, + "waitMax": 62 + } + ] + }, + { + "selector": "drr", + "contentionWorstShareOverWeight": { + "mean": 0.7900070943549204, + "min": 0.7692307692307692, + "max": 0.8181818181818181 + }, + "contentionJain": { + "mean": 0.9184573419314669, + "min": 0.9037433155080212, + "max": 0.937984496124031 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.4347826086956523, + "meanWait": 1235.5583333333334, + "waitP50": 1315, + "waitP99": 2139, + "waitMax": 2140 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.782608695652174, + "meanWait": 30.433333333333334, + "waitP50": 18, + "waitP99": 128, + "waitMax": 128 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.782608695652174, + "meanWait": 26.333333333333332, + "waitP50": 25, + "waitP99": 61, + "waitMax": 61 + } + ] + }, + { + "selector": "stride", + "contentionWorstShareOverWeight": { + "mean": 0.9092746009294808, + "min": 0.8910891089108911, + "max": 0.9183673469387755 + }, + "contentionJain": { + "mean": 0.9835072030385605, + "min": 0.9768265823996936, + "max": 0.9868475133579941 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.163265306122449, + "meanWait": 1237.4333333333334, + "waitP50": 1317, + "waitP99": 2139, + "waitMax": 2140 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9183673469387755, + "meanWait": 23.933333333333334, + "waitP50": 13, + "waitP99": 96, + "waitMax": 96 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9183673469387755, + "meanWait": 20.3, + "waitP50": 14, + "waitP99": 62, + "waitMax": 62 + } + ] + }, + { + "selector": "codel(sfq)", + "contentionWorstShareOverWeight": { + "mean": 0.9092746009294808, + "min": 0.8910891089108911, + "max": 0.9183673469387755 + }, + "contentionJain": { + "mean": 0.9835072030385605, + "min": 0.9768265823996936, + "max": 0.9868475133579941 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 1.163265306122449, + "meanWait": 1237.4333333333334, + "waitP50": 1317, + "waitP99": 2139, + "waitMax": 2140 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9183673469387755, + "meanWait": 23.933333333333334, + "waitP50": 13, + "waitP99": 96, + "waitMax": 96 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.9183673469387755, + "meanWait": 20.3, + "waitP50": 14, + "waitP99": 62, + "waitMax": 62 + } + ] + }, + { + "selector": "codel(baseline)", + "contentionWorstShareOverWeight": { + "mean": 0.01818181818181818, + "min": 0, + "max": 0.05454545454545454 + }, + "contentionJain": { + "mean": 0.42049894668433757, + "min": 0.4153846153846154, + "max": 0.4307276092837818 + }, + "detailSeed0": [ + { + "groupId": "bulk", + "dequeued": 240, + "weight": 1, + "share": 0.8, + "contentionShareOverWeight": 2.6666666666666665, + "meanWait": 871.9875, + "waitP50": 875, + "waitP99": 1644, + "waitMax": 1646 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0, + "meanWait": 1487.9333333333334, + "waitP50": 1569, + "waitP99": 1903, + "waitMax": 1903 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.1, + "contentionShareOverWeight": 0.3333333333333333, + "meanWait": 1027.9333333333334, + "waitP50": 1014, + "waitP99": 1653, + "waitMax": 1653 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/xtask-crossTaskSkew.json b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/xtask-crossTaskSkew.json new file mode 100644 index 0000000000..5ef0270251 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/results/xtask-crossTaskSkew.json @@ -0,0 +1,192 @@ +{ + "scenario": "crossTaskSkew", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "envLimit": 4, + "heavyTotalCap": 2, + "lightKey": "light-1", + "weights": { + "heavy": 1, + "light-1": 1, + "light-2": 1 + }, + "perTreatment": [ + { + "treatment": "baseline(fairqueue)", + "lightWait": { + "mean": 474.9555555555556, + "min": 352, + "max": 597.7333333333333 + }, + "heavyWait": { + "mean": 833.5527777777778, + "min": 761.4041666666667, + "max": 956.3708333333333 + }, + "makespan": { + "mean": 2039, + "min": 1765, + "max": 2285 + }, + "contentionWorst": { + "mean": 0.09807312252964427, + "min": 0.0703125, + "max": 0.1171875 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8888888888888888, + "contentionShareOverWeight": 2.70703125, + "meanWait": 956.3708333333333, + "waitP50": 961, + "waitP99": 1791, + "waitMax": 1796 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05555555555555555, + "contentionShareOverWeight": 0.17578125, + "meanWait": 597.7333333333333, + "waitP50": 709, + "waitP99": 935, + "waitMax": 935 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05555555555555555, + "contentionShareOverWeight": 0.1171875, + "meanWait": 529.6, + "waitP50": 695, + "waitP99": 919, + "waitMax": 919 + } + ] + }, + { + "treatment": "heavyTotalCap", + "lightWait": { + "mean": 2, + "min": 0, + "max": 4.533333333333333 + }, + "heavyWait": { + "mean": 1556.2055555555555, + "min": 1426.3708333333334, + "max": 1773.0791666666667 + }, + "makespan": { + "mean": 3039.3333333333335, + "min": 2863, + "max": 3312 + }, + "contentionWorst": { + "mean": 0.43014705882352944, + "min": 0.1875, + "max": 0.75 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8888888888888888, + "contentionShareOverWeight": 0.75, + "meanWait": 1773.0791666666667, + "waitP50": 1768, + "waitP99": 3296, + "waitMax": 3312 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05555555555555555, + "contentionShareOverWeight": 1.125, + "meanWait": 4.533333333333333, + "waitP50": 0, + "waitP99": 32, + "waitMax": 32 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05555555555555555, + "contentionShareOverWeight": 1.125, + "meanWait": 3.533333333333333, + "waitP50": 0, + "waitP99": 49, + "waitMax": 49 + } + ] + }, + { + "treatment": "sfq", + "lightWait": { + "mean": 13.555555555555555, + "min": 7.933333333333334, + "max": 21.466666666666665 + }, + "heavyWait": { + "mean": 896.6930555555555, + "min": 813.3458333333333, + "max": 1030.9541666666667 + }, + "makespan": { + "mean": 2039, + "min": 1765, + "max": 2285 + }, + "contentionWorst": { + "mean": 0.695078031212485, + "min": 0.6470588235294118, + "max": 0.7647058823529411 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 240, + "weight": 1, + "share": 0.8888888888888888, + "contentionShareOverWeight": 1.3529411764705883, + "meanWait": 1030.9541666666667, + "waitP50": 1071, + "waitP99": 1864, + "waitMax": 1866 + }, + { + "groupId": "light-1", + "dequeued": 15, + "weight": 1, + "share": 0.05555555555555555, + "contentionShareOverWeight": 0.8823529411764707, + "meanWait": 21.466666666666665, + "waitP50": 15, + "waitP99": 79, + "waitMax": 79 + }, + { + "groupId": "light-2", + "dequeued": 15, + "weight": 1, + "share": 0.05555555555555555, + "contentionShareOverWeight": 0.7647058823529411, + "meanWait": 11.266666666666667, + "waitP50": 13, + "waitP99": 26, + "waitMax": 26 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/ckDriver.smoke.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/ckDriver.smoke.test.ts new file mode 100644 index 0000000000..1f6adf5565 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/ckDriver.smoke.test.ts @@ -0,0 +1,45 @@ +import { redisTest } from "@internal/testcontainers"; +import { describe, expect } from "vitest"; +import { runCkScenario } from "../harness/ckDriver.js"; +import { buildWorkload } from "../../fairness-spike/harness/workload.js"; +import { BaselineCk, SfqCk } from "../disciplines.js"; + +describe("ck driver smoke + fidelity", () => { + redisTest( + "baseline starves light keys (age order); SFQ fixes it", + async ({ redisContainer }) => { + const redis = { + keyPrefix: "rq:ck-smoke:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + // one heavy concurrency key with a big backlog, three light keys + const workload = buildWorkload({ + seed: "ck-smoke", + envConcurrencyLimit: 3, + tenants: [ + { tenantId: "heavy", runCount: 90, holdMsMean: 25 }, + { tenantId: "light-1", runCount: 10, holdMsMean: 25 }, + { tenantId: "light-2", runCount: 10, holdMsMean: 25 }, + { tenantId: "light-3", runCount: 10, holdMsMean: 25 }, + ], + }); + const total = 120; + + const baseline = await runCkScenario({ redis, discipline: new BaselineCk(), workload }); + const sfq = await runCkScenario({ redis, discipline: new SfqCk(), workload }); + + // full drain both ways (harness health) + expect(baseline.totalDequeued).toBe(total); + expect(sfq.totalDequeued).toBe(total); + + // fidelity: baseline age-order lets the heavy key dominate contention, so a + // light key is under-served; SFQ gives the light keys their fair share + expect(baseline.contentionWorstShareOverWeight).toBeLessThan(0.6); + expect(sfq.contentionWorstShareOverWeight).toBeGreaterThan( + baseline.contentionWorstShareOverWeight + 0.2 + ); + }, + 120_000 + ); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/ckReader.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/ckReader.test.ts new file mode 100644 index 0000000000..a3456e5114 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/ckReader.test.ts @@ -0,0 +1,81 @@ +import { redisTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { Logger } from "@trigger.dev/core/logger"; +import { createRedisClient } from "@internal/redis"; +import { Decimal } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { RunQueue } from "../../index.js"; +import { FairQueueSelectionStrategy } from "../../fairQueueSelectionStrategy.js"; +import { RunQueueFullKeyProducer } from "../../keyProducer.js"; +import type { InputPayload } from "../../types.js"; +import { CkReader } from "../ckReader.js"; + +const keys = new RunQueueFullKeyProducer(); + +const env = { + id: "e-ck", + type: "PRODUCTION" as const, + maximumConcurrencyLimit: 10, + concurrencyLimitBurstFactor: new Decimal(1.0), + project: { id: "p-ck" }, + organization: { id: "o-ck" }, +}; + +function msg(concurrencyKey: string, runId: string): InputPayload { + return { + runId, + orgId: "o-ck", + projectId: "p-ck", + environmentId: "e-ck", + environmentType: "PRODUCTION", + queue: "task/base", + concurrencyKey, + timestamp: Date.now(), + attempt: 0, + }; +} + +describe("CkReader", () => { + redisTest("reads concurrency keys from the ck index", async ({ redisContainer }) => { + const redisOptions = { + keyPrefix: "rq:ck:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + const queue = new RunQueue({ + name: "rq-ck", + tracer: trace.getTracer("rq-ck"), + logger: new Logger("RunQueueCk", "error"), + defaultEnvConcurrency: 10, + shardCount: 1, + masterQueueConsumersDisabled: true, + workerOptions: { disabled: true }, + keys, + redis: redisOptions, + queueSelectionStrategy: new FairQueueSelectionStrategy({ redis: redisOptions, keys }), + }); + const raw = createRedisClient(redisOptions); + + try { + await queue.enqueueMessage({ env, message: msg("ck1", "r1"), workerQueue: "e-ck", skipDequeueProcessing: true }); + await queue.enqueueMessage({ env, message: msg("ck2", "r2"), workerQueue: "e-ck", skipDequeueProcessing: true }); + + // Discover the raw member format for grounding. + const ckIndexKey = keys.ckIndexKeyFromQueue(keys.queueKey(env, "task/base", "ck1")); + const dump = await raw.zrange(ckIndexKey, 0, -1, "WITHSCORES"); + // eslint-disable-next-line no-console + console.log("CK_INDEX_DUMP", JSON.stringify(dump)); + + const reader = new CkReader(raw, keys, "rq:ck:"); + const active = await reader.readActiveCks(keys.queueKey(env, "task/base", "ck1")); + // eslint-disable-next-line no-console + console.log("CK_ACTIVE", JSON.stringify(active)); + + expect(active.map((a) => a.concurrencyKey).sort()).toEqual(["ck1", "ck2"]); + expect(active.every((a) => typeof a.headScore === "number")).toBe(true); + } finally { + await raw.quit(); + await queue.quit(); + } + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/ckRescorer.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/ckRescorer.test.ts new file mode 100644 index 0000000000..73aad4bfa2 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/ckRescorer.test.ts @@ -0,0 +1,63 @@ +import { redisTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { Logger } from "@trigger.dev/core/logger"; +import { createRedisClient } from "@internal/redis"; +import { Decimal } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { RunQueue } from "../../index.js"; +import { FairQueueSelectionStrategy } from "../../fairQueueSelectionStrategy.js"; +import { RunQueueFullKeyProducer } from "../../keyProducer.js"; +import type { InputPayload } from "../../types.js"; +import { CkReader } from "../ckReader.js"; +import { rescoreCkIndex } from "../ckRescorer.js"; + +const keys = new RunQueueFullKeyProducer(); +const env = { + id: "e-ck", + type: "PRODUCTION" as const, + maximumConcurrencyLimit: 10, + concurrencyLimitBurstFactor: new Decimal(1.0), + project: { id: "p-ck" }, + organization: { id: "o-ck" }, +}; +function msg(ck: string, runId: string): InputPayload { + return { + runId, orgId: "o-ck", projectId: "p-ck", environmentId: "e-ck", + environmentType: "PRODUCTION", queue: "task/base", concurrencyKey: ck, + timestamp: Date.now(), attempt: 0, + }; +} + +describe("rescoreCkIndex", () => { + redisTest("makes the Lua's oldest-first pick follow the given order", async ({ redisContainer }) => { + const redisOptions = { keyPrefix: "rq:ck:", host: redisContainer.getHost(), port: redisContainer.getPort() }; + const queue = new RunQueue({ + name: "rq-ck", tracer: trace.getTracer("rq-ck"), logger: new Logger("RunQueueCk", "error"), + defaultEnvConcurrency: 10, shardCount: 1, masterQueueConsumersDisabled: true, + workerOptions: { disabled: true }, keys, redis: redisOptions, + queueSelectionStrategy: new FairQueueSelectionStrategy({ redis: redisOptions, keys }), + }); + const raw = createRedisClient(redisOptions); + try { + for (const ck of ["ck1", "ck2", "ck3"]) { + await queue.enqueueMessage({ env, message: msg(ck, `${ck}-r`), workerQueue: "e-ck", skipDequeueProcessing: true }); + } + const base = keys.queueKey(env, "task/base", "ck1"); + const reader = new CkReader(raw, keys, "rq:ck:"); + const active = await reader.readActiveCks(base); + const ckOf = (k: string) => active.find((a) => a.concurrencyKey === k)!.ckQueue; + + // Force order ck3, ck1, ck2 + const now = Date.now(); + await rescoreCkIndex(raw, keys, base, [ckOf("ck3"), ckOf("ck1"), ckOf("ck2")], now); + + const ckIndexKey = keys.ckIndexKeyFromQueue(base); + const ordered = await raw.zrangebyscore(ckIndexKey, "-inf", now); + const cks = ordered.map((m) => keys.descriptorFromQueue(m).concurrencyKey); + expect(cks).toEqual(["ck3", "ck1", "ck2"]); + } finally { + await raw.quit(); + await queue.quit(); + } + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/disciplines.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/disciplines.test.ts new file mode 100644 index 0000000000..a581e504ac --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike-ck/tests/disciplines.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect } from "vitest"; +import { BaselineCk, SfqCk, DrrCk, StrideCk, CodelCk } from "../disciplines.js"; +import type { ActiveCk } from "../ckReader.js"; + +function ck(concurrencyKey: string, headScore: number): ActiveCk { + return { ckQueue: `q:${concurrencyKey}`, concurrencyKey, headScore }; +} + +describe("CK disciplines", () => { + it("baseline orders by head age (oldest first) and does not rescore", () => { + const b = new BaselineCk(); + expect(b.rescore).toBe(false); + const order = b.order([ck("a", 30), ck("b", 10), ck("c", 20)], 100); + expect(order).toEqual(["q:b", "q:c", "q:a"]); + }); + + it("SFQ puts an under-served key ahead of an over-served one", () => { + const s = new SfqCk(); + for (let i = 0; i < 10; i++) s.onServiced("a"); + const order = s.order([ck("a", 5), ck("b", 5)], 100); + expect(order[0]).toBe("q:b"); + }); + + it("DRR alternates evenly between two keys", () => { + const d = new DrrCk(); + const active = [ck("a", 5), ck("b", 5)]; + let a = 0; + let b = 0; + for (let i = 0; i < 10; i++) { + const top = d.order(active, 100)[0]; + if (top === "q:a") { + a++; + d.onServiced("a"); + } else { + b++; + d.onServiced("b"); + } + } + expect(Math.abs(a - b)).toBeLessThanOrEqual(1); + }); + + it("stride serves an under-served key first", () => { + const s = new StrideCk(); + for (let i = 0; i < 5; i++) s.onServiced("a"); + expect(s.order([ck("a", 5), ck("b", 5)], 100)[0]).toBe("q:b"); + }); + + it("SFQ monotonic floor: a returning idle key does not monopolise", () => { + const s = new SfqCk(); + // advance both, then 'b' idles while 'a' keeps getting served + s.order([ck("a", 1), ck("b", 1)], 10); + for (let i = 0; i < 20; i++) { + s.order([ck("a", 1)], 10 + i); + s.onServiced("a"); + } + // b returns with its stale (low) clock; floor should have advanced, so b is + // pulled up rather than sitting far below and monopolising + const order = s.order([ck("a", 1), ck("b", 1)], 40); + // b is served next (it is behind), but its clock jumps to the floor, so after + // one serve a is competitive again rather than b taking 20 in a row + expect(order[0]).toBe("q:b"); + s.onServiced("b"); + const after = s.order([ck("a", 1), ck("b", 1)], 41); + expect(after[0]).toBe("q:a"); + }); + + it("CoDel hoists a stale key then reverts", () => { + // stub base always prefers a over b, so hoisting b is observable + const stub = { + name: "stub", + rescore: true as const, + order: (active: ActiveCk[]) => + [...active].sort((x, y) => (x.concurrencyKey < y.concurrencyKey ? -1 : 1)).map((a) => a.ckQueue), + onServiced: () => {}, + reset: () => {}, + }; + const codel = new CodelCk(stub, 50, 100); + // b's head is old (sojourn large); a is fresh + codel.order([ck("a", 390), ck("b", 0)], 200); // start tracking b's overage + const hoisted = codel.order([ck("a", 390), ck("b", 0)], 400); // interval elapsed + expect(hoisted[0]).toBe("q:b"); + const reverted = codel.order([ck("a", 390), ck("b", 399)], 400); // b now fresh + expect(reverted[0]).toBe("q:a"); + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/FINDINGS.md b/internal-packages/run-engine/src/run-queue/fairness-spike/FINDINGS.md new file mode 100644 index 0000000000..e386aba8d5 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/FINDINGS.md @@ -0,0 +1,168 @@ +# Fair-queueing spike: findings + +This is a throwaway spike. It ships nothing and should be deleted before any +merge to main; it exists to inform a design decision, not to become code. + +Read the caveats section before quoting any single number. Two of them matter up +front: (1) the candidate selectors are handed tenant identity (parsed from the +queue name) and the exact per-tenant weights, and the fairness target is defined +by those same groups and weights, so the candidate win over the tenant-blind +baseline is closer to definitional than discovered. (2) The harness anchors +message scores far in the past, which flattens all queue ages, so the baseline's +production age bias is not exercised here; the baseline measured is closer to a +uniform-random shuffle than the real one. + +Bottom line: at the base-queue grain, virtual-time ordering (SFQ) and stride give +tight, seed-stable proportional fairness, honour weights, and cut a starved +tenant's wait hard. DRR lands within noise of them (its small shortfall is a +measurement artifact of the harness, not an intrinsic property). The baseline is +fair on average but seed-variant and has no weight concept. The CoDel wrapper is +not worth shipping as built: it is a forced no-op under bulk arrival and it +actively hurt fairness on the one trickle-arrival workload that could exercise it. +The biggest single result is architectural: per-concurrency-key fairness (the +actual #2617 grain) cannot be expressed through the `RunQueueSelectionStrategy` +interface at all; it lives below that interface, in the CK-dequeue Lua. + +Every number comes from the real `RunQueue` against a testcontainers Redis, one +selector per run, real enqueue/dequeue/ack and real concurrency gating. Each +scenario runs over 3 seeds; tables show the mean and min..max spread. Per-tenant +detail (first seed) is in `results/*.json`. + +## Grain, and why it is not the concurrency key + +A tenant is the fairness group; a tenant owns one or more base queues. The +adversarial scenario gives one tenant 30 queues and the light tenants one each, +which is how the #2617 starvation shows up at the base-queue grain: an ordering +blind to tenant identity lets the many-queue tenant win most of the selection +chances. + +The concurrency-key grain #2617 asks for is not reachable through the strategy +interface. `FairQueueSelectionStrategy` reads the master-queue members verbatim, +and CK runs enqueue a single CK-wildcard entry per base queue. The per-CK pick +runs later inside `dequeueMessagesFromCkQueueTracked`, where `ckIndexKey` is a +ZSET of CK-queues scored by head timestamp and the Lua serves them oldest-first. +That age ordering is the unfairness. Fixing it means changing that Lua or the +`ckIndex` scoring, not the selection strategy. That is the follow-on spike. + +## How fairness is measured (and its limits) + +Because the sim drains every run, final throughput share is fixed by the workload +and cannot tell selectors apart. Two measures do: + +- contention share: a tenant's share of dequeues at instants when at least two + tenants have arrived, unserved work, over its expected weighted share. + `contWorstS/W` is the least-served contender; 1.0 is fair, near 0 means starved + while others had work. Getting this right took two corrections a review caught: + it must only count a tenant once its runs have actually arrived (else poisson + arrival looks like starvation), and the virtual-time floor must be monotonic + (else a returning idle tenant monopolises and skews the window). Even so, when + tenants have very different volumes (trickleStale: 30 runs vs 300) the + low-volume tenant can legitimately be over- or under-represented in the window, + so read this metric together with wait, not alone. +- wait: dequeue time minus enqueue time, per tenant, in the JSON. This is the + clean anti-staleness signal. (Note: `worstWaitP99` in the JSON is NOT an + anti-staleness win signal; it is dominated by the highest-volume tenant, which + a fair selector deliberately delays, so a fairer selector scores worse on it. + Use per-tenant wait.) + +## Results + +`contWorstS/W` mean over 3 seeds (min..max). Higher is fairer. + +| scenario | baseline | sfq | drr | stride | codel-sfq | codel-baseline | +| --------------- | ------------------- | ----- | ------------------- | ------ | --------- | -------------- | +| balanced | 0.889 (0.774..0.954)| 0.985 | 0.954 (0.923..0.970)| 0.985 | 0.985 | 0.889 | +| adversarialSkew | 0.288 (0.261..0.310)| 1.000 | 0.978 (0.968..0.984)| 1.000 | 1.000 | 0.288 | +| weighted | 0.703 (0.679..0.719)| 1.000 | 0.990 (0.977..1.000)| 1.000 | 1.000 | 0.703 | +| burst | 0.978 (0.966..0.992)| 0.992 | 0.958 (0.941..0.975)| 0.992 | 0.992 | 0.978 | +| longHold | 0.828 (0.800..0.842)| 0.981 | 0.981 | 0.981 | 0.981 | 0.828 | +| trickleStale | 0.208 (0.179..0.235)| 0.804 (0.769..0.826)| 0.776 (0.769..0.783)| 0.804 | 0.366 | 0.195 | + +Per-tenant mean wait (seed-a, logical ms), the anti-staleness signal: + +| scenario / selector | low-volume tenant wait | heavy tenant wait | +| ------------------------ | ---------------------- | ----------------- | +| adversarialSkew baseline | 1380 | 805 | +| adversarialSkew sfq | 319 | 1324 | +| trickleStale baseline | 1359 | 1234 | +| trickleStale sfq | 19 | 1353 | +| trickleStale codel-sfq | 213 | 1315 | + +Reading these: the fair selectors cut the light tenant's wait (skew 1380 to 319, +trickle 1359 to 19) by making the heavy tenant wait its fair turn. The heavy +tenant is not punished, it stops jumping the queue. CoDel undoes part of the +trickle win (19 back up to 213). + +## Verdict per mechanism + +- SFQ (start-time virtual time, the start-tag form of WFQ): the strongest result. + Perfect contention fairness under skew and weighting, seed-stable (zero variance + across seeds), and the largest cut to the starved tenant's wait. The floor is + now monotonic (a review found the earlier version let a returning idle tenant + monopolise; fixed). Recommended as the leaf ordering. +- Stride: identical to SFQ to the decimal on every scenario. The spike does not + separate them. Stride carries slightly less state. +- DRR: within noise of SFQ. It trails by a couple of points on balanced (0.954) + and burst (0.958) and matches SFQ elsewhere. That small shortfall is a + measurement artifact, not an intrinsic property: the driver drains a whole + capacity batch from a single strategy snapshot and only advances DRR's deficit + after the batch (via `onServiced`), so DRR's current-winner group, whose queues + it fronts together, grabs several slots before its deficit updates and its + deficit runs negative. Served one-at-a-time DRR is exactly fair (see + `drr.test.ts`). Note the earlier claim that "virtual-time sorts an over-served + group's queues to the back and so avoids this" was wrong: at a tie all of a + group's queues share one clock, so SFQ fronts them together too; the schemes + only separate after their state advances. DRR is O(1) and composes weight + trivially, so it is a fine choice if per-op cost matters, subject to that + caveat. +- CoDel wrapper: do not ship as built. Under bulk arrival it is a forced no-op: + all of a queue's runs share one enqueue timestamp, so every tenant's sojourn is + identical and they all cross the target together, so hoisting everyone collapses + to the base order (this is why codel-sfq equals sfq and codel-baseline equals + baseline to the decimal on those scenarios; it is one workload shape confirming + a null result, not five independent tests). On trickleStale, the one scenario + where sojourns diverge, the sojourn-hoist overshoots: it drops SFQ from 0.804 to + 0.366 and pushes the trickle tenant's wait from 19 back to 213. A staleness + monitor may still help on top of an unfair base or behind a hard concurrency + wall, but that needs a different construction and this spike does not support it. +- Baseline (`FairQueueSelectionStrategy`): fair on average on the easy scenarios + but seed-variant (balanced 0.774..0.954), no weight concept (weighted 0.703), + and it starves a light tenant under queue-count skew (0.288) and under trickle + arrival (0.208, trickle wait 1359). Remember its age bias is not exercised here + (see caveats), so this is a floor on its unfairness, not the production picture. + +## Caveats + +- Grain is base queues, not concurrency keys. The disciplines are grain-agnostic + so the ranking should carry over, but the #2617 gap itself needs the CK-Lua + spike. adversarialSkew is a proxy for that gap, not a measurement of it. +- Definitional advantage: candidates get tenant identity and exact weights the + real interface does not carry; the baseline structurally cannot. +- Baseline age bias inert: scores are anchored ~600s in the past, so all queue + ages are near-equal and the baseline degenerates to near-uniform selection. The + production age bias (which would give a heavy tenant's older heads more weight, + i.e. make skew worse) is not measured. +- Selection-only seam: the driver feeds serviced descriptors back via an + `onServiced` hook; production would advance selector state inside the ack/dequeue + Lua. The spike proves ordering logic, not that wiring. +- Cost was not rigorously measured. `selectionRounds` is roughly equal across + selectors (646..729) but is not comparable between them (a candidate reads all + queues per call; the baseline short-circuits at capacity), and there is no load + benchmark. DRR's "O(1)" advantage is a theory claim, not a spike measurement. +- Scenario quality varies. balanced best shows the baseline's variance; + adversarialSkew and weighted carry the clear separation; longHold and burst + barely separate the candidates; trickleStale's contention number only became + meaningful after two metric fixes and should be read with wait. Per-tenant p99 + equals max for the small (20 to 30 run) tenants, so "p99" there is just the max. +- Single Redis shard; single sequential consumer (not the multi-consumer, + Redis-hash-state design the spec sketched); simulated holds on a logical clock. + Three seeds shows the baseline's variance and the virtual-time schemes' + stability but is not a statistical study. + +## Recommended direction + +Use virtual-time (SFQ, or stride) for leaf ordering and compose weight with it. +DRR is an acceptable O(1) fallback given the batch caveat. Do not adopt the CoDel +wrapper as built. Then run the follow-on spike against the CK-dequeue Lua / +`ckIndex` scoring, because that is where per-tenant fairness actually has to land +in the current design. diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/fairnessSpike.bench.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/fairnessSpike.bench.test.ts new file mode 100644 index 0000000000..7815a03fd5 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/fairnessSpike.bench.test.ts @@ -0,0 +1,189 @@ +import { redisTest } from "@internal/testcontainers"; +import { createRedisClient, type RedisOptions } from "@internal/redis"; +import { describe } from "vitest"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { FairQueueSelectionStrategy } from "../fairQueueSelectionStrategy.js"; +import { RunQueueFullKeyProducer } from "../keyProducer.js"; +import type { RunQueueSelectionStrategy } from "../types.js"; +import { runScenario } from "./harness/driver.js"; +import { buildWorkload, weightsOf, type WorkloadConfig } from "./harness/workload.js"; +import { SCENARIOS } from "./harness/scenarios.js"; +import { SfqStrategy } from "./strategies/sfqStrategy.js"; +import { DrrStrategy } from "./strategies/drrStrategy.js"; +import { StrideStrategy } from "./strategies/strideStrategy.js"; +import { CodelWrapper } from "./strategies/codelWrapper.js"; +import type { GroupMetrics, RunMetrics } from "./harness/metrics.js"; +import type { SpikeSelectionStrategy } from "./types.js"; + +const keys = new RunQueueFullKeyProducer(); +const RESULTS_DIR = join(dirname(fileURLToPath(import.meta.url)), "results"); +const SEEDS = ["seed-a", "seed-b", "seed-c"]; + +const SELECTOR_NAMES = ["baseline", "sfq", "drr", "stride", "codel-sfq", "codel-baseline"] as const; +type SelectorName = (typeof SELECTOR_NAMES)[number]; + +type Built = { + strategy: RunQueueSelectionStrategy & Partial; + cleanup: () => Promise; +}; + +function makeBaseline(redis: RedisOptions): FairQueueSelectionStrategy { + return new FairQueueSelectionStrategy({ + redis, + keys, + seed: "spike", + biases: { concurrencyLimitBias: 0.75, availableCapacityBias: 0.3, queueAgeRandomization: 0.25 }, + }); +} + +/** Presents a bare RunQueueSelectionStrategy as a SpikeSelectionStrategy so it can be wrapped. */ +function asSpike(name: string, base: RunQueueSelectionStrategy): SpikeSelectionStrategy { + return { + name, + distributeFairQueuesFromParentQueue: (p, c) => base.distributeFairQueuesFromParentQueue(p, c), + onServiced() {}, + }; +} + +function buildSelector(name: SelectorName, redis: RedisOptions, weight: (g: string) => number): Built { + if (name === "baseline") { + return { strategy: makeBaseline(redis), cleanup: async () => {} }; + } + + const client = createRedisClient(redis); + const cleanup = async () => { + await client.quit(); + }; + + switch (name) { + case "sfq": + return { strategy: new SfqStrategy({ redis: client, keys, weight }), cleanup }; + case "drr": + return { strategy: new DrrStrategy({ redis: client, keys, weight }), cleanup }; + case "stride": + return { strategy: new StrideStrategy({ redis: client, keys, weight }), cleanup }; + case "codel-sfq": + return { + strategy: new CodelWrapper({ + base: new SfqStrategy({ redis: client, keys, weight }), + redis: client, + keys, + targetMs: 200, + intervalMs: 100, + }), + cleanup, + }; + case "codel-baseline": + return { + strategy: new CodelWrapper({ + base: asSpike("baseline", makeBaseline(redis)), + redis: client, + keys, + targetMs: 200, + intervalMs: 100, + }), + cleanup, + }; + } +} + +function stats(xs: number[]) { + const mean = xs.reduce((a, b) => a + b, 0) / xs.length; + return { mean, min: Math.min(...xs), max: Math.max(...xs) }; +} + +function fmt(n: number, digits = 3): string { + return Number.isFinite(n) ? n.toFixed(digits) : String(n); +} + +type SeedRun = { seed: string; metrics: RunMetrics }; + +describe("fairness spike bench", () => { + mkdirSync(RESULTS_DIR, { recursive: true }); + + for (const [scenarioName, baseConfig] of Object.entries(SCENARIOS)) { + redisTest( + `scenario: ${scenarioName}`, + async ({ redisContainer }) => { + const runsBySelector = new Map(); + for (const name of SELECTOR_NAMES) runsBySelector.set(name, []); + + for (const seed of SEEDS) { + const config: WorkloadConfig = { ...baseConfig, seed }; + const workload = buildWorkload(config); + const expectedTotal = workload.tenants.reduce((n, t) => n + t.runCount, 0); + const weights = weightsOf(workload); + const weight = (g: string) => weights[g] ?? 1; + + for (const name of SELECTOR_NAMES) { + const redis: RedisOptions = { + keyPrefix: `rq:spike:${scenarioName}:${name}:${seed}:`, + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + const built = buildSelector(name, redis, weight); + try { + const metrics = await runScenario({ redis, strategy: built.strategy, workload }); + if (metrics.totalDequeued !== expectedTotal) { + throw new Error( + `${scenarioName}/${name}/${seed}: dequeued ${metrics.totalDequeued} of ${expectedTotal}` + ); + } + runsBySelector.get(name)!.push({ seed, metrics }); + } finally { + await built.cleanup(); + } + } + } + + // Aggregate across seeds; keep full per-tenant detail for the first seed. + const perSelector = SELECTOR_NAMES.map((name) => { + const runs = runsBySelector.get(name)!; + const cwsw = stats(runs.map((r) => r.metrics.contentionWorstShareOverWeight)); + const jain = stats(runs.map((r) => r.metrics.contentionJain)); + const worstWaitP99 = stats(runs.map((r) => r.metrics.worstWaitP99)); + return { + selector: name, + contentionWorstShareOverWeight: cwsw, + contentionJain: jain, + worstWaitP99: worstWaitP99, + // Rough cost proxies. redisOps here is the number of strategy + // invocations, which is NOT comparable across selectors (a candidate + // reads all queues per call; the baseline short-circuits when the env + // is at capacity), so treat as illustrative only. + selectionRounds: stats(runs.map((r) => r.metrics.redisOps)), + wallClockMs: stats(runs.map((r) => r.metrics.wallClockMs)), + detailSeed0: runs[0].metrics.perGroup as GroupMetrics[], + }; + }); + + const firstWorkload = buildWorkload({ ...baseConfig, seed: SEEDS[0] }); + writeFileSync( + join(RESULTS_DIR, `${scenarioName}.json`), + JSON.stringify( + { scenario: scenarioName, seeds: SEEDS, weights: weightsOf(firstWorkload), perSelector }, + null, + 2 + ) + ); + + const lines = [ + ``, + `### ${scenarioName} (${SEEDS.length} seeds)`, + `selector contWorstS/W (min..max) contJain worstWaitP99(mean)`, + ...perSelector.map((r) => { + const c = r.contentionWorstShareOverWeight; + return `${r.selector.padEnd(15)} ${fmt(c.mean).padStart(7)} (${fmt(c.min)}..${fmt( + c.max + )}) ${fmt(r.contentionJain.mean).padStart(6)} ${fmt(r.worstWaitP99.mean, 0).padStart(8)}`; + }), + ``, + ]; + process.stdout.write(lines.join("\n") + "\n"); + }, + 400_000 + ); + } +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/harness/driver.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/harness/driver.ts new file mode 100644 index 0000000000..84e216c107 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/harness/driver.ts @@ -0,0 +1,184 @@ +import { trace } from "@internal/tracing"; +import { Logger } from "@trigger.dev/core/logger"; +import { createRedisClient } from "@internal/redis"; +import { Decimal } from "@trigger.dev/database"; +import { RunQueue } from "../../index.js"; +import { RunQueueFullKeyProducer } from "../../keyProducer.js"; +import type { InputPayload, RunQueueSelectionStrategy } from "../../types.js"; +import { groupIdFromQueueName, type SpikeSelectionStrategy } from "../types.js"; +import { computeMetrics, type DequeueEvent, type RunMetrics } from "./metrics.js"; +import { expandEvents, totalsOf, weightsOf, type WorkloadSpec } from "./workload.js"; + +const keys = new RunQueueFullKeyProducer(); + +const ORG = "o-spike"; +const PROJECT = "p-spike"; +const ENV = "e-spike"; + +export type DriverConfig = { + redis: { host: string; port: number; keyPrefix: string }; + strategy: RunQueueSelectionStrategy & + Partial>; + workload: WorkloadSpec; + /** ceiling on logical time (ms); the loop is event-driven so this only guards runaway starvation */ + maxLogicalMs?: number; + /** + * Per-base-queue concurrency limit, keyed by the workload queue name + * (e.g. `heavy~0`). Sets the real per-queue concurrency gate the dequeue Lua + * enforces. For a keyless task (one base queue) this IS the per-task total cap, + * so it models the plan's Phase-1 total cap for cross-task isolation. + */ + perQueueCap?: Record; +}; + +function authenticatedEnv(limit: number) { + return { + id: ENV, + type: "PRODUCTION" as const, + maximumConcurrencyLimit: limit, + concurrencyLimitBurstFactor: new Decimal(1.0), + project: { id: PROJECT }, + organization: { id: ORG }, + }; +} + +/** + * Runs one workload through a real RunQueue against Redis and returns fairness / + * latency / cost metrics. Deterministic: a logical clock drives arrivals and + * concurrency-slot releases, and the loop jumps straight to the next event + * rather than ticking through idle time. + * + * The RunQueue's background worker is disabled and acks skip dequeue processing, + * so the ONLY thing that moves runs out of the queue is the driver's explicit + * `testDequeueFromMasterQueue` call. Otherwise the internal worker would drain + * runs through the normal worker-queue path behind the driver's back. + */ +export async function runScenario(config: DriverConfig): Promise { + const maxLogicalMs = config.maxLogicalMs ?? 600_000; + const limit = config.workload.envConcurrencyLimit; + const env = authenticatedEnv(limit); + + const admin = createRedisClient(config.redis); + // NOTE: flushes the whole Redis DB, ignoring keyPrefix. Safe against the + // dedicated testcontainer this spike runs on; do not point config.redis at a + // shared instance. + await admin.flushdb(); + + const queue = new RunQueue({ + name: "rq-spike", + tracer: trace.getTracer("rq-spike"), + logger: new Logger("RunQueueSpike", "error"), + defaultEnvConcurrency: limit, + shardCount: 1, + masterQueueConsumersDisabled: true, + workerOptions: { disabled: true }, + keys, + redis: config.redis, + queueSelectionStrategy: config.strategy, + }); + + await config.strategy.reset?.(); + + if (config.perQueueCap) { + for (const [queueName, cap] of Object.entries(config.perQueueCap)) { + await queue.updateQueueConcurrencyLimits(env, queueName, cap); + } + } + + const sorted = expandEvents(config.workload); + const total = sorted.length; + const holdByRun = new Map(); + const enqueueByRun = new Map(); + for (const e of sorted) { + holdByRun.set(e.runId, e.holdMs); + enqueueByRun.set(e.runId, e.enqueueAtMs); + } + + // Anchor message scores in the past so the dequeue Lua's `score <= now` filter + // always passes; arrival timing is enforced by the driver, not the score. + const scoreBase = Date.now() - maxLogicalMs - 5_000; + + const events: DequeueEvent[] = []; + const holding: Array<{ runId: string; releaseAtMs: number }> = []; + let enqCursor = 0; + let selectionRounds = 0; + let t = 0; + const startedAt = Date.now(); + + try { + while (events.length < total && t <= maxLogicalMs) { + // (a) release concurrency for holds that have expired by t + for (let i = holding.length - 1; i >= 0; i--) { + if (holding[i].releaseAtMs <= t) { + const [h] = holding.splice(i, 1); + await queue.acknowledgeMessage(ORG, h.runId, { skipDequeueProcessing: true }); + } + } + + // (b) enqueue all arrivals up to t + while (enqCursor < sorted.length && sorted[enqCursor].enqueueAtMs <= t) { + const e = sorted[enqCursor++]; + const message: InputPayload = { + runId: e.runId, + orgId: ORG, + projectId: PROJECT, + environmentId: ENV, + environmentType: "PRODUCTION", + queue: e.queueName, + timestamp: scoreBase + e.enqueueAtMs, + attempt: 0, + }; + await queue.enqueueMessage({ + env, + message, + workerQueue: ENV, + skipDequeueProcessing: true, + }); + } + + // (c) drain available env capacity at this instant, one run per queue per + // round, re-running the strategy each round so stateful selectors react. + config.strategy.setClock?.(scoreBase + t); + let progressed = true; + while (progressed) { + const msgs = await queue.testDequeueFromMasterQueue(0, ENV, 1); + selectionRounds++; + progressed = msgs.length > 0; + for (const m of msgs) { + const descriptor = keys.descriptorFromQueue(m.message.queue); + const runId = m.messageId; + events.push({ + groupId: groupIdFromQueueName(descriptor.queue), + runId, + enqueueAtMs: enqueueByRun.get(runId) ?? 0, + dequeueAtMs: t, + }); + await config.strategy.onServiced?.(descriptor, t); + holding.push({ runId, releaseAtMs: t + (holdByRun.get(runId) ?? 0) }); + } + } + + // advance to the next event (next arrival or next release) + const candidates: number[] = []; + if (enqCursor < sorted.length) candidates.push(sorted[enqCursor].enqueueAtMs); + if (holding.length > 0) candidates.push(Math.min(...holding.map((h) => h.releaseAtMs))); + if (candidates.length === 0) break; + const next = Math.min(...candidates); + t = Math.max(t + 1, next); + } + + return computeMetrics({ + events, + weights: weightsOf(config.workload), + totals: totalsOf(config.workload), + redisOps: selectionRounds, + wallClockMs: Date.now() - startedAt, + }); + } finally { + for (const h of holding) { + await queue.acknowledgeMessage(ORG, h.runId, { skipDequeueProcessing: true }).catch(() => {}); + } + await admin.quit(); + await queue.quit(); + } +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/harness/metrics.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/harness/metrics.ts new file mode 100644 index 0000000000..ed109a5ec5 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/harness/metrics.ts @@ -0,0 +1,188 @@ +import type { GroupId } from "../types.js"; + +export type DequeueEvent = { + groupId: GroupId; + runId: string; + enqueueAtMs: number; + dequeueAtMs: number; +}; + +export type GroupMetrics = { + groupId: GroupId; + dequeued: number; + weight: number; + /** share of total dequeues (fixed by the workload; sanity only) */ + share: number; + /** share of dequeues during the contention window, over expected weighted share */ + contentionShareOverWeight: number; + meanWait: number; + waitP50: number; + waitP99: number; + waitMax: number; +}; + +export type RunMetrics = { + perGroup: GroupMetrics[]; + totalDequeued: number; + redisOps: number; + wallClockMs: number; + /** + * Fairness during contention: the least-served group's share of the + * contention window, over its expected weighted share. 1.0 = perfectly fair, + * →0 = that group was starved while others had work. + */ + contentionWorstShareOverWeight: number; + contentionJain: number; + /** + * Logical time of the LAST dequeue (not completion; ~one holdMs before the final + * ack). Work-conservation signal, but read it ONLY on a service-bound workload + * with no late arrivals (e.g. ckHeavyIdle): there a non-work-conserving + * discipline that idles slots drains slower, so makespan is larger. On scenarios + * with poisson arrivals the last dequeue is partly set by the arrival tail, not + * the discipline, so makespan is arrival-confounded and not a clean signal there. + */ + makespanMs: number; + /** + * The largest per-group p99/max wait. NOTE: this is NOT an anti-staleness win + * signal. It is dominated by the highest-volume tenant, which a fair selector + * deliberately makes wait longer, so a fairer selector can score WORSE here. + * Read per-tenant waits (perGroup) for the anti-staleness story instead. + */ + worstWaitP99: number; + worstWaitMax: number; +}; + +function percentile(sortedAsc: number[], p: number): number { + if (sortedAsc.length === 0) return 0; + const idx = Math.min(sortedAsc.length - 1, Math.ceil((p / 100) * sortedAsc.length) - 1); + return sortedAsc[Math.max(0, idx)]; +} + +function jain(values: number[]): number { + if (values.length === 0) return 1; + const sum = values.reduce((a, b) => a + b, 0); + const sumSq = values.reduce((a, b) => a + b * b, 0); + if (sumSq === 0) return 1; + return (sum * sum) / (values.length * sumSq); +} + +/** + * Counts each group's dequeues that happen during genuine contention: a dequeue + * counts only if, at that instant, at least two groups have work that has + * already arrived (been enqueued) but not yet been served. This excludes both + * the drain tail (one group left) and, crucially, any stretch where a group's + * runs have not arrived yet under spread (poisson) arrival: a tenant only + * "contends" once its work exists, otherwise the metric would penalise it for + * the arrival process throttling its supply rather than the selector starving it. + */ +function contentionCounts(events: DequeueEvent[]): { + counts: Map; + windowSize: number; + contended: Set; +} { + const ordered = [...events].sort((a, b) => a.dequeueAtMs - b.dequeueAtMs); + + // Per-group sorted arrival (enqueue) times, so we can ask how many of a + // group's runs have arrived by a given instant. + const arrivalsByGroup = new Map(); + for (const e of events) { + const arr = arrivalsByGroup.get(e.groupId) ?? []; + arr.push(e.enqueueAtMs); + arrivalsByGroup.set(e.groupId, arr); + } + for (const arr of arrivalsByGroup.values()) arr.sort((a, b) => a - b); + + const arrivedBy = (g: GroupId, t: number): number => { + const arr = arrivalsByGroup.get(g) ?? []; + let n = 0; + for (const a of arr) { + if (a <= t) n++; + else break; + } + return n; + }; + + const dequeuedSoFar = new Map(); + const counts = new Map(); + const contended = new Set(); + let windowSize = 0; + + for (const e of ordered) { + const t = e.dequeueAtMs; + const withWork = [...arrivalsByGroup.keys()].filter( + (g) => arrivedBy(g, t) - (dequeuedSoFar.get(g) ?? 0) > 0 + ); + if (withWork.length >= 2) { + for (const g of withWork) contended.add(g); + counts.set(e.groupId, (counts.get(e.groupId) ?? 0) + 1); + windowSize++; + } + dequeuedSoFar.set(e.groupId, (dequeuedSoFar.get(e.groupId) ?? 0) + 1); + } + + return { counts, windowSize, contended }; +} + +export function computeMetrics(input: { + events: DequeueEvent[]; + weights: Record; + totals: Record; + redisOps: number; + wallClockMs: number; +}): RunMetrics { + const { events, weights } = input; + const groupIds = Object.keys(weights); + const total = events.length; + + const { counts: windowCounts, windowSize, contended } = contentionCounts(events); + const sumWindowWeights = [...contended].reduce((a, g) => a + (weights[g] ?? 1), 0); + + const waitsByGroup = new Map(); + for (const g of groupIds) waitsByGroup.set(g, []); + for (const e of events) { + waitsByGroup.get(e.groupId)?.push(e.dequeueAtMs - e.enqueueAtMs); + } + + const perGroup: GroupMetrics[] = groupIds.map((g) => { + const waits = (waitsByGroup.get(g) ?? []).slice().sort((a, b) => a - b); + const dequeued = waits.length; + const weight = weights[g] ?? 1; + const share = total > 0 ? dequeued / total : 0; + + const windowShare = windowSize > 0 ? (windowCounts.get(g) ?? 0) / windowSize : 0; + const expectedWindowShare = sumWindowWeights > 0 ? weight / sumWindowWeights : 0; + const contentionShareOverWeight = + expectedWindowShare > 0 ? windowShare / expectedWindowShare : 0; + + const meanWait = waits.length ? waits.reduce((a, b) => a + b, 0) / waits.length : 0; + + return { + groupId: g, + dequeued, + weight, + share, + contentionShareOverWeight, + meanWait, + waitP50: percentile(waits, 50), + waitP99: percentile(waits, 99), + waitMax: waits.length ? waits[waits.length - 1] : 0, + }; + }); + + // Only groups that actually had work during the contention window count + // toward fairness (a starved competitor contributes a 0, dragging worst down). + const competed = perGroup.filter((p) => contended.has(p.groupId)); + const cowVec = competed.map((p) => p.contentionShareOverWeight); + + return { + perGroup, + totalDequeued: total, + redisOps: input.redisOps, + wallClockMs: input.wallClockMs, + contentionWorstShareOverWeight: cowVec.length ? Math.min(...cowVec) : 1, + contentionJain: jain(cowVec), + makespanMs: events.length ? Math.max(...events.map((e) => e.dequeueAtMs)) : 0, + worstWaitP99: perGroup.length ? Math.max(...perGroup.map((p) => p.waitP99)) : 0, + worstWaitMax: perGroup.length ? Math.max(...perGroup.map((p) => p.waitMax)) : 0, + }; +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/harness/scenarios.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/harness/scenarios.ts new file mode 100644 index 0000000000..4d40d4d161 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/harness/scenarios.ts @@ -0,0 +1,71 @@ +import type { WorkloadConfig } from "./workload.js"; + +/** + * Named workloads. All seeded for determinism. A "tenant" is the fairness group; + * `queueCount` is how many base queues that tenant owns. The adversarial + * scenario gives one tenant many queues, which is how the #2617 starvation + * (a tenant multiplying its selection chances) shows up at the base-queue grain. + */ +export const SCENARIOS: Record> = { + balanced: { + envConcurrencyLimit: 5, + tenants: [ + { tenantId: "t-a", runCount: 50, holdMsMean: 30 }, + { tenantId: "t-b", runCount: 50, holdMsMean: 30 }, + { tenantId: "t-c", runCount: 50, holdMsMean: 30 }, + { tenantId: "t-d", runCount: 50, holdMsMean: 30 }, + ], + }, + + adversarialSkew: { + envConcurrencyLimit: 5, + tenants: [ + { tenantId: "heavy", runCount: 250, queueCount: 30, holdMsMean: 25 }, + { tenantId: "light-1", runCount: 20, holdMsMean: 25 }, + { tenantId: "light-2", runCount: 20, holdMsMean: 25 }, + { tenantId: "light-3", runCount: 20, holdMsMean: 25 }, + { tenantId: "light-4", runCount: 20, holdMsMean: 25 }, + { tenantId: "light-5", runCount: 20, holdMsMean: 25 }, + ], + }, + + weighted: { + envConcurrencyLimit: 4, + tenants: [ + { tenantId: "big", runCount: 300, weight: 3, holdMsMean: 30 }, + { tenantId: "small", runCount: 300, weight: 1, holdMsMean: 30 }, + ], + }, + + burst: { + envConcurrencyLimit: 6, + tenants: Array.from({ length: 6 }, (_, i) => ({ + tenantId: `burst-${i}`, + runCount: 100, + startAtMs: 500, + holdMsMean: 20, + })), + }, + + longHold: { + envConcurrencyLimit: 4, + tenants: [ + { tenantId: "slow-1", runCount: 40, holdMsMean: 500 }, + { tenantId: "slow-2", runCount: 40, holdMsMean: 500 }, + { tenantId: "fast-1", runCount: 40, holdMsMean: 20 }, + { tenantId: "fast-2", runCount: 40, holdMsMean: 20 }, + ], + }, + + // A bulk backlog whose heads stay old, plus light tenants trickling in later + // whose runs then wait. This makes the baseline's age bias live and gives a + // CoDel wrapper divergent per-tenant sojourns to react to. + trickleStale: { + envConcurrencyLimit: 4, + tenants: [ + { tenantId: "heavy", runCount: 300, queueCount: 20, holdMsMean: 25 }, + { tenantId: "trickle-1", runCount: 30, arrival: "poisson", ratePerSec: 25, holdMsMean: 25 }, + { tenantId: "trickle-2", runCount: 30, arrival: "poisson", ratePerSec: 25, holdMsMean: 25 }, + ], + }, +}; diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/harness/workload.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/harness/workload.ts new file mode 100644 index 0000000000..3214088656 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/harness/workload.ts @@ -0,0 +1,112 @@ +import seedrandom from "seedrandom"; +import { GROUP_SEPARATOR, type GroupId } from "../types.js"; + +/** + * Seeded synthetic workload generator. Everything is derived from `seed`, so + * the same config produces byte-identical events across runs and across + * selectors. Hold durations are precomputed per run (not sampled live) so every + * selector holds a given run for exactly the same time. + * + * A "tenant" is the fairness group. A tenant owns `queueCount` distinct base + * queues (named `${tenantId}~${index}`) and its runs are spread round-robin + * across them. A heavy tenant with many queues is how the spike reproduces the + * #2617 starvation dynamic at the base-queue grain. + */ + +export type ArrivalMode = "immediate" | "poisson"; + +export type TenantConfig = { + tenantId: string; + runCount: number; + weight?: number; + queueCount?: number; + arrival?: ArrivalMode; + ratePerSec?: number; + holdMsMean?: number; + startAtMs?: number; +}; + +export type WorkloadConfig = { + seed: string; + envConcurrencyLimit: number; + tenants: TenantConfig[]; +}; + +export type EnqueueEvent = { + groupId: GroupId; + queueName: string; + runId: string; + enqueueAtMs: number; + holdMs: number; +}; + +export type TenantSpec = { + tenantId: GroupId; + runCount: number; + weight: number; + queueCount: number; + events: EnqueueEvent[]; +}; + +export type WorkloadSpec = { + seed: string; + envConcurrencyLimit: number; + tenants: TenantSpec[]; +}; + +function expSample(rng: seedrandom.PRNG, meanMs: number): number { + return Math.max(1, Math.round(-Math.log(1 - rng()) * meanMs)); +} + +export function buildWorkload(config: WorkloadConfig): WorkloadSpec { + const rng = seedrandom(config.seed); + + const tenants: TenantSpec[] = config.tenants.map((t) => { + const weight = t.weight ?? 1; + const queueCount = t.queueCount ?? 1; + const arrival = t.arrival ?? "immediate"; + const holdMsMean = t.holdMsMean ?? 50; + const startAtMs = t.startAtMs ?? 0; + const ratePerSec = t.ratePerSec ?? 100; + + const events: EnqueueEvent[] = []; + let cursor = startAtMs; + + for (let i = 0; i < t.runCount; i++) { + if (arrival === "poisson" && i > 0) { + cursor += expSample(rng, 1000 / ratePerSec); + } + const holdMs = expSample(rng, holdMsMean); + const qi = i % queueCount; + events.push({ + groupId: t.tenantId, + queueName: `${t.tenantId}${GROUP_SEPARATOR}${qi}`, + runId: `${t.tenantId}${GROUP_SEPARATOR}${qi}#${i}`, + enqueueAtMs: arrival === "immediate" ? startAtMs : cursor, + holdMs, + }); + } + + return { tenantId: t.tenantId, runCount: t.runCount, weight, queueCount, events }; + }); + + return { seed: config.seed, envConcurrencyLimit: config.envConcurrencyLimit, tenants }; +} + +export function expandEvents(spec: WorkloadSpec): EnqueueEvent[] { + const all = spec.tenants.flatMap((t) => t.events); + return all.sort( + (a, b) => + a.enqueueAtMs - b.enqueueAtMs || + (a.groupId < b.groupId ? -1 : a.groupId > b.groupId ? 1 : 0) || + (a.runId < b.runId ? -1 : 1) + ); +} + +export function weightsOf(spec: WorkloadSpec): Record { + return Object.fromEntries(spec.tenants.map((t) => [t.tenantId, t.weight])); +} + +export function totalsOf(spec: WorkloadSpec): Record { + return Object.fromEntries(spec.tenants.map((t) => [t.tenantId, t.runCount])); +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/results/adversarialSkew.json b/internal-packages/run-engine/src/run-queue/fairness-spike/results/adversarialSkew.json new file mode 100644 index 0000000000..a46f5fe7f4 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/results/adversarialSkew.json @@ -0,0 +1,594 @@ +{ + "scenario": "adversarialSkew", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "weights": { + "heavy": 1, + "light-1": 1, + "light-2": 1, + "light-3": 1, + "light-4": 1, + "light-5": 1 + }, + "perSelector": [ + { + "selector": "baseline", + "contentionWorstShareOverWeight": { + "mean": 0.288387506534917, + "min": 0.2608695652173913, + "max": 0.3103448275862069 + }, + "contentionJain": { + "mean": 0.3111216639468977, + "min": 0.3083948698017876, + "max": 0.3132993915311066 + }, + "worstWaitP99": { + "mean": 1836, + "min": 1653, + "max": 1989 + }, + "selectionRounds": { + "mean": 646, + "min": 638, + "max": 658 + }, + "wallClockMs": { + "mean": 5439, + "min": 5087, + "max": 6100 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 250, + "weight": 1, + "share": 0.7142857142857143, + "contentionShareOverWeight": 4.310344827586207, + "meanWait": 805.244, + "waitP50": 805, + "waitP99": 1721, + "waitMax": 1756 + }, + { + "groupId": "light-1", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.3448275862068966, + "meanWait": 1437.9, + "waitP50": 1540, + "waitP99": 1948, + "waitMax": 1948 + }, + { + "groupId": "light-2", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.3448275862068966, + "meanWait": 1158.3, + "waitP50": 1310, + "waitP99": 1883, + "waitMax": 1883 + }, + { + "groupId": "light-3", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.3103448275862069, + "meanWait": 1498.35, + "waitP50": 1518, + "waitP99": 1989, + "waitMax": 1989 + }, + { + "groupId": "light-4", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.3448275862068966, + "meanWait": 1287.2, + "waitP50": 1268, + "waitP99": 1983, + "waitMax": 1983 + }, + { + "groupId": "light-5", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.3448275862068966, + "meanWait": 1518.5, + "waitP50": 1715, + "waitP99": 1959, + "waitMax": 1959 + } + ] + }, + { + "selector": "sfq", + "contentionWorstShareOverWeight": { + "mean": 1, + "min": 1, + "max": 1 + }, + "contentionJain": { + "mean": 1, + "min": 1, + "max": 1 + }, + "worstWaitP99": { + "mean": 1850.6666666666667, + "min": 1632, + "max": 2050 + }, + "selectionRounds": { + "mean": 646, + "min": 636, + "max": 653 + }, + "wallClockMs": { + "mean": 4455.333333333333, + "min": 3897, + "max": 4776 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 250, + "weight": 1, + "share": 0.7142857142857143, + "contentionShareOverWeight": 1, + "meanWait": 1323.616, + "waitP50": 1426, + "waitP99": 2050, + "waitMax": 2051 + }, + { + "groupId": "light-1", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 308.2, + "waitP50": 237, + "waitP99": 689, + "waitMax": 689 + }, + { + "groupId": "light-2", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 312.15, + "waitP50": 238, + "waitP99": 695, + "waitMax": 695 + }, + { + "groupId": "light-3", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 318.65, + "waitP50": 242, + "waitP99": 696, + "waitMax": 696 + }, + { + "groupId": "light-4", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 325.7, + "waitP50": 249, + "waitP99": 700, + "waitMax": 700 + }, + { + "groupId": "light-5", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 332.75, + "waitP50": 252, + "waitP99": 710, + "waitMax": 710 + } + ] + }, + { + "selector": "drr", + "contentionWorstShareOverWeight": { + "mean": 0.9783183500793231, + "min": 0.967741935483871, + "max": 0.9836065573770492 + }, + "contentionJain": { + "mean": 0.9973800577665314, + "min": 0.994824016563147, + "max": 0.9986580783682237 + }, + "worstWaitP99": { + "mean": 1848.3333333333333, + "min": 1626, + "max": 2051 + }, + "selectionRounds": { + "mean": 649.6666666666666, + "min": 639, + "max": 657 + }, + "wallClockMs": { + "mean": 4385.333333333333, + "min": 4233, + "max": 4534 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 250, + "weight": 1, + "share": 0.7142857142857143, + "contentionShareOverWeight": 1.1612903225806452, + "meanWait": 1322.768, + "waitP50": 1426, + "waitP99": 2051, + "waitMax": 2056 + }, + { + "groupId": "light-1", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.967741935483871, + "meanWait": 299.5, + "waitP50": 221, + "waitP99": 692, + "waitMax": 692 + }, + { + "groupId": "light-2", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.967741935483871, + "meanWait": 303.75, + "waitP50": 224, + "waitP99": 692, + "waitMax": 692 + }, + { + "groupId": "light-3", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.967741935483871, + "meanWait": 311.3, + "waitP50": 232, + "waitP99": 702, + "waitMax": 702 + }, + { + "groupId": "light-4", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.967741935483871, + "meanWait": 377.35, + "waitP50": 356, + "waitP99": 724, + "waitMax": 724 + }, + { + "groupId": "light-5", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.967741935483871, + "meanWait": 322.55, + "waitP50": 238, + "waitP99": 696, + "waitMax": 696 + } + ] + }, + { + "selector": "stride", + "contentionWorstShareOverWeight": { + "mean": 1, + "min": 1, + "max": 1 + }, + "contentionJain": { + "mean": 1, + "min": 1, + "max": 1 + }, + "worstWaitP99": { + "mean": 1850.6666666666667, + "min": 1632, + "max": 2050 + }, + "selectionRounds": { + "mean": 646, + "min": 636, + "max": 653 + }, + "wallClockMs": { + "mean": 4208, + "min": 4118, + "max": 4357 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 250, + "weight": 1, + "share": 0.7142857142857143, + "contentionShareOverWeight": 1, + "meanWait": 1323.616, + "waitP50": 1426, + "waitP99": 2050, + "waitMax": 2051 + }, + { + "groupId": "light-1", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 308.2, + "waitP50": 237, + "waitP99": 689, + "waitMax": 689 + }, + { + "groupId": "light-2", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 312.15, + "waitP50": 238, + "waitP99": 695, + "waitMax": 695 + }, + { + "groupId": "light-3", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 318.65, + "waitP50": 242, + "waitP99": 696, + "waitMax": 696 + }, + { + "groupId": "light-4", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 325.7, + "waitP50": 249, + "waitP99": 700, + "waitMax": 700 + }, + { + "groupId": "light-5", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 332.75, + "waitP50": 252, + "waitP99": 710, + "waitMax": 710 + } + ] + }, + { + "selector": "codel-sfq", + "contentionWorstShareOverWeight": { + "mean": 1, + "min": 1, + "max": 1 + }, + "contentionJain": { + "mean": 1, + "min": 1, + "max": 1 + }, + "worstWaitP99": { + "mean": 1850.6666666666667, + "min": 1632, + "max": 2050 + }, + "selectionRounds": { + "mean": 646, + "min": 636, + "max": 653 + }, + "wallClockMs": { + "mean": 4296.666666666667, + "min": 3828, + "max": 4587 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 250, + "weight": 1, + "share": 0.7142857142857143, + "contentionShareOverWeight": 1, + "meanWait": 1323.616, + "waitP50": 1426, + "waitP99": 2050, + "waitMax": 2051 + }, + { + "groupId": "light-1", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 308.2, + "waitP50": 237, + "waitP99": 689, + "waitMax": 689 + }, + { + "groupId": "light-2", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 312.15, + "waitP50": 238, + "waitP99": 695, + "waitMax": 695 + }, + { + "groupId": "light-3", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 318.65, + "waitP50": 242, + "waitP99": 696, + "waitMax": 696 + }, + { + "groupId": "light-4", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 325.7, + "waitP50": 249, + "waitP99": 700, + "waitMax": 700 + }, + { + "groupId": "light-5", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 1, + "meanWait": 332.75, + "waitP50": 252, + "waitP99": 710, + "waitMax": 710 + } + ] + }, + { + "selector": "codel-baseline", + "contentionWorstShareOverWeight": { + "mean": 0.288387506534917, + "min": 0.2608695652173913, + "max": 0.3103448275862069 + }, + "contentionJain": { + "mean": 0.3111216639468977, + "min": 0.3083948698017876, + "max": 0.3132993915311066 + }, + "worstWaitP99": { + "mean": 1836, + "min": 1653, + "max": 1989 + }, + "selectionRounds": { + "mean": 646, + "min": 638, + "max": 658 + }, + "wallClockMs": { + "mean": 5449.333333333333, + "min": 5333, + "max": 5508 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 250, + "weight": 1, + "share": 0.7142857142857143, + "contentionShareOverWeight": 4.310344827586207, + "meanWait": 805.244, + "waitP50": 805, + "waitP99": 1721, + "waitMax": 1756 + }, + { + "groupId": "light-1", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.3448275862068966, + "meanWait": 1437.9, + "waitP50": 1540, + "waitP99": 1948, + "waitMax": 1948 + }, + { + "groupId": "light-2", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.3448275862068966, + "meanWait": 1158.3, + "waitP50": 1310, + "waitP99": 1883, + "waitMax": 1883 + }, + { + "groupId": "light-3", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.3103448275862069, + "meanWait": 1498.35, + "waitP50": 1518, + "waitP99": 1989, + "waitMax": 1989 + }, + { + "groupId": "light-4", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.3448275862068966, + "meanWait": 1287.2, + "waitP50": 1268, + "waitP99": 1983, + "waitMax": 1983 + }, + { + "groupId": "light-5", + "dequeued": 20, + "weight": 1, + "share": 0.05714285714285714, + "contentionShareOverWeight": 0.3448275862068966, + "meanWait": 1518.5, + "waitP50": 1715, + "waitP99": 1959, + "waitMax": 1959 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/results/balanced.json b/internal-packages/run-engine/src/run-queue/fairness-spike/results/balanced.json new file mode 100644 index 0000000000..bf239f0efd --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/results/balanced.json @@ -0,0 +1,460 @@ +{ + "scenario": "balanced", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "weights": { + "t-a": 1, + "t-b": 1, + "t-c": 1, + "t-d": 1 + }, + "perSelector": [ + { + "selector": "baseline", + "contentionWorstShareOverWeight": { + "mean": 0.8890945931344537, + "min": 0.7741935483870968, + "max": 0.9543147208121827 + }, + "contentionJain": { + "mean": 0.9937815689184859, + "min": 0.9832878581173262, + "max": 0.9993047687712433 + }, + "worstWaitP99": { + "mean": 1201.3333333333333, + "min": 1126, + "max": 1283 + }, + "selectionRounds": { + "mean": 370.3333333333333, + "min": 365, + "max": 377 + }, + "wallClockMs": { + "mean": 930.3333333333334, + "min": 783, + "max": 1196 + }, + "detailSeed0": [ + { + "groupId": "t-a", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.015228426395939, + "meanWait": 621.4, + "waitP50": 552, + "waitP99": 1252, + "waitMax": 1252 + }, + { + "groupId": "t-b", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.015228426395939, + "meanWait": 697.3, + "waitP50": 657, + "waitP99": 1271, + "waitMax": 1271 + }, + { + "groupId": "t-c", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.015228426395939, + "meanWait": 439.86, + "waitP50": 401, + "waitP99": 939, + "waitMax": 939 + }, + { + "groupId": "t-d", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9543147208121827, + "meanWait": 833.88, + "waitP50": 959, + "waitP99": 1283, + "waitMax": 1283 + } + ] + }, + { + "selector": "sfq", + "contentionWorstShareOverWeight": { + "mean": 0.9849246231155778, + "min": 0.9849246231155779, + "max": 0.9849246231155779 + }, + "contentionJain": { + "mean": 0.9999242500757499, + "min": 0.9999242500757499, + "max": 0.9999242500757499 + }, + "worstWaitP99": { + "mean": 1193.3333333333333, + "min": 1109, + "max": 1271 + }, + "selectionRounds": { + "mean": 366.3333333333333, + "min": 357, + "max": 373 + }, + "wallClockMs": { + "mean": 734.3333333333334, + "min": 576, + "max": 874 + }, + "detailSeed0": [ + { + "groupId": "t-a", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0050251256281406, + "meanWait": 633.02, + "waitP50": 620, + "waitP99": 1270, + "waitMax": 1270 + }, + { + "groupId": "t-b", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0050251256281406, + "meanWait": 639.08, + "waitP50": 626, + "waitP99": 1270, + "waitMax": 1270 + }, + { + "groupId": "t-c", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0050251256281406, + "meanWait": 646.06, + "waitP50": 629, + "waitP99": 1271, + "waitMax": 1271 + }, + { + "groupId": "t-d", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9849246231155779, + "meanWait": 652.42, + "waitP50": 634, + "waitP99": 1271, + "waitMax": 1271 + } + ] + }, + { + "selector": "drr", + "contentionWorstShareOverWeight": { + "mean": 0.9541569541569542, + "min": 0.9230769230769231, + "max": 0.9696969696969697 + }, + "contentionJain": { + "mean": 0.9991398336529844, + "min": 0.9980314960629924, + "max": 0.9996940024479803 + }, + "worstWaitP99": { + "mean": 1199.6666666666667, + "min": 1120, + "max": 1279 + }, + "selectionRounds": { + "mean": 364.6666666666667, + "min": 359, + "max": 368 + }, + "wallClockMs": { + "mean": 772, + "min": 550, + "max": 925 + }, + "detailSeed0": [ + { + "groupId": "t-a", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0101010101010102, + "meanWait": 589.54, + "waitP50": 587, + "waitP99": 1188, + "waitMax": 1188 + }, + { + "groupId": "t-b", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0101010101010102, + "meanWait": 657.34, + "waitP50": 629, + "waitP99": 1276, + "waitMax": 1276 + }, + { + "groupId": "t-c", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0101010101010102, + "meanWait": 653.76, + "waitP50": 651, + "waitP99": 1235, + "waitMax": 1235 + }, + { + "groupId": "t-d", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9696969696969697, + "meanWait": 683.34, + "waitP50": 652, + "waitP99": 1279, + "waitMax": 1279 + } + ] + }, + { + "selector": "stride", + "contentionWorstShareOverWeight": { + "mean": 0.9849246231155778, + "min": 0.9849246231155779, + "max": 0.9849246231155779 + }, + "contentionJain": { + "mean": 0.9999242500757499, + "min": 0.9999242500757499, + "max": 0.9999242500757499 + }, + "worstWaitP99": { + "mean": 1193.3333333333333, + "min": 1109, + "max": 1271 + }, + "selectionRounds": { + "mean": 366.3333333333333, + "min": 357, + "max": 373 + }, + "wallClockMs": { + "mean": 757, + "min": 721, + "max": 819 + }, + "detailSeed0": [ + { + "groupId": "t-a", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0050251256281406, + "meanWait": 633.02, + "waitP50": 620, + "waitP99": 1270, + "waitMax": 1270 + }, + { + "groupId": "t-b", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0050251256281406, + "meanWait": 639.08, + "waitP50": 626, + "waitP99": 1270, + "waitMax": 1270 + }, + { + "groupId": "t-c", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0050251256281406, + "meanWait": 646.06, + "waitP50": 629, + "waitP99": 1271, + "waitMax": 1271 + }, + { + "groupId": "t-d", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9849246231155779, + "meanWait": 652.42, + "waitP50": 634, + "waitP99": 1271, + "waitMax": 1271 + } + ] + }, + { + "selector": "codel-sfq", + "contentionWorstShareOverWeight": { + "mean": 0.9849246231155778, + "min": 0.9849246231155779, + "max": 0.9849246231155779 + }, + "contentionJain": { + "mean": 0.9999242500757499, + "min": 0.9999242500757499, + "max": 0.9999242500757499 + }, + "worstWaitP99": { + "mean": 1193.3333333333333, + "min": 1109, + "max": 1271 + }, + "selectionRounds": { + "mean": 366.3333333333333, + "min": 357, + "max": 373 + }, + "wallClockMs": { + "mean": 807.6666666666666, + "min": 721, + "max": 871 + }, + "detailSeed0": [ + { + "groupId": "t-a", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0050251256281406, + "meanWait": 633.02, + "waitP50": 620, + "waitP99": 1270, + "waitMax": 1270 + }, + { + "groupId": "t-b", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0050251256281406, + "meanWait": 639.08, + "waitP50": 626, + "waitP99": 1270, + "waitMax": 1270 + }, + { + "groupId": "t-c", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0050251256281406, + "meanWait": 646.06, + "waitP50": 629, + "waitP99": 1271, + "waitMax": 1271 + }, + { + "groupId": "t-d", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9849246231155779, + "meanWait": 652.42, + "waitP50": 634, + "waitP99": 1271, + "waitMax": 1271 + } + ] + }, + { + "selector": "codel-baseline", + "contentionWorstShareOverWeight": { + "mean": 0.8890945931344537, + "min": 0.7741935483870968, + "max": 0.9543147208121827 + }, + "contentionJain": { + "mean": 0.9937815689184859, + "min": 0.9832878581173262, + "max": 0.9993047687712433 + }, + "worstWaitP99": { + "mean": 1201.3333333333333, + "min": 1126, + "max": 1283 + }, + "selectionRounds": { + "mean": 370.3333333333333, + "min": 365, + "max": 377 + }, + "wallClockMs": { + "mean": 883, + "min": 793, + "max": 1031 + }, + "detailSeed0": [ + { + "groupId": "t-a", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.015228426395939, + "meanWait": 621.4, + "waitP50": 552, + "waitP99": 1252, + "waitMax": 1252 + }, + { + "groupId": "t-b", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.015228426395939, + "meanWait": 697.3, + "waitP50": 657, + "waitP99": 1271, + "waitMax": 1271 + }, + { + "groupId": "t-c", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.015228426395939, + "meanWait": 439.86, + "waitP50": 401, + "waitP99": 939, + "waitMax": 939 + }, + { + "groupId": "t-d", + "dequeued": 50, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9543147208121827, + "meanWait": 833.88, + "waitP50": 959, + "waitP99": 1283, + "waitMax": 1283 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/results/burst.json b/internal-packages/run-engine/src/run-queue/fairness-spike/results/burst.json new file mode 100644 index 0000000000..6dcf98c022 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/results/burst.json @@ -0,0 +1,594 @@ +{ + "scenario": "burst", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "weights": { + "burst-0": 1, + "burst-1": 1, + "burst-2": 1, + "burst-3": 1, + "burst-4": 1, + "burst-5": 1 + }, + "perSelector": [ + { + "selector": "baseline", + "contentionWorstShareOverWeight": { + "mean": 0.9776566931568053, + "min": 0.9664429530201342, + "max": 0.991652754590985 + }, + "contentionJain": { + "mean": 0.9998782190081853, + "min": 0.99977483563001, + "max": 0.9999860648930061 + }, + "worstWaitP99": { + "mean": 2073, + "min": 1919, + "max": 2192 + }, + "selectionRounds": { + "mean": 1070, + "min": 1057, + "max": 1082 + }, + "wallClockMs": { + "mean": 2796.6666666666665, + "min": 2299, + "max": 3309 + }, + "detailSeed0": [ + { + "groupId": "burst-0", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.0067114093959733, + "meanWait": 1120.02, + "waitP50": 1164, + "waitP99": 1944, + "waitMax": 1953 + }, + { + "groupId": "burst-1", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.0067114093959733, + "meanWait": 1137.44, + "waitP50": 1100, + "waitP99": 2144, + "waitMax": 2145 + }, + { + "groupId": "burst-2", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.0067114093959733, + "meanWait": 1194.17, + "waitP50": 1218, + "waitP99": 2165, + "waitMax": 2186 + }, + { + "groupId": "burst-3", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.0067114093959733, + "meanWait": 1026.64, + "waitP50": 1059, + "waitP99": 2143, + "waitMax": 2147 + }, + { + "groupId": "burst-4", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 0.9664429530201342, + "meanWait": 1137.04, + "waitP50": 1129, + "waitP99": 2192, + "waitMax": 2192 + }, + { + "groupId": "burst-5", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.0067114093959733, + "meanWait": 1191.08, + "waitP50": 1251, + "waitP99": 2161, + "waitMax": 2164 + } + ] + }, + { + "selector": "sfq", + "contentionWorstShareOverWeight": { + "mean": 0.991652754590985, + "min": 0.991652754590985, + "max": 0.991652754590985 + }, + "contentionJain": { + "mean": 0.999986064893006, + "min": 0.9999860648930061, + "max": 0.9999860648930061 + }, + "worstWaitP99": { + "mean": 2050.3333333333335, + "min": 1897, + "max": 2163 + }, + "selectionRounds": { + "mean": 1065, + "min": 1037, + "max": 1091 + }, + "wallClockMs": { + "mean": 2479.6666666666665, + "min": 2198, + "max": 2807 + }, + "detailSeed0": [ + { + "groupId": "burst-0", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1126.45, + "waitP50": 1147, + "waitP99": 2143, + "waitMax": 2175 + }, + { + "groupId": "burst-1", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1130.07, + "waitP50": 1152, + "waitP99": 2149, + "waitMax": 2175 + }, + { + "groupId": "burst-2", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1133.47, + "waitP50": 1154, + "waitP99": 2154, + "waitMax": 2179 + }, + { + "groupId": "burst-3", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1137.05, + "waitP50": 1156, + "waitP99": 2156, + "waitMax": 2182 + }, + { + "groupId": "burst-4", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1140.28, + "waitP50": 1158, + "waitP99": 2157, + "waitMax": 2183 + }, + { + "groupId": "burst-5", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 0.991652754590985, + "meanWait": 1144.22, + "waitP50": 1164, + "waitP99": 2163, + "waitMax": 2187 + } + ] + }, + { + "selector": "drr", + "contentionWorstShareOverWeight": { + "mean": 0.9579452142360924, + "min": 0.9409780775716695, + "max": 0.9748743718592965 + }, + "contentionJain": { + "mean": 0.9996081887757097, + "min": 0.9993037676118377, + "max": 0.9998737565015399 + }, + "worstWaitP99": { + "mean": 2081.3333333333335, + "min": 1926, + "max": 2208 + }, + "selectionRounds": { + "mean": 1062.3333333333333, + "min": 1050, + "max": 1079 + }, + "wallClockMs": { + "mean": 2655, + "min": 2489, + "max": 2858 + }, + "detailSeed0": [ + { + "groupId": "burst-0", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.008403361344538, + "meanWait": 1154.38, + "waitP50": 1183, + "waitP99": 2177, + "waitMax": 2188 + }, + { + "groupId": "burst-1", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.008403361344538, + "meanWait": 1136.48, + "waitP50": 1154, + "waitP99": 2188, + "waitMax": 2201 + }, + { + "groupId": "burst-2", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.008403361344538, + "meanWait": 1094.09, + "waitP50": 1133, + "waitP99": 2054, + "waitMax": 2075 + }, + { + "groupId": "burst-3", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 0.9579831932773109, + "meanWait": 1154.97, + "waitP50": 1160, + "waitP99": 2208, + "waitMax": 2209 + }, + { + "groupId": "burst-4", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.008403361344538, + "meanWait": 1162.51, + "waitP50": 1214, + "waitP99": 2189, + "waitMax": 2200 + }, + { + "groupId": "burst-5", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.008403361344538, + "meanWait": 1124.42, + "waitP50": 1148, + "waitP99": 2145, + "waitMax": 2176 + } + ] + }, + { + "selector": "stride", + "contentionWorstShareOverWeight": { + "mean": 0.991652754590985, + "min": 0.991652754590985, + "max": 0.991652754590985 + }, + "contentionJain": { + "mean": 0.999986064893006, + "min": 0.9999860648930061, + "max": 0.9999860648930061 + }, + "worstWaitP99": { + "mean": 2050.3333333333335, + "min": 1897, + "max": 2163 + }, + "selectionRounds": { + "mean": 1065, + "min": 1037, + "max": 1091 + }, + "wallClockMs": { + "mean": 2575.6666666666665, + "min": 2489, + "max": 2699 + }, + "detailSeed0": [ + { + "groupId": "burst-0", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1126.45, + "waitP50": 1147, + "waitP99": 2143, + "waitMax": 2175 + }, + { + "groupId": "burst-1", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1130.07, + "waitP50": 1152, + "waitP99": 2149, + "waitMax": 2175 + }, + { + "groupId": "burst-2", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1133.47, + "waitP50": 1154, + "waitP99": 2154, + "waitMax": 2179 + }, + { + "groupId": "burst-3", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1137.05, + "waitP50": 1156, + "waitP99": 2156, + "waitMax": 2182 + }, + { + "groupId": "burst-4", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1140.28, + "waitP50": 1158, + "waitP99": 2157, + "waitMax": 2183 + }, + { + "groupId": "burst-5", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 0.991652754590985, + "meanWait": 1144.22, + "waitP50": 1164, + "waitP99": 2163, + "waitMax": 2187 + } + ] + }, + { + "selector": "codel-sfq", + "contentionWorstShareOverWeight": { + "mean": 0.991652754590985, + "min": 0.991652754590985, + "max": 0.991652754590985 + }, + "contentionJain": { + "mean": 0.999986064893006, + "min": 0.9999860648930061, + "max": 0.9999860648930061 + }, + "worstWaitP99": { + "mean": 2050.3333333333335, + "min": 1897, + "max": 2163 + }, + "selectionRounds": { + "mean": 1065, + "min": 1037, + "max": 1091 + }, + "wallClockMs": { + "mean": 2794.6666666666665, + "min": 2537, + "max": 3169 + }, + "detailSeed0": [ + { + "groupId": "burst-0", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1126.45, + "waitP50": 1147, + "waitP99": 2143, + "waitMax": 2175 + }, + { + "groupId": "burst-1", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1130.07, + "waitP50": 1152, + "waitP99": 2149, + "waitMax": 2175 + }, + { + "groupId": "burst-2", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1133.47, + "waitP50": 1154, + "waitP99": 2154, + "waitMax": 2179 + }, + { + "groupId": "burst-3", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1137.05, + "waitP50": 1156, + "waitP99": 2156, + "waitMax": 2182 + }, + { + "groupId": "burst-4", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.001669449081803, + "meanWait": 1140.28, + "waitP50": 1158, + "waitP99": 2157, + "waitMax": 2183 + }, + { + "groupId": "burst-5", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 0.991652754590985, + "meanWait": 1144.22, + "waitP50": 1164, + "waitP99": 2163, + "waitMax": 2187 + } + ] + }, + { + "selector": "codel-baseline", + "contentionWorstShareOverWeight": { + "mean": 0.9776566931568053, + "min": 0.9664429530201342, + "max": 0.991652754590985 + }, + "contentionJain": { + "mean": 0.9998782190081853, + "min": 0.99977483563001, + "max": 0.9999860648930061 + }, + "worstWaitP99": { + "mean": 2073, + "min": 1919, + "max": 2192 + }, + "selectionRounds": { + "mean": 1070, + "min": 1057, + "max": 1082 + }, + "wallClockMs": { + "mean": 3488.6666666666665, + "min": 3283, + "max": 3873 + }, + "detailSeed0": [ + { + "groupId": "burst-0", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.0067114093959733, + "meanWait": 1120.02, + "waitP50": 1164, + "waitP99": 1944, + "waitMax": 1953 + }, + { + "groupId": "burst-1", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.0067114093959733, + "meanWait": 1137.44, + "waitP50": 1100, + "waitP99": 2144, + "waitMax": 2145 + }, + { + "groupId": "burst-2", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.0067114093959733, + "meanWait": 1194.17, + "waitP50": 1218, + "waitP99": 2165, + "waitMax": 2186 + }, + { + "groupId": "burst-3", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.0067114093959733, + "meanWait": 1026.64, + "waitP50": 1059, + "waitP99": 2143, + "waitMax": 2147 + }, + { + "groupId": "burst-4", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 0.9664429530201342, + "meanWait": 1137.04, + "waitP50": 1129, + "waitP99": 2192, + "waitMax": 2192 + }, + { + "groupId": "burst-5", + "dequeued": 100, + "weight": 1, + "share": 0.16666666666666666, + "contentionShareOverWeight": 1.0067114093959733, + "meanWait": 1191.08, + "waitP50": 1251, + "waitP99": 2161, + "waitMax": 2164 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/results/longHold.json b/internal-packages/run-engine/src/run-queue/fairness-spike/results/longHold.json new file mode 100644 index 0000000000..7f26696d35 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/results/longHold.json @@ -0,0 +1,460 @@ +{ + "scenario": "longHold", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "weights": { + "slow-1": 1, + "slow-2": 1, + "fast-1": 1, + "fast-2": 1 + }, + "perSelector": [ + { + "selector": "baseline", + "contentionWorstShareOverWeight": { + "mean": 0.8280701754385964, + "min": 0.8, + "max": 0.8421052631578947 + }, + "contentionJain": { + "mean": 0.9901195295932138, + "min": 0.9868421052631579, + "max": 0.9917582417582418 + }, + "worstWaitP99": { + "mean": 9922.666666666666, + "min": 9031, + "max": 11179 + }, + "selectionRounds": { + "mean": 311.3333333333333, + "min": 310, + "max": 312 + }, + "wallClockMs": { + "mean": 732, + "min": 641, + "max": 844 + }, + "detailSeed0": [ + { + "groupId": "slow-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0526315789473684, + "meanWait": 3877.25, + "waitP50": 3748, + "waitP99": 8561, + "waitMax": 8561 + }, + { + "groupId": "slow-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.8421052631578947, + "meanWait": 7305.175, + "waitP50": 8408, + "waitP99": 11179, + "waitMax": 11179 + }, + { + "groupId": "fast-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0526315789473684, + "meanWait": 6438.525, + "waitP50": 7955, + "waitP99": 10208, + "waitMax": 10208 + }, + { + "groupId": "fast-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0526315789473684, + "meanWait": 5684.9, + "waitP50": 6095, + "waitP99": 9301, + "waitMax": 9301 + } + ] + }, + { + "selector": "sfq", + "contentionWorstShareOverWeight": { + "mean": 0.9811320754716982, + "min": 0.9811320754716981, + "max": 0.9811320754716981 + }, + "contentionJain": { + "mean": 0.9998813478879921, + "min": 0.9998813478879922, + "max": 0.9998813478879922 + }, + "worstWaitP99": { + "mean": 10007, + "min": 9250, + "max": 11211 + }, + "selectionRounds": { + "mean": 312.6666666666667, + "min": 312, + "max": 314 + }, + "wallClockMs": { + "mean": 610, + "min": 532, + "max": 656 + }, + "detailSeed0": [ + { + "groupId": "slow-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 5299.775, + "waitP50": 5560, + "waitP99": 11191, + "waitMax": 11191 + }, + { + "groupId": "slow-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9811320754716981, + "meanWait": 5407.45, + "waitP50": 5674, + "waitP99": 11211, + "waitMax": 11211 + }, + { + "groupId": "fast-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 5264.575, + "waitP50": 5537, + "waitP99": 11162, + "waitMax": 11162 + }, + { + "groupId": "fast-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 5283.025, + "waitP50": 5549, + "waitP99": 11170, + "waitMax": 11170 + } + ] + }, + { + "selector": "drr", + "contentionWorstShareOverWeight": { + "mean": 0.9811320754716982, + "min": 0.9811320754716981, + "max": 0.9811320754716981 + }, + "contentionJain": { + "mean": 0.9998813478879921, + "min": 0.9998813478879922, + "max": 0.9998813478879922 + }, + "worstWaitP99": { + "mean": 9975.666666666666, + "min": 9256, + "max": 11091 + }, + "selectionRounds": { + "mean": 313.3333333333333, + "min": 312, + "max": 314 + }, + "wallClockMs": { + "mean": 541.6666666666666, + "min": 467, + "max": 672 + }, + "detailSeed0": [ + { + "groupId": "slow-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9811320754716981, + "meanWait": 5330.75, + "waitP50": 5610, + "waitP99": 11091, + "waitMax": 11091 + }, + { + "groupId": "slow-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 5363.4, + "waitP50": 5644, + "waitP99": 10934, + "waitMax": 10934 + }, + { + "groupId": "fast-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 4963.775, + "waitP50": 5313, + "waitP99": 10609, + "waitMax": 10609 + }, + { + "groupId": "fast-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 5262.2, + "waitP50": 5599, + "waitP99": 11070, + "waitMax": 11070 + } + ] + }, + { + "selector": "stride", + "contentionWorstShareOverWeight": { + "mean": 0.9811320754716982, + "min": 0.9811320754716981, + "max": 0.9811320754716981 + }, + "contentionJain": { + "mean": 0.9998813478879921, + "min": 0.9998813478879922, + "max": 0.9998813478879922 + }, + "worstWaitP99": { + "mean": 10007, + "min": 9250, + "max": 11211 + }, + "selectionRounds": { + "mean": 312.6666666666667, + "min": 312, + "max": 314 + }, + "wallClockMs": { + "mean": 490, + "min": 407, + "max": 575 + }, + "detailSeed0": [ + { + "groupId": "slow-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 5299.775, + "waitP50": 5560, + "waitP99": 11191, + "waitMax": 11191 + }, + { + "groupId": "slow-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9811320754716981, + "meanWait": 5407.45, + "waitP50": 5674, + "waitP99": 11211, + "waitMax": 11211 + }, + { + "groupId": "fast-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 5264.575, + "waitP50": 5537, + "waitP99": 11162, + "waitMax": 11162 + }, + { + "groupId": "fast-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 5283.025, + "waitP50": 5549, + "waitP99": 11170, + "waitMax": 11170 + } + ] + }, + { + "selector": "codel-sfq", + "contentionWorstShareOverWeight": { + "mean": 0.9811320754716982, + "min": 0.9811320754716981, + "max": 0.9811320754716981 + }, + "contentionJain": { + "mean": 0.9998813478879921, + "min": 0.9998813478879922, + "max": 0.9998813478879922 + }, + "worstWaitP99": { + "mean": 10007, + "min": 9250, + "max": 11211 + }, + "selectionRounds": { + "mean": 312.6666666666667, + "min": 312, + "max": 314 + }, + "wallClockMs": { + "mean": 573, + "min": 437, + "max": 646 + }, + "detailSeed0": [ + { + "groupId": "slow-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 5299.775, + "waitP50": 5560, + "waitP99": 11191, + "waitMax": 11191 + }, + { + "groupId": "slow-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.9811320754716981, + "meanWait": 5407.45, + "waitP50": 5674, + "waitP99": 11211, + "waitMax": 11211 + }, + { + "groupId": "fast-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 5264.575, + "waitP50": 5537, + "waitP99": 11162, + "waitMax": 11162 + }, + { + "groupId": "fast-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0062893081761006, + "meanWait": 5283.025, + "waitP50": 5549, + "waitP99": 11170, + "waitMax": 11170 + } + ] + }, + { + "selector": "codel-baseline", + "contentionWorstShareOverWeight": { + "mean": 0.8280701754385964, + "min": 0.8, + "max": 0.8421052631578947 + }, + "contentionJain": { + "mean": 0.9901195295932138, + "min": 0.9868421052631579, + "max": 0.9917582417582418 + }, + "worstWaitP99": { + "mean": 9922.666666666666, + "min": 9031, + "max": 11179 + }, + "selectionRounds": { + "mean": 311.3333333333333, + "min": 310, + "max": 312 + }, + "wallClockMs": { + "mean": 795.6666666666666, + "min": 679, + "max": 1008 + }, + "detailSeed0": [ + { + "groupId": "slow-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0526315789473684, + "meanWait": 3877.25, + "waitP50": 3748, + "waitP99": 8561, + "waitMax": 8561 + }, + { + "groupId": "slow-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 0.8421052631578947, + "meanWait": 7305.175, + "waitP50": 8408, + "waitP99": 11179, + "waitMax": 11179 + }, + { + "groupId": "fast-1", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0526315789473684, + "meanWait": 6438.525, + "waitP50": 7955, + "waitP99": 10208, + "waitMax": 10208 + }, + { + "groupId": "fast-2", + "dequeued": 40, + "weight": 1, + "share": 0.25, + "contentionShareOverWeight": 1.0526315789473684, + "meanWait": 5684.9, + "waitP50": 6095, + "waitP99": 9301, + "waitMax": 9301 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/results/trickleStale.json b/internal-packages/run-engine/src/run-queue/fairness-spike/results/trickleStale.json new file mode 100644 index 0000000000..2e40ac43dc --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/results/trickleStale.json @@ -0,0 +1,393 @@ +{ + "scenario": "trickleStale", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "weights": { + "heavy": 1, + "trickle-1": 1, + "trickle-2": 1 + }, + "perSelector": [ + { + "selector": "baseline", + "contentionWorstShareOverWeight": { + "mean": 0.20846388554312076, + "min": 0.17948717948717952, + "max": 0.2346368715083799 + }, + "contentionJain": { + "mean": 0.4581819362853548, + "min": 0.44960094590600047, + "max": 0.4659627997614996 + }, + "worstWaitP99": { + "mean": 2180.3333333333335, + "min": 2018, + "max": 2396 + }, + "selectionRounds": { + "mean": 714.3333333333334, + "min": 707, + "max": 721 + }, + "wallClockMs": { + "mean": 4240, + "min": 4124, + "max": 4362 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 300, + "weight": 1, + "share": 0.8333333333333334, + "contentionShareOverWeight": 2.535211267605634, + "meanWait": 1234.28, + "waitP50": 1308, + "waitP99": 2396, + "waitMax": 2428 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.2535211267605634, + "meanWait": 1208.8, + "waitP50": 1421, + "waitP99": 1680, + "waitMax": 1680 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.21126760563380284, + "meanWait": 1509.1333333333334, + "waitP50": 1680, + "waitP99": 1845, + "waitMax": 1845 + } + ] + }, + { + "selector": "sfq", + "contentionWorstShareOverWeight": { + "mean": 0.8043668869356942, + "min": 0.7692307692307692, + "max": 0.8256880733944955 + }, + "contentionJain": { + "mean": 0.9281466214393069, + "min": 0.9037433155080212, + "max": 0.9427120526858683 + }, + "worstWaitP99": { + "mean": 2330.6666666666665, + "min": 2125, + "max": 2554 + }, + "selectionRounds": { + "mean": 726.6666666666666, + "min": 712, + "max": 735 + }, + "wallClockMs": { + "mean": 2706.3333333333335, + "min": 2513, + "max": 2982 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 300, + "weight": 1, + "share": 0.8333333333333334, + "contentionShareOverWeight": 1.3636363636363638, + "meanWait": 1353.3966666666668, + "waitP50": 1384, + "waitP99": 2554, + "waitMax": 2584 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.8181818181818181, + "meanWait": 15.966666666666667, + "waitP50": 14, + "waitP99": 54, + "waitMax": 54 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.8181818181818181, + "meanWait": 22.966666666666665, + "waitP50": 17, + "waitP99": 63, + "waitMax": 63 + } + ] + }, + { + "selector": "drr", + "contentionWorstShareOverWeight": { + "mean": 0.7759005112828201, + "min": 0.7692307692307692, + "max": 0.782608695652174 + }, + "contentionJain": { + "mean": 0.9086951769169557, + "min": 0.9037433155080212, + "max": 0.9136442141623488 + }, + "worstWaitP99": { + "mean": 2333, + "min": 2126, + "max": 2559 + }, + "selectionRounds": { + "mean": 729.3333333333334, + "min": 726, + "max": 732 + }, + "wallClockMs": { + "mean": 3059.3333333333335, + "min": 2720, + "max": 3299 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 300, + "weight": 1, + "share": 0.8333333333333334, + "contentionShareOverWeight": 1.4482758620689657, + "meanWait": 1353.3633333333332, + "waitP50": 1383, + "waitP99": 2559, + "waitMax": 2586 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.7758620689655173, + "meanWait": 15.766666666666667, + "waitP50": 14, + "waitP99": 46, + "waitMax": 46 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.7758620689655173, + "meanWait": 28.833333333333332, + "waitP50": 18, + "waitP99": 78, + "waitMax": 78 + } + ] + }, + { + "selector": "stride", + "contentionWorstShareOverWeight": { + "mean": 0.8043668869356942, + "min": 0.7692307692307692, + "max": 0.8256880733944955 + }, + "contentionJain": { + "mean": 0.9281466214393069, + "min": 0.9037433155080212, + "max": 0.9427120526858683 + }, + "worstWaitP99": { + "mean": 2330.6666666666665, + "min": 2125, + "max": 2554 + }, + "selectionRounds": { + "mean": 726.6666666666666, + "min": 712, + "max": 735 + }, + "wallClockMs": { + "mean": 2956, + "min": 2854, + "max": 3015 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 300, + "weight": 1, + "share": 0.8333333333333334, + "contentionShareOverWeight": 1.3636363636363638, + "meanWait": 1353.3966666666668, + "waitP50": 1384, + "waitP99": 2554, + "waitMax": 2584 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.8181818181818181, + "meanWait": 15.966666666666667, + "waitP50": 14, + "waitP99": 54, + "waitMax": 54 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.8181818181818181, + "meanWait": 22.966666666666665, + "waitP50": 17, + "waitP99": 63, + "waitMax": 63 + } + ] + }, + { + "selector": "codel-sfq", + "contentionWorstShareOverWeight": { + "mean": 0.365699086718357, + "min": 0.3202846975088968, + "max": 0.410958904109589 + }, + "contentionJain": { + "mean": 0.554773458150014, + "min": 0.5197435543005339, + "max": 0.5903400908385951 + }, + "worstWaitP99": { + "mean": 2333.3333333333335, + "min": 2127, + "max": 2557 + }, + "selectionRounds": { + "mean": 724, + "min": 720, + "max": 727 + }, + "wallClockMs": { + "mean": 3042, + "min": 2990, + "max": 3069 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 300, + "weight": 1, + "share": 0.8333333333333334, + "contentionShareOverWeight": 2.178082191780822, + "meanWait": 1314.9666666666667, + "waitP50": 1326, + "waitP99": 2557, + "waitMax": 2585 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.410958904109589, + "meanWait": 227.5, + "waitP50": 296, + "waitP99": 323, + "waitMax": 323 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.410958904109589, + "meanWait": 198.76666666666668, + "waitP50": 244, + "waitP99": 353, + "waitMax": 353 + } + ] + }, + { + "selector": "codel-baseline", + "contentionWorstShareOverWeight": { + "mean": 0.1951842074747103, + "min": 0.17142857142857143, + "max": 0.2346368715083799 + }, + "contentionJain": { + "mean": 0.4542690884000587, + "min": 0.447243519532676, + "max": 0.4659627997614996 + }, + "worstWaitP99": { + "mean": 2181.3333333333335, + "min": 2018, + "max": 2399 + }, + "selectionRounds": { + "mean": 718.3333333333334, + "min": 707, + "max": 733 + }, + "wallClockMs": { + "mean": 4676.666666666667, + "min": 4087, + "max": 5778 + }, + "detailSeed0": [ + { + "groupId": "heavy", + "dequeued": 300, + "weight": 1, + "share": 0.8333333333333334, + "contentionShareOverWeight": 2.5714285714285716, + "meanWait": 1208.61, + "waitP50": 1179, + "waitP99": 2399, + "waitMax": 2438 + }, + { + "groupId": "trickle-1", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.17142857142857143, + "meanWait": 1335.6666666666667, + "waitP50": 1478, + "waitP99": 1882, + "waitMax": 1882 + }, + { + "groupId": "trickle-2", + "dequeued": 30, + "weight": 1, + "share": 0.08333333333333333, + "contentionShareOverWeight": 0.2571428571428572, + "meanWait": 1445.9, + "waitP50": 1609, + "waitP99": 1726, + "waitMax": 1726 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/results/weighted.json b/internal-packages/run-engine/src/run-queue/fairness-spike/results/weighted.json new file mode 100644 index 0000000000..c57d211a02 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/results/weighted.json @@ -0,0 +1,326 @@ +{ + "scenario": "weighted", + "seeds": [ + "seed-a", + "seed-b", + "seed-c" + ], + "weights": { + "big": 3, + "small": 1 + }, + "perSelector": [ + { + "selector": "baseline", + "contentionWorstShareOverWeight": { + "mean": 0.7030070606172968, + "min": 0.6791171477079797, + "max": 0.7194244604316546 + }, + "contentionJain": { + "mean": 0.8267111619816848, + "min": 0.8090215031917964, + "max": 0.8389112476170302 + }, + "worstWaitP99": { + "mean": 4671, + "min": 4314, + "max": 4960 + }, + "selectionRounds": { + "mean": 1137.6666666666667, + "min": 1126, + "max": 1144 + }, + "wallClockMs": { + "mean": 1915.3333333333333, + "min": 1703, + "max": 2121 + }, + "detailSeed0": [ + { + "groupId": "big", + "dequeued": 300, + "weight": 3, + "share": 0.5, + "contentionShareOverWeight": 0.7104795737122558, + "meanWait": 2383.806666666667, + "waitP50": 2451, + "waitP99": 4640, + "waitMax": 4657 + }, + { + "groupId": "small", + "dequeued": 300, + "weight": 1, + "share": 0.5, + "contentionShareOverWeight": 1.8685612788632326, + "meanWait": 2712.1466666666665, + "waitP50": 2769, + "waitP99": 4960, + "waitMax": 4968 + } + ] + }, + { + "selector": "sfq", + "contentionWorstShareOverWeight": { + "mean": 1, + "min": 1, + "max": 1 + }, + "contentionJain": { + "mean": 1, + "min": 1, + "max": 1 + }, + "worstWaitP99": { + "mean": 4671.666666666667, + "min": 4312, + "max": 4959 + }, + "selectionRounds": { + "mean": 1148, + "min": 1144, + "max": 1151 + }, + "wallClockMs": { + "mean": 1418.3333333333333, + "min": 1290, + "max": 1588 + }, + "detailSeed0": [ + { + "groupId": "big", + "dequeued": 300, + "weight": 3, + "share": 0.5, + "contentionShareOverWeight": 1, + "meanWait": 1747.7433333333333, + "waitP50": 1816, + "waitP99": 3362, + "waitMax": 3389 + }, + { + "groupId": "small", + "dequeued": 300, + "weight": 1, + "share": 0.5, + "contentionShareOverWeight": 1, + "meanWait": 3366.1033333333335, + "waitP50": 3839, + "waitP99": 4959, + "waitMax": 4962 + } + ] + }, + { + "selector": "drr", + "contentionWorstShareOverWeight": { + "mean": 0.9899370592728672, + "min": 0.9773299748110831, + "max": 1 + }, + "contentionJain": { + "mean": 0.9999142991823375, + "min": 0.9997681487969498, + "max": 1 + }, + "worstWaitP99": { + "mean": 4671.333333333333, + "min": 4312, + "max": 4959 + }, + "selectionRounds": { + "mean": 1150.3333333333333, + "min": 1140, + "max": 1159 + }, + "wallClockMs": { + "mean": 1545.6666666666667, + "min": 1455, + "max": 1693 + }, + "detailSeed0": [ + { + "groupId": "big", + "dequeued": 300, + "weight": 3, + "share": 0.5, + "contentionShareOverWeight": 1.0025062656641603, + "meanWait": 1744.4666666666667, + "waitP50": 1816, + "waitP99": 3354, + "waitMax": 3371 + }, + { + "groupId": "small", + "dequeued": 300, + "weight": 1, + "share": 0.5, + "contentionShareOverWeight": 0.9924812030075187, + "meanWait": 3369.17, + "waitP50": 3838, + "waitP99": 4959, + "waitMax": 4962 + } + ] + }, + { + "selector": "stride", + "contentionWorstShareOverWeight": { + "mean": 1, + "min": 1, + "max": 1 + }, + "contentionJain": { + "mean": 1, + "min": 1, + "max": 1 + }, + "worstWaitP99": { + "mean": 4671, + "min": 4312, + "max": 4959 + }, + "selectionRounds": { + "mean": 1149, + "min": 1140, + "max": 1160 + }, + "wallClockMs": { + "mean": 1589.3333333333333, + "min": 1448, + "max": 1685 + }, + "detailSeed0": [ + { + "groupId": "big", + "dequeued": 300, + "weight": 3, + "share": 0.5, + "contentionShareOverWeight": 1, + "meanWait": 1748.03, + "waitP50": 1818, + "waitP99": 3364, + "waitMax": 3384 + }, + { + "groupId": "small", + "dequeued": 300, + "weight": 1, + "share": 0.5, + "contentionShareOverWeight": 1, + "meanWait": 3365.7633333333333, + "waitP50": 3839, + "waitP99": 4959, + "waitMax": 4962 + } + ] + }, + { + "selector": "codel-sfq", + "contentionWorstShareOverWeight": { + "mean": 1, + "min": 1, + "max": 1 + }, + "contentionJain": { + "mean": 1, + "min": 1, + "max": 1 + }, + "worstWaitP99": { + "mean": 4671.666666666667, + "min": 4312, + "max": 4959 + }, + "selectionRounds": { + "mean": 1148, + "min": 1144, + "max": 1151 + }, + "wallClockMs": { + "mean": 1599.6666666666667, + "min": 1521, + "max": 1677 + }, + "detailSeed0": [ + { + "groupId": "big", + "dequeued": 300, + "weight": 3, + "share": 0.5, + "contentionShareOverWeight": 1, + "meanWait": 1747.7433333333333, + "waitP50": 1816, + "waitP99": 3362, + "waitMax": 3389 + }, + { + "groupId": "small", + "dequeued": 300, + "weight": 1, + "share": 0.5, + "contentionShareOverWeight": 1, + "meanWait": 3366.1033333333335, + "waitP50": 3839, + "waitP99": 4959, + "waitMax": 4962 + } + ] + }, + { + "selector": "codel-baseline", + "contentionWorstShareOverWeight": { + "mean": 0.7030070606172968, + "min": 0.6791171477079797, + "max": 0.7194244604316546 + }, + "contentionJain": { + "mean": 0.8267111619816848, + "min": 0.8090215031917964, + "max": 0.8389112476170302 + }, + "worstWaitP99": { + "mean": 4671, + "min": 4314, + "max": 4960 + }, + "selectionRounds": { + "mean": 1137.6666666666667, + "min": 1126, + "max": 1144 + }, + "wallClockMs": { + "mean": 2081, + "min": 1844, + "max": 2285 + }, + "detailSeed0": [ + { + "groupId": "big", + "dequeued": 300, + "weight": 3, + "share": 0.5, + "contentionShareOverWeight": 0.7104795737122558, + "meanWait": 2383.806666666667, + "waitP50": 2451, + "waitP99": 4640, + "waitMax": 4657 + }, + { + "groupId": "small", + "dequeued": 300, + "weight": 1, + "share": 0.5, + "contentionShareOverWeight": 1.8685612788632326, + "meanWait": 2712.1466666666665, + "waitP50": 2769, + "waitP99": 4960, + "waitMax": 4968 + } + ] + } + ] +} \ No newline at end of file diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/base.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/base.ts new file mode 100644 index 0000000000..c78e7f0935 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/base.ts @@ -0,0 +1,29 @@ +import type { EnvQueues } from "../../types.js"; +import type { ActiveQueue } from "../types.js"; + +/** + * Turns a set of active queues into the `EnvQueues[]` the RunQueue expects, + * ordering queues by a discipline-supplied comparator. Ties fall back to head + * age (oldest first) then queue name for stable, deterministic output. Queues + * are grouped by environment preserving the sorted order. + */ +export function buildEnvQueues( + active: ActiveQueue[], + compare: (a: ActiveQueue, b: ActiveQueue) => number +): EnvQueues[] { + const sorted = [...active].sort( + (a, b) => + compare(a, b) || + (a.headScore ?? 0) - (b.headScore ?? 0) || + (a.queue < b.queue ? -1 : a.queue > b.queue ? 1 : 0) + ); + + const byEnv = new Map(); + for (const a of sorted) { + const arr = byEnv.get(a.env.envId) ?? []; + arr.push(a.queue); + byEnv.set(a.env.envId, arr); + } + + return [...byEnv.entries()].map(([envId, queues]) => ({ envId, queues })); +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/codelWrapper.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/codelWrapper.ts new file mode 100644 index 0000000000..df6a51d659 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/codelWrapper.ts @@ -0,0 +1,114 @@ +import type { Redis } from "@internal/redis"; +import type { EnvQueues, QueueDescriptor, RunQueueKeyProducer } from "../../types.js"; +import { + SpikeQueueReader, + groupIdFromQueueName, + type GroupId, + type SpikeSelectionStrategy, +} from "../types.js"; + +/** + * CoDel-style staleness controller. Not a selector on its own: it wraps a base + * selector and only intervenes when a group's minimum sojourn (time its oldest + * run has waited) stays above `targetMs` for a full `intervalMs`. Such groups + * are hoisted ahead of the base ordering until their sojourn recovers. This is + * the anti-staleness safety net, measuring time-in-queue rather than depth. + */ +export class CodelWrapper implements SpikeSelectionStrategy { + readonly name: string; + + private readonly base: SpikeSelectionStrategy; + private readonly reader: SpikeQueueReader; + private readonly targetMs: number; + private readonly intervalMs: number; + private currentNow = 0; + private firstAboveTargetAt = new Map(); + + constructor(opts: { + base: SpikeSelectionStrategy; + redis: Redis; + keys: RunQueueKeyProducer; + targetMs: number; + intervalMs: number; + }) { + this.base = opts.base; + this.reader = new SpikeQueueReader(opts.redis, opts.keys); + this.targetMs = opts.targetMs; + this.intervalMs = opts.intervalMs; + this.name = `codel(${opts.base.name})`; + } + + reset(): void { + this.firstAboveTargetAt = new Map(); + this.currentNow = 0; + this.base.reset?.(); + } + + setClock(now: number): void { + this.currentNow = now; + this.base.setClock?.(now); + } + + private escalatingGroups(minHeadByGroup: Map): Set { + const escalating = new Set(); + const seen = new Set(); + + for (const [group, minHead] of minHeadByGroup) { + seen.add(group); + const sojourn = this.currentNow - minHead; + if (sojourn > this.targetMs) { + const since = this.firstAboveTargetAt.get(group) ?? this.currentNow; + this.firstAboveTargetAt.set(group, since); + if (this.currentNow - since >= this.intervalMs) escalating.add(group); + } else { + this.firstAboveTargetAt.delete(group); + } + } + + // forget groups that are no longer active + for (const g of [...this.firstAboveTargetAt.keys()]) { + if (!seen.has(g)) this.firstAboveTargetAt.delete(g); + } + + return escalating; + } + + async distributeFairQueuesFromParentQueue( + parentQueue: string, + consumerId: string + ): Promise { + const [baseOrder, active] = await Promise.all([ + this.base.distributeFairQueuesFromParentQueue(parentQueue, consumerId), + this.reader.readActiveQueues(parentQueue), + ]); + + const groupOfQueue = new Map(); + const minHeadByGroup = new Map(); + for (const a of active) { + groupOfQueue.set(a.queue, a.groupId); + const head = a.headScore ?? Number.POSITIVE_INFINITY; + const cur = minHeadByGroup.get(a.groupId); + if (cur === undefined || head < cur) minHeadByGroup.set(a.groupId, head); + } + + const escalating = this.escalatingGroups(minHeadByGroup); + if (escalating.size === 0) return baseOrder; + + // Hoist escalating groups' queues to the front of each env, preserving the + // base order within the hoisted and non-hoisted partitions. + return baseOrder.map((env) => { + const hot: string[] = []; + const cold: string[] = []; + for (const q of env.queues) { + const g = groupOfQueue.get(q); + if (g !== undefined && escalating.has(g)) hot.push(q); + else cold.push(q); + } + return { envId: env.envId, queues: [...hot, ...cold] }; + }); + } + + onServiced(descriptor: QueueDescriptor, now: number): void | Promise { + return this.base.onServiced(descriptor, now); + } +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/drrStrategy.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/drrStrategy.ts new file mode 100644 index 0000000000..f107e0c9fa --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/drrStrategy.ts @@ -0,0 +1,98 @@ +import type { Redis } from "@internal/redis"; +import type { EnvQueues, QueueDescriptor, RunQueueKeyProducer } from "../../types.js"; +import { + SpikeQueueReader, + defaultWeight, + groupIdFromQueueName, + type GroupId, + type SpikeSelectionStrategy, + type WeightFn, +} from "../types.js"; +import { buildEnvQueues } from "./base.js"; + +/** + * Deficit round robin. A rotating cursor points at the group currently being + * served. When the cursor lands on a group its deficit is topped up by + * `quantum * weight`; the group keeps winning selection (and spending one unit + * of deficit per serve) until its deficit falls below the unit head cost, then + * the cursor advances. A 3x-weight group therefore serves ~3 runs per turn. + */ +export class DrrStrategy implements SpikeSelectionStrategy { + readonly name = "drr"; + + private readonly reader: SpikeQueueReader; + private readonly weight: WeightFn; + private readonly quantum: number; + private deficit = new Map(); + private ring: GroupId[] = []; + private cursor = 0; + + constructor(opts: { + redis: Redis; + keys: RunQueueKeyProducer; + weight?: WeightFn; + quantum?: number; + }) { + this.reader = new SpikeQueueReader(opts.redis, opts.keys); + this.weight = opts.weight ?? defaultWeight; + this.quantum = opts.quantum ?? 1; + } + + reset(): void { + this.deficit = new Map(); + this.ring = []; + this.cursor = 0; + } + + async distributeFairQueuesFromParentQueue( + parentQueue: string, + _consumerId: string + ): Promise { + const active = await this.reader.readActiveQueues(parentQueue); + if (active.length === 0) return []; + + const activeGroups = new Set(active.map((a) => a.groupId)); + for (const g of activeGroups) if (!this.ring.includes(g)) this.ring.push(g); + this.ring = this.ring.filter((g) => activeGroups.has(g)); + if (this.ring.length === 0) return []; + if (this.cursor >= this.ring.length) this.cursor = 0; + + // Find the group whose turn it is: top up deficits as the cursor passes + // until a group has enough to serve. + let winner = this.ring[this.cursor]; + for (let steps = 0; steps < this.ring.length; steps++) { + const g = this.ring[this.cursor]; + if ((this.deficit.get(g) ?? 0) < 1) { + this.deficit.set(g, (this.deficit.get(g) ?? 0) + this.quantum * this.weight(g)); + } + if ((this.deficit.get(g) ?? 0) >= 1) { + winner = g; + break; + } + this.cursor = (this.cursor + 1) % this.ring.length; + } + + // Rank: winner first, then the rest in ring order from the cursor. + const rank = new Map(); + rank.set(winner, -1); + for (let i = 0; i < this.ring.length; i++) { + const g = this.ring[(this.cursor + i) % this.ring.length]; + if (!rank.has(g)) rank.set(g, i); + } + + return buildEnvQueues( + active, + (a, b) => + (rank.get(a.groupId) ?? this.ring.length) - (rank.get(b.groupId) ?? this.ring.length) + ); + } + + onServiced(descriptor: QueueDescriptor): void { + const g = groupIdFromQueueName(descriptor.queue); + this.deficit.set(g, (this.deficit.get(g) ?? 0) - 1); + if ((this.deficit.get(g) ?? 0) < 1) { + const idx = this.ring.indexOf(g); + if (idx !== -1) this.cursor = (idx + 1) % this.ring.length; + } + } +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/sfqStrategy.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/sfqStrategy.ts new file mode 100644 index 0000000000..56922f02df --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/sfqStrategy.ts @@ -0,0 +1,83 @@ +import type { Redis } from "@internal/redis"; +import type { EnvQueues, QueueDescriptor, RunQueueKeyProducer } from "../../types.js"; +import { + SpikeQueueReader, + defaultWeight, + groupIdFromQueueName, + type GroupId, + type SpikeSelectionStrategy, + type WeightFn, +} from "../types.js"; +import { buildEnvQueues } from "./base.js"; + +/** + * Start-time fair queueing (SFQ), the start-tag form of WFQ. + * + * Each group carries a virtual clock = accumulated service scaled by 1/weight. + * On selection a group's start tag is max(its clock, the system floor); the + * floor is the min clock over active groups (the CFS `min_vruntime` analogue), + * so a newly active or long-idle group jumps to the floor rather than hoarding + * credit or being buried. Queues are ordered by ascending start tag. + * + * Note: this is plain start-time WFQ, not EEVDF. An earlier version carried an + * eligibility term, but with one clock per group the smallest-start-tag group is + * always the eligible one, so the term never changed the ordering; it was dropped. + */ +export class SfqStrategy implements SpikeSelectionStrategy { + readonly name = "sfq"; + + private readonly reader: SpikeQueueReader; + private readonly weight: WeightFn; + private readonly quantum: number; + private virtualClock = new Map(); + private floor = 0; + + constructor(opts: { + redis: Redis; + keys: RunQueueKeyProducer; + weight?: WeightFn; + quantum?: number; + }) { + this.reader = new SpikeQueueReader(opts.redis, opts.keys); + this.weight = opts.weight ?? defaultWeight; + this.quantum = opts.quantum ?? 1; + } + + reset(): void { + this.virtualClock = new Map(); + this.floor = 0; + } + + private clockOf(groupId: GroupId): number { + return this.virtualClock.get(groupId) ?? this.floor; + } + + private startTag(groupId: GroupId): number { + return Math.max(this.clockOf(groupId), this.floor); + } + + async distributeFairQueuesFromParentQueue( + parentQueue: string, + _consumerId: string + ): Promise { + const active = await this.reader.readActiveQueues(parentQueue); + if (active.length === 0) return []; + + // Advance the system floor like CFS min_vruntime: it is the min clock over + // active groups but is monotonic non-decreasing, so a group that went idle + // and returns with a stale low clock is pulled up to the floor (via + // startTag) instead of collapsing the floor and monopolising service. + const activeGroups = new Set(active.map((a) => a.groupId)); + let min = Infinity; + for (const g of activeGroups) min = Math.min(min, this.clockOf(g)); + if (Number.isFinite(min)) this.floor = Math.max(this.floor, min); + + return buildEnvQueues(active, (a, b) => this.startTag(a.groupId) - this.startTag(b.groupId)); + } + + onServiced(descriptor: QueueDescriptor): void { + const g = groupIdFromQueueName(descriptor.queue); + const next = this.startTag(g) + this.quantum / this.weight(g); + this.virtualClock.set(g, next); + } +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/strideStrategy.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/strideStrategy.ts new file mode 100644 index 0000000000..07fcb713b5 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/strategies/strideStrategy.ts @@ -0,0 +1,71 @@ +import type { Redis } from "@internal/redis"; +import type { EnvQueues, QueueDescriptor, RunQueueKeyProducer } from "../../types.js"; +import { + SpikeQueueReader, + defaultWeight, + groupIdFromQueueName, + type GroupId, + type SpikeSelectionStrategy, + type WeightFn, +} from "../types.js"; +import { buildEnvQueues } from "./base.js"; + +/** + * Stride scheduling: deterministic integer virtual-time WFQ. Each group has a + * stride = stride1 / weight and a pass counter; the lowest pass is served next, + * and servicing advances that group's pass by its stride. A newly active group + * starts at the current minimum pass, so it neither hoards credit nor jumps the + * queue (the classic late-arrival guard). + */ +export class StrideStrategy implements SpikeSelectionStrategy { + readonly name = "stride"; + + private readonly reader: SpikeQueueReader; + private readonly weight: WeightFn; + private readonly stride1: number; + private pass = new Map(); + private floor = 0; + + constructor(opts: { + redis: Redis; + keys: RunQueueKeyProducer; + weight?: WeightFn; + stride1?: number; + }) { + this.reader = new SpikeQueueReader(opts.redis, opts.keys); + this.weight = opts.weight ?? defaultWeight; + this.stride1 = opts.stride1 ?? 1_000_000; + } + + reset(): void { + this.pass = new Map(); + this.floor = 0; + } + + // Effective pass: an over-served group keeps its high pass, but a new or + // returned-from-idle group is pulled up to the monotonic floor so it cannot + // monopolise service with a stale low counter. + private passOf(groupId: GroupId): number { + return Math.max(this.pass.get(groupId) ?? this.floor, this.floor); + } + + async distributeFairQueuesFromParentQueue( + parentQueue: string, + _consumerId: string + ): Promise { + const active = await this.reader.readActiveQueues(parentQueue); + if (active.length === 0) return []; + + const activeGroups = new Set(active.map((a) => a.groupId)); + let min = Infinity; + for (const g of activeGroups) min = Math.min(min, this.pass.get(g) ?? this.floor); + if (Number.isFinite(min)) this.floor = Math.max(this.floor, min); + + return buildEnvQueues(active, (a, b) => this.passOf(a.groupId) - this.passOf(b.groupId)); + } + + onServiced(descriptor: QueueDescriptor): void { + const g = groupIdFromQueueName(descriptor.queue); + this.pass.set(g, this.passOf(g) + this.stride1 / this.weight(g)); + } +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/tests/codel.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/codel.test.ts new file mode 100644 index 0000000000..49e4d3b2d7 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/codel.test.ts @@ -0,0 +1,62 @@ +import { describe, it, expect } from "vitest"; +import type { Redis } from "@internal/redis"; +import { CodelWrapper } from "../strategies/codelWrapper.js"; +import type { SpikeSelectionStrategy } from "../types.js"; +import { keys, queueKeyFor } from "./support.js"; + +const parent = keys.masterQueueKeyForShard(0); + +/** Base selector that always orders a~0 before b~0. */ +const stubBase: SpikeSelectionStrategy = { + name: "stub", + async distributeFairQueuesFromParentQueue() { + return [{ envId: "e", queues: [queueKeyFor("a~0"), queueKeyFor("b~0")] }]; + }, + onServiced() {}, +}; + +/** Mutable fake redis whose queue heads can change between calls. */ +function mutableRedis(state: { active: Array<{ name: string; head: number }> }): Redis { + return { + async zrange(_key: string, _s: number, _e: number, ws?: string): Promise { + if (ws) return state.active.flatMap((q) => [queueKeyFor(q.name), String(q.head)]); + return state.active.map((q) => queueKeyFor(q.name)); + }, + } as unknown as Redis; +} + +describe("CodelWrapper", () => { + it("hoists a group whose sojourn exceeds target for a full interval, then reverts", async () => { + const state = { + active: [ + { name: "a~0", head: 1000 }, + { name: "b~0", head: 0 }, + ], + }; + const codel = new CodelWrapper({ + base: stubBase, + redis: mutableRedis(state), + keys, + targetMs: 50, + intervalMs: 100, + }); + + // t=200: b sojourn = 200 > target, but interval not yet elapsed + codel.setClock(200); + let order = await codel.distributeFairQueuesFromParentQueue(parent, "e"); + expect(order[0].queues[0]).toBe(queueKeyFor("a~0")); + + // t=350: b has been above target for 150ms >= interval -> hoisted + codel.setClock(350); + order = await codel.distributeFairQueuesFromParentQueue(parent, "e"); + expect(order[0].queues[0]).toBe(queueKeyFor("b~0")); + + // b's oldest run is now fresh -> sojourn drops below target -> reverts + state.active = [ + { name: "a~0", head: 1000 }, + { name: "b~0", head: 349 }, + ]; + order = await codel.distributeFairQueuesFromParentQueue(parent, "e"); + expect(order[0].queues[0]).toBe(queueKeyFor("a~0")); + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/tests/driver.smoke.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/driver.smoke.test.ts new file mode 100644 index 0000000000..3d30777947 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/driver.smoke.test.ts @@ -0,0 +1,44 @@ +import { redisTest } from "@internal/testcontainers"; +import { describe, expect } from "vitest"; +import { FairQueueSelectionStrategy } from "../../fairQueueSelectionStrategy.js"; +import { RunQueueFullKeyProducer } from "../../keyProducer.js"; +import { runScenario } from "../harness/driver.js"; +import { buildWorkload } from "../harness/workload.js"; + +const keys = new RunQueueFullKeyProducer(); + +describe("driver smoke (baseline)", () => { + redisTest("balanced workload dequeues every run and is roughly fair", async ({ redisContainer }) => { + const redis = { + keyPrefix: "runqueue:spike-smoke:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + + const workload = buildWorkload({ + seed: "smoke-1", + envConcurrencyLimit: 5, + tenants: [ + { tenantId: "a", runCount: 50, holdMsMean: 30 }, + { tenantId: "b", runCount: 50, holdMsMean: 30 }, + { tenantId: "c", runCount: 50, holdMsMean: 30 }, + { tenantId: "d", runCount: 50, holdMsMean: 30 }, + ], + }); + + const metrics = await runScenario({ + redis, + strategy: new FairQueueSelectionStrategy({ redis, keys }), + workload, + }); + + // Harness health: every run drains exactly once and every tenant completes. + // (Baseline fairness itself is seed-sensitive and is measured in the bench, + // not asserted here.) + expect(metrics.totalDequeued).toBe(200); + expect(metrics.perGroup).toHaveLength(4); + for (const g of metrics.perGroup) { + expect(g.dequeued).toBe(50); + } + }, 60_000); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/tests/drr.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/drr.test.ts new file mode 100644 index 0000000000..d341a71662 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/drr.test.ts @@ -0,0 +1,55 @@ +import { describe, it, expect } from "vitest"; +import { DrrStrategy } from "../strategies/drrStrategy.js"; +import { descriptorFor, fakeRedis, keys, queueKeyFor } from "./support.js"; + +const parent = keys.masterQueueKeyForShard(0); + +describe("DrrStrategy", () => { + it("alternates evenly between two equal-weight groups", async () => { + const redis = fakeRedis([ + { name: "a~0", head: 100 }, + { name: "b~0", head: 100 }, + ]); + const drr = new DrrStrategy({ redis, keys }); + + let a = 0; + let b = 0; + for (let i = 0; i < 10; i++) { + const order = await drr.distributeFairQueuesFromParentQueue(parent, "e"); + const head = order[0].queues[0]; + if (head === queueKeyFor("a~0")) { + a++; + drr.onServiced(descriptorFor("a~0")); + } else { + b++; + drr.onServiced(descriptorFor("b~0")); + } + } + expect(Math.abs(a - b)).toBeLessThanOrEqual(1); + }); + + it("splits capacity by weight (3:1)", async () => { + const redis = fakeRedis([ + { name: "a~0", head: 100 }, + { name: "b~0", head: 100 }, + ]); + const weight = (g: string) => (g === "a" ? 3 : 1); + const drr = new DrrStrategy({ redis, keys, weight }); + + let a = 0; + let b = 0; + for (let i = 0; i < 80; i++) { + const order = await drr.distributeFairQueuesFromParentQueue(parent, "e"); + const head = order[0].queues[0]; + if (head === queueKeyFor("a~0")) { + a++; + drr.onServiced(descriptorFor("a~0")); + } else { + b++; + drr.onServiced(descriptorFor("b~0")); + } + } + expect(a / b).toBeGreaterThan(2.3); + expect(a / b).toBeLessThan(3.7); + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/tests/metrics.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/metrics.test.ts new file mode 100644 index 0000000000..1158da333a --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/metrics.test.ts @@ -0,0 +1,79 @@ +import { describe, it, expect } from "vitest"; +import { computeMetrics, type DequeueEvent } from "../harness/metrics.js"; + +function ev(groupId: string, runId: string, dequeueAt: number, enqueueAt = 0): DequeueEvent { + return { groupId, runId, enqueueAtMs: enqueueAt, dequeueAtMs: dequeueAt }; +} + +describe("computeMetrics", () => { + it("flags a starved competitor as contention share ~0", () => { + // a is served for all 4 of its runs before b gets any; both had 4 to do. + const events: DequeueEvent[] = [ + ...Array.from({ length: 4 }, (_, i) => ev("a", `a${i}`, i + 1)), + ...Array.from({ length: 4 }, (_, i) => ev("b", `b${i}`, 100 + i)), + ]; + const m = computeMetrics({ + events, + weights: { a: 1, b: 1 }, + totals: { a: 4, b: 4 }, + redisOps: 0, + wallClockMs: 0, + }); + // during the window (while both had work) a took everything, b took nothing + expect(m.contentionWorstShareOverWeight).toBeCloseTo(0, 6); + const a = m.perGroup.find((g) => g.groupId === "a")!; + expect(a.contentionShareOverWeight).toBeGreaterThan(1.5); + }); + + it("scores an even interleave as fair", () => { + const events: DequeueEvent[] = []; + for (let i = 0; i < 4; i++) { + events.push(ev("a", `a${i}`, i * 2 + 1)); + events.push(ev("b", `b${i}`, i * 2 + 2)); + } + const m = computeMetrics({ + events, + weights: { a: 1, b: 1 }, + totals: { a: 4, b: 4 }, + redisOps: 0, + wallClockMs: 0, + }); + expect(m.contentionWorstShareOverWeight).toBeGreaterThan(0.8); + expect(m.contentionJain).toBeGreaterThan(0.95); + }); + + it("does not count a tenant as contending before its runs arrive", () => { + // a runs and finishes entirely before b's runs are ever enqueued. They never + // actually compete, so b must not be scored as starved. + const events: DequeueEvent[] = [ + ...Array.from({ length: 4 }, (_, i) => ev("a", `a${i}`, i + 1, 0)), + ...Array.from({ length: 2 }, (_, i) => ev("b", `b${i}`, 51 + i, 50)), + ]; + const m = computeMetrics({ + events, + weights: { a: 1, b: 1 }, + totals: { a: 4, b: 2 }, + redisOps: 0, + wallClockMs: 0, + }); + // no instant had two tenants with arrived, unserved work + expect(m.contentionWorstShareOverWeight).toBe(1); + }); + + it("reports per-group wait percentiles and the worst tail", () => { + const waits = [1, 2, 3, 4, 5, 6, 7, 8, 9, 100]; + const events = waits.map((w, i) => ev("a", `a${i}`, w)); + const m = computeMetrics({ + events, + weights: { a: 1 }, + totals: { a: 10 }, + redisOps: 0, + wallClockMs: 0, + }); + const a = m.perGroup[0]; + expect(a.waitP50).toBe(5); + expect(a.waitP99).toBe(100); + expect(a.waitMax).toBe(100); + expect(m.worstWaitP99).toBe(100); + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/tests/queueReader.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/queueReader.test.ts new file mode 100644 index 0000000000..fa117669b9 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/queueReader.test.ts @@ -0,0 +1,85 @@ +import { redisTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { Logger } from "@trigger.dev/core/logger"; +import { createRedisClient } from "@internal/redis"; +import { Decimal } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { RunQueue } from "../../index.js"; +import { FairQueueSelectionStrategy } from "../../fairQueueSelectionStrategy.js"; +import { RunQueueFullKeyProducer } from "../../keyProducer.js"; +import type { InputPayload } from "../../types.js"; +import { SpikeQueueReader } from "../types.js"; + +const keys = new RunQueueFullKeyProducer(); + +const authenticatedEnvProd = { + id: "e-spike", + type: "PRODUCTION" as const, + maximumConcurrencyLimit: 10, + concurrencyLimitBurstFactor: new Decimal(1.0), + project: { id: "p-spike" }, + organization: { id: "o-spike" }, +}; + +function makeMessage(queue: string, runId: string): InputPayload { + return { + runId, + orgId: "o-spike", + projectId: "p-spike", + environmentId: "e-spike", + environmentType: "PRODUCTION", + queue, + timestamp: Date.now(), + attempt: 0, + }; +} + +describe("SpikeQueueReader", () => { + redisTest("reads active base queues from the master queue", async ({ redisContainer }) => { + const redisOptions = { + keyPrefix: "runqueue:spike:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }; + + const queue = new RunQueue({ + name: "rq-spike", + tracer: trace.getTracer("rq-spike"), + logger: new Logger("RunQueueSpike", "error"), + defaultEnvConcurrency: 10, + shardCount: 1, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + keys, + redis: redisOptions, + queueSelectionStrategy: new FairQueueSelectionStrategy({ redis: redisOptions, keys }), + }); + + const rawRedis = createRedisClient(redisOptions); + + try { + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: makeMessage("task/g1", "task/g1-0"), + workerQueue: authenticatedEnvProd.id, + skipDequeueProcessing: true, + }); + await queue.enqueueMessage({ + env: authenticatedEnvProd, + message: makeMessage("task/g2", "task/g2-0"), + workerQueue: authenticatedEnvProd.id, + skipDequeueProcessing: true, + }); + + const reader = new SpikeQueueReader(rawRedis, keys); + const active = await reader.readActiveQueues(keys.masterQueueKeyForShard(0)); + + expect(active.map((a) => a.groupId).sort()).toEqual(["task/g1", "task/g2"]); + expect(active.every((a) => typeof a.headScore === "number")).toBe(true); + expect(active.every((a) => a.env.envId === "e-spike")).toBe(true); + } finally { + await rawRedis.quit(); + await queue.quit(); + } + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/tests/sfq.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/sfq.test.ts new file mode 100644 index 0000000000..c36376918b --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/sfq.test.ts @@ -0,0 +1,47 @@ +import { describe, it, expect } from "vitest"; +import { SfqStrategy } from "../strategies/sfqStrategy.js"; +import { descriptorFor, fakeRedis, keys, queueKeyFor } from "./support.js"; + +const parent = keys.masterQueueKeyForShard(0); + +describe("SfqStrategy", () => { + it("orders an under-served group ahead of an over-served one", async () => { + const redis = fakeRedis([ + { name: "a~0", head: 100 }, + { name: "b~0", head: 100 }, + ]); + const sfq = new SfqStrategy({ redis, keys }); + + // serve group a ten times; b never + for (let i = 0; i < 10; i++) sfq.onServiced(descriptorFor("a~0")); + + const order = await sfq.distributeFairQueuesFromParentQueue(parent, "e"); + expect(order).toHaveLength(1); + expect(order[0].queues[0]).toBe(queueKeyFor("b~0")); + }); + + it("respects weight: a 3x-weighted group is serviceable more often", async () => { + const redis = fakeRedis([ + { name: "a~0", head: 100 }, + { name: "b~0", head: 100 }, + ]); + const weight = (g: string) => (g === "a" ? 3 : 1); + const sfq = new SfqStrategy({ redis, keys, weight }); + + let a = 0; + let b = 0; + for (let i = 0; i < 40; i++) { + const order = await sfq.distributeFairQueuesFromParentQueue(parent, "e"); + const head = order[0].queues[0]; + if (head === queueKeyFor("a~0")) { + a++; + sfq.onServiced(descriptorFor("a~0")); + } else { + b++; + sfq.onServiced(descriptorFor("b~0")); + } + } + expect(a / b).toBeGreaterThan(2.3); + expect(a / b).toBeLessThan(3.7); + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/tests/stride.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/stride.test.ts new file mode 100644 index 0000000000..e992e73463 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/stride.test.ts @@ -0,0 +1,41 @@ +import { describe, it, expect } from "vitest"; +import { StrideStrategy } from "../strategies/strideStrategy.js"; +import { descriptorFor, fakeRedis, keys, queueKeyFor } from "./support.js"; + +const parent = keys.masterQueueKeyForShard(0); + +async function serviceRatio(weight: (g: string) => number, rounds: number) { + const redis = fakeRedis([ + { name: "a~0", head: 100 }, + { name: "b~0", head: 100 }, + ]); + const stride = new StrideStrategy({ redis, keys, weight }); + let a = 0; + let b = 0; + for (let i = 0; i < rounds; i++) { + const order = await stride.distributeFairQueuesFromParentQueue(parent, "e"); + const head = order[0].queues[0]; + if (head === queueKeyFor("a~0")) { + a++; + stride.onServiced(descriptorFor("a~0")); + } else { + b++; + stride.onServiced(descriptorFor("b~0")); + } + } + return a / b; +} + +describe("StrideStrategy", () => { + it("services 3:1-weighted groups roughly 3:1", async () => { + const ratio = await serviceRatio((g) => (g === "a" ? 3 : 1), 80); + expect(ratio).toBeGreaterThan(2.5); + expect(ratio).toBeLessThan(3.5); + }); + + it("services equal-weight groups roughly 1:1", async () => { + const ratio = await serviceRatio(() => 1, 80); + expect(ratio).toBeGreaterThan(0.8); + expect(ratio).toBeLessThan(1.2); + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/tests/support.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/support.ts new file mode 100644 index 0000000000..1eb3550591 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/support.ts @@ -0,0 +1,32 @@ +import type { Redis } from "@internal/redis"; +import { RunQueueFullKeyProducer } from "../../keyProducer.js"; +import type { QueueDescriptor } from "../../types.js"; + +export const keys = new RunQueueFullKeyProducer(); + +const ENV = { organization: { id: "o" }, project: { id: "p" }, id: "e" } as const; + +export function queueKeyFor(name: string): string { + return keys.queueKey(ENV as never, name); +} + +export function descriptorFor(name: string): QueueDescriptor { + return keys.descriptorFromQueue(queueKeyFor(name)); +} + +/** + * Minimal in-memory stand-in for the bits of ioredis that SpikeQueueReader uses + * (`zrange` over the master queue and over each queue's head). `active` is the + * list of queue base-names currently present, each with a head score. + */ +export function fakeRedis(active: Array<{ name: string; head: number }>): Redis { + return { + async zrange(_key: string, _start: number, _stop: number, withScores?: string): Promise { + // Only the master-queue WITHSCORES read is used by the reader now. + if (withScores) { + return active.flatMap((q) => [queueKeyFor(q.name), String(q.head)]); + } + return active.map((q) => queueKeyFor(q.name)); + }, + } as unknown as Redis; +} diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/tests/workload.test.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/workload.test.ts new file mode 100644 index 0000000000..85962e048a --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/tests/workload.test.ts @@ -0,0 +1,49 @@ +import { describe, it, expect } from "vitest"; +import { buildWorkload, expandEvents, type WorkloadConfig } from "../harness/workload.js"; +import { groupIdFromQueueName } from "../types.js"; + +const cfg: WorkloadConfig = { + seed: "spike-test", + envConcurrencyLimit: 5, + tenants: [ + { tenantId: "a", runCount: 6, queueCount: 3, arrival: "immediate" }, + { tenantId: "b", runCount: 3, arrival: "poisson", ratePerSec: 50, holdMsMean: 20 }, + ], +}; + +describe("workload generator", () => { + it("is deterministic for a given seed", () => { + const a = expandEvents(buildWorkload(cfg)); + const b = expandEvents(buildWorkload(cfg)); + expect(a).toEqual(b); + }); + + it("produces one event per run", () => { + const events = expandEvents(buildWorkload(cfg)); + expect(events).toHaveLength(9); + }); + + it("spreads a tenant's runs across its queueCount queues", () => { + const events = expandEvents(buildWorkload(cfg)).filter((e) => e.groupId === "a"); + const distinctQueues = new Set(events.map((e) => e.queueName)); + expect(distinctQueues.size).toBe(3); + expect(events.every((e) => groupIdFromQueueName(e.queueName) === "a")).toBe(true); + }); + + it("immediate arrivals all enqueue at startAt", () => { + const events = expandEvents(buildWorkload(cfg)).filter((e) => e.groupId === "a"); + expect(events.every((e) => e.enqueueAtMs === 0)).toBe(true); + }); + + it("poisson arrivals are non-decreasing in time", () => { + const events = expandEvents(buildWorkload(cfg)).filter((e) => e.groupId === "b"); + for (let i = 1; i < events.length; i++) { + expect(events[i].enqueueAtMs).toBeGreaterThanOrEqual(events[i - 1].enqueueAtMs); + } + }); + + it("assigns a positive hold to every run", () => { + const events = expandEvents(buildWorkload(cfg)); + expect(events.every((e) => e.holdMs >= 1)).toBe(true); + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/fairness-spike/types.ts b/internal-packages/run-engine/src/run-queue/fairness-spike/types.ts new file mode 100644 index 0000000000..c2a98a25f6 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/fairness-spike/types.ts @@ -0,0 +1,102 @@ +import type { Redis } from "@internal/redis"; +import type { + EnvDescriptor, + QueueDescriptor, + RunQueueKeyProducer, + RunQueueSelectionStrategy, +} from "../types.js"; + +/** + * Fairness-spike shared types. Throwaway: this whole directory is a bench for + * ranking fair-queueing disciplines and ships nothing. + * + * Grain = the base queue name (see the plan's Discovery note for why not the + * concurrency key). A "group" is one distinct base queue in one environment. + */ + +export type GroupId = string; + +export type WeightFn = (groupId: GroupId) => number; + +export const defaultWeight: WeightFn = () => 1; + +/** + * A "group" (tenant) can own more than one base queue. We encode the tenant in + * the queue name as `${tenant}~${index}` and recover it here. This is what lets + * the spike reproduce the #2617 dynamic at the base-queue grain: a heavy tenant + * owning many queues would out-select light tenants under any per-queue + * discipline that is blind to tenant identity (the current baseline), while a + * tenant-keyed discipline stays fair. + */ +export const GROUP_SEPARATOR = "~"; + +export function groupIdFromQueueName(queueName: string): GroupId { + const idx = queueName.indexOf(GROUP_SEPARATOR); + return idx === -1 ? queueName : queueName.slice(0, idx); +} + +/** + * A selection strategy under test. Extends the real RunQueue interface with an + * `onServiced` hook: the strategy interface is selection-only and is never told + * which queue actually got dequeued, so stateful disciplines (virtual clock, + * deficit, pass counter) need the driver to feed serviced descriptors back. + * In production that advance would live in the ack/dequeue Lua. + */ +export interface SpikeSelectionStrategy extends RunQueueSelectionStrategy { + readonly name: string; + onServiced(descriptor: QueueDescriptor, now: number): void | Promise; + reset?(): void | Promise; + /** + * The driver calls this with the current logical clock (in message-score + * space) before each drain, so time-aware disciplines (CoDel) can measure + * sojourn against the same clock the workload uses. + */ + setClock?(now: number): void; +} + +export type ActiveQueue = { + queue: string; + env: EnvDescriptor; + groupId: GroupId; + headScore: number | undefined; +}; + +/** + * Reads the current set of active base queues under a parent (master) queue, + * with each queue's head-message score (its oldest enqueue timestamp). The + * candidate selectors order these. + */ +export class SpikeQueueReader { + constructor( + private readonly redis: Redis, + private readonly keys: RunQueueKeyProducer + ) {} + + async readActiveQueues(parentQueue: string): Promise { + // The master queue ZSET scores each queue by its earliest (head) message + // timestamp, so one WITHSCORES read gives us every active queue and its head + // age with no per-queue round trips. This mirrors how the real + // FairQueueSelectionStrategy reads ages. + const raw = await this.redis.zrange(parentQueue, 0, -1, "WITHSCORES"); + const out: ActiveQueue[] = []; + + for (let i = 0; i + 1 < raw.length; i += 2) { + const queue = raw[i]; + const headScore = Number(raw[i + 1]); + + // Grain is base queues; a CK wildcard here means a workload leaked a + // concurrency key, which the spike does not use. + if (this.keys.isCkWildcard(queue)) continue; + + const d = this.keys.descriptorFromQueue(queue); + out.push({ + queue, + env: { orgId: d.orgId, projectId: d.projectId, envId: d.envId }, + groupId: groupIdFromQueueName(d.queue), + headScore, + }); + } + + return out; + } +}