Add load testing framework

simstudioai · icecrasher321 · Mar 27, 2026 · Mar 16, 2026 · Mar 16, 2026 · Mar 16, 2026
commit 53733e42635e9f52f84c301eec01837653f0db20
diff --git a/apps/sim/package.json b/apps/sim/package.json
@@ -13,6 +13,10 @@
     "dev:sockets": "bun run socket/index.ts",
     "dev:worker": "bun run worker/index.ts",
     "dev:full": "bunx concurrently -n \"App,Realtime,Worker\" -c \"cyan,magenta,yellow\" \"bun run dev\" \"bun run dev:sockets\" \"bun run dev:worker\"",
+    "load:workflow": "bun run load:workflow:baseline",
+    "load:workflow:baseline": "BASE_URL=${BASE_URL:-http://localhost:3000} WARMUP_DURATION=${WARMUP_DURATION:-10} WARMUP_RATE=${WARMUP_RATE:-2} PEAK_RATE=${PEAK_RATE:-8} HOLD_DURATION=${HOLD_DURATION:-20} bunx artillery run scripts/load/workflow-concurrency.yml",
+    "load:workflow:waves": "BASE_URL=${BASE_URL:-http://localhost:3000} WAVE_ONE_DURATION=${WAVE_ONE_DURATION:-10} WAVE_ONE_RATE=${WAVE_ONE_RATE:-6} QUIET_DURATION=${QUIET_DURATION:-5} WAVE_TWO_DURATION=${WAVE_TWO_DURATION:-15} WAVE_TWO_RATE=${WAVE_TWO_RATE:-8} WAVE_THREE_DURATION=${WAVE_THREE_DURATION:-20} WAVE_THREE_RATE=${WAVE_THREE_RATE:-10} bunx artillery run scripts/load/workflow-waves.yml",
+    "load:workflow:isolation": "BASE_URL=${BASE_URL:-http://localhost:3000} ISOLATION_DURATION=${ISOLATION_DURATION:-30} TOTAL_RATE=${TOTAL_RATE:-9} WORKSPACE_A_WEIGHT=${WORKSPACE_A_WEIGHT:-8} WORKSPACE_B_WEIGHT=${WORKSPACE_B_WEIGHT:-1} bunx artillery run scripts/load/workflow-isolation.yml",
     "build": "next build",
     "start": "next start",
     "worker": "NODE_ENV=production bun run worker/index.ts",

diff --git a/apps/sim/scripts/load/README.md b/apps/sim/scripts/load/README.md
@@ -0,0 +1,113 @@
+# Workflow Load Tests
+
+These local-only Artillery scenarios exercise `POST /api/workflows/[id]/execute` in async mode.
+
+## Requirements
+
+- The app should be running locally, for example with `bun run dev:full`
+- Each scenario needs valid workflow IDs and API keys
+- All scenarios default to `http://localhost:3000`
+
+The default rates are tuned for these local limits:
+
+- `ADMISSION_GATE_MAX_INFLIGHT=500`
+- `DISPATCH_MAX_QUEUE_PER_WORKSPACE=1000`
+- `DISPATCH_MAX_QUEUE_GLOBAL=50000`
+- `WORKSPACE_CONCURRENCY_FREE=5`
+- `WORKSPACE_CONCURRENCY_PRO=50`
+- `WORKSPACE_CONCURRENCY_TEAM=200`
+- `WORKSPACE_CONCURRENCY_ENTERPRISE=200`
+
+That means the defaults are intentionally aimed at forcing queueing for a Free workspace without overwhelming a single local dev server process.
+
+## Baseline Concurrency
+
+Use this to ramp traffic into one workflow and observe normal queueing behavior.
+
+Default profile:
+
+- Starts at `2` requests per second
+- Ramps to `8` requests per second
+- Holds there for `20` seconds
+- Good for validating queueing against a Free workspace concurrency of `5`
+
+```bash
+WORKFLOW_ID=<workflow-id> \
+SIM_API_KEY=<api-key> \
+bun run load:workflow:baseline
+```
+
+Optional variables:
+
+- `BASE_URL`
+- `WARMUP_DURATION`
+- `WARMUP_RATE`
+- `PEAK_RATE`
+- `HOLD_DURATION`
+
+For higher-plan workspaces, a good local starting point is:
+
+- Pro: `PEAK_RATE=20` to `40`
+- Team or Enterprise: `PEAK_RATE=50` to `100`
+
+## Queueing Waves
+
+Use this to send repeated bursts to one workflow in the same workspace.
+
+Default profile:
+
+- Wave 1: `6` requests per second for `10` seconds
+- Wave 2: `8` requests per second for `15` seconds
+- Wave 3: `10` requests per second for `20` seconds
+- Quiet gaps: `5` seconds
+
+```bash
+WORKFLOW_ID=<workflow-id> \
+SIM_API_KEY=<api-key> \
+bun run load:workflow:waves
+```
+
+Optional variables:
+
+- `BASE_URL`
+- `WAVE_ONE_DURATION`
+- `WAVE_ONE_RATE`
+- `QUIET_DURATION`
+- `WAVE_TWO_DURATION`
+- `WAVE_TWO_RATE`
+- `WAVE_THREE_DURATION`
+- `WAVE_THREE_RATE`
+
+## Two-Workspace Isolation
+
+Use this to send mixed traffic to two workflows from different workspaces and compare whether one workspace's queue pressure appears to affect the other.
+
+Default profile:
+
+- Total rate: `9` requests per second for `30` seconds
+- Weight split: `8:1`
+- In practice this sends heavy pressure to workspace A while still sending a light stream to workspace B
+
+```bash
+WORKFLOW_ID_A=<workspace-a-workflow-id> \
+SIM_API_KEY_A=<workspace-a-api-key> \
+WORKFLOW_ID_B=<workspace-b-workflow-id> \
+SIM_API_KEY_B=<workspace-b-api-key> \
+bun run load:workflow:isolation
+```
+
+Optional variables:
+
+- `BASE_URL`
+- `ISOLATION_DURATION`
+- `TOTAL_RATE`
+- `WORKSPACE_A_WEIGHT`
+- `WORKSPACE_B_WEIGHT`
+
+## Notes
+
+- `load:workflow` is an alias for `load:workflow:baseline`
+- All scenarios send `x-execution-mode: async`
+- Artillery output will show request counts and response codes, which is usually enough for quick local verification
+- At these defaults, you should observe queueing behavior before you approach `ADMISSION_GATE_MAX_INFLIGHT=500` or `DISPATCH_MAX_QUEUE_PER_WORKSPACE=1000`
+- If you still see lots of `429` or `ETIMEDOUT` responses locally, lower the rates again before increasing durations
diff --git a/apps/sim/scripts/load/workflow-concurrency.yml b/apps/sim/scripts/load/workflow-concurrency.yml
@@ -0,0 +1,24 @@
+config:
+  target: "{{ $processEnvironment.BASE_URL }}"
+  phases:
+    - duration: "{{ $processEnvironment.WARMUP_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.WARMUP_RATE }}"
+      rampTo: "{{ $processEnvironment.PEAK_RATE }}"
+      name: baseline-ramp
+    - duration: "{{ $processEnvironment.HOLD_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.PEAK_RATE }}"
+      name: baseline-hold
+  defaults:
+    headers:
+      content-type: application/json
+      x-api-key: "{{ $processEnvironment.SIM_API_KEY }}"
+      x-execution-mode: async
+scenarios:
+  - name: baseline-workflow-concurrency
+    flow:
+      - post:
+          url: "/api/workflows/{{ $processEnvironment.WORKFLOW_ID }}/execute"
+          json:
+            input:
+              source: artillery-baseline
+              runId: "{{ $uuid }}"
diff --git a/apps/sim/scripts/load/workflow-isolation.yml b/apps/sim/scripts/load/workflow-isolation.yml
@@ -0,0 +1,35 @@
+config:
+  target: "{{ $processEnvironment.BASE_URL }}"
+  phases:
+    - duration: "{{ $processEnvironment.ISOLATION_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.TOTAL_RATE }}"
+      name: mixed-workspace-load
+  defaults:
+    headers:
+      content-type: application/json
+      x-execution-mode: async
+scenarios:
+  - name: workspace-a-traffic
+    weight: "{{ $processEnvironment.WORKSPACE_A_WEIGHT }}"
+    flow:
+      - post:
+          url: "/api/workflows/{{ $processEnvironment.WORKFLOW_ID_A }}/execute"
+          headers:
+            x-api-key: "{{ $processEnvironment.SIM_API_KEY_A }}"
+          json:
+            input:
+              source: artillery-isolation
+              workspace: a
+              runId: "{{ $uuid }}"
+  - name: workspace-b-traffic
+    weight: "{{ $processEnvironment.WORKSPACE_B_WEIGHT }}"
+    flow:
+      - post:
+          url: "/api/workflows/{{ $processEnvironment.WORKFLOW_ID_B }}/execute"
+          headers:
+            x-api-key: "{{ $processEnvironment.SIM_API_KEY_B }}"
+          json:
+            input:
+              source: artillery-isolation
+              workspace: b
+              runId: "{{ $uuid }}"
diff --git a/apps/sim/scripts/load/workflow-waves.yml b/apps/sim/scripts/load/workflow-waves.yml
@@ -0,0 +1,33 @@
+config:
+  target: "{{ $processEnvironment.BASE_URL }}"
+  phases:
+    - duration: "{{ $processEnvironment.WAVE_ONE_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.WAVE_ONE_RATE }}"
+      name: wave-one
+    - duration: "{{ $processEnvironment.QUIET_DURATION }}"
+      arrivalRate: 1
+      name: quiet-gap
+    - duration: "{{ $processEnvironment.WAVE_TWO_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.WAVE_TWO_RATE }}"
+      name: wave-two
+    - duration: "{{ $processEnvironment.QUIET_DURATION }}"
+      arrivalRate: 1
+      name: quiet-gap-two
+    - duration: "{{ $processEnvironment.WAVE_THREE_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.WAVE_THREE_RATE }}"
+      name: wave-three
+  defaults:
+    headers:
+      content-type: application/json
+      x-api-key: "{{ $processEnvironment.SIM_API_KEY }}"
+      x-execution-mode: async
+scenarios:
+  - name: workflow-queue-waves
+    flow:
+      - post:
+          url: "/api/workflows/{{ $processEnvironment.WORKFLOW_ID }}/execute"
+          json:
+            input:
+              source: artillery-waves
+              runId: "{{ $uuid }}"
+              waveProfile: single-workspace