From fbbc9f2ba5e766fd45218df1be4e8c8aa8c90177 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Thu, 23 Jul 2026 01:16:14 -0700 Subject: [PATCH 1/2] test(e2e): finalize settings suite stabilization Co-authored-by: Cursor --- .github/workflows/ci.yml | 5 - .github/workflows/migrations.yml | 5 + .github/workflows/test-build.yml | 30 +- apps/docs/app/api/chat/route.ts | 6 +- .../docs/en/platform/enterprise/sso.mdx | 58 +++- apps/docs/lib/search-tool-schema.test.ts | 24 ++ apps/docs/lib/search-tool-schema.ts | 41 +++ apps/docs/package.json | 4 +- apps/sim/e2e/README.md | 146 ++++++-- apps/sim/e2e/STABILIZATION.md | 157 +++++++++ apps/sim/e2e/fixtures/validate-scenario.ts | 36 +- apps/sim/e2e/foundation/safety.spec.ts | 93 ++++- .../foundation/scenario-validation.spec.ts | 42 ++- apps/sim/e2e/scripts/capture-auth-states.ts | 16 +- apps/sim/e2e/scripts/options.ts | 49 ++- apps/sim/e2e/scripts/run.ts | 26 +- apps/sim/e2e/scripts/seed-world.ts | 37 +- .../authorization/contract-integrity.spec.ts | 6 +- .../e2e/settings/authorization/contracts.ts | 12 +- apps/sim/e2e/settings/navigation/contracts.ts | 12 +- .../e2e/settings/persona-contracts.spec.ts | 14 + apps/sim/e2e/settings/personas.ts | 19 +- .../sim/e2e/settings/workflows/people.spec.ts | 5 + apps/sim/e2e/settings/workflows/sso.spec.ts | 5 + .../e2e/settings/workflows/workflow-test.ts | 4 +- .../e2e/support/auth-capture-diagnostics.ts | 33 ++ apps/sim/playwright.config.ts | 7 +- bun.lock | 1 + docker-compose.prod.yml | 4 + helm/sim/templates/deployment-app.yaml | 19 ++ .../tests/sso-migration-preflight_test.yaml | 54 +++ helm/sim/values.yaml | 12 + packages/db/package.json | 1 + packages/db/scripts/audit-sso-providers.ts | 100 +----- packages/db/scripts/migrate.ts | 93 +++++ packages/db/scripts/rehearse-sso-migration.ts | 319 ++++++++++++++++++ packages/db/sso-provider-audit.test.ts | 174 ++++++++++ packages/db/sso-provider-audit.ts | 225 ++++++++++++ 38 files changed, 1656 insertions(+), 238 deletions(-) create mode 100644 apps/docs/lib/search-tool-schema.test.ts create mode 100644 apps/docs/lib/search-tool-schema.ts create mode 100644 apps/sim/e2e/STABILIZATION.md create mode 100644 apps/sim/e2e/support/auth-capture-diagnostics.ts create mode 100644 helm/sim/tests/sso-migration-preflight_test.yaml create mode 100644 packages/db/scripts/rehearse-sso-migration.ts create mode 100644 packages/db/sso-provider-audit.test.ts create mode 100644 packages/db/sso-provider-audit.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88be4166929..af93f0161c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,11 +20,6 @@ on: branches: [main, staging, dev] pull_request: branches: [main, staging, dev, e2e/settings-playwright] - # Docs content and markdown don't affect the app build or images; push - # runs stay unfiltered because they feed the deploy pipeline. - paths-ignore: - - 'apps/docs/content/**' - - '**/*.md' concurrency: group: ci-${{ github.ref }} diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index ab177ed0c5d..71c1fbc00a7 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -26,6 +26,7 @@ jobs: name: Apply Database Migrations runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 45 + environment: ${{ inputs.environment }} steps: - name: Checkout code @@ -62,6 +63,10 @@ jobs: DATABASE_URL: ${{ inputs.environment == 'production' && secrets.DATABASE_URL || inputs.environment == 'staging' && secrets.STAGING_DATABASE_URL || inputs.environment == 'dev' && secrets.DEV_DATABASE_URL || '' }} MIGRATION_DATABASE_URL: ${{ inputs.environment == 'production' && secrets.MIGRATION_DATABASE_URL || inputs.environment == 'staging' && secrets.STAGING_MIGRATION_DATABASE_URL || '' }} ENVIRONMENT: ${{ inputs.environment }} + # One-shot migration 0266 inputs. Configure them independently in + # each protected GitHub environment before its first rollout. + SSO_PROVIDER_WRITES_QUIESCED: ${{ secrets.SSO_PROVIDER_WRITES_QUIESCED }} + SSO_AUDIT_APPROVED_PROVIDER_IDS: ${{ secrets.SSO_AUDIT_APPROVED_PROVIDER_IDS }} run: | if [ -z "$DATABASE_URL" ]; then echo "ERROR: no database URL secret resolved for environment '${ENVIRONMENT}'" >&2 diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 16168dc5990..37eea7d4aeb 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -261,10 +261,11 @@ jobs: run: bunx turbo run build --filter=sim settings-e2e: - name: Settings E2E (informational) - runs-on: blacksmith-8vcpu-ubuntu-2404 + name: Settings E2E + # The production Next build peaks above the free GitHub runner's memory + # ceiling. Keep the break-glass provider predicate aligned with Build App. + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'linux-x64-8-core' }} timeout-minutes: 45 - continue-on-error: true services: postgres: @@ -295,27 +296,36 @@ jobs: with: node-version: 22 - - name: Mount Bun cache (Sticky Disk) - uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + - name: Mount Bun cache + uses: ./.github/actions/cache-mount with: + provider: ${{ vars.CI_PROVIDER }} key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ~/.bun/install/cache - - name: Mount node_modules (Sticky Disk) - uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + - name: Mount node_modules + uses: ./.github/actions/cache-mount with: + provider: ${{ vars.CI_PROVIDER }} key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ./node_modules - - name: Mount Playwright browsers (Sticky Disk) - uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + - name: Mount Playwright browsers + uses: ./.github/actions/cache-mount with: + provider: ${{ vars.CI_PROVIDER }} key: ${{ github.repository }}-playwright-browsers-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ~/.cache/ms-playwright - name: Install dependencies run: bun install --frozen-lockfile + - name: Rehearse SSO migration hardening + working-directory: packages/db + env: + E2E_PG_ADMIN_URL: 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' + run: bun run db:rehearse-sso-migration + - name: Configure E2E hostnames run: echo "127.0.0.1 e2e.sim.ai mcp.e2e.sim.ai" | sudo tee -a /etc/hosts @@ -323,7 +333,7 @@ jobs: working-directory: apps/sim run: bun run test:e2e:install-browsers -- --with-deps - - name: Run settings E2E foundation + - name: Run complete settings E2E suite timeout-minutes: 40 working-directory: apps/sim env: diff --git a/apps/docs/app/api/chat/route.ts b/apps/docs/app/api/chat/route.ts index 2150e044980..658787f8918 100644 --- a/apps/docs/app/api/chat/route.ts +++ b/apps/docs/app/api/chat/route.ts @@ -1,9 +1,9 @@ import { openai } from '@ai-sdk/openai' import { convertToModelMessages, stepCountIs, streamText, tool, type UIMessage } from 'ai' import { sql } from 'drizzle-orm' -import { z } from 'zod' import { db, docsEmbeddings } from '@/lib/db' import { generateSearchEmbedding } from '@/lib/embeddings' +import { searchDocsInputSchema } from '@/lib/search-tool-schema' export const runtime = 'nodejs' export const maxDuration = 30 @@ -332,9 +332,7 @@ export async function POST(req: Request) { searchDocs: tool({ description: 'Search the Sim documentation for relevant content. Use this before answering any question about Sim.', - inputSchema: z.object({ - query: z.string().describe('A focused natural-language search query.'), - }), + inputSchema: searchDocsInputSchema, execute: async ({ query }) => searchDocs(query, locale), }), }, diff --git a/apps/docs/content/docs/en/platform/enterprise/sso.mdx b/apps/docs/content/docs/en/platform/enterprise/sso.mdx index 112ec7f28ef..bb22c2fdd62 100644 --- a/apps/docs/content/docs/en/platform/enterprise/sso.mdx +++ b/apps/docs/content/docs/en/platform/enterprise/sso.mdx @@ -281,8 +281,8 @@ Self-hosted deployments use environment variables instead of the billing/plan ch SSO_ENABLED=true NEXT_PUBLIC_SSO_ENABLED=true -# Enable only after the SSO schema migration, provider audit, and approved -# domain-verification backfill described below. +# Enable only after the one-shot pre-migration audit, schema migration, +# separately approved domain-ownership backfill, and live TXT check below. SSO_DOMAIN_VERIFICATION_ENABLED=false # Required if you want users auto-added to your organization on first SSO sign-in @@ -307,22 +307,48 @@ SSO_TRUSTED_PROVIDER_IDS=custom-oidc,partner-saml - For upgrades, leave domain verification disabled until the database migration - has landed and - `SSO_AUDIT_APPROVED_PROVIDER_IDS=reviewed-provider-ids bun run --cwd packages/db db:audit-sso-providers` - passes. The approval list records the providers whose existing account links - and active sessions operators chose to retain; omit a provider until its - links are migrated or removed and its sessions are revoked. The audit reports - linked-user and active-session counts and requires one owner organization and - one registrable domain per provider. - Remediate legacy user-scoped, public-suffix, duplicate, or overlapping rows; - explicitly backfill `domain_verified` only for domains your operators have - verified; quiesce SSO writes and rerun the audit immediately before enabling - the flag. Internal pseudo-domains are not eligible for verified-domain - enforcement. Existing internal-domain providers can remain unchanged while - the flag is off, but must move to a registrable domain before activation. + Migration 0266 runs a one-shot legacy-provider audit before changing the + schema. If the legacy provider table exists, disable SSO/provider mutations + and keep them disabled through migration, then set + `SSO_PROVIDER_WRITES_QUIESCED=true` even when the table is empty. Provide + `SSO_AUDIT_APPROVED_PROVIDER_IDS` only for providers whose existing account + links and active sessions operators chose to retain. Omit a provider until + its links are migrated or removed and its sessions are revoked. + Remediate user-scoped, public-suffix, unknown-suffix, duplicate, or + overlapping rows before retrying. The quiescence flag is an acknowledgement, + not a technical lock against an older running application. + + Account/session approval does not prove domain ownership. After migration, + transactionally backfill `domain_verified` only for domains supported by + independently reviewed ownership evidence, read the values back, and keep + unknown rows false. Complete a live TXT verification before enabling + `SSO_DOMAIN_VERIFICATION_ENABLED`. Internal pseudo-domains are not eligible + for verified-domain enforcement. +For the first 0266 upgrade with Docker Compose, set the one-shot migration +inputs in the Compose environment while writes are actually disabled: + +```bash +export SSO_PROVIDER_WRITES_QUIESCED=true +export SSO_AUDIT_APPROVED_PROVIDER_IDS=reviewed-provider-ids +docker compose -f docker-compose.prod.yml up +``` + +For Helm, use inline migration values or a dedicated Secret: + +```yaml +migrations: + ssoPreflight: + providerWritesQuiesced: "true" + auditApprovedProviderIds: "reviewed-provider-ids" + # Or set existingSecret to a Secret containing + # SSO_PROVIDER_WRITES_QUIESCED and SSO_AUDIT_APPROVED_PROVIDER_IDS. +``` + +Remove the one-shot values after 0266 is journaled. Do not leave provider writes +disabled or interpret the flag itself as quiescence. + You can register providers through the **Settings UI** (same as cloud) or by running the registration script directly against your database. ### Script-based registration diff --git a/apps/docs/lib/search-tool-schema.test.ts b/apps/docs/lib/search-tool-schema.test.ts new file mode 100644 index 00000000000..a1c086bbc6e --- /dev/null +++ b/apps/docs/lib/search-tool-schema.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest' +import { validateSearchDocsInput } from './search-tool-schema' + +describe('search documentation tool schema', () => { + it('accepts exactly one string query', () => { + expect(validateSearchDocsInput({ query: 'SSO setup' })).toEqual({ + success: true, + value: { query: 'SSO setup' }, + }) + }) + + it.each([ + ['null', null], + ['array', []], + ['missing query', {}], + ['non-string query', { query: 7 }], + ['extra properties', { query: 'SSO setup', unexpected: true }], + ])('rejects malformed input: %s', (_label, value) => { + expect(validateSearchDocsInput(value)).toMatchObject({ + success: false, + error: expect.any(TypeError), + }) + }) +}) diff --git a/apps/docs/lib/search-tool-schema.ts b/apps/docs/lib/search-tool-schema.ts new file mode 100644 index 00000000000..7a6034f5e17 --- /dev/null +++ b/apps/docs/lib/search-tool-schema.ts @@ -0,0 +1,41 @@ +import { jsonSchema } from 'ai' + +interface SearchDocsInput { + query: string +} + +export function validateSearchDocsInput(value: unknown) { + if ( + typeof value !== 'object' || + value === null || + Array.isArray(value) || + Object.keys(value).length !== 1 || + !('query' in value) || + typeof value.query !== 'string' + ) { + return { + success: false as const, + error: new TypeError('Search documentation input must contain only a string query'), + } + } + + return { + success: true as const, + value: { query: value.query }, + } +} + +export const searchDocsInputSchema = jsonSchema( + { + type: 'object', + properties: { + query: { + type: 'string', + description: 'A focused natural-language search query.', + }, + }, + required: ['query'], + additionalProperties: false, + }, + { validate: validateSearchDocsInput } +) diff --git a/apps/docs/package.json b/apps/docs/package.json index 41a7490421c..052cddcb108 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -8,6 +8,7 @@ "build": "fumadocs-mdx && NODE_OPTIONS='--max-old-space-size=8192' next build", "start": "next start", "postinstall": "fumadocs-mdx", + "test": "vitest run", "type-check": "tsc --noEmit", "lint": "biome check --write --unsafe .", "lint:check": "biome check .", @@ -53,6 +54,7 @@ "@typescript/native-preview": "7.0.0-dev.20260707.2", "postcss": "^8.5.3", "tailwindcss": "^4.0.12", - "typescript": "^7.0.2" + "typescript": "^7.0.2", + "vitest": "^4.1.0" } } diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md index 6737e1e57d8..8d78e235a09 100644 --- a/apps/sim/e2e/README.md +++ b/apps/sim/e2e/README.md @@ -40,7 +40,7 @@ per-run pgvector database. bun run test:e2e:install-browsers ``` -## Run the foundation +## Run the complete suite From `apps/sim`: @@ -235,12 +235,13 @@ Member rows are accessible groups named by email, nested under count-free and Access Control expose fail-closed loading/error/ready states; tests do not locate mutable count text. -Traces and video are disabled for the workflows project because the Teammates -API necessarily returns pending invitation tokens to the application. Tests -never parse that token-bearing response, invoke Copy invite link, or issue an -extra workspace-invitation list request. Token-free organization rosters are -used for safe invitation IDs, kinds, roles, and grants. Failure-only screenshots -remain enabled because invitation tokens are never rendered. +The people workflow disables traces because the Teammates API necessarily +returns pending invitation tokens to the application. Tests never parse that +token-bearing response, invoke Copy invite link, or issue an extra +workspace-invitation list request. Token-free organization rosters are used for +safe invitation IDs, kinds, roles, and grants. Failure-only screenshots remain +enabled because invitation tokens are never rendered. Other workflow files keep +failure traces unless they carry a documented sensitive boundary. Invitations exercise the real mail-rendering path. The hermetic app and build environments expose none of the Resend, SES, SMTP, Azure ACS, or Gmail provider @@ -273,21 +274,48 @@ bodies must never be attached to reports or copied into logs. Better Auth native domain verification is controlled independently by `SSO_DOMAIN_VERIFICATION_ENABLED` and defaults off for upgrades and self-hosted -deployments. It must not be enabled in a hosted environment until the SSO schema -migration has landed separately and -`SSO_AUDIT_APPROVED_PROVIDER_IDS=reviewed-provider-ids bun run --cwd packages/db db:audit-sso-providers` -passes. The command validates every provider row and reports linked-user and -active-session counts; the approval list records each operator decision to -retain those links/sessions. Unapproved links must be migrated or removed and -their sessions revoked before the provider is approved. Only explicitly approved existing providers -may be backfilled as verified; unknown rows remain inactive, with link/session -disposition documented before enabling the flag. SSO writes must be quiesced -across that interval and the audit rerun immediately before the flag changes. -The hermetic profile sets it to true. Legacy user-scoped -provider rows must be assigned to an audited organization or removed before the -migration; its check constraints and preflight reject them. Providers with -linked Better Auth accounts cannot change issuer/domain or be deleted until an -operator completes the documented account-link and session migration. +deployments. Migration 0266 has a one-shot legacy-data gate: when the migration +is not yet journaled and the legacy provider table exists, the migration +process acquires the SSO mutation advisory lock, requires +`SSO_PROVIDER_WRITES_QUIESCED=true` even if that table is empty, and runs the +public-suffix/account-link audit before applying schema changes. +The flag is an operator acknowledgement, not a lock: the old deployment must +actually have SSO/provider mutation traffic disabled from audit through +migration because it may not participate in the advisory-lock protocol. + +Configure `SSO_PROVIDER_WRITES_QUIESCED` and +`SSO_AUDIT_APPROVED_PROVIDER_IDS` independently in the protected staging and +production GitHub environments whose databases predate 0266. The development +migration job currently uses `db:push`, not the versioned `migrate.ts` path, so +it does not execute this one-shot gate; audit, repair, or reset persistent +legacy development data before that schema push. The approval list records +only explicit retain-or-migrate decisions for existing Better Auth account +links and sessions. Unapproved links must be migrated or removed and their +sessions revoked. Legacy user-scoped providers must be assigned to an audited +organization or removed. The blocking gate stops after 0266 is journaled; +later SSO audits are operator reports and do not block unrelated migrations +merely because healthy providers have linked users or the public suffix list +changed. + +Domain ownership is a separate decision. After 0266, backfill only providers +whose ownership evidence was independently approved, perform the update +transactionally, and read it back. Keep unknown rows at `domain_verified=false`. +Successful live TXT verification is still required before enabling +`SSO_DOMAIN_VERIFICATION_ENABLED`; neither linked-account approval nor migration +success proves domain ownership. The hermetic profile sets verification to true +only for its isolated world. Providers with linked Better Auth accounts cannot +change issuer/domain or be deleted until an operator completes the documented +account-link and session migration. + +Rehearse the legacy audit, empty-table quiescence block, linked-account +approval, migration constraints/indexes/default, and post-0266 short-circuit +against the local pgvector instance: + +```bash +E2E_PG_ADMIN_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres \ + bun run --cwd packages/db db:rehearse-sso-migration +``` + Better Auth 1.6.13 does not honor explicit `requestSignUp` for SAML callbacks, so implicit signup remains enabled only behind the verified-domain gate to preserve intended JIT organization provisioning; @@ -378,6 +406,80 @@ the dedicated two-worker cross-world isolation project. Project dependencies serialize navigation, authorization, credentials, workflows, and persona contracts before the isolation project opens its two-worker pool. +## Stability gate and required CI + +The Step 7 source discovers 256 tests: 131 navigation/foundation, 84 +authorization, 13 credentials, 11 workflows, 15 persona contracts, and 2 +two-worker isolation tests. + +Retries remain fixed at zero while the suite is stabilized. The final gate is +five Playwright executions per test against one healthy production stack, not +five builds or five CI workflows. For an unscoped `--repeat-each`, the +orchestrator runs each canonical project sequentially with dependencies +suppressed after it has established their canonical order. Playwright otherwise +repeats the terminal project but runs dependency projects only once: + +```bash +rm -rf e2e/.cache/builds +bun run test:e2e -- --reuse-build +bun run test:e2e -- --reuse-build --repeat-each=5 +``` + +Final Step 7 local evidence (2026-07-23): + +- The clean cache-miss chain stored verified build + `f4a8f018238f3c0a27a10f8ec7a0e656c9d11acdd10024777e2b4206519dcbef` + and passed all 256 tests in 7 minutes 39 seconds end to end. +- The cache-hit two-repeat qualification executed 512 tests across all six + projects and passed in 9 minutes 4 seconds. +- The definitive cache-hit five-repeat gate executed 655 navigation, 420 + authorization, 65 credentials, 55 workflow, 75 persona-contract, and 10 + isolation tests: 1,280 retry-free executions in 20 minutes 39 seconds. +- Both repeated chains used one guarded database/seed/auth-capture/app/realtime + lifecycle. Their leak markers were written only after safe scanning; Stripe + and MCP fake logs contained no unexpected requests. + +The first command must record a verified cache miss, build, and store. Make no +tracked or untracked source change under the hashed app/package trees before +the second command; it must hit the same source/profile/`BUILD_ID`, then create +a new guarded database, seed/auth state, and app/realtime boot for all five +repetitions. A failure invalidates the gate: fix the cause, rerun focused proof, +then restart the complete gate from repetition one. Do not substitute retries. + +Ordinary PR CI runs the complete suite once. `Settings E2E` is a blocking job +on the same Blacksmith/GitHub provider switch as the production app build; the +GitHub fallback uses the paid high-memory runner. Pull-request workflows do not +ignore Markdown or docs-only changes because a missing required context would +leave those PRs permanently blocked. This intentionally spends the full CI +cost on every PR to a protected target. Failure diagnostics upload only after +the leak-scan marker is present. + +If retries are ever introduced after stabilization, the only sanctioned policy +is `retries: 1` together with Playwright `failOnFlakyTests: true`. The retry may +collect diagnostics and classify the flake, but the required check must still +fail. + +The temporary `e2e/settings-playwright` PR target remains until the Step 7 +branch has merged into it. Remove that target only in the umbrella PR to +`staging`, observe the exact `Test and Build / Settings E2E` context on the +latest staging and main PR commits, and only then configure it as required for +each protected branch. Those branch-protection changes are manual repository +operations. + +## Adding another browser suite + +Create feature tests under `e2e//` and compose their scenario from the +existing `E2EWorld` factories. Reuse a deployment profile, Better Auth storage +states, external fakes, browser-network guard, cleanup registry, diagnostics, +and one-shot lifecycle instead of copying the harness. + +Join an existing Playwright project only when the new suite has the same +deployment profile, process topology, isolation requirements, mutation +coupling, worker budget, artifact policy, and CI cadence. Otherwise add a +separate project/job with an explicit worker/shard and diagnostics policy. +Self-hosted, billing-disabled, and cross-browser profiles remain later nightly +coverage; they are not hidden variants of the hosted Chromium acceptance gate. + ## Diagnostics - HTML report: `playwright-report/` diff --git a/apps/sim/e2e/STABILIZATION.md b/apps/sim/e2e/STABILIZATION.md new file mode 100644 index 00000000000..625bfcc66ce --- /dev/null +++ b/apps/sim/e2e/STABILIZATION.md @@ -0,0 +1,157 @@ +# Settings E2E stabilization record + +This record is the non-waivable acceptance crosswalk for the first hosted, +billing-enabled Chromium milestone. It complements the operational runbook in +`README.md`; it does not replace the literal contracts or their assertions. + +## Review baseline + +- Integration head before Step 7: `1a631d9cd27cb917dbf131f178c934d5cf2250db` +- Reviewed staging reference: `26e4c308887de566144cdedaee223ce1c7f6c050` +- Inventory at that boundary: 194 changed files, 59,367 insertions, and 1,904 + deletions. Generated Drizzle snapshots are classified as generated review + material; migration SQL, schema declarations, and preflight code remain + hand-reviewed. +- Current Step 7 source discovers 256 tests: 131 navigation/foundation, 84 + authorization, 13 credentials, 11 workflows, 15 persona contracts, and 2 + isolation tests. The pre-Step-7 baseline was 254; Step 7 adds two foundation + safety policies for selective trace suppression and auth-screenshot clearing. +- Regenerate the exact inventory instead of copying a stale list: + + ```bash + git diff --name-status \ + 26e4c308887de566144cdedaee223ce1c7f6c050...1a631d9cd27cb917dbf131f178c934d5cf2250db + ``` + +The final Step 7 SHA, timings, and gate evidence are recorded in the PR after +the source is frozen. Raw reports, auth states, and secret-bearing artifacts +are never committed. + +## Architecture proof + +- `playwright.config.ts` fixes Chromium, zero retries, project dependencies, and + worker budgets. `e2e/scripts/run.ts` is the only test entry point. +- `e2e/support/deployment-profile.ts` gives build, app, realtime, migration, + seed, auth capture, and Playwright separate allowlisted environments. +- `e2e/support/database.ts` creates and drops only guarded per-run + `sim_e2e_*` databases. Migrations run before any scenario is seeded. +- `e2e/fixtures` and `e2e/settings/personas.ts` define deterministic users, + organizations, memberships, subscriptions, workspaces, grants, permission + groups, and 14 named personas plus an independent isolation twin. Scenario + validation rejects duplicate organization membership. +- `e2e/scripts/capture-auth-states.ts` signs in through the real Better Auth UI + and writes mode-0600 storage states. Playwright receives no password, + database URL, or admin key. +- Same-origin Sim APIs stay real. Stripe and MCP are strict loopback fakes; + mail uses the production mock-success path with provider credentials absent. + Browser HTTP(S) egress is allowlisted. Provider log scans are residual-risk + tripwires, not proof of an OS-level server firewall. +- `e2e/support/leak-canary.ts` scans eligible diagnostics and fails closed. + Auth/private/home roots are excluded from uploads, and CI uploads only after + the successful scan marker exists. + +## Navigation contract + +The explicit datasets in `e2e/settings/navigation/contracts.ts` are independent +of `SETTINGS_SECTION_REGISTRY`: 37 canonical sections, 21 special route cases, +and 11 representative visibility cases. + +- `canonical-navigation.spec.ts` iterates every literal account, organization, + and workspace section, clicks its semantic sidebar item, asserts the exact + pathname, heading, description, active item, dynamic API response when + required, and semantic readiness without an error state. +- `persona-visibility.spec.ts` asserts complete visible sets and important + hidden items for personal, organization, workspace, restricted, and platform + personas. +- `route-cases.spec.ts` owns account/organization/workspace default redirects, + aliases, legacy redirects, unavailable states, unknown sections, and direct + non-member/member outcomes. +- `smoke/unauthenticated.spec.ts` owns all three login redirects. +- `history.spec.ts` owns browser Back, app Back, direct-entry fallback, and + account/organization return destinations. +- `navigation/contract-integrity.spec.ts` rejects duplicate identities, + incoherent drivers, and accidental coupling between the scenario and + acceptance datasets. + +No navigation requirement may be deferred. Product route/copy changes require +an explicit product decision and a paired update to the independent contract. + +## Authorization and entitlement contract + +`e2e/settings/authorization/contracts.ts` owns literal access outcomes and +semantic mutation probes: 47 direct access cases and 31 mutation-control cases. +Its integrity spec requires every declared gate axis. + +- Read, write, and admin workspace personas cover view/mutation boundaries for + Secrets, Custom tools, MCP tools, workflow MCP servers, Recently deleted, + Teammates, BYOK, API keys, Inbox, Forks, and Custom blocks. +- Permission-group visibility and direct-route cases deny Secrets, API keys, + Inbox, MCP tools, and Custom tools. +- Organization member cases prove Members visibility without management and + direct denial of admin-only sections. Admin cases prove eligible controls. +- Lapsed/free and entitled organizations provide negative and positive + Enterprise gates. Inbox has both locked-upgrade and enabled-Max proof. +- Account and workspace Admin/Mothership have platform-admin positives and + non-platform direct-route negatives. +- `unsaved-changes.spec.ts` covers Keep editing and Discard changes for sidebar, + app Back, browser history, and native `beforeunload`. Credential detail tests + add in-app and popstate guards. + +Existing Step 3 proof IDs are referenced rather than rerun; integrity tests +fail if a reference disappears. Every mandatory gate needs allowed and denied +browser proof, including direct URL access. + +## Critical workflows + +All workflows use the real UI and same-origin production contracts, register +cleanup before mutation, use unique resources, and verify observable API or +database-backed state. + +1. Secrets: `credentials/secrets.spec.ts` covers personal creation, workspace + create/edit/discard/save/delete, detail routes, read-only behavior, + cross-workspace binding rejection, and permission-group denial. +2. API keys: `credentials/api-keys.spec.ts` covers personal/workspace + create/revoke, write-user denial, and workspace personal-key policy. +3. People: `workflows/people.spec.ts` covers workspace and organization invites, + role/grant changes, revocation, existing-member permission/role changes, and + removal with exact baseline restoration. +4. Access control: `workflows/access-control.spec.ts` creates and assigns a + permission group, proves five restrictions in a fresh persona context, then + deletes it and proves restoration. +5. SSO: `workflows/sso.spec.ts` creates, edits/discards/saves, requests + verification instructions, and deletes a pending SAML provider without an + IdP login or provider egress. +6. Data retention: `workflows/data-retention.spec.ts` edits defaults, adds and + removes a workspace override, restores the exact snapshot, and is followed + by an orchestrator database probe. +7. MCP tools: `workflows/mcp.spec.ts` proves a denied domain, then + test/connect/create/discover/edit/reprobe/delete against the strict + multi-session loopback fake. + +`credentials/contract-integrity.spec.ts` and +`workflows/contract-integrity.spec.ts` keep these workflows tied to durable +navigation, persona, authorization, and lifecycle proof IDs. + +## Quality and diagnostics gates + +- No raw Playwright invocation, retries, worker override, arbitrary browser + sleep, CSS-class assertion, `test.only`, shared mutable fixture, or + unexplained skip is allowed. +- Credentials suppress traces, screenshots, videos, and authored attachments. + People and SSO suppress network-bearing traces. Safe workflows retain + failure traces and screenshots. Every exception is security-motivated and + remains covered by the leak scan. +- Existing Vitest navigation, billing, SSO, route, permission, and + business-rule tests remain required. +- The ordinary required CI job runs the full chain once. The final stability + gate runs `--repeat-each=5` with retries zero against one verified production + build and one healthy app/realtime boot, as specified by roadmap Step 7. + +## Explicit milestone boundaries + +The only deferred product boundaries are those authorized by the objective: +real payment execution, real email delivery, live SSO login/TXT verification, +destructive fork synchronization, and later self-hosted, billing-disabled, and +cross-browser nightly profiles. Production SSO quiescence, migration approval, +domain-ownership backfill/read-back, live TXT proof, and branch-protection +changes are manual operational gates rather than missing browser coverage. diff --git a/apps/sim/e2e/fixtures/validate-scenario.ts b/apps/sim/e2e/fixtures/validate-scenario.ts index a6414b04755..1bd7efddf4f 100644 --- a/apps/sim/e2e/fixtures/validate-scenario.ts +++ b/apps/sim/e2e/fixtures/validate-scenario.ts @@ -665,7 +665,18 @@ function validatePersonaWorkspaceExpectation( subscriptionsByKey: ReadonlyMap, issues: string[] ): void { - const actual = deriveWorkspaceAccess(definition, user.key, workspace) + const declaredSubscription = workspace.subscriptionKey + ? subscriptionsByKey.get(workspace.subscriptionKey) + : undefined + const entitledSubscription = [...subscriptionsByKey.values()].find( + (candidate) => + isEntitledSubscription(candidate) && + billingReferenceKey(candidate) === payerReferenceKey(workspace) + ) + const organizationWillDetach = Boolean( + workspace.organizationKey && declaredSubscription?.status === 'lapsed' && !entitledSubscription + ) + const actual = deriveWorkspaceAccess(definition, user.key, workspace, !organizationWillDetach) if (expected.access !== actual.access || expected.roleSource !== actual.roleSource) { issues.push( `persona "${personaKey}" has incoherent access/roleSource for workspace "${workspace.key}"` @@ -681,24 +692,13 @@ function validatePersonaWorkspaceExpectation( `persona "${personaKey}" owner/admin expectation for "${workspace.key}" is below admin` ) } - const declaredSubscription = workspace.subscriptionKey - ? subscriptionsByKey.get(workspace.subscriptionKey) - : undefined - const entitledSubscription = [...subscriptionsByKey.values()].find( - (candidate) => - isEntitledSubscription(candidate) && - billingReferenceKey(candidate) === payerReferenceKey(workspace) - ) const actualPlan = entitledSubscription?.plan ?? 'free' const actualMembership = actual.isOwner ? 'owner' : actual.organizationRole ? 'member' : 'external' - const actualPayerScope = - workspace.organizationKey && declaredSubscription?.status === 'lapsed' - ? 'user' - : workspace.payer.kind + const actualPayerScope = organizationWillDetach ? 'user' : workspace.payer.kind if ( expected.hostContext.isOwner !== actual.isOwner || expected.hostContext.hostMembership !== actualMembership || @@ -714,7 +714,8 @@ function validatePersonaWorkspaceExpectation( function deriveWorkspaceAccess( definition: ScenarioDefinition, userKey: ResourceKey, - workspace: ScenarioWorkspace + workspace: ScenarioWorkspace, + includeOrganizationMembership = true ): { access: ExpectedWorkspaceAccess roleSource: 'owner' | 'explicit' | 'org-admin' | 'none' @@ -722,9 +723,10 @@ function deriveWorkspaceAccess( organizationRole?: OrganizationRole } { const isOwner = workspace.ownerUserKey === userKey - const organizationRole = workspace.organizationKey - ? membershipFor(definition, workspace.organizationKey, userKey)?.role - : undefined + const organizationRole = + includeOrganizationMembership && workspace.organizationKey + ? membershipFor(definition, workspace.organizationKey, userKey)?.role + : undefined if (isOwner) return { access: 'admin', roleSource: 'owner', isOwner, organizationRole } if (organizationRole === 'owner' || organizationRole === 'admin') { return { access: 'admin', roleSource: 'org-admin', isOwner, organizationRole } diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index 34e67e000f9..5f73a29e800 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -3,8 +3,9 @@ import { createServer } from 'node:net' import os from 'node:os' import path from 'node:path' import { loadEnvConfig } from '@next/env' -import { expect, test } from '@playwright/test' -import { parseRunOptions } from '../scripts/options' +import { expect, type Page, test } from '@playwright/test' +import { buildPlaywrightInvocations, parseRunOptions } from '../scripts/options' +import { captureSafeAuthFailureScreenshot } from '../support/auth-capture-diagnostics' import { assertLoopbackPostgresUrl, assertSafeDatabaseName, @@ -190,6 +191,71 @@ test.describe('foundation safety guards', () => { } }) + test('token-bearing workflow files explicitly disable network traces', () => { + for (const relativePath of [ + 'e2e/settings/workflows/people.spec.ts', + 'e2e/settings/workflows/sso.spec.ts', + ]) { + const source = readFileSync(path.join(process.cwd(), relativePath), 'utf8') + expect(source).toContain("test.use({ trace: 'off' })") + } + }) + + test('auth diagnostics prove native password fields safe before screenshots', async () => { + const submittedPassword = 'synthetic-password' + const runCase = async ({ + values, + fillFails = false, + clearPersists = false, + }: { + values: string[] + fillFails?: boolean + clearPersists?: boolean + }) => { + const currentValues = [...values] + let screenshots = 0 + const page = { + locator: () => ({ + count: async () => currentValues.length, + nth: (index: number) => ({ + fill: async (value: string) => { + if (fillFails) throw new Error('page closed') + if (!clearPersists) currentValues[index] = value + }, + inputValue: async () => currentValues[index], + }), + }), + screenshot: async () => { + screenshots += 1 + }, + } as unknown as Page + const safe = await captureSafeAuthFailureScreenshot( + page, + '/tmp/auth-failure.png', + submittedPassword + ) + return { safe, screenshots } + } + + await expect(runCase({ values: [] })).resolves.toEqual({ safe: true, screenshots: 1 }) + await expect(runCase({ values: [submittedPassword] })).resolves.toEqual({ + safe: true, + screenshots: 1, + }) + await expect(runCase({ values: ['email@example.com', submittedPassword] })).resolves.toEqual({ + safe: true, + screenshots: 1, + }) + await expect(runCase({ values: [submittedPassword], fillFails: true })).resolves.toEqual({ + safe: false, + screenshots: 0, + }) + await expect(runCase({ values: [submittedPassword], clearPersists: true })).resolves.toEqual({ + safe: false, + screenshots: 0, + }) + }) + test('database guards reject shared or remote targets', () => { expect(() => assertSafeDatabaseName('simstudio')).toThrow() expect(() => assertSafeDatabaseName('sim_e2e_valid_run')).not.toThrow() @@ -290,6 +356,15 @@ test.describe('foundation safety guards', () => { }) test('repeat-each can increase coverage without weakening orchestration', () => { + expect(buildPlaywrightInvocations(parseRunOptions(['--repeat-each=5']))).toEqual([ + ['--repeat-each=5', '--project=hosted-billing-chromium-navigation', '--no-deps'], + ['--repeat-each=5', '--project=hosted-billing-chromium-authorization', '--no-deps'], + ['--repeat-each=5', '--project=hosted-billing-chromium-credentials', '--no-deps'], + ['--repeat-each=5', '--project=hosted-billing-chromium-workflows', '--no-deps'], + ['--repeat-each=5', '--project=hosted-billing-chromium-personas', '--no-deps'], + ['--repeat-each=5', '--project=hosted-billing-chromium-persona-isolation', '--no-deps'], + ]) + expect(buildPlaywrightInvocations(parseRunOptions(['--repeat-each', '2']))).toHaveLength(6) expect( parseRunOptions( ['--project=hosted-billing-chromium-workflows', '--no-deps', '--repeat-each=2'], @@ -299,6 +374,20 @@ test.describe('foundation safety guards', () => { expect(() => parseRunOptions(['--project=hosted-billing-chromium-workflows', '--repeat-each']) ).toThrow(/requires a value/) + expect(() => + parseRunOptions([ + '--project=hosted-billing-chromium-navigation', + '--project=hosted-billing-chromium-authorization', + '--repeat-each=2', + ]) + ).toThrow(/at most one explicit project/) + for (const filteredRepeat of [ + ['--repeat-each=2', '--grep=workspace'], + ['--repeat-each=2', '-g', 'workspace'], + ['--repeat-each=2', 'e2e/settings'], + ]) { + expect(() => parseRunOptions(filteredRepeat)).toThrow(/complete suite without test filters/) + } }) test('Playwright CLI arguments cannot override orchestration invariants', () => { diff --git a/apps/sim/e2e/foundation/scenario-validation.spec.ts b/apps/sim/e2e/foundation/scenario-validation.spec.ts index dc008140606..b4c91caa479 100644 --- a/apps/sim/e2e/foundation/scenario-validation.spec.ts +++ b/apps/sim/e2e/foundation/scenario-validation.spec.ts @@ -81,10 +81,21 @@ test.describe('pure scenario validation', () => { seats: 4, enterprise: { seats: 4 }, }) - expect(workspaceExpectation(primary, 'freeOrganizationOwner').hostContext).toMatchObject({ - payerScope: 'user', - plan: 'free', + expect(workspaceExpectation(primary, 'freeOrganizationAdmin')).toMatchObject({ + access: 'admin', + roleSource: 'explicit', + hostContext: { + hostMembership: 'external', + payerScope: 'user', + plan: 'free', + isOwner: false, + }, }) + expect( + scenarios.primary.organizationMemberships.find( + ({ userKey }) => userKey === 'free-organization-admin' + )?.role + ).toBe('admin') expect(primary.subscriptionsByKey.get('lapsed-team-subscription')?.status).toBe('lapsed') const restricted = primary.personasByKey.get('permissionGroupRestricted') @@ -256,21 +267,26 @@ test.describe('pure scenario validation', () => { : workspace ) scenario.personas = scenario.personas.map((persona) => - persona.key === 'freeOrganizationOwner' + persona.key === 'freeOrganizationAdmin' ? { ...persona, - workspaces: persona.workspaces.map((expectation) => ({ - ...expectation, - hostContext: { - ...expectation.hostContext, - payerScope: 'organization', - plan: 'team_6000', - }, - })), + workspaces: persona.workspaces.map((workspace) => + workspace.workspaceKey === 'lapsed-organization-workspace' + ? { + ...workspace, + roleSource: 'org-admin' as const, + hostContext: { + ...workspace.hostContext, + hostMembership: 'member' as const, + payerScope: 'organization' as const, + plan: 'team_6000' as const, + }, + } + : workspace + ), } : persona ) - const resolved = validateScenario(scenario) if (lapsed.billingReference.kind !== 'organization') { throw new Error('Expected an organization subscription') diff --git a/apps/sim/e2e/scripts/capture-auth-states.ts b/apps/sim/e2e/scripts/capture-auth-states.ts index be4975a4fa2..272be24b67f 100644 --- a/apps/sim/e2e/scripts/capture-auth-states.ts +++ b/apps/sim/e2e/scripts/capture-auth-states.ts @@ -9,6 +9,7 @@ import { scenarioManifestSchema, writeJsonAtomic, } from '../fixtures/e2e-world' +import { captureSafeAuthFailureScreenshot } from '../support/auth-capture-diagnostics' import { installBrowserNetworkGuard } from '../support/browser-network' const baseUrl = requiredEnv('E2E_BASE_URL') @@ -97,16 +98,11 @@ async function main(): Promise { throw new Error(`Storage state permissions are not private for persona: ${personaKey}`) } } catch (error) { - await page - .getByRole('textbox', { name: 'Password' }) - .fill('') - .catch(() => {}) - await page - .screenshot({ - path: path.join(screenshotsDirectory, `${personaKey}.failure.png`), - fullPage: true, - }) - .catch(() => {}) + await captureSafeAuthFailureScreenshot( + page, + path.join(screenshotsDirectory, `${personaKey}.failure.png`), + login.password + ) failures.push(error) } try { diff --git a/apps/sim/e2e/scripts/options.ts b/apps/sim/e2e/scripts/options.ts index 08d70bad441..7d6bb75a43d 100644 --- a/apps/sim/e2e/scripts/options.ts +++ b/apps/sim/e2e/scripts/options.ts @@ -4,14 +4,15 @@ const CREDENTIALS_PROJECT = 'hosted-billing-chromium-credentials' const WORKFLOWS_PROJECT = 'hosted-billing-chromium-workflows' const PERSONAS_PROJECT = 'hosted-billing-chromium-personas' const PERSONA_ISOLATION_PROJECT = 'hosted-billing-chromium-persona-isolation' -const E2E_PROJECTS = new Set([ +export const E2E_PROJECT_NAMES = [ NAVIGATION_PROJECT, AUTHORIZATION_PROJECT, CREDENTIALS_PROJECT, WORKFLOWS_PROJECT, PERSONAS_PROJECT, PERSONA_ISOLATION_PROJECT, -]) +] as const +const E2E_PROJECTS = new Set(E2E_PROJECT_NAMES) const FORBIDDEN_OPTIONS = [ '--config', '-c', @@ -35,6 +36,7 @@ const SAFE_VALUE_OPTIONS = new Set(['--grep', '--grep-invert', '--repeat-each', export interface E2eRunOptions { playwrightArgs: string[] reuseBuild: boolean + repeatAllProjects?: boolean } export function parseRunOptions( @@ -97,7 +99,35 @@ export function parseRunOptions( ) } - return { playwrightArgs, reuseBuild } + const hasRepeatEach = hasOption(playwrightArgs, '--repeat-each') + if (hasRepeatEach && projects.length > 1) { + throw new Error('--repeat-each accepts at most one explicit project') + } + const repeatAllProjects = hasRepeatEach && projects.length === 0 + if ( + repeatAllProjects && + (hasOption(playwrightArgs, '--grep') || + hasOption(playwrightArgs, '--grep-invert') || + hasOption(playwrightArgs, '-g') || + getPositionalArgs(playwrightArgs).length > 0) + ) { + throw new Error('Unscoped --repeat-each must run the complete suite without test filters') + } + + return { + playwrightArgs, + reuseBuild, + ...(repeatAllProjects ? { repeatAllProjects: true } : {}), + } +} + +export function buildPlaywrightInvocations(options: E2eRunOptions): string[][] { + if (!options.repeatAllProjects) return [options.playwrightArgs] + return E2E_PROJECT_NAMES.map((project) => [ + ...options.playwrightArgs, + `--project=${project}`, + '--no-deps', + ]) } function assertSupportedPlaywrightArgs(args: string[]): void { @@ -131,3 +161,16 @@ function hasOption(args: string[], name: string): boolean { function getEqualsOptionValues(args: string[], name: string): string[] { return args.filter((arg) => arg.startsWith(`${name}=`)).map((arg) => arg.slice(name.length + 1)) } + +function getPositionalArgs(args: string[]): string[] { + const positional: string[] = [] + for (let index = 0; index < args.length; index += 1) { + const argument = args[index] + if (!argument.startsWith('-')) { + positional.push(argument) + continue + } + if (SAFE_VALUE_OPTIONS.has(argument)) index += 1 + } + return positional +} diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts index cf38b65badd..c935cdf9665 100644 --- a/apps/sim/e2e/scripts/run.ts +++ b/apps/sim/e2e/scripts/run.ts @@ -59,7 +59,7 @@ import { startApp, startRealtime, } from '../support/stack' -import { parseRunOptions } from './options' +import { buildPlaywrightInvocations, parseRunOptions } from './options' const DEFAULT_ADMIN_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' @@ -404,14 +404,22 @@ async function main(): Promise { markerDirectory, manifestPath ) - await runPlaywright( - { - nodeExecutable, - env: playwrightEnvironment, - logsDirectory, - }, - options.playwrightArgs - ) + const playwrightInvocations = buildPlaywrightInvocations(options) + for (const [index, playwrightArgs] of playwrightInvocations.entries()) { + if (playwrightInvocations.length > 1) { + console.log( + `Playwright repeat gate ${index + 1}/${playwrightInvocations.length}: ${playwrightArgs.find((argument) => argument.startsWith('--project='))}` + ) + } + await runPlaywright( + { + nodeExecutable, + env: playwrightEnvironment, + logsDirectory, + }, + playwrightArgs + ) + } const provisioning = await inspectFoundationUsers(runDatabase.url, runId) assertAuthenticatedSmokeEffectsIfPresent( stripeFake, diff --git a/apps/sim/e2e/scripts/seed-world.ts b/apps/sim/e2e/scripts/seed-world.ts index 5f9768c5db4..f12d7d503cc 100644 --- a/apps/sim/e2e/scripts/seed-world.ts +++ b/apps/sim/e2e/scripts/seed-world.ts @@ -410,20 +410,33 @@ async function recordProductionPermissionIds(world: E2EWorld): Promise { } async function assertTrustedWorldInvariants(world: E2EWorld): Promise { + const workspaceIds = [...world.records.workspaces.values()].map(({ id }) => id) + const persistedWorkspaceHosts = new Map( + ( + await db + .select({ id: workspace.id, organizationId: workspace.organizationId }) + .from(workspace) + .where(inArray(workspace.id, workspaceIds)) + ).map((row) => [row.id, row]) + ) + if (persistedWorkspaceHosts.size !== workspaceIds.length) { + throw new Error('Every scenario workspace must remain persisted') + } + for (const persona of world.scenario.definition.personas) { const userId = required(world.records.users, persona.userKey, 'invariant user').id for (const expectation of persona.workspaces) { if (expectation.access === 'none') continue - const workspaceDefinition = required( - world.scenario.workspacesByKey, - expectation.workspaceKey, - 'invariant workspace definition' - ) const workspaceId = required( world.records.workspaces, expectation.workspaceKey, 'invariant workspace' ).id + const persistedWorkspace = required( + persistedWorkspaceHosts, + workspaceId, + 'persisted invariant workspace' + ) if (expectation.roleSource === 'org-admin') { const explicitRows = await db .select({ id: permissions.id }) @@ -443,17 +456,17 @@ async function assertTrustedWorldInvariants(world: E2EWorld): Promise { } if ( expectation.hostContext.hostMembership === 'external' && - workspaceDefinition.organizationKey + persistedWorkspace.organizationId ) { - const organizationId = required( - world.records.organizations, - workspaceDefinition.organizationKey, - 'external host organization' - ).id const memberships = await db .select({ id: member.id }) .from(member) - .where(and(eq(member.userId, userId), eq(member.organizationId, organizationId))) + .where( + and( + eq(member.userId, userId), + eq(member.organizationId, persistedWorkspace.organizationId) + ) + ) if (memberships.length !== 0) { throw new Error( `External workspace persona unexpectedly received host membership: ${persona.key}` diff --git a/apps/sim/e2e/settings/authorization/contract-integrity.spec.ts b/apps/sim/e2e/settings/authorization/contract-integrity.spec.ts index a2543654c85..0e7d1133fb9 100644 --- a/apps/sim/e2e/settings/authorization/contract-integrity.spec.ts +++ b/apps/sim/e2e/settings/authorization/contract-integrity.spec.ts @@ -114,7 +114,7 @@ test('access contracts declare every required Step 4 gate axis', () => { expectSections( accessGateCases.filter( ({ caseId }) => - caseId.startsWith('organization-lapsed-owner-') && caseId.endsWith('-plan-denied') + caseId.startsWith('organization-lapsed-admin-') && caseId.endsWith('-plan-denied') ), ['access-control', 'audit-logs', 'sso', 'data-retention', 'data-drains', 'whitelabeling'] ) @@ -153,8 +153,8 @@ test('access contracts declare every required Step 4 gate axis', () => { 'account-personal-paid-billing', 'account-non-platform-admin-denied', 'account-non-platform-mothership-denied', - 'organization-lapsed-owner-members', - 'organization-lapsed-owner-billing', + 'organization-lapsed-admin-members', + 'organization-lapsed-admin-billing', 'organization-team-owner-billing', 'workspace-foreign-access-denied', 'workspace-external-admin-organization-denied', diff --git a/apps/sim/e2e/settings/authorization/contracts.ts b/apps/sim/e2e/settings/authorization/contracts.ts index 453f34b6a13..4d2f5b0962d 100644 --- a/apps/sim/e2e/settings/authorization/contracts.ts +++ b/apps/sim/e2e/settings/authorization/contracts.ts @@ -260,11 +260,11 @@ export const accessGateCases: readonly AccessGateCase[] = [ outcome: { kind: 'organization-unavailable', title: 'Settings unavailable' }, }, { - caseId: 'organization-lapsed-owner-members', + caseId: 'organization-lapsed-admin-members', plane: 'organization', pathTemplate: '/organization/{organizationId}/settings/members', driver: { - personaKey: 'freeOrganizationOwner', + personaKey: 'freeOrganizationAdmin', binding: { worldKey: 'settings-primary', resourceKind: 'organization', @@ -279,11 +279,11 @@ export const accessGateCases: readonly AccessGateCase[] = [ }, }, { - caseId: 'organization-lapsed-owner-billing', + caseId: 'organization-lapsed-admin-billing', plane: 'organization', pathTemplate: '/organization/{organizationId}/settings/billing', driver: { - personaKey: 'freeOrganizationOwner', + personaKey: 'freeOrganizationAdmin', binding: { worldKey: 'settings-primary', resourceKind: 'organization', @@ -330,11 +330,11 @@ export const accessGateCases: readonly AccessGateCase[] = [ ].map( ([section, label]) => ({ - caseId: `organization-lapsed-owner-${section}-plan-denied`, + caseId: `organization-lapsed-admin-${section}-plan-denied`, plane: 'organization', pathTemplate: `/organization/{organizationId}/settings/${section}`, driver: { - personaKey: 'freeOrganizationOwner', + personaKey: 'freeOrganizationAdmin', binding: { worldKey: 'settings-primary', resourceKind: 'organization', diff --git a/apps/sim/e2e/settings/navigation/contracts.ts b/apps/sim/e2e/settings/navigation/contracts.ts index 7935d8af6b0..3dd1ce17a12 100644 --- a/apps/sim/e2e/settings/navigation/contracts.ts +++ b/apps/sim/e2e/settings/navigation/contracts.ts @@ -113,7 +113,7 @@ const enterpriseOrganization = primaryOrganization( 'enterpriseOrganizationAdmin', 'enterprise-organization' ) -const lapsedOrganization = primaryOrganization('freeOrganizationOwner', 'lapsed-organization') +const lapsedOrganization = primaryOrganization('freeOrganizationAdmin', 'lapsed-organization') const foreignEnterpriseOrganization = primaryOrganization( 'personalFreeOwner', 'enterprise-organization' @@ -916,15 +916,14 @@ export const personaVisibilityCases = [ representativeLabel: 'Access control', }, { - caseId: 'workspace-free-organization-owner', + caseId: 'workspace-free-organization-admin', plane: 'workspace', - driver: primaryWorkspace('freeOrganizationOwner', 'lapsed-organization-workspace'), + driver: primaryWorkspace('freeOrganizationAdmin', 'lapsed-organization-workspace'), expectedVisibleSectionIds: [ 'general', 'secrets', 'custom-tools', 'mcp', - 'billing', 'teammates', 'apikeys', 'workflow-mcp-servers', @@ -938,7 +937,6 @@ export const personaVisibilityCases = [ 'Secrets', 'Custom tools', 'MCP tools', - 'Billing', 'Teammates', 'Sim API keys', 'MCP servers', @@ -947,8 +945,8 @@ export const personaVisibilityCases = [ 'Sim mailer', 'Recently deleted', ], - importantHiddenSectionIds: ['organization', 'access-control', 'admin'], - importantHiddenLabels: ['Organization', 'Access control', 'Admin'], + importantHiddenSectionIds: ['billing', 'organization', 'access-control', 'admin'], + importantHiddenLabels: ['Billing', 'Organization', 'Access control', 'Admin'], representativeSectionId: 'secrets', representativeLabel: 'Secrets', }, diff --git a/apps/sim/e2e/settings/persona-contracts.spec.ts b/apps/sim/e2e/settings/persona-contracts.spec.ts index a4bbddda3d5..99b4c4aa48d 100644 --- a/apps/sim/e2e/settings/persona-contracts.spec.ts +++ b/apps/sim/e2e/settings/persona-contracts.spec.ts @@ -48,6 +48,20 @@ for (const personaKey of SETTINGS_PERSONA_KEYS) { role: persona.expectedPlatformRole, }) expect(session.session?.activeOrganizationId ?? null).toBe(persona.expectedActiveOrganizationId) + if (personaKey === 'freeOrganizationAdmin') { + const organizationId = persona.expectedActiveOrganizationId + if (!organizationId) throw new Error('Free organization admin has no active organization') + const rosterResponse = await context.request.get( + `/api/organizations/${encodeURIComponent(organizationId)}/roster` + ) + expect(rosterResponse.status()).toBe(200) + const roster = (await rosterResponse.json()) as { + data?: { members?: Array<{ userId?: string; role?: string }> } + } + expect(roster.data?.members).toContainEqual( + expect.objectContaining({ userId: persona.userId, role: 'admin' }) + ) + } const settingsResponse = await context.request.get('/api/users/me/settings') expect(settingsResponse.status()).toBe(200) const userSettings = (await settingsResponse.json()) as { diff --git a/apps/sim/e2e/settings/personas.ts b/apps/sim/e2e/settings/personas.ts index d73859ae6c1..3ce083f8922 100644 --- a/apps/sim/e2e/settings/personas.ts +++ b/apps/sim/e2e/settings/personas.ts @@ -24,7 +24,7 @@ export const SETTINGS_PERSONA_KEYS = [ 'teamWorkflowMember', 'enterpriseOrganizationAdmin', 'enterpriseWorkflowMember', - 'freeOrganizationOwner', + 'freeOrganizationAdmin', 'permissionGroupRestricted', 'platformAdmin', ] as const @@ -60,6 +60,7 @@ export function createPrimarySettingsScenario(namespace: ScenarioNamespace): Sce 'enterprise-organization-admin', 'enterprise-workflow-member', 'free-organization-owner', + 'free-organization-admin', 'permission-group-restricted', 'platform-admin', ] as const @@ -104,6 +105,7 @@ export function createPrimarySettingsScenario(namespace: ScenarioNamespace): Sce membership('enterprise-organization', 'enterprise-workflow-member', 'member'), membership('enterprise-organization', 'permission-group-restricted', 'member'), membership('lapsed-organization', 'free-organization-owner', 'owner'), + membership('lapsed-organization', 'free-organization-admin', 'admin'), ] as const const subscriptions = [ @@ -150,7 +152,7 @@ export function createPrimarySettingsScenario(namespace: ScenarioNamespace): Sce plan: 'team_6000', status: 'lapsed', billingReference: { kind: 'organization', organizationKey: 'lapsed-organization' }, - seats: 1, + seats: 2, ...HOSTED_BILLING, }, ] as const @@ -216,6 +218,7 @@ export function createPrimarySettingsScenario(namespace: ScenarioNamespace): Sce grant('team-workspace', 'team-workflow-member', 'read'), grant('enterprise-workspace', 'permission-group-restricted', 'read'), grant('enterprise-workspace', 'enterprise-workflow-member', 'read'), + grant('lapsed-organization-workspace', 'free-organization-admin', 'admin'), ] as const const permissionGroups = [ @@ -315,8 +318,16 @@ export function createPrimarySettingsScenario(namespace: ScenarioNamespace): Sce false ), ]), - persona(namespace, 'freeOrganizationOwner', 'free-organization-owner', [ - expected('lapsed-organization-workspace', 'admin', 'owner', 'owner', 'user', 'free', true), + persona(namespace, 'freeOrganizationAdmin', 'free-organization-admin', [ + expected( + 'lapsed-organization-workspace', + 'admin', + 'explicit', + 'external', + 'user', + 'free', + false + ), ]), persona( namespace, diff --git a/apps/sim/e2e/settings/workflows/people.spec.ts b/apps/sim/e2e/settings/workflows/people.spec.ts index 2ba0a99852a..f430e20c8e2 100644 --- a/apps/sim/e2e/settings/workflows/people.spec.ts +++ b/apps/sim/e2e/settings/workflows/people.spec.ts @@ -17,6 +17,11 @@ import { } from './helpers' import { expect, test } from './workflow-test' +// Workspace invitation responses contain bearer tokens even though the tests +// never inspect them. Keep Playwright's network-bearing trace artifact off for +// this file; failure screenshots remain safe because tokens are never rendered. +test.use({ trace: 'off' }) + test('workspace invitation can be sent as Read and revoked without exposing its token', async ({ contextForPersona, personaManifest, diff --git a/apps/sim/e2e/settings/workflows/sso.spec.ts b/apps/sim/e2e/settings/workflows/sso.spec.ts index 286135e191f..8bd241f76d5 100644 --- a/apps/sim/e2e/settings/workflows/sso.spec.ts +++ b/apps/sim/e2e/settings/workflows/sso.spec.ts @@ -11,6 +11,11 @@ import { } from './helpers' import { expect, test } from './workflow-test' +// SSO registration and verification responses can contain certificates and +// verification values. The workflow deliberately proves behavior without +// retaining Playwright's network-bearing trace artifact. +test.use({ trace: 'off' }) + test('SAML provider lifecycle stays pending and uses scoped management APIs', async ({ contextForPersona, personaManifest, diff --git a/apps/sim/e2e/settings/workflows/workflow-test.ts b/apps/sim/e2e/settings/workflows/workflow-test.ts index 5a4e7457448..c4b58b76677 100644 --- a/apps/sim/e2e/settings/workflows/workflow-test.ts +++ b/apps/sim/e2e/settings/workflows/workflow-test.ts @@ -19,9 +19,9 @@ export { expect } from '@playwright/test' function assertWorkflowArtifactPolicy(testInfo: TestInfo): void { const { trace, screenshot, video } = testInfo.project.use - if (trace !== 'off' || screenshot !== 'only-on-failure' || video !== 'off') { + if (trace !== 'retain-on-failure' || screenshot !== 'only-on-failure' || video !== 'off') { throw new Error( - 'Settings workflows must disable trace/video while retaining failure-only screenshots' + 'Settings workflows must retain failure traces/screenshots and keep video disabled by default' ) } } diff --git a/apps/sim/e2e/support/auth-capture-diagnostics.ts b/apps/sim/e2e/support/auth-capture-diagnostics.ts new file mode 100644 index 00000000000..bd48064acf4 --- /dev/null +++ b/apps/sim/e2e/support/auth-capture-diagnostics.ts @@ -0,0 +1,33 @@ +import type { Page } from '@playwright/test' + +/** + * Captures an auth failure only after proving that no entered password remains + * in the login control. PNGs are intentionally outside the text/ZIP leak + * scanner, so an uncertain clear must suppress the artifact. + */ +export async function captureSafeAuthFailureScreenshot( + page: Page, + screenshotPath: string, + submittedPassword: string +): Promise { + const formFields = page.locator('input, textarea') + try { + const count = await formFields.count() + for (let index = 0; index < count; index += 1) { + const field = formFields.nth(index) + if ((await field.inputValue()).includes(submittedPassword)) await field.fill('') + } + for (let index = 0; index < count; index += 1) { + if ((await formFields.nth(index).inputValue()).includes(submittedPassword)) return false + } + } catch { + return false + } + + try { + await page.screenshot({ path: screenshotPath, fullPage: true }) + return true + } catch { + return false + } +} diff --git a/apps/sim/playwright.config.ts b/apps/sim/playwright.config.ts index 562373b940e..b3b59767d95 100644 --- a/apps/sim/playwright.config.ts +++ b/apps/sim/playwright.config.ts @@ -13,7 +13,7 @@ export default defineConfig({ testDir: './e2e', testMatch: '**/*.spec.ts', fullyParallel: true, - forbidOnly: isCI, + forbidOnly: true, retries: 0, workers: 2, timeout: 60_000, @@ -72,11 +72,6 @@ export default defineConfig({ dependencies: ['hosted-billing-chromium-credentials'], fullyParallel: false, workers: 1, - use: { - trace: 'off', - screenshot: 'only-on-failure', - video: 'off', - }, }, { name: 'hosted-billing-chromium-personas', diff --git a/bun.lock b/bun.lock index 29d04010627..43266b313fe 100644 --- a/bun.lock +++ b/bun.lock @@ -61,6 +61,7 @@ "postcss": "^8.5.3", "tailwindcss": "^4.0.12", "typescript": "^7.0.2", + "vitest": "^4.1.0", }, }, "apps/pii": { diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 67266fbc780..28790cb49f2 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -78,6 +78,10 @@ services: working_dir: /app/packages/db environment: - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} + # Required once for an established pre-0266 database, including an empty + # SSO provider table. Set true only while provider writes are quiesced. + - SSO_PROVIDER_WRITES_QUIESCED=${SSO_PROVIDER_WRITES_QUIESCED:-} + - SSO_AUDIT_APPROVED_PROVIDER_IDS=${SSO_AUDIT_APPROVED_PROVIDER_IDS:-} depends_on: db: condition: service_healthy diff --git a/helm/sim/templates/deployment-app.yaml b/helm/sim/templates/deployment-app.yaml index 77a3d792e7f..9c623dcdbc9 100644 --- a/helm/sim/templates/deployment-app.yaml +++ b/helm/sim/templates/deployment-app.yaml @@ -52,6 +52,25 @@ spec: cd /app/packages/db export DATABASE_URL="{{ include "sim.databaseUrl" . }}" bun run db:migrate + env: + - name: SSO_PROVIDER_WRITES_QUIESCED + {{- if .Values.migrations.ssoPreflight.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.migrations.ssoPreflight.existingSecret | quote }} + key: {{ .Values.migrations.ssoPreflight.providerWritesQuiescedKey | quote }} + {{- else }} + value: {{ .Values.migrations.ssoPreflight.providerWritesQuiesced | quote }} + {{- end }} + - name: SSO_AUDIT_APPROVED_PROVIDER_IDS + {{- if .Values.migrations.ssoPreflight.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.migrations.ssoPreflight.existingSecret | quote }} + key: {{ .Values.migrations.ssoPreflight.auditApprovedProviderIdsKey | quote }} + {{- else }} + value: {{ .Values.migrations.ssoPreflight.auditApprovedProviderIds | quote }} + {{- end }} envFrom: {{- if .Values.postgresql.enabled }} - secretRef: diff --git a/helm/sim/tests/sso-migration-preflight_test.yaml b/helm/sim/tests/sso-migration-preflight_test.yaml new file mode 100644 index 00000000000..10e58c27165 --- /dev/null +++ b/helm/sim/tests/sso-migration-preflight_test.yaml @@ -0,0 +1,54 @@ +suite: SSO migration preflight environment +release: + name: t + namespace: sim + +tests: + - it: passes inline one-shot migration values + template: deployment-app.yaml + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + migrations.ssoPreflight.providerWritesQuiesced: "true" + migrations.ssoPreflight.auditApprovedProviderIds: acme-saml + asserts: + - contains: + path: spec.template.spec.initContainers[0].env + content: + name: SSO_PROVIDER_WRITES_QUIESCED + value: "true" + - contains: + path: spec.template.spec.initContainers[0].env + content: + name: SSO_AUDIT_APPROVED_PROVIDER_IDS + value: acme-saml + + - it: supports a dedicated Secret for one-shot migration values + template: deployment-app.yaml + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + migrations.ssoPreflight.existingSecret: sso-migration-gate + asserts: + - contains: + path: spec.template.spec.initContainers[0].env + content: + name: SSO_PROVIDER_WRITES_QUIESCED + valueFrom: + secretKeyRef: + name: sso-migration-gate + key: SSO_PROVIDER_WRITES_QUIESCED + - contains: + path: spec.template.spec.initContainers[0].env + content: + name: SSO_AUDIT_APPROVED_PROVIDER_IDS + valueFrom: + secretKeyRef: + name: sso-migration-gate + key: SSO_AUDIT_APPROVED_PROVIDER_IDS diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 3f8d0a183c4..1465d222188 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -549,6 +549,18 @@ migrations: tag: "" digest: "" pullPolicy: IfNotPresent + + # One-shot pre-0266 SSO migration gate. Established databases must quiesce + # provider writes and set providerWritesQuiesced to "true" for that rollout. + # auditApprovedProviderIds is a comma-separated retain/migrate decision for + # providers with existing Better Auth account links or active sessions. + ssoPreflight: + providerWritesQuiesced: "" + auditApprovedProviderIds: "" + # Optional Secret-backed mode. When set, both values above are ignored. + existingSecret: "" + providerWritesQuiescedKey: SSO_PROVIDER_WRITES_QUIESCED + auditApprovedProviderIdsKey: SSO_AUDIT_APPROVED_PROVIDER_IDS # Resource limits and requests resources: diff --git a/packages/db/package.json b/packages/db/package.json index 7e6a051dfb2..f5dfd4d7a74 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -22,6 +22,7 @@ "db:push": "bunx drizzle-kit push --config=./drizzle.config.ts", "db:migrate": "bun --env-file=.env run ./scripts/migrate.ts", "db:audit-sso-providers": "bun --env-file=.env run ./scripts/audit-sso-providers.ts", + "db:rehearse-sso-migration": "bun --no-env-file run ./scripts/rehearse-sso-migration.ts", "db:reconcile-workspace-storage": "bun --env-file=.env run ./scripts/reconcile-workspace-storage.ts", "db:studio": "bunx drizzle-kit studio --config=./drizzle.config.ts", "test": "vitest run", diff --git a/packages/db/scripts/audit-sso-providers.ts b/packages/db/scripts/audit-sso-providers.ts index f3adaeff334..996de96e12f 100644 --- a/packages/db/scripts/audit-sso-providers.ts +++ b/packages/db/scripts/audit-sso-providers.ts @@ -1,37 +1,17 @@ #!/usr/bin/env bun -import { drizzle } from 'drizzle-orm/postgres-js' import postgres from 'postgres' -import { parse as parseDomain } from 'tldts' -import { account, session, ssoProvider } from '../schema' -import { ssoDomainsOverlap } from '../sso-domain' +import { + auditSSOProviderSnapshot, + loadLegacySSOProviderAuditSnapshot, + parseApprovedSSOProviderIds, +} from '../sso-provider-audit' const connectionString = process.env.POSTGRES_URL ?? process.env.DATABASE_URL if (!connectionString) { throw new Error('POSTGRES_URL or DATABASE_URL is required') } -const RESERVED_PROVIDER_IDS = new Set(['google', 'github', 'email-password']) - -function isValidProviderId(value: string): boolean { - return ( - value.length <= 44 && - /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(value) && - !RESERVED_PROVIDER_IDS.has(value) - ) -} - -function isRegistrableDomain(value: string): boolean { - const parsed = parseDomain(value, { allowPrivateDomains: true, validateHostname: true }) - return ( - value === value.trim().toLowerCase() && - !value.includes(',') && - parsed.hostname === value && - Boolean(parsed.publicSuffix) && - parsed.publicSuffix !== value - ) -} - const client = postgres(connectionString, { prepare: false, max: 1, @@ -39,74 +19,22 @@ const client = postgres(connectionString, { }) try { - const db = drizzle(client) - const [providers, accountLinks, activeSessions] = await Promise.all([ - db.select().from(ssoProvider), - db.select({ providerId: account.providerId, userId: account.userId }).from(account), - db.select({ userId: session.userId, expiresAt: session.expiresAt }).from(session), - ]) - const findings: string[] = [] - const approvedProviderIds = new Set( - (process.env.SSO_AUDIT_APPROVED_PROVIDER_IDS ?? '') - .split(',') - .map((value) => value.trim()) - .filter(Boolean) + const result = auditSSOProviderSnapshot( + await loadLegacySSOProviderAuditSnapshot(client), + parseApprovedSSOProviderIds(process.env.SSO_AUDIT_APPROVED_PROVIDER_IDS) ) - const now = Date.now() - - for (const provider of providers) { - if (!provider.organizationId) { - findings.push(`${provider.providerId}: missing organization ownership`) - } - if (!isValidProviderId(provider.providerId)) { - findings.push(`${provider.providerId}: invalid provider ID`) - } - if (!isRegistrableDomain(provider.domain)) { - findings.push(`${provider.providerId}: '${provider.domain}' is not a registrable domain`) - } - - const linkedUserIds = new Set( - accountLinks - .filter((link) => link.providerId === provider.providerId) - .map((link) => link.userId) + for (const link of result.links) { + console.log( + `${link.providerId}: ${link.linkedUserCount} linked user(s), ${link.activeSessionCount} active session(s)` ) - const sessionCount = activeSessions.filter( - (candidate) => linkedUserIds.has(candidate.userId) && candidate.expiresAt.getTime() > now - ).length - if (linkedUserIds.size > 0) { - console.log( - `${provider.providerId}: ${linkedUserIds.size} linked user(s), ${sessionCount} active session(s)` - ) - if (!approvedProviderIds.has(provider.providerId)) { - findings.push( - `${provider.providerId}: linked users/sessions require an explicit retain-or-migrate decision in SSO_AUDIT_APPROVED_PROVIDER_IDS` - ) - } - } - } - - for (let leftIndex = 0; leftIndex < providers.length; leftIndex += 1) { - for (let rightIndex = leftIndex + 1; rightIndex < providers.length; rightIndex += 1) { - const left = providers[leftIndex] - const right = providers[rightIndex] - if (left.providerId === right.providerId) { - findings.push(`duplicate provider ID: ${left.providerId}`) - } - if (left.organizationId && left.organizationId === right.organizationId) { - findings.push(`organization ${left.organizationId} owns multiple providers`) - } - if (ssoDomainsOverlap(left.domain, right.domain)) { - findings.push(`overlapping domains: ${left.domain} and ${right.domain}`) - } - } } - if (findings.length > 0) { + if (result.findings.length > 0) { console.error('SSO provider audit failed:') - for (const finding of [...new Set(findings)]) console.error(`- ${finding}`) + for (const finding of result.findings) console.error(`- ${finding}`) process.exitCode = 1 } else { - console.log(`SSO provider audit passed (${providers.length} provider rows checked).`) + console.log(`SSO provider audit passed (${result.providerCount} provider rows checked).`) } } finally { await client.end({ timeout: 5 }) diff --git a/packages/db/scripts/migrate.ts b/packages/db/scripts/migrate.ts index d08bb7a32a1..fd34f1e528e 100644 --- a/packages/db/scripts/migrate.ts +++ b/packages/db/scripts/migrate.ts @@ -5,6 +5,13 @@ import { drizzle } from 'drizzle-orm/postgres-js' import { migrate } from 'drizzle-orm/postgres-js/migrator' import postgres from 'postgres' import { runScriptMigrations } from '../script-migrations/index' +import { SSO_PROVIDER_MUTATION_LOCK_KEY } from '../sso-lock' +import { + auditSSOProviderSnapshot, + isSSOHardeningMigrationApplied, + loadLegacySSOProviderAuditSnapshot, + parseApprovedSSOProviderIds, +} from '../sso-provider-audit' /** * Concurrent-index convention: plain `CREATE INDEX` write-blocks large/hot @@ -62,6 +69,8 @@ const client = postgres(url, { const MIGRATION_LOCK_KEY = 4_961_002_270n const LOCK_ACQUIRE_DEADLINE_MS = 30 * 60_000 const LOCK_RETRY_INTERVAL_MS = 5_000 +const SSO_LOCK_ACQUIRE_DEADLINE_MS = 5 * 60_000 +const SSO_LOCK_RETRY_INTERVAL_MS = 1_000 /** * Max time a migration statement may queue for a table lock (SQLSTATE 55P03 on @@ -105,17 +114,20 @@ function isTransientConnectError(error: unknown): boolean { /** Backend pid of the lock-holding session; a change means the lock was lost. */ let lockSessionPid = 0 +let ssoMutationLockHeld = false try { await connectWithRetry() await acquireMigrationLock() try { + await runPendingSSOHardeningAudit() await runMigrationsWithRetry() console.log('Migrations applied successfully.') await assertLockSessionHeld() await runScriptMigrations(client) await warnOnInvalidIndexes() } finally { + await releaseSSOMutationLock() await releaseMigrationLock() } } catch (error) { @@ -198,6 +210,87 @@ async function assertLockSessionHeld(): Promise { } } +/** + * Migration 0266 strengthens legacy SSO rows using public-suffix validation + * that cannot be expressed by its SQL checks. Run that audit exactly once, + * while 0266 is still absent from drizzle's journal. + * + * The advisory lock serializes writers that use the hardened application and + * scripts. It cannot stop an older deployment that does not participate, so a + * non-empty legacy table also requires an explicit operator acknowledgement + * that SSO/provider writes are quiesced for the audit-to-migration interval. + */ +async function runPendingSSOHardeningAudit(): Promise { + if (await isSSOHardeningMigrationApplied(client)) return + + const [{ table_exists: tableExists }] = await client< + Array<{ table_exists: boolean }> + >`SELECT to_regclass('public.sso_provider') IS NOT NULL AS table_exists` + if (!tableExists) { + console.log('SSO migration preflight: no legacy provider table exists.') + return + } + + await acquireSSOMutationLock() + const snapshot = await loadLegacySSOProviderAuditSnapshot(client) + if (process.env.SSO_PROVIDER_WRITES_QUIESCED !== 'true') { + const environment = process.env.ENVIRONMENT || 'unknown' + throw new Error( + `SSO migration preflight blocked for ${environment}: set ` + + 'SSO_PROVIDER_WRITES_QUIESCED=true only after disabling provider mutations ' + + 'and keep them quiesced until migration 0266 completes' + ) + } + + const result = auditSSOProviderSnapshot( + snapshot, + parseApprovedSSOProviderIds(process.env.SSO_AUDIT_APPROVED_PROVIDER_IDS) + ) + for (const link of result.links) { + console.log( + `SSO migration preflight: ${link.providerId} has ${link.linkedUserCount} linked user(s) ` + + `and ${link.activeSessionCount} active session(s).` + ) + } + if (result.findings.length > 0) { + throw new Error(`SSO migration preflight failed:\n- ${result.findings.join('\n- ')}`) + } + console.log(`SSO migration preflight passed (${result.providerCount} provider rows checked).`) +} + +async function acquireSSOMutationLock(): Promise { + const deadline = Date.now() + SSO_LOCK_ACQUIRE_DEADLINE_MS + while (true) { + const [{ acquired }] = await client< + Array<{ acquired: boolean }> + >`SELECT pg_try_advisory_lock(${SSO_PROVIDER_MUTATION_LOCK_KEY}) AS acquired` + if (acquired) { + ssoMutationLockHeld = true + return + } + if (Date.now() >= deadline) { + throw new Error( + 'Timed out acquiring the SSO mutation advisory lock; verify that provider writes are quiesced and no SSO mutation transaction is wedged' + ) + } + await sleep(SSO_LOCK_RETRY_INTERVAL_MS) + } +} + +async function releaseSSOMutationLock(): Promise { + if (!ssoMutationLockHeld) return + try { + await client`SELECT pg_advisory_unlock(${SSO_PROVIDER_MUTATION_LOCK_KEY})` + } catch (unlockError) { + console.error( + 'WARN: SSO mutation advisory unlock failed; the session lock will auto-release on disconnect.', + unlockError + ) + } finally { + ssoMutationLockHeld = false + } +} + async function runMigrationsWithRetry(): Promise { for (let attempt = 1; ; attempt++) { await assertLockSessionHeld() diff --git a/packages/db/scripts/rehearse-sso-migration.ts b/packages/db/scripts/rehearse-sso-migration.ts new file mode 100644 index 00000000000..f395207c9fb --- /dev/null +++ b/packages/db/scripts/rehearse-sso-migration.ts @@ -0,0 +1,319 @@ +#!/usr/bin/env bun + +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { drizzle } from 'drizzle-orm/postgres-js' +import { migrate } from 'drizzle-orm/postgres-js/migrator' +import postgres, { type Sql } from 'postgres' +import { isSSOHardeningMigrationApplied, SSO_HARDENING_MIGRATION_TAG } from '../sso-provider-audit' + +const adminUrl = requireGuardedAdminUrl() +const runId = `${Date.now()}_${process.pid}` +const databaseName = `sim_e2e_sso_rehearsal_${runId}`.toLowerCase() +const databaseUrl = new URL(adminUrl) +databaseUrl.pathname = `/${databaseName}` +const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const temporaryRoot = mkdtempSync(path.join(os.tmpdir(), 'sim-sso-migration-rehearsal-')) +const preMigrationFolder = path.join(temporaryRoot, 'migrations') +const admin = postgres(adminUrl, { max: 1, prepare: false, onnotice: () => {} }) +let database: Sql | undefined + +try { + await admin.unsafe(`CREATE DATABASE "${databaseName}"`) + database = postgres(databaseUrl.toString(), { max: 1, prepare: false, onnotice: () => {} }) + createPre0266MigrationFolder(preMigrationFolder) + await migrate(drizzle(database), { migrationsFolder: preMigrationFolder }) + + const [{ has_domain_verified: hasDomainVerified }] = await database< + Array<{ has_domain_verified: boolean }> + >` + SELECT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'sso_provider' + AND column_name = 'domain_verified' + ) AS has_domain_verified + ` + assert.equal(hasDomainVerified, false) + assert.equal(await isSSOHardeningMigrationApplied(database), false) + + await seedAuditFailureMatrix(database) + const invalidResult = runMigrate({ SSO_PROVIDER_WRITES_QUIESCED: 'true' }) + assert.equal(invalidResult.status, 1, invalidResult.output) + for (const finding of [ + 'missing-org: missing organization ownership', + 'Bad_Id: invalid provider ID', + 'google: invalid provider ID', + "public-suffix: 'com' is not a registrable domain", + "unknown-suffix: 'tenant.invalid' is not a registrable domain", + 'duplicate provider ID: duplicate-id', + 'organization organization-multi owns multiple providers', + 'overlapping domains: duplicate.example.com and duplicate.example.com', + 'overlapping domains: login.parent.example.net and parent.example.net', + ]) { + assert.match(invalidResult.output, new RegExp(escapeRegExp(finding))) + } + + await database`DELETE FROM sso_provider` + const emptyUnquiescedResult = runMigrate() + assert.equal(emptyUnquiescedResult.status, 1, emptyUnquiescedResult.output) + assert.match(emptyUnquiescedResult.output, /SSO_PROVIDER_WRITES_QUIESCED=true/) + + await seedValidLinkedProvider(database) + + const unquiescedResult = runMigrate() + assert.equal(unquiescedResult.status, 1, unquiescedResult.output) + assert.match(unquiescedResult.output, /SSO_PROVIDER_WRITES_QUIESCED=true/) + + const unapprovedResult = runMigrate({ SSO_PROVIDER_WRITES_QUIESCED: 'true' }) + assert.equal(unapprovedResult.status, 1, unapprovedResult.output) + assert.match(unapprovedResult.output, /linked users\/sessions require an explicit/) + + const approvedResult = runMigrate({ + SSO_PROVIDER_WRITES_QUIESCED: 'true', + SSO_AUDIT_APPROVED_PROVIDER_IDS: 'acme-saml', + }) + assert.equal(approvedResult.status, 0, approvedResult.output) + assert.equal(await isSSOHardeningMigrationApplied(database), true) + + const migratedProviders = await database< + Array<{ provider_id: string; domain_verified: boolean }> + >` + SELECT provider_id, domain_verified + FROM sso_provider + ORDER BY provider_id + ` + assert.deepEqual(migratedProviders, [ + { provider_id: 'acme-saml', domain_verified: false }, + { provider_id: 'private-suffix', domain_verified: false }, + ]) + + const constraints = await database>` + SELECT conname, convalidated + FROM pg_constraint + WHERE conrelid = 'sso_provider'::regclass + AND conname IN ( + 'sso_provider_provider_id_format_check', + 'sso_provider_provider_id_not_reserved_check', + 'sso_provider_domain_format_check', + 'sso_provider_organization_required_check' + ) + ORDER BY conname + ` + assert.equal(constraints.length, 4) + assert.ok(constraints.every(({ convalidated }) => convalidated)) + + const indexes = await database< + Array<{ indexrelid: string; indisunique: boolean; indisvalid: boolean }> + >` + SELECT indexrelid::regclass::text AS indexrelid, indisunique, indisvalid + FROM pg_index + WHERE indexrelid::regclass::text IN ( + 'sso_provider_provider_id_unique', + 'sso_provider_domain_lower_unique', + 'sso_provider_organization_id_unique' + ) + ORDER BY indexrelid::regclass::text + ` + assert.equal(indexes.length, 3) + assert.ok(indexes.every(({ indisunique, indisvalid }) => indisunique && indisvalid)) + + const repeatResult = runMigrate() + assert.equal(repeatResult.status, 0, repeatResult.output) + console.log( + 'SSO migration rehearsal passed for pre-0266 failures, rollout gate, and post-0266 schema.' + ) +} finally { + if (database) await database.end({ timeout: 5 }).catch(() => {}) + await admin.unsafe( + `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${databaseName}' AND pid <> pg_backend_pid()` + ) + await admin.unsafe(`DROP DATABASE IF EXISTS "${databaseName}" WITH (FORCE)`) + await admin.end({ timeout: 5 }) + rmSync(temporaryRoot, { recursive: true, force: true }) +} + +function createPre0266MigrationFolder(destination: string): void { + const source = path.join(packageRoot, 'migrations') + const journal = JSON.parse(readFileSync(path.join(source, 'meta', '_journal.json'), 'utf8')) as { + entries: Array<{ + idx: number + tag: string + when: number + version: string + breakpoints: boolean + }> + } + const hardeningEntry = journal.entries.find(({ tag }) => tag === SSO_HARDENING_MIGRATION_TAG) + if (!hardeningEntry) + throw new Error(`Migration journal is missing ${SSO_HARDENING_MIGRATION_TAG}`) + const entries = journal.entries.filter(({ idx }) => idx < hardeningEntry.idx) + + mkdirSync(path.join(destination, 'meta'), { recursive: true }) + writeFileSync( + path.join(destination, 'meta', '_journal.json'), + `${JSON.stringify({ version: '7', dialect: 'postgresql', entries }, null, 2)}\n` + ) + for (const { tag } of entries) { + copyFileSync(path.join(source, `${tag}.sql`), path.join(destination, `${tag}.sql`)) + } +} + +async function seedAuditFailureMatrix(sql: Sql): Promise { + await seedUsersAndOrganizations(sql) + const providers = [ + ['row-missing-org', 'missing-org', null, 'missing.example.com'], + ['row-invalid-id', 'Bad_Id', 'organization-1', 'invalid-id.example.com'], + ['row-reserved-id', 'google', 'organization-2', 'reserved.example.com'], + ['row-public-suffix', 'public-suffix', 'organization-3', 'com'], + ['row-unknown-suffix', 'unknown-suffix', 'organization-10', 'tenant.invalid'], + ['row-duplicate-id-a', 'duplicate-id', 'organization-4', 'duplicate-a.example.com'], + ['row-duplicate-id-b', 'duplicate-id', 'organization-5', 'duplicate-b.example.com'], + ['row-duplicate-domain-a', 'duplicate-domain-a', 'organization-6', 'duplicate.example.com'], + ['row-duplicate-domain-b', 'duplicate-domain-b', 'organization-7', 'duplicate.example.com'], + ['row-parent-domain', 'parent-domain', 'organization-8', 'parent.example.net'], + ['row-child-domain', 'child-domain', 'organization-9', 'login.parent.example.net'], + ['row-multi-a', 'multi-a', 'organization-multi', 'multi-a.example.org'], + ['row-multi-b', 'multi-b', 'organization-multi', 'multi-b.example.org'], + ] as const + for (const [id, providerId, organizationId, domain] of providers) { + await sql` + INSERT INTO sso_provider ( + id, issuer, domain, oidc_config, saml_config, user_id, provider_id, organization_id + ) + VALUES ( + ${id}, + ${`https://${providerId.toLowerCase()}.example.invalid`}, + ${domain}, + NULL, + NULL, + 'audit-user', + ${providerId}, + ${organizationId} + ) + ` + } +} + +async function seedValidLinkedProvider(sql: Sql): Promise { + await sql` + INSERT INTO sso_provider ( + id, issuer, domain, oidc_config, saml_config, user_id, provider_id, organization_id + ) + VALUES ( + 'row-acme', + 'https://idp.acme.invalid', + 'login.acme.com', + NULL, + NULL, + 'audit-user', + 'acme-saml', + 'organization-1' + ) + ` + await sql` + INSERT INTO sso_provider ( + id, issuer, domain, oidc_config, saml_config, user_id, provider_id, organization_id + ) + VALUES ( + 'row-private-suffix', + 'https://tenant.github.io', + 'tenant.github.io', + NULL, + NULL, + 'audit-user', + 'private-suffix', + 'organization-2' + ) + ` + await sql` + INSERT INTO account ( + id, account_id, provider_id, user_id, created_at, updated_at + ) + VALUES ('account-acme', 'external-acme', 'acme-saml', 'audit-user', now(), now()) + ` + await sql` + INSERT INTO session ( + id, expires_at, token, created_at, updated_at, user_id + ) + VALUES ( + 'session-acme', + now() + interval '1 hour', + 'rehearsal-session-token', + now(), + now(), + 'audit-user' + ) + ` +} + +async function seedUsersAndOrganizations(sql: Sql): Promise { + await sql` + INSERT INTO "user" (id, name, email, email_verified, created_at, updated_at) + VALUES ('audit-user', 'Audit User', 'audit-user@example.com', true, now(), now()) + ON CONFLICT (id) DO NOTHING + ` + const organizationIds = [ + ...Array.from({ length: 10 }, (_, index) => `organization-${index + 1}`), + 'organization-multi', + ] + for (const id of organizationIds) { + await sql` + INSERT INTO organization (id, name, slug, created_at, updated_at) + VALUES (${id}, ${id}, ${id}, now(), now()) + ON CONFLICT (id) DO NOTHING + ` + } +} + +function runMigrate(overrides: Record = {}): { + status: number | null + output: string +} { + const result = spawnSync( + process.execPath, + ['--no-env-file', path.join(packageRoot, 'scripts', 'migrate.ts')], + { + cwd: packageRoot, + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + TMPDIR: process.env.TMPDIR, + DATABASE_URL: databaseUrl.toString(), + MIGRATION_DATABASE_URL: databaseUrl.toString(), + ENVIRONMENT: 'sso-rehearsal', + ...overrides, + }, + encoding: 'utf8', + maxBuffer: 50 * 1024 * 1024, + timeout: 180_000, + } + ) + return { + status: result.status, + output: `${result.stdout ?? ''}\n${result.stderr ?? ''}`, + } +} + +function requireGuardedAdminUrl(): string { + const raw = process.env.E2E_PG_ADMIN_URL + if (!raw) throw new Error('E2E_PG_ADMIN_URL is required for the SSO migration rehearsal') + const url = new URL(raw) + if ( + url.protocol !== 'postgresql:' || + url.hostname !== '127.0.0.1' || + url.pathname !== '/postgres' + ) { + throw new Error('SSO migration rehearsal requires a 127.0.0.1 PostgreSQL /postgres admin URL') + } + return url.toString() +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} diff --git a/packages/db/sso-provider-audit.test.ts b/packages/db/sso-provider-audit.test.ts new file mode 100644 index 00000000000..787770e99ed --- /dev/null +++ b/packages/db/sso-provider-audit.test.ts @@ -0,0 +1,174 @@ +import type { Sql } from 'postgres' +import { describe, expect, it, vi } from 'vitest' +import { + auditSSOProviderSnapshot, + isSSOHardeningMigrationApplied, + loadLegacySSOProviderAuditSnapshot, + parseApprovedSSOProviderIds, + resolveSSOHardeningMigrationTimestamp, + SSO_HARDENING_MIGRATION_TAG, + type SSOProviderAuditSnapshot, +} from './sso-provider-audit' + +const VALID_PROVIDER = { + id: 'provider-row-1', + providerId: 'acme-saml', + organizationId: 'organization-1', + domain: 'login.acme.com', +} + +describe('SSO provider audit', () => { + it('accepts a valid organization provider without account links', () => { + expect( + auditSSOProviderSnapshot({ providers: [VALID_PROVIDER], links: [] }, new Set()).findings + ).toEqual([]) + }) + + it('accepts a registrable tenant under a known private suffix', () => { + const provider = { + ...VALID_PROVIDER, + providerId: 'private-suffix', + domain: 'tenant.github.io', + } + expect( + auditSSOProviderSnapshot({ providers: [provider], links: [] }, new Set()).findings + ).toEqual([]) + }) + + it('rejects domains below an unknown public suffix', () => { + const provider = { ...VALID_PROVIDER, domain: 'tenant.invalid' } + expect( + auditSSOProviderSnapshot({ providers: [provider], links: [] }, new Set()).findings + ).toEqual(["acme-saml: 'tenant.invalid' is not a registrable domain"]) + }) + + it('reports every legacy invariant that requires remediation', () => { + const snapshot: SSOProviderAuditSnapshot = { + providers: [ + { + id: 'provider-row-1', + providerId: 'google', + organizationId: null, + domain: 'COM', + }, + { + id: 'provider-row-2', + providerId: 'duplicate', + organizationId: 'organization-2', + domain: 'acme.com', + }, + { + id: 'provider-row-3', + providerId: 'duplicate', + organizationId: 'organization-2', + domain: 'login.acme.com', + }, + ], + links: [], + } + + expect(auditSSOProviderSnapshot(snapshot, new Set()).findings).toEqual( + expect.arrayContaining([ + 'google: missing organization ownership', + 'google: invalid provider ID', + "google: 'COM' is not a registrable domain", + 'duplicate provider ID: duplicate', + 'organization organization-2 owns multiple providers', + 'overlapping domains: acme.com and login.acme.com', + ]) + ) + }) + + it('keeps linked-account approval separate and explicit', () => { + const snapshot: SSOProviderAuditSnapshot = { + providers: [VALID_PROVIDER], + links: [ + { + providerId: VALID_PROVIDER.providerId, + linkedUserCount: 2, + activeSessionCount: 1, + }, + ], + } + + expect(auditSSOProviderSnapshot(snapshot, new Set()).findings).toEqual([ + 'acme-saml: linked users/sessions require an explicit retain-or-migrate decision in SSO_AUDIT_APPROVED_PROVIDER_IDS', + ]) + expect( + auditSSOProviderSnapshot(snapshot, parseApprovedSSOProviderIds(' other, acme-saml ')).findings + ).toEqual([]) + }) + + it('loads only columns available before migration 0266', async () => { + const queries: string[] = [] + const fakeSql = ((strings: TemplateStringsArray) => { + const query = strings.join('?').replace(/\s+/g, ' ').trim() + queries.push(query) + if (query.includes('SELECT id, provider_id')) { + return Promise.resolve([ + { + id: VALID_PROVIDER.id, + provider_id: VALID_PROVIDER.providerId, + organization_id: VALID_PROVIDER.organizationId, + domain: VALID_PROVIDER.domain, + }, + ]) + } + return Promise.resolve([ + { + provider_id: VALID_PROVIDER.providerId, + linked_user_count: 1, + active_session_count: 1, + }, + ]) + }) as unknown as Sql + + await expect(loadLegacySSOProviderAuditSnapshot(fakeSql)).resolves.toEqual({ + providers: [VALID_PROVIDER], + links: [ + { + providerId: VALID_PROVIDER.providerId, + linkedUserCount: 1, + activeSessionCount: 1, + }, + ], + }) + expect(queries.join('\n')).not.toContain('domain_verified') + expect(queries[0]).toContain('SELECT id, provider_id, organization_id, domain') + }) +}) + +describe('SSO hardening migration state', () => { + it('is pending before the drizzle journal exists', async () => { + const unsafe = vi.fn() + const fakeSql = Object.assign( + (() => Promise.resolve([{ journal_exists: false }])) as unknown as Sql, + { unsafe } + ) + + await expect(isSSOHardeningMigrationApplied(fakeSql)).resolves.toBe(false) + expect(unsafe).not.toHaveBeenCalled() + }) + + it('uses the 0266 journal timestamp instead of post-migration columns', async () => { + const unsafe = vi.fn().mockResolvedValue([{ applied: true }]) + const fakeSql = Object.assign( + (() => Promise.resolve([{ journal_exists: true }])) as unknown as Sql, + { unsafe } + ) + + await expect(isSSOHardeningMigrationApplied(fakeSql)).resolves.toBe(true) + expect(unsafe).toHaveBeenCalledWith(expect.stringContaining('created_at = 1784759628537')) + }) + + it('resolves the migration timestamp by tag and fails closed if it is renamed', () => { + expect( + resolveSSOHardeningMigrationTimestamp({ + entries: [{ tag: SSO_HARDENING_MIGRATION_TAG, when: 123 }], + }) + ).toBe(123) + expect(() => resolveSSOHardeningMigrationTimestamp({ entries: [] })).toThrow( + `missing ${SSO_HARDENING_MIGRATION_TAG}` + ) + }) +}) diff --git a/packages/db/sso-provider-audit.ts b/packages/db/sso-provider-audit.ts new file mode 100644 index 00000000000..465618a0263 --- /dev/null +++ b/packages/db/sso-provider-audit.ts @@ -0,0 +1,225 @@ +import { readFileSync } from 'node:fs' +import type { Sql } from 'postgres' +import { parse as parseDomain } from 'tldts' +import { ssoDomainsOverlap } from './sso-domain' + +export const SSO_HARDENING_MIGRATION_TAG = '0266_zippy_the_phantom' + +const RESERVED_PROVIDER_IDS = new Set(['google', 'github', 'email-password']) + +export interface LegacySSOProviderRow { + id: string + providerId: string + organizationId: string | null + domain: string +} + +export interface SSOProviderLinkSummary { + providerId: string + linkedUserCount: number + activeSessionCount: number +} + +export interface SSOProviderAuditSnapshot { + providers: LegacySSOProviderRow[] + links: SSOProviderLinkSummary[] +} + +export interface SSOProviderAuditResult { + findings: string[] + links: SSOProviderLinkSummary[] + providerCount: number +} + +export function parseApprovedSSOProviderIds(value: string | undefined): Set { + return new Set( + (value ?? '') + .split(',') + .map((providerId) => providerId.trim()) + .filter(Boolean) + ) +} + +export function auditSSOProviderSnapshot( + snapshot: SSOProviderAuditSnapshot, + approvedProviderIds: ReadonlySet +): SSOProviderAuditResult { + const findings: string[] = [] + const linkByProviderId = new Map(snapshot.links.map((summary) => [summary.providerId, summary])) + + for (const provider of snapshot.providers) { + if (!provider.organizationId) { + findings.push(`${provider.providerId}: missing organization ownership`) + } + if (!isValidProviderId(provider.providerId)) { + findings.push(`${provider.providerId}: invalid provider ID`) + } + if (!isRegistrableDomain(provider.domain)) { + findings.push(`${provider.providerId}: '${provider.domain}' is not a registrable domain`) + } + + const linkSummary = linkByProviderId.get(provider.providerId) + if ( + linkSummary && + linkSummary.linkedUserCount > 0 && + !approvedProviderIds.has(provider.providerId) + ) { + findings.push( + `${provider.providerId}: linked users/sessions require an explicit retain-or-migrate decision in SSO_AUDIT_APPROVED_PROVIDER_IDS` + ) + } + } + + for (let leftIndex = 0; leftIndex < snapshot.providers.length; leftIndex += 1) { + for (let rightIndex = leftIndex + 1; rightIndex < snapshot.providers.length; rightIndex += 1) { + const left = snapshot.providers[leftIndex] + const right = snapshot.providers[rightIndex] + if (left.providerId === right.providerId) { + findings.push(`duplicate provider ID: ${left.providerId}`) + } + if (left.organizationId && left.organizationId === right.organizationId) { + findings.push(`organization ${left.organizationId} owns multiple providers`) + } + if (ssoDomainsOverlap(left.domain, right.domain)) { + findings.push(`overlapping domains: ${left.domain} and ${right.domain}`) + } + } + } + + return { + findings: [...new Set(findings)], + links: snapshot.links, + providerCount: snapshot.providers.length, + } +} + +/** + * Loads only columns that existed before migration 0266. This must remain + * compatible with the legacy schema because it is the migration's preflight. + */ +export async function loadLegacySSOProviderAuditSnapshot( + sql: Sql +): Promise { + const providers = await sql< + Array<{ + id: string + provider_id: string + organization_id: string | null + domain: string + }> + >` + SELECT id, provider_id, organization_id, domain + FROM sso_provider + ORDER BY id + ` + const links = await sql< + Array<{ + provider_id: string + linked_user_count: number + active_session_count: number + }> + >` + SELECT + linked.provider_id, + count(DISTINCT linked.user_id)::integer AS linked_user_count, + count(session.id) FILTER (WHERE session.expires_at > now())::integer + AS active_session_count + FROM ( + SELECT DISTINCT account.provider_id, account.user_id + FROM account + INNER JOIN sso_provider + ON sso_provider.provider_id = account.provider_id + ) AS linked + LEFT JOIN session + ON session.user_id = linked.user_id + GROUP BY linked.provider_id + ORDER BY linked.provider_id + ` + + return { + providers: providers.map((provider) => ({ + id: provider.id, + providerId: provider.provider_id, + organizationId: provider.organization_id, + domain: provider.domain, + })), + links: links.map((summary) => ({ + providerId: summary.provider_id, + linkedUserCount: summary.linked_user_count, + activeSessionCount: summary.active_session_count, + })), + } +} + +export async function isSSOHardeningMigrationApplied(sql: Sql): Promise { + const [{ journal_exists: journalExists }] = await sql< + Array<{ journal_exists: boolean }> + >`SELECT to_regclass('drizzle.__drizzle_migrations') IS NOT NULL AS journal_exists` + if (!journalExists) return false + + const migrationTimestamp = readSSOHardeningMigrationTimestamp() + const [{ applied }] = await sql.unsafe>( + `SELECT EXISTS ( + SELECT 1 + FROM drizzle.__drizzle_migrations + WHERE created_at = ${migrationTimestamp} + ) AS applied` + ) + return applied +} + +export function resolveSSOHardeningMigrationTimestamp(journal: unknown): number { + if ( + typeof journal !== 'object' || + journal === null || + !('entries' in journal) || + !Array.isArray(journal.entries) + ) { + throw new Error('Drizzle migration journal has no entries array') + } + const entry = journal.entries.find( + (candidate) => + typeof candidate === 'object' && + candidate !== null && + 'tag' in candidate && + candidate.tag === SSO_HARDENING_MIGRATION_TAG + ) + if ( + typeof entry !== 'object' || + entry === null || + !('when' in entry) || + typeof entry.when !== 'number' + ) { + throw new Error(`Drizzle migration journal is missing ${SSO_HARDENING_MIGRATION_TAG}`) + } + return entry.when +} + +function readSSOHardeningMigrationTimestamp(): number { + const journal = JSON.parse( + readFileSync(new URL('./migrations/meta/_journal.json', import.meta.url), 'utf8') + ) + return resolveSSOHardeningMigrationTimestamp(journal) +} + +function isValidProviderId(value: string): boolean { + return ( + value.length <= 44 && + /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(value) && + !RESERVED_PROVIDER_IDS.has(value) + ) +} + +function isRegistrableDomain(value: string): boolean { + const parsed = parseDomain(value, { allowPrivateDomains: true, validateHostname: true }) + return ( + value === value.trim().toLowerCase() && + !value.includes(',') && + !parsed.isIp && + Boolean(parsed.domain) && + parsed.hostname === value && + Boolean(parsed.publicSuffix) && + Boolean(parsed.isIcann || parsed.isPrivate) && + parsed.publicSuffix !== value + ) +} From 66eb9a587df32d8d997b5051a35e415deeaf0502 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Thu, 23 Jul 2026 06:56:47 -0700 Subject: [PATCH 2/2] fix(helm): tolerate retired SSO preflight keys Co-authored-by: Cursor --- apps/docs/content/docs/en/platform/enterprise/sso.mdx | 6 ++++++ helm/sim/templates/deployment-app.yaml | 2 ++ helm/sim/tests/sso-migration-preflight_test.yaml | 2 ++ helm/sim/values.yaml | 3 +++ 4 files changed, 13 insertions(+) diff --git a/apps/docs/content/docs/en/platform/enterprise/sso.mdx b/apps/docs/content/docs/en/platform/enterprise/sso.mdx index bb22c2fdd62..7c483f30523 100644 --- a/apps/docs/content/docs/en/platform/enterprise/sso.mdx +++ b/apps/docs/content/docs/en/platform/enterprise/sso.mdx @@ -346,6 +346,12 @@ migrations: # SSO_PROVIDER_WRITES_QUIESCED and SSO_AUDIT_APPROVED_PROVIDER_IDS. ``` +Secret-backed keys are optional at Kubernetes pod admission so removing retired +one-shot keys cannot block deployments after 0266. A pending 0266 migration +still fails closed unless `SSO_PROVIDER_WRITES_QUIESCED` resolves to `true`; +`SSO_AUDIT_APPROVED_PROVIDER_IDS` may be absent when no linked providers require +an explicit approval. + Remove the one-shot values after 0266 is journaled. Do not leave provider writes disabled or interpret the flag itself as quiescence. diff --git a/helm/sim/templates/deployment-app.yaml b/helm/sim/templates/deployment-app.yaml index 9c623dcdbc9..ff0f4931eb1 100644 --- a/helm/sim/templates/deployment-app.yaml +++ b/helm/sim/templates/deployment-app.yaml @@ -59,6 +59,7 @@ spec: secretKeyRef: name: {{ .Values.migrations.ssoPreflight.existingSecret | quote }} key: {{ .Values.migrations.ssoPreflight.providerWritesQuiescedKey | quote }} + optional: true {{- else }} value: {{ .Values.migrations.ssoPreflight.providerWritesQuiesced | quote }} {{- end }} @@ -68,6 +69,7 @@ spec: secretKeyRef: name: {{ .Values.migrations.ssoPreflight.existingSecret | quote }} key: {{ .Values.migrations.ssoPreflight.auditApprovedProviderIdsKey | quote }} + optional: true {{- else }} value: {{ .Values.migrations.ssoPreflight.auditApprovedProviderIds | quote }} {{- end }} diff --git a/helm/sim/tests/sso-migration-preflight_test.yaml b/helm/sim/tests/sso-migration-preflight_test.yaml index 10e58c27165..3e0fa310b14 100644 --- a/helm/sim/tests/sso-migration-preflight_test.yaml +++ b/helm/sim/tests/sso-migration-preflight_test.yaml @@ -44,6 +44,7 @@ tests: secretKeyRef: name: sso-migration-gate key: SSO_PROVIDER_WRITES_QUIESCED + optional: true - contains: path: spec.template.spec.initContainers[0].env content: @@ -52,3 +53,4 @@ tests: secretKeyRef: name: sso-migration-gate key: SSO_AUDIT_APPROVED_PROVIDER_IDS + optional: true diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 1465d222188..3cfe319a22c 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -558,6 +558,9 @@ migrations: providerWritesQuiesced: "" auditApprovedProviderIds: "" # Optional Secret-backed mode. When set, both values above are ignored. + # Secret/key references are optional at pod admission so retired one-shot + # values cannot block post-0266 pods. The migration still fails closed + # before 0266 when providerWritesQuiesced is absent or not "true". existingSecret: "" providerWritesQuiescedKey: SSO_PROVIDER_WRITES_QUIESCED auditApprovedProviderIdsKey: SSO_AUDIT_APPROVED_PROVIDER_IDS