Skip to content
Closed
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
4f91839
Bulldozer DB
N2D4 Mar 24, 2026
9694c33
declareGroupByTable
N2D4 Mar 24, 2026
09ba416
Fix Prisma schema
N2D4 Mar 24, 2026
4232a94
declareMapTable
N2D4 Mar 24, 2026
31b6ac6
Performance tests
N2D4 Mar 24, 2026
2f7f09a
Load tests
N2D4 Mar 24, 2026
863ee05
Interface updates
N2D4 Mar 24, 2026
109cf5d
Bulldozer Studio
N2D4 Mar 25, 2026
53f7302
Remove unnecessary table
N2D4 Mar 25, 2026
006cf5e
Add flat map interface
N2D4 Mar 25, 2026
4cdc057
FlatMap table
N2D4 Mar 25, 2026
49dc922
Flat map fuzz tests
N2D4 Mar 25, 2026
3eccc22
Build MapTable from FlatMapTable
N2D4 Mar 25, 2026
e041e7d
Filter tables
N2D4 Mar 25, 2026
e2b0d3d
Limit tables
N2D4 Mar 25, 2026
f7f21aa
Speed up fuzzing
N2D4 Mar 26, 2026
90f61ec
Concat table
N2D4 Mar 26, 2026
69b3d4f
Bulldozer Studio: Better node placing algorithm
N2D4 Mar 26, 2026
e306770
Add left-join table
N2D4 Mar 26, 2026
5036c2a
Sort table
N2D4 Mar 27, 2026
d55470b
LFold table
N2D4 Mar 27, 2026
43d7304
Left join table
N2D4 Mar 27, 2026
a33faa0
Improve left join performance
N2D4 Mar 27, 2026
aaeb7d1
Improved performance for most tables
N2D4 Mar 27, 2026
ef9915a
Refactor Bulldozer into individual files
N2D4 Mar 27, 2026
037d20f
Merge branch 'dev' into bulldozer-db
N2D4 Mar 27, 2026
2403c17
Update apps/backend/src/lib/bulldozer/db/tables/group-by-table.ts
N2D4 Mar 27, 2026
d3a2daa
Performance improvements
N2D4 Mar 28, 2026
b459240
PR comments
N2D4 Mar 29, 2026
c01d931
Some more perf changes
N2D4 Mar 30, 2026
2d49d23
Fix various comparison key bugs
N2D4 Mar 31, 2026
2bb89c2
Performance improvements
N2D4 Mar 31, 2026
199cdf2
Lint fixes
N2D4 Apr 1, 2026
1dc76dc
Merge remote-tracking branch 'origin/dev' into bulldozer-db
N2D4 Apr 1, 2026
e4a5221
More fixes...
N2D4 Apr 3, 2026
e353232
Various changes
N2D4 Apr 6, 2026
4b400e2
Comments from Aman
N2D4 Apr 8, 2026
5b69a18
Fix tests
N2D4 Apr 8, 2026
a7f999f
Improve perf tests
N2D4 Apr 10, 2026
19abfb3
TimeFold table
N2D4 Apr 11, 2026
9cb5f5b
Clean up Prisma schema
N2D4 Apr 11, 2026
e3c3865
Make migration warnings into errors
N2D4 Apr 13, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,6 @@
"typescript.tsdk": "node_modules/typescript/lib",
"editor.tabSize": 2,
"cSpell.words": [
"glassmorphic",
"sparkline",
"Clickhouse",
"pushable",
"autoupdate",
"backlinks",
"Cancelation",
Expand All @@ -19,14 +15,15 @@
"chinthakagodawita",
"Ciphertext",
"cjsx",
"Clickhouse",
"clsx",
"dbgenerated",
"cmdk",
"codegen",
"crockford",
"Crudl",
"ctsx",
"datapoints",
"dbgenerated",
"deindent",
"Deindentable",
"deindented",
Expand All @@ -42,6 +39,7 @@
"fkey",
"frontends",
"geoip",
"glassmorphic",
"healthcheck",
"hookform",
"hostable",
Expand All @@ -51,6 +49,7 @@
"JWTs",
"katex",
"localstack",
"ltree",
"lucide",
"Luma",
"midfix",
Expand All @@ -64,6 +63,7 @@
"nicified",
"nicify",
"oidc",
"onnotice",
"openapi",
"opentelemetry",
"otel",
Expand All @@ -81,6 +81,7 @@
"Prefetchers",
"Proxied",
"psql",
"pushable",
"qrcode",
"QSTASH",
"quetzallabs",
Expand All @@ -89,6 +90,7 @@
"retryable",
"RPID",
"simplewebauthn",
"sparkline",
"spoofable",
"stackable",
"stackauth",
Expand All @@ -109,6 +111,7 @@
"Unmigrated",
"unsubscribers",
"upsert",
"upserted",
"Upvotes",
"upvoting",
"webapi",
Expand Down
3 changes: 2 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ This file provides guidance to coding agents when working with code in this repo
### Essential Commands
- **Install dependencies**: `pnpm install`
- **Run tests**: `pnpm test run` (uses Vitest). You can filter with `pnpm test run <file-filters>`. The `run` is important to not trigger watch mode
- **Lint code**: `pnpm lint`. `pnpm lint --fix` will fix some of the linting errors, prefer that over fixing them manually.
- **Lint code**: `pnpm lint`. `pnpm lint --fix` will fix some of the linting errors, prefer that over fixing them manually. Use `pnpm -C <package> lint` to lint a specific package.
- **Type check**: `pnpm typecheck`

#### Extra commands
Expand Down Expand Up @@ -108,6 +108,7 @@ To see all development ports, refer to the index.html of `apps/dev-launchpad/pub
- Any design components you add or modify in the dashboard, update the Playground page accordingly to showcase the changes.
- Unless very clearly equivalent from types, prefer explicit null/undefinedness checks over boolean checks, eg. `foo == null` instead of `!foo`.
- Ensure **aggressively** that all code has low coupling and high cohesion. This is really important as it makes sure our code remains consistent and maintainable. Eagerly refactor things into better abstractions and look out for them actively.
- Always let me know about the tradeoffs and decisions you make while implementing a non-trivial change.

### Code-related
- Use ES6 maps instead of records wherever you can.
6 changes: 4 additions & 2 deletions apps/backend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
"with-env:dev": "dotenv -c development --",
"with-env:prod": "dotenv -c production --",
"with-env:test": "dotenv -c test --",
"dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue,cron-jobs\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\" \"pnpm run run-cron-jobs\"",
"dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue,cron-jobs,bulldozer-studio\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\" \"pnpm run run-cron-jobs\" \"pnpm run run-bulldozer-studio\"",
"dev:inspect": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--inspect\" pnpm run dev",
"dev:profile": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--experimental-cpu-prof\" pnpm run dev",
"build": "pnpm run codegen && next build",
Expand Down Expand Up @@ -48,7 +48,8 @@
"run-cron-jobs": "pnpm run with-env:dev tsx scripts/run-cron-jobs.ts",
"run-cron-jobs:test": "pnpm run with-env:test tsx scripts/run-cron-jobs.ts",
"verify-data-integrity": "pnpm run with-env:dev tsx scripts/verify-data-integrity/index.ts",
"run-email-queue": "pnpm run with-env:dev tsx scripts/run-email-queue.ts"
"run-email-queue": "pnpm run with-env:dev tsx scripts/run-email-queue.ts",
"run-bulldozer-studio": "pnpm run with-env:dev tsx watch --clear-screen=false scripts/run-bulldozer-studio.ts"
},
"prisma": {
"seed": "pnpm run db-seed-script"
Expand Down Expand Up @@ -93,6 +94,7 @@
"chokidar-cli": "^3.0.0",
"dotenv": "^16.4.5",
"dotenv-cli": "^7.3.0",
"elkjs": "^0.11.1",
"emailable": "^3.1.1",
"freestyle-sandboxes": "^0.1.6",
"jiti": "^2.6.1",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
-- CreateTable
CREATE TABLE "BulldozerStorageEngine" (
"id" UUID NOT NULL DEFAULT gen_random_uuid(),
"keyPath" JSONB[] NOT NULL,
"keyPathParent" JSONB[] GENERATED ALWAYS AS (
CASE
WHEN cardinality("keyPath") = 0 THEN NULL
ELSE "keyPath"[1:cardinality("keyPath") - 1]
END
) STORED,
"value" JSONB NOT NULL,

CONSTRAINT "BulldozerStorageEngine_pkey" PRIMARY KEY ("id"),
CONSTRAINT "BulldozerStorageEngine_keyPath_key" UNIQUE ("keyPath"),
CONSTRAINT "BulldozerStorageEngine_keyPathParent_fkey"
FOREIGN KEY ("keyPathParent")
REFERENCES "BulldozerStorageEngine"("keyPath")
ON DELETE CASCADE
);

-- Seed root hierarchy rows used by all tables.
INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
VALUES
('00000000-0000-0000-0000-000000000100'::uuid, ARRAY[]::jsonb[], 'null'::jsonb);

INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
VALUES
('00000000-0000-0000-0000-000000000101'::uuid, ARRAY[to_jsonb('table'::text)]::jsonb[], 'null'::jsonb);

-- CreateIndex
CREATE INDEX "BulldozerStorageEngine_keyPathParent_idx" ON "BulldozerStorageEngine"("keyPathParent");
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
import type { Sql } from "postgres";
import { expect } from "vitest";

export const postMigration = async (sql: Sql) => {
await sql`
INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
VALUES
('00000000-0000-0000-0000-000000000001'::uuid, ARRAY[to_jsonb('root'::text)]::jsonb[], '{"node":"root"}'::jsonb),
('00000000-0000-0000-0000-000000000002'::uuid, ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text)]::jsonb[], '{"node":"branch"}'::jsonb),
('00000000-0000-0000-0000-000000000003'::uuid, ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text), to_jsonb('leaf'::text)]::jsonb[], '{"node":"leaf"}'::jsonb),
('00000000-0000-0000-0000-000000000004'::uuid, ARRAY[to_jsonb('root'::text), to_jsonb('other'::text)]::jsonb[], '{"node":"other"}'::jsonb)
`;

const exactRows = await sql`
SELECT "value"
FROM "BulldozerStorageEngine"
WHERE "keyPath" = ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text), to_jsonb('leaf'::text)]::jsonb[]
`;

expect(exactRows).toHaveLength(1);
expect(exactRows[0].value).toEqual({ node: "leaf" });

const nestedRows = await sql`
SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath"
FROM "BulldozerStorageEngine"
WHERE "keyPath"[1:cardinality(ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text)]::jsonb[])] = ARRAY[to_jsonb('root'::text), to_jsonb('branch'::text)]::jsonb[]
ORDER BY "keyPath"
`;

expect(nestedRows.map((row) => row.keyPath)).toEqual([
"root.branch",
"root.branch.leaf",
]);

const directChildrenRows = await sql`
SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath"
FROM "BulldozerStorageEngine"
WHERE "keyPathParent" = ARRAY[to_jsonb('root'::text)]::jsonb[]
ORDER BY "keyPath"
`;

expect(directChildrenRows.map((row) => row.keyPath)).toEqual([
"root.branch",
"root.other",
]);

const indexRows = await sql`
SELECT "indexname"
FROM pg_indexes
WHERE schemaname = 'public'
AND tablename = 'BulldozerStorageEngine'
AND indexname IN (
'BulldozerStorageEngine_keyPath_key',
'BulldozerStorageEngine_keyPathParent_idx'
)
ORDER BY "indexname"
`;

expect(indexRows.map((row) => row.indexname)).toEqual([
"BulldozerStorageEngine_keyPathParent_idx",
"BulldozerStorageEngine_keyPath_key",
]);

const seededRootRows = await sql`
SELECT array_to_string(ARRAY(SELECT x #>> '{}' FROM unnest("keyPath") AS x), '.') AS "keyPath"
FROM "BulldozerStorageEngine"
WHERE "keyPath" IN (ARRAY[]::jsonb[], ARRAY[to_jsonb('table'::text)]::jsonb[])
ORDER BY cardinality("keyPath")
`;

expect(seededRootRows.map((row) => row.keyPath)).toEqual([
"",
"table",
]);

const generatedColumnRows = await sql`
SELECT attname
FROM pg_attribute
WHERE attrelid = 'public."BulldozerStorageEngine"'::regclass
AND attname = 'keyPathParent'
AND attgenerated = 's'
`;

expect(generatedColumnRows).toHaveLength(1);

const fkConstraintRows = await sql`
SELECT conname
FROM pg_constraint
WHERE conrelid = 'public."BulldozerStorageEngine"'::regclass
AND conname = 'BulldozerStorageEngine_keyPathParent_fkey'
`;

expect(fkConstraintRows).toHaveLength(1);

await expect(sql`
INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "keyPathParent", "value")
VALUES (
'00000000-0000-0000-0000-000000000005'::uuid,
ARRAY[to_jsonb('root'::text), to_jsonb('mismatch'::text)]::jsonb[],
ARRAY[]::jsonb[],
'{"node":"invalid"}'::jsonb
)
`).rejects.toThrow('cannot insert a non-DEFAULT value into column "keyPathParent"');

await expect(sql`
INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
VALUES (
'00000000-0000-0000-0000-000000000006'::uuid,
ARRAY[to_jsonb('missing-parent'::text), to_jsonb('child'::text)]::jsonb[],
'{"node":"invalid-fk"}'::jsonb
)
`).rejects.toThrow('BulldozerStorageEngine_keyPathParent_fkey');
};
14 changes: 12 additions & 2 deletions apps/backend/prisma/schema.prisma
Original file line number Diff line number Diff line change
Expand Up @@ -283,8 +283,8 @@ model ProjectUser {
signUpEmailBase String?

// Sign-up risk scores (0-100, set at sign-up time)
signUpRiskScoreBot Int @default(0) @db.SmallInt
signUpRiskScoreFreeTrialAbuse Int @default(0) @db.SmallInt
signUpRiskScoreBot Int @default(0) @db.SmallInt
signUpRiskScoreFreeTrialAbuse Int @default(0) @db.SmallInt

projectUserOAuthAccounts ProjectUserOAuthAccount[]
teamMembers TeamMember[]
Expand Down Expand Up @@ -1247,6 +1247,16 @@ model OutgoingRequest {
@@index([startedFulfillingAt, deduplicationKey])
}

model BulldozerStorageEngine {
Comment thread
N2D4 marked this conversation as resolved.
id String @id @default(uuid()) @db.Uuid
keyPath Json[]
keyPathParent Unsupported("jsonb[]")
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: if this is nullable, shouldn't it be Unsupported("jsonb[]")? ?

value Json

@@unique([keyPath], map: "BulldozerStorageEngine_keyPath_key")
@@index([keyPathParent], map: "BulldozerStorageEngine_keyPathParent_idx")
}

model DeletedRow {
id String @id @default(uuid()) @db.Uuid
tenancyId String @db.Uuid
Expand Down
Loading
Loading