Skip to content

Commit 4b400e2

Browse files
N2D4N2D4
committed
Comments from Aman
1 parent e353232 commit 4b400e2

File tree

11 files changed

+578
-135
lines changed

11 files changed

+578
-135
lines changed

‎apps/backend/src/lib/bulldozer/db/index.test.ts‎

Lines changed: 311 additions & 0 deletions
Large diffs are not rendered by default.

‎apps/backend/src/lib/bulldozer/db/index.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,11 @@ export type Table<GK extends Json, SK extends Json, RD extends RowData> = {
1515

1616
// Query groups and rows
1717
listGroups(options: { start: SqlExpression<GK> | "start", end: SqlExpression<GK> | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery<Iterable<{ groupKey: GK }>>,
18-
listRowsInGroup(options: { groupKey?: SqlExpression<GK>, start: SqlExpression<SK> | "start", end: SqlExpression<SK> | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery<Iterable<{ rowIdentifier: RowIdentifier, rowSortKey: SK, rowData: RD }>>,
18+
/**
19+
* Rows queried across all groups may include `groupKey`; rows queried for a specific `groupKey`
20+
* may omit it.
21+
*/
22+
listRowsInGroup(options: { groupKey?: SqlExpression<GK>, start: SqlExpression<SK> | "start", end: SqlExpression<SK> | "end", startInclusive: boolean, endInclusive: boolean }): SqlQuery<Iterable<{ groupKey?: GK, rowIdentifier: RowIdentifier, rowSortKey: SK, rowData: RD }>>,
1923

2024
// Sorting and grouping
2125
compareGroupKeys(a: SqlExpression<GK>, b: SqlExpression<GK>): SqlExpression<number>,

‎apps/backend/src/lib/bulldozer/db/tables/concat-table.ts‎

Lines changed: 63 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,10 @@ export function declareConcatTable<
3030
throw new StackAssertionError("declareConcatTable requires at least one input table", { tableId: options.tableId });
3131
})();
3232
const referenceCompareGroupKeysSql = firstTable.compareGroupKeys(sqlExpression`$1`, sqlExpression`$2`).sql;
33-
const referenceCompareSortKeysSql = firstTable.compareSortKeys(sqlExpression`$1`, sqlExpression`$2`).sql;
3433
for (const table of tables) {
3534
const compareGroupKeysSql = table.compareGroupKeys(sqlExpression`$1`, sqlExpression`$2`).sql;
36-
const compareSortKeysSql = table.compareSortKeys(sqlExpression`$1`, sqlExpression`$2`).sql;
37-
if (compareGroupKeysSql !== referenceCompareGroupKeysSql || compareSortKeysSql !== referenceCompareSortKeysSql) {
38-
throw new StackAssertionError("declareConcatTable requires comparator-compatible input tables", {
35+
if (compareGroupKeysSql !== referenceCompareGroupKeysSql) {
36+
throw new StackAssertionError("declareConcatTable requires group-comparator-compatible input tables", {
3937
tableId: options.tableId,
4038
tableDebugId: tableIdToDebugString(table.tableId),
4139
});
@@ -98,27 +96,43 @@ export function declareConcatTable<
9896
`;
9997
}).join("\nUNION ALL\n");
10098
};
101-
102-
tables.forEach((table, tableIndex) => {
103-
table.registerRowChangeTrigger((changesTable) => {
104-
const concatChangesTableName = `concat_changes_${generateSecureRandomString()}`;
105-
return [
106-
sqlQuery`
107-
SELECT
108-
"changes"."groupKey" AS "groupKey",
109-
${rawExpression<RowIdentifier>(createConcatenatedRowIdentifierSql(tableIndex, `"changes"."rowIdentifier"`))} AS "rowIdentifier",
110-
'null'::jsonb AS "oldRowSortKey",
111-
'null'::jsonb AS "newRowSortKey",
112-
"changes"."oldRowData" AS "oldRowData",
113-
"changes"."newRowData" AS "newRowData"
114-
FROM ${changesTable} AS "changes"
115-
WHERE ${isInitializedExpression}
116-
AND ${rawExpression<boolean>(getInputInitializedSql(table))}
117-
`.toStatement(concatChangesTableName),
118-
...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(concatChangesTableName))),
119-
];
99+
const createInputTriggerStatements = (
100+
table: Table<GK, any, RD>,
101+
tableIndex: number,
102+
changesTable: SqlExpression<{ __brand: "$SQL_Table" }>,
103+
) => {
104+
const concatChangesTableName = `concat_changes_${generateSecureRandomString()}`;
105+
return [
106+
sqlQuery`
107+
SELECT
108+
"changes"."groupKey" AS "groupKey",
109+
${rawExpression<RowIdentifier>(createConcatenatedRowIdentifierSql(tableIndex, `"changes"."rowIdentifier"`))} AS "rowIdentifier",
110+
'null'::jsonb AS "oldRowSortKey",
111+
'null'::jsonb AS "newRowSortKey",
112+
"changes"."oldRowData" AS "oldRowData",
113+
"changes"."newRowData" AS "newRowData"
114+
FROM ${changesTable} AS "changes"
115+
WHERE ${isInitializedExpression}
116+
AND ${rawExpression<boolean>(getInputInitializedSql(table))}
117+
`.toStatement(concatChangesTableName),
118+
...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(concatChangesTableName))),
119+
];
120+
};
121+
let inputTriggerRegistrations: Array<{ deregister: () => void }> = [];
122+
const ensureInputTriggerRegistrations = () => {
123+
if (inputTriggerRegistrations.length > 0) return;
124+
inputTriggerRegistrations = tables.map((table, tableIndex) => {
125+
return table.registerRowChangeTrigger((changesTable) => {
126+
return createInputTriggerStatements(table, tableIndex, changesTable);
127+
});
120128
});
121-
});
129+
};
130+
const deregisterInputTriggers = () => {
131+
for (const registration of inputTriggerRegistrations) {
132+
registration.deregister();
133+
}
134+
inputTriggerRegistrations = [];
135+
};
122136

123137
return {
124138
tableId: options.tableId,
@@ -166,25 +180,31 @@ export function declareConcatTable<
166180
`,
167181
compareGroupKeys: firstTable.compareGroupKeys,
168182
compareSortKeys: () => sqlExpression`0`,
169-
init: () => [sqlStatement`
170-
INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
171-
VALUES
172-
(gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb),
173-
(gen_random_uuid(), ${sqlArray([...getTablePathSegments(options.tableId), quoteSqlJsonbLiteral("table")])}::jsonb[], 'null'::jsonb),
174-
(gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}::jsonb[], 'null'::jsonb),
175-
(gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[], '{ "version": 1 }'::jsonb)
176-
`],
177-
delete: () => [sqlStatement`
178-
WITH RECURSIVE "pathsToDelete" AS (
179-
SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
180-
UNION ALL
181-
SELECT "BulldozerStorageEngine"."keyPath" AS "path"
182-
FROM "BulldozerStorageEngine"
183-
INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
184-
)
185-
DELETE FROM "BulldozerStorageEngine"
186-
WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
187-
`],
183+
init: () => {
184+
ensureInputTriggerRegistrations();
185+
return [sqlStatement`
186+
INSERT INTO "BulldozerStorageEngine" ("id", "keyPath", "value")
187+
VALUES
188+
(gen_random_uuid(), ${getTablePath(options.tableId)}, 'null'::jsonb),
189+
(gen_random_uuid(), ${sqlArray([...getTablePathSegments(options.tableId), quoteSqlJsonbLiteral("table")])}::jsonb[], 'null'::jsonb),
190+
(gen_random_uuid(), ${getStorageEnginePath(options.tableId, [])}::jsonb[], 'null'::jsonb),
191+
(gen_random_uuid(), ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[], '{ "version": 1 }'::jsonb)
192+
`];
193+
},
194+
delete: () => {
195+
deregisterInputTriggers();
196+
return [sqlStatement`
197+
WITH RECURSIVE "pathsToDelete" AS (
198+
SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
199+
UNION ALL
200+
SELECT "BulldozerStorageEngine"."keyPath" AS "path"
201+
FROM "BulldozerStorageEngine"
202+
INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
203+
)
204+
DELETE FROM "BulldozerStorageEngine"
205+
WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
206+
`];
207+
},
188208
isInitialized: () => isInitializedExpression,
189209
registerRowChangeTrigger: (trigger) => {
190210
const id = generateSecureRandomString();

‎apps/backend/src/lib/bulldozer/db/tables/flat-map-table.ts‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,7 @@ export function declareFlatMapTable<
3333
WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[]
3434
)
3535
`;
36-
37-
options.fromTable.registerRowChangeTrigger((fromChangesTable) => {
36+
const createFromTableTriggerStatements = (fromChangesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => {
3837
const mappedChangesTableName = `mapped_changes_${generateSecureRandomString()}`;
3938
const oldFlatRowsTableName = `old_flat_rows_${generateSecureRandomString()}`;
4039
const newFlatRowsTableName = `new_flat_rows_${generateSecureRandomString()}`;
@@ -204,7 +203,18 @@ export function declareFlatMapTable<
204203
`.toStatement(flatMapChangesTableName),
205204
...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(flatMapChangesTableName))),
206205
];
207-
});
206+
};
207+
let fromTableTriggerRegistration: null | { deregister: () => void } = null;
208+
const ensureFromTableTriggerRegistration = () => {
209+
if (fromTableTriggerRegistration != null) return;
210+
fromTableTriggerRegistration = options.fromTable.registerRowChangeTrigger((fromChangesTable) => {
211+
return createFromTableTriggerStatements(fromChangesTable);
212+
});
213+
};
214+
const deregisterFromTableTrigger = () => {
215+
fromTableTriggerRegistration?.deregister();
216+
fromTableTriggerRegistration = null;
217+
};
208218

209219
return {
210220
tableId: options.tableId,
@@ -218,6 +228,7 @@ export function declareFlatMapTable<
218228
compareGroupKeys: options.fromTable.compareGroupKeys,
219229
compareSortKeys: (a, b) => sqlExpression` 0 `,
220230
init: () => {
231+
ensureFromTableTriggerRegistration();
221232
const fromGroupsTableName = `from_groups_${generateSecureRandomString()}`;
222233
const fromRowsTableName = `from_rows_${generateSecureRandomString()}`;
223234
const mappedRowsTableName = `mapped_rows_${generateSecureRandomString()}`;
@@ -324,17 +335,20 @@ export function declareFlatMapTable<
324335
`,
325336
];
326337
},
327-
delete: () => [sqlStatement`
328-
WITH RECURSIVE "pathsToDelete" AS (
329-
SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
330-
UNION ALL
331-
SELECT "BulldozerStorageEngine"."keyPath" AS "path"
332-
FROM "BulldozerStorageEngine"
333-
INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
334-
)
335-
DELETE FROM "BulldozerStorageEngine"
336-
WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
337-
`],
338+
delete: () => {
339+
deregisterFromTableTrigger();
340+
return [sqlStatement`
341+
WITH RECURSIVE "pathsToDelete" AS (
342+
SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
343+
UNION ALL
344+
SELECT "BulldozerStorageEngine"."keyPath" AS "path"
345+
FROM "BulldozerStorageEngine"
346+
INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
347+
)
348+
DELETE FROM "BulldozerStorageEngine"
349+
WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
350+
`];
351+
},
338352
isInitialized: () => isInitializedExpression,
339353
listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery`
340354
SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey

‎apps/backend/src/lib/bulldozer/db/tables/group-by-table.ts‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,7 @@ export function declareGroupByTable<
3333
WHERE "keyPath" = ${getStorageEnginePath(options.tableId, ["metadata"])}::jsonb[]
3434
)
3535
`;
36-
37-
options.fromTable.registerRowChangeTrigger((fromChangesTable) => {
36+
const createFromTableTriggerStatements = (fromChangesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => {
3837
const mappedChangesTableName = `mapped_changes_${generateSecureRandomString()}`;
3938
const groupedChangesTableName = `grouped_changes_${generateSecureRandomString()}`;
4039

@@ -184,7 +183,18 @@ export function declareGroupByTable<
184183
`.toStatement(groupedChangesTableName),
185184
...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(groupedChangesTableName))),
186185
];
187-
});
186+
};
187+
let fromTableTriggerRegistration: null | { deregister: () => void } = null;
188+
const ensureFromTableTriggerRegistration = () => {
189+
if (fromTableTriggerRegistration != null) return;
190+
fromTableTriggerRegistration = options.fromTable.registerRowChangeTrigger((fromChangesTable) => {
191+
return createFromTableTriggerStatements(fromChangesTable);
192+
});
193+
};
194+
const deregisterFromTableTrigger = () => {
195+
fromTableTriggerRegistration?.deregister();
196+
fromTableTriggerRegistration = null;
197+
};
188198

189199
return {
190200
tableId: options.tableId,
@@ -198,6 +208,7 @@ export function declareGroupByTable<
198208
compareGroupKeys,
199209
compareSortKeys: (a, b) => sqlExpression` 0 `,
200210
init: () => {
211+
ensureFromTableTriggerRegistration();
201212
const fromTableAllRowsTableName = `from_table_all_rows_${generateSecureRandomString()}`;
202213
const fromTableRowsWithGroupKeyTableName = `from_table_rows_with_group_key_${generateSecureRandomString()}`;
203214

@@ -262,17 +273,20 @@ export function declareGroupByTable<
262273
`,
263274
];
264275
},
265-
delete: () => [sqlStatement`
266-
WITH RECURSIVE "pathsToDelete" AS (
267-
SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
268-
UNION ALL
269-
SELECT "BulldozerStorageEngine"."keyPath" AS "path"
270-
FROM "BulldozerStorageEngine"
271-
INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
272-
)
273-
DELETE FROM "BulldozerStorageEngine"
274-
WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
275-
`],
276+
delete: () => {
277+
deregisterFromTableTrigger();
278+
return [sqlStatement`
279+
WITH RECURSIVE "pathsToDelete" AS (
280+
SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
281+
UNION ALL
282+
SELECT "BulldozerStorageEngine"."keyPath" AS "path"
283+
FROM "BulldozerStorageEngine"
284+
INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
285+
)
286+
DELETE FROM "BulldozerStorageEngine"
287+
WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
288+
`];
289+
},
276290
isInitialized: () => sqlExpression`
277291
EXISTS (
278292
SELECT 1 FROM "BulldozerStorageEngine"

‎apps/backend/src/lib/bulldozer/db/tables/l-fold-table.ts‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ export function declareLFoldTable<
107107
}
108108
`;
109109

110-
sourceSortTable.registerRowChangeTrigger((fromChangesTable) => {
110+
const createSourceSortTriggerStatements = (fromChangesTable: SqlExpression<{ __brand: "$SQL_Table" }>) => {
111111
const normalizedChangesTableName = `normalized_changes_${generateSecureRandomString()}`;
112112
const boundaryCandidatesTableName = `boundary_candidates_${generateSecureRandomString()}`;
113113
const earliestBoundaryCandidatesTableName = `earliest_boundary_candidates_${generateSecureRandomString()}`;
@@ -490,7 +490,18 @@ export function declareLFoldTable<
490490
`.toStatement(lfoldChangesTableName),
491491
...[...triggers.values()].flatMap((trigger) => trigger(quoteSqlIdentifier(lfoldChangesTableName))),
492492
];
493-
});
493+
};
494+
let sourceSortTriggerRegistration: null | { deregister: () => void } = null;
495+
const ensureSourceSortTriggerRegistration = () => {
496+
if (sourceSortTriggerRegistration != null) return;
497+
sourceSortTriggerRegistration = sourceSortTable.registerRowChangeTrigger((fromChangesTable) => {
498+
return createSourceSortTriggerStatements(fromChangesTable);
499+
});
500+
};
501+
const deregisterSourceSortTrigger = () => {
502+
sourceSortTriggerRegistration?.deregister();
503+
sourceSortTriggerRegistration = null;
504+
};
494505

495506
return {
496507
tableId: options.tableId,
@@ -505,6 +516,7 @@ export function declareLFoldTable<
505516
compareGroupKeys: options.fromTable.compareGroupKeys,
506517
compareSortKeys: options.fromTable.compareSortKeys,
507518
init: () => {
519+
ensureSourceSortTriggerRegistration();
508520
const firstSourceRowsTableName = `first_source_rows_${generateSecureRandomString()}`;
509521
const recomputedSourceStatesTableName = `recomputed_source_states_${generateSecureRandomString()}`;
510522
const newFoldRowsTableName = `new_fold_rows_${generateSecureRandomString()}`;
@@ -689,17 +701,20 @@ export function declareLFoldTable<
689701
`,
690702
];
691703
},
692-
delete: () => [sqlStatement`
693-
WITH RECURSIVE "pathsToDelete" AS (
694-
SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
695-
UNION ALL
696-
SELECT "BulldozerStorageEngine"."keyPath" AS "path"
697-
FROM "BulldozerStorageEngine"
698-
INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
699-
)
700-
DELETE FROM "BulldozerStorageEngine"
701-
WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
702-
`],
704+
delete: () => {
705+
deregisterSourceSortTrigger();
706+
return [sqlStatement`
707+
WITH RECURSIVE "pathsToDelete" AS (
708+
SELECT ${getTablePath(options.tableId)}::jsonb[] AS "path"
709+
UNION ALL
710+
SELECT "BulldozerStorageEngine"."keyPath" AS "path"
711+
FROM "BulldozerStorageEngine"
712+
INNER JOIN "pathsToDelete" ON "BulldozerStorageEngine"."keyPathParent" = "pathsToDelete"."path"
713+
)
714+
DELETE FROM "BulldozerStorageEngine"
715+
WHERE "keyPath" IN (SELECT "path" FROM "pathsToDelete")
716+
`];
717+
},
703718
isInitialized: () => isInitializedExpression,
704719
listGroups: ({ start, end, startInclusive, endInclusive }) => sqlQuery`
705720
SELECT "groupPath"."keyPath"[cardinality("groupPath"."keyPath")] AS groupKey

0 commit comments

Comments
 (0)