Skip to content

Commit 484c3a6

Browse files
authored
clickhouse setup (#1032)
1 parent 4fc5ddc commit 484c3a6

30 files changed

Lines changed: 3980 additions & 5 deletions

File tree

.github/workflows/docker-server-build-run.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,13 @@ jobs:
2222
docker run -d --name db -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=password -e POSTGRES_DB=stackframe -p 8128:5432 postgres:latest
2323
sleep 5
2424
docker logs db
25+
26+
- name: Setup clickhouse
27+
run: |
28+
docker run -d --name clickhouse -e CLICKHOUSE_DB=analytics -e CLICKHOUSE_USER=stackframe -e CLICKHOUSE_PASSWORD=password -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -p 8133:8123 clickhouse/clickhouse-server:25.10
29+
sleep 5
30+
docker logs clickhouse
31+
2532
2633
- name: Build Docker image
2734
run: docker build -f docker/server/Dockerfile -t server .

apps/backend/.env

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,13 @@ STACK_QSTASH_TOKEN=
8181
STACK_QSTASH_CURRENT_SIGNING_KEY=
8282
STACK_QSTASH_NEXT_SIGNING_KEY=
8383

84+
# Clickhouse
85+
STACK_CLICKHOUSE_URL=# URL of the Clickhouse instance
86+
STACK_CLICKHOUSE_ADMIN_USER=# username of the admin account
87+
STACK_CLICKHOUSE_ADMIN_PASSWORD=# password of the admin account
88+
STACK_CLICKHOUSE_EXTERNAL_PASSWORD=# a randomly generated secure string. The user account will be created automatically
89+
90+
8491
# Misc
8592
STACK_ACCESS_TOKEN_EXPIRATION_TIME=# enter the expiration time for the access token here. Optional, don't specify it for default value
8693
STACK_SETUP_ADMIN_GITHUB_ID=# enter the account ID of the admin user here, and after running the seed script they will be able to access the internal project in the Stack dashboard. Optional, don't specify it for default value

apps/backend/.env.development

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,3 +73,9 @@ STACK_QSTASH_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}25
7373
STACK_QSTASH_TOKEN=eyJVc2VySUQiOiJkZWZhdWx0VXNlciIsIlBhc3N3b3JkIjoiZGVmYXVsdFBhc3N3b3JkIn0=
7474
STACK_QSTASH_CURRENT_SIGNING_KEY=sig_7kYjw48mhY7kAjqNGcy6cr29RJ6r
7575
STACK_QSTASH_NEXT_SIGNING_KEY=sig_5ZB6DVzB1wjE8S6rZ7eenA8Pdnhs
76+
77+
# Clickhouse
78+
STACK_CLICKHOUSE_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}36
79+
STACK_CLICKHOUSE_ADMIN_USER=stackframe
80+
STACK_CLICKHOUSE_ADMIN_PASSWORD=PASSWORD-PLACEHOLDER--9gKyMxJeMx
81+
STACK_CLICKHOUSE_EXTERNAL_PASSWORD=PASSWORD-PLACEHOLDER--EZeHscBMzE

apps/backend/package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
"codegen": "pnpm run with-env pnpm run generate-migration-imports && pnpm run with-env bash -c 'if [ \"$STACK_ACCELERATE_ENABLED\" = \"true\" ]; then pnpm run prisma generate --no-engine; else pnpm run codegen-prisma; fi' && pnpm run codegen-docs && pnpm run codegen-route-info",
2626
"codegen:watch": "concurrently -n \"prisma,docs,route-info,migration-imports\" -k \"pnpm run codegen-prisma:watch\" \"pnpm run codegen-docs:watch\" \"pnpm run codegen-route-info:watch\" \"pnpm run generate-migration-imports:watch\"",
2727
"psql-inner": "psql $(echo $STACK_DATABASE_CONNECTION_STRING | sed 's/\\?.*$//')",
28+
"clickhouse": "pnpm run with-env clickhouse-client --host localhost --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}37 --user stackframe --password PASSWORD-PLACEHOLDER--9gKyMxJeMx",
2829
"psql": "pnpm run with-env:dev pnpm run psql-inner",
2930
"prisma-studio": "pnpm run with-env:dev prisma studio --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}06 --browser none",
3031
"prisma:dev": "pnpm run with-env:dev prisma",
@@ -50,6 +51,7 @@
5051
"dependencies": {
5152
"@ai-sdk/openai": "^1.3.23",
5253
"@aws-sdk/client-s3": "^3.855.0",
54+
"@clickhouse/client": "^1.14.0",
5355
"@node-oauth/oauth2-server": "^5.1.0",
5456
"@opentelemetry/api": "^1.9.0",
5557
"@opentelemetry/api-logs": "^0.53.0",
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import { getClickhouseAdminClient } from "@/lib/clickhouse";
2+
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
3+
4+
export async function runClickhouseMigrations() {
5+
console.log("[Clickhouse] Running Clickhouse migrations...");
6+
const client = getClickhouseAdminClient();
7+
const clickhouseExternalPassword = getEnvVariable("STACK_CLICKHOUSE_EXTERNAL_PASSWORD");
8+
await client.exec({
9+
query: "CREATE USER IF NOT EXISTS limited_user IDENTIFIED WITH sha256_password BY {clickhouseExternalPassword:String}",
10+
query_params: { clickhouseExternalPassword },
11+
});
12+
// todo: create migration files
13+
await client.exec({ query: EXTERNAL_ANALYTICS_DB_SQL });
14+
await client.exec({ query: EVENTS_TABLE_BASE_SQL });
15+
await client.exec({ query: EVENTS_VIEW_SQL });
16+
const queries = [
17+
"REVOKE ALL PRIVILEGES ON *.* FROM limited_user;",
18+
"REVOKE ALL FROM limited_user;",
19+
"GRANT SELECT ON default.events TO limited_user;",
20+
];
21+
await client.exec({
22+
query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user",
23+
});
24+
for (const query of queries) {
25+
await client.exec({ query });
26+
}
27+
console.log("[Clickhouse] Clickhouse migrations complete");
28+
await client.close();
29+
}
30+
31+
const EVENTS_TABLE_BASE_SQL = `
32+
CREATE TABLE IF NOT EXISTS analytics_internal.events (
33+
event_type LowCardinality(String),
34+
event_at DateTime64(3, 'UTC'),
35+
data JSON,
36+
project_id String,
37+
branch_id String,
38+
user_id String,
39+
team_id String,
40+
refresh_token_id String,
41+
is_anonymous Boolean,
42+
session_id String,
43+
ip_address String,
44+
created_at DateTime64(3, 'UTC') DEFAULT now64(3)
45+
)
46+
ENGINE MergeTree
47+
PARTITION BY toYYYYMM(event_at)
48+
ORDER BY (project_id, branch_id, event_at);
49+
`;
50+
51+
const EVENTS_VIEW_SQL = `
52+
CREATE OR REPLACE VIEW default.events
53+
SQL SECURITY DEFINER
54+
AS
55+
SELECT *
56+
FROM analytics_internal.events;
57+
`;
58+
59+
const EXTERNAL_ANALYTICS_DB_SQL = `
60+
CREATE DATABASE IF NOT EXISTS analytics_internal;
61+
`;

apps/backend/scripts/db-migrations.ts

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,25 @@ import { applyMigrations } from "@/auto-migrations";
22
import { MIGRATION_FILES_DIR, getMigrationFiles } from "@/auto-migrations/utils";
33
import { Prisma } from "@/generated/prisma/client";
44
import { globalPrismaClient, globalPrismaSchema, sqlQuoteIdent } from "@/prisma-client";
5-
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
65
import { spawnSync } from "child_process";
76
import fs from "fs";
87
import path from "path";
98
import * as readline from "readline";
109
import { seed } from "../prisma/seed";
10+
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
11+
import { runClickhouseMigrations } from "./clickhouse-migrations";
12+
import { getClickhouseAdminClient } from "@/lib/clickhouse";
13+
14+
const getClickhouseClient = () => getClickhouseAdminClient();
1115

1216
const dropSchema = async () => {
1317
await globalPrismaClient.$executeRaw(Prisma.sql`DROP SCHEMA ${sqlQuoteIdent(globalPrismaSchema)} CASCADE`);
1418
await globalPrismaClient.$executeRaw(Prisma.sql`CREATE SCHEMA ${sqlQuoteIdent(globalPrismaSchema)}`);
1519
await globalPrismaClient.$executeRaw(Prisma.sql`GRANT ALL ON SCHEMA ${sqlQuoteIdent(globalPrismaSchema)} TO postgres`);
1620
await globalPrismaClient.$executeRaw(Prisma.sql`GRANT ALL ON SCHEMA ${sqlQuoteIdent(globalPrismaSchema)} TO public`);
21+
const clickhouseClient = getClickhouseClient();
22+
await clickhouseClient.command({ query: "DROP DATABASE IF EXISTS analytics_internal" });
23+
await clickhouseClient.command({ query: "CREATE DATABASE IF NOT EXISTS analytics_internal" });
1724
};
1825

1926

@@ -163,6 +170,8 @@ const migrate = async (selectedMigrationFiles?: { migrationName: string, sql: st
163170

164171
console.log('='.repeat(60) + '\n');
165172

173+
await runClickhouseMigrations();
174+
166175
return result;
167176
};
168177

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import { getClickhouseExternalClient, getQueryTimingStats, isClickhouseConfigured } from "@/lib/clickhouse";
2+
import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
3+
import { KnownErrors } from "@stackframe/stack-shared";
4+
import { adaptSchema, adminAuthTypeSchema, jsonSchema, yupBoolean, yupMixed, yupNumber, yupObject, yupRecord, yupString } from "@stackframe/stack-shared/dist/schema-fields";
5+
import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
6+
import { Result } from "@stackframe/stack-shared/dist/utils/results";
7+
import { randomUUID } from "crypto";
8+
9+
export const POST = createSmartRouteHandler({
10+
metadata: { hidden: true },
11+
request: yupObject({
12+
auth: yupObject({
13+
type: adminAuthTypeSchema,
14+
tenancy: adaptSchema,
15+
}).defined(),
16+
body: yupObject({
17+
include_all_branches: yupBoolean().default(false),
18+
query: yupString().defined().nonEmpty(),
19+
params: yupRecord(yupString().defined(), yupMixed().defined()).default({}),
20+
timeout_ms: yupNumber().integer().min(1_000).default(10_000),
21+
}).defined(),
22+
}),
23+
response: yupObject({
24+
statusCode: yupNumber().oneOf([200]).defined(),
25+
bodyType: yupString().oneOf(["json"]).defined(),
26+
body: yupObject({
27+
result: jsonSchema.defined(),
28+
stats: yupObject({
29+
cpu_time: yupNumber().defined(),
30+
wall_clock_time: yupNumber().defined(),
31+
}).defined(),
32+
}).defined(),
33+
}),
34+
async handler({ body, auth }) {
35+
if (body.include_all_branches) {
36+
throw new StackAssertionError("include_all_branches is not supported yet");
37+
}
38+
if (!isClickhouseConfigured()) {
39+
throw new StackAssertionError("ClickHouse is not configured");
40+
}
41+
const client = getClickhouseExternalClient();
42+
const queryId = randomUUID();
43+
const resultSet = await Result.fromPromise(client.query({
44+
query: body.query,
45+
query_id: queryId,
46+
query_params: body.params,
47+
clickhouse_settings: {
48+
SQL_project_id: auth.tenancy.project.id,
49+
SQL_branch_id: auth.tenancy.branchId,
50+
max_execution_time: body.timeout_ms / 1000,
51+
readonly: "1",
52+
allow_ddl: 0,
53+
max_result_rows: MAX_RESULT_ROWS.toString(),
54+
max_result_bytes: MAX_RESULT_BYTES.toString(),
55+
result_overflow_mode: "throw",
56+
},
57+
format: "JSONEachRow",
58+
}));
59+
60+
if (resultSet.status === "error") {
61+
const message = getSafeClickhouseErrorMessage(resultSet.error);
62+
if (message === null) {
63+
throw new StackAssertionError("Unknown Clickhouse error", { cause: resultSet.error });
64+
}
65+
throw new KnownErrors.AnalyticsQueryError(message);
66+
}
67+
68+
const rows = await resultSet.data.json<Record<string, unknown>[]>();
69+
const stats = await getQueryTimingStats(client, queryId);
70+
71+
return {
72+
statusCode: 200,
73+
bodyType: "json",
74+
body: {
75+
result: rows,
76+
stats: {
77+
cpu_time: stats.cpu_time_ms,
78+
wall_clock_time: stats.wall_clock_time_ms,
79+
},
80+
},
81+
};
82+
},
83+
});
84+
85+
const SAFE_CLICKHOUSE_ERROR_CODES = [
86+
62, // SYNTAX_ERROR
87+
159, // TIMEOUT_EXCEEDED
88+
164, // READONLY
89+
158, // TOO_MANY_ROWS
90+
396, // TOO_MANY_ROWS_OR_BYTES
91+
636, // CANNOT_EXTRACT_TABLE_STRUCTURE
92+
];
93+
94+
const UNSAFE_CLICKHOUSE_ERROR_CODES = [
95+
36, // BAD_ARGUMENTS
96+
60, // UNKNOWN_TABLE
97+
497, // ACCESS_DENIED
98+
];
99+
100+
const DEFAULT_CLICKHOUSE_ERROR_MESSAGE = "Error during execution of this query.";
101+
const MAX_RESULT_ROWS = 10_000;
102+
const MAX_RESULT_BYTES = 10 * 1024 * 1024;
103+
104+
function getSafeClickhouseErrorMessage(error: unknown): string | null {
105+
if (typeof error !== "object" || error === null || !("code" in error) || typeof error.code !== "string") {
106+
return null;
107+
}
108+
const errorCode = Number(error.code);
109+
if (isNaN(errorCode)) {
110+
return null;
111+
}
112+
const message = "message" in error && typeof error.message === "string" ? error.message : null;
113+
if (SAFE_CLICKHOUSE_ERROR_CODES.includes(errorCode)) {
114+
return message;
115+
}
116+
if (UNSAFE_CLICKHOUSE_ERROR_CODES.includes(errorCode)) {
117+
return DEFAULT_CLICKHOUSE_ERROR_MESSAGE;
118+
}
119+
return null;
120+
}

0 commit comments

Comments
 (0)