From aea62269d507295f47129ff44662292a8179b09a Mon Sep 17 00:00:00 2001 From: arturka Date: Thu, 7 May 2026 15:35:09 +0200 Subject: [PATCH 1/3] feat: add postgres storage source --- Dockerfile | 4 ++ README.md | 7 ++- app/lib/utils.ts | 2 +- drizzle.config.ts | 32 +++++++++- env-example | 2 + package.json | 1 + pnpm-lock.yaml | 104 ++++++++++++++++++++++++++++++++- server/api/tasks/backup.get.ts | 22 ++++++- server/plugins/startup.ts | 14 ++--- shared/db/constants.ts | 14 +++++ shared/db/index.ts | 63 +++++++++++++++++--- shared/db/init.ts | 30 ++++++++++ shared/db/schema.postgres.ts | 50 ++++++++++++++++ shared/db/schema.sqlite.ts | 45 ++++++++++++++ shared/db/schema.ts | 52 ++++------------- shared/env.ts | 42 ++++++++++++- tests/helpers/db-test-utils.ts | 8 +-- tests/task-flow.nuxt.spec.ts | 8 ++- 18 files changed, 422 insertions(+), 78 deletions(-) create mode 100644 shared/db/constants.ts create mode 100644 shared/db/init.ts create mode 100644 shared/db/schema.postgres.ts create mode 100644 shared/db/schema.sqlite.ts diff --git a/Dockerfile b/Dockerfile index b1a0d2f..90324bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,9 @@ RUN corepack enable WORKDIR /app +ENV DB_DRIVER=sqlite +ENV DB_FILE_PATH=/tmp/taskiq-admin.db + # install dependencies COPY ./package.json /app/ COPY ./pnpm-lock.yaml /app/ @@ -33,6 +36,7 @@ COPY --from=build /app/entrypoint.sh /usr/app/entrypoint.sh EXPOSE 3000 ENV HOST=0.0.0.0 NODE_ENV=production +ENV DB_DRIVER=sqlite ENV DB_FILE_PATH=/usr/database/database.db ENV BACKUP_FILE_PATH=/usr/database/backup.db diff --git a/README.md b/README.md index 7054fa9..a79fec9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## Broker-agnostic admin panel for Taskiq -Standalone admin panel with all data stored in SQLite database +Standalone admin panel with task results storage in SQLite or Postgres database - [Broker-agnostic admin panel for Taskiq](#broker-agnostic-admin-panel-for-taskiq) @@ -91,6 +91,11 @@ volumes: 2) run `make dev` to run it locally in dev mode 3) run `make prod` to run it locally in prod mode +Environment variables for DB setup: +- `DB_DRIVER`: required, one of `sqlite` or `postgres` +- `DB_FILE_PATH`: required when `DB_DRIVER=sqlite` +- `DB_URL`: required when `DB_DRIVER=postgres` + ### Task States Let's assume we have a task 'do_smth', there are all states it can embrace: 1) `queued` - the task has been sent to the queue without an error diff --git a/app/lib/utils.ts b/app/lib/utils.ts index 6475307..86cad5a 100644 --- a/app/lib/utils.ts +++ b/app/lib/utils.ts @@ -2,7 +2,7 @@ import dayjs from 'dayjs' import utc from 'dayjs/plugin/utc.js' import { type ClassValue, clsx } from 'clsx' import { twMerge } from 'tailwind-merge' -import type { TaskSelect } from '~/server/db/schema' +import type { TaskSelect } from '../../shared/db/schema' dayjs.extend(utc) diff --git a/drizzle.config.ts b/drizzle.config.ts index 7cd020c..e298746 100644 --- a/drizzle.config.ts +++ b/drizzle.config.ts @@ -1,11 +1,37 @@ import 'dotenv/config' import { defineConfig } from 'drizzle-kit' +const dbDriver = process.env.DB_DRIVER + +if (!dbDriver) { + throw new Error('Environment variable DB_DRIVER is required') +} + +if (dbDriver !== 'sqlite' && dbDriver !== 'postgres') { + throw new Error( + 'Environment variable DB_DRIVER must be "sqlite" or "postgres"' + ) +} + +const dbUrl = + dbDriver === 'sqlite' ? process.env.DB_FILE_PATH : process.env.DB_URL + +if (!dbUrl) { + throw new Error( + dbDriver === 'sqlite' + ? 'Environment variable DB_FILE_PATH is required for sqlite driver' + : 'Environment variable DB_URL is required for postgres driver' + ) +} + export default defineConfig({ out: './drizzle', - schema: './shared/db/schema.ts', - dialect: 'sqlite', + schema: + dbDriver === 'sqlite' + ? './shared/db/schema.sqlite.ts' + : './shared/db/schema.postgres.ts', + dialect: dbDriver === 'sqlite' ? 'sqlite' : 'postgresql', dbCredentials: { - url: process.env.DB_FILE_PATH! + url: dbUrl } }) diff --git a/env-example b/env-example index 6cccaf3..4254473 100644 --- a/env-example +++ b/env-example @@ -1,3 +1,5 @@ +DB_DRIVER=sqlite DB_FILE_PATH=database/database.db +DB_URL=postgres://postgres:postgres@localhost:5432/taskiq_admin BACKUP_FILE_PATH=database/backup.db TASKIQ_ADMIN_API_TOKEN=supersecret \ No newline at end of file diff --git a/package.json b/package.json index 713c795..6efe7dd 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "drizzle-orm": "^0.45.1", "lucide-vue-next": "^0.524.0", "nuxt": "^4.2.2", + "pg": "^8.16.3", "reka-ui": "^2.6.1", "tailwind-merge": "^3.4.0", "tailwindcss": "^4.1.17", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2efe4ad..02f4628 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,13 +37,16 @@ dependencies: version: 17.2.3 drizzle-orm: specifier: ^0.45.1 - version: 0.45.1(@types/better-sqlite3@7.6.13)(better-sqlite3@12.6.2) + version: 0.45.1(@types/better-sqlite3@7.6.13)(better-sqlite3@12.6.2)(pg@8.20.0) lucide-vue-next: specifier: ^0.524.0 version: 0.524.0(vue@3.5.27) nuxt: specifier: ^4.2.2 version: 4.2.2(@vue/compiler-sfc@3.5.27)(better-sqlite3@12.6.2)(drizzle-orm@0.45.1)(tsx@4.21.0)(typescript@5.9.3)(vite@7.3.1) + pg: + specifier: ^8.16.3 + version: 8.20.0 reka-ui: specifier: ^2.6.1 version: 2.6.1(typescript@5.9.3)(vue@3.5.27) @@ -4422,7 +4425,7 @@ packages: optional: true dependencies: better-sqlite3: 12.6.2 - drizzle-orm: 0.45.1(@types/better-sqlite3@7.6.13)(better-sqlite3@12.6.2) + drizzle-orm: 0.45.1(@types/better-sqlite3@7.6.13)(better-sqlite3@12.6.2)(pg@8.20.0) dev: false /debug@4.4.1: @@ -4588,7 +4591,7 @@ packages: - supports-color dev: true - /drizzle-orm@0.45.1(@types/better-sqlite3@7.6.13)(better-sqlite3@12.6.2): + /drizzle-orm@0.45.1(@types/better-sqlite3@7.6.13)(better-sqlite3@12.6.2)(pg@8.20.0): resolution: {integrity: sha512-Te0FOdKIistGNPMq2jscdqngBRfBpC8uMFVwqjf6gtTVJHIQ/dosgV/CLBU2N4ZJBsXL5savCba9b0YJskKdcA==} peerDependencies: '@aws-sdk/client-rds-data': '>=3' @@ -4682,6 +4685,7 @@ packages: dependencies: '@types/better-sqlite3': 7.6.13 better-sqlite3: 12.6.2 + pg: 8.20.0 dev: false /duplexer@0.1.2: @@ -6544,6 +6548,68 @@ packages: /perfect-debounce@2.0.0: resolution: {integrity: sha512-fkEH/OBiKrqqI/yIgjR92lMfs2K8105zt/VT6+7eTjNwisrsh47CeIED9z58zI7DfKdH3uHAn25ziRZn3kgAow==} + /pg-cloudflare@1.3.0: + resolution: {integrity: sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==} + requiresBuild: true + dev: false + optional: true + + /pg-connection-string@2.12.0: + resolution: {integrity: sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==} + dev: false + + /pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + dev: false + + /pg-pool@3.13.0(pg@8.20.0): + resolution: {integrity: sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==} + peerDependencies: + pg: '>=8.0' + dependencies: + pg: 8.20.0 + dev: false + + /pg-protocol@1.13.0: + resolution: {integrity: sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==} + dev: false + + /pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + dev: false + + /pg@8.20.0: + resolution: {integrity: sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + dependencies: + pg-connection-string: 2.12.0 + pg-pool: 3.13.0(pg@8.20.0) + pg-protocol: 1.13.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.3.0 + dev: false + + /pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + dependencies: + split2: 4.2.0 + dev: false + /picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -6880,6 +6946,28 @@ packages: picocolors: 1.1.1 source-map-js: 1.2.1 + /postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + dev: false + + /postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + dev: false + + /postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + dev: false + + /postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + dependencies: + xtend: 4.0.2 + dev: false + /prebuild-install@7.1.3: resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} engines: {node: '>=10'} @@ -7374,6 +7462,11 @@ packages: engines: {node: '>=0.10.0'} dev: false + /split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + dev: false + /srvx@0.10.1: resolution: {integrity: sha512-A//xtfak4eESMWWydSRFUVvCTQbSwivnGCEf8YGPe2eHU0+Z6znfUTCPF0a7oV3sObSOcrXHlL6Bs9vVctfXdg==} engines: {node: '>=20.16.0'} @@ -8578,6 +8671,11 @@ packages: is-wsl: 3.1.0 dev: false + /xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + dev: false + /y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} diff --git a/server/api/tasks/backup.get.ts b/server/api/tasks/backup.get.ts index 213f5fc..0dbf5b4 100644 --- a/server/api/tasks/backup.get.ts +++ b/server/api/tasks/backup.get.ts @@ -1,11 +1,27 @@ import fs from 'fs' -import { db } from '../../../shared/db' +import { backupDatabase } from '../../../shared/db' import { utcNow } from '../../../shared/utils' import { envVariables } from '../../../shared/env' -import { defineEventHandler, sendStream, setHeader } from '#imports' +import { + createError, + defineEventHandler, + sendStream, + setHeader +} from '#imports' export default defineEventHandler(async (event) => { - await db.$client.backup(envVariables.backupFilePath) + try { + await backupDatabase(envVariables.backupFilePath) + } catch (error) { + throw createError({ + status: 400, + statusMessage: + error instanceof Error + ? error.message + : 'Could not create database backup' + }) + } + const stream = fs.createReadStream(envVariables.backupFilePath) const now = utcNow() diff --git a/server/plugins/startup.ts b/server/plugins/startup.ts index 4cc686d..b3806ec 100644 --- a/server/plugins/startup.ts +++ b/server/plugins/startup.ts @@ -1,18 +1,15 @@ -import fs from 'fs/promises' -import { db } from '../../shared/db' +import { db, initializeDatabase } from '../../shared/db' import { defineNitroPlugin } from '#imports' import { taskiqAdminSettingsTable } from '~~/shared/db/schema' import { SETTINGS } from '~~/shared/constants/settings' -export default defineNitroPlugin(async (nitroApp) => { +export default defineNitroPlugin(async () => { console.log('Running DB initialization...') - const sqlScript = await fs.readFile('dbschema.sql', 'utf-8') - - // executing the SQL script to create tables and indexes - db.$client.exec(sqlScript) + await initializeDatabase() // seeding default settings - db.insert(taskiqAdminSettingsTable) + await db + .insert(taskiqAdminSettingsTable) .values( Object.entries(SETTINGS).map(([_, value]) => ({ key: value.key, @@ -20,7 +17,6 @@ export default defineNitroPlugin(async (nitroApp) => { })) ) .onConflictDoNothing() - .run() console.log('DB initialization completed.') }) diff --git a/shared/db/constants.ts b/shared/db/constants.ts new file mode 100644 index 0000000..867a799 --- /dev/null +++ b/shared/db/constants.ts @@ -0,0 +1,14 @@ +export const DB_TABLE_NAMES = { + tasks: 'taskiq_admin_tasks', + settings: 'taskiq_admin_settings' +} as const + +export const DB_INDEX_NAMES = { + tasksState: 'idx_taskiq_admin_tasks__state', + tasksQueuedAt: 'idx_taskiq_admin_tasks__queued_at', + tasksStartedAt: 'idx_taskiq_admin_tasks__started_at', + tasksFinishedAt: 'idx_taskiq_admin_tasks__finished_at', + tasksExecutionTime: 'idx_taskiq_admin_tasks__execution_time', + tasksName: 'idx_taskiq_admin_tasks__name', + tasksWorker: 'idx_taskiq_admin_tasks__worker' +} as const diff --git a/shared/db/index.ts b/shared/db/index.ts index 60ce9a2..b08ff88 100644 --- a/shared/db/index.ts +++ b/shared/db/index.ts @@ -1,11 +1,60 @@ import Database from 'better-sqlite3' -import { drizzle } from 'drizzle-orm/better-sqlite3' +import { drizzle as drizzleSqlite } from 'drizzle-orm/better-sqlite3' +import { sql } from 'drizzle-orm' +import { drizzle as drizzlePostgres } from 'drizzle-orm/node-postgres' +import { Pool } from 'pg' +import { envVariables } from '../env' +import { dbInitializationStatements } from './init' -const conn = new Database(process.env.DB_FILE_PATH!) +const sqliteClient = + envVariables.dbDriver === 'sqlite' + ? new Database(envVariables.dbFilePath) + : null -conn.pragma('synchronous = NORMAL') -conn.pragma('journal_size_limit = 6144000') +const postgresClient = + envVariables.dbDriver === 'postgres' + ? new Pool({ connectionString: envVariables.dbUrl }) + : null -export const db = drizzle({ - client: conn -}) +const sqliteDb = sqliteClient ? drizzleSqlite({ client: sqliteClient }) : null +const postgresDb = postgresClient + ? drizzlePostgres({ client: postgresClient }) + : null + +export const db: any = + envVariables.dbDriver === 'sqlite' ? sqliteDb! : postgresDb! + +const executeStatement = async (statement: string) => { + if (envVariables.dbDriver === 'sqlite') { + sqliteClient!.exec(statement) + return + } + await postgresDb!.execute(sql.raw(statement)) +} + +export const initializeDatabase = async () => { + for (const statement of dbInitializationStatements[envVariables.dbDriver]) { + await executeStatement(statement) + } +} + +export const backupDatabase = async (backupFilePath: string) => { + if (envVariables.dbDriver !== 'sqlite') { + throw new Error('Database backup is supported only with sqlite driver') + } + await sqliteClient!.backup(backupFilePath) +} + +export const resetDatabaseForTests = async () => { + if (envVariables.dbDriver === 'sqlite') { + sqliteClient!.exec('PRAGMA foreign_keys = OFF') + sqliteClient!.exec('DROP TABLE IF EXISTS taskiq_admin_tasks') + sqliteClient!.exec('DROP TABLE IF EXISTS taskiq_admin_settings') + await initializeDatabase() + return + } + + await postgresDb!.execute(sql`DROP TABLE IF EXISTS taskiq_admin_tasks`) + await postgresDb!.execute(sql`DROP TABLE IF EXISTS taskiq_admin_settings`) + await initializeDatabase() +} diff --git a/shared/db/init.ts b/shared/db/init.ts new file mode 100644 index 0000000..197d85a --- /dev/null +++ b/shared/db/init.ts @@ -0,0 +1,30 @@ +import type { DBDriver } from '../env' +import { DB_INDEX_NAMES, DB_TABLE_NAMES } from './constants' + +export const dbInitializationStatements: Record = { + sqlite: [ + 'PRAGMA journal_mode = WAL', + 'PRAGMA synchronous = NORMAL', + 'PRAGMA journal_size_limit = 6144000', + `CREATE TABLE IF NOT EXISTS \`${DB_TABLE_NAMES.settings}\` (\`key\` text PRIMARY KEY NOT NULL, \`value\` text)`, + `CREATE TABLE IF NOT EXISTS \`${DB_TABLE_NAMES.tasks}\` (\`id\` text PRIMARY KEY NOT NULL, \`name\` text NOT NULL, \`state\` text NOT NULL, \`error\` text, \`worker\` text, \`execution_time\` real, \`queued_at\` integer NOT NULL, \`started_at\` integer, \`finished_at\` integer, \`args\` text, \`kwargs\` text, \`return_value\` text)`, + `CREATE INDEX IF NOT EXISTS \`${DB_INDEX_NAMES.tasksState}\` ON \`${DB_TABLE_NAMES.tasks}\` (\`state\`)`, + `CREATE INDEX IF NOT EXISTS \`${DB_INDEX_NAMES.tasksQueuedAt}\` ON \`${DB_TABLE_NAMES.tasks}\` (\`queued_at\`)`, + `CREATE INDEX IF NOT EXISTS \`${DB_INDEX_NAMES.tasksStartedAt}\` ON \`${DB_TABLE_NAMES.tasks}\` (\`started_at\`)`, + `CREATE INDEX IF NOT EXISTS \`${DB_INDEX_NAMES.tasksFinishedAt}\` ON \`${DB_TABLE_NAMES.tasks}\` (\`finished_at\`)`, + `CREATE INDEX IF NOT EXISTS \`${DB_INDEX_NAMES.tasksExecutionTime}\` ON \`${DB_TABLE_NAMES.tasks}\` (\`execution_time\`)`, + `CREATE INDEX IF NOT EXISTS \`${DB_INDEX_NAMES.tasksName}\` ON \`${DB_TABLE_NAMES.tasks}\` (name COLLATE NOCASE)`, + `CREATE INDEX IF NOT EXISTS \`${DB_INDEX_NAMES.tasksWorker}\` ON \`${DB_TABLE_NAMES.tasks}\` (worker COLLATE NOCASE)` + ], + postgres: [ + `CREATE TABLE IF NOT EXISTS \"${DB_TABLE_NAMES.settings}\" (\"key\" text PRIMARY KEY, \"value\" text)`, + `CREATE TABLE IF NOT EXISTS \"${DB_TABLE_NAMES.tasks}\" (\"id\" text PRIMARY KEY, \"name\" text NOT NULL, \"state\" text NOT NULL, \"error\" text, \"worker\" text, \"execution_time\" real, \"queued_at\" timestamp with time zone NOT NULL, \"started_at\" timestamp with time zone, \"finished_at\" timestamp with time zone, \"args\" jsonb, \"kwargs\" jsonb, \"return_value\" jsonb)`, + `CREATE INDEX IF NOT EXISTS \"${DB_INDEX_NAMES.tasksState}\" ON \"${DB_TABLE_NAMES.tasks}\" (\"state\")`, + `CREATE INDEX IF NOT EXISTS \"${DB_INDEX_NAMES.tasksQueuedAt}\" ON \"${DB_TABLE_NAMES.tasks}\" (\"queued_at\")`, + `CREATE INDEX IF NOT EXISTS \"${DB_INDEX_NAMES.tasksStartedAt}\" ON \"${DB_TABLE_NAMES.tasks}\" (\"started_at\")`, + `CREATE INDEX IF NOT EXISTS \"${DB_INDEX_NAMES.tasksFinishedAt}\" ON \"${DB_TABLE_NAMES.tasks}\" (\"finished_at\")`, + `CREATE INDEX IF NOT EXISTS \"${DB_INDEX_NAMES.tasksExecutionTime}\" ON \"${DB_TABLE_NAMES.tasks}\" (\"execution_time\")`, + `CREATE INDEX IF NOT EXISTS \"${DB_INDEX_NAMES.tasksName}\" ON \"${DB_TABLE_NAMES.tasks}\" (\"name\")`, + `CREATE INDEX IF NOT EXISTS \"${DB_INDEX_NAMES.tasksWorker}\" ON \"${DB_TABLE_NAMES.tasks}\" (\"worker\")` + ] +} diff --git a/shared/db/schema.postgres.ts b/shared/db/schema.postgres.ts new file mode 100644 index 0000000..91c5f2d --- /dev/null +++ b/shared/db/schema.postgres.ts @@ -0,0 +1,50 @@ +import { type InferSelectModel } from 'drizzle-orm' +import { + index, + jsonb, + pgTable, + real, + text, + timestamp +} from 'drizzle-orm/pg-core' +import { DB_INDEX_NAMES, DB_TABLE_NAMES } from './constants' + +export const tasksTable = pgTable( + DB_TABLE_NAMES.tasks, + { + id: text().primaryKey(), + name: text().notNull(), + state: text().notNull(), + error: text(), + worker: text(), + executionTime: real('execution_time'), + queuedAt: timestamp('queued_at', { withTimezone: true, mode: 'date' }).notNull(), + startedAt: timestamp('started_at', { withTimezone: true, mode: 'date' }), + finishedAt: timestamp('finished_at', { withTimezone: true, mode: 'date' }), + args: jsonb().$type>(), + kwargs: jsonb().$type>(), + returnValue: jsonb('return_value').$type<{ + return_value: any + }>() + }, + (t) => [ + index(DB_INDEX_NAMES.tasksState).on(t.state), + index(DB_INDEX_NAMES.tasksQueuedAt).on(t.queuedAt), + index(DB_INDEX_NAMES.tasksStartedAt).on(t.startedAt), + index(DB_INDEX_NAMES.tasksFinishedAt).on(t.finishedAt), + index(DB_INDEX_NAMES.tasksExecutionTime).on(t.executionTime), + index(DB_INDEX_NAMES.tasksName).on(t.name), + index(DB_INDEX_NAMES.tasksWorker).on(t.worker) + ] +) + +export type TaskSelect = InferSelectModel + +export const taskiqAdminSettingsTable = pgTable(DB_TABLE_NAMES.settings, { + key: text('key').primaryKey(), + value: text('value') +}) + +export type TaskiqAdminSettings = InferSelectModel< + typeof taskiqAdminSettingsTable +> diff --git a/shared/db/schema.sqlite.ts b/shared/db/schema.sqlite.ts new file mode 100644 index 0000000..4e2c9cb --- /dev/null +++ b/shared/db/schema.sqlite.ts @@ -0,0 +1,45 @@ +import { type InferSelectModel, sql } from 'drizzle-orm' +import { int, text, real, sqliteTable, index } from 'drizzle-orm/sqlite-core' +import { DB_INDEX_NAMES, DB_TABLE_NAMES } from './constants' + +export const tasksTable = sqliteTable( + DB_TABLE_NAMES.tasks, + { + id: text().primaryKey(), + name: text().notNull(), + state: text({ + enum: ['queued', 'running', 'success', 'failure', 'abandoned'] + }).notNull(), + error: text(), + worker: text(), + executionTime: real('execution_time'), + queuedAt: int('queued_at', { mode: 'timestamp_ms' }).notNull(), + startedAt: int('started_at', { mode: 'timestamp_ms' }), + finishedAt: int('finished_at', { mode: 'timestamp_ms' }), + args: text({ mode: 'json' }).$type>(), + kwargs: text({ mode: 'json' }).$type>(), + returnValue: text('return_value', { mode: 'json' }).$type<{ + return_value: any + }>() + }, + (t) => [ + index(DB_INDEX_NAMES.tasksState).on(t.state), + index(DB_INDEX_NAMES.tasksQueuedAt).on(t.queuedAt), + index(DB_INDEX_NAMES.tasksStartedAt).on(t.startedAt), + index(DB_INDEX_NAMES.tasksFinishedAt).on(t.finishedAt), + index(DB_INDEX_NAMES.tasksExecutionTime).on(t.executionTime), + index(DB_INDEX_NAMES.tasksName).on(sql`name COLLATE NOCASE`), + index(DB_INDEX_NAMES.tasksWorker).on(sql`worker COLLATE NOCASE`) + ] +) + +export type TaskSelect = InferSelectModel + +export const taskiqAdminSettingsTable = sqliteTable(DB_TABLE_NAMES.settings, { + key: text('key').primaryKey(), + value: text('value') +}) + +export type TaskiqAdminSettings = InferSelectModel< + typeof taskiqAdminSettingsTable +> diff --git a/shared/db/schema.ts b/shared/db/schema.ts index ab44399..31fa4fc 100644 --- a/shared/db/schema.ts +++ b/shared/db/schema.ts @@ -1,44 +1,14 @@ -import { type InferSelectModel, sql } from 'drizzle-orm' -import { int, text, real, sqliteTable, index } from 'drizzle-orm/sqlite-core' +import { envVariables } from '../env' +import * as sqliteSchema from './schema.sqlite' +import * as postgresSchema from './schema.postgres' -export const tasksTable = sqliteTable( - 'tasks', - { - id: text().primaryKey(), - name: text().notNull(), - state: text({ - enum: ['queued', 'running', 'success', 'failure', 'abandoned'] - }).notNull(), - error: text(), - worker: text(), - executionTime: real('execution_time'), - queuedAt: int('queued_at', { mode: 'timestamp_ms' }).notNull(), - startedAt: int('started_at', { mode: 'timestamp_ms' }), - finishedAt: int('finished_at', { mode: 'timestamp_ms' }), - args: text({ mode: 'json' }).$type>(), - kwargs: text({ mode: 'json' }).$type>(), - returnValue: text('return_value', { mode: 'json' }).$type<{ - return_value: any - }>() - }, - (t) => [ - index('idx_tasks__state').on(t.state), - index('idx_tasks__queued_at').on(t.queuedAt), - index('idx_tasks__started_at').on(t.startedAt), - index('idx_tasks__finished_at').on(t.finishedAt), - index('idx_tasks__execution_time').on(t.executionTime), - index('idx_tasks__name').on(sql`name COLLATE NOCASE`), - index('idx_tasks__worker').on(sql`worker COLLATE NOCASE`) - ] -) +const schema = + envVariables.dbDriver === 'sqlite' ? sqliteSchema : postgresSchema -export type TaskSelect = InferSelectModel +export const tasksTable = schema.tasksTable +export const taskiqAdminSettingsTable = schema.taskiqAdminSettingsTable -export const taskiqAdminSettingsTable = sqliteTable('taskiq_admin_settings', { - key: text('key').primaryKey(), - value: text('value') -}) - -export type TaskiqAdminSettings = InferSelectModel< - typeof taskiqAdminSettingsTable -> +export type TaskSelect = sqliteSchema.TaskSelect | postgresSchema.TaskSelect +export type TaskiqAdminSettings = + | sqliteSchema.TaskiqAdminSettings + | postgresSchema.TaskiqAdminSettings diff --git a/shared/env.ts b/shared/env.ts index ecbaaa9..d4a8141 100644 --- a/shared/env.ts +++ b/shared/env.ts @@ -1,5 +1,41 @@ +export type DBDriver = 'sqlite' | 'postgres' + +const getRequiredEnv = (name: string) => { + const value = process.env[name] + if (!value) { + throw new Error(`Environment variable ${name} is required`) + } + return value +} + +const getDbDriver = (): DBDriver => { + const value = process.env.DB_DRIVER + if (!value) { + throw new Error('Environment variable DB_DRIVER is required') + } + if (value !== 'sqlite' && value !== 'postgres') { + throw new Error( + 'Environment variable DB_DRIVER must be "sqlite" or "postgres"' + ) + } + return value +} + +const dbDriver = getDbDriver() + export const envVariables = { - dbFilePath: process.env.DB_FILE_PATH!, - backupFilePath: process.env.BACKUP_FILE_PATH!, - taskiqAdminApiToken: process.env.TASKIQ_ADMIN_API_TOKEN! + dbDriver, + dbFilePath: + dbDriver === 'sqlite' + ? getRequiredEnv('DB_FILE_PATH') + : process.env.DB_FILE_PATH || '', + dbUrl: + dbDriver === 'postgres' + ? getRequiredEnv('DB_URL') + : process.env.DB_URL || '', + backupFilePath: + dbDriver === 'sqlite' + ? getRequiredEnv('BACKUP_FILE_PATH') + : process.env.BACKUP_FILE_PATH || '', + taskiqAdminApiToken: getRequiredEnv('TASKIQ_ADMIN_API_TOKEN') } diff --git a/tests/helpers/db-test-utils.ts b/tests/helpers/db-test-utils.ts index 565462f..11f0b27 100644 --- a/tests/helpers/db-test-utils.ts +++ b/tests/helpers/db-test-utils.ts @@ -1,9 +1,5 @@ -import fs from 'node:fs' -import { db } from '~~/shared/db' +import { resetDatabaseForTests } from '../../shared/db' export async function resetDatabase() { - const schema = fs.readFileSync('dbschema.sql', 'utf8') - db.$client.exec('PRAGMA foreign_keys = OFF;') - db.$client.exec('DROP TABLE IF EXISTS tasks;') // в schema уже IF NOT EXISTS - db.$client.exec(schema) + await resetDatabaseForTests() } diff --git a/tests/task-flow.nuxt.spec.ts b/tests/task-flow.nuxt.spec.ts index 2f08c38..673660d 100644 --- a/tests/task-flow.nuxt.spec.ts +++ b/tests/task-flow.nuxt.spec.ts @@ -8,6 +8,7 @@ await setup({ server: true, // up the Nitro server env: { NODE_ENV: 'test', + DB_DRIVER: 'sqlite', DB_FILE_PATH: ':memory:', BACKUP_FILE_PATH: ':memory:', TASKIQ_ADMIN_API_TOKEN: 'supersecret' @@ -56,7 +57,12 @@ test('started → queued → executed saves dates', async () => { } }) - const task = await $fetch(`/api/tasks/${id}`) + const task = await $fetch<{ + state: string + queuedAt: string + startedAt: string + finishedAt: string + }>(`/api/tasks/${id}`) expect(task?.state).toBe('success') expect(task?.queuedAt).toBe(queued.toISOString()) From dd8078aea048c94fb671f9cb43338a1768d85d93 Mon Sep 17 00:00:00 2001 From: arturka Date: Thu, 7 May 2026 15:41:49 +0200 Subject: [PATCH 2/3] feat: env names updated --- Dockerfile | 10 +++++----- README.md | 6 +++--- drizzle.config.ts | 14 ++++++++------ env-example | 8 ++++---- shared/env.ts | 18 +++++++++--------- tests/task-flow.nuxt.spec.ts | 6 +++--- 6 files changed, 32 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 90324bc..c1239d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,8 +9,8 @@ RUN corepack enable WORKDIR /app -ENV DB_DRIVER=sqlite -ENV DB_FILE_PATH=/tmp/taskiq-admin.db +ENV TASKIQ_ADMIN_DB_DRIVER=sqlite +ENV TASKIQ_ADMIN_DB_FILE_PATH=/tmp/taskiq-admin.db # install dependencies COPY ./package.json /app/ @@ -36,9 +36,9 @@ COPY --from=build /app/entrypoint.sh /usr/app/entrypoint.sh EXPOSE 3000 ENV HOST=0.0.0.0 NODE_ENV=production -ENV DB_DRIVER=sqlite -ENV DB_FILE_PATH=/usr/database/database.db -ENV BACKUP_FILE_PATH=/usr/database/backup.db +ENV TASKIQ_ADMIN_DB_DRIVER=sqlite +ENV TASKIQ_ADMIN_DB_FILE_PATH=/usr/database/database.db +ENV TASKIQ_ADMIN_BACKUP_FILE_PATH=/usr/database/backup.db RUN npm install dotenv diff --git a/README.md b/README.md index a79fec9..b8bd753 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,9 @@ volumes: 3) run `make prod` to run it locally in prod mode Environment variables for DB setup: -- `DB_DRIVER`: required, one of `sqlite` or `postgres` -- `DB_FILE_PATH`: required when `DB_DRIVER=sqlite` -- `DB_URL`: required when `DB_DRIVER=postgres` +- `TASKIQ_ADMIN_DB_DRIVER`: required, one of `sqlite` or `postgres` +- `TASKIQ_ADMIN_DB_FILE_PATH`: required when `TASKIQ_ADMIN_DB_DRIVER=sqlite` +- `TASKIQ_ADMIN_DB_URL`: required when `TASKIQ_ADMIN_DB_DRIVER=postgres` ### Task States Let's assume we have a task 'do_smth', there are all states it can embrace: diff --git a/drizzle.config.ts b/drizzle.config.ts index e298746..9f18769 100644 --- a/drizzle.config.ts +++ b/drizzle.config.ts @@ -1,26 +1,28 @@ import 'dotenv/config' import { defineConfig } from 'drizzle-kit' -const dbDriver = process.env.DB_DRIVER +const dbDriver = process.env.TASKIQ_ADMIN_DB_DRIVER if (!dbDriver) { - throw new Error('Environment variable DB_DRIVER is required') + throw new Error('Environment variable TASKIQ_ADMIN_DB_DRIVER is required') } if (dbDriver !== 'sqlite' && dbDriver !== 'postgres') { throw new Error( - 'Environment variable DB_DRIVER must be "sqlite" or "postgres"' + 'Environment variable TASKIQ_ADMIN_DB_DRIVER must be "sqlite" or "postgres"' ) } const dbUrl = - dbDriver === 'sqlite' ? process.env.DB_FILE_PATH : process.env.DB_URL + dbDriver === 'sqlite' + ? process.env.TASKIQ_ADMIN_DB_FILE_PATH + : process.env.TASKIQ_ADMIN_DB_URL if (!dbUrl) { throw new Error( dbDriver === 'sqlite' - ? 'Environment variable DB_FILE_PATH is required for sqlite driver' - : 'Environment variable DB_URL is required for postgres driver' + ? 'Environment variable TASKIQ_ADMIN_DB_FILE_PATH is required for sqlite driver' + : 'Environment variable TASKIQ_ADMIN_DB_URL is required for postgres driver' ) } diff --git a/env-example b/env-example index 4254473..ae3142d 100644 --- a/env-example +++ b/env-example @@ -1,5 +1,5 @@ -DB_DRIVER=sqlite -DB_FILE_PATH=database/database.db -DB_URL=postgres://postgres:postgres@localhost:5432/taskiq_admin -BACKUP_FILE_PATH=database/backup.db +TASKIQ_ADMIN_DB_DRIVER=sqlite +TASKIQ_ADMIN_DB_FILE_PATH=database/database.db +TASKIQ_ADMIN_DB_URL=postgres://postgres:postgres@localhost:5432/taskiq_admin +TASKIQ_ADMIN_BACKUP_FILE_PATH=database/backup.db TASKIQ_ADMIN_API_TOKEN=supersecret \ No newline at end of file diff --git a/shared/env.ts b/shared/env.ts index d4a8141..1ae4ebd 100644 --- a/shared/env.ts +++ b/shared/env.ts @@ -9,13 +9,13 @@ const getRequiredEnv = (name: string) => { } const getDbDriver = (): DBDriver => { - const value = process.env.DB_DRIVER + const value = process.env.TASKIQ_ADMIN_DB_DRIVER if (!value) { - throw new Error('Environment variable DB_DRIVER is required') + throw new Error('Environment variable TASKIQ_ADMIN_DB_DRIVER is required') } if (value !== 'sqlite' && value !== 'postgres') { throw new Error( - 'Environment variable DB_DRIVER must be "sqlite" or "postgres"' + 'Environment variable TASKIQ_ADMIN_DB_DRIVER must be "sqlite" or "postgres"' ) } return value @@ -27,15 +27,15 @@ export const envVariables = { dbDriver, dbFilePath: dbDriver === 'sqlite' - ? getRequiredEnv('DB_FILE_PATH') - : process.env.DB_FILE_PATH || '', + ? getRequiredEnv('TASKIQ_ADMIN_DB_FILE_PATH') + : process.env.TASKIQ_ADMIN_DB_FILE_PATH || '', dbUrl: dbDriver === 'postgres' - ? getRequiredEnv('DB_URL') - : process.env.DB_URL || '', + ? getRequiredEnv('TASKIQ_ADMIN_DB_URL') + : process.env.TASKIQ_ADMIN_DB_URL || '', backupFilePath: dbDriver === 'sqlite' - ? getRequiredEnv('BACKUP_FILE_PATH') - : process.env.BACKUP_FILE_PATH || '', + ? getRequiredEnv('TASKIQ_ADMIN_BACKUP_FILE_PATH') + : process.env.TASKIQ_ADMIN_BACKUP_FILE_PATH || '', taskiqAdminApiToken: getRequiredEnv('TASKIQ_ADMIN_API_TOKEN') } diff --git a/tests/task-flow.nuxt.spec.ts b/tests/task-flow.nuxt.spec.ts index 673660d..c281093 100644 --- a/tests/task-flow.nuxt.spec.ts +++ b/tests/task-flow.nuxt.spec.ts @@ -8,9 +8,9 @@ await setup({ server: true, // up the Nitro server env: { NODE_ENV: 'test', - DB_DRIVER: 'sqlite', - DB_FILE_PATH: ':memory:', - BACKUP_FILE_PATH: ':memory:', + TASKIQ_ADMIN_DB_DRIVER: 'sqlite', + TASKIQ_ADMIN_DB_FILE_PATH: ':memory:', + TASKIQ_ADMIN_BACKUP_FILE_PATH: ':memory:', TASKIQ_ADMIN_API_TOKEN: 'supersecret' } }) From c33702cf3c97a938680672ea8febf21e3429c2c2 Mon Sep 17 00:00:00 2001 From: arturka Date: Thu, 7 May 2026 15:51:17 +0200 Subject: [PATCH 3/3] feat: properly create sqlite file --- shared/db/index.ts | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/shared/db/index.ts b/shared/db/index.ts index b08ff88..4d5bb5f 100644 --- a/shared/db/index.ts +++ b/shared/db/index.ts @@ -2,13 +2,31 @@ import Database from 'better-sqlite3' import { drizzle as drizzleSqlite } from 'drizzle-orm/better-sqlite3' import { sql } from 'drizzle-orm' import { drizzle as drizzlePostgres } from 'drizzle-orm/node-postgres' +import { mkdirSync } from 'node:fs' +import { dirname, isAbsolute, resolve } from 'node:path' import { Pool } from 'pg' import { envVariables } from '../env' import { dbInitializationStatements } from './init' +const isInMemorySqlitePath = (sqlitePath: string) => { + return sqlitePath === ':memory:' || sqlitePath.startsWith('file::memory:') +} + +const ensureSqliteFilePath = (sqlitePath: string) => { + if (isInMemorySqlitePath(sqlitePath) || sqlitePath.startsWith('file:')) { + return sqlitePath + } + + const resolvedPath = isAbsolute(sqlitePath) + ? sqlitePath + : resolve(process.cwd(), sqlitePath) + mkdirSync(dirname(resolvedPath), { recursive: true }) + return resolvedPath +} + const sqliteClient = envVariables.dbDriver === 'sqlite' - ? new Database(envVariables.dbFilePath) + ? new Database(ensureSqliteFilePath(envVariables.dbFilePath)) : null const postgresClient = @@ -42,7 +60,7 @@ export const backupDatabase = async (backupFilePath: string) => { if (envVariables.dbDriver !== 'sqlite') { throw new Error('Database backup is supported only with sqlite driver') } - await sqliteClient!.backup(backupFilePath) + await sqliteClient!.backup(ensureSqliteFilePath(backupFilePath)) } export const resetDatabaseForTests = async () => {