diff --git a/.github/workflows/test-web-console-e2e.yml b/.github/workflows/test-web-console-e2e.yml index bfa94dd84f1..31f116d2069 100644 --- a/.github/workflows/test-web-console-e2e.yml +++ b/.github/workflows/test-web-console-e2e.yml @@ -44,6 +44,13 @@ jobs: - name: Verify pipeline-manager is reachable run: curl -fsSL --retry 5 --retry-delay 2 --retry-connrefused http://localhost:8080/healthz + - name: Run vitest integration tests + if: ${{ vars.CI_DRY_RUN != 'true' }} + run: bun run test-integration + working-directory: js-packages/web-console + env: + FELDERA_API_URL: http://localhost:8080 + - name: Run Playwright e2e tests if: ${{ vars.CI_DRY_RUN != 'true' }} run: bun run test-e2e diff --git a/Cargo.lock b/Cargo.lock index 3cd9d96da5f..179949b262e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3761,7 +3761,7 @@ dependencies = [ [[package]] name = "dbsp" -version = "0.281.0" +version = "0.282.0" dependencies = [ "anyhow", "arc-swap", @@ -3849,7 +3849,7 @@ dependencies = [ [[package]] name = "dbsp_adapters" -version = "0.281.0" +version = "0.282.0" dependencies = [ "actix", "actix-codec", @@ -3986,7 +3986,7 @@ dependencies = [ [[package]] name = "dbsp_nexmark" -version = "0.281.0" +version = "0.282.0" dependencies = [ "anyhow", "ascii_table", @@ -4861,7 +4861,7 @@ dependencies = [ [[package]] name = "fda" -version = "0.281.0" +version = "0.282.0" dependencies = [ "anyhow", "arrow", @@ -4913,7 +4913,7 @@ dependencies = [ [[package]] name = "feldera-adapterlib" -version = "0.281.0" +version = "0.282.0" dependencies = [ "actix-web", "anyhow", @@ -4944,7 +4944,7 @@ dependencies = [ [[package]] name = "feldera-buffer-cache" -version = "0.281.0" +version = "0.282.0" dependencies = [ "crossbeam-utils", "enum-map", @@ -4972,7 +4972,7 @@ dependencies = [ [[package]] name = "feldera-datagen" -version = "0.281.0" +version = "0.282.0" dependencies = [ "anyhow", "async-channel 2.5.0", @@ -4998,7 +4998,7 @@ dependencies = [ [[package]] name = "feldera-fxp" -version = "0.281.0" +version = "0.282.0" dependencies = [ "bytecheck", "dbsp", @@ -5018,7 +5018,7 @@ dependencies = [ [[package]] name = "feldera-iceberg" -version = "0.281.0" +version = "0.282.0" dependencies = [ "anyhow", "chrono", @@ -5038,7 +5038,7 @@ dependencies = [ [[package]] name = "feldera-ir" -version = "0.281.0" +version = "0.282.0" dependencies = [ "proptest", "proptest-derive", @@ -5050,7 +5050,7 @@ dependencies = [ [[package]] name = "feldera-macros" -version = "0.281.0" +version = "0.282.0" dependencies = [ "prettyplease", "proc-macro2", @@ -5060,7 +5060,7 @@ dependencies = [ [[package]] name = "feldera-observability" -version = "0.281.0" +version = "0.282.0" dependencies = [ "actix-http", "awc", @@ -5075,7 +5075,7 @@ dependencies = [ [[package]] name = "feldera-rest-api" -version = "0.281.0" +version = "0.282.0" dependencies = [ "chrono", "feldera-observability", @@ -5109,7 +5109,7 @@ dependencies = [ [[package]] name = "feldera-sqllib" -version = "0.281.0" +version = "0.282.0" dependencies = [ "arcstr", "base58", @@ -5150,7 +5150,7 @@ dependencies = [ [[package]] name = "feldera-storage" -version = "0.281.0" +version = "0.282.0" dependencies = [ "anyhow", "crossbeam", @@ -5173,7 +5173,7 @@ dependencies = [ [[package]] name = "feldera-types" -version = "0.281.0" +version = "0.282.0" dependencies = [ "actix-web", "anyhow", @@ -8094,7 +8094,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pipeline-manager" -version = "0.281.0" +version = "0.282.0" dependencies = [ "actix-cors", "actix-files", @@ -9188,7 +9188,7 @@ dependencies = [ [[package]] name = "readers" -version = "0.281.0" +version = "0.282.0" dependencies = [ "async-std", "csv", @@ -10764,7 +10764,7 @@ dependencies = [ [[package]] name = "sltsqlvalue" -version = "0.281.0" +version = "0.282.0" dependencies = [ "dbsp", "feldera-sqllib", @@ -11067,7 +11067,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "storage-test-compat" -version = "0.281.0" +version = "0.282.0" dependencies = [ "dbsp", "derive_more 1.0.0", diff --git a/Cargo.toml b/Cargo.toml index 848c71cb476..10d170e2164 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace.package] authors = ["Feldera Team "] -version = "0.281.0" +version = "0.282.0" license = "MIT OR Apache-2.0" homepage = "https://github.com/feldera/feldera" repository = "https://github.com/feldera/feldera" @@ -101,7 +101,7 @@ csv = "1.2.2" csv-core = "0.1.10" dashmap = "6.1.0" datafusion = "51.0" -dbsp = { path = "crates/dbsp", version = "0.281.0" } +dbsp = { path = "crates/dbsp", version = "0.282.0" } dbsp_nexmark = { path = "crates/nexmark" } deadpool-postgres = "0.14.1" #deltalake = "0.30.2" @@ -121,19 +121,19 @@ erased-serde = "0.3.31" fake = "2.10" fastbloom = "0.14.0" fdlimit = "0.3.0" -feldera-buffer-cache = { version = "0.281.0", path = "crates/buffer-cache" } +feldera-buffer-cache = { version = "0.282.0", path = "crates/buffer-cache" } feldera-cloud1-client = "0.1.2" feldera-datagen = { path = "crates/datagen" } -feldera-fxp = { version = "0.281.0", path = "crates/fxp", features = ["dbsp"] } +feldera-fxp = { version = "0.282.0", path = "crates/fxp", features = ["dbsp"] } feldera-iceberg = { path = "crates/iceberg" } -feldera-observability = { version = "0.281.0", path = "crates/feldera-observability" } -feldera-macros = { version = "0.281.0", path = "crates/feldera-macros" } -feldera-sqllib = { version = "0.281.0", path = "crates/sqllib" } -feldera-storage = { version = "0.281.0", path = "crates/storage" } -feldera-types = { version = "0.281.0", path = "crates/feldera-types" } -feldera-rest-api = { version = "0.281.0", path = "crates/rest-api" } -feldera-ir = { version = "0.281.0", path = "crates/ir" } -feldera-adapterlib = { version = "0.281.0", path = "crates/adapterlib" } +feldera-observability = { version = "0.282.0", path = "crates/feldera-observability" } +feldera-macros = { version = "0.282.0", path = "crates/feldera-macros" } +feldera-sqllib = { version = "0.282.0", path = "crates/sqllib" } +feldera-storage = { version = "0.282.0", path = "crates/storage" } +feldera-types = { version = "0.282.0", path = "crates/feldera-types" } +feldera-rest-api = { version = "0.282.0", path = "crates/rest-api" } +feldera-ir = { version = "0.282.0", path = "crates/ir" } +feldera-adapterlib = { version = "0.282.0", path = "crates/adapterlib" } flate2 = "1.1.0" form_urlencoded = "1.2.0" futures = "0.3.30" diff --git a/js-packages/web-console/package.json b/js-packages/web-console/package.json index 6e651106883..5108adf7399 100644 --- a/js-packages/web-console/package.json +++ b/js-packages/web-console/package.json @@ -132,7 +132,8 @@ "test-prepare": "git clone --depth 1 https://github.com/feldera/playwright-snapshots.git || true && mkdir -p playwright-snapshots/e2e playwright-snapshots/component", "test-update-snapshots": "bun run test -- --update && bun playwright test --update-snapshots", "test-unit": "vitest", - "test": "bun run test-unit -- --run" + "test-integration": "vitest --run --project integration --project integration-client", + "test": "bun run test-unit -- --run --project client --project server" }, "trustedDependencies": ["@axa-fr/oidc-client", "sk-oidc-oauth", "svelte-preprocess"], "type": "module" diff --git a/js-packages/web-console/src/lib/components/pipelines/editor/performance/ConnectorErrors.svelte b/js-packages/web-console/src/lib/components/pipelines/editor/performance/ConnectorErrors.svelte index 3f765991d8a..5e1ac9c0b06 100644 --- a/js-packages/web-console/src/lib/components/pipelines/editor/performance/ConnectorErrors.svelte +++ b/js-packages/web-console/src/lib/components/pipelines/editor/performance/ConnectorErrors.svelte @@ -5,6 +5,7 @@ import InlineDropdown from '$lib/components/common/InlineDropdown.svelte' import { Tooltip } from '$lib/components/common/Tooltip.svelte' import InlineDrawer from '$lib/components/layout/InlineDrawer.svelte' + import { getCaseDependentName } from '$lib/functions/felderaRelation' import { formatDateTime } from '$lib/functions/format' import { type ConnectorError, @@ -61,6 +62,10 @@ tagsFilter = filter }) + const strippedConnectorName = $derived( + connectorName.slice(getCaseDependentName(relationName).name.length + 1) + ) + $effect(() => { pipelineName relationName @@ -69,11 +74,10 @@ loading = true status = null - const stripped = connectorName.slice(connectorName.indexOf('.') + 1) const request = direction === 'input' - ? getInputConnectorStatus(pipelineName, relationName, stripped) - : getOutputConnectorStatus(pipelineName, relationName, stripped) + ? getInputConnectorStatus(pipelineName, relationName, strippedConnectorName) + : getOutputConnectorStatus(pipelineName, relationName, strippedConnectorName) request.then((s) => { status = s @@ -137,7 +141,7 @@
-
{connectorName.replace('.', ' · ')}
+
{relationName} · {strippedConnectorName}
diff --git a/js-packages/web-console/src/lib/functions/felderaRelation.ts b/js-packages/web-console/src/lib/functions/felderaRelation.ts index 61ca0105842..5b289aaf005 100644 --- a/js-packages/web-console/src/lib/functions/felderaRelation.ts +++ b/js-packages/web-console/src/lib/functions/felderaRelation.ts @@ -21,10 +21,7 @@ export const normalizeCaseIndependentName = ({ export const getCaseDependentName = (caseIndependentName: string) => { const case_sensitive = isCaseSensitive(caseIndependentName) return { - name: normalizeCaseIndependentName({ - name: caseIndependentName.replaceAll('"', ''), - case_sensitive - }), + name: caseIndependentName.replaceAll('"', ''), case_sensitive } } diff --git a/js-packages/web-console/src/lib/services/pipelineManager.test.ts b/js-packages/web-console/src/lib/services/pipelineManager.test.ts new file mode 100644 index 00000000000..1df08520473 --- /dev/null +++ b/js-packages/web-console/src/lib/services/pipelineManager.test.ts @@ -0,0 +1,85 @@ +/** + * Integration tests for pipelineManager.ts that require a running Feldera instance. + * Run via the 'integration' vitest project: + * + * bun run test-integration + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { + getInputConnectorStatus, + getOutputConnectorStatus, + putPipeline +} from '$lib/services/pipelineManager' +import { + cleanupPipeline, + configureTestClient, + startPipelineAndWaitForRunning, + waitForCompilation +} from '$lib/services/testPipelineHelpers' + +// Pipeline name is URL-safe by convention, but table/view/connector names are not. +const PIPELINE_NAME = 'test-special-chars-connector-status' + +// Names with dots and URL-unsafe characters +const TABLE_NAME = 'my.input.table' +const VIEW_NAME = 'my.output.view' +const INPUT_CONNECTOR_NAME = 'http-input_special-name' +const OUTPUT_CONNECTOR_NAME = 'http-output_special-name' + +describe('pipelineManager connector status with special characters', () => { + beforeAll(async () => { + configureTestClient() + + // Clean up any leftover pipeline from a previous run + await cleanupPipeline(PIPELINE_NAME) + + // SQL program with quoted identifiers containing dots + const programCode = ` + CREATE TABLE "${TABLE_NAME}" (id INT NOT NULL, val VARCHAR) + WITH ( + 'connectors' = '[{ + "name": "${INPUT_CONNECTOR_NAME}", + "transport": { "name": "url_input", "config": { "path": "https://feldera.com/test-data.json" } }, + "format": { "name": "json", "config": { "update_format": "raw" } } + }]' + ); + CREATE VIEW "${VIEW_NAME}" + WITH ( + 'connectors' = '[{ + "name": "${OUTPUT_CONNECTOR_NAME}", + "transport": { "name": "file_output", "config": { "path": "/tmp/feldera-test-output.json" } }, + "format": { "name": "json" } + }]' + ) + AS SELECT * FROM "${TABLE_NAME}"; + ` + + // Create the pipeline + await putPipeline(PIPELINE_NAME, { + name: PIPELINE_NAME, + description: 'Integration test for special character handling', + program_code: programCode, + runtime_config: {} + }) + + await waitForCompilation(PIPELINE_NAME, 120_000) + await startPipelineAndWaitForRunning(PIPELINE_NAME, 60_000) + }, 180_000) + + afterAll(async () => { + await cleanupPipeline(PIPELINE_NAME) + }, 60_000) + + it('getInputConnectorStatus succeeds with dot and URL-unsafe characters in table and connector name', async () => { + const body = await getInputConnectorStatus(PIPELINE_NAME, TABLE_NAME, INPUT_CONNECTOR_NAME) + console.log('body', JSON.stringify(body)) + expect(body).toHaveProperty(['metrics', 'num_parse_errors']) + expect(body).toHaveProperty(['metrics', 'num_transport_errors']) + }) + + it('getOutputConnectorStatus succeeds with dot and URL-unsafe characters in view and connector name', async () => { + const body = await getOutputConnectorStatus(PIPELINE_NAME, VIEW_NAME, OUTPUT_CONNECTOR_NAME) + expect(body).toHaveProperty(['metrics', 'num_encode_errors']) + expect(body).toHaveProperty(['metrics', 'num_transport_errors']) + }) +}) diff --git a/js-packages/web-console/src/lib/services/pipelineManager.ts b/js-packages/web-console/src/lib/services/pipelineManager.ts index b34a6c378b7..d6af2f21359 100644 --- a/js-packages/web-console/src/lib/services/pipelineManager.ts +++ b/js-packages/web-console/src/lib/services/pipelineManager.ts @@ -364,7 +364,7 @@ export const getExtendedPipeline = async ( ) => { return mapResponse( _getPipeline({ - path: { pipeline_name: encodeURIComponent(pipeline_name) }, + path: { pipeline_name }, ...options }), toExtendedPipeline, @@ -398,7 +398,7 @@ export const putPipeline = async ( await mapResponse( _putPipeline({ body: newPipeline, - path: { pipeline_name: encodeURIComponent(pipeline_name) }, + path: { pipeline_name }, ...options }), (v) => v @@ -412,7 +412,7 @@ export const patchPipeline = async ( ) => { return mapResponse( _patchPipeline({ - path: { pipeline_name: encodeURIComponent(pipeline_name) }, + path: { pipeline_name }, body: fromPipeline(pipeline), ...options }), @@ -433,7 +433,7 @@ export const getPipelines = async (options?: FetchOptions): Promise { return mapResponse( _getPipeline({ - path: { pipeline_name: encodeURIComponent(pipeline_name) }, + path: { pipeline_name }, query: { selector: 'status' }, ...options }), @@ -450,7 +450,7 @@ export const getPipelineStatus = async (pipeline_name: string, options?: FetchOp export const getPipelineStats = async (pipeline_name: string, options?: FetchOptions) => { return mapResponse( _getPipelineStats({ - path: { pipeline_name: encodeURIComponent(pipeline_name) }, + path: { pipeline_name }, ...options }), (status) => ({ @@ -578,7 +578,7 @@ export const getClusterEvent = (eventId: string) => const getSamplyProfileStream = (pipelineName: string, latest: boolean) => { const result = streamingFetch( getAuthenticatedFetch(), - `${felderaEndpoint}/v0/pipelines/${encodeURIComponent(pipelineName)}/samply_profile${latest ? '?latest=true' : ''}`, + `${felderaEndpoint}/v0/pipelines/${pipelineName}/samply_profile${latest ? '?latest=true' : ''}`, { method: 'GET' } @@ -967,5 +967,5 @@ export const getPipelineSupportBundleUrl = ( for (const [key, value] of Object.entries(options)) { query.append(key, String(value)) } - return `${felderaEndpoint}/v0/pipelines/${encodeURIComponent(pipelineName)}/support_bundle?${query.toString()}` + return `${felderaEndpoint}/v0/pipelines/${pipelineName}/support_bundle?${query.toString()}` } diff --git a/js-packages/web-console/src/lib/services/testPipelineHelpers.ts b/js-packages/web-console/src/lib/services/testPipelineHelpers.ts new file mode 100644 index 00000000000..e8d5febdff4 --- /dev/null +++ b/js-packages/web-console/src/lib/services/testPipelineHelpers.ts @@ -0,0 +1,179 @@ +/** + * Shared test helpers for pipeline lifecycle management. + * + * Used by both Playwright e2e tests and Vitest integration tests to avoid + * duplicating pipeline create/compile/start/stop/delete boilerplate. + */ + +import { client } from '$lib/services/manager/client.gen' +import { + deletePipeline, + type ExtendedPipeline, + getExtendedPipeline, + getPipelineStatus, + postPipelineAction, + programStatusOf, + putPipeline +} from '$lib/services/pipelineManager' + +export { type ExtendedPipeline } + +/** + * Configure the API client base URL for tests. + * + * Reads from `FELDERA_TEST_API_ORIGIN` (shared) or `PLAYWRIGHT_API_ORIGIN` + * (Playwright-specific), falling back to `http://localhost:8080`. + */ +export function configureTestClient() { + const origin = ( + process.env.FELDERA_TEST_API_ORIGIN ?? + process.env.PLAYWRIGHT_API_ORIGIN ?? + 'http://localhost:8080' + ).replace(/\/$/, '') + client.setConfig({ baseUrl: origin }) + return origin +} + +/** + * Poll `getExtendedPipeline` until `predicate` returns true or `timeoutMs` elapses. + */ +export async function waitForExtendedPipeline( + pipelineName: string, + predicate: (p: ExtendedPipeline) => boolean, + timeoutMs = 120_000 +): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + const pipeline = await getExtendedPipeline(pipelineName) + if (predicate(pipeline)) return pipeline + await new Promise((r) => setTimeout(r, 2000)) + } + const pipeline = await getExtendedPipeline(pipelineName) + throw new Error( + `waitForExtendedPipeline timed out for "${pipelineName}". ` + + `deploy=${pipeline.deploymentStatus} status=${JSON.stringify(pipeline.status)} storage=${pipeline.storageStatus}` + ) +} + +/** + * Poll `getPipelineStatus` until `predicate` returns true or `timeoutMs` elapses. + */ +export async function waitForPipelineStatus( + pipelineName: string, + predicate: (status: string) => boolean, + timeoutMs = 120_000 +) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + const { status } = await getPipelineStatus(pipelineName) + if (predicate(status as string)) return status + await new Promise((r) => setTimeout(r, 1000)) + } + const { status } = await getPipelineStatus(pipelineName) + throw new Error( + `waitForPipelineStatus timed out for "${pipelineName}". status=${JSON.stringify(status)}` + ) +} + +/** + * Wait for a pipeline's SQL/Rust compilation to finish successfully. + * Throws on compilation errors. + */ +export async function waitForCompilation(pipelineName: string, timeoutMs = 600_000) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + const { status } = await getPipelineStatus(pipelineName) + const programStatus = programStatusOf(status) + if (programStatus === 'Success') return + if ( + programStatus === 'SqlError' || + programStatus === 'RustError' || + programStatus === 'SystemError' + ) { + throw new Error(`Compilation failed for "${pipelineName}": ${programStatus}`) + } + await new Promise((r) => setTimeout(r, 2000)) + } + throw new Error(`Compilation timed out for "${pipelineName}"`) +} + +/** + * Start a pipeline and wait until it reaches the Running state. + * Automatically handles the AwaitingApproval → approve_changes transition. + */ +export async function startPipelineAndWaitForRunning(pipelineName: string, timeoutMs = 60_000) { + await postPipelineAction(pipelineName, 'start') + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + const { status } = await getPipelineStatus(pipelineName) + if (status === 'Running') return + if (status === 'AwaitingApproval') { + await postPipelineAction(pipelineName, 'approve_changes') + } + await new Promise((r) => setTimeout(r, 1000)) + } + throw new Error(`Pipeline "${pipelineName}" did not reach Running within ${timeoutMs}ms`) +} + +/** + * Stop a pipeline and wait until it reaches the Stopped state. + */ +export async function stopPipelineAndWaitForStopped(pipelineName: string, timeoutMs = 30_000) { + try { + await postPipelineAction(pipelineName, 'stop') + } catch { + // Ignore if already stopped + } + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + const { status } = await getPipelineStatus(pipelineName) + if (status === 'Stopped') return + await new Promise((r) => setTimeout(r, 1000)) + } + throw new Error(`Pipeline "${pipelineName}" did not reach Stopped within ${timeoutMs}ms`) +} + +/** + * Fully clean up a pipeline: stop → wait → delete. Ignores errors at every step. + */ +export async function cleanupPipeline(pipelineName: string) { + try { + await stopPipelineAndWaitForStopped(pipelineName) + } catch {} + try { + await deletePipeline(pipelineName) + } catch {} +} + +const WARMUP_PIPELINE = '__test_warmup__' + +/** + * Warm the Rust compilation cache by creating, compiling, and deleting a + * minimal pipeline. The first compilation in a fresh Feldera image compiles + * many Rust crates from scratch, so this prevents individual tests from + * timing out. + */ +export async function warmCompilationCache() { + console.log('Warming Rust compilation cache…') + + // Clean up any leftover warmup pipeline + try { + await deletePipeline(WARMUP_PIPELINE) + } catch {} + + try { + await putPipeline(WARMUP_PIPELINE, { + name: WARMUP_PIPELINE, + program_code: 'CREATE TABLE _warmup (id INT);', + program_config: { profile: 'unoptimized' } + }) + } catch (e) { + console.error('warmCompilationCache: putPipeline failed:', e) + throw e + } + + await waitForCompilation(WARMUP_PIPELINE) + console.log('Compilation cache is warm.') + + await deletePipeline(WARMUP_PIPELINE) +} diff --git a/js-packages/web-console/src/lib/vitest-integration-setup.ts b/js-packages/web-console/src/lib/vitest-integration-setup.ts new file mode 100644 index 00000000000..e0a1611f443 --- /dev/null +++ b/js-packages/web-console/src/lib/vitest-integration-setup.ts @@ -0,0 +1,13 @@ +/** + * Vitest globalSetup for integration test projects. + * + * Configures the API client and warms the Rust compilation cache, + * mirroring what tests/global-setup.ts does for Playwright. + */ + +import { configureTestClient, warmCompilationCache } from '$lib/services/testPipelineHelpers' + +export async function setup() { + configureTestClient() + await warmCompilationCache() +} diff --git a/js-packages/web-console/tests/clearStorageDialog.e2e.ts b/js-packages/web-console/tests/clearStorageDialog.e2e.ts index 8438b4d24c1..77ef2f50995 100644 --- a/js-packages/web-console/tests/clearStorageDialog.e2e.ts +++ b/js-packages/web-console/tests/clearStorageDialog.e2e.ts @@ -1,31 +1,15 @@ import { expect, test } from '@playwright/test' -import { client } from '$lib/services/manager/client.gen' +import { getExtendedPipeline, postPipelineAction, putPipeline } from '$lib/services/pipelineManager' import { - deletePipeline, - type ExtendedPipeline, - getExtendedPipeline, - postPipelineAction, - putPipeline -} from '$lib/services/pipelineManager' + cleanupPipeline, + configureTestClient, + waitForExtendedPipeline +} from '$lib/services/testPipelineHelpers' -const API_ORIGIN = (process.env.PLAYWRIGHT_API_ORIGIN ?? 'http://localhost:8080').replace(/\/$/, '') -client.setConfig({ baseUrl: API_ORIGIN }) +configureTestClient() const PIPELINE_NAME = `test-clear-storage-${Date.now()}` -async function waitForStatus(predicate: (p: ExtendedPipeline) => boolean, timeoutMs = 120_000) { - const start = Date.now() - while (Date.now() - start < timeoutMs) { - const pipeline = await getExtendedPipeline(PIPELINE_NAME) - if (predicate(pipeline)) return pipeline - await new Promise((r) => setTimeout(r, 1000)) - } - const pipeline = await getExtendedPipeline(PIPELINE_NAME) - throw new Error( - `Timed out. deploy=${pipeline.deploymentStatus} status=${JSON.stringify(pipeline.status)} storage=${pipeline.storageStatus}` - ) -} - async function createPipelineWithStorage() { await putPipeline(PIPELINE_NAME, { name: PIPELINE_NAME, @@ -37,32 +21,15 @@ async function createPipelineWithStorage() { }, program_config: { profile: 'unoptimized' } }) - await waitForStatus((p) => p.status === 'Stopped') + await waitForExtendedPipeline(PIPELINE_NAME, (p) => p.status === 'Stopped') } async function startAndStopToCreateStorage() { await postPipelineAction(PIPELINE_NAME, 'start') - await waitForStatus((p) => p.deploymentStatus === 'Running') + await waitForExtendedPipeline(PIPELINE_NAME, (p) => p.deploymentStatus === 'Running') await postPipelineAction(PIPELINE_NAME, 'kill') - await waitForStatus((p) => p.deploymentStatus === 'Stopped') - await waitForStatus((p) => p.storageStatus !== 'Cleared', 10_000) -} - -async function cleanupPipeline() { - try { - const p = await getExtendedPipeline(PIPELINE_NAME) - if (p.deploymentStatus !== 'Stopped') { - await postPipelineAction(PIPELINE_NAME, 'kill') - await waitForStatus((p) => p.deploymentStatus === 'Stopped', 30_000) - } - if (p.storageStatus !== 'Cleared') { - await postPipelineAction(PIPELINE_NAME, 'clear') - await waitForStatus((p) => p.storageStatus === 'Cleared', 30_000) - } - } catch {} - try { - await deletePipeline(PIPELINE_NAME) - } catch {} + await waitForExtendedPipeline(PIPELINE_NAME, (p) => p.deploymentStatus === 'Stopped') + await waitForExtendedPipeline(PIPELINE_NAME, (p) => p.storageStatus !== 'Cleared', 10_000) } /** Intercept the next PATCH to return a storage-not-cleared error, then open @@ -108,14 +75,14 @@ test.describe('Clear storage dialog', () => { test.beforeAll(async ({}, testInfo) => { testInfo.setTimeout(120_000) - await cleanupPipeline() + await cleanupPipeline(PIPELINE_NAME) await createPipelineWithStorage() await startAndStopToCreateStorage() }) test.afterAll(async ({}, testInfo) => { testInfo.setTimeout(60_000) - await cleanupPipeline() + await cleanupPipeline(PIPELINE_NAME) }) test('changing config triggers clear storage dialog and completes the flow', async ({ page }) => { diff --git a/js-packages/web-console/tests/global-setup.ts b/js-packages/web-console/tests/global-setup.ts index fc1a13f0183..a5206db3417 100644 --- a/js-packages/web-console/tests/global-setup.ts +++ b/js-packages/web-console/tests/global-setup.ts @@ -6,48 +6,9 @@ * the test suite keeps individual test timeouts tight. */ -import { client } from '$lib/services/manager/client.gen' -import { deletePipeline, getExtendedPipeline, putPipeline } from '$lib/services/pipelineManager' - -const API_ORIGIN = (process.env.PLAYWRIGHT_API_ORIGIN ?? 'http://localhost:8080').replace(/\/$/, '') -client.setConfig({ baseUrl: API_ORIGIN }) - -const WARMUP_PIPELINE = '__e2e_warmup__' +import { configureTestClient, warmCompilationCache } from '$lib/services/testPipelineHelpers' export default async function globalSetup() { - console.log('Warming Rust compilation cache…') - - // Clean up any leftover warmup pipeline - try { - await deletePipeline(WARMUP_PIPELINE) - } catch {} - - // Create a minimal pipeline - try { - await putPipeline(WARMUP_PIPELINE, { - name: WARMUP_PIPELINE, - program_code: 'CREATE TABLE _warmup (id INT);', - program_config: { profile: 'unoptimized' } - }) - } catch (e) { - console.error(`putPipeline failed (API_ORIGIN=${API_ORIGIN}):`, e) - throw e - } - - // Wait for compilation (up to 10 minutes for a cold cache) - const deadline = Date.now() + 600_000 - while (Date.now() < deadline) { - const pipeline = await getExtendedPipeline(WARMUP_PIPELINE) - if (pipeline.status === 'Stopped') { - console.log('Compilation cache is warm.') - break - } - const status = pipeline.status - if (status === 'SqlError' || status === 'RustError' || status === 'SystemError') { - throw new Error(`Warmup compilation failed: ${status}`) - } - await new Promise((r) => setTimeout(r, 2000)) - } - - await deletePipeline(WARMUP_PIPELINE) + configureTestClient() + await warmCompilationCache() } diff --git a/js-packages/web-console/vite.config.ts b/js-packages/web-console/vite.config.ts index 7ba2d08e599..24ea9b9e42a 100644 --- a/js-packages/web-console/vite.config.ts +++ b/js-packages/web-console/vite.config.ts @@ -7,7 +7,11 @@ import { playwright } from '@vitest/browser-playwright' import { type PluginOption } from 'vite' import devtoolsJson from 'vite-plugin-devtools-json' import virtual from 'vite-plugin-virtual' -import { defineConfig, type ViteUserConfigExport } from 'vitest/config' +import { + defineConfig, + type TestProjectInlineConfiguration, + type ViteUserConfigExport +} from 'vitest/config' import { felderaApiJsonSchemas } from './src/lib/functions/felderaApiJsonSchemas' import { svelteCssVirtualModuleFallback } from './src/lib/vite-plugins/svelte-css-virtual-module-fallback' @@ -61,6 +65,36 @@ const testOptimizeDepsInclude = [ 'virtua/svelte' ] +const browserTestProject = ({ + name, + include, + exclude +}: { + name: string + include: string[] + exclude?: string[] +}): TestProjectInlineConfiguration => ({ + extends: './vite.config.ts', + test: { + name, + browser: { + enabled: true, + provider: playwright({ contextOptions: {} }), + instances: [{ browser: 'chromium', headless: true }], + expect: { + toMatchScreenshot: { + resolveScreenshotPath({ testFileName, arg, ext }) { + return path.join(snapshotsDir, 'component', testFileName, `${arg}${ext}`) + } + } + } + }, + setupFiles: ['src/lib/vitest-browser-setup.ts'], + include, + exclude: exclude ?? ['src/lib/server/**'] + } +}) + // TODO: remove Prettier export default defineConfig(async () => { return { @@ -179,39 +213,35 @@ export default defineConfig(async () => { ) }, projects: [ + // Unit tests: *.spec.ts (run with `bun run test`) + browserTestProject({ name: 'client', include: ['src/**/*.svelte.spec.{js,ts}'] }), + { extends: './vite.config.ts', test: { - name: 'client', - browser: { - enabled: true, - provider: playwright({ - contextOptions: {} - }), - instances: [{ browser: 'chromium', headless: true }], - expect: { - toMatchScreenshot: { - resolveScreenshotPath({ testFileName, arg, ext }) { - return path.join(snapshotsDir, 'component', testFileName, `${arg}${ext}`) - } - } - } - }, - setupFiles: ['src/lib/vitest-browser-setup.ts'], - include: ['src/**/*.svelte.{test,spec}.{js,ts}'], - exclude: ['src/lib/server/**'] + name: 'server', + environment: 'node', + include: ['src/**/*.spec.{js,ts}'], + exclude: ['src/**/*.svelte.spec.{js,ts}'] } }, + // Integration tests: *.test.ts (require a Feldera instance, run with `bun run test-integration`) { extends: './vite.config.ts', test: { - name: 'server', + name: 'integration', environment: 'node', - include: ['src/**/*.{test,spec}.{js,ts}'], - exclude: ['src/**/*.svelte.{test,spec}.{js,ts}'] + globalSetup: ['src/lib/vitest-integration-setup.ts'], + include: ['src/**/*.test.{js,ts}'], + exclude: ['src/**/*.svelte.test.{js,ts}'] } - } + }, + + browserTestProject({ + name: 'integration-client', + include: ['src/**/*.svelte.test.{js,ts}'] + }) ] } } satisfies ViteUserConfigExport diff --git a/openapi.json b/openapi.json index 20ede772528..3b951bec1a4 100644 --- a/openapi.json +++ b/openapi.json @@ -10,7 +10,7 @@ "license": { "name": "MIT OR Apache-2.0" }, - "version": "0.281.0" + "version": "0.282.0" }, "paths": { "/config/authentication": { diff --git a/python/pyproject.toml b/python/pyproject.toml index 4de5966d8c4..6d2d94693f4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "feldera" readme = "README.md" description = "The feldera python client" -version = "0.281.0" +version = "0.282.0" license = "MIT" requires-python = ">=3.10" authors = [ diff --git a/python/uv.lock b/python/uv.lock index 7876f86939e..036a757848d 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -12,7 +12,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-03-26T19:13:33.259555Z" +exclude-newer = "2026-03-28T07:51:53.482034304Z" exclude-newer-span = "P1W" [[package]] @@ -221,7 +221,7 @@ wheels = [ [[package]] name = "feldera" -version = "0.281.0" +version = "0.282.0" source = { editable = "." } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },