Skip to content

Commit 833ba07

Browse files
Karakatiza666Karakatiza666
committed
[web-console] Integration tests for retrieving connector errors
Signed-off-by: Karakatiza666 <bulakh.96@gmail.com>
1 parent af4c983 commit 833ba07

File tree

7 files changed

+190
-36
lines changed

7 files changed

+190
-36
lines changed

‎.github/workflows/test-web-console-e2e.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@ jobs:
4444
- name: Verify pipeline-manager is reachable
4545
run: curl -fsSL --retry 5 --retry-delay 2 --retry-connrefused http://localhost:8080/healthz
4646

47+
- name: Run vitest integration tests
48+
if: ${{ vars.CI_DRY_RUN != 'true' }}
49+
run: bun run test-integration
50+
working-directory: js-packages/web-console
51+
env:
52+
FELDERA_API_URL: http://localhost:8080
53+
4754
- name: Run Playwright e2e tests
4855
if: ${{ vars.CI_DRY_RUN != 'true' }}
4956
run: bun run test-e2e

‎js-packages/web-console/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,8 @@
132132
"test-prepare": "git clone --depth 1 https://github.com/feldera/playwright-snapshots.git || true && mkdir -p playwright-snapshots/e2e playwright-snapshots/component",
133133
"test-update-snapshots": "bun run test -- --update && bun playwright test --update-snapshots",
134134
"test-unit": "vitest",
135-
"test": "bun run test-unit -- --run"
135+
"test-integration": "vitest --run --project integration --project integration-client",
136+
"test": "bun run test-unit -- --run --project client --project server"
136137
},
137138
"trustedDependencies": ["@axa-fr/oidc-client", "sk-oidc-oauth", "svelte-preprocess"],
138139
"type": "module"

‎js-packages/web-console/src/lib/components/pipelines/editor/performance/ConnectorErrors.svelte‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
type OutputEndpointStatus
1313
} from '$lib/services/manager'
1414
import { getInputConnectorStatus, getOutputConnectorStatus } from '$lib/services/pipelineManager'
15+
import { getCaseDependentName } from '$lib/functions/felderaRelation'
1516
1617
export type ConnectorErrorFilter = 'all' | 'parse' | 'transport' | 'encode'
1718
@@ -61,6 +62,8 @@
6162
tagsFilter = filter
6263
})
6364
65+
const strippedConnectorName = $derived(connectorName.slice(getCaseDependentName(relationName).name.length + 1))
66+
6467
$effect(() => {
6568
pipelineName
6669
relationName
@@ -69,11 +72,10 @@
6972
loading = true
7073
status = null
7174
72-
const stripped = connectorName.slice(connectorName.indexOf('.') + 1)
7375
const request =
7476
direction === 'input'
75-
? getInputConnectorStatus(pipelineName, relationName, stripped)
76-
: getOutputConnectorStatus(pipelineName, relationName, stripped)
77+
? getInputConnectorStatus(pipelineName, relationName, strippedConnectorName)
78+
: getOutputConnectorStatus(pipelineName, relationName, strippedConnectorName)
7779
7880
request.then((s) => {
7981
status = s
@@ -137,7 +139,7 @@
137139
<div class="bg-white-dark flex h-full flex-col gap-2 rounded p-4">
138140
<div class="flex items-start justify-between">
139141
<div>
140-
<div class="font-medium">{connectorName.replace('.', ' · ')}</div>
142+
<div class="font-medium">{relationName} · {strippedConnectorName}</div>
141143
</div>
142144
<button class="fd fd-x text-[20px]" onclick={() => (open = false)} aria-label="Close"
143145
></button>

‎js-packages/web-console/src/lib/functions/felderaRelation.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,7 @@ export const normalizeCaseIndependentName = ({
2121
export const getCaseDependentName = (caseIndependentName: string) => {
2222
const case_sensitive = isCaseSensitive(caseIndependentName)
2323
return {
24-
name: normalizeCaseIndependentName({
25-
name: caseIndependentName.replaceAll('"', ''),
26-
case_sensitive
27-
}),
24+
name: caseIndependentName.replaceAll('"', ''),
2825
case_sensitive
2926
}
3027
}

‎js-packages/web-console/src/lib/services/pipelineManager.test.ts‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
/**
2+
* Integration tests for pipelineManager.ts that require a running Feldera instance.
3+
* Run via the 'integration' vitest project:
4+
*
5+
* FELDERA_API_URL=http://localhost:8080 bun run test-integration
6+
*/
7+
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
8+
9+
const BASE_URL = process.env.FELDERA_API_URL ?? 'http://localhost:8080'
10+
11+
// Pipeline name is URL-safe by convention, but table/view/connector names are not.
12+
const PIPELINE_NAME = 'test-special-chars-connector-status'
13+
14+
// Names with dots and URL-unsafe characters
15+
const TABLE_NAME = 'my.input.table'
16+
const VIEW_NAME = 'my.output.view'
17+
const INPUT_CONNECTOR_NAME = 'http.input/special&name'
18+
const OUTPUT_CONNECTOR_NAME = 'http.output/special&name'
19+
20+
const api = (path: string, init?: RequestInit) =>
21+
fetch(`${BASE_URL}${path}`, {
22+
...init,
23+
headers: { 'Content-Type': 'application/json', ...init?.headers }
24+
})
25+
26+
describe('pipelineManager connector status with special characters', () => {
27+
beforeAll(async () => {
28+
// Clean up any leftover pipeline from a previous run
29+
const del = await api(`/v0/pipelines/${PIPELINE_NAME}`, { method: 'DELETE' })
30+
if (del.ok || del.status === 404) {
31+
// ok
32+
} else {
33+
throw new Error(`Cleanup failed: ${del.status} ${await del.text()}`)
34+
}
35+
36+
// SQL program with quoted identifiers containing dots
37+
const programCode = `
38+
CREATE TABLE "${TABLE_NAME}" (id INT NOT NULL, val VARCHAR)
39+
WITH (
40+
'connectors' = '[{
41+
"name": "${INPUT_CONNECTOR_NAME}",
42+
"transport": { "name": "http_input" },
43+
"format": { "name": "json", "config": { "update_format": "raw" } }
44+
}]'
45+
);
46+
CREATE VIEW "${VIEW_NAME}" AS SELECT * FROM "${TABLE_NAME}"
47+
WITH (
48+
'connectors' = '[{
49+
"name": "${OUTPUT_CONNECTOR_NAME}",
50+
"transport": { "name": "http_output" },
51+
"format": { "name": "json", "config": { "update_format": "raw" } }
52+
}]'
53+
);
54+
`
55+
56+
// Create the pipeline
57+
const put = await api(`/v0/pipelines/${PIPELINE_NAME}`, {
58+
method: 'PUT',
59+
body: JSON.stringify({
60+
name: PIPELINE_NAME,
61+
description: 'Integration test for special character handling',
62+
program_code: programCode,
63+
runtime_config: {}
64+
})
65+
})
66+
expect(put.ok, `PUT pipeline: ${put.status} ${await put.clone().text()}`).toBe(true)
67+
68+
// Wait for compilation
69+
for (let i = 0; i < 120; i++) {
70+
const res = await api(`/v0/pipelines/${PIPELINE_NAME}`)
71+
const info = await res.json()
72+
if (info.program_status === 'Success') break
73+
if (info.program_status?.SqlError || info.program_status?.RustError || info.program_status?.SystemError) {
74+
throw new Error(`Compilation failed: ${JSON.stringify(info.program_status)}`)
75+
}
76+
await new Promise((r) => setTimeout(r, 1000))
77+
}
78+
79+
// Start the pipeline
80+
const start = await api(`/v0/pipelines/${PIPELINE_NAME}/start`, { method: 'POST' })
81+
expect(start.ok, `POST start: ${start.status} ${await start.clone().text()}`).toBe(true)
82+
83+
// Wait for pipeline to be running
84+
for (let i = 0; i < 60; i++) {
85+
const res = await api(`/v0/pipelines/${PIPELINE_NAME}`)
86+
const info = await res.json()
87+
if (info.deployment_status === 'Running') break
88+
if (info.deployment_error) {
89+
throw new Error(`Deployment failed: ${JSON.stringify(info.deployment_error)}`)
90+
}
91+
await new Promise((r) => setTimeout(r, 1000))
92+
}
93+
}, 180_000)
94+
95+
afterAll(async () => {
96+
// Shutdown and delete
97+
await api(`/v0/pipelines/${PIPELINE_NAME}/shutdown`, { method: 'POST' })
98+
// Wait for shutdown
99+
for (let i = 0; i < 30; i++) {
100+
const res = await api(`/v0/pipelines/${PIPELINE_NAME}`)
101+
const info = await res.json()
102+
if (info.deployment_status === 'Shutdown') break
103+
await new Promise((r) => setTimeout(r, 1000))
104+
}
105+
await api(`/v0/pipelines/${PIPELINE_NAME}`, { method: 'DELETE' })
106+
}, 60_000)
107+
108+
it('getInputConnectorStatus succeeds with dot and URL-unsafe characters in table and connector name', async () => {
109+
const url = `/v0/pipelines/${PIPELINE_NAME}/tables/${encodeURIComponent(TABLE_NAME)}/connectors/${encodeURIComponent(INPUT_CONNECTOR_NAME)}/status`
110+
const res = await api(url)
111+
expect(res.ok, `GET input connector status: ${res.status} ${await res.clone().text()}`).toBe(true)
112+
const body = await res.json()
113+
expect(body).toHaveProperty('num_parse_errors')
114+
expect(body).toHaveProperty('num_transport_errors')
115+
})
116+
117+
it('getOutputConnectorStatus succeeds with dot and URL-unsafe characters in view and connector name', async () => {
118+
const url = `/v0/pipelines/${PIPELINE_NAME}/views/${encodeURIComponent(VIEW_NAME)}/connectors/${encodeURIComponent(OUTPUT_CONNECTOR_NAME)}/status`
119+
const res = await api(url)
120+
expect(res.ok, `GET output connector status: ${res.status} ${await res.clone().text()}`).toBe(true)
121+
const body = await res.json()
122+
expect(body).toHaveProperty('num_encode_errors')
123+
expect(body).toHaveProperty('num_transport_errors')
124+
})
125+
})

‎js-packages/web-console/src/lib/services/pipelineManager.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,7 @@ export const getInputConnectorStatus = (
477477
) =>
478478
mapResponse(
479479
_getPipelineInputConnectorStatus({
480-
path: { pipeline_name, table_name: encodeURIComponent(table_name), connector_name: encodeURIComponent(connector_name) },
480+
path: { pipeline_name, table_name, connector_name },
481481
...options
482482
}),
483483
(v) => v
@@ -491,7 +491,7 @@ export const getOutputConnectorStatus = (
491491
) =>
492492
mapResponse(
493493
_getPipelineOutputConnectorStatus({
494-
path: { pipeline_name, view_name: encodeURIComponent(view_name), connector_name: encodeURIComponent(connector_name) },
494+
path: { pipeline_name, view_name, connector_name },
495495
...options
496496
}),
497497
(v) => v
@@ -550,7 +550,7 @@ export const postApiKey = (name: string, options?: FetchOptions) =>
550550

551551
export const deleteApiKey = (name: string, options?: FetchOptions) =>
552552
mapResponse(
553-
_deleteApiKey({ path: { api_key_name: encodeURIComponent(name) }, ...options }),
553+
_deleteApiKey({ path: { api_key_name: name }, ...options }),
554554
(v) => v,
555555
() => {
556556
throw new Error(`Failed to delete ${name} API key`)
@@ -929,7 +929,7 @@ export const relationIngress = async (
929929
options?: FetchOptions
930930
) => {
931931
return httpInput({
932-
path: { pipeline_name: pipelineName, table_name: encodeURIComponent(relationName) },
932+
path: { pipeline_name: pipelineName, table_name: relationName },
933933
parseAs: 'text', // Response is empty, so no need to parse it as JSON
934934
query: { format: 'json', array: true, update_format: 'insert_delete', force: !!force },
935935
body: data as any,

‎js-packages/web-console/vite.config.ts‎

Lines changed: 45 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { playwright } from '@vitest/browser-playwright'
77
import { type PluginOption } from 'vite'
88
import devtoolsJson from 'vite-plugin-devtools-json'
99
import virtual from 'vite-plugin-virtual'
10-
import { defineConfig, type ViteUserConfigExport } from 'vitest/config'
10+
import { defineConfig, type TestProjectInlineConfiguration, type ViteUserConfigExport } from 'vitest/config'
1111
import { felderaApiJsonSchemas } from './src/lib/functions/felderaApiJsonSchemas'
1212
import { svelteCssVirtualModuleFallback } from './src/lib/vite-plugins/svelte-css-virtual-module-fallback'
1313

@@ -61,6 +61,36 @@ const testOptimizeDepsInclude = [
6161
'virtua/svelte'
6262
]
6363

64+
const browserTestProject = ({
65+
name,
66+
include,
67+
exclude
68+
}: {
69+
name: string
70+
include: string[]
71+
exclude?: string[]
72+
}): TestProjectInlineConfiguration => ({
73+
extends: './vite.config.ts',
74+
test: {
75+
name,
76+
browser: {
77+
enabled: true,
78+
provider: playwright({ contextOptions: {} }),
79+
instances: [{ browser: 'chromium', headless: true }],
80+
expect: {
81+
toMatchScreenshot: {
82+
resolveScreenshotPath({ testFileName, arg, ext }) {
83+
return path.join(snapshotsDir, 'component', testFileName, `${arg}${ext}`)
84+
}
85+
}
86+
}
87+
},
88+
setupFiles: ['src/lib/vitest-browser-setup.ts'],
89+
include,
90+
exclude: exclude ?? ['src/lib/server/**']
91+
}
92+
})
93+
6494
// TODO: remove Prettier
6595
export default defineConfig(async () => {
6696
return {
@@ -179,39 +209,31 @@ export default defineConfig(async () => {
179209
)
180210
},
181211
projects: [
212+
// Unit tests: *.spec.ts (run with `bun run test`)
213+
browserTestProject({ name: 'client', include: ['src/**/*.svelte.spec.{js,ts}'] }),
214+
182215
{
183216
extends: './vite.config.ts',
184217
test: {
185-
name: 'client',
186-
browser: {
187-
enabled: true,
188-
provider: playwright({
189-
contextOptions: {}
190-
}),
191-
instances: [{ browser: 'chromium', headless: true }],
192-
expect: {
193-
toMatchScreenshot: {
194-
resolveScreenshotPath({ testFileName, arg, ext }) {
195-
return path.join(snapshotsDir, 'component', testFileName, `${arg}${ext}`)
196-
}
197-
}
198-
}
199-
},
200-
setupFiles: ['src/lib/vitest-browser-setup.ts'],
201-
include: ['src/**/*.svelte.{test,spec}.{js,ts}'],
202-
exclude: ['src/lib/server/**']
218+
name: 'server',
219+
environment: 'node',
220+
include: ['src/**/*.spec.{js,ts}'],
221+
exclude: ['src/**/*.svelte.spec.{js,ts}']
203222
}
204223
},
205224

225+
// Integration tests: *.test.ts (require a Feldera instance, run with `bun run test-integration`)
206226
{
207227
extends: './vite.config.ts',
208228
test: {
209-
name: 'server',
229+
name: 'integration',
210230
environment: 'node',
211-
include: ['src/**/*.{test,spec}.{js,ts}'],
212-
exclude: ['src/**/*.svelte.{test,spec}.{js,ts}']
231+
include: ['src/**/*.test.{js,ts}'],
232+
exclude: ['src/**/*.svelte.test.{js,ts}']
213233
}
214-
}
234+
},
235+
236+
browserTestProject({ name: 'integration-client', include: ['src/**/*.svelte.test.{js,ts}'] })
215237
]
216238
}
217239
} satisfies ViteUserConfigExport

0 commit comments

Comments
 (0)