Skip to content

Commit 5af34fe

Browse files
blpblp
committed
web-console: Update OpenAPI and explain how to fix it repeatably.
`yarn generate-openapi` generates some code with bugs in it. This commit regenerates the OpenAPI with those bugs fixes and adds instructions on how to do that again. Thanks to @Karakatiza666 for explaining the fix. Signed-off-by: Ben Pfaff <blp@feldera.com>
1 parent f33c402 commit 5af34fe

6 files changed

Lines changed: 105 additions & 29 deletions

File tree

‎web-console/README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,9 @@ The bindings for OpenAPI (under src/lib/services/manager) are generated using
7070
If you change the API, execute the following steps to update the bindings:
7171

7272
```bash
73+
yarn build-openapi
7374
yarn generate-openapi
74-
yarn format
75+
patch -p2 < openapi-fixes.patch
7576
```
7677

7778
Note sometimes strange caching errors may warrant deleting `node_modules` after

‎web-console/openapi-fixes.patch‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
This patch fixes bugs in output from `yarn generate-openapi && yarn format`.
2+
3+
diff --git b/web-console/src/lib/services/manager/models/ColumnType.ts a/web-console/src/lib/services/manager/models/ColumnType.ts
4+
index 8484d2bea..953d53760 100644
5+
--- b/web-console/src/lib/services/manager/models/ColumnType.ts
6+
+++ a/web-console/src/lib/services/manager/models/ColumnType.ts
7+
@@ -57,5 +57,5 @@ export type ColumnType = {
8+
* - `DECIMAL(1,2)` sets scale to `2`.
9+
*/
10+
scale?: number | null
11+
- type?: SqlType
12+
+ type: SqlType
13+
}
14+
diff --git b/web-console/src/lib/services/manager/models/TransportConfig.ts a/web-console/src/lib/services/manager/models/TransportConfig.ts
15+
index 8c3a364bc..273fa0299 100644
16+
--- b/web-console/src/lib/services/manager/models/TransportConfig.ts
17+
+++ a/web-console/src/lib/services/manager/models/TransportConfig.ts
18+
@@ -16,33 +16,38 @@ import type { UrlInputConfig } from './UrlInputConfig'
19+
export type TransportConfig =
20+
| {
21+
config: FileInputConfig
22+
- name: TransportConfig.name
23+
+ name: TransportConfig.name.FILE_INPUT
24+
}
25+
| {
26+
config: FileOutputConfig
27+
- name: TransportConfig.name
28+
+ name: TransportConfig.name.FILE_OUTPUT
29+
}
30+
| {
31+
config: KafkaInputConfig
32+
- name: TransportConfig.name
33+
+ name: TransportConfig.name.KAFKA_INPUT
34+
}
35+
| {
36+
config: KafkaOutputConfig
37+
- name: TransportConfig.name
38+
+ name: TransportConfig.name.KAFKA_OUTPUT
39+
}
40+
| {
41+
config: UrlInputConfig
42+
- name: TransportConfig.name
43+
+ name: TransportConfig.name.URL_INPUT
44+
}
45+
| {
46+
config: S3InputConfig
47+
- name: TransportConfig.name
48+
- }
49+
- | {
50+
- name: TransportConfig.name
51+
+ name: TransportConfig.name.S3_INPUT
52+
}
53+
+
54+
export namespace TransportConfig {
55+
export enum name {
56+
- FILE_INPUT = 'file_input'
57+
+ FILE_INPUT = 'file_input',
58+
+ FILE_OUTPUT = 'file_output',
59+
+ KAFKA_INPUT = 'kafka_input',
60+
+ KAFKA_OUTPUT = 'kafka_output',
61+
+ URL_INPUT = 'url_input',
62+
+ S3_INPUT = 's3_input',
63+
+ HTTP_INPUT = 'http_input',
64+
+ HTTP_OUTPUT = 'http_output'
65+
}
66+
}

‎web-console/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
"lint": "eslint --max-warnings 0 --fix \"src/**/*.{js,jsx,ts,tsx}\"",
2323
"format": "prettier --write \"{src,tests}/**/*.{js,jsx,ts,tsx}\"",
2424
"format:check": "prettier --check \"src/**/*.{js,jsx,ts,tsx}\"",
25-
"generate-openapi": "openapi --input ../openapi.json --output ./src/lib/services/manager",
25+
"generate-openapi": "openapi --input ../openapi.json --output ./src/lib/services/manager && yarn format",
2626
"build-openapi": "cd .. && cargo run --bin pipeline-manager -- --dump-openapi",
2727
"test": "PLAYWRIGHT_API_ORIGIN=http://localhost:8080/ PLAYWRIGHT_APP_ORIGIN=http://localhost:8080/ DISPLAY= yarn playwright test",
2828
"test:ui": "PLAYWRIGHT_API_ORIGIN=http://localhost:8080/ PLAYWRIGHT_APP_ORIGIN=http://localhost:8080/ DISPLAY= yarn playwright test --ui-port=0",

‎web-console/src/lib/services/manager/models/ConnectorConfig.ts‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,34 @@
33
/* tslint:disable */
44
/* eslint-disable */
55
import type { FormatConfig } from './FormatConfig'
6+
import type { OutputBufferConfig } from './OutputBufferConfig'
67
import type { TransportConfig } from './TransportConfig'
78
/**
89
* A data connector's configuration
910
*/
10-
export type ConnectorConfig = {
11+
export type ConnectorConfig = OutputBufferConfig & {
1112
format: FormatConfig
1213
/**
1314
* Backpressure threshold.
1415
*
15-
* Maximal amount of records buffered by the endpoint before the endpoint
16-
* is paused by the backpressure mechanism. Note that this is not a
17-
* hard bound: there can be a small delay between the backpressure
18-
* mechanism is triggered and the endpoint is paused, during which more
19-
* data may be received.
16+
* Maximal number of records queued by the endpoint before the endpoint
17+
* is paused by the backpressure mechanism.
18+
*
19+
* For input endpoints, this setting bounds the number of records that have
20+
* been received from the input transport but haven't yet been consumed by
21+
* the circuit since the circuit, since the circuit is still busy processing
22+
* previous inputs.
23+
*
24+
* For output endpoints, this setting bounds the number of records that have
25+
* been produced by the circuit but not yet sent via the output transport endpoint
26+
* nor stored in the output buffer (see `enable_output_buffer`).
27+
*
28+
* Note that this is not a hard bound: there can be a small delay between
29+
* the backpressure mechanism is triggered and the endpoint is paused, during
30+
* which more data may be queued.
2031
*
2132
* The default is 1 million.
2233
*/
23-
max_buffered_records?: number
34+
max_queued_records?: number
2435
transport: TransportConfig
2536
}

‎web-console/src/lib/services/manager/models/OutputBufferConfig.ts‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,19 @@ export type OutputBufferConfig = {
1616
* small files every few milliseconds.
1717
*
1818
* To achieve such input/output decoupling, users can enable output buffering by
19-
* setting the `enable_buffer` flag to `true`. When buffering is enabled, output
19+
* setting the `enable_output_buffer` flag to `true`. When buffering is enabled, output
2020
* updates produced by the pipeline are consolidated in an internal buffer and are
2121
* pushed to the output transport when one of several conditions is satisfied:
2222
*
23-
* * data has been accumulated in the buffer for more than `max_buffer_time_millis`
23+
* * data has been accumulated in the buffer for more than `max_output_buffer_time_millis`
2424
* milliseconds.
25-
* * buffer size exceeds `max_buffered_records` records.
25+
* * buffer size exceeds `max_output_buffer_size_records` records.
2626
*
2727
* This flag is `false` by default.
2828
*/
29-
enable_buffer?: boolean
29+
enable_output_buffer?: boolean
3030
/**
31-
* Maximum number of output updates to be kept in the buffer.
31+
* Maximum number of updates to be kept in the output buffer.
3232
*
3333
* This parameter bounds the maximal size of the buffer.
3434
* Note that the size of the buffer is not always equal to the
@@ -38,20 +38,20 @@ export type OutputBufferConfig = {
3838
* By default, the buffer can grow indefinitely until one of
3939
* the other output conditions is satisfied.
4040
*
41-
* NOTE: this configuration option requires the `enable_buffer` flag
41+
* NOTE: this configuration option requires the `enable_output_buffer` flag
4242
* to be set.
4343
*/
44-
max_buffer_size_records?: number
44+
max_output_buffer_size_records?: number
4545
/**
46-
* Maximum time in milliseconds data is kept in the buffer.
46+
* Maximum time in milliseconds data is kept in the output buffer.
4747
*
4848
* By default, data is kept in the buffer indefinitely until one of
4949
* the other output conditions is satisfied. When this option is
5050
* set the buffer will be flushed at most every
51-
* `max_buffer_time_millis` milliseconds.
51+
* `max_output_buffer_time_millis` milliseconds.
5252
*
53-
* NOTE: this configuration option requires the `enable_buffer` flag
53+
* NOTE: this configuration option requires the `enable_output_buffer` flag
5454
* to be set.
5555
*/
56-
max_buffer_time_millis?: number
56+
max_output_buffer_time_millis?: number
5757
}

‎web-console/src/lib/services/manager/models/OutputEndpointConfig.ts‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,13 @@
33
/* tslint:disable */
44
/* eslint-disable */
55
import type { ConnectorConfig } from './ConnectorConfig'
6-
import type { OutputBufferConfig } from './OutputBufferConfig'
76
/**
87
* Describes an output connector configuration
98
*/
10-
export type OutputEndpointConfig = ConnectorConfig &
11-
OutputBufferConfig & {
12-
/**
13-
* The name of the output stream of the circuit that this endpoint is
14-
* connected to.
15-
*/
16-
stream: string
17-
}
9+
export type OutputEndpointConfig = ConnectorConfig & {
10+
/**
11+
* The name of the output stream of the circuit that this endpoint is
12+
* connected to.
13+
*/
14+
stream: string
15+
}

0 commit comments

Comments
 (0)