Skip to content

Commit f850cae

Browse files
snkasKarakatiza666
snkas
and
Karakatiza666
committed
demo: packaged SQL-only demos
- Support multiple demos directories - Ignore directories and files not ending with '.sql' within a demo directory - Python SQL preamble validation script which is called in Earthfile - pipeline-manager does SQL preamble validation - Copy demos in Earthfile/Dockerfile and add argument to entry points - First five packaged demos - Web Console uses new endpoint format Co-authored-by: Karakatiza666 <bulakh.96@gmail.com> Signed-off-by: Karakatiza666 <bulakh.96@gmail.com> Signed-off-by: Simon Kassing <simon.kassing@feldera.com>
1 parent ebda7af commit f850cae

22 files changed

Lines changed: 1400 additions & 364 deletions

File tree

‎Earthfile‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,11 @@ install-python:
113113
RUN pip install --user -v python/
114114
SAVE ARTIFACT /root/.local/lib/python3.10
115115

116+
demo-packaged-sql-formatting-check:
117+
FROM +install-python
118+
COPY --dir demo/packaged demo/packaged
119+
RUN (cd demo/packaged && python3 validate-preamble.py sql/*.sql)
120+
116121
build-webui-deps:
117122
FROM +install-deps
118123

@@ -302,6 +307,10 @@ build-pipeline-manager-container:
302307
RUN mkdir -p .feldera/cargo_workspace
303308
COPY --chown=feldera Cargo.lock .feldera/cargo_workspace/Cargo.lock
304309

310+
# Copy over demos
311+
RUN mkdir -p demos
312+
COPY demo/packaged/sql demos
313+
305314
# Then copy over the crates needed by the sql compiler
306315
COPY crates/dbsp database-stream-processor/crates/dbsp
307316
COPY crates/feldera-types database-stream-processor/crates/feldera-types
@@ -322,12 +331,12 @@ build-pipeline-manager-container:
322331
ENV PATH="$PATH:/home/feldera/.cargo/bin"
323332

324333
RUN ./pipeline-manager --bind-address=0.0.0.0 --sql-compiler-home=/home/feldera/database-stream-processor/sql-to-dbsp-compiler --compilation-profile=unoptimized --dbsp-override-path=/home/feldera/database-stream-processor --precompile
325-
ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/feldera/database-stream-processor/sql-to-dbsp-compiler", "--dbsp-override-path=/home/feldera/database-stream-processor", "--compilation-profile=unoptimized"]
334+
ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/feldera/database-stream-processor/sql-to-dbsp-compiler", "--dbsp-override-path=/home/feldera/database-stream-processor", "--compilation-profile=unoptimized", "--demos-dir", "/home/feldera/demos"]
326335

327336
# Same as the above, but with a permissive CORS setting, else playwright doesn't work
328337
pipeline-manager-container-cors-all:
329338
FROM +build-pipeline-manager-container
330-
ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/feldera/database-stream-processor/sql-to-dbsp-compiler", "--dbsp-override-path=/home/feldera/database-stream-processor", "--dev-mode", "--compilation-profile=unoptimized"]
339+
ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/feldera/database-stream-processor/sql-to-dbsp-compiler", "--dbsp-override-path=/home/feldera/database-stream-processor", "--dev-mode", "--compilation-profile=unoptimized", "--demos-dir", "/home/feldera/demos"]
331340

332341
# TODO: mirrors the Dockerfile. See note above.
333342
build-demo-container:
@@ -551,6 +560,7 @@ ci-tests:
551560
BUILD +openapi-checker
552561
BUILD +test-sql
553562
BUILD +integration-tests
563+
BUILD +demo-packaged-sql-formatting-check
554564
# BUILD +test-docker-compose-stable
555565
# TODO: Temporarily disabled while we port the demo script
556566
# BUILD +test-snowflake

‎crates/pipeline-manager/src/api/config_api.rs‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
// Configuration API to retrieve the current authentication configuration and list of demos
2-
use crate::demo::{read_demos_from_directory, Demo};
32
use actix_web::{get, web::Data as WebData, HttpRequest, HttpResponse};
43
use serde::Serialize;
54
use serde_json::json;
6-
use std::path::Path;
75
use utoipa::ToSchema;
86

97
use super::{ManagerError, ServerState};
@@ -69,11 +67,11 @@ async fn get_config_authentication(
6967
path="/config/demos",
7068
responses(
7169
(status = OK
72-
, description = "List of demos."
70+
, description = "List of demos"
7371
, content_type = "application/json"
7472
, body = Vec<Demo>),
7573
(status = INTERNAL_SERVER_ERROR
76-
, description = "Failed to read demos from the demos directory."
74+
, description = "Failed to read demos from the demos directories"
7775
, body = ErrorResponse),
7876
),
7977
context_path = "/v0",
@@ -82,12 +80,7 @@ async fn get_config_authentication(
8280
)]
8381
#[get("/config/demos")]
8482
async fn get_config_demos(state: WebData<ServerState>) -> Result<HttpResponse, ManagerError> {
85-
match &state._config.demos_dir {
86-
None => Ok(HttpResponse::Ok().json(Vec::<Demo>::new())),
87-
Some(demos_dir) => {
88-
Ok(HttpResponse::Ok().json(read_demos_from_directory(Path::new(&demos_dir))?))
89-
}
90-
}
83+
Ok(HttpResponse::Ok().json(&state.demos))
9184
}
9285

9386
#[derive(Serialize, ToSchema)]

‎crates/pipeline-manager/src/api/mod.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ mod pipeline;
2626
use crate::auth::JwkCache;
2727
use crate::config::ApiServerConfig;
2828
use crate::db::storage_postgres::StoragePostgres;
29+
use crate::demo::{read_demos_from_directories, Demo};
2930
use crate::error::ManagerError;
3031
use crate::probe::Probe;
3132
use crate::runner::interaction::RunnerInteraction;
@@ -327,18 +328,21 @@ pub(crate) struct ServerState {
327328
_config: ApiServerConfig,
328329
pub jwk_cache: Arc<Mutex<JwkCache>>,
329330
probe: Arc<Mutex<Probe>>,
331+
demos: Vec<Demo>,
330332
}
331333

332334
impl ServerState {
333335
pub async fn new(config: ApiServerConfig, db: Arc<Mutex<StoragePostgres>>) -> AnyResult<Self> {
334336
let runner = RunnerInteraction::new(config.clone(), db.clone());
335337
let db_copy = db.clone();
338+
let demos = read_demos_from_directories(&config.demos_dir);
336339
Ok(Self {
337340
db,
338341
runner,
339342
_config: config,
340343
jwk_cache: Arc::new(Mutex::new(JwkCache::new())),
341344
probe: Probe::new(db_copy).await,
345+
demos,
342346
})
343347
}
344348
}

‎crates/pipeline-manager/src/auth.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -721,7 +721,7 @@ mod test {
721721
dump_openapi: false,
722722
config_file: None,
723723
allowed_origins: None,
724-
demos_dir: None,
724+
demos_dir: vec![],
725725
telemetry: "".to_owned(),
726726
runner_hostname_port: "127.0.0.1:8089".to_string(),
727727
};

‎crates/pipeline-manager/src/config.rs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@ fn default_compilation_profile() -> CompilationProfile {
5050
CompilationProfile::Optimized
5151
}
5252

53+
fn default_demos_dir() -> Vec<String> {
54+
vec!["demo/packaged/sql".to_string()]
55+
}
56+
5357
/// Pipeline manager configuration read from a YAML config file or from command
5458
/// line arguments.
5559
#[derive(Parser, Deserialize, Debug, Clone)]
@@ -170,13 +174,16 @@ pub struct ApiServerConfig {
170174
#[arg(long)]
171175
pub dev_mode: bool,
172176

173-
/// Local directory in which demos are stored for supplying clients like the UI with
177+
/// Local directories in which demos are stored for supplying clients like the UI with
174178
/// a set of demos to present to the user. Administrators can use this option to set
175179
/// up environment-specific demos for users (e.g., ones that connect to an internal
176180
/// data source).
177-
#[serde(default)]
178-
#[arg(long)]
179-
pub demos_dir: Option<String>,
181+
///
182+
/// For each directory, the files are read sorted on the filename.
183+
/// For multiple directories, the lists of demos are appended one after the other into a single one.
184+
/// Files which do not end in `.sql` and directories are ignored. Symlinks are followed.
185+
#[arg(long, default_values_t = default_demos_dir())]
186+
pub demos_dir: Vec<String>,
180187

181188
/// Telemetry key.
182189
///

0 commit comments

Comments
 (0)