diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2e721ea10b2..70ac3717fd3 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -34,7 +34,7 @@ "biomejs.biome" ], "settings": { - "editor.formatOnSave": true, + "editor.formatOnSave": false, "terminal.integrated.defaultProfile.linux": "bash", "files.exclude": { "**/CODE_OF_CONDUCT.md": true, diff --git a/.github/workflows/build-docker-dev.yml b/.github/workflows/build-docker-dev.yml index e097c936b83..8929d05483c 100644 --- a/.github/workflows/build-docker-dev.yml +++ b/.github/workflows/build-docker-dev.yml @@ -96,7 +96,7 @@ jobs: needs: build-docker-ci-dev steps: - name: Download digests - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: path: ${{ runner.temp }}/digests pattern: digests-* diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 74ad8970b84..2da8edd0bb4 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -52,19 +52,19 @@ jobs: - name: Download fda id: binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: fda-${{ matrix.rust_target }} path: build - name: Download pipeline-manager - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: pipeline-manager-${{ matrix.rust_target }} path: build - name: Download Compiler Binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: feldera-sql-compiler path: build @@ -141,7 +141,7 @@ jobs: needs: build-docker steps: - name: Download digests - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: path: ${{ runner.temp }}/digests pattern: digests-* diff --git a/.github/workflows/test-adapters.yml b/.github/workflows/test-adapters.yml index 5a4e8a570ee..d0937fe723f 100644 --- a/.github/workflows/test-adapters.yml +++ b/.github/workflows/test-adapters.yml @@ -62,7 +62,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Download Test Binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: feldera-test-binaries-${{ matrix.target }} path: build @@ -142,7 +142,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Download Test Binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: feldera-test-binaries-x86_64-unknown-linux-gnu path: build diff --git a/.github/workflows/test-integration-platform.yml b/.github/workflows/test-integration-platform.yml index 61c06b4e2d1..fa022ad6b05 100644 --- a/.github/workflows/test-integration-platform.yml +++ b/.github/workflows/test-integration-platform.yml @@ -234,7 +234,7 @@ jobs: CI_K8S_MINIO_SECRET_ACCESS_KEY: ${{ secrets.CI_K8S_MINIO_TEST_BUCKET_SECRET_ACCESS_KEY }} - name: Download fda binary - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: fda-${{ matrix.target }} path: build diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index acc92e228dd..37e1152a32a 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -33,13 +33,13 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Download Test Binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: feldera-test-binaries-${{ matrix.target }} path: build - name: Download Compiler Binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: feldera-sql-compiler path: sql-build diff --git a/Cargo.lock b/Cargo.lock index 8fd21ea182d..bba2c6970b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1882,7 +1882,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "regex", @@ -1900,7 +1900,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "regex", @@ -1918,7 +1918,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "log", "prettyplease", "proc-macro2", @@ -2600,7 +2600,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] @@ -3952,7 +3952,7 @@ dependencies = [ [[package]] name = "dbsp" -version = "0.314.0" +version = "0.315.0" dependencies = [ "anyhow", "arc-swap", @@ -4042,7 +4042,7 @@ dependencies = [ [[package]] name = "dbsp_adapters" -version = "0.314.0" +version = "0.315.0" dependencies = [ "actix", "actix-codec", @@ -4185,7 +4185,7 @@ dependencies = [ [[package]] name = "dbsp_nexmark" -version = "0.314.0" +version = "0.315.0" dependencies = [ "anyhow", "ascii_table", @@ -5157,7 +5157,7 @@ dependencies = [ [[package]] name = "fda" -version = "0.314.0" +version = "0.315.0" dependencies = [ "anyhow", "arrow", @@ -5209,7 +5209,7 @@ dependencies = [ [[package]] name = "feldera-adapterlib" -version = "0.314.0" +version = "0.315.0" dependencies = [ "actix-web", "anyhow", @@ -5243,7 +5243,7 @@ dependencies = [ [[package]] name = "feldera-buffer-cache" -version = "0.314.0" +version = "0.315.0" dependencies = [ "crossbeam-utils", "enum-map", @@ -5271,7 +5271,7 @@ dependencies = [ [[package]] name = "feldera-datagen" -version = "0.314.0" +version = "0.315.0" dependencies = [ "anyhow", "async-channel 2.5.0", @@ -5297,7 +5297,7 @@ dependencies = [ [[package]] name = "feldera-fxp" -version = "0.314.0" +version = "0.315.0" dependencies = [ "bytecheck", "dbsp", @@ -5317,7 +5317,7 @@ dependencies = [ [[package]] name = "feldera-iceberg" -version = "0.314.0" +version = "0.315.0" dependencies = [ "anyhow", "chrono", @@ -5339,7 +5339,7 @@ dependencies = [ [[package]] name = "feldera-ir" -version = "0.314.0" +version = "0.315.0" dependencies = [ "proptest", "proptest-derive", @@ -5351,7 +5351,7 @@ dependencies = [ [[package]] name = "feldera-macros" -version = "0.314.0" +version = "0.315.0" dependencies = [ "prettyplease", "proc-macro2", @@ -5361,7 +5361,7 @@ dependencies = [ [[package]] name = "feldera-observability" -version = "0.314.0" +version = "0.315.0" dependencies = [ "actix-http", "awc", @@ -5376,7 +5376,7 @@ dependencies = [ [[package]] name = "feldera-rest-api" -version = "0.314.0" +version = "0.315.0" dependencies = [ "chrono", "feldera-observability", @@ -5395,7 +5395,7 @@ dependencies = [ [[package]] name = "feldera-samply" -version = "0.314.0" +version = "0.315.0" dependencies = [ "crossbeam", "feldera-size-of", @@ -5431,7 +5431,7 @@ dependencies = [ [[package]] name = "feldera-sqllib" -version = "0.314.0" +version = "0.315.0" dependencies = [ "arcstr", "base58", @@ -5475,7 +5475,7 @@ dependencies = [ [[package]] name = "feldera-storage" -version = "0.314.0" +version = "0.315.0" dependencies = [ "anyhow", "crossbeam", @@ -5498,7 +5498,7 @@ dependencies = [ [[package]] name = "feldera-types" -version = "0.314.0" +version = "0.315.0" dependencies = [ "actix-web", "anyhow", @@ -7244,7 +7244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.53.5", + "windows-targets 0.48.5", ] [[package]] @@ -8761,7 +8761,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pipeline-manager" -version = "0.314.0" +version = "0.315.0" dependencies = [ "actix-cors", "actix-files", @@ -9695,9 +9695,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.8" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -9706,7 +9706,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls", - "socket2 0.5.10", + "socket2 0.6.1", "thiserror 2.0.17", "tokio", "tracing", @@ -9715,9 +9715,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", @@ -9971,7 +9971,7 @@ dependencies = [ [[package]] name = "readers" -version = "0.314.0" +version = "0.315.0" dependencies = [ "async-std", "csv", @@ -11670,7 +11670,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "sltsqlvalue" -version = "0.314.0" +version = "0.315.0" dependencies = [ "dbsp", "feldera-sqllib", @@ -12024,7 +12024,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "storage-test-compat" -version = "0.314.0" +version = "0.315.0" dependencies = [ "dbsp", "derive_more 1.0.0", @@ -13815,7 +13815,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 17f6f29dd3a..80fd8b7a5da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace.package] authors = ["Feldera Team "] -version = "0.314.0" +version = "0.315.0" license = "MIT OR Apache-2.0" homepage = "https://github.com/feldera/feldera" repository = "https://github.com/feldera/feldera" @@ -108,7 +108,7 @@ csv = "1.2.2" csv-core = "0.1.10" dashmap = "6.1.0" datafusion = "53.1" -dbsp = { path = "crates/dbsp", version = "0.314.0" } +dbsp = { path = "crates/dbsp", version = "0.315.0" } dbsp_nexmark = { path = "crates/nexmark" } deadpool-postgres = "0.14.1" # Feldera fork of delta-rs: upstream tag `rust-v0.32.3` + 2 patches, on branch @@ -133,20 +133,20 @@ erased-serde = "0.3.31" fake = "2.10" fastbloom = "0.14.0" fdlimit = "0.3.0" -feldera-buffer-cache = { version = "0.314.0", path = "crates/buffer-cache" } +feldera-buffer-cache = { version = "0.315.0", path = "crates/buffer-cache" } feldera-cloud1-client = "0.1.2" feldera-datagen = { path = "crates/datagen" } -feldera-fxp = { version = "0.314.0", path = "crates/fxp", features = ["dbsp"] } +feldera-fxp = { version = "0.315.0", path = "crates/fxp", features = ["dbsp"] } feldera-iceberg = { path = "crates/iceberg" } -feldera-observability = { version = "0.314.0", path = "crates/feldera-observability" } -feldera-macros = { version = "0.314.0", path = "crates/feldera-macros" } -feldera-sqllib = { version = "0.314.0", path = "crates/sqllib" } -feldera-storage = { version = "0.314.0", path = "crates/storage" } -feldera-types = { version = "0.314.0", path = "crates/feldera-types" } -feldera-rest-api = { version = "0.314.0", path = "crates/rest-api" } -feldera-ir = { version = "0.314.0", path = "crates/ir" } -feldera-adapterlib = { version = "0.314.0", path = "crates/adapterlib" } -feldera-samply = { version = "0.314.0", path = "crates/samply" } +feldera-observability = { version = "0.315.0", path = "crates/feldera-observability" } +feldera-macros = { version = "0.315.0", path = "crates/feldera-macros" } +feldera-sqllib = { version = "0.315.0", path = "crates/sqllib" } +feldera-storage = { version = "0.315.0", path = "crates/storage" } +feldera-types = { version = "0.315.0", path = "crates/feldera-types" } +feldera-rest-api = { version = "0.315.0", path = "crates/rest-api" } +feldera-ir = { version = "0.315.0", path = "crates/ir" } +feldera-adapterlib = { version = "0.315.0", path = "crates/adapterlib" } +feldera-samply = { version = "0.315.0", path = "crates/samply" } flate2 = "1.1.0" form_urlencoded = "1.2.0" fs_extra = "1.3.0" diff --git a/crates/adapterlib/src/catalog.rs b/crates/adapterlib/src/catalog.rs index 6260edc6c33..29f2c6d8954 100644 --- a/crates/adapterlib/src/catalog.rs +++ b/crates/adapterlib/src/catalog.rs @@ -21,7 +21,7 @@ use dbsp::dynamic::{ClonableTrait, DynData, DynVec, Erase, Factory}; use dbsp::operator::StagedBuffers; use dyn_clone::DynClone; use feldera_sqllib::Variant; -use feldera_types::format::csv::CsvParserConfig; +use feldera_types::format::csv::CsvFormatConfig; use feldera_types::format::json::JsonFlavor; use feldera_types::program_schema::{Relation, SqlIdentifier}; use feldera_types::serde_with_context::SqlSerdeConfig; @@ -43,7 +43,7 @@ pub enum RecordFormat { // raw encoding of this column only. This is particularly useful for // tables that store raw JSON or binary data to be parsed using SQL. Json(JsonFlavor), - Csv(CsvParserConfig), + Csv(CsvFormatConfig), Parquet(SqlSerdeConfig), #[cfg(feature = "with-avro")] Avro, diff --git a/crates/adapters/README.md b/crates/adapters/README.md index 16b6ad85841..1baba6270b9 100644 --- a/crates/adapters/README.md +++ b/crates/adapters/README.md @@ -51,3 +51,265 @@ $ cargo run --example server --features="with-kafka server" ``` Then open a web browser and open the following URL: `http://localhost:8080` + +## Connector configuration validation + +The SQL compiler validates connector configurations at compile time, reporting +warnings for unknown fields and invalid values before the pipeline ever runs. +This section explains how the validation works and what to do when you add or +change a Rust connector config struct. + +### Architecture + +The validation lives in the SQL compiler: + +``` +sql-to-dbsp-compiler/SQL-compiler/src/main/java/org/dbsp/sqlCompiler/compiler/frontend/connectors/ +├── ConnectorValidator.java — dispatch: routes format/transport names to POJO classes +├── ConfigReporter.java — emits positioned warnings via JSON Pointer lookup +├── IValidateConfig.java — interface: boolean validate(ConfigReporter) +└── config/ — one POJO class (and helper enums) per Rust config struct + ├── CsvParserConfig.java + ├── KafkaInputConfig.java + └── ... +``` + +`ConnectorValidator.validateFormatConfig()` and `validateTransportConfig()` are called +once per connector during SQL compilation. Each method looks at the transport/format +name and deserializes the JSON `config` block into the matching POJO class. Unknown +fields produce a warning pointing at the exact source location of the typo. After +deserialization the POJO's `validate()` method checks cross-field constraints. + +By default every unknown field is rejected. A POJO opts out of that check for specific +keys by declaring a `@JsonAnySetter` method: + +```java +/** Absorbs flattened extra keys so they are not rejected as unknown. */ +@JsonAnySetter +public void setExtra(String key, Object value) {} +``` + +Jackson calls the method for unknown keys instead of failing. Since the +method body is empty the key is silently absorbed. This is used for configs that have +a `#[serde(flatten)] HashMap` in Rust (e.g. Kafka, Delta Table, Iceberg), +where any string key is valid and should not be reported as an error. + +### Currently validated connectors + +#### Format (input) +| Name | POJO class | +|------------|-----------------------| +| `avro` | `AvroParserConfig` | +| `csv` | `CsvParserConfig` | +| `json` | `JsonParserConfig` | +| `parquet` | `ParquetParserConfig` | +| `raw` | `RawParserConfig` | + +#### Format (output) +| Name | POJO class | +|------------|------------------------| +| `avro` | `AvroEncoderConfig` | +| `csv` | `CsvEncoderConfig` | +| `json` | `JsonEncoderConfig` | +| `parquet` | `ParquetEncoderConfig` | + +#### Transport (input) +| Name | POJO class | +|----------------------|---------------------------| +| `clock` | `ClockConfig` | +| `delta_table_input` | `DeltaTableReaderConfig` | +| `file_input` | `FileInputConfig` | +| `iceberg_input` | `IcebergReaderConfig` | +| `kafka_input` | `KafkaInputConfig` | +| `nats_input` | `NatsInputConfig` | +| `postgres_input` | `PostgresReaderConfig` | +| `postgres_cdc_input` | `PostgresCdcReaderConfig` | +| `pub_sub_input` | `PubSubInputConfig` | +| `s3_input` | `S3InputConfig` | +| `url_input` | `UrlInputConfig` | + +#### Transport (output) +| Name | POJO class | +|----------------------|--------------------------| +| `delta_table_output` | `DeltaTableWriterConfig` | +| `dynamodb_output` | `DynamoDBWriterConfig` | +| `file_output` | `FileOutputConfig` | +| `http_output` | `HttpOutputConfig` | +| `kafka_output` | `KafkaOutputConfig` | +| `postgres_output` | `PostgresWriterConfig` | +| `redis_output` | `RedisOutputConfig` | + +Not validated by the SQL compiler: `datagen`, `nexmark`. + +Note: flattened string-map configs (e.g., `kafka_options`) cannot be +validated to detect typos. + +#### Data-carrying enum variants + +Some Rust enums have variants that carry data, e.g.: + +```rust +pub enum DeliverPolicy { + All, + Last, + ByStartSequence { start_sequence: u64 }, + ByStartTime { start_time: OffsetDateTime }, +} +``` + +Rust's default serde serialization is **externally tagged**: unit variants become +plain strings (`"All"`), struct variants become single-key objects +(`{"ByStartSequence": {"start_sequence": 5}}`). Jackson has no direct equivalent +without a custom deserializer. The recommended approach is to declare the field +as `JsonNode` in the Java POJO and skip content validation: + +```java +/** Required. Unit variants ("All", "Last", ...) and struct variants + * ({"ByStartSequence": {...}}) are accepted as-is. */ +@Nullable +@JsonProperty("deliver_policy") +public JsonNode deliverPolicy = null; +``` + +Presence can still be checked in `validate()` via +`deliverPolicy == null || deliverPolicy.isNull()`. + +### How to update the Java code + +The Java compiler validates connector configuration, but only gives +warnings. The ultimate real validation is done by the connector code +itself, in Rust, at runtime. + +#### When you add a field to an existing Rust struct + +Find the matching POJO class in `config/` and add the field following the +mapping rules below. + +#### When you add a new Rust config struct + +1. **Create a POJO class** in `config/` implementing `IValidateConfig`: + + ```java + package org.dbsp.sqlCompiler.compiler.frontend.connectors.config; + + import org.dbsp.sqlCompiler.compiler.frontend.connectors.ConfigReporter; + import org.dbsp.sqlCompiler.compiler.frontend.connectors.IValidateConfig; + import com.fasterxml.jackson.annotation.JsonProperty; + import javax.annotation.Nullable; + + public class MyTransportConfig implements IValidateConfig { + @JsonProperty("some_field") + public String someField = ""; + + @Override + public boolean validate(ConfigReporter reporter) { + boolean ok = true; + if (someField.isBlank()) { + reporter.warnPath("some_field", "Invalid configuration", + "required field \"some_field\" is missing or empty"); + ok = false; + } + return ok; + } + } + ``` + +2. **Wire it** in `ConnectorValidator.validateTransportConfig()` (or + `validateFormatConfig()` for formats): + + ```java + case "my_transport_input": + validateConfig(transportConfig, outerJson, configPointer, + outerStart, MyTransportConfig.class, reporter); + break; + ``` + +3. **Add tests** in `ConnectorTests.java` using the `tableConnectorTest()` / + `viewConnectorTest()` helpers. + +#### When you rename or remove a field + +Update the `@JsonProperty` annotation (or remove the field from the POJO). +If the old name should still be accepted as a fallback, add `@JsonAlias("old_name")`. + +#### When you add a new enum variant + +Add the variant to the Java enum with a matching `@JsonProperty`. If the Rust +variant has `#[serde(skip)]`, omit it from the Java enum entirely. + +### Rust-to-Java field mapping + +| Rust | Java | +|---------------------------------------------------|-------------------------------------------------------------------| +| `field: String` | `@JsonProperty("field") public String field = "";` | +| `field: Option` | `@Nullable @JsonProperty("field") public String field = null;` | +| `flag: bool` (default false) | `@JsonProperty("flag") public boolean flag = false;` | +| `#[serde(default = "f")] n: u32` | `@JsonProperty("n") public int n = /* value of f() */;` | +| `#[serde(rename = "x")] field: T` | `@JsonProperty("x") public T field = ...;` | +| `#[serde(alias = "old")] field: T` | `@JsonAlias("old") @JsonProperty("new") public T field;` | +| `#[serde(flatten)] extra: HashMap`| `@JsonAnySetter public void set(String k, Object v) {}` | +| `#[serde(flatten)] inner: InnerStruct` | Inline all fields of `InnerStruct` directly (see below) | +| enum variant `#[serde(rename = "x")] Foo` | `@JsonProperty("x") Foo,` | +| enum variant `#[serde(skip)] Bar` | Omit from the Java enum | + +#### Java field initializers + +Java field initializers in POJO classes serve **validation logic only** — they are +not used by the runtime, which has its own Rust defaults. Set an initializer only +when `validate()` depends on it: + +- `String field = ""` — sentinel for a required field; `validate()` checks + `field.isBlank()` to detect omission. +- `int n = k` — when `validate()` checks a range or relationship involving `n` and + the Rust default `k` is the boundary (e.g. `n == 0` is invalid but the Rust + default is `1`, so the initializer must be `1` for the check to fire correctly on + omission). + +Do **not** copy Rust defaults just for documentation — the Rust and Java values are +not kept in sync automatically, so a stale initializer is worse than no initializer. +For fields that `validate()` never inspects, omit the initializer or use the Java +natural default (`0`, `false`, `null`). + +#### Flattened structs + +Copy all fields from the nested Rust struct directly into the Java POJO class: + +```rust +// Rust +pub struct OuterConfig { + pub name: String, + #[serde(flatten)] + pub tls: TlsConfig, // has fields ssl_ca_pem, verify_hostname, ... +} +``` + +```java +// Java — inline the TLS fields directly +public class OuterConfig implements IValidateConfig { + @JsonProperty("name") public String name = ""; + // TLS fields (inlined from TlsConfig) + @Nullable @JsonProperty("ssl_ca_pem") public String sslCaPem = null; + @Nullable @JsonProperty("verify_hostname") public Boolean verifyHostname = null; + // ... +} +``` + +### Emitting warnings from `validate()` + +```java +// Point at a specific field's key in the source +reporter.warnPath("field_name", "Invalid configuration", "message"); + +// Point at the whole config block (when the error isn't tied to one field) +reporter.warn("Invalid configuration", "message"); +``` + +`warnPath` accepts a slash-separated JSON Pointer suffix relative to the config +block, e.g. `"field"` or `"nested/subfield"`. + +### Running the tests + +```bash +cd sql-to-dbsp-compiler/SQL-compiler +mvn test -Dtest=ConnectorTests +``` diff --git a/crates/adapters/src/format/csv.rs b/crates/adapters/src/format/csv.rs index dca7b8a9b0b..05c4b435648 100644 --- a/crates/adapters/src/format/csv.rs +++ b/crates/adapters/src/format/csv.rs @@ -12,7 +12,7 @@ use feldera_adapterlib::{ConnectorMetadata, catalog::SerCursorFlattened}; use feldera_sqllib::Variant; use feldera_types::{ config::ConnectorConfig, - format::csv::{CsvEncoderConfig, CsvParserConfig}, + format::csv::{CsvEncoderConfig, CsvFormatConfig}, }; use serde::Deserialize; use serde_json::json; @@ -31,6 +31,34 @@ use super::{InputBuffer, Splitter}; /// truncate it to `MAX_RECORD_LEN_IN_ERRMSG` bytes. static MAX_RECORD_LEN_IN_ERRMSG: usize = 4096; +fn csv_trim(t: &feldera_types::format::csv::CsvTrim) -> csv::Trim { + use feldera_types::format::csv::CsvTrim; + match t { + CsvTrim::None => csv::Trim::None, + CsvTrim::Headers => csv::Trim::Headers, + CsvTrim::Fields => csv::Trim::Fields, + CsvTrim::All => csv::Trim::All, + } +} + +/// Build a [`csv::ReaderBuilder`] fully configured from `config`. +/// +/// `has_headers` is always `false` — the adapter layer skips the header row +/// itself and must not pass it to the underlying CSV reader. +pub(crate) fn csv_reader_builder(config: &CsvFormatConfig) -> csv::ReaderBuilder { + let mut b = csv::ReaderBuilder::new(); + b.has_headers(false) + .delimiter(config.delimiter().0) + .quote(config.quote as u8) + .escape(config.escape.map(|c| c as u8)) + .double_quote(config.double_quote) + .quoting(config.quoting) + .comment(config.comment.map(|c| c as u8)) + .flexible(config.flexible) + .trim(csv_trim(&config.trim)); + b +} + /// CSV format parser. pub struct CsvInputFormat; @@ -47,7 +75,7 @@ impl InputFormat for CsvInputFormat { _endpoint_name: &str, _request: &HttpRequest, ) -> Result, ControllerError> { - Ok(Box::new(CsvParserConfig::default())) + Ok(Box::new(CsvFormatConfig::default())) } fn new_parser( @@ -57,7 +85,7 @@ impl InputFormat for CsvInputFormat { config: &serde_json::Value, ) -> Result, ControllerError> { let config = if config.is_null() { &json!({}) } else { config }; - let config = CsvParserConfig::deserialize(config).map_err(|e| { + let config = CsvFormatConfig::deserialize(config).map_err(|e| { ControllerError::parser_config_parse_error( endpoint_name, &e, @@ -65,11 +93,10 @@ impl InputFormat for CsvInputFormat { ) })?; - let headers = config.headers; let input_stream = input_stream .handle - .configure_deserializer(RecordFormat::Csv(config))?; - Ok(Box::new(CsvParser::new(input_stream, headers)) as Box) + .configure_deserializer(RecordFormat::Csv(config.clone()))?; + Ok(Box::new(CsvParser::new(input_stream, &config)) as Box) } } @@ -80,18 +107,37 @@ struct CsvParser { /// Whether the input starts with a header row. headers: bool, + /// Quote character (as a byte), used for record splitting. + quote: u8, + + /// Whether quoting is enabled. When `false`, every newline ends a record. + quoting: bool, + + /// Optional comment character. Lines whose first byte matches this value + /// are skipped and never passed to the deserializer. + comment: Option, + last_event_number: u64, } impl CsvParser { - fn new(input_stream: Box, headers: bool) -> Self { + fn new(input_stream: Box, config: &CsvFormatConfig) -> Self { Self { input_stream, last_event_number: 0, - headers, + headers: config.headers, + quote: config.quote as u8, + quoting: config.quoting, + comment: config.comment.map(|c| c as u8), } } + /// Returns `true` if `record` is a comment line (its first byte matches + /// the configured comment character). + fn is_comment(&self, record: &[u8]) -> bool { + self.comment.is_some_and(|c| record.first() == Some(&c)) + } + fn parse_record( &mut self, record: &[u8], @@ -118,26 +164,15 @@ impl CsvParser { /// Tries to split `buffer` into a CSV record followed by any remaining text. /// - /// A CSV record is usually one line that ends in a new-line, but it can - /// span multiple lines if new-lines are enclosed in double quotes. + /// A CSV record is usually one line that ends in a newline, but it can + /// span multiple lines if newlines appear inside a quoted field. fn split_record<'a>(&mut self, buffer: &'a [u8]) -> Option<(&'a [u8], &'a [u8])> { - // This uses the simple rule that a new-line ends a record if it is not - // in double quotes. The "standard" format for CSV escapes double - // quotes by doubling them (e.g. `"a""b""c"` unescapes to `a"b"c`), so - // that any even number of quotes followed by a new-line ends a reocrd, - // but any odd number followed by a new-line is a continuation of the - // field. This means that this rule properly handles escapes. - // - // If we allow the user to configure the CSV format used, we'll need to - // adjust this to match the configuration. let mut quoted = false; - for (offset, c) in buffer.iter().enumerate() { - match c { - b'"' => quoted = !quoted, - b'\n' if !quoted => { - return Some(buffer.split_at(offset + 1)); - } - _ => (), + for (offset, &c) in buffer.iter().enumerate() { + if self.quoting && c == self.quote { + quoted = !quoted; + } else if c == b'\n' && !quoted { + return Some(buffer.split_at(offset + 1)); } } None @@ -146,11 +181,23 @@ impl CsvParser { impl Parser for CsvParser { fn fork(&self) -> Box { - Box::new(Self::new(self.input_stream.fork(), self.headers)) + Box::new(Self { + input_stream: self.input_stream.fork(), + headers: self.headers, + quote: self.quote, + quoting: self.quoting, + comment: self.comment, + last_event_number: 0, + }) } fn splitter(&self) -> Box { - Box::new(CsvSplitter::new(self.headers)) + Box::new(CsvSplitter::new( + self.headers, + self.quote, + self.quoting, + self.comment, + )) } fn parse( @@ -162,11 +209,13 @@ impl Parser for CsvParser { let mut errors = Vec::new(); while let Some((record, rest)) = self.split_record(data) { - self.parse_record(record, &metadata, &mut errors); - self.last_event_number += 1; + if !self.is_comment(record) { + self.parse_record(record, &metadata, &mut errors); + self.last_event_number += 1; + } data = rest; } - if !data.is_empty() { + if !data.is_empty() && !self.is_comment(data) { self.parse_record(data, &metadata, &mut errors); } (self.input_stream.take_all(), errors) @@ -180,39 +229,61 @@ impl Parser for CsvParser { struct CsvSplitter { quoted: bool, headers: bool, + quote: u8, + quoting: bool, + /// Optional comment character: lines whose first byte matches this value + /// are skipped entirely. + comment: Option, + /// True when the next byte to be processed begins a new line. + at_line_start: bool, + /// True while consuming a comment line (until the terminating newline). + in_comment: bool, } impl CsvSplitter { - fn new(headers: bool) -> Self { + fn new(headers: bool, quote: u8, quoting: bool, comment: Option) -> Self { Self { quoted: false, headers, + quote, + quoting, + comment, + at_line_start: true, + in_comment: false, } } } impl Splitter for CsvSplitter { fn input(&mut self, data: &[u8]) -> Option { - // This uses the simple rule that a new-line ends a record if it is not - // in double quotes. The "standard" format for CSV escapes double - // quotes by doubling them (e.g. `"a""b""c"` unescapes to `a"b"c`), so - // that any even number of quotes followed by a new-line ends a reocrd, - // but any odd number followed by a new-line is a continuation of the - // field. This means that this rule properly handles escapes. - // - // If we allow the user to configure the CSV format used, we'll need to - // adjust this to match the configuration. - for (offset, c) in data.iter().enumerate() { - match c { - b'"' => self.quoted = !self.quoted, - b'\n' if !self.quoted => { - if self.headers { - self.headers = false; - } else { - return Some(offset + 1); - } + for (offset, &c) in data.iter().enumerate() { + // While inside a comment line, skip everything up to the newline. + if self.in_comment { + if c == b'\n' { + self.in_comment = false; + self.at_line_start = true; + } + continue; + } + + // At the start of a line: check whether this is a comment line. + if self.at_line_start { + self.at_line_start = false; + if self.comment == Some(c) { + self.in_comment = true; + continue; + } + } + + if self.quoting && c == self.quote { + self.quoted = !self.quoted; + } else if c == b'\n' && !self.quoted { + self.at_line_start = true; + if self.headers { + self.headers = false; + } else { + return Some(offset + 1); } - _ => (), } } None @@ -220,6 +291,8 @@ impl Splitter for CsvSplitter { fn clear(&mut self) { self.quoted = false; + self.in_comment = false; + self.at_line_start = true; } } @@ -331,13 +404,13 @@ impl Encoder for CsvEncoder { let mut cursor = if self.input_is_indexed { CursorWithPolarity::new(Box::new(SerCursorFlattened::new(batch.cursor( - RecordFormat::Csv(CsvParserConfig { + RecordFormat::Csv(CsvFormatConfig { delimiter: self.config.delimiter, ..Default::default() }), )?))) } else { - CursorWithPolarity::new(batch.cursor(RecordFormat::Csv(CsvParserConfig { + CursorWithPolarity::new(batch.cursor(RecordFormat::Csv(CsvFormatConfig { delimiter: self.config.delimiter, ..Default::default() }))?) @@ -407,6 +480,10 @@ mod test { use feldera_types::deserialize_table_record; use feldera_types::serde_with_context::{DeserializeWithContext, SqlSerdeConfig}; + use super::CsvSplitter; + use crate::format::Splitter; + use feldera_types::format::csv::{CsvFormatConfig, CsvTrim}; + #[derive(Debug, Eq, PartialEq)] #[allow(non_snake_case)] struct CaseSensitive { @@ -453,4 +530,310 @@ true,bar,buzz"#; ) ); } + + // ---- CsvSplitter option tests ---- + + // `headers = true`: the first newline ends the header row and is consumed + // without producing a record; subsequent newlines are record boundaries. + #[test] + fn splitter_headers() { + let mut s = CsvSplitter::new(true, b'"', true, None); + // Header row: consumed silently, no record returned. + assert_eq!(s.input(b"col1,col2\n"), None); + // First data record. + assert_eq!(s.input(b"foo,bar\n"), Some(8)); + } + + // `headers = true` with a comment before the header line. + #[test] + fn splitter_headers_with_leading_comment() { + let mut s = CsvSplitter::new(true, b'"', true, Some(b'#')); + // Comment before the header: skipped. + assert_eq!(s.input(b"# preamble\n"), None); + // Header row: consumed silently. + assert_eq!(s.input(b"col1,col2\n"), None); + // First data record. + assert_eq!(s.input(b"foo,bar\n"), Some(8)); + } + + // Default quoting: a newline inside a double-quoted span does not split. + #[test] + fn splitter_default_no_split_inside_quotes() { + let mut s = CsvSplitter::new(false, b'"', true, None); + // Opening quote is present; newline is inside the quoted span. + assert_eq!(s.input(b"\"foo\n"), None); + // Closing quote followed by newline terminates the record. + assert_eq!(s.input(b"bar\"\n"), Some(5)); + } + + // Custom quote character: single-quote acts as the quoting delimiter. + #[test] + fn splitter_custom_quote() { + let mut s = CsvSplitter::new(false, b'\'', true, None); + // Newline inside single-quoted span — no split. + assert_eq!(s.input(b"foo,'bar\n"), None); + // Closing single-quote then newline — split. + assert_eq!(s.input(b"baz'\n"), Some(5)); + } + + // `quoting = false`: every newline ends a record even when inside quotes. + #[test] + fn splitter_quoting_disabled() { + let mut s = CsvSplitter::new(false, b'"', false, None); + // Newline inside "quotes" splits immediately because quoting is off. + assert_eq!(s.input(b"\"foo\n"), Some(5)); + } + + // `comment`: comment lines are skipped; the next real record is returned. + #[test] + fn splitter_comment() { + let mut s = CsvSplitter::new(false, b'"', true, Some(b'#')); + // A comment line is consumed without being returned as a record. + assert_eq!(s.input(b"# comment\n"), None); + // The following data record is returned normally. + assert_eq!(s.input(b"foo,bar\n"), Some(8)); + } + + // Comment line interleaved with data records in a single chunk. + #[test] + fn splitter_comment_interleaved() { + let mut s = CsvSplitter::new(false, b'"', true, Some(b'#')); + // First record returned; comment + second record still pending. + assert_eq!(s.input(b"a,b\n# skip\nc,d\n"), Some(4)); + // After the first record is consumed, the comment is skipped and the + // second record is returned. + assert_eq!(s.input(b"# skip\nc,d\n"), Some(11)); + } + + // ---- End-to-end tests via mock_parser_pipeline ---- + // + // These tests push bytes through a real CsvParser (the same code path that + // a live input connector uses) and verify the deserialized records. + + use crate::FormatConfig; + use crate::format::InputBuffer; + use crate::test::mock_parser_pipeline; + use feldera_adapterlib::format::Parser; + use feldera_types::deserialize_without_context; + use feldera_types::program_schema::Relation; + use serde::{Deserialize, Serialize}; + use std::borrow::Cow; + + /// Two-string record used by the end-to-end tests below. + #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] + struct TwoStrings { + first: String, + second: String, + } + + deserialize_without_context!(TwoStrings); + + /// Push `data` through a fully-configured `CsvParser` and return the + /// inserted `TwoStrings` records. + fn e2e_parse_csv(config: CsvFormatConfig, data: &[u8]) -> Vec { + let format_config = FormatConfig { + name: Cow::from("csv"), + config: serde_json::to_value(config).unwrap(), + }; + let (_, mut parser, input_handle) = + mock_parser_pipeline::(&Relation::empty(), &format_config) + .unwrap(); + let (buf, errors) = parser.parse(data, None); + assert!(errors.is_empty(), "unexpected parse errors: {errors:?}"); + if let Some(mut buf) = buf { + buf.flush(); + } + input_handle + .state() + .flushed + .iter() + .map(|upd| upd.unwrap_insert().clone()) + .collect() + } + + // `quote`: single-quote wraps a field containing a comma. + #[test] + fn csv_e2e_quote() { + let rows = e2e_parse_csv( + CsvFormatConfig { + quote: '\'', + ..Default::default() + }, + b"foo,'bar,baz'\nqux,'quux,corge'\n", + ); + assert_eq!(rows.len(), 2); + assert_eq!( + rows[0], + TwoStrings { + first: "foo".into(), + second: "bar,baz".into() + } + ); + assert_eq!( + rows[1], + TwoStrings { + first: "qux".into(), + second: "quux,corge".into() + } + ); + } + + // `escape` + `double_quote = false`: backslash escapes a literal quote. + #[test] + fn csv_e2e_escape() { + let rows = e2e_parse_csv( + CsvFormatConfig { + escape: Some('\\'), + double_quote: false, + ..Default::default() + }, + // "foo\"bar" → foo"bar + b"\"foo\\\"bar\",baz\n", + ); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].first, "foo\"bar"); + assert_eq!(rows[0].second, "baz"); + } + + // `double_quote = true` (default): `""` inside a quoted field is one `"`. + #[test] + fn csv_e2e_double_quote() { + let rows = e2e_parse_csv( + CsvFormatConfig::default(), + // "foo""bar" → foo"bar + b"\"foo\"\"bar\",baz\n", + ); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].first, "foo\"bar"); + assert_eq!(rows[0].second, "baz"); + } + + // `double_quote = false`: `""` is not collapsed; the first `"` ends the + // quoted field, and the remainder of the token is appended unquoted. + // (Matches csv-core's `quote_escapes_no_double` test: `"a""b"` → `a"b"`.) + #[test] + fn csv_e2e_double_quote_disabled() { + let rows = e2e_parse_csv( + CsvFormatConfig { + double_quote: false, + ..Default::default() + }, + b"\"foo\"\"bar\",baz\n", + ); + assert_eq!(rows.len(), 1); + assert_ne!(rows[0].first, "foo\"bar"); // not collapsed + assert_eq!(rows[0].second, "baz"); + } + + // `trim = None` (default): leading and trailing whitespace is preserved. + #[test] + fn csv_e2e_trim_none() { + let rows = e2e_parse_csv(CsvFormatConfig::default(), b" foo , bar \n"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].first, " foo "); + assert_eq!(rows[0].second, " bar "); + } + + // `quoting = false`: quote characters are treated as ordinary bytes. + #[test] + fn csv_e2e_quoting_disabled() { + let rows = e2e_parse_csv( + CsvFormatConfig { + quoting: false, + ..Default::default() + }, + // The outer " chars are kept verbatim in the field value. + b"\"hello\",world\n", + ); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].first, "\"hello\""); + assert_eq!(rows[0].second, "world"); + } + + // `comment`: lines starting with `#` are skipped. + #[test] + fn csv_e2e_comment() { + let rows = e2e_parse_csv( + CsvFormatConfig { + comment: Some('#'), + ..Default::default() + }, + b"# opening comment\nfoo,bar\n# mid comment\nbaz,qux\n", + ); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].first, "foo"); + assert_eq!(rows[1].first, "baz"); + } + + // `flexible = true` (default): a record with extra fields is accepted; + // only the first two are used for deserialization. + #[test] + fn csv_e2e_flexible_true() { + let rows = e2e_parse_csv( + CsvFormatConfig::default(), // flexible=true + b"foo,bar\nqux,quux,extra\n", + ); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].first, "foo"); + assert_eq!(rows[1].first, "qux"); + assert_eq!(rows[1].second, "quux"); + } + + // `flexible = false`: a record whose field count differs from the first + // record's count produces a parse error. + #[test] + fn csv_e2e_flexible_false() { + let format_config = FormatConfig { + name: Cow::from("csv"), + config: serde_json::to_value(CsvFormatConfig { + flexible: false, + ..Default::default() + }) + .unwrap(), + }; + let (_, mut parser, _) = + mock_parser_pipeline::(&Relation::empty(), &format_config) + .unwrap(); + + // Suppress the default panic-on-error so we can inspect the error list. + parser.on_error(Some(Box::new(|_fatal, _err| {}))); + + // Record 1: 2 fields (sets the expected count). + // Record 2: 3 fields (different count → error with flexible=false). + let (_buf, errors) = parser.parse(b"foo,bar\nqux,quux,extra\n", None); + assert!( + !errors.is_empty(), + "expected a parse error for flexible=false with mismatched field counts" + ); + } + + // `trim = Fields`: leading/trailing whitespace is stripped from field values. + #[test] + fn csv_e2e_trim_fields() { + let rows = e2e_parse_csv( + CsvFormatConfig { + trim: CsvTrim::Fields, + ..Default::default() + }, + b" foo , bar \n", + ); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].first, "foo"); + assert_eq!(rows[0].second, "bar"); + } + + // `headers = true`: the first line is a header row and is skipped. + #[test] + fn csv_e2e_headers() { + let rows = e2e_parse_csv( + CsvFormatConfig { + headers: true, + ..Default::default() + }, + b"first,second\nfoo,bar\n", + ); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].first, "foo"); + assert_eq!(rows[0].second, "bar"); + } } diff --git a/crates/adapters/src/static_compile/deinput.rs b/crates/adapters/src/static_compile/deinput.rs index a76109a99e8..03f6f7f8553 100644 --- a/crates/adapters/src/static_compile/deinput.rs +++ b/crates/adapters/src/static_compile/deinput.rs @@ -2,7 +2,7 @@ use crate::catalog::AvroStream; #[cfg(feature = "with-avro")] use crate::format::avro::from_avro_value; -use crate::format::csv::deserializer::ByteRecordDeserializer; +use crate::format::csv::{csv_reader_builder, deserializer::ByteRecordDeserializer}; use crate::format::raw::{RawDeserializer, raw_serde_config}; use crate::{ ControllerError, DeCollectionHandle, @@ -25,7 +25,7 @@ use erased_serde::Deserializer as ErasedDeserializer; use feldera_adapterlib::catalog::AvroSchemaRefs; use feldera_adapterlib::format::{BufferSize, flatten_nested}; use feldera_sqllib::Variant; -use feldera_types::format::csv::CsvParserConfig; +use feldera_types::format::csv::CsvFormatConfig; use feldera_types::serde_with_context::{DeserializeWithContext, SqlSerdeConfig}; use serde_arrow::Deserializer as ArrowDeserializer; use serde_json::de::SliceRead; @@ -94,16 +94,12 @@ pub struct CsvDeserializerFromBytes { config: SqlSerdeConfig, } -impl DeserializerFromBytes<(SqlSerdeConfig, CsvParserConfig)> for CsvDeserializerFromBytes { - fn create((serde_config, csv_config): (SqlSerdeConfig, CsvParserConfig)) -> Self { +impl DeserializerFromBytes<(SqlSerdeConfig, CsvFormatConfig)> for CsvDeserializerFromBytes { + fn create((serde_config, csv_config): (SqlSerdeConfig, CsvFormatConfig)) -> Self { CsvDeserializerFromBytes { - reader: csv::ReaderBuilder::new() - // We skip the headers ourselves, without passing them to the - // reader, so we unconditionally turn off headers in the reader. - .has_headers(false) - .flexible(true) - .delimiter(csv_config.delimiter().0) - .from_reader(VecDeque::new()), + // `csv_reader_builder` sets `has_headers(false)` — the adapter + // layer handles header-row skipping itself. + reader: csv_reader_builder(&csv_config).from_reader(VecDeque::new()), record: csv::ByteRecord::new(), config: serde_config, } diff --git a/crates/adapters/src/static_compile/seroutput.rs b/crates/adapters/src/static_compile/seroutput.rs index 574093ba546..e9afa7de941 100644 --- a/crates/adapters/src/static_compile/seroutput.rs +++ b/crates/adapters/src/static_compile/seroutput.rs @@ -41,7 +41,7 @@ use feldera_types::serde_with_context::serialize::{ SerializeFieldsWithContextWrapper, SerializeWithContextWrapper, }; use feldera_types::{ - format::csv::CsvParserConfig, + format::csv::CsvFormatConfig, serde_with_context::{SerializeWithContext, SqlSerdeConfig}, }; use rand::thread_rng; @@ -197,7 +197,7 @@ struct CsvSerializer { } impl CsvSerializer { - fn create(config: CsvParserConfig) -> Self { + fn create(config: CsvFormatConfig) -> Self { Self { writer: CsvWriterBuilder::new() .has_headers(false) diff --git a/crates/dbsp/Cargo.toml b/crates/dbsp/Cargo.toml index d820af34634..8af54c76821 100644 --- a/crates/dbsp/Cargo.toml +++ b/crates/dbsp/Cargo.toml @@ -217,6 +217,10 @@ path = "examples/tutorial/tutorial10.rs" name = "tutorial11" path = "examples/tutorial/tutorial11.rs" +[[example]] +name = "tutorial12" +path = "examples/tutorial/tutorial12/tutorial12.rs" + [[example]] name = "coord" path = "examples/dist/coord.rs" diff --git a/crates/dbsp/examples/tutorial/tutorial10.rs b/crates/dbsp/examples/tutorial/tutorial10.rs index 005bdffb659..38090af4e62 100644 --- a/crates/dbsp/examples/tutorial/tutorial10.rs +++ b/crates/dbsp/examples/tutorial/tutorial10.rs @@ -34,7 +34,7 @@ fn main() -> Result<()> { // | | // ------------------3------------------ // zset_set! { Tup3(1,2,1), Tup3(4, 0, 3)} - ] as [_; STEPS]) + ] as [OrdZSet>; STEPS]) .into_iter(); let edges = root_circuit.add_source(Generator::new(move || edges_data.next().unwrap())); @@ -99,7 +99,7 @@ fn main() -> Result<()> { // This does not matter, as the computation does not terminate // anymore due to the cycle. // zset! {}, - ] as [_; STEPS]) + ] as [OrdZSet>; STEPS]) .into_iter(); for i in 0..STEPS { diff --git a/crates/dbsp/examples/tutorial/tutorial11.rs b/crates/dbsp/examples/tutorial/tutorial11.rs index 8b3f01368b4..09580d4b1ea 100644 --- a/crates/dbsp/examples/tutorial/tutorial11.rs +++ b/crates/dbsp/examples/tutorial/tutorial11.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use dbsp::OrdZSet; use dbsp::typed_batch::IndexedZSetReader; use dbsp::{ Circuit, NestedCircuit, OrdIndexedZSet, Runtime, Stream, indexed_zset, @@ -32,7 +33,7 @@ fn main() -> Result<()> { // | | // ------------------3------------------ zset_set! { Tup3(4, 0, 3)} - ] as [_; STEPS]) + ] as [OrdZSet>; STEPS]) .into_iter(); let edges = root_circuit.add_source(Generator::new(move || edges_data.next().unwrap())); diff --git a/crates/dbsp/examples/tutorial/tutorial12/README.md b/crates/dbsp/examples/tutorial/tutorial12/README.md new file mode 100644 index 00000000000..1bed451b49b --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/README.md @@ -0,0 +1,166 @@ +# Tutorial 12 + +This tutorial deals with static program analysis. Specifically, we aim to +find out two things about a program in some fictional language using +polymorphism: + +1. To which class type a variable can point to throughout the execution of the + program (later called `VarPointsTo`). +2. Which exact method is invoked at every call site in the presence of + polymorphism (later called `CallGraph`). + +Look at the following program in our fictional language. We have a super class +`Animal` and two subclasses `Cat` and `Dog`. There is also the `Greeter` class +whose `greet()` method expects a value of type `Animal` upon which it calls +the `speak()` method (this is the polymorphic part): + +``` +class Greeter { void greet(Animal x) { x.speak(); } } // s3 +class Animal { void speak() {} } +class Dog extends Animal { void speak() {...} } +class Cat extends Animal { void speak() {...} } + +void main() { + Greeter g = new Greeter(); + Dog d = new Dog(); + g.greet(d); // s1 + Cat c = new Cat(); + Animal ac = c; + g.greet(ac); // s2 +} +``` + +For instance, we may want to find out which types the variable `x` in +`Greeter.greet` actually assumes (`VarPointsTo`) and to which (concrete) methods +the call site `x.speak()` resolves to (`CallGraph`). + +To succinctly express `VarPointsTo` and `CallGraph` we resort to Datalog. +A fictional driver program preparing the static analysis could generate the +following base facts about the program being analyzed: + +1. `Alloc(var, obj)` whenever `var = new Obj()` (new allocation). +2. `Assign(dst, src)` whenever `dst = src` (new assignment). +3. `VirtualCall(site, recv, sig)` whenever `recv.sig(...)` at call site `site` + (imagine the driver program keeping track of all call sites as indicated + above in the comments: `s1`, `s2`, and `s3`). +4. `HeapType(obj, ty)` stores the runtime type `ty` of an allocated + object `obj`. +5. `Dispatch(ty, sig, meth)` stores for each type `ty` and each + signature `sig` its method `meth`. +6. `ActualArg(site, arg)` stores for each call site `site` the given + argument `arg`. +7. `FormalParam(meth, param)` stores for each method `meth` its + parameter `param`. + +The properties of our interest, `VarPointsTo` and `CallGraph`, can be expressed +with Datalog as follows: + +```dl +// The base case is an allocation. +VarPointsTo(V, Obj) :- Alloc(V, Obj). +// The self-recursive case is an alias. +VarPointsTo(Dst, Obj) :- + Assign(Dst, Src), + VarPointsTo(Src, Obj). +// This resolves to which object a parameter points to. Notice that this is +// self-recursive and mutually recursive with `CallGraph`! +VarPointsTo(Param, Obj) :- + CallGraph(Site, Meth), + ActualArg(Site, Arg), + FormalParam(Meth, Param), + VarPointsTo(Arg, Obj). + +// This resolves to which concrete method each call site resolves to. +// Notice that this is mutually recursive with `VarPointsTo`. +CallGraph(Site, Meth) :- + VirtualCall(Site, Recv, Sig), + VarPointsTo(Recv, Obj), + HeapType(Obj, Ty), + Dispatch(Ty, Sig, Meth). +``` + +[Souffle](https://souffle-lang.github.io) is a Datalog engine that can actually +run this query. The folder `data_step_1` contains the base facts to the +fictional program shown above. This is how you can run it: + +```sh +souffle program_analysis.dl --fact-dir data_step_1 +``` + +Notice how the analysis tells us, among other things, that the variable +`x` can point to both an instance of `Dog` and `Cat` (as part of +`VarPointsTo`) and that call site `s3` (corresponding to `x.speak()`) +uses both `Dog.speak` and `Cat.speak` (see `CallGraph`). + +Now the fictional program is modified, say, by its programmer, to introduce +another subclass called `Mouse` of `Animal`. Also `main()` is extended by two +lines to create a mouse and have it be greeted by the greeter: + +``` +class Greeter { void greet(Animal x) { x.speak(); } } // s3 +class Animal { void speak() {} } +class Dog extends Animal { void speak() {...} } +class Cat extends Animal { void speak() {...} } +class Mouse extends Animal { void speak() {...} } + +void main() { + Greeter g = new Greeter(); // oG + Dog d = new Dog(); // oDog + g.greet(d); // s1 + Cat c = new Cat(); // oCat + Animal ac = c; // alias (exercises the Assign rule) + g.greet(ac); // s2 + Mouse m = new Mouse() // oMouse + g.greet(m); // s4 +} +``` + +The new base facts _and_ the old ones are stored in the `data_step_2` folder. +Again you can run this via: + +```sh +souffle program_analysis.dl --fact-dir data_step_2 +``` + +This now tells us that both `x` and `m` can point to variables of type `Mouse` +and that call site `s3` (`x.speak()`) is also invoking `Mouse.speak`, +plus that call site `s4` is only using `Greeter.greet`. + +Finally, we face a deletion of code. The alias `ac` is gone and, consequently, +its usage afterwards is removed, too: + +``` +class Greeter { void greet(Animal x) { x.speak(); } } // s3 +class Animal { void speak() {} } +class Dog extends Animal { void speak() {...} } +class Cat extends Animal { void speak() {...} } +class Mouse extends Animal { void speak() {...} } + +void main() { + Greeter g = new Greeter(); // oG + Dog d = new Dog(); // oDog + g.greet(d); // s1 + Cat c = new Cat(); // oCat + Mouse m = new Mouse() // oMouse + g.greet(m); // s4 +} +``` + +As you might be guessing already, some previously derived facts are not part +of the output anymore. You can verify which ones by running: + +```sh +souffle program_analysis.dl --fact-dir data_step_3 +``` + +While each computation caused by a code change starts from scratch every time +if we rely on Souffle, we now turn towards incremental computations with DBSP. +The file `tutorial12.rs` contains the analogous DBSP circuit for the computation +of `VarPointsTo` and `CallGraph`. Note that the translation from Datalog to a +DBSP circuit has been done manually here. To observe how DBSP computes the +output _changes_ in response to the program modifications discussed above +(as opposed to a full recomputation of the state with Souffle), run: + +```sh +cargo run --example tutorial12 +``` diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_1/ActualArg.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/ActualArg.facts new file mode 100644 index 00000000000..1bfd659c257 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/ActualArg.facts @@ -0,0 +1,3 @@ +site,arg +s1,d +s2,ac diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_1/Alloc.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/Alloc.facts new file mode 100644 index 00000000000..cadadee1331 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/Alloc.facts @@ -0,0 +1,4 @@ +var,obj +g,oG +d,oDog +c,oCat diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_1/Assign.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/Assign.facts new file mode 100644 index 00000000000..b5d2643f2b3 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/Assign.facts @@ -0,0 +1,2 @@ +dst,src +ac,c diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_1/Dispatch.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/Dispatch.facts new file mode 100644 index 00000000000..45fb55b8076 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/Dispatch.facts @@ -0,0 +1,4 @@ +site,recv,sig +Greeter,greet,Greeter.greet +Dog,speak,Dog.speak +Cat,speak,Cat.speak diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_1/FormalParam.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/FormalParam.facts new file mode 100644 index 00000000000..96030e0c01e --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/FormalParam.facts @@ -0,0 +1,2 @@ +meth,param +Greeter.greet,x diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_1/HeapType.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/HeapType.facts new file mode 100644 index 00000000000..a376ee59bb6 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/HeapType.facts @@ -0,0 +1,4 @@ +obj,ty +oG,Greeter +oDog,Dog +oCat,Cat diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_1/VirtualCall.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/VirtualCall.facts new file mode 100644 index 00000000000..a3ee6486edf --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_1/VirtualCall.facts @@ -0,0 +1,4 @@ +site,recv,sig +s1,g,greet +s2,g,greet +s3,x,speak diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_2/ActualArg.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/ActualArg.facts new file mode 100644 index 00000000000..29622479d0f --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/ActualArg.facts @@ -0,0 +1,4 @@ +site,arg +s1,d +s2,ac +s4,m diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_2/Alloc.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/Alloc.facts new file mode 100644 index 00000000000..ca517628b6d --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/Alloc.facts @@ -0,0 +1,5 @@ +var,obj +g,oG +d,oDog +c,oCat +m,oMouse diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_2/Assign.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/Assign.facts new file mode 100644 index 00000000000..b5d2643f2b3 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/Assign.facts @@ -0,0 +1,2 @@ +dst,src +ac,c diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_2/Dispatch.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/Dispatch.facts new file mode 100644 index 00000000000..111d4a44d1b --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/Dispatch.facts @@ -0,0 +1,5 @@ +site,recv,sig +Greeter,greet,Greeter.greet +Dog,speak,Dog.speak +Cat,speak,Cat.speak +Mouse,speak,Mouse.speak diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_2/FormalParam.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/FormalParam.facts new file mode 100644 index 00000000000..96030e0c01e --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/FormalParam.facts @@ -0,0 +1,2 @@ +meth,param +Greeter.greet,x diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_2/HeapType.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/HeapType.facts new file mode 100644 index 00000000000..7a9a123bb34 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/HeapType.facts @@ -0,0 +1,5 @@ +obj,ty +oG,Greeter +oDog,Dog +oCat,Cat +oMouse,Mouse diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_2/VirtualCall.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/VirtualCall.facts new file mode 100644 index 00000000000..6f39b499aa4 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_2/VirtualCall.facts @@ -0,0 +1,5 @@ +site,recv,sig +s1,g,greet +s2,g,greet +s3,x,speak +s4,g,greet diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_3/ActualArg.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/ActualArg.facts new file mode 100644 index 00000000000..3c02e13c0ae --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/ActualArg.facts @@ -0,0 +1,3 @@ +site,arg +s1,d +s4,m diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_3/Alloc.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/Alloc.facts new file mode 100644 index 00000000000..ca517628b6d --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/Alloc.facts @@ -0,0 +1,5 @@ +var,obj +g,oG +d,oDog +c,oCat +m,oMouse diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_3/Assign.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/Assign.facts new file mode 100644 index 00000000000..e08ed75eac6 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/Assign.facts @@ -0,0 +1 @@ +dst,src diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_3/Dispatch.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/Dispatch.facts new file mode 100644 index 00000000000..111d4a44d1b --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/Dispatch.facts @@ -0,0 +1,5 @@ +site,recv,sig +Greeter,greet,Greeter.greet +Dog,speak,Dog.speak +Cat,speak,Cat.speak +Mouse,speak,Mouse.speak diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_3/FormalParam.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/FormalParam.facts new file mode 100644 index 00000000000..96030e0c01e --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/FormalParam.facts @@ -0,0 +1,2 @@ +meth,param +Greeter.greet,x diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_3/HeapType.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/HeapType.facts new file mode 100644 index 00000000000..7a9a123bb34 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/HeapType.facts @@ -0,0 +1,5 @@ +obj,ty +oG,Greeter +oDog,Dog +oCat,Cat +oMouse,Mouse diff --git a/crates/dbsp/examples/tutorial/tutorial12/data_step_3/VirtualCall.facts b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/VirtualCall.facts new file mode 100644 index 00000000000..28c2261e169 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/data_step_3/VirtualCall.facts @@ -0,0 +1,4 @@ +site,recv,sig +s1,g,greet +s3,x,speak +s4,g,greet diff --git a/crates/dbsp/examples/tutorial/tutorial12/program_analysis.dl b/crates/dbsp/examples/tutorial/tutorial12/program_analysis.dl new file mode 100644 index 00000000000..e601ffea506 --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/program_analysis.dl @@ -0,0 +1,53 @@ +//============================================================================ +// Run: souffle program_analysis.dl --fact-dir data_step_{1,2,3} +//============================================================================ + +//--------------------------- Input relations (EDB) -------------------------- +.decl Alloc(var:symbol, obj:symbol) // var = new ... (obj = alloc site) +.input Alloc(delimiter=",", headers=true) +.decl Assign(dst:symbol, src:symbol) // dst = src +.input Assign(delimiter=",", headers=true) +.decl VirtualCall(site:symbol, recv:symbol, sig:symbol) // recv.sig(...) at `site` +.input VirtualCall(delimiter=",", headers=true) +.decl HeapType(obj:symbol, ty:symbol) // runtime type of an allocated object +.input HeapType(delimiter=",", headers=true) +.decl Dispatch(ty:symbol, sig:symbol, meth:symbol) // type + signature -> method +.input Dispatch(delimiter=",", headers=true) +.decl ActualArg(site:symbol, arg:symbol) // argument variable passed at a call +.input ActualArg(delimiter=",", headers=true) +.decl FormalParam(meth:symbol, param:symbol) // a method's parameter variable +.input FormalParam(delimiter=",", headers=true) + +//------------------------- Derived relations (IDB) -------------------------- +.decl VarPointsTo(var:symbol, obj:symbol) +.decl CallGraph(site:symbol, meth:symbol) +.output VarPointsTo(IO=stdout) +.output CallGraph(IO=stdout) + +//--------------------------------- Rules ------------------------------------ + +// A variable points to whatever it was directly allocated. +VarPointsTo(V, Obj) :- Alloc(V, Obj). + +// Copies propagate points-to facts (transitively). +VarPointsTo(Dst, Obj) :- + Assign(Dst, Src), + VarPointsTo(Src, Obj). + +// Once a call edge exists, the actual arguments flow into the callee's +// formal parameters. +// --> this rule READS CallGraph and WRITES VarPointsTo, closing the loop. +VarPointsTo(Param, Obj) :- + CallGraph(Site, Meth), + ActualArg(Site, Arg), + FormalParam(Meth, Param), + VarPointsTo(Arg, Obj). + +// Resolve a virtual call: look at what the receiver points to, take that +// object's type, and dispatch the signature on that type. +// --> this rule READS VarPointsTo. +CallGraph(Site, Meth) :- + VirtualCall(Site, Recv, Sig), + VarPointsTo(Recv, Obj), + HeapType(Obj, Ty), + Dispatch(Ty, Sig, Meth). diff --git a/crates/dbsp/examples/tutorial/tutorial12/tutorial12.rs b/crates/dbsp/examples/tutorial/tutorial12/tutorial12.rs new file mode 100644 index 00000000000..696966a542d --- /dev/null +++ b/crates/dbsp/examples/tutorial/tutorial12/tutorial12.rs @@ -0,0 +1,406 @@ +use anyhow::Result; +use dbsp::typed_batch::IndexedZSetReader; +use dbsp::{ + NestedCircuit, Runtime, Stream, + utils::{Tup2, Tup3}, +}; +use dbsp::{OrdZSet, ZWeight, zset}; + +// Some helper types. +type WeightedValue = Tup2; +type String2 = Tup2; +type String3 = Tup3; + +// And some helper functions. +fn owned_string2(((s1, s2), weight): ((&str, &str), ZWeight)) -> WeightedValue { + Tup2(Tup2(s1.to_owned(), s2.to_owned()), weight) +} +fn owned_string3(((s1, s2, s3), weight): ((&str, &str, &str), ZWeight)) -> WeightedValue { + Tup2(Tup3(s1.to_owned(), s2.to_owned(), s3.to_owned()), weight) +} + +fn main() -> Result<()> { + const STEPS: usize = 3; + + let threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + + let ( + mut circuit, + ( + ( + alloc_input, + assign_input, + virtual_call_input, + heap_type_input, + dispatch_input, + actual_arg_input, + formal_param_input, + ), + (var_points_to_output, call_graph_output), + ), + ) = Runtime::init_circuit(threads, move |root_circuit| { + let (alloc, alloc_input) = root_circuit.add_input_zset::(); + let (assign, assign_input) = root_circuit.add_input_zset::(); + let (virtual_call, virtual_call_input) = root_circuit.add_input_zset::(); + let (heap_type, heap_type_input) = root_circuit.add_input_zset::(); + let (dispatch, dispatch_input) = root_circuit.add_input_zset::(); + let (actual_arg, actual_arg_input) = root_circuit.add_input_zset::(); + let (formal_param, formal_param_input) = root_circuit.add_input_zset::(); + + let (var_points_to, call_graph) = root_circuit.recursive( + // Note that recursive has an implicit distinct (see its Docs)! + |child_circuit, + // var_points_to and call_graph are the recursive computations we are + // interested in. + (var_points_to, call_graph): ( + Stream>, + Stream>, + )| { + // Import streams from the parent circuit into the child circuit. + let alloc = alloc.delta0(child_circuit); + let assign = assign.delta0(child_circuit); + let virtual_call = virtual_call.delta0(child_circuit); + let heap_type = heap_type.delta0(child_circuit); + let dispatch = dispatch.delta0(child_circuit); + let actual_arg = actual_arg.delta0(child_circuit); + let formal_param = formal_param.delta0(child_circuit); + + // call_graph_next computes this Datalog query: + // CallGraph(Site, Meth) :- + // VirtualCall(Site, Recv, Sig), + // VarPointsTo(Recv, Obj), + // HeapType(Obj, Ty), + // Dispatch(Ty, Sig, Meth). + let call_graph_next = virtual_call + .map_index(|Tup3(site, recv, sig)| { + (recv.clone(), Tup3(site.clone(), recv.clone(), sig.clone())) + }) + .join_index( + // 1. virtual_call JOIN var_points_to ON recv + // Mutual recursion: call_graph uses var_points_to + &var_points_to.map_index(|Tup2(recv, obj)| { + (recv.clone(), Tup2(recv.clone(), obj.clone())) + }), + |_recv, Tup3(site, _, sig), Tup2(_, obj)| { + Some((obj.clone(), Tup3(site.clone(), sig.clone(), obj.clone()))) + }, + ) + .join_index( + // 2. ... JOIN heap_type ON obj + &heap_type.map_index(|Tup2(obj, ty)| { + (obj.clone(), Tup2(obj.clone(), ty.clone())) + }), + |_obj, Tup3(site, sig, _), Tup2(_, ty)| { + Some(( + Tup2(ty.clone(), sig.clone()), + Tup2(site.clone(), ty.clone()), + )) + }, + ) + .join_index( + // 3. ... JOIN dispatch ON ty and sig + &dispatch.map_index(|Tup3(ty, sig, meth)| { + (Tup2(ty.clone(), sig.clone()), meth.clone()) + }), + |_, Tup2(site, _), meth| { + Some(( + Tup2(site.clone(), meth.clone()), + Tup2(site.clone(), meth.clone()), + )) + }, + ); + + // var_points_to_next computes this Datalog query: + // VarPointsTo(V, Obj) :- Alloc(V, Obj). + // VarPointsTo(Dst, Obj) :- + // Assign(Dst, Src), + // VarPointsTo(Src, Obj). + // VarPointsTo(Param, Obj) :- + // CallGraph(Site, Meth), + // ActualArg(Site, Arg), + // FormalParam(Meth, Param), + // VarPointsTo(Arg, Obj). + let var_points_to_next = var_points_to // Recursion: var_points_to is also self-recursive + .map_index(|Tup2(src, obj)| (src.clone(), Tup2(src.clone(), obj.clone()))) + .join_index( + // var_points_to JOIN assign ON src + &assign.map_index(|Tup2(dst, src)| { + (src.clone(), Tup2(dst.clone(), src.clone())) + }), + |_src, Tup2(_, obj), Tup2(dst, _)| { + Some(( + Tup2(dst.clone(), obj.clone()), + Tup2(dst.clone(), obj.clone()), + )) + }, + ) + .plus( + // "base case": alloc feeds into var_points_to + &alloc.map_index(|Tup2(var, obj)| { + ( + Tup2(var.clone(), obj.clone()), + Tup2(var.clone(), obj.clone()), + ) + }), + ) + .plus( + // This argument to plus() computes this part of the Datalog query: + // VarPointsTo(Param, Obj) :- + // CallGraph(Site, Meth), + // ActualArg(Site, Arg), + // FormalParam(Meth, Param), + // VarPointsTo(Arg, Obj). + // Mutual recursion: var_points_to uses call_graph + &call_graph + .map_index(|Tup2(site, meth)| { + (site.clone(), Tup2(site.clone(), meth.clone())) + }) + .join_index( + // 1. call_graph JOIN actual_arg ON site + &actual_arg.map_index(|Tup2(site, arg)| { + (site.clone(), Tup2(site.clone(), arg.clone())) + }), + |_site, Tup2(_, meth), Tup2(_, arg)| { + Some((meth.clone(), Tup2(meth.clone(), arg.clone()))) + }, + ) + .join_index( + // .2. ... JOIN formal_param ON meth + &formal_param.map_index(|Tup2(meth, param)| { + (meth.clone(), Tup2(meth.clone(), param.clone())) + }), + |_meth, Tup2(_, arg), Tup2(_, param)| { + Some(((arg.clone()), Tup2(arg.clone(), param.clone()))) + }, + ) + .join_index( + // 3. ... JOIN var_points_to ON arg + &var_points_to.map_index(|Tup2(arg, obj)| { + (arg.clone(), Tup2(arg.clone(), obj.clone())) + }), + |_arg, Tup2(_, param), Tup2(_, obj)| { + Some(( + Tup2(param.clone(), obj.clone()), + Tup2(param.clone(), obj.clone()), + )) + }, + ), + ); + + Ok(( + var_points_to_next + .map(|(Tup2(param, obj), _)| Tup2(param.clone(), obj.clone())), + call_graph_next.map(|(Tup2(site, meth), _)| Tup2(site.clone(), meth.clone())), + )) + }, + )?; + + Ok(( + ( + alloc_input, + assign_input, + virtual_call_input, + heap_type_input, + dispatch_input, + actual_arg_input, + formal_param_input, + ), + ( + var_points_to.accumulate_output(), + call_graph.accumulate_output(), + ), + )) + })?; + + // Define the inputs at each "step". + + let mut alloc_inputs = ([ + vec![(("g", "oG"), 1), (("d", "oDog"), 1), (("c", "oCat"), 1)] + .into_iter() + .map(owned_string2) + .collect(), + vec![(("m", "oMouse"), 1)] + .into_iter() + .map(owned_string2) + .collect(), + vec![], + ] as [Vec>; STEPS]) + .into_iter(); + + let mut assign_inputs = ([ + vec![(("ac", "c"), 1)] + .into_iter() + .map(owned_string2) + .collect(), + vec![], + vec![(("ac", "c"), -1)] + .into_iter() + .map(owned_string2) + .collect(), + ] as [Vec>; STEPS]) + .into_iter(); + + let mut virtual_call_inputs = ([ + vec![ + (("s1", "g", "greet"), 1), + (("s2", "g", "greet"), 1), + (("s3", "x", "speak"), 1), + ] + .into_iter() + .map(owned_string3) + .collect(), + vec![(("s4", "g", "greet"), 1)] + .into_iter() + .map(owned_string3) + .collect(), + vec![(("s2", "g", "greet"), -1)] + .into_iter() + .map(owned_string3) + .collect(), + ] as [Vec>; STEPS]) + .into_iter(); + + let mut heap_type_inputs = ([ + vec![ + (("oG", "Greeter"), 1), + (("oDog", "Dog"), 1), + (("oCat", "Cat"), 1), + ] + .into_iter() + .map(owned_string2) + .collect(), + vec![(("oMouse", "Mouse"), 1)] + .into_iter() + .map(owned_string2) + .collect(), + vec![], + ] as [Vec>; STEPS]) + .into_iter(); + + let mut dispatch_inputs = ([ + vec![ + (("Greeter", "greet", "Greeter.greet"), 1), + (("Dog", "speak", "Dog.speak"), 1), + (("Cat", "speak", "Cat.speak"), 1), + ] + .into_iter() + .map(owned_string3) + .collect(), + vec![(("Mouse", "speak", "Mouse.speak"), 1)] + .into_iter() + .map(owned_string3) + .collect(), + vec![], + ] as [Vec>; STEPS]) + .into_iter(); + + let mut actual_arg_inputs = ([ + vec![(("s1", "d"), 1), (("s2", "ac"), 1)] + .into_iter() + .map(owned_string2) + .collect(), + vec![(("s4", "m"), 1)] + .into_iter() + .map(owned_string2) + .collect(), + vec![(("s2", "ac"), 1)] + .into_iter() + .map(owned_string2) + .collect(), + ] as [Vec>; STEPS]) + .into_iter(); + + let mut formal_param_inputs = ([ + vec![(("Greeter.greet", "x"), 1)] + .into_iter() + .map(owned_string2) + .collect(), + vec![], + vec![], + ] as [Vec>; STEPS]) + .into_iter(); + + // Define the expected outputs at each "step". + + let mut var_points_to_expected_outputs = ([ + zset! { + Tup2("ac".to_string(), "oCat".to_string()) => 1, + Tup2("c".to_string(), "oCat".to_string()) => 1, + Tup2("d".to_string(), "oDog".to_string()) => 1, + Tup2("g".to_string(), "oG".to_string()) => 1, + Tup2("x".to_string(), "oDog".to_string()) => 1, + Tup2("x".to_string(), "oCat".to_string()) => 1, + }, + zset! { + Tup2("m".to_string(), "oMouse".to_string()) => 1, + Tup2("x".to_string(), "oMouse".to_string()) => 1, + }, + zset! { + Tup2("ac".to_string(), "oCat".to_string()) => -1, + Tup2("x".to_string(), "oCat".to_string()) => -1, + }, + ] as [OrdZSet; STEPS]) + .into_iter(); + + let mut call_graph_expected_outputs = ([ + zset! { + Tup2("s1".to_string(), "Greeter.greet".to_string()) => 1, + Tup2("s2".to_string(), "Greeter.greet".to_string()) => 1, + Tup2("s3".to_string(), "Dog.speak".to_string()) => 1, + Tup2("s3".to_string(), "Cat.speak".to_string()) => 1, + }, + zset! { + Tup2("s3".to_string(), "Mouse.speak".to_string()) => 1, + Tup2("s4".to_string(), "Greeter.greet".to_string()) => 1, + }, + zset! { + Tup2("s2".to_string(), "Greeter.greet".to_string()) => -1, + Tup2("s3".to_string(), "Cat.speak".to_string()) => -1, + }, + ] as [OrdZSet; STEPS]) + .into_iter(); + + // Execute the circuit. + for i in 1..=STEPS { + // 1. Feed input data. + alloc_input.append(&mut alloc_inputs.next().unwrap()); + assign_input.append(&mut assign_inputs.next().unwrap()); + virtual_call_input.append(&mut virtual_call_inputs.next().unwrap()); + heap_type_input.append(&mut heap_type_inputs.next().unwrap()); + dispatch_input.append(&mut dispatch_inputs.next().unwrap()); + actual_arg_input.append(&mut actual_arg_inputs.next().unwrap()); + formal_param_input.append(&mut formal_param_inputs.next().unwrap()); + + // 2. Execute the transaction. + circuit.transaction()?; + + // 3. Print and check outputs. + println!("=== Outputs Iteration {i} ==="); + println!("=== VarPointsTo Output ==="); + let var_points_to_output = var_points_to_output.concat(); + var_points_to_output + .iter() + .for_each(|(Tup2(var, o_type), _, z_weight)| { + println!("var: {var} -> object_type: {o_type} => {z_weight}"); + }); + assert_eq!( + var_points_to_output.consolidate(), + var_points_to_expected_outputs.next().unwrap(), + ); + println!("=== CallGraph Output ==="); + let call_graph_output = call_graph_output.concat(); + call_graph_output + .iter() + .for_each(|(Tup2(call_site, method), _, z_weight)| { + println!("call_site {call_site} -> method {method} => {z_weight}"); + }); + assert_eq!( + call_graph_output.consolidate(), + call_graph_expected_outputs.next().unwrap(), + ); + } + + Ok(()) +} diff --git a/crates/dbsp/src/circuit.rs b/crates/dbsp/src/circuit.rs index dd3573a1402..44b217523e2 100644 --- a/crates/dbsp/src/circuit.rs +++ b/crates/dbsp/src/circuit.rs @@ -34,8 +34,9 @@ pub mod trace; mod replay_tests; pub use circuit_builder::{ - ChildCircuit, Circuit, CircuitHandle, ExportId, ExportStream, FeedbackConnector, GlobalNodeId, - NestedCircuit, NodeId, OwnershipPreference, RegionName, RootCircuit, Scope, Stream, WithClock, + ChildCircuit, Circuit, CircuitHandle, ElapsedTime, ExportId, ExportStream, FeedbackConnector, + GlobalNodeId, NestedCircuit, NodeId, OwnershipPreference, RegionName, RootCircuit, Scope, + Stream, ThreadCpuTime, WithClock, }; pub use dbsp_handle::{ CheckpointCommitter, CircuitConfig, CircuitStorageConfig, DBSPHandle, Host, Layout, diff --git a/crates/dbsp/src/circuit/circuit_builder.rs b/crates/dbsp/src/circuit/circuit_builder.rs index 872c1918dc3..fdc1a7937a6 100644 --- a/crates/dbsp/src/circuit/circuit_builder.rs +++ b/crates/dbsp/src/circuit/circuit_builder.rs @@ -59,8 +59,13 @@ use feldera_ir::{LirCircuit, LirNodeId}; use feldera_samply::Span; use feldera_storage::{FileCommitter, StoragePath}; use itertools::Itertools; +use nix::{ + sys::time::TimeValLike, + time::{ClockId, clock_gettime}, +}; use pin_project_lite::pin_project; use serde::{Deserialize, Serialize, Serializer, de::DeserializeOwned}; +use size_of::SizeOf; use std::{ any::{Any, TypeId, type_name_of_val}, borrow::Cow, @@ -71,7 +76,7 @@ use std::{ io::ErrorKind, marker::PhantomData, mem::{take, transmute}, - ops::Deref, + ops::{AddAssign, Deref}, panic::Location, pin::Pin, rc::Rc, @@ -3799,20 +3804,26 @@ where circuit.nodes.borrow()[id.0].borrow().as_ref(), )); - let span = Span::new("eval") - .with_category("Operator") - .with_tooltip(|| { - let nodes = circuit.nodes.borrow(); - let node = nodes[id.0].borrow(); - format!("{} {}", node.name(), node.global_id().node_identifier()) - }); - let (result, duration) = Timed::new(circuit.nodes.borrow()[id.0].borrow_mut().eval()).await; + let span = Span::new("eval").with_category("Operator"); + let (result, elapsed_time) = + Timed::new(circuit.nodes.borrow()[id.0].borrow_mut().eval()).await; let progress = result?; - span.record(); + span.with_tooltip(|| { + let nodes = circuit.nodes.borrow(); + let node = nodes[id.0].borrow(); + format!( + "{} {} used {}μs real time, {}μs CPU time", + node.name(), + node.global_id().node_identifier(), + elapsed_time.real.as_micros(), + elapsed_time.cpu.as_micros(), + ) + }) + .record(); circuit.log_scheduler_event(&SchedulerEvent::eval_end( circuit.nodes.borrow()[id.0].borrow().as_ref(), - duration, + elapsed_time, )); Ok(progress) @@ -4552,7 +4563,7 @@ where let res = constructor(child)?; let child_clone = child.clone(); - let consensus = Consensus::new(); + let consensus = Consensus::new("fixed point"); let termination_check = async move || { // Send local fixed point status to all peers. @@ -7913,14 +7924,35 @@ impl CircuitHandle { } } +/// Real time and CPU time. +#[derive(Copy, Clone, Debug, Default, SizeOf)] +pub struct ElapsedTime { + /// Real time running a task. + pub real: Duration, + + /// CPU time running a task. + /// + /// In the ordinary course, if `cpu` is much less than `real`, then it + /// indicates that the task blocked its thread, e.g. for synchronous I/O, + /// without yielding to the tokio scheduler. + pub cpu: Duration, +} + +impl AddAssign for ElapsedTime { + fn add_assign(&mut self, rhs: Self) { + self.real += rhs.real; + self.cpu += rhs.cpu; + } +} + pin_project! { /// An async task that measures its runtime. /// /// This uses the same wrapper technique as [tokio_metrics::Instrumented] or /// [tracing::Instrument]: by implementing [Future] manually, it wraps each - /// poll with elapsed time measurement. When the future eventually - /// completes, it outputs the wrapped future's output value plus the total - /// elapsed time. + /// poll with elapsed real and CPU time measurement. When the future + /// eventually completes, it outputs the wrapped future's output value plus + /// the [ElapsedTime]. /// /// [tokio_metrics::Instrumented]: https://docs.rs/tokio-metrics/latest/tokio_metrics/struct.Instrumented.html /// [tracing::Instrument]: https://docs.rs/tracing/latest/tracing/trait.Instrument.html @@ -7930,8 +7962,8 @@ pin_project! { #[pin] task: T, - // Time spent running this task. - elapsed: Duration, + // Elapsed time running this task. + elapsed: ElapsedTime, } } @@ -7939,22 +7971,48 @@ impl Timed { fn new(task: T) -> Self { Self { task, - elapsed: Duration::ZERO, + elapsed: ElapsedTime::default(), } } } +/// Amount of time elapsed running a thread. +pub struct ThreadCpuTime(pub Duration); + +impl ThreadCpuTime { + /// Returns the current time elapsed running the current thread. + pub fn now() -> Self { + let nanos = clock_gettime(ClockId::CLOCK_THREAD_CPUTIME_ID) + .unwrap() + .num_nanoseconds(); + Self(Duration::from_nanos(nanos.max(0).cast_unsigned())) + } + + /// Returns the time elapsed running the current thread since this + /// `ThreadCpuTime`. + /// + /// This only makes sense if this `ThreadCpuTime` was for the currently + /// running thread. + /// + /// Returns zero if the current time is earlier than self. + pub fn elapsed(&self) -> Duration { + Self::now().0.saturating_sub(self.0) + } +} + impl Future for Timed where T: Future, { - type Output = (T::Output, Duration); + type Output = (T::Output, ElapsedTime); fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.project(); let start = Instant::now(); + let start_cputime = ThreadCpuTime::now(); let ret = this.task.poll(cx); - *this.elapsed += start.elapsed(); + this.elapsed.real += start.elapsed(); + this.elapsed.cpu += start_cputime.elapsed(); ret.map(|value| (value, take(&mut *this.elapsed))) } } diff --git a/crates/dbsp/src/circuit/dbsp_handle.rs b/crates/dbsp/src/circuit/dbsp_handle.rs index faa9d95d86a..853b4c124c5 100644 --- a/crates/dbsp/src/circuit/dbsp_handle.rs +++ b/crates/dbsp/src/circuit/dbsp_handle.rs @@ -414,11 +414,20 @@ pub fn adaptive_joins_enabled() -> bool { Runtime::with_dev_tweaks(|d| d.adaptive_joins()) } -pub fn max_level0_batch_size_records() -> u16 { - Runtime::with_dev_tweaks(|d| { +pub fn max_level0_batch_size_records() -> usize { + let max_level0_batch_size_records = Runtime::with_dev_tweaks(|d| { d.max_level0_batch_size_records .unwrap_or(MAX_LEVEL0_BATCH_SIZE_RECORDS) - }) + }) as usize; + assert!( + max_level0_batch_size_records > 0, + "max_level0_batch_size_records must be greater than 0" + ); + assert!( + max_level0_batch_size_records <= 99_999, + "max_level0_batch_size_records must be less than or equal to 99_999" + ); + max_level0_batch_size_records } pub fn negative_weight_multiplier() -> u16 { diff --git a/crates/dbsp/src/circuit/metadata.rs b/crates/dbsp/src/circuit/metadata.rs index e9377ccde06..e48ef450d1f 100644 --- a/crates/dbsp/src/circuit/metadata.rs +++ b/crates/dbsp/src/circuit/metadata.rs @@ -160,6 +160,9 @@ pub const MERGE_BACKPRESSURE_WAIT_TIME_SECONDS: MetricId = MetricId(Cow::Borrowed("merge_backpressure_wait_time_seconds")); pub const INVOCATIONS_COUNT: MetricId = MetricId(Cow::Borrowed("invocations_count")); pub const RUNTIME_SECONDS: MetricId = MetricId(Cow::Borrowed("runtime_seconds")); +/// The fraction of an operator's runtime that it is actually running as opposed +/// to blocking in the kernel (e.g. because it is waiting for I/O). +pub const RUNTIME_NONBLOCKING_PERCENT: MetricId = MetricId(Cow::Borrowed("nonblocking_percent")); pub const RUNTIME_PERCENT: MetricId = MetricId(Cow::Borrowed("runtime_percent")); pub const CIRCUIT_WAIT_TIME_SECONDS: MetricId = MetricId(Cow::Borrowed("circuit_wait_time_seconds")); @@ -177,7 +180,7 @@ pub const PREFIX_BATCHES_STATS: MetricId = MetricId(Cow::Borrowed("prefix_batche pub const INPUT_INTEGRAL_RECORDS_COUNT: MetricId = MetricId(Cow::Borrowed("input_integral_records_count")); -pub const CIRCUIT_METRICS: [CircuitMetric; 75] = [ +pub const CIRCUIT_METRICS: [CircuitMetric; 76] = [ // State CircuitMetric { name: USED_MEMORY_BYTES, @@ -506,6 +509,12 @@ pub const CIRCUIT_METRICS: [CircuitMetric; 75] = [ advanced: false, description: "Time spent waiting for backpressure.", }, + CircuitMetric { + name: RUNTIME_NONBLOCKING_PERCENT, + category: CircuitMetricCategory::Time, + advanced: true, + description: "Fraction of operator runtime spent running, as opposed to blocking in the kernel (e.g. because it is waiting for I/O) or descheduled by the kernel (because there are fewer CPUs than runnable threads).", + }, // Balancer CircuitMetric { name: KEY_DISTRIBUTION, diff --git a/crates/dbsp/src/circuit/runtime.rs b/crates/dbsp/src/circuit/runtime.rs index 766268bc267..8e24494cb96 100644 --- a/crates/dbsp/src/circuit/runtime.rs +++ b/crates/dbsp/src/circuit/runtime.rs @@ -24,7 +24,7 @@ use core_affinity::{CoreId, get_core_ids}; use crossbeam::sync::{Parker, Unparker}; use enum_map::{Enum, EnumMap, enum_map}; use feldera_buffer_cache::ThreadType; -use feldera_samply::LongSpanBuilder; +use feldera_samply::{LongSpanBuilder, Span}; use feldera_storage::fbuf::FBuf; use feldera_storage::fbuf::slab::{FBufSlabs, FBufSlabsStats, set_thread_slab_pool}; use feldera_types::config::{DevTweaks, StorageCompression, StorageConfig, StorageOptions}; @@ -1351,8 +1351,8 @@ impl Runtime { pub(crate) struct Consensus(Broadcast); impl Consensus { - pub fn new() -> Self { - Self(Broadcast::new()) + pub fn new(name: impl Display) -> Self { + Self(Broadcast::new(name)) } /// Returns `true` if all workers vote `true`. @@ -1379,12 +1379,12 @@ impl Broadcast where T: Clone + Debug + Send + serde::Serialize + for<'de> serde::Deserialize<'de> + 'static, { - pub fn new() -> Self { + pub fn new(name: impl Display) -> Self { match Runtime::runtime() { Some(runtime) if Runtime::num_workers() > 1 => { let exchange_id = runtime.sequence_next().try_into().unwrap(); let exchange = Exchange::with_runtime(&runtime, exchange_id); - let identifier = Arc::new(format!("broadcast {exchange_id}")); + let identifier = Arc::new(format!("broadcast {name} (exchange {exchange_id})")); Self::MultiThreaded { exchange, @@ -1406,24 +1406,32 @@ where Self::MultiThreaded { exchange, identifier, - } => Runtime::runtime() - .unwrap() - .cancellation_token() - .run_until_cancelled_owned(async { - exchange - .send_all_with_serializer(identifier, repeat(local.clone()), |local| { - let mut fbuf = FBuf::new(); - serde_json::to_writer(&mut fbuf, &local).unwrap(); - fbuf - }) - .await; - - exchange - .receive_all(|data| serde_json::from_slice(&data).unwrap()) - .await - }) - .await - .ok_or(SchedulerError::Killed), + } => { + // This span shows elapsed time from sending our broadcast to + // getting all the others' results back. + let _span = Span::new("broadcast") + .with_category("Exchange") + .with_tooltip(|| format!("collect for {}", identifier)); + + Runtime::runtime() + .unwrap() + .cancellation_token() + .run_until_cancelled_owned(async { + exchange + .send_all_with_serializer(identifier, repeat(local.clone()), |local| { + let mut fbuf = FBuf::new(); + serde_json::to_writer(&mut fbuf, &local).unwrap(); + fbuf + }) + .await; + + exchange + .receive_all(|data| serde_json::from_slice(&data).unwrap()) + .await + }) + .await + .ok_or(SchedulerError::Killed) + } } } } diff --git a/crates/dbsp/src/circuit/schedule/dynamic_scheduler.rs b/crates/dbsp/src/circuit/schedule/dynamic_scheduler.rs index 75581b41221..8a0010ca5e9 100644 --- a/crates/dbsp/src/circuit/schedule/dynamic_scheduler.rs +++ b/crates/dbsp/src/circuit/schedule/dynamic_scheduler.rs @@ -358,8 +358,8 @@ impl Inner { handles: JoinSet::new(), waiting: false, transaction_phase: TransactionPhase::CommitComplete, - global_commit_consensus: Broadcast::new(), - metadata_broadcast: Broadcast::new(), + global_commit_consensus: Broadcast::new("global commit consensus"), + metadata_broadcast: Broadcast::new("metadata"), before_first_step: true, flush_state: false, }; diff --git a/crates/dbsp/src/circuit/trace.rs b/crates/dbsp/src/circuit/trace.rs index 302244a72e7..d36c8c19666 100644 --- a/crates/dbsp/src/circuit/trace.rs +++ b/crates/dbsp/src/circuit/trace.rs @@ -17,12 +17,11 @@ //! quickly, with any expensive processing completed asynchronously. use super::{GlobalNodeId, NodeId, OwnershipPreference, circuit_builder::Node}; -use crate::circuit::{RegionName, metadata::OperatorLocation}; +use crate::circuit::{RegionName, circuit_builder::ElapsedTime, metadata::OperatorLocation}; use std::{ borrow::Cow, fmt::{self, Display}, hash::Hash, - time::Duration, }; /// Type of edge in a circuit graph. @@ -556,7 +555,7 @@ pub enum SchedulerEvent<'a> { }, EvalEnd { node: &'a dyn Node, - duration: Duration, + elapsed_time: ElapsedTime, }, WaitStart { circuit_id: &'a GlobalNodeId, @@ -581,8 +580,8 @@ impl<'a> SchedulerEvent<'a> { } /// Create a [`SchedulerEvent::EvalEnd`] event instance. - pub fn eval_end(node: &'a dyn Node, duration: Duration) -> Self { - Self::EvalEnd { node, duration } + pub fn eval_end(node: &'a dyn Node, elapsed_time: ElapsedTime) -> Self { + Self::EvalEnd { node, elapsed_time } } /// Create a [`SchedulerEvent::WaitStart`] event instance. @@ -622,12 +621,13 @@ impl Display for SchedulerEvent<'_> { Self::EvalStart { node } => { write!(f, "EvalStart({})", node.global_id()) } - Self::EvalEnd { node, duration } => { + Self::EvalEnd { node, elapsed_time } => { write!( f, - "EvalEnd({}, {}μs)", + "EvalEnd({}, {}μs real, {}μs CPU)", node.global_id(), - duration.as_micros() + elapsed_time.real.as_micros(), + elapsed_time.cpu.as_micros(), ) } Self::WaitStart { circuit_id } => { diff --git a/crates/dbsp/src/monitor.rs b/crates/dbsp/src/monitor.rs index 03af2e65f77..81c4da720de 100644 --- a/crates/dbsp/src/monitor.rs +++ b/crates/dbsp/src/monitor.rs @@ -641,7 +641,10 @@ impl TraceMonitorInternal { } } } - SchedulerEvent::EvalEnd { node, duration: _ } => match self.current_state() { + SchedulerEvent::EvalEnd { + node, + elapsed_time: _, + } => match self.current_state() { CircuitState::Eval(visited_nodes, eval_node_id) => { if eval_node_id != &node.local_id() { return Err(TraceError::InvalidEvent(Cow::from(format!( diff --git a/crates/dbsp/src/operator/non_incremental.rs b/crates/dbsp/src/operator/non_incremental.rs index 96cacaf4665..6204b6c8941 100644 --- a/crates/dbsp/src/operator/non_incremental.rs +++ b/crates/dbsp/src/operator/non_incremental.rs @@ -96,7 +96,7 @@ impl NonIterativeExecutor { Self { scheduler: DynamicScheduler::new(), flush: Cell::new(false), - flush_consensus: Consensus::new(), + flush_consensus: Consensus::new("flush"), } } } diff --git a/crates/dbsp/src/profile.rs b/crates/dbsp/src/profile.rs index fae4e355beb..7b149675c07 100644 --- a/crates/dbsp/src/profile.rs +++ b/crates/dbsp/src/profile.rs @@ -9,8 +9,8 @@ use crate::{ BACKGROUND_CACHE_OCCUPANCY, CIRCUIT_IDLE_TIME_SECONDS, CIRCUIT_METRICS, CIRCUIT_RUNTIME_ELAPSED_SECONDS, CIRCUIT_RUNTIME_SECONDS, CIRCUIT_WAIT_TIME_SECONDS, CircuitMetric, FOREGROUND_CACHE_OCCUPANCY, INVOCATIONS_COUNT, MetaItem, MetricId, - MetricReading, OperatorMeta, RUNTIME_PERCENT, RUNTIME_SECONDS, - SPINE_STORAGE_SIZE_BYTES, STEPS_COUNT, USED_MEMORY_BYTES, + MetricReading, OperatorMeta, RUNTIME_NONBLOCKING_PERCENT, RUNTIME_PERCENT, + RUNTIME_SECONDS, SPINE_STORAGE_SIZE_BYTES, STEPS_COUNT, USED_MEMORY_BYTES, }, }, monitor::{TraceMonitor, visual_graph::Graph}, @@ -334,7 +334,7 @@ impl Profiler { let mut total_time: Duration = Duration::default(); for (node_id, _) in metadata.iter_mut() { if let Some(profile) = self.cpu_profiler.operator_profile(node_id) { - total_time += profile.total_time(); + total_time += profile.real_time(); } } @@ -358,13 +358,21 @@ impl Profiler { MetricReading::new( RUNTIME_SECONDS, Vec::new(), - MetaItem::Duration(profile.total_time()), + MetaItem::Duration(profile.real_time()), + ), + MetricReading::new( + RUNTIME_NONBLOCKING_PERCENT, + Vec::new(), + MetaItem::Percent { + numerator: profile.cpu_time().as_micros() as u64, + denominator: profile.real_time().as_micros() as u64, + }, ), MetricReading::new( RUNTIME_PERCENT, Vec::new(), MetaItem::Percent { - numerator: profile.total_time().as_micros() as u64, + numerator: profile.real_time().as_micros() as u64, denominator: total_time.as_micros() as u64, }, ), @@ -376,10 +384,10 @@ impl Profiler { // Additional metadata for circuit nodes. if let Some(profile) = self.cpu_profiler.circuit_profile(node_id) { let default_meta = metadata![ - CIRCUIT_WAIT_TIME_SECONDS => profile.wait_profile.total_time(), + CIRCUIT_WAIT_TIME_SECONDS => profile.wait_profile.real_time(), STEPS_COUNT => profile.step_profile.invocations(), - CIRCUIT_RUNTIME_SECONDS => profile.step_profile.total_time(), - CIRCUIT_IDLE_TIME_SECONDS => profile.idle_profile.total_time(), + CIRCUIT_RUNTIME_SECONDS => profile.step_profile.real_time(), + CIRCUIT_IDLE_TIME_SECONDS => profile.idle_profile.real_time(), CIRCUIT_RUNTIME_ELAPSED_SECONDS => runtime_elapsed, ]; diff --git a/crates/dbsp/src/profile/cpu.rs b/crates/dbsp/src/profile/cpu.rs index 6aa79f35f0d..e240fa0222b 100644 --- a/crates/dbsp/src/profile/cpu.rs +++ b/crates/dbsp/src/profile/cpu.rs @@ -18,13 +18,15 @@ use std::{ #[derive(Clone, Default, Debug)] pub struct OperatorCPUProfile { invocations: usize, - total_time: Duration, + real_time: Duration, + cpu_time: Duration, } impl OperatorCPUProfile { - pub fn add_event(&mut self, duration: Duration) { + pub fn add_event(&mut self, real_time: Duration, cpu_time: Duration) { self.invocations += 1; - self.total_time += duration; + self.real_time += real_time; + self.cpu_time += cpu_time; } /// Returns the number of times the operator has been invoked. @@ -34,9 +36,14 @@ impl OperatorCPUProfile { self.invocations } - /// Total time spent evaluating the operator across all invocations. - pub fn total_time(&self) -> Duration { - self.total_time + /// Total elapsed time spent evaluating the operator across all invocations. + pub fn real_time(&self) -> Duration { + self.real_time + } + + /// Total CPU time spent evaluating the operator across all invocations. + pub fn cpu_time(&self) -> Duration { + self.cpu_time } } @@ -78,7 +85,9 @@ impl CPUProfilerInner { .circuit_profiles .entry((*circuit_id).clone()) .or_insert_with(Default::default); - circuit_profile.idle_profile.add_event(duration); + circuit_profile + .idle_profile + .add_event(duration, Duration::ZERO); }; self.step_start_times @@ -91,18 +100,20 @@ impl CPUProfilerInner { .circuit_profiles .entry((*circuit_id).clone()) .or_insert_with(Default::default); - circuit_profile.step_profile.add_event(duration); + circuit_profile + .step_profile + .add_event(duration, Duration::ZERO); }; self.step_end_times .insert((*circuit_id).clone(), Instant::now()); } SchedulerEvent::EvalStart { .. } => {} - SchedulerEvent::EvalEnd { node, duration } => { + SchedulerEvent::EvalEnd { node, elapsed_time } => { let op_profile = self .operators .entry(node.global_id().clone()) .or_insert_with(Default::default); - op_profile.add_event(*duration); + op_profile.add_event(elapsed_time.real, elapsed_time.cpu); // println!("{}:{}:{:?}", crate::Runtime::worker_index(), // node.global_id(), duration); } @@ -117,7 +128,9 @@ impl CPUProfilerInner { .circuit_profiles .entry((*circuit_id).clone()) .or_insert_with(Default::default); - circuit_profile.wait_profile.add_event(duration); + circuit_profile + .wait_profile + .add_event(duration, Duration::ZERO); }; } _ => (), diff --git a/crates/dbsp/src/trace/spine_async.rs b/crates/dbsp/src/trace/spine_async.rs index 1ed606da5d2..0953144464b 100644 --- a/crates/dbsp/src/trace/spine_async.rs +++ b/crates/dbsp/src/trace/spine_async.rs @@ -9,7 +9,7 @@ use crate::{ Error, NumEntries, Runtime, circuit::{ - max_level0_batch_size_records, + ElapsedTime, ThreadCpuTime, max_level0_batch_size_records, metadata::{ BLOOM_FILTER_BITS_PER_KEY, BLOOM_FILTER_HIT_RATE_PERCENT, BLOOM_FILTER_HITS_COUNT, BLOOM_FILTER_MISSES_COUNT, BLOOM_FILTER_SIZE_BYTES, COMPACTION_STATE, COMPLETED_MERGES, @@ -208,8 +208,8 @@ where /// Invariant: the batches must be non-empty. loose_batches: VecDeque>, - /// Amount of time spent merging batches at this level. - elapsed: Duration, + /// Time spent merging batches at this level. + elapsed: ElapsedTime, /// Number of completed merges at this level. n_merged: usize, @@ -237,7 +237,7 @@ where Self { merging_batches: None, loose_batches: VecDeque::new(), - elapsed: Duration::ZERO, + elapsed: ElapsedTime::default(), n_merged: 0, n_merged_batches: 0, n_steps: 0, @@ -464,7 +464,7 @@ where new_batch: Arc, new_level: usize, start: Instant, - elapsed: Duration, + elapsed: ElapsedTime, n_steps: usize, ) { let slot = &mut self.slots[level]; @@ -485,11 +485,13 @@ where .with_start(start) .with_tooltip(|| { format!( - "{} in worker {} merged {} batches ({pre_len} -> {post_len}) in {n_steps} steps using {:.1} ms CPU", + "{} in worker {} merged {} batches ({pre_len} -> {post_len}) in {} steps using {:.1} ms real time and {:.1} ms CPU time", &self.name, Runtime::worker_index(), batches.len(), - elapsed.as_secs_f64() * 1000.0 + n_steps, + elapsed.real.as_secs_f64() * 1000.0, + elapsed.cpu.as_secs_f64() * 1000.0 ) }) .record(); @@ -683,7 +685,7 @@ where let no_backpressure = Arc::new(Notify::new()); let state = Arc::new(Mutex::new(SharedState::new(factories, name))); - let max_level0_batch_size_records = max_level0_batch_size_records() as usize; + let max_level0_batch_size_records = max_level0_batch_size_records(); assert!( max_level0_batch_size_records > 0, "max_level0_batch_size_records must be greater than 0" @@ -907,8 +909,12 @@ where ), (Cow::Borrowed("steps"), MetaItem::Count(slot.n_steps)), ( - Cow::Borrowed("avg_step_time"), - MetaItem::Duration(slot.elapsed / slot.n_steps as u32), + Cow::Borrowed("avg_step_seconds"), + MetaItem::Duration(slot.elapsed.real / slot.n_steps as u32), + ), + ( + Cow::Borrowed("avg_step_cpu_seconds"), + MetaItem::Duration(slot.elapsed.cpu / slot.n_steps as u32), ), ])), )]); @@ -1377,7 +1383,7 @@ where start: Instant, /// Time spent running merge steps. - elapsed: Duration, + elapsed: ElapsedTime, /// Number of merge steps executed. n_steps: usize, @@ -1416,7 +1422,7 @@ where builder, fuel: 0, start: Instant::now(), - elapsed: Duration::ZERO, + elapsed: ElapsedTime::default(), n_steps: 0, done: false, frontier, @@ -1442,6 +1448,7 @@ where debug_assert!(fuel > 0); let supplied_fuel = fuel; let start = Instant::now(); + let start_cpu = ThreadCpuTime::now(); match &mut self.inner { MergeInner::ListMerger(merger) => { merger.work(&mut self.builder, &self.frontier, &mut fuel); @@ -1457,7 +1464,10 @@ where } } }; - self.elapsed += start.elapsed(); + self.elapsed += ElapsedTime { + real: start.elapsed(), + cpu: start_cpu.elapsed(), + }; self.n_steps += 1; let consumed_fuel = supplied_fuel - fuel; self.fuel += consumed_fuel; diff --git a/crates/dbsp/src/tutorial.rs b/crates/dbsp/src/tutorial.rs index ef218cb03dd..2c4d5887693 100644 --- a/crates/dbsp/src/tutorial.rs +++ b/crates/dbsp/src/tutorial.rs @@ -2002,7 +2002,10 @@ //! Fixed-point computations are useful if you want to repeat a query until //! its result does not change anymore. Then, a fixed-point is reached and //! the query processing terminates, yielding the result. SQL provides this -//! mechanism through recursive common table expressions (CTEs). +//! mechanism through recursive common table expressions (CTEs). DBSP supports +//! both self and mutual recursion. +//! +//! ## Fixed-Point Computation with Self-Recursion //! //! A classical use case for a fixed-point computation is the [transitive closure //! of a graph](https://en.wikipedia.org/wiki/Transitive_closure#In_graph_theory). @@ -2283,6 +2286,14 @@ //! optimum of the aggregation function (here, the minimum function), //! even though there exists a finite solution. //! +//! ## Fixed-Point Computation with Mutual Recursion +//! +//! The examples on the transitive closure above demonstrate how to express +//! self-recursive queries. DBSP also supports _mutually_-recursive queries. +//! As this is a little bit more involved than what would fit in here, we defer +//! the interested reader to the example given in `tutorial12` which +//! demonstrates mutual recursion in the domain of static program analysis. +//! //! # Next steps //! //! We've shown how input, computation, and output work in DBSP. That's all diff --git a/crates/fda/src/cli.rs b/crates/fda/src/cli.rs index 9509739912f..e0da94e7f73 100644 --- a/crates/fda/src/cli.rs +++ b/crates/fda/src/cli.rs @@ -320,6 +320,12 @@ pub enum PipelineAction { /// The compilation profile to use. #[arg(default_value = "optimized")] profile: CompilationProfile, + /// A tag to assign to the pipeline. + /// + /// Repeat the flag to assign several tags, e.g. `--tag prod --tag team-billing`. + /// Tags are deduplicated and stored in sorted order. + #[arg(long = "tag", value_hint = ValueHint::Other)] + tags: Vec, /// Read the program code from stdin. /// /// EXAMPLES: @@ -515,6 +521,31 @@ pub enum PipelineAction { /// The new value for the configuration. value: String, }, + /// Retrieve the tags of a pipeline. + /// + /// Prints the tags as a comma-separated list, in sorted order. + Tags { + /// The name of the pipeline. + #[arg(value_hint = ValueHint::Other, add = ArgValueCompleter::new(pipeline_names))] + name: String, + }, + /// Replace the tags of a pipeline. + /// + /// Takes the new tags as a single comma-separated list, replacing whatever the + /// pipeline carried before; pass an empty list to clear all tags. To append + /// instead, include the current tags, e.g. + /// `fda set-tags my-pipeline $(fda tags my-pipeline),d,e`. + /// + /// A tag containing spaces must be quoted, e.g. `"team billing",prod`. Each tag + /// may be named alone; its color is filled in from the same tag used elsewhere. + SetTags { + /// The name of the pipeline. + #[arg(value_hint = ValueHint::Other, add = ArgValueCompleter::new(pipeline_names))] + name: String, + /// The new tags, as a comma-separated list. Omit to clear all tags. + #[arg(default_value = "")] + tags: String, + }, /// Recompile a pipeline with the Feldera runtime version included in the /// currently installed Feldera platform. UpdateRuntime { diff --git a/crates/fda/src/main.rs b/crates/fda/src/main.rs index ef8a13a11c4..5f71c69e25d 100644 --- a/crates/fda/src/main.rs +++ b/crates/fda/src/main.rs @@ -33,6 +33,7 @@ mod bench; mod cli; mod debug; mod shell; +mod tags; pub(crate) const UPGRADE_NOTICE: &str = "Try upgrading to the latest CLI version to resolve this issue. Also make sure the pipeline is recompiled with the latest version of feldera. Report it on github.com/feldera/feldera if the issue persists."; @@ -677,6 +678,98 @@ async fn wait_for_checkpoint(client: &Client, name: String, seq_number: u64, wai } } +/// Fetch a pipeline's current tags. +async fn get_pipeline_tags(client: &Client, name: &str) -> Vec { + client + .get_pipeline() + .pipeline_name(name) + .send() + .await + .map_err(handle_errors_fatal( + client.baseurl().clone(), + "Failed to get pipeline tags", + 1, + )) + .unwrap() + .into_inner() + .tags +} + +/// Collect the tags used across every pipeline. +/// +/// This is the pool of "known tags": when a tag name is colored on one pipeline, +/// setting it on another borrows that color so the name stays one consistent +/// color everywhere. +async fn get_known_tags(client: &Client) -> Vec { + client + .list_pipelines() + .send() + .await + .map_err(handle_errors_fatal( + client.baseurl().clone(), + "Failed to list pipeline tags", + 1, + )) + .unwrap() + .into_inner() + .into_iter() + .flat_map(|pipeline| pipeline.tags) + .collect() +} + +/// Replace a pipeline's tags and return the stored result. +async fn set_pipeline_tags(client: &Client, name: &str, tags: Vec) -> Vec { + client + .patch_pipeline() + .pipeline_name(name) + .body(PatchPipeline { + description: None, + tags: Some(tags), + name: None, + program_code: None, + udf_rust: None, + udf_toml: None, + program_config: None, + runtime_config: None, + }) + .send() + .await + .map_err(handle_errors_fatal( + client.baseurl().clone(), + "Failed to update pipeline tags", + 1, + )) + .unwrap() + .into_inner() + .tags +} + +/// Print a pipeline's tags as a comma-separated list of display names, or as a +/// JSON array of the raw stored strings. +/// +/// Text output shows display names: the color suffix that encodes a tag's color +/// is stripped, since it is noise to a human reader and the same comma-separated +/// list feeds straight back into `set-tags`. JSON output keeps the raw strings, +/// so a tag's color round-trips for machine consumers. +fn print_pipeline_tags(format: OutputFormat, tags: &[String]) { + match format { + OutputFormat::Text => { + let names: Vec<&str> = tags.iter().map(|tag| tags::tag_display_name(tag)).collect(); + println!("{}", names.join(",")); + } + OutputFormat::Json => { + println!( + "{}", + serde_json::to_string_pretty(tags).expect("Failed to serialize tags") + ); + } + _ => { + eprintln!("Unsupported output format: {}", format); + std::process::exit(1); + } + } +} + async fn pipeline(format: OutputFormat, action: PipelineAction, client: Client) { match action { PipelineAction::Create { @@ -685,6 +778,7 @@ async fn pipeline(format: OutputFormat, action: PipelineAction, client: Client) runtime_version, use_platform_compiler, profile, + tags, udf_rs, udf_toml, stdin, @@ -694,11 +788,19 @@ async fn pipeline(format: OutputFormat, action: PipelineAction, client: Client) read_file(udf_rs).await, read_file(udf_toml).await, ) { + // Each `--tag` borrows its color from the tags already used across + // pipelines, keeping a name one consistent color. The lookup is + // skipped when no tags were given. + let tags = if tags.is_empty() { + Vec::new() + } else { + tags::set_tags(tags, &get_known_tags(&client).await) + }; let response = client .post_pipeline() .body(PostPutPipeline { description: None, - tags: Vec::new(), + tags, name: name.to_string(), program_code: program_code.unwrap_or_default(), udf_rust, @@ -1438,6 +1540,22 @@ async fn pipeline(format: OutputFormat, action: PipelineAction, client: Client) } } } + PipelineAction::Tags { name } => { + let current = get_pipeline_tags(&client, &name).await; + print_pipeline_tags(format, ¤t); + } + PipelineAction::SetTags { name, tags } => { + let requested = tags::split_tag_list(&tags); + // A tag may be named alone; its color is borrowed from the same tag + // used across pipelines. + let known = get_known_tags(&client).await; + let updated = + set_pipeline_tags(&client, &name, tags::set_tags(requested, &known)).await; + if matches!(format, OutputFormat::Text) { + println!("Tags updated successfully."); + } + print_pipeline_tags(format, &updated); + } PipelineAction::UpdateRuntime { name } => { let response = client .post_update_runtime() diff --git a/crates/fda/src/tags.rs b/crates/fda/src/tags.rs new file mode 100644 index 00000000000..2c2088b902b --- /dev/null +++ b/crates/fda/src/tags.rs @@ -0,0 +1,239 @@ +//! Pipeline tag conventions shared across Feldera clients. +//! +//! The backend stores a tag as an opaque string, validates it against a fixed +//! character set, and knows nothing about color. A tag's color is encoded into +//! the string itself as a suffix that character set permits: the visible name, +//! then a `|` separator, then a six-digit `rrggbb` hex color (no leading `#`, +//! which the backend disallows) -- e.g. `"prod|ef4444"`. The suffix is optional, +//! so a bare `"prod"` is an uncolored tag; `"prod|ef4444"` and `"prod|22c55e"` +//! are the same tag in two colors. +//! +//! The CLI treats a tag's *display name* -- its name with any color suffix +//! removed -- as the tag's identity, and enforces two invariants whenever a +//! pipeline's tags are written: +//! +//! * **Variant exclusivity.** A pipeline carries at most one tag per display +//! name. When the input holds several color variants of the same name, the +//! last one wins. +//! * **Lexicographic order.** Tags are stored sorted, so the stored order is +//! stable regardless of the order they were supplied in. +//! +//! Both operate on the full string, so a tag's color is preserved; only the +//! display name decides which tags collide. A name alone is enough to refer to a +//! tag: its color is filled in from the same tag elsewhere, so no command +//! requires the user to type the raw color suffix. + +use std::collections::HashMap; + +/// Split a comma-separated tag list into individual tags. +/// +/// Empty segments are dropped, so an empty string yields no tags -- the way +/// `set-tags` clears a pipeline -- and a stray trailing comma is harmless. Only +/// the comma separates tags, so spaces are kept and a quoted tag such as +/// `"team billing"` survives intact. +pub fn split_tag_list(input: &str) -> Vec { + input + .split(',') + .filter(|tag| !tag.is_empty()) + .map(|tag| tag.to_string()) + .collect() +} + +/// A tag's visible text: its name, with any trailing `|rrggbb` color suffix +/// removed. +/// +/// The suffix is recognized only when the text after the last `|` is exactly six +/// hex digits, so a name that itself contains `|` (without a trailing color) is +/// returned whole. +pub fn tag_display_name(tag: &str) -> &str { + if let Some(separator) = tag.rfind('|') + && is_hex_color(&tag[separator + 1..]) + { + return &tag[..separator]; + } + tag +} + +/// Whether `text` is a six-digit `rrggbb` hex color. +fn is_hex_color(text: &str) -> bool { + text.len() == 6 && text.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// Collapse color variants of the same tag and sort the result. +/// +/// Tags that share a display name are deduplicated, keeping the last occurrence, +/// and the survivors are returned in lexicographic order. The color suffix that +/// encodes a tag's color is preserved. +pub fn normalize_tags(tags: I) -> Vec +where + I: IntoIterator, + S: AsRef, +{ + let mut by_display_name: HashMap = HashMap::new(); + for tag in tags { + let tag = tag.as_ref(); + by_display_name.insert(tag_display_name(tag).to_string(), tag.to_string()); + } + let mut result: Vec = by_display_name.into_values().collect(); + result.sort(); + result +} + +/// Index a pool of tags by display name, mapping each name to one colored +/// variant. +/// +/// When the pool holds several color variants of the same name -- which can +/// happen across pipelines if some client diverged -- the lexicographically +/// smallest one wins, so the choice is deterministic. +fn variant_index(pool: &[String]) -> HashMap { + let mut sorted = pool.to_vec(); + sorted.sort(); + let mut index: HashMap = HashMap::new(); + for tag in sorted { + index + .entry(tag_display_name(&tag).to_string()) + .or_insert(tag); + } + index +} + +/// Choose the colored variant to store for `name`. +/// +/// The CLI cannot express a color, so it borrows one rather than inventing it: +/// any variant of this name known across all pipelines (`known`) wins, and only +/// a name seen nowhere is stored uncolored. This keeps a given name one +/// consistent color everywhere. +fn resolve_variant(name: &str, known: &HashMap) -> String { + known + .get(tag_display_name(name)) + .cloned() + .unwrap_or_else(|| name.to_string()) +} + +/// Resolve `requested` against the colors in use across all pipelines and return +/// the normalized result. +/// +/// Each requested name borrows a color via [`resolve_variant`]: a variant known +/// across all pipelines (`known`), else uncolored. +pub fn set_tags(requested: I, known: &[String]) -> Vec +where + I: IntoIterator, + S: AsRef, +{ + let known = variant_index(known); + normalize_tags( + requested + .into_iter() + .map(|tag| resolve_variant(tag.as_ref(), &known)), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display_name_strips_color_suffix() { + assert_eq!(tag_display_name("prod"), "prod"); + assert_eq!(tag_display_name("prod|ef4444"), "prod"); + assert_eq!(tag_display_name("team billing|22c55e"), "team billing"); + // Uppercase hex digits are a valid color. + assert_eq!(tag_display_name("prod|ABCDEF"), "prod"); + // A name containing '|' but no valid trailing color is returned whole. + assert_eq!(tag_display_name("a|b"), "a|b"); + assert_eq!(tag_display_name("env|staging"), "env|staging"); + // Wrong-length or malformed suffixes are not colors. + assert_eq!(tag_display_name("prod|fff"), "prod|fff"); + assert_eq!(tag_display_name("prod|gggggg"), "prod|gggggg"); + // Only the final '|' segment is the color; an inner '|' stays in the name. + assert_eq!(tag_display_name("a|b|ef4444"), "a|b"); + assert_eq!(tag_display_name(""), ""); + } + + #[test] + fn sorts_lexicographically() { + assert_eq!( + normalize_tags(["prod", "dev", "staging"]), + vec!["dev", "prod", "staging"] + ); + } + + #[test] + fn is_idempotent() { + let once = normalize_tags(["beta", "alpha", "alpha|ef4444"]); + assert_eq!(normalize_tags(&once), once); + } + + #[test] + fn empty_input() { + assert_eq!(normalize_tags(Vec::::new()), Vec::::new()); + } + + #[test] + fn collapses_color_variants_keeping_last() { + assert_eq!(normalize_tags(["prod", "prod|ef4444"]), vec!["prod|ef4444"]); + assert_eq!(normalize_tags(["prod|ef4444", "prod"]), vec!["prod"]); + } + + #[test] + fn preserves_color_of_survivor() { + assert_eq!( + normalize_tags(["dev", "prod|ef4444", "qa"]), + vec!["dev", "prod|ef4444", "qa"] + ); + } + + #[test] + fn distinct_names_with_colors_all_kept() { + assert_eq!( + normalize_tags(["b|ef4444", "a|22c55e", "c"]), + vec!["a|22c55e", "b|ef4444", "c"] + ); + } + + #[test] + fn split_drops_empty_segments_and_keeps_spaces() { + assert_eq!(split_tag_list(""), Vec::::new()); + assert_eq!(split_tag_list("a,b,c"), vec!["a", "b", "c"]); + // A trailing or doubled comma yields no empty tag. + assert_eq!(split_tag_list("a,,b,"), vec!["a", "b"]); + // A quoted tag arrives as one argument with its space intact. + assert_eq!( + split_tag_list("team billing,prod"), + vec!["team billing", "prod"] + ); + } + + #[test] + fn set_borrows_color_from_known_pool() { + // "prod" is requested by name and colored in the known pool; setting it + // adopts that color, while "dev" is unknown and stored uncolored. + assert_eq!( + set_tags(["prod", "dev"], &["prod|ef4444".to_string()]), + vec!["dev", "prod|ef4444"] + ); + } + + #[test] + fn set_color_conflict_is_deterministic() { + // The known pool colors "prod" two ways; the lexicographically smallest + // variant ("...|22c55e" < "...|ef4444") wins, so the choice is stable. + assert_eq!( + set_tags( + ["prod"], + &["prod|ef4444".to_string(), "prod|22c55e".to_string()] + ), + vec!["prod|22c55e"] + ); + } + + #[test] + fn set_adds_uncolored_name_when_absent() { + assert_eq!(set_tags(["prod"], &[]), vec!["prod"]); + } + + #[test] + fn set_with_no_tags_clears() { + assert_eq!(set_tags(Vec::::new(), &[]), Vec::::new()); + } +} diff --git a/crates/feldera-types/src/format/csv.rs b/crates/feldera-types/src/format/csv.rs index 84566466f04..2e588aaa5d7 100644 --- a/crates/feldera-types/src/format/csv.rs +++ b/crates/feldera-types/src/format/csv.rs @@ -3,29 +3,114 @@ use std::fmt::Debug; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; -#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] +/// Whitespace trimming policy applied to CSV fields or header names. +#[derive(Clone, Debug, Default, Deserialize, Serialize, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CsvTrim { + /// Do not trim whitespace (default). + #[default] + None, + /// Trim whitespace from header names only. + Headers, + /// Trim whitespace from field values only. + Fields, + /// Trim whitespace from both header names and field values. + All, +} + +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema, PartialEq)] #[serde(default)] -pub struct CsvParserConfig { +pub struct CsvFormatConfig { /// Field delimiter (default `','`). /// - /// This must be an ASCII character. + /// Must be an ASCII character. + /// + /// Used by: input and output. pub delimiter: char, - /// Whether the input begins with a header line (which is ignored). + /// Whether the input begins with a header line (which is skipped). + /// + /// Used by: input only. pub headers: bool, + + /// The quote character (default `'"'`). + /// + /// Must be an ASCII character. Set `quoting` to `false` to disable + /// quoting entirely. + /// + /// Used by: input only. + pub quote: char, + + /// The escape character for quoted fields (default: `None`). + /// + /// When `None` (the default), the CSV parser uses the double-quote + /// convention: a literal quote inside a quoted field is written as two + /// consecutive quote characters. When set, the given character is used + /// as the escape prefix instead (e.g. `Some('\\')` for backslash + /// escaping). + /// + /// Must be an ASCII character. + /// + /// Used by: input only. + pub escape: Option, + + /// Enable double-quote escaping (default `true`). + /// + /// When `true`, a quote character inside a quoted field may be escaped by + /// doubling it. Setting this to `false` disables double-quote escaping + /// (an explicit `escape` character can still be used). + /// + /// Used by: input only. + pub double_quote: bool, + + /// Enable quoting (default `true`). + /// + /// When `false`, the `quote` and `escape` characters have no special + /// meaning and every newline terminates a record regardless of context. + /// + /// Used by: input only. + pub quoting: bool, + + /// Comment character (default: `None`). + /// + /// When set, lines whose first byte matches this character are treated as + /// comments and skipped entirely. Must be an ASCII character. + /// + /// Used by: input only. + pub comment: Option, + + /// Allow records with a variable number of fields (default `true`). + /// + /// When `true`, records that have fewer or more fields than expected are + /// accepted rather than treated as errors. + /// + /// Used by: input only. + pub flexible: bool, + + /// Whitespace trimming policy (default [`CsvTrim::None`]). + /// + /// Used by: input only. + pub trim: CsvTrim, } -impl CsvParserConfig { +impl CsvFormatConfig { pub fn delimiter(&self) -> CsvDelimiter { self.delimiter.into() } } -impl Default for CsvParserConfig { +impl Default for CsvFormatConfig { fn default() -> Self { Self { delimiter: CsvDelimiter::default().0.into(), headers: false, + quote: '"', + escape: None, + double_quote: true, + quoting: true, + comment: None, + flexible: true, + trim: CsvTrim::None, } } } diff --git a/crates/iceberg/src/test/requirements.ci.txt b/crates/iceberg/src/test/requirements.ci.txt index 34810ec9daa..2ae2a45c01e 100644 --- a/crates/iceberg/src/test/requirements.ci.txt +++ b/crates/iceberg/src/test/requirements.ci.txt @@ -1,7 +1,7 @@ # Only include requirements used in CI (no s3 or glue pyiceberg features). numpy==2.2.0 pandas==2.2.3 -pyarrow==17.0.0 -pyiceberg[sql-sqlite]==0.8.1 +pyarrow==23.0.1 +pyiceberg[sql-sqlite,pyiceberg-core]==0.11.1 # Transitive dependency via pyiceberg. -pydantic==2.11.10 +pydantic==2.13.4 diff --git a/crates/iceberg/src/test/requirements.txt b/crates/iceberg/src/test/requirements.txt index 075c265c13c..4c375ca5ce5 100644 --- a/crates/iceberg/src/test/requirements.txt +++ b/crates/iceberg/src/test/requirements.txt @@ -1,6 +1,6 @@ numpy==2.2.0 pandas==2.2.3 -pyarrow==17.0.0 -pyiceberg[s3fs,glue,sql-sqlite]==0.8.1 +pyarrow==23.0.1 +pyiceberg[s3fs,glue,sql-sqlite,pyiceberg-core]==0.11.1 # Transitive dependency via pyiceberg. -pydantic==2.11.10 +pydantic==2.13.4 diff --git a/crates/sqllib/src/map.rs b/crates/sqllib/src/map.rs index bae0e80ebda..fc4b07e2506 100644 --- a/crates/sqllib/src/map.rs +++ b/crates/sqllib/src/map.rs @@ -313,3 +313,23 @@ where } some_function1!(map_values [I: Ord + Clone, T: Clone], Map, Array); + +#[doc(hidden)] +pub fn map_concat__(left: Map, right: Map) -> Map +where + I: Ord + Clone, + T: Clone, +{ + if right.is_empty() { + return left; + } + if left.is_empty() { + return right; + } + let mut out = BTreeMap::new(); + out.extend(left.iter().map(|(k, v)| (k.clone(), v.clone()))); + out.extend(right.iter().map(|(k, v)| (k.clone(), v.clone()))); + out.into() +} + +some_function2!(map_concat [I: Ord + Clone, T: Clone], Map, Map, Map); diff --git a/docs.feldera.com/docs/formats/csv.md b/docs.feldera.com/docs/formats/csv.md index 8fb320799a7..3a560691e29 100644 --- a/docs.feldera.com/docs/formats/csv.md +++ b/docs.feldera.com/docs/formats/csv.md @@ -177,4 +177,15 @@ create table FAILED_BANKS ( ``` - `delimiter`: A single character that delimits fields. The default is `","`. -- `headers`: Whether the first line of input is a header line. If this is set to true, Feldera ignores the first line. This applies only to input connectors (Feldera never writes a header line for CSV output). The default is `false`. +- `headers`: Whether the first line of input is a header line. If this is set to `true`, Feldera ignores the first line. This applies only to input connectors (Feldera never writes a header line for CSV output). The default is `false`. +- `quote`: The quote character used to wrap fields that contain the delimiter or newlines. Must be an ASCII character. The default is `'"'`. Set `quoting` to `false` to disable quoting entirely. +- `escape`: An escape character that can precede a quote inside a quoted field to produce a literal quote byte (e.g., `"\\"` for backslash). When unset (the default), the `double_quote` convention is used instead. +- `double_quote`: When `true` (the default), a quote character inside a quoted field may be escaped by doubling it (e.g., `""` represents a single `"`). Setting this to `false` disables that convention; use `escape` to escape quotes in that case. +- `quoting`: When `true` (the default), the `quote` and `escape` characters have their special meaning. When `false`, every newline terminates a record regardless of context, and neither `quote` nor `escape` is treated specially. +- `comment`: A single ASCII character that marks a line as a comment. Lines whose first byte matches this character are skipped entirely. Unset by default. +- `flexible`: When `true` (the default), records may have fewer or more fields than expected. Missing fields receive empty values. When `false`, every record must have exactly the same number of fields as the first record; a mismatch is reported as an error. +- `trim`: Controls whitespace trimming. Allowed values: + - `"none"` (default) — no trimming. + - `"fields"` — strip leading and trailing ASCII whitespace from every field value. + - `"headers"` — strip whitespace from header names only (effective only when `headers` is `true`). + - `"all"` — strip whitespace from both field values and header names. diff --git a/docs.feldera.com/docs/interface/cli.md b/docs.feldera.com/docs/interface/cli.md index 46709d04a1e..53bb5eec13a 100644 --- a/docs.feldera.com/docs/interface/cli.md +++ b/docs.feldera.com/docs/interface/cli.md @@ -261,6 +261,15 @@ Enable storage for `p1`: fda set-config p1 storage true ``` +View and set the tags of `p1`: + +```bash +fda tags p1 # prints the tags as a comma-separated list +fda set-tags p1 prod,team-billing +fda set-tags p1 `fda tags p1`,qa # append to the existing tags +fda set-tags p1 # clear all tags +``` + Add Rust UDF code to `p1`: ```bash diff --git a/docs.feldera.com/docs/sql/function-index.md b/docs.feldera.com/docs/sql/function-index.md index 00b1ca87410..43d673ab4f0 100644 --- a/docs.feldera.com/docs/sql/function-index.md +++ b/docs.feldera.com/docs/sql/function-index.md @@ -163,6 +163,7 @@ * `MAKE_TIMESTAMP`: [datetime](datetime.md#make_timestamp) * `MAP` (aggregate): [aggregates](aggregates.md#map) * `MAP`: [map](map.md#map-literals) +* `MAP_CONCAT`: [map](map.md#map_concat) * `MAP_CONTAINS_KEY`: [map](map.md#map_contains_key) * `MAP_KEYS`: [map](map.md#map_keys) * `MAP_VALUES`: [map](map.md#map_values) diff --git a/docs.feldera.com/docs/sql/grammar.md b/docs.feldera.com/docs/sql/grammar.md index f3f10e90ef8..d6277f3d538 100644 --- a/docs.feldera.com/docs/sql/grammar.md +++ b/docs.feldera.com/docs/sql/grammar.md @@ -339,7 +339,7 @@ query | query MINUS [ DISTINCT ] query | query INTERSECT [ DISTINCT ] query } - [ ORDER BY orderItem [, orderItem ]* ] + [ ORDER BY { ALL [ ASC | DESC ] [ NULLS FIRST | NULLS LAST ] | orderItem [, orderItem]* } ] [ LIMIT [ start, ] { count | ALL } ] [ OFFSET start [ { ROW | ROWS } ] ] [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ONLY ] @@ -360,15 +360,25 @@ values : { VALUES | VALUE } expression [, expression ]* select - : SELECT [ hintComment ] [ ALL | DISTINCT ] - { projectItem [, projectItem ]* } + : SELECT [ hintComment ] [ ALL | DISTINCT [ ON '(' expression [ ',' expression ]* ')' ] ] { projectItem [, projectItem ]* } FROM tableExpression [ WHERE booleanExpression ] - [ GROUP BY [ ALL | DISTINCT ] { groupItem [, groupItem ]* } ] + [ GROUP BY { ALL | [ ALL | DISTINCT ] { groupItem [, groupItem ]* } } ] [ HAVING booleanExpression ] [ QUALIFY booleanExpression ] ``` +`DISTINCT ON` allows you to eliminate duplicate rows based on +specified expressions, keeping the first row in each group as +determined by the `ORDER BY` clause. When using `DISTINCT ON`, the +expressions in the `DISTINCT ON` clause must match a prefix of the +`ORDER BY` clause. + +`GROUP BY ALL` without any groupItem specified groups by every +expression in the `SELECT` clause that is not an aggregate function. +For example, `SELECT deptno, SUM(sal) FROM emp GROUP BY ALL` is +equivalent to `SELECT deptno, SUM(sal) FROM emp GROUP BY deptno`. + ``` tablePrimary @@ -406,6 +416,12 @@ orderItem : expression [ ASC | DESC ] [ NULLS FIRST | NULLS LAST ] ``` +`ORDER BY ALL` will order on all expressions of the `SELECT` +statement, in the order they appear. E.g., `SELECT x, y FROM t ORDER +BY ALL` is equivalent to `SELECT x, y FROM t ORDER BY x, y`. An +optional trailing `ASC/DESC/NULLS FIRST/NULLS LAST` applies to all +expressions. + ``` projectItem diff --git a/docs.feldera.com/docs/sql/map.md b/docs.feldera.com/docs/sql/map.md index 8240a485c9e..ac4deeea377 100644 --- a/docs.feldera.com/docs/sql/map.md +++ b/docs.feldera.com/docs/sql/map.md @@ -49,9 +49,10 @@ Comparison operations (`=`, `<>`, `!=`, `>`, `<`, `>=`, `<=`) can be applied to |------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------| | _map_`[`_key_`]` | Returns the element in the map with the specified key. If there is no such key, the result is `NULL`. | `MAP['x', 4, 'y', 3]['x']` => 4 | | `CARDINALITY`(map) | Returns the number of key-value pairs in the map. | `CARDINALITY(MAP['x', 4])` => 1 | +| `MAP_CONCAT`(map1, map2) | Returns a map with the elements in both maps; keys in the second map win. Returns `NULL` if any argument is `NULL` | `MAP_CONCAT(MAP['x', 4], MAP['y', 2])` => `MAP['x', 4, 'y', 2]` | | `MAP_CONTAINS_KEY`(map, key) | Returns true when the map has an item with the specified key; `NULL` if any argument is `NULL`. | `MAP_CONTAINS_KEY(MAP['x', 4], 'x')` => `true` | | `MAP_KEYS`(map) | Returns a sorted ARRAY of the appropriate type with all the keys of the MAP. | `MAP_KEYS(MAP['x', 4, 'y', 5])` => `['x', 'y']` | -| `MAP_VALUES`(map) | Returns an ARRAY of the appropriate type with all the values of the MAP. | `MAP_VALUES(MAP['x', 4, 'y', 5])` => `[4, 5]` | +| `MAP_VALUES`(map) | Returns an ARRAY of the appropriate type with all the values of the MAP sorted by the order of the keys. | `MAP_VALUES(MAP['x', 4, 'y', 5])` => `[4, 5]` | ## The `UNNEST` Operator diff --git a/docs.feldera.com/docs/sql/unsupported-operations.md b/docs.feldera.com/docs/sql/unsupported-operations.md index a4cfe21bff8..8d4018309e6 100644 --- a/docs.feldera.com/docs/sql/unsupported-operations.md +++ b/docs.feldera.com/docs/sql/unsupported-operations.md @@ -79,12 +79,10 @@ Several `MAP` functions are not yet implemented: | Function | Status | |----------|--------| -| `MAP_CONCAT` | Not supported | | `MAP_ENTRIES` | Not supported | | `MAP_FROM_ARRAYS` | Not supported | | `MAP_FROM_ENTRIES` | Not supported | | `STR_TO_MAP` | Not supported | -| Building a `MAP` from a subquery returning a pair of columns | Not supported | A list of supported MAP operations is available [here](./map.md). See [#1907](https://github.com/feldera/feldera/issues/1907). diff --git a/docs.feldera.com/package.json b/docs.feldera.com/package.json index 6b9917d1548..11af4fcbf2b 100644 --- a/docs.feldera.com/package.json +++ b/docs.feldera.com/package.json @@ -58,8 +58,11 @@ "serialize-javascript": "^7.0.5", "gray-matter/js-yaml": "3.14.2", "json-refs/js-yaml": "3.14.2", - "*/js-yaml": "4.1.1", + "*/js-yaml": "4.2.0", "yaml": "^1.10.3", + "dompurify": "3.4.11", + "http-proxy-middleware": "2.0.10", + "webpack-dev-server": "5.2.5", "path-to-regexp": "0.1.13", "brace-expansion": "1.1.13", "postcss": "8.5.10", @@ -68,6 +71,6 @@ "@babel/plugin-transform-modules-systemjs": "^7.29.4", "fast-uri": "^3.1.2", "qs": "6.15.2", - "ws": "8.20.1" + "ws": "8.21.0" } } diff --git a/docs.feldera.com/yarn.lock b/docs.feldera.com/yarn.lock index 87582d18320..c167d79edb9 100644 --- a/docs.feldera.com/yarn.lock +++ b/docs.feldera.com/yarn.lock @@ -4768,10 +4768,10 @@ domhandler@^5.0.2, domhandler@^5.0.3: dependencies: domelementtype "^2.3.0" -dompurify@^3.0.6: - version "3.4.10" - resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.4.10.tgz#96704295b4d8aeefcc8c7a90caa579b0ad69e46a" - integrity sha512-0xzNv0e7oYC6yyuOGZIABPM4qtg3QxLFniDNPP4ZP90wR8Yq3zgwpRbrNiT4N3IKqDbbYFEJLV+JWEs19aZ//w== +dompurify@3.4.11, dompurify@^3.0.6: + version "3.4.11" + resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.4.11.tgz#29c8ba496475f279ef4015784068452fb14a0680" + integrity sha512-zhlUV12GsaRzMsf9q5M254YhA4+VuF0fG+QFqu6aYpoGlKtz+w8//jBcGVYBgQkR5GHjUomejY84AV+/uPbWdw== optionalDependencies: "@types/trusted-types" "^2.0.7" @@ -5894,10 +5894,10 @@ http-parser-js@>=0.5.1: resolved "https://registry.yarnpkg.com/http-parser-js/-/http-parser-js-0.5.10.tgz#b3277bd6d7ed5588e20ea73bf724fcbe44609075" integrity sha512-Pysuw9XpUq5dVc/2SMHpuTY01RFl8fttgcyunjL7eEMhGM3cI4eOmiCycJDVCo/7O7ClfQD3SaI6ftDzqOXYMA== -http-proxy-middleware@^2.0.9: - version "2.0.9" - resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz#e9e63d68afaa4eee3d147f39149ab84c0c2815ef" - integrity sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q== +http-proxy-middleware@2.0.10, http-proxy-middleware@^2.0.9: + version "2.0.10" + resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.10.tgz#b2df7b705203d7a8c269ac8450cf96b00c532f94" + integrity sha512-RKzRWNPxUZqbuk3BC5mGVJbBnWgr+diEnjJexIOytFbBzDy88Fbh/YvBr3DsNrl1jYAfjWfpATEv0NO35FDuPQ== dependencies: "@types/http-proxy" "^1.17.8" http-proxy "^1.18.1" @@ -6329,7 +6329,14 @@ js-yaml@3.14.2, js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" -js-yaml@4.1.0, js-yaml@4.1.1, js-yaml@^4.1.0: +js-yaml@4.1.0, js-yaml@4.2.0, js-yaml@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.2.0.tgz#2bd9e85682dd91bd469afb809d816043b3d49524" + integrity sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw== + dependencies: + argparse "^2.0.1" + +js-yaml@4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.1.tgz#854c292467705b699476e1a2decc0c8a3458806b" integrity sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA== @@ -10423,10 +10430,10 @@ webpack-dev-middleware@^7.4.2: range-parser "^1.2.1" schema-utils "^4.0.0" -webpack-dev-server@^5.2.2: - version "5.2.4" - resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-5.2.4.tgz#6e6306ce59848ed322c235e48b326632b1eed6d6" - integrity sha512-GqDPGZN9bRqKBTkp4aWkobDDHMsrXKoGSdOH56smIri8qR0JG8gfL8/v/f/OZR3/OKXjG8uwJbFVhKm/FNU/UA== +webpack-dev-server@5.2.5, webpack-dev-server@^5.2.2: + version "5.2.5" + resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-5.2.5.tgz#648fceaac6a5736b0935e5c1e55d6aa1d0626119" + integrity sha512-4wZtCquSuv9CKX8oybo+mqxtxZqWz47uM1Ch94lxowBztOhWCbhqvRbfC/mODOwxgV2brY+JGZpHq58/SuVFYg== dependencies: "@types/bonjour" "^3.5.13" "@types/connect-history-api-fallback" "^1.5.4" @@ -10599,10 +10606,10 @@ write-file-atomic@^3.0.3: signal-exit "^3.0.2" typedarray-to-buffer "^3.1.5" -ws@8.20.1, ws@^7.3.1, ws@^8.18.0: - version "8.20.1" - resolved "https://registry.yarnpkg.com/ws/-/ws-8.20.1.tgz#91a9ae2b312ccf98e0a85ec499b48cef45ab0ddb" - integrity sha512-It4dO0K5v//JtTXuPkfEOaI3uUN87iYPnqo/ZzqCoG3g8uhA66QUMs/SrM0YK7/NAu+r4LMh/9dq2A7k+rHs+w== +ws@8.21.0, ws@^7.3.1, ws@^8.18.0: + version "8.21.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.21.0.tgz#012e413fc07429945121b0c153158c4343086951" + integrity sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g== wsl-utils@^0.1.0: version "0.1.0" diff --git a/js-packages/web-console/src/assets/fonts/feldera-material-icons.css b/js-packages/web-console/src/assets/fonts/feldera-material-icons.css index 8284f29ff7b..22b32d51e98 100644 --- a/js-packages/web-console/src/assets/fonts/feldera-material-icons.css +++ b/js-packages/web-console/src/assets/fonts/feldera-material-icons.css @@ -2,8 +2,8 @@ font-family: "feldera-material-icons"; font-display: block; src: - url("feldera-material-icons.woff2?7e3c6a8d23bb66b7a89348ec07b22a96") format("woff2"), - url("feldera-material-icons.woff?7e3c6a8d23bb66b7a89348ec07b22a96") format("woff"); + url("feldera-material-icons.woff2?06f8372fd6db29f5f189dc16b2f9c8af") format("woff2"), + url("feldera-material-icons.woff?06f8372fd6db29f5f189dc16b2f9c8af") format("woff"); } .fd { @@ -20,201 +20,201 @@ .fd-chevron-down:before { content: "\f101"; } -.fd-arrow-down:before { +.fd-square:before { content: "\f102"; } -.fd-pencil-line:before { +.fd-wine:before { content: "\f103"; } -.fd-save:before { +.fd-trash-2:before { content: "\f104"; } -.fd-more_horiz:before { +.fd-book-open:before { content: "\f105"; } -.fd-menu:before { +.fd-moon:before { content: "\f106"; } -.fd-panel-right:before { +.fd-circle-x:before { content: "\f107"; } -.fd-dot:before { +.fd-mic-off:before { content: "\f108"; } -.fd-chevron-left:before { +.fd-plus:before { content: "\f109"; } -.fd-server-off:before { +.fd-circle-play:before { content: "\f10a"; } -.fd-circle-x:before { +.fd-chevron-left:before { content: "\f10b"; } -.fd-server-cog:before { +.fd-triangle-alert:before { content: "\f10c"; } -.fd-log-out:before { +.fd-paintbrush:before { content: "\f10d"; } -.fd-copy:before { +.fd-circle-stop:before { content: "\f10e"; } -.fd-file-down:before { +.fd-chevron-right:before { content: "\f10f"; } -.fd-play:before { +.fd-mic:before { content: "\f110"; } -.fd-message-circle-question:before { +.fd-square-power:before { content: "\f111"; } -.fd-chevron-right:before { +.fd-circle-check-big:before { content: "\f112"; } -.fd-lock-open:before { +.fd-circle-dot:before { content: "\f113"; } -.fd-lightbulb:before { +.fd-panel-right:before { content: "\f114"; } -.fd-key:before { +.fd-user:before { content: "\f115"; } -.fd-circle-play:before { +.fd-circle-stop-locked:before { content: "\f116"; } -.fd-square-power:before { +.fd-sliders-horizontal:before { content: "\f117"; } -.fd-circle-stop:before { +.fd-panel-left:before { content: "\f118"; } -.fd-network:before { +.fd-pencil-line:before { content: "\f119"; } -.fd-square:before { +.fd-info:before { content: "\f11a"; } -.fd-panel-bottom:before { +.fd-database-off:before { content: "\f11b"; } -.fd-database-off:before { +.fd-sun:before { content: "\f11c"; } .fd-file-text:before { content: "\f11d"; } -.fd-book-marked:before { +.fd-panel-bottom:before { content: "\f11e"; } -.fd-settings:before { +.fd-x:before { content: "\f11f"; } .fd-layout-panel-top:before { content: "\f120"; } -.fd-circle-user:before { +.fd-settings:before { content: "\f121"; } -.fd-eraser:before { +.fd-play:before { content: "\f122"; } -.fd-circle-pause:before { +.fd-rocket:before { content: "\f123"; } -.fd-trash-2:before { +.fd-server:before { content: "\f124"; } -.fd-plus:before { +.fd-circle-plus:before { content: "\f125"; } -.fd-moon:before { +.fd-more_horiz:before { content: "\f126"; } -.fd-mic-off:before { +.fd-key:before { content: "\f127"; } -.fd-wine:before { +.fd-copy:before { content: "\f128"; } -.fd-sun:before { +.fd-arrow-down:before { content: "\f129"; } -.fd-database:before { +.fd-circle-pause:before { content: "\f12a"; } -.fd-paintbrush:before { +.fd-menu:before { content: "\f12b"; } -.fd-circle-check-big:before { +.fd-lightbulb:before { content: "\f12c"; } -.fd-layout-panel-left:before { +.fd-server-cog:before { content: "\f12d"; } -.fd-arrow-right:before { +.fd-save:before { content: "\f12e"; } -.fd-x:before { +.fd-database:before { content: "\f12f"; } -.fd-circle-slash:before { +.fd-lock-open:before { content: "\f130"; } -.fd-panel-left:before { +.fd-log-out:before { content: "\f131"; } -.fd-circle-dot:before { +.fd-layout-panel-left:before { content: "\f132"; } -.fd-pause:before { +.fd-server-off:before { content: "\f133"; } -.fd-triangle-alert:before { +.fd-circle-slash:before { content: "\f134"; } -.fd-circle-divide:before { +.fd-arrow-right:before { content: "\f135"; } -.fd-info:before { +.fd-construction:before { content: "\f136"; } -.fd-construction:before { +.fd-file-down:before { content: "\f137"; } -.fd-rocket:before { +.fd-circle-divide:before { content: "\f138"; } -.fd-heart-crack:before { +.fd-network:before { content: "\f139"; } -.fd-circle-plus:before { +.fd-pause:before { content: "\f13a"; } -.fd-server:before { +.fd-copy-plus:before { content: "\f13b"; } -.fd-circle-stop-locked:before { +.fd-book-marked:before { content: "\f13c"; } -.fd-receipt-text:before { +.fd-dot:before { content: "\f13d"; } -.fd-check:before { +.fd-circle-user:before { content: "\f13e"; } -.fd-book-open:before { +.fd-circle-alert:before { content: "\f13f"; } -.fd-user:before { +.fd-message-circle-question:before { content: "\f140"; } -.fd-sliders-horizontal:before { +.fd-receipt-text:before { content: "\f141"; } -.fd-mic:before { +.fd-eraser:before { content: "\f142"; } -.fd-circle-alert:before { +.fd-check:before { content: "\f143"; } diff --git a/js-packages/web-console/src/assets/fonts/feldera-material-icons.svg b/js-packages/web-console/src/assets/fonts/feldera-material-icons.svg index 2e25e399e40..88d9035756f 100644 --- a/js-packages/web-console/src/assets/fonts/feldera-material-icons.svg +++ b/js-packages/web-console/src/assets/fonts/feldera-material-icons.svg @@ -13,402 +13,402 @@ - - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + - - - - - - - + + - - + + - - - - - - - + horiz-adv-x="128" d="M60.16 122.4373333333333C52.3573333333333 121.952 45.3546666666667 120.064 38.08 116.48C31.904 113.44 27.4186666666667 110.2293333333333 22.5973333333333 105.4026666666667C17.7706666666667 100.5813333333333 14.56 96.096 11.52 89.92C7.824 82.4213333333333 6.0533333333333 75.7013333333333 5.5466666666667 67.2693333333333C4.9333333333333 57.1413333333333 6.928 47.392 11.52 38.08C14.56 31.904 17.7706666666667 27.4186666666667 22.5973333333333 22.5973333333333C27.4133333333333 17.7813333333333 31.888 14.576 38.08 11.5093333333333C46.4693333333333 7.36 54.464 5.5093333333333 64 5.5093333333333C73.536 5.5093333333333 81.5306666666667 7.36 89.92 11.5093333333333C96.112 14.576 100.5866666666667 17.7813333333333 105.4026666666667 22.5973333333333C110.2186666666667 27.4133333333333 113.424 31.888 116.4906666666667 38.08C120.64 46.4693333333333 122.4906666666667 54.464 122.4906666666667 64C122.4906666666667 73.536 120.64 81.5306666666667 116.4906666666667 89.92C113.424 96.112 110.2186666666667 100.5866666666667 105.4026666666667 105.4026666666667C100.5813333333333 110.2293333333333 96.096 113.44 89.92 116.48C82.448 120.1653333333333 75.728 121.9413333333333 67.3173333333333 122.4586666666667C63.8026666666667 122.672 63.9786666666667 122.6773333333333 60.16 122.4373333333333M69.2906666666667 111.7706666666667C87.872 109.7226666666667 103.808 96.7626666666667 109.6586666666667 78.9333333333333C113.3706666666667 67.6213333333333 112.6986666666667 55.1893333333333 107.776 44.2133333333333C103.856 35.4666666666667 96.848 27.648 88.4266666666667 22.6346666666667C76.7253333333333 15.664 62.0906666666667 14.064 49.0666666666667 18.3413333333333C34.5706666666667 23.0986666666667 23.0986666666667 34.576 18.336 49.072C16.7306666666667 53.9626666666667 16 58.6293333333333 16 64C16 67.8133333333333 16.2613333333333 70.4426666666667 16.992 73.9146666666667C19.7386666666667 86.9866666666667 27.9893333333333 98.464 39.616 105.392C45.0613333333333 108.6346666666667 52.2133333333333 111.056 58.4533333333333 111.7653333333333C60.8 112.032 66.9013333333333 112.032 69.2906666666667 111.7706666666667M46.9333333333333 85.136C45.3653333333333 84.816 44.064 83.792 43.232 82.2453333333333C42.7733333333333 81.3866666666667 42.7733333333333 81.3813333333334 42.7733333333333 64C42.7733333333333 46.6186666666667 42.7733333333333 46.6133333333333 43.232 45.7546666666667C43.8133333333333 44.6666666666667 44.6666666666667 43.8133333333333 45.7546666666667 43.232C46.6133333333333 42.7733333333333 46.6186666666667 42.7733333333333 64 42.7733333333333C81.3813333333333 42.7733333333333 81.3866666666667 42.7733333333333 82.2453333333333 43.232C83.3333333333333 43.8133333333333 84.1866666666667 44.6666666666667 84.768 45.7546666666667C85.2266666666667 46.6133333333333 85.2266666666667 46.6186666666667 85.2266666666667 64C85.2266666666667 81.3813333333334 85.2266666666667 81.3866666666667 84.768 82.2453333333333C84.1866666666667 83.3333333333333 83.3333333333333 84.1866666666667 82.2453333333333 84.768C81.3866666666667 85.2266666666667 81.3386666666667 85.2266666666667 64.5333333333333 85.2586666666667C55.264 85.2746666666667 47.344 85.2213333333333 46.9333333333333 85.136M74.6666666666667 64L74.6666666666667 53.3333333333333L64 53.3333333333333L53.3333333333333 53.3333333333333L53.3333333333333 64L53.3333333333333 74.6666666666667L64 74.6666666666667L74.6666666666667 74.6666666666667L74.6666666666667 64" /> + - - - - - - - - + + - + + + + + + + + + + + + - - + + - - + + - - - + horiz-adv-x="128" d="M60.16 122.4373333333333C48.3626666666667 121.7066666666667 36.6826666666667 117.1786666666667 27.312 109.712C24.704 107.632 20.368 103.296 18.288 100.688C13.088 94.16 9.0293333333333 85.904 7.0666666666667 77.8346666666667C4.9173333333333 69.0346666666667 4.9173333333333 58.9653333333333 7.0666666666667 50.1653333333333C9.0293333333333 42.096 13.088 33.84 18.288 27.312C20.368 24.704 24.704 20.368 27.312 18.288C32.192 14.4 39.0506666666667 10.608 44.7253333333333 8.6666666666667C59.7493333333333 3.52 75.8186666666667 4.5333333333333 89.92 11.5093333333333C96.112 14.576 100.5866666666667 17.7813333333333 105.4026666666667 22.5973333333333C110.2186666666667 27.4133333333333 113.424 31.888 116.4906666666667 38.08C120.64 46.4693333333333 122.4906666666667 54.464 122.4906666666667 64C122.4906666666667 73.536 120.64 81.5306666666667 116.4906666666667 89.92C113.424 96.112 110.2186666666667 100.5866666666667 105.4026666666667 105.4026666666667C100.5813333333333 110.2293333333333 96.096 113.44 89.92 116.48C82.448 120.1653333333333 75.728 121.9413333333333 67.3173333333333 122.4586666666667C63.8026666666667 122.672 63.9786666666667 122.6773333333333 60.16 122.4373333333333M69.2906666666667 111.7706666666667C87.872 109.7226666666667 103.808 96.7626666666667 109.6586666666667 78.9333333333333C113.3706666666667 67.6213333333333 112.6986666666667 55.1893333333333 107.776 44.2133333333333C103.856 35.4666666666667 96.848 27.648 88.4266666666667 22.6346666666667C76.7253333333333 15.664 62.0906666666667 14.064 49.0666666666667 18.3413333333333C34.5706666666667 23.0986666666667 23.0986666666667 34.576 18.336 49.072C16.7306666666667 53.9626666666667 16 58.6293333333333 16 64C16 67.8133333333333 16.2613333333333 70.4426666666667 16.992 73.9146666666667C19.7386666666667 86.9866666666667 27.9893333333333 98.464 39.616 105.392C45.0613333333333 108.6346666666667 52.2133333333333 111.056 58.4533333333333 111.7653333333333C60.8 112.032 66.9013333333333 112.032 69.2906666666667 111.7706666666667M62.9333333333333 90.4693333333333C58.9013333333333 89.6426666666667 57.3813333333333 84.544 60.2933333333333 81.6266666666667C61.7706666666667 80.1546666666667 64.4 79.6746666666667 66.24 80.5493333333333C67.9946666666667 81.3813333333334 69.3333333333333 83.4506666666667 69.3333333333333 85.3333333333333C69.3333333333333 87.1786666666667 67.9893333333333 89.3013333333333 66.336 90.064C65.4133333333333 90.4906666666667 63.9093333333333 90.6666666666667 62.9333333333333 90.4693333333333M62.9333333333333 69.136C61.3706666666667 68.816 60.064 67.792 59.232 66.2453333333333C58.7786666666667 65.3973333333333 58.7733333333333 65.2586666666667 58.7733333333333 53.3333333333333C58.7733333333333 41.408 58.7786666666667 41.2693333333333 59.232 40.4213333333333C60.2773333333333 38.4693333333333 61.8773333333333 37.4826666666667 64 37.4826666666667C66.1226666666667 37.4826666666667 67.7226666666667 38.4693333333333 68.768 40.4213333333333C69.2213333333333 41.2693333333333 69.2266666666667 41.408 69.2266666666667 53.3333333333333C69.2266666666667 65.2586666666667 69.2213333333333 65.3973333333333 68.768 66.2453333333333C67.584 68.4533333333334 65.248 69.6106666666667 62.9333333333333 69.136" /> + + + - - - + + - + horiz-adv-x="128" d="M29.9573333333333 100.816C28.944 100.3786666666667 27.7066666666667 99.1893333333333 27.28 98.24C26.8053333333333 97.2 26.7573333333333 95.008 27.184 93.9786666666667C27.5786666666667 93.04 30.0053333333333 90.5173333333333 44.4693333333333 76L56.4266666666667 64L44.4693333333333 52C29.8613333333333 37.344 27.5786666666667 34.96 27.1626666666667 33.968C26.7413333333333 32.9653333333334 26.752 31.0453333333333 27.184 30.032C27.6053333333333 29.04 28.672 27.808 29.472 27.3973333333333C30.48 26.8746666666667 32 26.6773333333333 33.136 26.9173333333333C33.8666666666667 27.072 34.5386666666667 27.4506666666667 35.4826666666667 28.2293333333333C36.208 28.832 42.9226666666667 35.4133333333333 50.4 42.8533333333333L64 56.384L77.6 42.8533333333333C85.0826666666667 35.4133333333333 91.792 28.832 92.5173333333333 28.2293333333333C94.0533333333333 26.96 95.28 26.5813333333333 96.9013333333333 26.8906666666667C98.2453333333333 27.1413333333333 98.9973333333333 27.552 99.808 28.4746666666667C100.7413333333333 29.536 101.152 30.6186666666667 101.1573333333333 32.0106666666667C101.168 34.3893333333333 101.856 33.6106666666667 83.424 52.1066666666667L71.5733333333333 64L83.424 75.8933333333334C98.08 90.6026666666667 100.4266666666667 93.0453333333333 100.8373333333333 94.0266666666667C101.3173333333333 95.1786666666667 101.2533333333333 96.9973333333333 100.688 98.2026666666667C99.8026666666667 100.0906666666667 98.08 101.12 95.808 101.12C93.5733333333333 101.12 94.2933333333333 101.76 76 83.5306666666667L64 71.5733333333333L52.32 83.216C36.6453333333333 98.832 34.7786666666667 100.6133333333334 33.7706666666667 100.8906666666667C32.6613333333333 101.2 30.752 101.1626666666667 29.9573333333333 100.816" /> + - - - + + - - + + - - + + - - + + - - + + - - - - + + + + - - + + - - + + + + + + + + + + + + - - - - + + + + - - - - + + - - - - - - - - - - - - + horiz-adv-x="128" d="M61.9573333333333 106.1493333333333C60.944 105.712 59.7066666666667 104.5226666666667 59.28 103.5733333333333C58.8053333333333 102.528 58.7573333333333 100.336 59.184 99.312C59.5733333333333 98.3786666666667 62.064 95.792 76.544 81.2586666666667L88.3626666666667 69.3973333333333L56.7786666666667 69.2853333333333L25.1946666666667 69.1733333333334L24.2773333333333 68.6346666666667C22.3573333333333 67.4986666666667 21.5466666666667 66.128 21.5466666666667 64C21.5466666666667 61.7973333333333 22.5386666666667 60.224 24.5813333333333 59.1786666666667L25.584 58.6666666666667L57.008 58.6666666666667L88.4266666666667 58.6666666666667L76.4693333333333 46.6666666666667C61.8613333333333 32.0106666666667 59.5786666666667 29.6266666666667 59.1626666666667 28.6346666666667C58.7413333333333 27.632 58.752 25.712 59.184 24.6986666666667C59.6746666666667 23.5413333333333 60.7786666666667 22.384 61.84 21.9146666666667C62.976 21.4133333333333 64.88 21.36 65.9306666666667 21.808C67.728 22.5706666666667 105.4186666666667 60.224 106.16 62.0053333333333C106.5386666666667 62.9066666666667 106.5386666666667 65.0933333333333 106.1653333333333 65.9893333333333C105.7013333333333 67.0986666666667 87.136 86.0053333333333 73.3386666666667 99.424C67.9786666666667 104.6346666666667 66.4 106.0533333333333 65.7653333333333 106.2293333333333C64.6613333333333 106.5333333333333 62.7466666666667 106.496 61.9573333333333 106.1493333333333" /> + - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + - - - - - - - - - - diff --git a/js-packages/web-console/src/assets/fonts/feldera-material-icons.ttf b/js-packages/web-console/src/assets/fonts/feldera-material-icons.ttf index 2b05d6f7f9c..8b1d7fa9769 100644 Binary files a/js-packages/web-console/src/assets/fonts/feldera-material-icons.ttf and b/js-packages/web-console/src/assets/fonts/feldera-material-icons.ttf differ diff --git a/js-packages/web-console/src/assets/fonts/feldera-material-icons.woff b/js-packages/web-console/src/assets/fonts/feldera-material-icons.woff index ff6b30eee3a..3e552c9e818 100644 Binary files a/js-packages/web-console/src/assets/fonts/feldera-material-icons.woff and b/js-packages/web-console/src/assets/fonts/feldera-material-icons.woff differ diff --git a/js-packages/web-console/src/assets/fonts/feldera-material-icons.woff2 b/js-packages/web-console/src/assets/fonts/feldera-material-icons.woff2 index 929b374f02b..199c0aa8dd6 100644 Binary files a/js-packages/web-console/src/assets/fonts/feldera-material-icons.woff2 and b/js-packages/web-console/src/assets/fonts/feldera-material-icons.woff2 differ diff --git a/js-packages/web-console/src/assets/fonts/generic-icons.css b/js-packages/web-console/src/assets/fonts/generic-icons.css index c2124d9fa4d..34ae10699cb 100644 --- a/js-packages/web-console/src/assets/fonts/generic-icons.css +++ b/js-packages/web-console/src/assets/fonts/generic-icons.css @@ -2,8 +2,8 @@ font-family: "generic-icons"; font-display: block; src: - url("generic-icons.woff2?204a1f3771f46f1024a2c650b43800c5") format("woff2"), - url("generic-icons.woff?204a1f3771f46f1024a2c650b43800c5") format("woff"); + url("generic-icons.woff2?93adf303867ea39c2b4e5835c473116a") format("woff2"), + url("generic-icons.woff?93adf303867ea39c2b4e5835c473116a") format("woff"); } .gc { @@ -17,18 +17,18 @@ vertical-align: top; } -.gc-boiling-flask:before { +.gc-layout-panel-right:before { content: "\f101"; } -.gc-http-get:before { +.gc-circle-solid:before { content: "\f102"; } -.gc-circle-solid:before { +.gc-loader-alt:before { content: "\f103"; } -.gc-loader-alt:before { +.gc-boiling-flask:before { content: "\f104"; } -.gc-layout-panel-right:before { +.gc-http-get:before { content: "\f105"; } diff --git a/js-packages/web-console/src/assets/fonts/generic-icons.svg b/js-packages/web-console/src/assets/fonts/generic-icons.svg index e89c317f776..b3269604155 100644 --- a/js-packages/web-console/src/assets/fonts/generic-icons.svg +++ b/js-packages/web-console/src/assets/fonts/generic-icons.svg @@ -7,36 +7,36 @@ units-per-em="128" ascent="128" descent="0" /> - - - - + horiz-adv-x="128" d="M91.818 92H36.1818C32.7818 92 30 89.2 30 85.7776V42.2224C30 38.8 32.7818 36 36.1818 36H91.818C95.218 36 98 38.8 98 42.2224V85.7776C98 89.2 95.218 92 91.818 92zM91.818 42.2224H36.1818V85.7776H91.818V42.2224zM74 82H88V45.3332H74V82z" /> + - + + - + horiz-adv-x="128" d="" /> + diff --git a/js-packages/web-console/src/assets/fonts/generic-icons.ttf b/js-packages/web-console/src/assets/fonts/generic-icons.ttf index 9f8f3f55f57..e9db6e59e2d 100644 Binary files a/js-packages/web-console/src/assets/fonts/generic-icons.ttf and b/js-packages/web-console/src/assets/fonts/generic-icons.ttf differ diff --git a/js-packages/web-console/src/assets/fonts/generic-icons.woff b/js-packages/web-console/src/assets/fonts/generic-icons.woff index 5de327d83e7..102d845addb 100644 Binary files a/js-packages/web-console/src/assets/fonts/generic-icons.woff and b/js-packages/web-console/src/assets/fonts/generic-icons.woff differ diff --git a/js-packages/web-console/src/assets/fonts/generic-icons.woff2 b/js-packages/web-console/src/assets/fonts/generic-icons.woff2 index 310e19f57ce..0a75081b4e8 100644 Binary files a/js-packages/web-console/src/assets/fonts/generic-icons.woff2 and b/js-packages/web-console/src/assets/fonts/generic-icons.woff2 differ diff --git a/js-packages/web-console/src/assets/icons/feldera-material-icons/copy-plus.svg b/js-packages/web-console/src/assets/icons/feldera-material-icons/copy-plus.svg new file mode 100755 index 00000000000..b22ed9668b7 --- /dev/null +++ b/js-packages/web-console/src/assets/icons/feldera-material-icons/copy-plus.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/js-packages/web-console/src/lib/components/dialogs/GenericDialog.svelte.spec.ts b/js-packages/web-console/src/lib/components/dialogs/GenericDialog.svelte.spec.ts index 34d2e8381bf..5d343fde28c 100644 --- a/js-packages/web-console/src/lib/components/dialogs/GenericDialog.svelte.spec.ts +++ b/js-packages/web-console/src/lib/components/dialogs/GenericDialog.svelte.spec.ts @@ -18,6 +18,10 @@ async function renderDialog( return render(GenericDialog, { content, ...opts }) } +const clickButton = (locator: ReturnType) => { + ;(locator.element() as HTMLButtonElement).click() +} + describe('GenericDialog.svelte', () => { describe('A. Basic rendering', () => { it('renders title', async () => { @@ -79,7 +83,7 @@ describe('GenericDialog.svelte', () => { onSuccess: { name: 'Confirm', callback: onSuccess, 'data-testid': 'btn-dialog-success' } }) ) - await page.getByTestId('btn-dialog-success').click() + clickButton(page.getByTestId('btn-dialog-success')) expect(onSuccess).toHaveBeenCalledOnce() }) @@ -149,7 +153,7 @@ describe('GenericDialog.svelte', () => { onCancel: { name: 'Back', callback: onCancel } }) ) - await page.getByTestId('btn-dialog-cancel').click() + clickButton(page.getByTestId('btn-dialog-cancel')) expect(onCancel).toHaveBeenCalledOnce() }) @@ -160,7 +164,7 @@ describe('GenericDialog.svelte', () => { onCancel: { callback: onCancel } }) ) - await page.getByLabelText('Close dialog').click() + ;(page.getByLabelText('Close dialog').element() as HTMLButtonElement).click() expect(onCancel).toHaveBeenCalledOnce() }) }) diff --git a/js-packages/web-console/src/lib/components/layout/pipelines/PipelineMemoryGraph.svelte b/js-packages/web-console/src/lib/components/layout/pipelines/PipelineMemoryGraph.svelte index aff4f84b66a..67c733ebce6 100644 --- a/js-packages/web-console/src/lib/components/layout/pipelines/PipelineMemoryGraph.svelte +++ b/js-packages/web-console/src/lib/components/layout/pipelines/PipelineMemoryGraph.svelte @@ -16,6 +16,7 @@ import { humanSize } from '$lib/functions/common/string' import { tuple } from '$lib/functions/common/tuple' import { multihostMemoryLimitMb, timeSeriesAxisMax } from '$lib/functions/pipelineMetrics' + import type { MemoryPressure } from '$lib/services/manager' import type { Pipeline } from '$lib/services/pipelineManager' import type { TimeSeriesEntry } from '$lib/types/pipelineManager' @@ -23,13 +24,23 @@ pipeline, metrics, refetchMs, - keepMs + keepMs, + memoryPressure }: { pipeline: { current: Pipeline } metrics: TimeSeriesEntry[] refetchMs: number keepMs: number + memoryPressure?: MemoryPressure } = $props() + + const pressureChips: Record = { + low: { label: '', class: 'hidden' }, + moderate: { label: 'Moderate pressure', class: 'bg-warning-100-900' }, + high: { label: 'High pressure', class: 'bg-error-50-950' }, + critical: { label: 'Critical pressure', class: 'bg-error-50-950' } + } + const pressureChip = $derived(memoryPressure ? pressureChips[memoryPressure] : undefined) use([ LineChart, GridComponent, @@ -209,8 +220,13 @@
-
- Used memory: {humanSize(metrics.at(-1)?.m ?? 0)} +
+ Used memory: {humanSize(metrics.at(-1)?.m ?? 0)} + {#if pressureChip} +
+ {pressureChip.label} +
+ {/if}
{#key pipelineName} (ref = init(dom, theme, opts))} {options} /> diff --git a/js-packages/web-console/src/lib/components/pipelines/List.svelte.spec.ts b/js-packages/web-console/src/lib/components/pipelines/List.svelte.spec.ts index d80f33b86ef..1ad1a1d52df 100644 --- a/js-packages/web-console/src/lib/components/pipelines/List.svelte.spec.ts +++ b/js-packages/web-console/src/lib/components/pipelines/List.svelte.spec.ts @@ -1,3 +1,4 @@ +import { tick } from 'svelte' import { describe, expect, it } from 'vitest' import { page } from 'vitest/browser' import { render } from 'vitest-browser-svelte' @@ -19,9 +20,18 @@ const visibleNames = () => .elements() .map((el) => el.textContent?.trim()) +const setSearch = async (value: string) => { + const search = page.getByPlaceholder('Search pipelines...') + await expect.element(search).toBeInTheDocument() + const input = search.element() as HTMLInputElement + input.value = value + input.dispatchEvent(new Event('input', { bubbles: true })) + await tick() +} + describe('pipeline list search', () => { it('lists every pipeline before any search term is entered', async () => { - render(List, { pipelineName: '', pipelines }) + await render(List, { pipelineName: '', pipelines }) await expect.element(page.getByText('orders')).toBeInTheDocument() await expect.element(page.getByText('payments')).toBeInTheDocument() @@ -30,9 +40,9 @@ describe('pipeline list search', () => { }) it('filters to pipelines whose name contains the search term', async () => { - render(List, { pipelineName: '', pipelines }) + await render(List, { pipelineName: '', pipelines }) - await page.getByPlaceholder('Search pipelines...').fill('pay') + await setSearch('pay') await expect.element(page.getByText('payments')).toBeInTheDocument() await expect.element(page.getByText('orders')).not.toBeInTheDocument() @@ -41,37 +51,36 @@ describe('pipeline list search', () => { }) it('matches case-insensitively', async () => { - render(List, { pipelineName: '', pipelines }) + await render(List, { pipelineName: '', pipelines }) - await page.getByPlaceholder('Search pipelines...').fill('PAY') + await setSearch('PAY') expect(visibleNames()).toEqual(['payments']) }) it('matches a substring anywhere in the name, not only a prefix', async () => { - render(List, { pipelineName: '', pipelines }) + await render(List, { pipelineName: '', pipelines }) - await page.getByPlaceholder('Search pipelines...').fill('detect') + await setSearch('detect') expect(visibleNames()).toEqual(['fraud-detection']) }) it('shows no pipelines when the term matches nothing', async () => { - render(List, { pipelineName: '', pipelines }) + await render(List, { pipelineName: '', pipelines }) - await page.getByPlaceholder('Search pipelines...').fill('no-such-pipeline') + await setSearch('no-such-pipeline') expect(visibleNames()).toHaveLength(0) }) it('restores the full list when the search term is cleared', async () => { - render(List, { pipelineName: '', pipelines }) - const search = page.getByPlaceholder('Search pipelines...') + await render(List, { pipelineName: '', pipelines }) - await search.fill('pay') + await setSearch('pay') expect(visibleNames()).toEqual(['payments']) - await search.fill('') + await setSearch('') expect(visibleNames()).toHaveLength(3) }) }) diff --git a/js-packages/web-console/src/lib/components/pipelines/Table.svelte b/js-packages/web-console/src/lib/components/pipelines/Table.svelte index ab8d571e223..8ec51e0d53c 100644 --- a/js-packages/web-console/src/lib/components/pipelines/Table.svelte +++ b/js-packages/web-console/src/lib/components/pipelines/Table.svelte @@ -6,6 +6,7 @@ import PipelineStatus from '$lib/components/pipelines/list/PipelineStatus.svelte' import ThSort from '$lib/components/pipelines/table/ThSort.svelte' import { useElapsedTime } from '$lib/compositions/common/useElapsedTime' + import { useLayoutSettings } from '$lib/compositions/layout/useLayoutSettings.svelte' import { dateMax } from '$lib/functions/common/date' import { matchesSubstring } from '$lib/functions/common/string' import { type NamesInUnion, unionName } from '$lib/functions/common/union' @@ -84,6 +85,15 @@ const { formatElapsedTime } = useElapsedTime() const td = 'py-1 text-base border-t-[0.5px]' + + // Persist the active sort and restore it on the matching column. Each ThSort + // owns the upstream `direction`/`onSort` API; the column identity lives here. + const { pipelinesTableSort } = useLayoutSettings() + const sortColumn = (column: string) => ({ + direction: + pipelinesTableSort.value.column === column ? pipelinesTableSort.value.direction : undefined, + onSort: (direction: 'asc' | 'desc') => (pipelinesTableSort.value = { column, direction }) + })
@@ -134,13 +144,13 @@ onclick={() => table.selectAll()} /> - Pipeline name Storage - Status p.connectors?.numErrors} + {...sortColumn('numErrors')} > Errors - + Runtime - Status changed - Deployed on diff --git a/js-packages/web-console/src/lib/components/pipelines/Table.svelte.spec.ts b/js-packages/web-console/src/lib/components/pipelines/Table.svelte.spec.ts new file mode 100644 index 00000000000..9982ac08175 --- /dev/null +++ b/js-packages/web-console/src/lib/components/pipelines/Table.svelte.spec.ts @@ -0,0 +1,152 @@ +/** + * Mouse interaction tests for column sorting in the pipelines `Table`. + * + * The real pipelines `Table` is mounted with a handful of pipeline thumbs whose name order + * and "status changed" order deliberately disagree, so every assertion about row + * order can only pass if the click actually re-sorted the rows. The default sort + * (name ascending) and its persistence both live in `useLayoutSettings`, so the + * test also checks the localStorage key the component writes through. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { page } from 'vitest/browser' +import { render } from 'vitest-browser-svelte' +import type { PipelineThumb } from '$lib/services/pipelineManager' +import { useLayoutSettings } from '$lib/compositions/layout/useLayoutSettings.svelte' + +const SORT_KEY = 'layout/pipelines/table/sort' + +// `vi.mock` factories are hoisted above the imports and run before ordinary +// module-scope consts initialize, so a factory can't read them. `vi.hoisted` is +// hoisted too, so its result is available to both the factory and the fixtures. +const { PLATFORM_VERSION } = vi.hoisted(() => ({ PLATFORM_VERSION: '1.0.0' })) + +// Table and its PipelineVersion child both read `page.data.feldera`. A `version` +// equal to every thumb's platformVersion keeps the runtime column in its +// "latest" state, so no update Popover is rendered. +vi.mock('$app/state', () => ({ + page: { data: { feldera: { version: PLATFORM_VERSION, unstableFeatures: [] } } } +})) + +import Table from './Table.svelte' + +// `lastStatusSince` is derived from these timestamps. The date order +// (bravo < delta < charlie < alpha) is a rotation of the name order +// (alpha < bravo < charlie < delta), so sorting by either column yields a +// distinct, unambiguous row sequence. +const lastChange: Record = { + alpha: '2024-01-04T00:00:00Z', + bravo: '2024-01-01T00:00:00Z', + charlie: '2024-01-03T00:00:00Z', + delta: '2024-01-02T00:00:00Z' +} + +// Only the fields the Table template reads are populated; the rest of +// PipelineThumb is irrelevant to sorting, hence the cast. +const thumb = (name: string): PipelineThumb => + ({ + name, + description: '', + status: 'Stopped', + storageStatus: 'Cleared', + deploymentStatusSince: lastChange[name], + programStatusSince: lastChange[name], + deploymentError: undefined, + platformVersion: PLATFORM_VERSION, + programConfig: { runtime_version: null }, + deploymentResourcesStatus: 'Stopped', + deploymentResourcesStatusSince: new Date(lastChange[name]), + connectors: { numErrors: 0 } + }) as unknown as PipelineThumb + +// Fed deliberately unsorted so a passing default-order assertion proves the +// table sorted them rather than echoing the input order. +const pipelines = [thumb('charlie'), thumb('alpha'), thumb('delta'), thumb('bravo')] + +// Each rendered row carries data-testid="box-row-"; reading them back in +// DOM order gives the visible row sequence. +const rowOrder = () => + Array.from(document.querySelectorAll('tbody tr[data-testid^="box-row-"]')).map((tr) => + tr.getAttribute('data-testid')!.slice('box-row-'.length) + ) + +// The sortable headers render their label inside the clickable ; the +// gains the `active` class while it is the column the table is sorted by. +const header = (label: string) => page.getByText(label, { exact: true }) +const headerCell = (label: string) => header(label).element().closest('th')! + +const persistedSort = () => JSON.parse(localStorage.getItem(SORT_KEY)!) + +const mountTable = () => render(Table, { props: { pipelines, selectedPipelines: [] } } as any) + +describe('Table — column sorting', () => { + beforeEach(() => { + // The persisted sort is a process-wide singleton (useLocalStorage caches by + // key), so reset both the backing store and the cached value before each test + // to keep them independent. + localStorage.clear() + useLayoutSettings().pipelinesTableSort.value = { column: 'name', direction: 'asc' } + }) + + afterEach(async () => { + // @vincjo/datatables' setRows() defers a scroll-position restore via + // setTimeout(..., 2) that dereferences table.element. On unmount Svelte nulls + // that binding, so a timer still in flight throws "Cannot set properties of + // null (setting 'scrollTop')". vitest-browser-svelte unmounts after afterEach + // runs, so waiting out the 2 ms window here lets the timer fire while the + // component — and its element — is still alive. + await new Promise((resolve) => setTimeout(resolve, 10)) + localStorage.clear() + }) + + it('sorts by pipeline name ascending by default', async () => { + mountTable() + + await expect.poll(rowOrder).toEqual(['alpha', 'bravo', 'charlie', 'delta']) + expect(headerCell('Pipeline name').classList.contains('active')).toBe(true) + }) + + it('clicking the name header toggles ascending → descending → ascending', async () => { + mountTable() + await expect.poll(rowOrder).toEqual(['alpha', 'bravo', 'charlie', 'delta']) + + await header('Pipeline name').click() + await expect.poll(rowOrder).toEqual(['delta', 'charlie', 'bravo', 'alpha']) + + await header('Pipeline name').click() + await expect.poll(rowOrder).toEqual(['alpha', 'bravo', 'charlie', 'delta']) + }) + + it('clicking a different header sorts by that column and moves the active marker', async () => { + mountTable() + await expect.poll(rowOrder).toEqual(['alpha', 'bravo', 'charlie', 'delta']) + + // "Status changed" sorts by lastStatusSince, ascending on first click. + await header('Status changed').click() + await expect.poll(rowOrder).toEqual(['bravo', 'delta', 'charlie', 'alpha']) + + expect(headerCell('Status changed').classList.contains('active')).toBe(true) + expect(headerCell('Pipeline name').classList.contains('active')).toBe(false) + }) + + it('persists the active sort to localStorage', async () => { + mountTable() + await expect.poll(rowOrder).toEqual(['alpha', 'bravo', 'charlie', 'delta']) + + await header('Pipeline name').click() + await expect.poll(persistedSort).toEqual({ column: 'name', direction: 'desc' }) + + await header('Status changed').click() + await expect.poll(persistedSort).toEqual({ column: 'lastStatusSince', direction: 'asc' }) + }) + + it('restores a persisted non-default sort on mount', async () => { + // Simulate a returning user whose last sort was name descending. + useLayoutSettings().pipelinesTableSort.value = { column: 'name', direction: 'desc' } + + mountTable() + + await expect.poll(rowOrder).toEqual(['delta', 'charlie', 'bravo', 'alpha']) + expect(headerCell('Pipeline name').classList.contains('active')).toBe(true) + }) +}) diff --git a/js-packages/web-console/src/lib/components/pipelines/editor/MonitoringPanel.svelte b/js-packages/web-console/src/lib/components/pipelines/editor/MonitoringPanel.svelte index c43d5d5d14f..e2acfa7bd6d 100755 --- a/js-packages/web-console/src/lib/components/pipelines/editor/MonitoringPanel.svelte +++ b/js-packages/web-console/src/lib/components/pipelines/editor/MonitoringPanel.svelte @@ -140,6 +140,13 @@ const connectorsWithErrorsCount = $derived(numConnectorsWithProblems(metrics.current)) + const memoryPressureErrorCount = $derived.by(() => { + const memoryPressure = metrics.current.global?.memory_pressure + return memoryPressure === 'high' || memoryPressure === 'critical' ? 1 : 0 + }) + + const runtimeErrorsCount = $derived(connectorsWithErrorsCount + memoryPressureErrorCount) + // Local input binding. The committed search (what the list actually runs) lives in // `logSearch` and only advances on Enter — so typing doesn't search as-you-type. let logSearchInput = $state('') @@ -187,9 +194,9 @@ {#snippet TabControlPerformance()} {@render TabPerformance.Label()} - {#if connectorsWithErrorsCount > 0} + {#if runtimeErrorsCount > 0} - {connectorsWithErrorsCount} + {runtimeErrorsCount} {/if} {/snippet} diff --git a/js-packages/web-console/src/lib/components/pipelines/editor/TabPerformance.svelte b/js-packages/web-console/src/lib/components/pipelines/editor/TabPerformance.svelte index b0952d05d4e..43a78e592cf 100644 --- a/js-packages/web-console/src/lib/components/pipelines/editor/TabPerformance.svelte +++ b/js-packages/web-console/src/lib/components/pipelines/editor/TabPerformance.svelte @@ -306,7 +306,12 @@ >
-
diff --git a/js-packages/web-console/src/lib/components/pipelines/editor/performance/CheckpointDialog.svelte.spec.ts b/js-packages/web-console/src/lib/components/pipelines/editor/performance/CheckpointDialog.svelte.spec.ts index e02008598ad..65ffbc6dc8c 100644 --- a/js-packages/web-console/src/lib/components/pipelines/editor/performance/CheckpointDialog.svelte.spec.ts +++ b/js-packages/web-console/src/lib/components/pipelines/editor/performance/CheckpointDialog.svelte.spec.ts @@ -3,6 +3,10 @@ import { page } from 'vitest/browser' import { render } from 'vitest-browser-svelte' import CheckpointDialog from './CheckpointDialog.svelte' +const clickButton = (locator: ReturnType) => { + ;(locator.element() as HTMLButtonElement).click() +} + describe('CheckpointDialog.svelte', () => { describe('A. Content', () => { it('renders the title', async () => { @@ -31,14 +35,14 @@ describe('CheckpointDialog.svelte', () => { it('calls onConfirm when Checkpoint is clicked', async () => { const onConfirm = vi.fn() await render(CheckpointDialog, { onConfirm }) - await page.getByTestId('btn-confirm-checkpoint').click() + clickButton(page.getByTestId('btn-confirm-checkpoint')) expect(onConfirm).toHaveBeenCalledOnce() }) it('does not call onConfirm when Cancel is clicked', async () => { const onConfirm = vi.fn() await render(CheckpointDialog, { onConfirm }) - await page.getByTestId('btn-dialog-cancel').click() + clickButton(page.getByTestId('btn-dialog-cancel')) expect(onConfirm).not.toHaveBeenCalled() }) }) diff --git a/js-packages/web-console/src/lib/components/pipelines/editor/performance/MetricsTables.svelte.spec.ts b/js-packages/web-console/src/lib/components/pipelines/editor/performance/MetricsTables.svelte.spec.ts index 8292fef09aa..b6ea301e951 100644 --- a/js-packages/web-console/src/lib/components/pipelines/editor/performance/MetricsTables.svelte.spec.ts +++ b/js-packages/web-console/src/lib/components/pipelines/editor/performance/MetricsTables.svelte.spec.ts @@ -402,13 +402,23 @@ describe('MetricsTables.svelte', () => { // Initially collapsed await expect.element(page.getByTestId('box-connector-row-alpha')).not.toBeInTheDocument() + const toggle = page.getByTestId('box-relation-row-t1').getByText('t1', { exact: true }) + + // `force` bypasses the pointer hit-test. Once expanded, the connector rows render + // icon tooltips (placement="top") that float up over this row; while the pointer + // travels toward the label it can open one, and the portaled tooltip's high z-index + // intercepts the hit-test so the click never resolves (flaky CI timeout on the + // second click). The toggle is wired to the row's onclick, so the forced dispatch + // still fires it. `position` keeps the dispatch over the chevron, clear of tooltips. + const click = { position: { x: 1, y: 1 }, force: true } + // Click to expand - await page.getByTestId('box-relation-row-t1').click({ position: { x: 1, y: 0 } }) + await toggle.click(click) await expect.element(page.getByTestId('box-connector-row-alpha')).toBeInTheDocument() await expect.element(page.getByTestId('box-connector-row-beta')).toBeInTheDocument() // Click again to collapse - await page.getByTestId('box-relation-row-t1').click({ position: { x: 1, y: 0 } }) + await toggle.click(click) await expect.element(page.getByTestId('box-connector-row-alpha')).not.toBeInTheDocument() }) diff --git a/js-packages/web-console/src/lib/components/pipelines/list/Actions.svelte b/js-packages/web-console/src/lib/components/pipelines/list/Actions.svelte index 14c95abdd03..448a2251f97 100644 --- a/js-packages/web-console/src/lib/components/pipelines/list/Actions.svelte +++ b/js-packages/web-console/src/lib/components/pipelines/list/Actions.svelte @@ -49,9 +49,14 @@ groups related actions into multi-action dropdowns when multiple options are ava import DeleteDialog, { deleteDialogProps } from '$lib/components/dialogs/DeleteDialog.svelte' import JSONDialog from '$lib/components/dialogs/JSONDialog.svelte' import PipelineConfigurationsPopup from '$lib/components/layout/pipelines/PipelineConfigurationsPopup.svelte' + import { duplicatePipeline, duplicatePipelineTooltip } from '$lib/compositions/duplicatePipeline' import { useGlobalDialog } from '$lib/compositions/layout/useGlobalDialog.svelte' import { useIsMobile } from '$lib/compositions/layout/useIsMobile.svelte' import { usePipelineActionCallbacks } from '$lib/compositions/pipelines/usePipelineActionCallbacks.svelte' + import { + usePipelineList, + useUpdatePipelineList + } from '$lib/compositions/pipelines/usePipelineList.svelte' import { getPipelineAction } from '$lib/compositions/usePipelineAction.svelte' import { usePipelineManager } from '$lib/compositions/usePipelineManager.svelte' import { usePremiumFeatures } from '$lib/compositions/usePremiumFeatures.svelte' @@ -84,11 +89,26 @@ groups related actions into multi-action dropdowns when multiple options are ava const globalDialog = useGlobalDialog() const api = usePipelineManager() + const pipelineList = usePipelineList() + const { discardPendingListRefresh, updatePipeline, updatePipelines } = useUpdatePipelineList() const deletePipeline = async (pipelineName: string) => { await api.deletePipeline(pipelineName) onDeletePipeline?.(pipelineName) goto(resolve('/')) } + const duplicateCurrentPipeline = async () => { + const pipelines = pipelineList.pipelines + if (!pipelines || unsavedChanges) { + return + } + + const newPipeline = await duplicatePipeline(api, pipeline.current, pipelines, { + discardPendingListRefresh, + updatePipeline, + updatePipelines + }) + await goto(resolve(`/pipelines/${encodeURIComponent(newPipeline.name)}/`)) + } const { toastError } = useToast() const actions = { @@ -105,7 +125,8 @@ groups related actions into multi-action dropdowns when multiple options are ava _stop, _multiStop, _multiStart, - _delete, + _duplicate, + _more, _spacer_short, _spacer_long, _spinner, @@ -135,7 +156,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('Preparing', 'Provisioning', 'Initializing', () => [ '_kill', @@ -143,7 +164,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('Pausing', 'Resuming', () => [ ...stopButtons, @@ -151,7 +172,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('Unavailable', () => [ ...stopButtons, @@ -159,7 +180,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('Running', () => [ ...stopButtons, @@ -167,7 +188,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('Paused', () => [ ...stopButtons, @@ -175,7 +196,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('Suspending', () => [ '_kill', @@ -183,7 +204,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('Suspended', () => [ '_spinner', @@ -191,7 +212,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('Standby', () => [ '_kill', @@ -199,7 +220,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('Bootstrapping', () => [ '_kill', @@ -207,7 +228,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('Replaying', () => [ '_kill', @@ -215,14 +236,14 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('AwaitingApproval', () => [ '_kill', '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with('Stopping', () => [ '_kill', @@ -230,7 +251,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .with( { Queued: P.any }, @@ -243,7 +264,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ] ) .with('SqlError', 'RustError', 'SystemError', () => [ @@ -251,7 +272,7 @@ groups related actions into multi-action dropdowns when multiple options are ava '_saveFile', '_configurations', '_storage_indicator', - '_delete' + '_more' ]) .exhaustive() } @@ -288,7 +309,7 @@ groups related actions into multi-action dropdowns when multiple options are ava const active = $derived.by(() => { const rawActions = getRawActions(pipeline.current.status) // Group actions into dropdowns - return processActionsForDropdowns(rawActions) + return [...processActionsForDropdowns(rawActions), '_duplicate' as const] }) function processActionsForDropdowns( @@ -637,18 +658,52 @@ groups related actions into multi-action dropdowns when multiple options are ava {/snippet} --> -{#snippet _delete()} -
- -
- {#if editConfigDisabled} - Stop the pipeline to delete it - {/if} +{#snippet _more()} + + {#snippet trigger(toggle)} + + {/snippet} + {#snippet content(close)} +
+ +
+ {/snippet} + +{/snippet} +{#snippet _duplicate()} + + + {#if unsavedChanges} + Save the program before duplicating. + {:else} + {duplicatePipelineTooltip} + {/if} + {/snippet} {#snippet start({ text, diff --git a/js-packages/web-console/src/lib/components/pipelines/table/AvailableActions.svelte b/js-packages/web-console/src/lib/components/pipelines/table/AvailableActions.svelte index 43f410c7c2d..cac7709db28 100644 --- a/js-packages/web-console/src/lib/components/pipelines/table/AvailableActions.svelte +++ b/js-packages/web-console/src/lib/components/pipelines/table/AvailableActions.svelte @@ -2,6 +2,7 @@ import { join } from 'array-join' import { match, P } from 'ts-pattern' import DeleteDialog, { deleteDialogProps } from '$lib/components/dialogs/DeleteDialog.svelte' + import { duplicatePipeline, duplicatePipelineTooltip } from '$lib/compositions/duplicatePipeline' import { useGlobalDialog } from '$lib/compositions/layout/useGlobalDialog.svelte' import { useUpdatePipelineList } from '$lib/compositions/pipelines/usePipelineList.svelte' import { getPipelineAction } from '$lib/compositions/usePipelineAction.svelte' @@ -15,9 +16,10 @@ pipelines, selectedPipelines = $bindable() }: { pipelines: PipelineThumb[]; selectedPipelines: string[] } = $props() - const { updatePipeline } = useUpdatePipelineList() + const { discardPendingListRefresh, updatePipeline, updatePipelines } = useUpdatePipelineList() const sortedSelectedPipelines = $derived([...selectedPipelines].sort()) const availableActions = [ + 'duplicate' as const, 'start' as const, 'resume' as const, 'pause' as const, @@ -26,12 +28,14 @@ 'delete' as const, 'clear' as const ] + type AvailableAction = (typeof availableActions)[number] + type StatusAction = Exclude const isPremium = usePremiumFeatures() const stop = isPremium.value ? ['stop' as const] : [] const statusActions = ({ status, storageStatus }: (typeof selected)[number]) => { const storageAction = storageStatus === 'InUse' ? ['clear' as const] : [] return match(status) - .returnType<(typeof availableActions)[number][]>() + .returnType() .with( { Queued: P.any }, { CompilingSql: P.any }, @@ -69,7 +73,7 @@ (_, p) => p ) ) - const eligibleFor = (action: (typeof availableActions)[number]) => + const eligibleFor = (action: StatusAction) => selected .filter((pipeline) => statusActions(pipeline).includes(action)) .map((p) => p.name) @@ -78,11 +82,15 @@ if (selected.length === 0) { return [] } - const supportedByAny = new Set(selected.flatMap(statusActions)) + const supportedByAny = new Set(selected.flatMap(statusActions)) + if (selected.length === 1) { + supportedByAny.add('duplicate') + } return availableActions .filter((action) => supportedByAny.has(action)) .map((action) => match(action) + .with('duplicate', () => btnDuplicate) .with('start', () => btnStart) .with('resume', () => btnResume) .with('pause', () => btnPause) @@ -96,7 +104,7 @@ const globalDialog = useGlobalDialog() const api = usePipelineManager() - const postPipelinesAction = (action: Exclude<(typeof availableActions)[number], 'delete'>) => { + const postPipelinesAction = (action: Exclude) => { selected .filter((pipeline) => statusActions(pipeline).includes(action)) .forEach((pipeline) => { @@ -128,6 +136,20 @@ }) selectedPipelines = [] } + + const duplicateSelectedPipeline = async () => { + const pipeline = selected[0] + if (!pipeline) { + return + } + + await duplicatePipeline(api, pipeline, pipelines, { + discardPendingListRefresh, + updatePipeline, + updatePipelines + }) + selectedPipelines = [] + } {#snippet btnStart()} @@ -166,6 +188,16 @@ Delete {/snippet} +{#snippet btnDuplicate()} + +{/snippet} {#snippet btnClear()}