diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a99f3c0effb..06fd09d9de1 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -84,6 +84,7 @@ RUN curl -fsSL https://claude.ai/install.sh | bash RUN \ rustup install $RUST_VERSION && \ + rustup default $RUST_VERSION && \ rustup component add rustfmt && \ rustup component add clippy && \ cargo install cargo-make diff --git a/.github/workflows/docs-linkcheck.yml b/.github/workflows/docs-linkcheck.yml index d4159bd94b5..f5212dc1cb1 100644 --- a/.github/workflows/docs-linkcheck.yml +++ b/.github/workflows/docs-linkcheck.yml @@ -18,7 +18,9 @@ jobs: uses: astral-sh/setup-uv@6dfebec6ddbcd197e02256fbdf54deb334fb7f06 # v2 # ignored crates.io due to https://github.com/rust-lang/crates.io/issues/788 - - name: Check links on feldera.com + # browser-like user-agent avoids WAFs dropping the connection on the + # default LinkChecker UA (the classic UNEXPECTED_EOF cause) + - name: Check links on docs.feldera.com run: | - uv run --locked linkchecker https://docs.feldera.com --check-extern --no-warnings --ignore-url "https?://localhost|https://crates.io|https://www.linkedin.com|https://ieeexplore.ieee.org|https://x.com|https?://127.0.0.1" --no-robots + uv run --locked linkchecker https://docs.feldera.com --check-extern --no-warnings --ignore-url "https?://localhost|https://crates.io|https://www.linkedin.com|https://ieeexplore.ieee.org|https://x.com|https?://127.0.0.1|https://join.slack.com|https://felderacommunity.slack.com" --no-robots working-directory: docs.feldera.com diff --git a/Cargo.lock b/Cargo.lock index 8f8059a5e0e..337c2ee257a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3976,7 +3976,7 @@ dependencies = [ [[package]] name = "dbsp" -version = "0.320.0" +version = "0.321.0" dependencies = [ "anyhow", "arc-swap", @@ -4066,7 +4066,7 @@ dependencies = [ [[package]] name = "dbsp_adapters" -version = "0.320.0" +version = "0.321.0" dependencies = [ "actix", "actix-codec", @@ -4210,7 +4210,7 @@ dependencies = [ [[package]] name = "dbsp_nexmark" -version = "0.320.0" +version = "0.321.0" dependencies = [ "anyhow", "ascii_table", @@ -4305,7 +4305,7 @@ checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" [[package]] name = "deltalake" version = "0.32.3" -source = "git+https://github.com/feldera/delta-rs.git?rev=5b35c0988b07ab921d5eee5b8da8da55e4d65945#5b35c0988b07ab921d5eee5b8da8da55e4d65945" +source = "git+https://github.com/feldera/delta-rs.git?rev=297f2dd146f9947795f8b77f25db8da9daff2ade#297f2dd146f9947795f8b77f25db8da9daff2ade" dependencies = [ "buoyant_kernel", "ctor 0.10.1", @@ -4319,7 +4319,7 @@ dependencies = [ [[package]] name = "deltalake-aws" version = "0.15.0" -source = "git+https://github.com/feldera/delta-rs.git?rev=5b35c0988b07ab921d5eee5b8da8da55e4d65945#5b35c0988b07ab921d5eee5b8da8da55e4d65945" +source = "git+https://github.com/feldera/delta-rs.git?rev=297f2dd146f9947795f8b77f25db8da9daff2ade#297f2dd146f9947795f8b77f25db8da9daff2ade" dependencies = [ "async-trait", "aws-config", @@ -4345,7 +4345,7 @@ dependencies = [ [[package]] name = "deltalake-azure" version = "0.15.0" -source = "git+https://github.com/feldera/delta-rs.git?rev=5b35c0988b07ab921d5eee5b8da8da55e4d65945#5b35c0988b07ab921d5eee5b8da8da55e4d65945" +source = "git+https://github.com/feldera/delta-rs.git?rev=297f2dd146f9947795f8b77f25db8da9daff2ade#297f2dd146f9947795f8b77f25db8da9daff2ade" dependencies = [ "bytes", "deltalake-core", @@ -4358,7 +4358,7 @@ dependencies = [ [[package]] name = "deltalake-catalog-unity" version = "0.16.0" -source = "git+https://github.com/feldera/delta-rs.git?rev=5b35c0988b07ab921d5eee5b8da8da55e4d65945#5b35c0988b07ab921d5eee5b8da8da55e4d65945" +source = "git+https://github.com/feldera/delta-rs.git?rev=297f2dd146f9947795f8b77f25db8da9daff2ade#297f2dd146f9947795f8b77f25db8da9daff2ade" dependencies = [ "async-trait", "chrono", @@ -4385,7 +4385,7 @@ dependencies = [ [[package]] name = "deltalake-core" version = "0.32.3" -source = "git+https://github.com/feldera/delta-rs.git?rev=5b35c0988b07ab921d5eee5b8da8da55e4d65945#5b35c0988b07ab921d5eee5b8da8da55e4d65945" +source = "git+https://github.com/feldera/delta-rs.git?rev=297f2dd146f9947795f8b77f25db8da9daff2ade#297f2dd146f9947795f8b77f25db8da9daff2ade" dependencies = [ "arrow", "arrow-arith", @@ -4439,7 +4439,7 @@ dependencies = [ [[package]] name = "deltalake-derive" version = "1.0.0" -source = "git+https://github.com/feldera/delta-rs.git?rev=5b35c0988b07ab921d5eee5b8da8da55e4d65945#5b35c0988b07ab921d5eee5b8da8da55e4d65945" +source = "git+https://github.com/feldera/delta-rs.git?rev=297f2dd146f9947795f8b77f25db8da9daff2ade#297f2dd146f9947795f8b77f25db8da9daff2ade" dependencies = [ "convert_case 0.9.0", "itertools 0.14.0", @@ -4451,7 +4451,7 @@ dependencies = [ [[package]] name = "deltalake-gcp" version = "0.16.0" -source = "git+https://github.com/feldera/delta-rs.git?rev=5b35c0988b07ab921d5eee5b8da8da55e4d65945#5b35c0988b07ab921d5eee5b8da8da55e4d65945" +source = "git+https://github.com/feldera/delta-rs.git?rev=297f2dd146f9947795f8b77f25db8da9daff2ade#297f2dd146f9947795f8b77f25db8da9daff2ade" dependencies = [ "async-trait", "bytes", @@ -5194,7 +5194,7 @@ dependencies = [ [[package]] name = "fda" -version = "0.320.0" +version = "0.321.0" dependencies = [ "anyhow", "arrow", @@ -5246,7 +5246,7 @@ dependencies = [ [[package]] name = "feldera-adapterlib" -version = "0.320.0" +version = "0.321.0" dependencies = [ "actix-web", "anyhow", @@ -5280,7 +5280,7 @@ dependencies = [ [[package]] name = "feldera-buffer-cache" -version = "0.320.0" +version = "0.321.0" dependencies = [ "crossbeam-utils", "enum-map", @@ -5308,7 +5308,7 @@ dependencies = [ [[package]] name = "feldera-datagen" -version = "0.320.0" +version = "0.321.0" dependencies = [ "anyhow", "async-channel 2.5.0", @@ -5334,7 +5334,7 @@ dependencies = [ [[package]] name = "feldera-fxp" -version = "0.320.0" +version = "0.321.0" dependencies = [ "bytecheck", "dbsp", @@ -5354,7 +5354,7 @@ dependencies = [ [[package]] name = "feldera-iceberg" -version = "0.320.0" +version = "0.321.0" dependencies = [ "anyhow", "chrono", @@ -5377,7 +5377,7 @@ dependencies = [ [[package]] name = "feldera-ir" -version = "0.320.0" +version = "0.321.0" dependencies = [ "proptest", "proptest-derive", @@ -5389,7 +5389,7 @@ dependencies = [ [[package]] name = "feldera-macros" -version = "0.320.0" +version = "0.321.0" dependencies = [ "prettyplease", "proc-macro2", @@ -5399,7 +5399,7 @@ dependencies = [ [[package]] name = "feldera-observability" -version = "0.320.0" +version = "0.321.0" dependencies = [ "actix-http", "awc", @@ -5414,7 +5414,7 @@ dependencies = [ [[package]] name = "feldera-rest-api" -version = "0.320.0" +version = "0.321.0" dependencies = [ "chrono", "feldera-observability", @@ -5433,7 +5433,7 @@ dependencies = [ [[package]] name = "feldera-samply" -version = "0.320.0" +version = "0.321.0" dependencies = [ "crossbeam", "feldera-size-of", @@ -5469,7 +5469,7 @@ dependencies = [ [[package]] name = "feldera-sqllib" -version = "0.320.0" +version = "0.321.0" dependencies = [ "arcstr", "base58", @@ -5513,7 +5513,7 @@ dependencies = [ [[package]] name = "feldera-storage" -version = "0.320.0" +version = "0.321.0" dependencies = [ "anyhow", "crossbeam", @@ -5536,7 +5536,7 @@ dependencies = [ [[package]] name = "feldera-types" -version = "0.320.0" +version = "0.321.0" dependencies = [ "actix-web", "anyhow", @@ -8813,7 +8813,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pipeline-manager" -version = "0.320.0" +version = "0.321.0" dependencies = [ "actix-cors", "actix-files", @@ -10023,7 +10023,7 @@ dependencies = [ [[package]] name = "readers" -version = "0.320.0" +version = "0.321.0" dependencies = [ "async-std", "csv", @@ -11722,7 +11722,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "sltsqlvalue" -version = "0.320.0" +version = "0.321.0" dependencies = [ "dbsp", "feldera-sqllib", @@ -12193,7 +12193,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "storage-test-compat" -version = "0.320.0" +version = "0.321.0" dependencies = [ "dbsp", "derive_more 1.0.0", diff --git a/Cargo.toml b/Cargo.toml index 2985273bea9..ffecd064b40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace.package] authors = ["Feldera Team "] -version = "0.320.0" +version = "0.321.0" license = "MIT OR Apache-2.0" homepage = "https://github.com/feldera/feldera" repository = "https://github.com/feldera/feldera" @@ -108,20 +108,23 @@ csv = "1.2.2" csv-core = "0.1.10" dashmap = "6.1.0" datafusion = "53.1" -dbsp = { path = "crates/dbsp", version = "0.320.0" } +dbsp = { path = "crates/dbsp", version = "0.321.0" } dbsp_nexmark = { path = "crates/nexmark" } deadpool-postgres = "0.14.1" -# Feldera fork of delta-rs: upstream tag `rust-v0.32.3` + 2 patches, on branch -# `v0.32.3-feldera`. When bumping the pinned revision, preserve these patches: +# Feldera fork of delta-rs: upstream tag `rust-v0.32.3` + 3 patches, on branch +# `v0.32.3-feldera-fix`. When bumping the pinned revision, preserve these patches: # - 7f8bb445 "Disable unsupported feature check." — relax the writer-side # protocol-feature gate so we can write tables produced by newer clients. # - 5b35c098 "delta-rs: round-robin pre-split unpartitioned scans into # target_partitions FileGroups" — improves snapshot read parallelism for # unpartitioned tables; pairs with the DataFusion knobs we tune in adapters. +# - 9a58fbef "delta-rs: resolve column-mapped Uniform/Iceberg columns by +# Parquet field-id" — fixes `col- is missing from the physical schema` +# when reading UC-Uniform-over-Iceberg tables (columnMapping.mode=id). #6557. # Diff vs. upstream: -# https://github.com/delta-io/delta-rs/compare/rust-v0.32.3...feldera:delta-rs:v0.32.3-feldera -deltalake = { git = "https://github.com/feldera/delta-rs.git", rev = "5b35c0988b07ab921d5eee5b8da8da55e4d65945" } -deltalake-catalog-unity = { git = "https://github.com/feldera/delta-rs.git", rev = "5b35c0988b07ab921d5eee5b8da8da55e4d65945" } +# https://github.com/delta-io/delta-rs/compare/rust-v0.32.3...feldera:delta-rs:v0.32.3-feldera-fix +deltalake = { git = "https://github.com/feldera/delta-rs.git", rev = "297f2dd146f9947795f8b77f25db8da9daff2ade" } +deltalake-catalog-unity = { git = "https://github.com/feldera/delta-rs.git", rev = "297f2dd146f9947795f8b77f25db8da9daff2ade" } delta_kernel = { package = "buoyant_kernel", version = "0.22" } derive_more = { version = "1.0.0" } @@ -133,20 +136,20 @@ erased-serde = "0.3.31" fake = "2.10" fastbloom = "0.14.0" fdlimit = "0.3.0" -feldera-buffer-cache = { version = "0.320.0", path = "crates/buffer-cache" } +feldera-buffer-cache = { version = "0.321.0", path = "crates/buffer-cache" } feldera-cloud1-client = "0.1.2" feldera-datagen = { path = "crates/datagen" } -feldera-fxp = { version = "0.320.0", path = "crates/fxp", features = ["dbsp"] } +feldera-fxp = { version = "0.321.0", path = "crates/fxp", features = ["dbsp"] } feldera-iceberg = { path = "crates/iceberg" } -feldera-observability = { version = "0.320.0", path = "crates/feldera-observability" } -feldera-macros = { version = "0.320.0", path = "crates/feldera-macros" } -feldera-sqllib = { version = "0.320.0", path = "crates/sqllib" } -feldera-storage = { version = "0.320.0", path = "crates/storage" } -feldera-types = { version = "0.320.0", path = "crates/feldera-types" } -feldera-rest-api = { version = "0.320.0", path = "crates/rest-api" } -feldera-ir = { version = "0.320.0", path = "crates/ir" } -feldera-adapterlib = { version = "0.320.0", path = "crates/adapterlib" } -feldera-samply = { version = "0.320.0", path = "crates/samply" } +feldera-observability = { version = "0.321.0", path = "crates/feldera-observability" } +feldera-macros = { version = "0.321.0", path = "crates/feldera-macros" } +feldera-sqllib = { version = "0.321.0", path = "crates/sqllib" } +feldera-storage = { version = "0.321.0", path = "crates/storage" } +feldera-types = { version = "0.321.0", path = "crates/feldera-types" } +feldera-rest-api = { version = "0.321.0", path = "crates/rest-api" } +feldera-ir = { version = "0.321.0", path = "crates/ir" } +feldera-adapterlib = { version = "0.321.0", path = "crates/adapterlib" } +feldera-samply = { version = "0.321.0", path = "crates/samply" } flate2 = "1.1.0" form_urlencoded = "1.2.0" fs_extra = "1.3.0" diff --git a/crates/pipeline-manager/src/api/endpoints/pipeline_management.rs b/crates/pipeline-manager/src/api/endpoints/pipeline_management.rs index 58ca1a34b09..ee66d6bf2a9 100644 --- a/crates/pipeline-manager/src/api/endpoints/pipeline_management.rs +++ b/crates/pipeline-manager/src/api/endpoints/pipeline_management.rs @@ -1775,27 +1775,9 @@ pub struct PostPipelineTesting { /// /// This endpoint is used as part of the test harness. Only available if the `testing` /// unstable feature is enabled. Do not use in production. -#[utoipa::path( - context_path = "/v0", - security(("JSON web token (JWT) or API key" = [])), - params( - ("pipeline_name" = String, Path, description = "Unique pipeline name"), - PostPipelineTesting, - ), - responses( - (status = OK - , description = "Request successfully processed"), - (status = NOT_FOUND - , description = "Pipeline with that name does not exist" - , body = ErrorResponse - , example = json!(examples::error_unknown_pipeline_name())), - (status = METHOD_NOT_ALLOWED - , description = "Endpoint is disabled. Set FELDERA_UNSTABLE_FEATURES=\"testing\" to enable." - , body = ErrorResponse - ) - ), - tag = "Metrics & Debugging" -)] +/// +/// Deliberately excluded from the public OpenAPI spec (not in `ApiDoc`'s `paths()`): it is a +/// test-only hook, not a supported API. The route stays registered so the test harness can call it. #[post("/pipelines/{pipeline_name}/testing")] pub(crate) async fn post_pipeline_testing( state: WebData, diff --git a/crates/pipeline-manager/src/api/main.rs b/crates/pipeline-manager/src/api/main.rs index e7f26ed1509..4121329c5c4 100644 --- a/crates/pipeline-manager/src/api/main.rs +++ b/crates/pipeline-manager/src/api/main.rs @@ -189,7 +189,6 @@ It contains the following fields: endpoints::pipeline_management::post_pipeline, endpoints::pipeline_management::put_pipeline, endpoints::pipeline_management::patch_pipeline, - endpoints::pipeline_management::post_pipeline_testing, endpoints::pipeline_management::post_update_runtime, endpoints::pipeline_management::delete_pipeline, endpoints::pipeline_management::post_pipeline_start, @@ -639,6 +638,7 @@ fn public_scope(api_config: &ApiServerConfig) -> Scope { ) .service(SwaggerUi::new("/swagger-ui/{_:.*}").url("/api-doc/openapi.json", openapi)) .service(healthz) + .service(robots_txt) .service( web::scope("") .wrap(middleware::from_fn(add_cache_headers)) @@ -1036,6 +1036,16 @@ async fn healthz(state: WebData) -> Result HttpResponse { + HttpResponse::Ok() + .content_type("text/plain; charset=utf-8") + .body("User-agent: *\nDisallow: /\n") +} + #[cfg(test)] mod tests { //! Tests covering the static-asset caching middleware and the CORS surface @@ -1102,6 +1112,28 @@ mod tests { assert!(res.headers().get(header::EXPIRES).is_none()); } + /// `/robots.txt` must be answered by the backend with a blanket crawler + /// disallow, not fall through to the SPA catch-all. Drives the production + /// App from `build_app`. Without the explicit `robots_txt` route this path + /// resolves to the web-console bundle (index.html in production, a 404 in + /// the bundle-less test build) and crawlers get no valid robots rules — so + /// they index the sandbox's generated `/create?...` deep-links. Removing + /// the `.service(robots_txt)` wiring flips the status to 404 and fails here. + #[actix_web::test] + async fn robots_txt_disallows_all_crawlers() { + let cfg = ApiServerConfig::test_config(); + let app = test::init_service(build_app(&cfg, &None)).await; + let req = test::TestRequest::get().uri("/robots.txt").to_request(); + let res = test::call_service(&app, req).await; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!( + res.headers().get(header::CONTENT_TYPE).unwrap(), + "text/plain; charset=utf-8", + ); + let body = test::read_body(res).await; + assert_eq!(&body[..], b"User-agent: *\nDisallow: /\n"); + } + // -------- CORS surface integration tests -------- // // All integration tests below use `build_app(&cfg, &None)` — the same diff --git a/docs.feldera.com/docs/connectors/sinks/confluent-jdbc.md b/docs.feldera.com/docs/connectors/sinks/confluent-jdbc.md index 55031f94cc6..5e7c803e3bd 100644 --- a/docs.feldera.com/docs/connectors/sinks/confluent-jdbc.md +++ b/docs.feldera.com/docs/connectors/sinks/confluent-jdbc.md @@ -16,7 +16,7 @@ Avro format expected by this connector. In this format, the key of the Kafka me of the target table. The value component of the message contains the new or updated value for this primary key or `null` if the key is to be deleted from the table. -Because this connector uses the [Kafka output connector](kafka), it +Because this connector uses the [Kafka output connector](kafka.md), it supports [fault tolerance](/pipelines/fault-tolerance) too. Setting up a Confluent JDBC Sink Connector integration involves three steps: diff --git a/docs.feldera.com/docs/connectors/sinks/snowflake.md b/docs.feldera.com/docs/connectors/sinks/snowflake.md index 8fd485aba92..80d4bac75bd 100644 --- a/docs.feldera.com/docs/connectors/sinks/snowflake.md +++ b/docs.feldera.com/docs/connectors/sinks/snowflake.md @@ -3,7 +3,7 @@ The Feldera Snowflake connector ingests data change events produced by a Feldera pipeline into a Snowflake database in near-realtime. -Because this connector uses the [Kafka output connector](kafka), it +Because this connector uses the [Kafka output connector](kafka.md), it supports [fault tolerance](/pipelines/fault-tolerance) too. :::caution Experimental feature diff --git a/docs.feldera.com/docs/connectors/sources/debezium.md b/docs.feldera.com/docs/connectors/sources/debezium.md index d25eb09e5be..5e7ae507777 100644 --- a/docs.feldera.com/docs/connectors/sources/debezium.md +++ b/docs.feldera.com/docs/connectors/sources/debezium.md @@ -130,7 +130,7 @@ Configure an input connector for each Feldera SQL table that must ingest changes Use the `kafka_input` transport with either `json` or `avro` format. Debezium automatically creates a Kafka topic for each database table. -Because input from Debezium uses the [Kafka input connector](kafka), it +Because input from Debezium uses the [Kafka input connector](kafka.md), it supports [fault tolerance](/pipelines/fault-tolerance) too. ### JSON diff --git a/docs.feldera.com/docs/connectors/sources/kafka.md b/docs.feldera.com/docs/connectors/sources/kafka.md index e9657d18448..875c68088ba 100644 --- a/docs.feldera.com/docs/connectors/sources/kafka.md +++ b/docs.feldera.com/docs/connectors/sources/kafka.md @@ -240,7 +240,7 @@ the latter will be overwritten. The Kafka connector can be used to ingest data from a source via Debezium. For information on how to setup Debezium integration for Feldera, see -[Debezium connector documentation](debezium). +[Debezium connector documentation](debezium.md). ### Connecting to AWS MSK with IAM SASL diff --git a/docs.feldera.com/docs/get-started/enterprise/helm-chart-reference.md b/docs.feldera.com/docs/get-started/enterprise/helm-chart-reference.md index 68181fda62c..9d4b1b39795 100644 --- a/docs.feldera.com/docs/get-started/enterprise/helm-chart-reference.md +++ b/docs.feldera.com/docs/get-started/enterprise/helm-chart-reference.md @@ -1,7 +1,7 @@ # Helm Chart Reference This page documents all configurable values for the Feldera Enterprise Helm chart. -See the [Helm guide](./helm-guide) for installation instructions. +See the [Helm guide](./helm-guide.md) for installation instructions. ## Required Values @@ -111,7 +111,7 @@ Controls which Kubernetes secrets can be mounted in connectors via [secret refer ## Parallel Compilation -Parallel compilation runs multiple compiler server replicas to reduce total compile time. See the [parallel compilation guide](./parallel-compilation) for additional details. +Parallel compilation runs multiple compiler server replicas to reduce total compile time. See the [parallel compilation guide](./parallel-compilation.md) for additional details. | Key | Default | Description | |-----|---------|-------------| @@ -137,12 +137,12 @@ When running multiple compiler replicas, sccache allows them to share compiled a ## Authentication -When `auth.enabled` is `true`, the API server requires OIDC authentication. See the guides for [AWS Cognito](./authentication/aws-cognito), [Okta](./authentication/okta-sso), or [generic OIDC providers](./authentication/generic-oidc). +When `auth.enabled` is `true`, the API server requires OIDC authentication. See the guides for [AWS Cognito](./authentication/aws-cognito.md), [Okta](./authentication/okta-sso.md), or [generic OIDC providers](./authentication/generic-oidc.md). | Key | Default | Description | |-----|---------|-------------| | `auth.enabled` | `false` | Enable authentication for the API server. | -| `auth.provider` | `"aws-cognito"` | OIDC provider type. Set the appropriate value based on your [supported authentication provider](./authentication) | +| `auth.provider` | `"aws-cognito"` | OIDC provider type. Set the appropriate value based on your [supported authentication provider](./authentication/index.mdx) | | `auth.clientId` | `""` | OIDC application client ID. | | `auth.issuer` | `""` | OIDC issuer URL. | | `auth.cognitoLoginUrl` | `""` | Cognito hosted UI login URL (Cognito only). | @@ -292,7 +292,7 @@ annotations: ## HTTPS / TLS -Configure HTTPS for all Feldera components. See the [HTTPS guide](./https) for certificate generation and setup. +Configure HTTPS for all Feldera components. See the [HTTPS guide](./https.md) for certificate generation and setup. | Key | Default | Description | |-----|---------|-------------| diff --git a/docs.feldera.com/docs/get-started/enterprise/helm-guide.md b/docs.feldera.com/docs/get-started/enterprise/helm-guide.md index b77079fcd57..cf19080f5db 100644 --- a/docs.feldera.com/docs/get-started/enterprise/helm-guide.md +++ b/docs.feldera.com/docs/get-started/enterprise/helm-guide.md @@ -126,7 +126,7 @@ The `values.yaml` file can be retrieved for an existing Feldera release or from Remote Feldera registry: Run `helm show values oci://public.ecr.aws/feldera/feldera-chart --version {version} >> values.yaml` where `version` is the Feldera platform version for which you want to retrieve values. -Existing Feldera release: Run `helm get values {feldera-release-name} --all -o yaml >> values.yaml` where `feldera-release-name` is the name of your release. The value is usually `feldera`, unless the value has been configured during the [installation](./helm-guide#installing-feldera-enterprise) process. +Existing Feldera release: Run `helm get values {feldera-release-name} --all -o yaml >> values.yaml` where `feldera-release-name` is the name of your release. The value is usually `feldera`, unless the value has been configured during the [installation](#installing-feldera-enterprise) process. ### Configure custom database credentials diff --git a/docs.feldera.com/docs/literature/papers.md b/docs.feldera.com/docs/literature/papers.md index 60ca7a31ae2..ded260e34c1 100644 --- a/docs.feldera.com/docs/literature/papers.md +++ b/docs.feldera.com/docs/literature/papers.md @@ -4,8 +4,8 @@ The following publications and awards describe Feldera's theoretical foundation, ## Awards -* :trophy: [DBSP in ACM SIGMOD Research Highlights](https://www.feldera.com/blog/sigmod-research-highlights/) -* :trophy: [DBSP wins Best Paper award at VLDB 2023 ](https://www.feldera.com/blog/Best-Research-Paper-VLDB-2023/) +* :trophy: [DBSP in ACM SIGMOD Research Highlights](https://www.feldera.com/blog/sigmod-research-highlights) +* :trophy: [DBSP wins Best Paper award at VLDB 2023 ](https://www.feldera.com/blog/Best-Research-Paper-VLDB-2023) ## Publications diff --git a/docs.feldera.com/docs/operations/guide.md b/docs.feldera.com/docs/operations/guide.md index 89eaa502365..0b15740dce8 100644 --- a/docs.feldera.com/docs/operations/guide.md +++ b/docs.feldera.com/docs/operations/guide.md @@ -94,7 +94,7 @@ ALTER TABLE my_table SET TBLPROPERTIES ( **Error**: `The pipeline container has restarted. This was likely caused by an Out-Of-Memory (OOM) crash.` Feldera runs each pipeline in a separate container with configurable memory limits. -See [documentation on the pipeline's memory usage](memory) for a detailed breakdown +See [documentation on the pipeline's memory usage](memory.md) for a detailed breakdown of how memory is used and available control knobs. ### Out-of-storage Errors diff --git a/docs.feldera.com/docs/sql/grammar.md b/docs.feldera.com/docs/sql/grammar.md index 9b36a77a66c..14c57d086c7 100644 --- a/docs.feldera.com/docs/sql/grammar.md +++ b/docs.feldera.com/docs/sql/grammar.md @@ -310,7 +310,7 @@ See [Materialized Tables and Views](materialized.md) for more details. `DECLARE RECURSIVE VIEW` is used to declare a view that can afterwards be used in a recursive SQL query. The syntax of this statement is reminiscent of a table declaration, without constraints. Recursive -queries are documented in [this section](recursion). +queries are documented in [this section](recursion.mdx). ``` declareRecursiveViewStatement: diff --git a/docs.feldera.com/docs/sql/recursion.mdx b/docs.feldera.com/docs/sql/recursion.mdx index 5e32fb497a9..393033b35a0 100644 --- a/docs.feldera.com/docs/sql/recursion.mdx +++ b/docs.feldera.com/docs/sql/recursion.mdx @@ -250,4 +250,4 @@ The following are currently not supported in recursive queries: ### Known Limitations -- Recursive queries mixed with [streaming annotations](streaming) currently can not leverage garbage collection. \ No newline at end of file +- Recursive queries mixed with [streaming annotations](streaming.md) currently can not leverage garbage collection. \ No newline at end of file diff --git a/docs.feldera.com/docs/tutorials/basics/part4.md b/docs.feldera.com/docs/tutorials/basics/part4.md index 6ddffca76de..a691abdc9bc 100644 --- a/docs.feldera.com/docs/tutorials/basics/part4.md +++ b/docs.feldera.com/docs/tutorials/basics/part4.md @@ -161,4 +161,4 @@ changes in `PREFERRED_VENDOR` view approximately every second. To summarize Part 4 of the tutorial, we can attach a random generator to Feldera tables to simulate different scenarios such as backfill, continuous evaluation or a combination of the two. -You'll find a complete datagen reference in the [connectors section](../../connectors/sources/datagen). +You'll find a complete datagen reference in the [connectors section](../../connectors/sources/datagen.md). diff --git a/docs.feldera.com/docs/use_cases/batch/part3.md b/docs.feldera.com/docs/use_cases/batch/part3.md index 1049bc14c2c..59c0ca1030c 100644 --- a/docs.feldera.com/docs/use_cases/batch/part3.md +++ b/docs.feldera.com/docs/use_cases/batch/part3.md @@ -7,7 +7,7 @@ accumulated in the source database over an extended period (months or years) before processing new real-time inputs. This process is known as **backfill**. In some cases, both historical and real-time data can be ingested from the same -data source. For example, in [Part 2](part2) of this tutorial, we configured the +data source. For example, in [Part 2](part2.md) of this tutorial, we configured the Delta Lake connector to read the current snapshot of the table before following new updates in the table's transaction log. diff --git a/docs.feldera.com/docs/use_cases/fine_grained_authorization/static.md b/docs.feldera.com/docs/use_cases/fine_grained_authorization/static.md index a5bbf401101..34a85cb430f 100644 --- a/docs.feldera.com/docs/use_cases/fine_grained_authorization/static.md +++ b/docs.feldera.com/docs/use_cases/fine_grained_authorization/static.md @@ -282,7 +282,7 @@ where not users.is_banned; ``` -Start the pipeline and populate the object graph to match the [example](intro#object-graph) by issuing the following +Start the pipeline and populate the object graph to match the [example](intro.md#object-graph) by issuing the following ad hoc queries: ```sql diff --git a/docs.feldera.com/docs/use_cases/fraud_detection/fraud_detection.md b/docs.feldera.com/docs/use_cases/fraud_detection/fraud_detection.md index d0fbf791bb2..30e77c82c01 100644 --- a/docs.feldera.com/docs/use_cases/fraud_detection/fraud_detection.md +++ b/docs.feldera.com/docs/use_cases/fraud_detection/fraud_detection.md @@ -483,4 +483,4 @@ input. ## Resources A version of this use case focusing on integration with the Delta Lake and Spark ecosystem -is published as a [blog post](https://www.feldera.com/blog/feature-engineering-part2/). +is published as a [blog post](https://www.feldera.com/blog/feature-engineering-part2). diff --git a/docs.feldera.com/docs/what-is-feldera.md b/docs.feldera.com/docs/what-is-feldera.md index 17b23388f74..a8402436819 100644 --- a/docs.feldera.com/docs/what-is-feldera.md +++ b/docs.feldera.com/docs/what-is-feldera.md @@ -18,7 +18,7 @@ Our defining features: 3. **Datasets larger than RAM**. Feldera is designed to handle datasets that exceed the available RAM by spilling efficiently to disk, taking advantage of recent advances in NVMe storage. -4. **Strong guarantees on consistency and freshness**. Feldera is strongly consistent: it [guarantees](https://www.feldera.com/blog/synchronous-streaming/) that the state of the views always corresponds to what you'd get if you ran the queries in a batch system for the same input. +4. **Strong guarantees on consistency and freshness**. Feldera is strongly consistent: it [guarantees](https://www.feldera.com/blog/synchronous-streaming) that the state of the views always corresponds to what you'd get if you ran the queries in a batch system for the same input. 5. **Connectors for your favorite data sources and destinations**. Feldera connects to myriad batch and streaming data sources, like Kafka, HTTP, CDC streams, S3, Data Lakes, Warehouses and more. If you need a connector that we don't yet support, [let us know](https://github.com/feldera/feldera/issues/new/choose). diff --git a/docs.feldera.com/docusaurus.config.ts b/docs.feldera.com/docusaurus.config.ts index 56b5a9a15cc..aef1ca3b012 100644 --- a/docs.feldera.com/docusaurus.config.ts +++ b/docs.feldera.com/docusaurus.config.ts @@ -15,6 +15,11 @@ const config: Config = { // For GitHub pages deployment, it is often '//' baseUrl: "/", + // The static host serves canonical pages with a trailing slash and 301s the + // non-slash form to it. Match that so generated links, canonical tags, and + // sitemap entries point at the served URL instead of a redirect. + trailingSlash: true, + // GitHub pages deployment config. // If you aren't using GitHub pages, you don't need these. organizationName: "feldera", // Usually your GitHub org/user name. @@ -54,6 +59,13 @@ const config: Config = { theme: { customCss: "./src/css/custom.css", }, + sitemap: { + // Emit a real freshness signal; changefreq/priority are ignored by + // search engines, but lastmod is honored. + lastmod: "date", + // Keep utility routes and the raw OpenAPI spec out of the sitemap. + ignorePatterns: ["/search", "/**/openapi.json/**", "/**/openapi.json"], + }, } satisfies PresetOpenapi.Options, ], [ @@ -262,6 +274,18 @@ const config: Config = { from: "/api/retrieve-the-status-of-an-output-connector", to: "/api/get-output-status", }, + { + from: "/get-started/enterprise/kubernetes-guides/eks/ingress", + to: "/get-started/enterprise/kubernetes-guides/eks/envoy-gateway", + }, + { + from: "/enterprise/kubernetes-guides/k3d", + to: "/get-started/enterprise/kubernetes-guides/k3d", + }, + { + from: "/connectors/sources/kafka/debezium", + to: "/connectors/sources/debezium", + }, ], }, ], diff --git a/docs.feldera.com/src/pages/markdown-page.md b/docs.feldera.com/src/pages/markdown-page.md deleted file mode 100644 index 9756c5b6685..00000000000 --- a/docs.feldera.com/src/pages/markdown-page.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: Markdown page example ---- - -# Markdown page example - -You don't need React to write simple standalone pages. diff --git a/docs.feldera.com/src/theme/CodeBlock/SandboxButton/index.tsx b/docs.feldera.com/src/theme/CodeBlock/SandboxButton/index.tsx index 6a3ff77243f..dc1ce916b76 100644 --- a/docs.feldera.com/src/theme/CodeBlock/SandboxButton/index.tsx +++ b/docs.feldera.com/src/theme/CodeBlock/SandboxButton/index.tsx @@ -21,7 +21,7 @@ console.log('styles', styles)

- Load a Feldera support bundle (.zip file) containing circuit profile data to begin - visualization. + Load a Feldera support bundle (.zip file) to explore the circuit profile, + program code, logs and the pipeline config.

{#if isLoading} diff --git a/js-packages/profiler-layout/src/lib/components/SupportBundleViewerLayout.svelte b/js-packages/profiler-layout/src/lib/components/SupportBundleViewerLayout.svelte index 96d588c24c5..21b6f7ec623 100644 --- a/js-packages/profiler-layout/src/lib/components/SupportBundleViewerLayout.svelte +++ b/js-packages/profiler-layout/src/lib/components/SupportBundleViewerLayout.svelte @@ -39,7 +39,8 @@ type AnalysisTab = (typeof TABS)[number] interface Props { - profileData: JsonProfiles + /** Parsed circuit profile */ + profileData?: JsonProfiles dataflowData: Dataflow | undefined programCode: string[] | undefined logText?: string @@ -81,6 +82,12 @@ onRenderingChange }: Props = $props() + const hasProfile = $derived(profileData !== undefined) + + const graphPaneDefaultSize = 55 + const graphPaneMinSize = 20 + const belowGraphPaneDefaultSize = 100 - graphPaneDefaultSize + let profilerDiagram: ProfilerDiagram | undefined = $state() // The loaded profile's toplevel node id, so the analysis panel can recognise overview data. const diagramRootNodeId = $derived(profilerDiagram?.getProfile()?.rootNodeId) @@ -137,7 +144,6 @@ const attrs = data.match({ some: (v) => v, none: () => null }) untrack(() => { if (attrs) { - currentTab = 'Metrics' tooltipData = { nodeAttributes: attrs } // Only cache while we're on a node view, so an `'overview'` round-trip can't // overwrite the operator the user actually inspected. @@ -205,19 +211,18 @@ profilerDiagram?.selectMetric(selectedMetricId) }) - // Show the overview each time a profile loads. `analysisView` is reset first so the + // Reset to the overview whenever the loaded bundle changes. `analysisView` is reset first so the // SegmentedControl indicator follows the new view, and so the sticky `displayNodeAttributes` // that `showGlobalMetrics` triggers sees a non-node view and can't overwrite `lastNodeData` // with the overview payload. $effect(() => { void profileData const diagram = profilerDiagram - if (!diagram) { - return - } queueMicrotask(() => { analysisView = 'overview' - diagram.showGlobalMetrics(true) + tooltipData = null + lastNodeData = null + diagram?.showGlobalMetrics(true) }) }) @@ -316,42 +321,54 @@ {#snippet graphPanel()} -
+
{@render loadProfileControl?.()} -
- e.key === 'Enter' && handleSearch()} - class="input h-6 w-36 text-sm" - /> -
+ {#if hasProfile} +
+ e.key === 'Enter' && handleSearch()} + class="input h-6 w-36 text-sm" + /> +
+ {:else} +

+ This bundle has no circuit profile, so the dataflow graph is unavailable. +

+ {/if}
-
- {#if error} -
- {error} -
- {/if} - {#if message} -
- {message} -
- {/if} -
+ {#if hasProfile} +
+ {#if error} +
+ {error} +
+ {/if} + {#if message} +
+ {message} +
+ {/if} +
+ {/if}
{/snippet} @@ -446,7 +463,7 @@ items={[ { value: 'overview', label: 'Overview' }, { value: 'node', label: 'Node', disabled: !lastNodeData }, - { value: 'top-nodes', label: 'Top nodes' } + { value: 'top-nodes', label: 'Top nodes', disabled: !hasProfile } ]} class="px-2" /> @@ -491,11 +508,17 @@ {#if !sqlPanelFullHeight} - - {@render graphPanel()} - - - + {#if hasProfile} + + {@render graphPanel()} + + + {:else} +
+ {@render graphPanel()} +
+ {/if} +
@@ -516,11 +539,21 @@ - - {@render graphPanel()} - - - + {#if hasProfile} + + {@render graphPanel()} + + + {:else} +
+ {@render graphPanel()} +
+ {/if} + {@render analysisPanel()} @@ -532,15 +565,17 @@ in graphPanel. Surviving the layout toggle (and any other conditional re-render of graphPanel) is what keeps the Visualizer's internal state intact instead of being torn down and re-created from `profileData` each time. --> - - - +{#if profileData} + + + +{/if}