[adapters] Remove cmake dependency for rdkafka.

Leonid Ryzhyk · ryzhyk · commit 3e16df1a7733 · 2025-04-01T20:59:44.000Z
Building Kafka with cmake started causing issues recently (most likely due to pip installing cmake 4.0 and this issue: confluentinc/librdkafka#4594 (comment)) This feature should only be needed on Windows, which we don't support anyway, so this commit removes it. Hopefully this will make rdkafka builds less fragile in the future. Signed-off-by: Leonid Ryzhyk <leonid@feldera.com>
diff --git a/.github/workflows/build-java.yml b/.github/workflows/build-java.yml
@@ -8,7 +8,7 @@ jobs:
     name: Build Compiler
     runs-on: [self-hosted, skylake40]
     container:
-      image: ghcr.io/feldera/feldera-dev:0a775f772b2ebde13708744d3e6a219ca4a492d2
+      image: ghcr.io/feldera/feldera-dev:f4797ad926773a38483275405728a9eb3b9005b5
       options: --user=ubuntu
     steps:
       - name: Checkout repository
diff --git a/.github/workflows/build-rust.yml b/.github/workflows/build-rust.yml
@@ -29,7 +29,7 @@ jobs:
     runs-on: ${{ matrix.runner }}
 
     container:
-      image: ghcr.io/feldera/feldera-dev:3012ef118aaaa2b7d49e1dfeda938288a70ba66c
+      image: ghcr.io/feldera/feldera-dev:f4797ad926773a38483275405728a9eb3b9005b5
       options: --user=ubuntu --privileged
       volumes:
         - /sccache:/sccache
diff --git a/.github/workflows/ci-post-release.yml b/.github/workflows/ci-post-release.yml
@@ -20,7 +20,7 @@ jobs:
   adjust-versions:
     runs-on: [self-hosted, skylake40]
     container:
-      image: ghcr.io/feldera/feldera-dev:39c1b55d0972ccd26375cf2f7cbc8554f4609da8
+      image: ghcr.io/feldera/feldera-dev:f4797ad926773a38483275405728a9eb3b9005b5
       options: --user=ubuntu
       volumes:
         - /sccache:/sccache
diff --git a/.github/workflows/ci-pre-mergequeue.yml b/.github/workflows/ci-pre-mergequeue.yml
@@ -9,6 +9,11 @@ jobs:
   main:
     runs-on: ubuntu-latest
     steps:
+      # Required by librdkafka.
+      - name: Install libsasl2-dev
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libsasl2-dev
       - uses: actions/checkout@v3
       - uses: oven-sh/setup-bun@v2
       - uses: actions-rust-lang/setup-rust-toolchain@v1
diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml
@@ -11,7 +11,7 @@ jobs:
   deploy:
     runs-on: [self-hosted, skylake40]
     container:
-      image: ghcr.io/feldera/feldera-dev:3012ef118aaaa2b7d49e1dfeda938288a70ba66c
+      image: ghcr.io/feldera/feldera-dev:f4797ad926773a38483275405728a9eb3b9005b5
       options: --user=ubuntu
       volumes:
         - /sccache:/sccache
@@ -32,4 +32,4 @@ jobs:
           cargo publish ${{ vars.CARGO_PUBLISH_FLAGS }} --package fda
           cargo publish ${{ vars.CARGO_PUBLISH_FLAGS }} --package feldera-sqllib
         env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
@@ -9,7 +9,7 @@ jobs:
     name: Integration Tests
     runs-on: [self-hosted, skylake40]
     container:
-      image: ghcr.io/feldera/feldera-dev:0a775f772b2ebde13708744d3e6a219ca4a492d2
+      image: ghcr.io/feldera/feldera-dev:f4797ad926773a38483275405728a9eb3b9005b5
 
     services:
       pipeline-manager:
diff --git a/.github/workflows/test-java.yml b/.github/workflows/test-java.yml
@@ -8,7 +8,7 @@ jobs:
     name: Execute Java Tests
     runs-on: [self-hosted, skylake40]
     container:
-      image: ghcr.io/feldera/feldera-dev:0a775f772b2ebde13708744d3e6a219ca4a492d2
+      image: ghcr.io/feldera/feldera-dev:f4797ad926773a38483275405728a9eb3b9005b5
       options: --user=ubuntu
     steps:
       - name: Checkout repository
diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
@@ -12,7 +12,7 @@ jobs:
     name: Rust Unit Tests
     runs-on: [self-hosted, skylake40]
     container:
-      image: ghcr.io/feldera/feldera-dev:0a775f772b2ebde13708744d3e6a219ca4a492d2
+      image: ghcr.io/feldera/feldera-dev:f4797ad926773a38483275405728a9eb3b9005b5
       options: --user=ubuntu
 
     steps:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -22,6 +22,7 @@ Our dependencies are:
         - this will need a C and C++ compiler installed (e.g., gcc, gcc++)
     - cmake
     - libssl-dev
+    - libsasl2-dev
 - SQL Compiler
     - a Java Virtual Machine (at least Java 19)
     - maven
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/README.md b/README.md
@@ -112,6 +112,7 @@ To run Feldera from sources, first install required dependencies:
 - [Rust tool chain](https://www.rust-lang.org/tools/install)
 - cmake
 - libssl-dev
+- libsasl2-dev
 - Java Development Kit (JDK), version 19 or newer
 - maven
 - [Bun](https://bun.sh/docs/installation)
diff --git a/crates/adapters/Cargo.toml b/crates/adapters/Cargo.toml
@@ -52,8 +52,7 @@ serde_json = { version = "1.0.127", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
 form_urlencoded = "1.2.0"
 csv = "1.2.2"
-# cmake-build is required on Windows.
-rdkafka = { version = "0.37.0", features = ["cmake-build", "ssl-vendored", "gssapi-vendored", "zstd", "libz"], optional = true }
+rdkafka = { version = "0.37.0", features = ["ssl-vendored", "gssapi-vendored", "zstd", "libz"], optional = true }
 aws-sdk-s3 = { version = "1.17.0", features = ["behavior-version-latest"] }
 aws-types = "1.1.7"
 actix = "0.13.1"
diff --git a/crates/nexmark/Cargo.toml b/crates/nexmark/Cargo.toml
@@ -30,8 +30,7 @@ regex = { version = "1.6.0" }
 time = { version = "0.3.14", features = ["formatting"] }
 paste = { version = "1.0.9" }
 rand = { version = "0.8", features = ["small_rng"] }
-# cmake-build is required on Windows.
-rdkafka = { version = "0.37.0", features = ["cmake-build", "ssl-vendored", "gssapi-vendored"], optional = true }
+rdkafka = { version = "0.37.0", features = ["ssl-vendored", "gssapi-vendored"], optional = true }
 clap = { version = "3.2.8", features = ["derive", "env"] }
 cached = { version = "0.38.0" }
 serde = { version = "1.0", features = ["derive"] }
diff --git a/deploy/Dockerfile b/deploy/Dockerfile
@@ -15,8 +15,9 @@ ENV OPENSSL_STATIC=1
 RUN apt update --fix-missing && apt install \
   # pkg-config is required for cargo to find libssl
   libssl-dev pkg-config \
-  # rdkafka dependency needs cmake and a CXX compiler
-  cmake build-essential \
+  cmake \
+  # rdkafka dependency needs libsasl2-dev and the CXX compiler
+  libsasl2-dev build-essential \
   # To install rust
   curl  \
   # For running the SQL compiler
@@ -80,4 +81,4 @@ ENV RUSTFLAGS="-C link-arg=-fuse-ld=mold"
 RUN ./pipeline-manager --bind-address=0.0.0.0 --sql-compiler-home=/home/ubuntu/lib/sql-to-dbsp-compiler --compilation-cargo-lock-path=/home/ubuntu/Cargo.lock --dbsp-override-path=/home/ubuntu/lib --precompile
 RUN ./pipeline-manager --bind-address=0.0.0.0 --sql-compiler-home=/home/ubuntu/lib/sql-to-dbsp-compiler --compilation-cargo-lock-path=/home/ubuntu/Cargo.lock --dbsp-override-path=/home/ubuntu/lib --preinstall-pg-embed
 ENV BANNER_ADDR=localhost
-ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/ubuntu/lib/sql-to-dbsp-compiler", "--compilation-cargo-lock-path=/home/ubuntu/Cargo.lock", "--dbsp-override-path=/home/ubuntu/lib", "--demos-dir", "/home/ubuntu/demos"]
+ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--sql-compiler-home=/home/ubuntu/lib/sql-to-dbsp-compiler", "--compilation-cargo-lock-path=/home/ubuntu/Cargo.lock", "--dbsp-override-path=/home/ubuntu/lib", "--demos-dir", "/home/ubuntu/demos"]
diff --git a/deploy/build.Dockerfile b/deploy/build.Dockerfile
@@ -15,8 +15,9 @@ ENV OPENSSL_STATIC=1
 RUN apt-get update --fix-missing && apt-get install -y \
     # pkg-config is required for cargo to find libssl
     libssl-dev pkg-config \
-    # rdkafka dependency needs cmake and a CXX compiler
-    cmake build-essential \
+    cmake \
+    # rdkafka dependency needs libsasl2-dev and a CXX compiler
+    libsasl2-dev build-essential \
     # To install rust
     curl  \
     # For running the SQL compiler
@@ -91,4 +92,4 @@ RUN  arch=`dpkg --print-architecture | sed "s/arm64/aarch64/g" | sed "s/amd64/x8
     && chmod +x /home/ubuntu/.cargo/bin/sccache
 
 ENV RUSTFLAGS="-C link-arg=-fuse-ld=mold"
-RUN rustup default stable
+RUN rustup default stable
