Custom Kafka Connect docker image.

Leonid Ryzhyk · ryzhyk · commit 96932fa482a3 · 2023-10-07T14:41:00.000-07:00
In preparation for adding support for Snowflake output connector, we
replace the pre-built Kafka Connect image from Debezium with a custom
one that adds Snowflake connector classes to the image.  There are a
couple of Kafka Connect base images we could build upon, including one
from Confluent, but that one seems to assume Apache Kafka and may not
work well with Redpanda (first thing it did when I tried it is
complain about not having Zookeeper around :) ).

In addition, we want the image to contain Debezium connectors, so
building on the Debezium image from RedHat seemed like a good choice.

This commit:
1. Adds a Kafka Connect container build step to Earthfile
2. Adds identical instructions for building the connector to Dockerfile
3. Sets up github CI to build the container along with other Feldera
   containers.

While I was at it, I also added a trigger to the `containers.yml` to
allow triggering the container build workflow manually.  This is useful
if we want to build the containers without waiting for the cronjob.

Signed-off-by: Leonid Ryzhyk &lt;leonid@feldera.com&gt;
diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml
@@ -3,11 +3,13 @@ on:
     - cron: '0 0 * * *'
   release:
       types: [published]
+  workflow_dispatch:
 
 env:
   REGISTRY: ghcr.io
   PIPELINE_MANAGER_IMAGE: ghcr.io/${{ github.repository_owner }}/pipeline-manager
   DEMO_IMAGE: ghcr.io/${{ github.repository_owner }}/demo-container
+  KAFKA_CONNECT_IMAGE:  ghcr.io/${{ github.repository_owner }}/kafka-connect
 
 jobs:
   build-and-push-image:
@@ -43,6 +45,19 @@ jobs:
           tags: |
             type=semver,pattern={{version}}
 
+      - name: Docker meta (kafka-connect)
+        id: meta_kafka_connect
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.KAFKA_CONNECT_IMAGE }}
+          tags: |
+            type=semver,pattern={{version}}
+
+      - name: Build Kafka Connect container
+        run: |
+          cd deploy && \
+          docker build --target kafka-connect -t ${{ env.KAFKA_CONNECT_IMAGE }}  .
+
       - name: Run integration tests
         run: |
           cd deploy && \
@@ -71,7 +86,7 @@ jobs:
       # Push untagged images when the workflow is not triggered by a release
       - name: Push containers
         if: github.event_name != 'release'
-        run: docker push ${{ env.PIPELINE_MANAGER_IMAGE }} && docker push ${{ env.DEMO_IMAGE }}
+        run: docker push ${{ env.PIPELINE_MANAGER_IMAGE }} && docker push ${{ env.DEMO_IMAGE }} && docker push ${{ env.KAFKA_CONNECT_IMAGE }}
 
       # Tagged DBSP image
       - name: Push (pipeline-manager)
@@ -96,6 +111,18 @@ jobs:
           tags: ${{ steps.meta_demo.outputs.tags }}
           labels: ${{ steps.meta_demo.outputs.labels }}
 
+      # Tagged Kafka Connect image
+      - name: Push (kafka-connect)
+        uses: docker/build-push-action@v4
+        if: github.event_name == 'release'
+        with:
+          context: .
+          file: deploy/Dockerfile
+          target: kafka-connect
+          push: true
+          tags: ${{ steps.meta_kafka_connect.outputs.tags }}
+          labels: ${{ steps.meta_kafka_connect.outputs.labels }}
+
       - uses: actions/delete-package-versions@v4
         with:
           package-name: pipeline-manager
diff --git a/Earthfile b/Earthfile
@@ -514,7 +514,6 @@ build-pipeline-manager-container:
     COPY sql-to-dbsp-compiler/temp /database-stream-processor/sql-to-dbsp-compiler/temp
     RUN ./pipeline-manager --bind-address=0.0.0.0 --api-server-working-directory=/working-dir --compiler-working-directory=/working-dir --runner-working-directory=/working-dir --sql-compiler-home=/database-stream-processor/sql-to-dbsp-compiler --dbsp-override-path=/database-stream-processor --precompile
     ENTRYPOINT ["./pipeline-manager", "--bind-address=0.0.0.0", "--api-server-working-directory=/working-dir", "--compiler-working-directory=/working-dir", "--runner-working-directory=/working-dir", "--sql-compiler-home=/database-stream-processor/sql-to-dbsp-compiler", "--dbsp-override-path=/database-stream-processor"]
-    SAVE IMAGE ghcr.io/feldera/pipeline-manager
 
 # TODO: mirrors the Dockerfile. See note above.
 build-demo-container:
@@ -530,7 +529,9 @@ build-demo-container:
     COPY +install-python/bin /root/.local/bin
     COPY demo demo
     CMD bash
-    SAVE IMAGE ghcr.io/feldera/demo-container
+
+build-kafka-connect-container:
+    FROM DOCKERFILE -f deploy/Dockerfile --target kafka-connect .
 
 test-docker-compose:
     FROM earthly/dind:alpine
@@ -550,10 +551,10 @@ test-debezium:
     ENV FELDERA_VERSION=latest
     WITH DOCKER --pull postgres \
                 --pull docker.redpanda.com/vectorized/redpanda:v23.2.3 \
-                --pull debezium/connect:2.3 \
                 --pull debezium/example-mysql:2.3 \
                 --load ghcr.io/feldera/pipeline-manager:latest=+build-pipeline-manager-container \
-                --load ghcr.io/feldera/demo-container:latest=+build-demo-container
+                --load ghcr.io/feldera/demo-container:latest=+build-demo-container \
+                --load ghcr.io/feldera/kafka-connect:latest=+build-kafka-connect-container
         RUN COMPOSE_HTTP_TIMEOUT=120 RUST_LOG=debug,tokio_postgres=info docker-compose -f docker-compose.yml -f docker-compose-debezium.yml --profile debezium up --force-recreate --exit-code-from debezium-demo
     END
 
diff --git a/deploy/Dockerfile b/deploy/Dockerfile
@@ -167,6 +167,13 @@ RUN apt update && apt install pkg-config \
   && apt remove python3-pip unzip pkg-config -y && apt autoremove -y
 CMD bash
 
+# Kafka connect with all Debezium connectors + the Snowflake connector.
+FROM debezium/connect:2.3 AS kafka-connect
+RUN mkdir /kafka/connect/snowflake-kafka-connector
+RUN cd /kafka/connect/snowflake-kafka-connector \
+    && curl -LO https://repo1.maven.org/maven2/com/snowflake/snowflake-kafka-connector/2.1.0/snowflake-kafka-connector-2.1.0.jar \
+    && curl -LO https://repo1.maven.org/maven2/org/bouncycastle/bc-fips/1.0.1/bc-fips-1.0.1.jar \
+    && curl -LO https://repo1.maven.org/maven2/org/bouncycastle/bcpkix-fips/1.0.3/bcpkix-fips-1.0.3.jar
 
 # By default, only build the release version
 FROM release
diff --git a/deploy/docker-compose-debezium.yml b/deploy/docker-compose-debezium.yml
@@ -4,7 +4,7 @@ services:
   # Kafka connect.
   # TODO: add a healthcheck to this container.
   connect:
-    image: debezium/connect:2.3
+    image: ghcr.io/feldera/kafka-connect:${FELDERA_VERSION:-0.1.6}
     depends_on:
       redpanda:
         condition: service_healthy
@@ -28,7 +28,17 @@ services:
       MYSQL_USER: mysqluser
       MYSQL_PASSWORD: mysqlpw
     healthcheck:
-      test: ["CMD", "mysqladmin" , "-u", "$$MYSQL_USER",  "-p$$MYSQL_PASSWORD" ,"ping", "-h", "localhost"]
+      test:
+        [
+          "CMD",
+          "mysqladmin",
+          "-u",
+          "$$MYSQL_USER",
+          "-p$$MYSQL_PASSWORD",
+          "ping",
+          "-h",
+          "localhost"
+        ]
       interval: 5s
       timeout: 20s
       # MySQL can be _very_ slow to start.
