Skip to content

Commit 2c912a5

Browse files
Karakatiza666Karakatiza666
committed
[CI] Add arm64 release image build to containers.yml
Update .devcontainer Signed-off-by: Karakatiza666 <bulakh.96@gmail.com>
1 parent ccefb56 commit 2c912a5

File tree

6 files changed

+203
-51
lines changed

6 files changed

+203
-51
lines changed

‎.devcontainer/Dockerfile‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM ubuntu:22.04 AS base
1+
FROM ubuntu:24.04 AS base
22

33
## Add non-root user for startup of pg-embed with rust backend
44

@@ -33,17 +33,14 @@ RUN apt-get update --fix-missing && \
3333

3434
RUN apt-get install -y \
3535
gcc clang libclang-dev python3-pip python3-plumbum \
36-
hub numactl openjdk-19-jre-headless maven
36+
hub numactl openjdk-21-jre-headless maven
3737

38-
## Install nodejs and global packages (optional)
39-
# - nodejs
38+
## Install nodejs (optional)
4039
RUN mkdir -p /etc/apt/keyrings
4140
RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
4241
ENV NODE_MAJOR=20
4342
RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
4443
RUN apt update --fix-missing && apt install nodejs
45-
RUN npm install --global yarn
46-
RUN npm install --global openapi-typescript-codegen
4744

4845
## Install Redpanda rpk to enable preparing demos
4946

@@ -55,8 +52,7 @@ RUN ARCH=`dpkg --print-architecture`; \
5552

5653
## Install python dependencies for the demos
5754

58-
RUN \
59-
pip3 install gdown kafka-python
55+
RUN pip3 install --break-system-packages gdown kafka-python-ng
6056

6157
## Switch to non-root user
6258

@@ -71,13 +67,12 @@ ENV PATH="$HOME/.cargo/bin:$PATH"
7167
## Install Bun.js
7268
RUN curl -fsSL https://bun.sh/install | bash
7369
ENV PATH="$HOME/.bun/bin:$PATH"
70+
RUN $HOME/.bun/bin/bun install --global @hey-api/openapi-ts
7471

7572
RUN \
76-
rustup install nightly && \
73+
rustup install 1.82.0 && \
7774
rustup component add rustfmt && \
78-
rustup component add rustfmt --toolchain nightly && \
7975
rustup component add clippy && \
80-
rustup component add clippy --toolchain nightly && \
8176
cargo install cargo-make
8277

8378
ENV PATH="$HOME/.local/bin:$PATH"

‎.devcontainer/devcontainer.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"name": "DBSP",
2+
"name": "Feldera dev",
33
"dockerComposeFile": [
44
"./docker-compose.devcontainer.yml",
55
"../deploy/docker-compose.yml"
@@ -8,7 +8,7 @@
88
"runServices": [
99
"redpanda"
1010
],
11-
"workspaceFolder": "/workspaces/dbsp",
11+
"workspaceFolder": "/workspaces/feldera",
1212
"shutdownAction": "stopCompose",
1313
"customizations": {
1414
"vscode": {
@@ -42,5 +42,5 @@
4242
"mounts": [
4343
"source=${localWorkspaceFolder},target=${containerWorkspaceFolder},type=bind"
4444
],
45-
"postCreateCommand": "chmod +x /workspaces/dbsp/.devcontainer/postCreate.sh && /workspaces/dbsp/.devcontainer/postCreate.sh"
45+
"postCreateCommand": "bash /workspaces/feldera/.devcontainer/postCreate.sh"
4646
}

‎.github/workflows/containers.yml‎

Lines changed: 75 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Docker Build
1+
name: Build release images
22

33
on:
44
schedule:
@@ -10,41 +10,43 @@ on:
1010
env:
1111
REGISTRY: ghcr.io
1212
PIPELINE_MANAGER_IMAGE: ghcr.io/${{ github.repository_owner }}/pipeline-manager
13-
KAFKA_CONNECT_IMAGE: ghcr.io/${{ github.repository_owner }}/kafka-connect
1413

1514
jobs:
1615
build-and-push-image:
17-
runs-on: [self-hosted, skylake40]
16+
runs-on: ${{ matrix.runner }}
17+
strategy:
18+
matrix:
19+
include:
20+
- runner: [self-hosted, skylake40]
21+
os: linux
22+
arch: amd64
23+
- runner: blacksmith-8vcpu-ubuntu-2204-arm
24+
os: linux
25+
arch: arm64
1826
permissions:
1927
contents: read
2028
packages: write
2129

2230
steps:
2331
- name: Checkout repository
24-
uses: actions/checkout@v3
32+
uses: actions/checkout@v4
2533
with:
2634
submodules: true
2735

2836
- name: Docker meta (pipeline-manager)
2937
id: meta_pipeline_manager
30-
uses: docker/metadata-action@v4
38+
uses: docker/metadata-action@v5
3139
with:
3240
images: ${{ env.PIPELINE_MANAGER_IMAGE }}
3341
tags: |
3442
type=semver,pattern={{version}}
35-
36-
- name: Docker meta (kafka-connect)
37-
id: meta_kafka_connect
38-
uses: docker/metadata-action@v4
39-
with:
40-
images: ${{ env.KAFKA_CONNECT_IMAGE }}
41-
tags: |
42-
type=semver,pattern={{version}}
43-
44-
- name: Build Kafka Connect container
45-
run: |
46-
cd deploy && \
47-
docker build -f Dockerfile.kafka-connect --target kafka-connect -t ${{ env.KAFKA_CONNECT_IMAGE }} .
43+
type=raw,value=latest
44+
# Single image description
45+
labels: |
46+
org.opencontainers.image.description=The Pipeline Manager lets you create and run pipelines inside Feldera instance
47+
# Multiarch manifest description
48+
annotations: |
49+
org.opencontainers.image.description=The Pipeline Manager lets you create and run pipelines inside Feldera instance
4850
4951
- name: Run standard demo that checks a couple endpoints are reachable
5052
run: |
@@ -56,31 +58,71 @@ jobs:
5658
--exit-code-from demo-standard --renew-anon-volumes
5759
5860
- name: Log in to the Container registry
59-
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
61+
uses: docker/login-action@v3
6062
with:
6163
registry: ${{ env.REGISTRY }}
6264
username: ${{ github.actor }}
6365
password: ${{ secrets.GITHUB_TOKEN }}
6466

65-
# Push untagged images when the workflow is not triggered by a release
66-
- name: Push containers
67-
if: github.event_name != 'release'
68-
run: docker push ${{ env.PIPELINE_MANAGER_IMAGE }} && docker push ${{ env.KAFKA_CONNECT_IMAGE }}
69-
70-
# Tagged pipeline-manager image
71-
- name: Push (pipeline-manager)
72-
uses: docker/build-push-action@v4
73-
if: github.event_name == 'release'
67+
- name: Build and push pipeline-manager image
68+
id: build_pipeline_manager
69+
uses: docker/build-push-action@v6
7470
with:
7571
context: .
72+
platforms: ${{matrix.os}}/${{matrix.arch}}
7673
file: deploy/Dockerfile
7774
push: true
7875
tags: ${{ steps.meta_pipeline_manager.outputs.tags }}
7976
labels: ${{ steps.meta_pipeline_manager.outputs.labels }}
77+
annotations: ${{ steps.meta_pipeline_manager.outputs.annotations }}
78+
79+
- name: Export multiarch image digest
80+
id: export_image_digest
81+
run: |
82+
echo "digest-pipeline-manager-${{matrix.os}}-${{matrix.arch}}=${{ steps.build_pipeline_manager.outputs.digest }}" >> $GITHUB_OUTPUT
8083
81-
- uses: actions/delete-package-versions@v4
84+
# - uses: actions/delete-package-versions@v4
85+
# with:
86+
# package-name: pipeline-manager
87+
# package-type: 'container'
88+
# min-versions-to-keep: 5
89+
# delete-only-untagged-versions: 'true'
90+
91+
outputs:
92+
digest-pipeline-manager-linux-amd64: ${{ steps.export_image_digest.outputs.digest-pipeline-manager-linux-amd64 }}
93+
digest-pipeline-manager-linux-arm64: ${{ steps.export_image_digest.outputs.digest-pipeline-manager-linux-arm64 }}
94+
95+
push-docker-manifest:
96+
needs: build-and-push-image
97+
runs-on: blacksmith-2vcpu-ubuntu-2204-arm
98+
permissions:
99+
contents: read
100+
packages: write
101+
steps:
102+
- name: Docker meta
103+
id: meta_pipeline_manager
104+
uses: docker/metadata-action@v5
105+
with:
106+
images: |
107+
${{ env.PIPELINE_MANAGER_IMAGE }}
108+
tags: |
109+
type=semver,pattern={{version}}
110+
- name: Log in to Docker Hub
111+
uses: docker/login-action@v3
82112
with:
83-
package-name: pipeline-manager
84-
package-type: 'container'
85-
min-versions-to-keep: 5
86-
delete-only-untagged-versions: 'true'
113+
registry: ${{ env.REGISTRY }}
114+
username: ${{ github.actor }}
115+
password: ${{ secrets.GITHUB_TOKEN }}
116+
- name: Push latest manifest
117+
run: |
118+
docker manifest create ${{ env.PIPELINE_MANAGER_IMAGE }} \
119+
${{ env.PIPELINE_MANAGER_IMAGE }}@${{needs.build-and-push-image.outputs.digest-pipeline-manager-linux-amd64}} \
120+
${{ env.PIPELINE_MANAGER_IMAGE }}@${{needs.build-and-push-image.outputs.digest-pipeline-manager-linux-arm64}}
121+
docker manifest push ${{ env.PIPELINE_MANAGER_IMAGE }}:latest
122+
- name: Push versioned manifest
123+
if: github.event_name == 'release'
124+
run: |
125+
docker manifest create ${{ env.PIPELINE_MANAGER_IMAGE }} \
126+
${{ env.PIPELINE_MANAGER_IMAGE }}@${{needs.build-and-push-image.outputs.digest-pipeline-manager-linux-amd64}} \
127+
${{ env.PIPELINE_MANAGER_IMAGE }}@${{needs.build-and-push-image.outputs.digest-pipeline-manager-linux-arm64}}
128+
docker manifest push ${{ env.PIPELINE_MANAGER_IMAGE }}:${{ github.event.release.tag_name }}

‎.github/workflows/kafka-connect-image.yml‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
name: Build kafka-connect image
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
version:
7+
description: 'kafka-connect image version'
8+
required: true
9+
type: string
10+
11+
env:
12+
REGISTRY: ghcr.io
13+
KAFKA_CONNECT_IMAGE: ghcr.io/${{ github.repository_owner }}/kafka-connect
14+
15+
jobs:
16+
build-and-push-image:
17+
runs-on: ${{ matrix.runner }}
18+
strategy:
19+
matrix:
20+
include:
21+
- runner: [self-hosted, skylake40]
22+
os: linux
23+
arch: amd64
24+
- runner: blacksmith-8vcpu-ubuntu-2204-arm
25+
os: linux
26+
arch: arm64
27+
permissions:
28+
contents: read
29+
packages: write
30+
31+
steps:
32+
- name: Checkout repository
33+
uses: actions/checkout@v4
34+
with:
35+
submodules: true
36+
37+
- name: Docker meta (kafka-connect)
38+
id: meta_kafka_connect
39+
uses: docker/metadata-action@v5
40+
with:
41+
images: ${{ env.KAFKA_CONNECT_IMAGE }}
42+
tags: |
43+
type=raw,value=${{ inputs.version }}
44+
type=raw,value=latest
45+
46+
- name: Log in to the Container registry
47+
uses: docker/login-action@v3
48+
with:
49+
registry: ${{ env.REGISTRY }}
50+
username: ${{ github.actor }}
51+
password: ${{ secrets.GITHUB_TOKEN }}
52+
53+
- name: Build and push kafka-connect image
54+
id: build_kafka_connect
55+
uses: docker/build-push-action@v6
56+
with:
57+
context: deploy/
58+
platforms: ${{matrix.os}}/${{matrix.arch}}
59+
file: deploy/Dockerfile.kafka-connect
60+
push: true
61+
tags: ${{ steps.meta_kafka_connect.outputs.tags }}
62+
63+
- name: Export multiarch image digest
64+
id: export_image_digest
65+
run: |
66+
echo "digest-kafka-connect-${{matrix.os}}-${{matrix.arch}}=${{ steps.build_kafka_connect.outputs.digest }}" >> $GITHUB_OUTPUT
67+
68+
# - uses: actions/delete-package-versions@v4
69+
# with:
70+
# package-name: kafka-connect
71+
# package-type: 'container'
72+
# min-versions-to-keep: 5
73+
# delete-only-untagged-versions: 'true'
74+
75+
outputs:
76+
digest-kafka-connect-linux-amd64: ${{ steps.export_image_digest.outputs.digest-kafka-connect-linux-amd64 }}
77+
digest-kafka-connect-linux-arm64: ${{ steps.export_image_digest.outputs.digest-kafka-connect-linux-arm64 }}
78+
79+
push-docker-manifest:
80+
needs: build-and-push-image
81+
runs-on: blacksmith-2vcpu-ubuntu-2204-arm
82+
permissions:
83+
contents: read
84+
packages: write
85+
steps:
86+
- name: Docker meta
87+
id: meta_kafka_connect
88+
uses: docker/metadata-action@v5
89+
with:
90+
images: |
91+
${{ env.KAFKA_CONNECT_IMAGE }}
92+
tags: |
93+
type=raw,value=${{ inputs.version }}
94+
- name: Log in to Docker Hub
95+
uses: docker/login-action@v3
96+
with:
97+
registry: ${{ env.REGISTRY }}
98+
username: ${{ github.actor }}
99+
password: ${{ secrets.GITHUB_TOKEN }}
100+
- name: Push latest manifest
101+
run: |
102+
docker manifest create ${{ env.KAFKA_CONNECT_IMAGE }} \
103+
${{ env.KAFKA_CONNECT_IMAGE }}@${{needs.build-and-push-image.outputs.digest-kafka-connect-linux-amd64}} \
104+
${{ env.KAFKA_CONNECT_IMAGE }}@${{needs.build-and-push-image.outputs.digest-kafka-connect-linux-arm64}}
105+
docker manifest push ${{ env.KAFKA_CONNECT_IMAGE }}:latest
106+
- name: Push versioned manifest
107+
run: |
108+
docker manifest create ${{ env.KAFKA_CONNECT_IMAGE }} \
109+
${{ env.KAFKA_CONNECT_IMAGE }}@${{needs.build-and-push-image.outputs.digest-kafka-connect-linux-amd64}} \
110+
${{ env.KAFKA_CONNECT_IMAGE }}@${{needs.build-and-push-image.outputs.digest-kafka-connect-linux-arm64}}
111+
docker manifest push ${{ env.KAFKA_CONNECT_IMAGE }}:${{ inputs.version }}

‎deploy/Dockerfile‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# as of now. This will change later.
44
FROM ubuntu:24.04 AS base
55
ENV DEBIAN_FRONTEND=noninteractive
6+
67
# These two environment variables are used to make openssl-sys pick
78
# up libssl-dev and statically link it. Without it, our build defaults
89
# to building a vendored version of OpenSSL.
@@ -77,6 +78,8 @@ RUN /root/.cargo/bin/cargo chef prepare --recipe-path recipe.json
7778
# layer for faster incremental builds of source-code only changes
7879
FROM chef AS builder
7980
ENV CARGO_INCREMENTAL=0
81+
# Making gcc, installed as a part of build-essentials, a default C compiler to build Rust dependencies
82+
ENV CC=gcc-13
8083
COPY --from=planner /app/recipe.json recipe.json
8184
RUN /root/.cargo/bin/cargo chef cook --release --recipe-path recipe.json --bin=pipeline-manager --features=pg-embed
8285
COPY Cargo.toml Cargo.toml
@@ -85,11 +88,10 @@ COPY crates crates
8588
RUN mkdir sql-to-dbsp-compiler || true
8689
COPY sql-to-dbsp-compiler/lib sql-to-dbsp-compiler/lib
8790
COPY --from=web-ui-builder /web-console/build web-console-build
88-
ENV WEBUI_BUILD_DIR=/app/web-console-out
8991
ENV WEBCONSOLE_BUILD_DIR=/app/web-console-build
9092
RUN /root/.cargo/bin/cargo build --release --bin=pipeline-manager --features=pg-embed
9193

92-
# Java build can be performed in parallel
94+
# Java build can be performed in parallel. Outputs should be architecture independent
9395
FROM base AS javabuild
9496
RUN apt install maven git -y
9597
RUN mkdir sql

‎deploy/Dockerfile.demo‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@ RUN apt update && apt install pkg-config \
1717
&& rpk version \
1818
&& rm rpk-linux-$arch.zip \
1919
# Install snowsql
20-
&& curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.28-linux_x86_64.bash \
21-
&& SNOWSQL_DEST=/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.28-linux_x86_64.bash \
20+
# The download URL for mold uses x86_64/aarch64 whereas dpkg --print-architecture says amd64/arm64
21+
&& arch=`dpkg --print-architecture | sed "s/arm64/aarch64/g" | sed "s/amd64/x86_64/g"`; \
22+
curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.3/linux_$arch/snowsql-1.3.2-linux_$arch.bash \
23+
&& SNOWSQL_DEST=/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.3.2-linux_$arch.bash \
2224
&& pip3 install --break-system-packages snowflake-connector-python \
2325
# TODO: only required for running the fraud detection demo. Remove when we clean that up.
2426
&& pip3 install --break-system-packages gdown \

0 commit comments

Comments
 (0)