Skip to content

Commit 6de01a8

Browse files
gzgz
committed
Rework CI.
- Build binaries, jars and docker image once - Reuse binaries by saving them as artifacts in github actions - Refactor earthly CI jobs into smaller github actions (gets rid of earthly) Signed-off-by: Gerd Zellweger <mail@gerdzellweger.com>
1 parent 7375374 commit 6de01a8

36 files changed

Lines changed: 1170 additions & 1583 deletions

‎.github/workflows/benchmark-command.yml‎

Lines changed: 0 additions & 48 deletions
This file was deleted.

‎.github/workflows/build-docker-dev.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: "Docker CI Image"
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
paths:
7+
- ".github/workflows/build-docker-dev.yml"
8+
- "deploy/build.Dockerfile"
9+
10+
env:
11+
REGISTRY: ghcr.io
12+
IMAGE_NAME: ghcr.io/feldera/feldera-dev
13+
14+
jobs:
15+
build-docker-ci-dev:
16+
name: Build Docker Container used by Actions
17+
runs-on: ubuntu-latest
18+
19+
steps:
20+
- name: Checkout Repository
21+
uses: actions/checkout@v4
22+
23+
- name: Set up Docker Buildx
24+
uses: docker/setup-buildx-action@v3
25+
26+
- name: Login to GHCR Registry
27+
uses: docker/login-action@v3
28+
with:
29+
registry: ${{ env.REGISTRY }}
30+
username: ${{ github.actor }}
31+
password: ${{ secrets.GITHUB_TOKEN }}
32+
33+
- name: Build and push Multi-Arch Docker Image
34+
run: |
35+
docker buildx build \
36+
--platform linux/amd64,linux/arm64 \
37+
-t ${{ env.IMAGE_NAME }}:latest \
38+
-t ${{ env.IMAGE_NAME }}:${{ github.sha }} \
39+
-f build.Dockerfile . \
40+
--push
41+
working-directory: deploy

‎.github/workflows/build-docker.yml‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
name: Build and Publish Docker Image
2+
3+
on:
4+
workflow_call:
5+
6+
permissions:
7+
contents: read
8+
packages: write
9+
10+
env:
11+
REGISTRY: ghcr.io
12+
13+
# This would be easier if we could just build all platforms in one job, but
14+
# building the ARM version on x86 is very slow due to the pre-compile step we do
15+
# in the Dockerfile.
16+
#
17+
# See also https://github.com/docker/build-push-action/issues/671 and especially the linked solution
18+
# https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
19+
# for more details on how this produces a single image with multiple architectures from different runners.
20+
jobs:
21+
build-docker:
22+
name: Package Docker Image
23+
24+
strategy:
25+
matrix:
26+
include:
27+
- runner: [self-hosted, skylake40]
28+
rust_target: x86_64-unknown-linux-gnu
29+
docker_arch: amd64
30+
docker_platform: linux/amd64
31+
- runner: blacksmith-16vcpu-ubuntu-2204-arm
32+
rust_target: aarch64-unknown-linux-gnu
33+
docker_arch: arm64
34+
docker_platform: linux/arm64
35+
runs-on: ${{ matrix.runner }}
36+
37+
steps:
38+
- name: Checkout Repository
39+
uses: actions/checkout@v4
40+
41+
- name: Download Binaries
42+
id: binaries
43+
uses: actions/download-artifact@v4
44+
with:
45+
name: feldera-binaries-${{ matrix.rust_target }}
46+
path: build
47+
#run-id: ${{ inputs.run-id }}
48+
# Token is only needed when run-id is set
49+
#github-token: ${{ secrets.GITHUB_TOKEN }}
50+
51+
- name: Download Compiler Binaries
52+
uses: actions/download-artifact@v4
53+
with:
54+
name: feldera-sql-compiler
55+
path: build
56+
57+
# Remove if https://github.com/actions/upload-artifact/issues/38 ever gets fixed
58+
- name: Make binaries executable
59+
run: |
60+
chmod +x ${{ steps.binaries.outputs.download-path }}/*
61+
ls -la ${{ steps.binaries.outputs.download-path }}
62+
63+
- name: Prepare Platform Environment Variable
64+
run: |
65+
platform=${{ matrix.docker_platform }}
66+
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
67+
68+
- name: Docker meta
69+
id: meta
70+
uses: docker/metadata-action@v5
71+
with:
72+
images: ${{ vars.FELDERA_IMAGE_NAME }}
73+
74+
- name: Set up Docker Buildx
75+
uses: docker/setup-buildx-action@v3
76+
77+
- name: Login to GHCR
78+
uses: docker/login-action@v3
79+
with:
80+
registry: ${{ env.REGISTRY }}
81+
username: ${{ github.actor }}
82+
password: ${{ secrets.GITHUB_TOKEN }}
83+
84+
- name: Build and push by digest
85+
id: build
86+
uses: docker/build-push-action@v6
87+
with:
88+
# `context` is somehow important to set, otherwise the build will use it's own git checkout instead of the one
89+
# checkout out earlier that also has the artifacts (!?)
90+
context: .
91+
file: deploy/Dockerfile
92+
platforms: ${{ matrix.docker_platform }}
93+
labels: ${{ steps.meta.outputs.labels }}
94+
tags: ${{ vars.FELDERA_IMAGE_NAME }}
95+
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
96+
cache-from: type=gha
97+
cache-to: type=gha,mode=max
98+
99+
- name: Export digest
100+
run: |
101+
mkdir -p ${{ runner.temp }}/digests
102+
digest="${{ steps.build.outputs.digest }}"
103+
touch "${{ runner.temp }}/digests/${digest#sha256:}"
104+
105+
- name: Upload digest
106+
uses: actions/upload-artifact@v4
107+
with:
108+
name: digests-${{ env.PLATFORM_PAIR }}
109+
path: ${{ runner.temp }}/digests/*
110+
if-no-files-found: error
111+
retention-days: 7
112+
113+
# One problem with this workflow is the unknown/unknown architecture
114+
# it adds in the github UI. It's a github bug:
115+
# https://github.com/orgs/community/discussions/45969
116+
merge-manifests:
117+
name: Merge Docker Manifests
118+
runs-on: ubuntu-latest
119+
needs: build-docker
120+
steps:
121+
- name: Download digests
122+
uses: actions/download-artifact@v4
123+
with:
124+
path: ${{ runner.temp }}/digests
125+
pattern: digests-*
126+
merge-multiple: true
127+
128+
- name: Login to GHCR
129+
uses: docker/login-action@v3
130+
with:
131+
registry: ${{ env.REGISTRY }}
132+
username: ${{ github.actor }}
133+
password: ${{ secrets.GITHUB_TOKEN }}
134+
135+
- name: Set up Docker Buildx
136+
uses: docker/setup-buildx-action@v3
137+
138+
- name: Docker meta
139+
id: meta
140+
uses: docker/metadata-action@v5
141+
with:
142+
images: ${{ vars.FELDERA_IMAGE_NAME }}
143+
tags: |
144+
type=ref,event=branch
145+
type=ref,event=pr
146+
type=semver,pattern={{version}}
147+
type=semver,pattern={{major}}.{{minor}}
148+
type=sha,format=long
149+
150+
- name: Create manifest list and push
151+
working-directory: ${{ runner.temp }}/digests
152+
run: |
153+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
154+
$(printf '${{ vars.FELDERA_IMAGE_NAME }}@sha256:%s ' *)
155+
156+
- name: Inspect image
157+
run: |
158+
docker buildx imagetools inspect ${{ vars.FELDERA_IMAGE_NAME }}:${{ steps.meta.outputs.version }}

‎.github/workflows/build-java.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Build Java Sources
2+
3+
on:
4+
workflow_call:
5+
6+
jobs:
7+
build-jar:
8+
name: Build Compiler
9+
runs-on: [self-hosted, skylake40]
10+
container:
11+
image: ghcr.io/feldera/feldera-dev:0a775f772b2ebde13708744d3e6a219ca4a492d2
12+
options: --user=ubuntu
13+
steps:
14+
- name: Checkout repository
15+
uses: actions/checkout@v4
16+
17+
- name: Cache Maven dependencies
18+
uses: actions/cache@v4
19+
with:
20+
path: ~/.m2/repository
21+
key: maven-${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
22+
restore-keys: |
23+
maven-${{ runner.os }}-
24+
25+
- name: Build JAR
26+
run: bash build.sh
27+
working-directory: ./sql-to-dbsp-compiler
28+
29+
- name: Copy JAR to artifacts directory
30+
run: |
31+
mkdir -p build-artifacts
32+
cp ./sql-to-dbsp-compiler/SQL-compiler/target/sql2dbsp.jar build-artifacts/
33+
cp ./sql-to-dbsp-compiler/SQL-compiler/target/sql2dbsp-jar-with-dependencies.jar build-artifacts/
34+
cp ./sql-to-dbsp-compiler/slt/target/slt-jar-with-dependencies.jar build-artifacts/
35+
cp ./sql-to-dbsp-compiler/slt/target/slt.jar build-artifacts/
36+
37+
- name: Upload build artifact
38+
uses: actions/upload-artifact@v4
39+
with:
40+
name: feldera-sql-compiler
41+
path: build-artifacts
42+
retention-days: 7

‎.github/workflows/build-rust.yml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
name: Build Rust Sources
2+
3+
on:
4+
workflow_call:
5+
6+
env:
7+
CARGO_FLAGS: "--release --locked --all-targets --features pubsub-emulator-test,iceberg-tests-fs,iceberg-tests-glue"
8+
FELDERA_PLATFORM_VERSION_SUFFIX: ${{ github.sha }}
9+
10+
jobs:
11+
build-rust:
12+
name: Build Rust Binaries
13+
14+
# We run this on two different architectures (x86_64 and aarch64)
15+
strategy:
16+
matrix:
17+
include:
18+
- runner: [self-hosted, skylake40]
19+
arch: x86_64
20+
target: x86_64-unknown-linux-gnu
21+
- runner: blacksmith-16vcpu-ubuntu-2204-arm
22+
arch: aarch64
23+
target: aarch64-unknown-linux-gnu
24+
25+
runs-on: ${{ matrix.runner }}
26+
27+
container:
28+
image: ghcr.io/feldera/feldera-dev:0a775f772b2ebde13708744d3e6a219ca4a492d2
29+
options: --user=ubuntu
30+
31+
steps:
32+
- name: Checkout repository
33+
uses: actions/checkout@v4
34+
35+
# The docker container when executed in the action runs with a different home directory
36+
# than we set in the dev container (?), hence this step is necessary (sigh)
37+
# https://github.com/actions/runner/issues/863
38+
- name: Rustup set default toolchain
39+
run: rustup default stable
40+
41+
- name: Cache dependencies
42+
uses: Swatinem/rust-cache@v2
43+
44+
- name: Build Rust binaries
45+
run: |
46+
cargo build ${{ env.CARGO_FLAGS }} --target=${{ matrix.target }}
47+
48+
# Get list of executables
49+
- name: Collect executables
50+
id: collect
51+
run: |
52+
# Run again with --message-format=json to list out executables
53+
# (No real recompile since nothing has changed).
54+
# Then transform newlines to spaces for the artifact step.
55+
EXES=$(cargo build ${{ env.CARGO_FLAGS }} --target=${{ matrix.target }} --message-format=json \
56+
| jq -r '.executable | select(. != null)' | tr '\n' ' ')
57+
echo "Found executables: $EXES"
58+
# Save it as an output variable for subsequent steps
59+
echo "executables=$EXES" >> $GITHUB_OUTPUT
60+
61+
# Copy all executables into a single directory because upload-artifact does not support
62+
# multiple paths or `|` in glob patterns
63+
- name: Copy executables
64+
run: |
65+
mkdir -p build-artifacts
66+
for exe in ${{ steps.collect.outputs.executables }}; do
67+
cp "$exe" build-artifacts/
68+
done
69+
mkdir -p build-release-artifacts
70+
# Move the executables we ship to users to a separate directory
71+
mv build-artifacts/fda build-release-artifacts/
72+
mv build-artifacts/pipeline-manager build-release-artifacts/
73+
74+
# Upload test binaries as one artifact
75+
- name: Upload build artifacts
76+
uses: actions/upload-artifact@v4
77+
with:
78+
name: feldera-test-binaries-${{ matrix.target }}
79+
path: build-artifacts
80+
retention-days: 7
81+
82+
# Upload binaries to run the product as another artifact
83+
- name: Upload build artifacts
84+
uses: actions/upload-artifact@v4
85+
with:
86+
name: feldera-binaries-${{ matrix.target }}
87+
path: build-release-artifacts
88+
retention-days: 7

0 commit comments

Comments
 (0)