From c11d4968d24dbb5ccb35ac1fc5248205bcb41071 Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Wed, 6 Dec 2023 03:46:26 +0100 Subject: [PATCH 01/11] Use quay.io as a cache for the Docker images --- .github/workflows/main.yml | 12 ++++++++++++ ci/build-in-docker.sh | 17 ++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a2674df6..41037758 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,8 +48,20 @@ jobs: - name: Set up QEMU integration for Docker run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # using such an action is more secure than using custom scripts + - name: Log into quay.io + uses: docker/login-action@v3 + # secrets are not available when building a pull request + if: ${{ github.event_name != "pull_request" }} + with: + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSPHRASE }} + - name: Build run: bash ci/build-in-docker.sh + env: + CACHE_FROM: 1 + PUSH_IMAGE: ${{ github.event_name == "pull_request" && "1" || "" }} - name: Archive artifacts uses: actions/upload-artifact@v2 diff --git a/ci/build-in-docker.sh b/ci/build-in-docker.sh index 322a0b59..98bd0fb9 100755 --- a/ci/build-in-docker.sh +++ b/ci/build-in-docker.sh @@ -57,7 +57,16 @@ else build_args+=("--pull") fi -image_tag="linuxdeploy-build:$ARCH" +image_tag="quay.io/theassassin/linuxdeploy-build:$ARCH" + +if [[ "${CACHE_FROM:-}" != "" ]]; then + warning "using cached image from quay: $image_tag" + + build_args+=( + "--cache-from" + "$image_tag" + ) +fi docker build \ --build-arg ARCH="$ARCH" \ @@ -66,6 +75,12 @@ docker build \ -t "$image_tag" \ "$this_dir"/docker +# by default, we are not logged into the registry and therefore must not attempt to push the image +if [[ "${PUSH_IMAGE:-}" ]]; then + warning "pushing image to quay (requires login): $image_tag" + docker push "$image_tag" +fi + docker_args=() # only if there's more than 1G of free space in RAM, we can build in a RAM disk if [[ "${GITHUB_ACTIONS:-}" != "" ]]; then From 72a843a88ffc75b687b2238b79a66cd117b821aa Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Wed, 6 Dec 2023 03:59:33 +0100 Subject: [PATCH 02/11] fixup! Use quay.io as a cache for the Docker images --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 41037758..846a541e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,7 +52,7 @@ jobs: - name: Log into quay.io uses: docker/login-action@v3 # secrets are not available when building a pull request - if: ${{ github.event_name != "pull_request" }} + if: ${{ github.event_name != 'pull_request' }} with: username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSPHRASE }} @@ -61,7 +61,7 @@ jobs: run: bash ci/build-in-docker.sh env: CACHE_FROM: 1 - PUSH_IMAGE: ${{ github.event_name == "pull_request" && "1" || "" }} + PUSH_IMAGE: ${{ github.event_name == 'pull_request' && '1' || '' }} - name: Archive artifacts uses: actions/upload-artifact@v2 From 23d2c6b8b18bf878656e0c2d1619485cc46bb8fb Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Wed, 6 Dec 2023 04:01:19 +0100 Subject: [PATCH 03/11] fixup! fixup! Use quay.io as a cache for the Docker images --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 846a541e..39d4f833 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,6 +56,7 @@ jobs: with: username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSPHRASE }} + registry: quay.io - name: Build run: bash ci/build-in-docker.sh From e99b99fd115dd02826677a0b0b0083e1390e50a2 Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Wed, 6 Dec 2023 04:04:47 +0100 Subject: [PATCH 04/11] fixup! fixup! fixup! Use quay.io as a cache for the Docker images --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 39d4f833..d845385a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,17 +15,20 @@ jobs: ARCH: [x86_64, i386, armhf, aarch64] BUILD_TYPE: ["appimage"] USE_STATIC_RUNTIME: [""] + DISABLE_PUSH: [""] include: # test build - ARCH: x86_64 DOCKER_ARCH: amd64 BUILD_TYPE: coverage + DISABLE_PUSH: 1 # experimental build - ARCH: x86_64 BUILD_TYPE: appimage USE_STATIC_RUNTIME: 1 + DISABLE_PUSH: 1 fail-fast: false @@ -62,7 +65,7 @@ jobs: run: bash ci/build-in-docker.sh env: CACHE_FROM: 1 - PUSH_IMAGE: ${{ github.event_name == 'pull_request' && '1' || '' }} + PUSH_IMAGE: ${{ matrix.DISABLE_PUSH == '' && github.event_name == 'pull_request' && '1' || '' }} - name: Archive artifacts uses: actions/upload-artifact@v2 From 311bd5b30c988dc02ee96331c9b12e8993aeb616 Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Wed, 6 Dec 2023 04:14:13 +0100 Subject: [PATCH 05/11] fixup! fixup! fixup! fixup! Use quay.io as a cache for the Docker images --- .github/workflows/main.yml | 2 +- ci/build-in-docker.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d845385a..d6401a9f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -65,7 +65,7 @@ jobs: run: bash ci/build-in-docker.sh env: CACHE_FROM: 1 - PUSH_IMAGE: ${{ matrix.DISABLE_PUSH == '' && github.event_name == 'pull_request' && '1' || '' }} + PUSH_IMAGE: ${{ (matrix.DISABLE_PUSH == '' && github.event_name == 'pull_request') && '1' || '' }} - name: Archive artifacts uses: actions/upload-artifact@v2 diff --git a/ci/build-in-docker.sh b/ci/build-in-docker.sh index 98bd0fb9..2b9ab6a5 100755 --- a/ci/build-in-docker.sh +++ b/ci/build-in-docker.sh @@ -79,6 +79,8 @@ docker build \ if [[ "${PUSH_IMAGE:-}" ]]; then warning "pushing image to quay (requires login): $image_tag" docker push "$image_tag" +else + warning "\$PUSH_IMAGE not set, not pushing image" fi docker_args=() From 6e74559cabb35b865941fef5526c674681e8fc72 Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Wed, 6 Dec 2023 04:19:29 +0100 Subject: [PATCH 06/11] fixup! fixup! fixup! fixup! fixup! Use quay.io as a cache for the Docker images --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d6401a9f..854a5e01 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -65,7 +65,7 @@ jobs: run: bash ci/build-in-docker.sh env: CACHE_FROM: 1 - PUSH_IMAGE: ${{ (matrix.DISABLE_PUSH == '' && github.event_name == 'pull_request') && '1' || '' }} + PUSH_IMAGE: ${{ (matrix.DISABLE_PUSH == '' && github.event_name != 'pull_request') && '1' || '' }} - name: Archive artifacts uses: actions/upload-artifact@v2 From 123bdabf0b01d429cbce7ce7b5dcc6649f139133 Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Sun, 10 Dec 2023 20:26:35 +0100 Subject: [PATCH 07/11] Make output of free stable --- ci/build-in-docker.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/build-in-docker.sh b/ci/build-in-docker.sh index 2b9ab6a5..f51087e4 100755 --- a/ci/build-in-docker.sh +++ b/ci/build-in-docker.sh @@ -87,7 +87,7 @@ docker_args=() # only if there's more than 1G of free space in RAM, we can build in a RAM disk if [[ "${GITHUB_ACTIONS:-}" != "" ]]; then warning "Building on GitHub actions, which does not support --tmpfs flag -> building on regular disk" -elif [[ "$(free -m | grep "Mem:" | awk '{print $4}')" -gt 1024 ]]; then +elif [[ "$(env LC_ALL=C free -m | grep "Mem:" | awk '{print $4}')" -gt 1024 ]]; then info "Host system has enough free memory -> building in RAM disk" docker_args+=( "--tmpfs" @@ -109,7 +109,6 @@ if [ -t 1 ]; then docker_args+=("-t") fi -DOCKER_OPTS=() # fix for https://stackoverflow.com/questions/51195528/rcc-error-in-resource-qrc-cannot-find-file-png if [ "${CI:-}" != "" ]; then docker_args+=( From 2c911f77a1528ef65f156296506e2aa9f40964aa Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Mon, 11 Dec 2023 01:24:39 +0100 Subject: [PATCH 08/11] fixup! fixup! fixup! fixup! fixup! fixup! Use quay.io as a cache for the Docker images --- ci/build-in-docker.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/build-in-docker.sh b/ci/build-in-docker.sh index f51087e4..566da852 100755 --- a/ci/build-in-docker.sh +++ b/ci/build-in-docker.sh @@ -66,6 +66,9 @@ if [[ "${CACHE_FROM:-}" != "" ]]; then "--cache-from" "$image_tag" ) + + # need to be pulled manually for versions that do not use buildkit (yet), apparently + docker pull "$image_tag" fi docker build \ From 35cd01d0ad70202a1698fd541de0c79ad83d69c5 Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Mon, 11 Dec 2023 01:32:48 +0100 Subject: [PATCH 09/11] Trigger build From d391f38357c2939d510372801c7394507712a8b8 Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Mon, 11 Dec 2023 01:40:20 +0100 Subject: [PATCH 10/11] Log Docker commands --- ci/build-in-docker.sh | 46 ++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/ci/build-in-docker.sh b/ci/build-in-docker.sh index 566da852..488af4c3 100755 --- a/ci/build-in-docker.sh +++ b/ci/build-in-docker.sh @@ -71,12 +71,15 @@ if [[ "${CACHE_FROM:-}" != "" ]]; then docker pull "$image_tag" fi -docker build \ - --build-arg ARCH="$ARCH" \ - --build-arg docker_arch="$docker_arch" \ - "${build_args[@]}" \ - -t "$image_tag" \ - "$this_dir"/docker +( + set -x + docker build \ + --build-arg ARCH="$ARCH" \ + --build-arg docker_arch="$docker_arch" \ + "${build_args[@]}" \ + -t "$image_tag" \ + "$this_dir"/docker +) # by default, we are not logged into the registry and therefore must not attempt to push the image if [[ "${PUSH_IMAGE:-}" ]]; then @@ -125,17 +128,20 @@ fi # b) allow the build scripts to "mv" the binaries into the /out directory uid="${UID:-"$(id -u)"}" info "Running build with uid $uid" -docker run \ - --rm \ - -i \ - -e GITHUB_RUN_NUMBER \ - -e ARCH \ - -e BUILD_TYPE \ - -e USE_STATIC_RUNTIME \ - -e CI \ - --user "$uid" \ - "${docker_args[@]}" \ - -v "$(readlink -f "$this_dir"/..):/ws" \ - -w /ws \ - "$image_tag" \ - bash -xc "$build_script" +( + set -x + docker run \ + --rm \ + -i \ + -e GITHUB_RUN_NUMBER \ + -e ARCH \ + -e BUILD_TYPE \ + -e USE_STATIC_RUNTIME \ + -e CI \ + --user "$uid" \ + "${docker_args[@]}" \ + -v "$(readlink -f "$this_dir"/..):/ws" \ + -w /ws \ + "$image_tag" \ + bash -xc "$build_script" +) From 08e41e95a35deb3724a0c8ff2a8f73e62d39a582 Mon Sep 17 00:00:00 2001 From: TheAssassin Date: Mon, 11 Dec 2023 01:41:34 +0100 Subject: [PATCH 11/11] fixup! Log Docker commands --- ci/build-in-docker.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/build-in-docker.sh b/ci/build-in-docker.sh index 488af4c3..8c78fd42 100755 --- a/ci/build-in-docker.sh +++ b/ci/build-in-docker.sh @@ -68,7 +68,10 @@ if [[ "${CACHE_FROM:-}" != "" ]]; then ) # need to be pulled manually for versions that do not use buildkit (yet), apparently - docker pull "$image_tag" + ( + set -x + docker pull "$image_tag" + ) fi (