Skip to content

Commit ae0acdf

Browse files
chandrasekharan-zipstackchandrasekharan-zipstack
authored
[MISC] Updated workflows to use github's cache for faster builds (#1368)
* [misc] Updated workflows to use github's cache for docker builds * minor: Updated workflows to not push images for testing * [MISC] Updated workflow build to use correct Dockerfile for services * minor: Updated workflows to push images again post testing * [MISC] Refactored build push workflows to use, marked push to false for testing * [MISC] Fixed production build workflow issue related to VERSION env and context resolution * [MISC] Refactored build workflow to simplify tag resolution * [MISC] Moved commented line in build workflow * [MISC] Uncomment push option in workflow runs that was added for testing * [MISC] Updated build definition to add tools in a separate profile, removed related fix added to run-platform.sh, updated Github tool build workflow to also use bake * [MISC] Updated Dockerfile paths and context for tools to make workflow build easily * Revert "[MISC] Updated build definition to add tools in a separate profile, removed related fix added to run-platform.sh, updated Github tool build workflow to also use bake" This reverts commit 9e7fe1c.
1 parent 9e08566 commit ae0acdf

2 files changed

Lines changed: 74 additions & 23 deletions

File tree

‎.github/workflows/docker-tools-build-push.yaml‎

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,23 +30,42 @@ jobs:
3030
- name: Checkout code
3131
uses: actions/checkout@v4
3232

33+
# Set up Docker Buildx for better caching
34+
- name: Set up Docker Buildx
35+
uses: docker/setup-buildx-action@v3
36+
37+
# Log in to Docker Hub
3338
- name: Login to Docker Hub
3439
uses: docker/login-action@v3
3540
with:
3641
username: ${{ secrets.DOCKERHUB_USERNAME }}
3742
password: ${{ secrets.DOCKERHUB_TOKEN }}
3843

39-
- name: Build tool-classifier
40-
if: github.event.inputs.service_name=='tool-classifier'
41-
run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/classifier
42-
- name: Build tool-structure
43-
if: github.event.inputs.service_name=='tool-structure'
44-
run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/structure
45-
- name: Build tool-text-extractor
46-
if: github.event.inputs.service_name=='tool-text-extractor'
47-
run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} ./tools/text_extractor
48-
- name: Build tool-sidecar
49-
if: github.event.inputs.service_name=='tool-sidecar'
50-
run: docker build -t unstract/${{github.event.inputs.service_name}}:${{ github.event.inputs.tag }} -f docker/dockerfiles/tool-sidecar.Dockerfile .
51-
- name: Push Docker image to Docker Hub
52-
run: docker push unstract/${{ github.event.inputs.service_name }}:${{ github.event.inputs.tag }}
44+
# Define build configuration based on service name
45+
- name: Set build configuration
46+
id: build-config
47+
run: |
48+
if [ "${{ github.event.inputs.service_name }}" == "tool-classifier" ]; then
49+
echo "context=./tools/classifier" >> $GITHUB_OUTPUT
50+
echo "dockerfile=" >> $GITHUB_OUTPUT
51+
elif [ "${{ github.event.inputs.service_name }}" == "tool-structure" ]; then
52+
echo "context=./tools/structure" >> $GITHUB_OUTPUT
53+
echo "dockerfile=" >> $GITHUB_OUTPUT
54+
elif [ "${{ github.event.inputs.service_name }}" == "tool-text-extractor" ]; then
55+
echo "context=./tools/text_extractor" >> $GITHUB_OUTPUT
56+
echo "dockerfile=" >> $GITHUB_OUTPUT
57+
elif [ "${{ github.event.inputs.service_name }}" == "tool-sidecar" ]; then
58+
echo "context=." >> $GITHUB_OUTPUT
59+
echo "dockerfile=docker/dockerfiles/tool-sidecar.Dockerfile" >> $GITHUB_OUTPUT
60+
fi
61+
62+
# Build and push Docker image
63+
- name: Build and push ${{ github.event.inputs.service_name }}
64+
uses: docker/build-push-action@v5
65+
with:
66+
context: ${{ steps.build-config.outputs.context }}
67+
file: ${{ steps.build-config.outputs.dockerfile }}
68+
push: true
69+
tags: unstract/${{ github.event.inputs.service_name }}:${{ github.event.inputs.tag }}
70+
cache-from: type=gha
71+
cache-to: type=gha,mode=max

‎.github/workflows/production-build.yaml‎

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,26 +27,58 @@ jobs:
2727
with:
2828
ref: ${{ github.event.release.tag_name }}
2929

30-
3130
- name: Checkout code for branch
3231
if: github.event_name != 'release'
3332
uses: actions/checkout@v4
3433

34+
# Set up Docker Buildx for better caching
35+
- name: Set up Docker Buildx
36+
uses: docker/setup-buildx-action@v3
3537

38+
# Log in to Docker Hub
3639
- name: Login to Docker Hub
3740
uses: docker/login-action@v3
3841
with:
3942
username: ${{ secrets.DOCKERHUB_USERNAME }}
4043
password: ${{ secrets.DOCKERHUB_TOKEN }}
4144

42-
- name: Build and push images
43-
working-directory: ./docker
45+
# Set version tag based on event type
46+
- name: Set version tag
47+
id: set-tag
48+
run: echo "DOCKER_VERSION_TAG=${{ github.event.release.tag_name || github.event.inputs.tag }}" >> $GITHUB_ENV
49+
50+
# Set up additional tags for release builds
51+
- name: Set image tags
52+
id: tags
4453
run: |
45-
DOCKER_VERSION_TAG=${{ github.event.release.tag_name || github.event.inputs.tag}}
46-
VERSION=$DOCKER_VERSION_TAG docker compose -f docker-compose.build.yaml build --no-cache ${{ matrix.service_name }}
47-
docker push unstract/${{ matrix.service_name }}:$DOCKER_VERSION_TAG
54+
# Check if service exists in the config
55+
echo "Checking if service ${{ matrix.service_name }} exists in docker-compose.build.yaml"
56+
if ! grep -q "^ ${{ matrix.service_name }}:" ./docker/docker-compose.build.yaml; then
57+
echo "Service ${{ matrix.service_name }} not found in docker-compose.build.yaml" && exit 1
58+
fi
59+
60+
# Set latest tag for releases
4861
if [ "${{ github.event_name }}" = "release" ]; then
49-
echo "This workflow was triggered by a release event."
50-
docker tag unstract/${{ matrix.service_name }}:$DOCKER_VERSION_TAG unstract/${{ matrix.service_name }}:latest
51-
docker push unstract/${{ matrix.service_name }}:latest
62+
echo "latest=true" >> $GITHUB_OUTPUT
63+
echo "IMAGE_TAGS=unstract/${{ matrix.service_name }}:${{ env.DOCKER_VERSION_TAG }},unstract/${{ matrix.service_name }}:latest" >> $GITHUB_ENV
64+
else
65+
echo "latest=false" >> $GITHUB_OUTPUT
66+
echo "IMAGE_TAGS=unstract/${{ matrix.service_name }}:${{ env.DOCKER_VERSION_TAG }}" >> $GITHUB_ENV
5267
fi
68+
69+
# Build and push using Docker Bake
70+
- name: Build and push image
71+
uses: docker/bake-action@v5
72+
env:
73+
VERSION: ${{ env.DOCKER_VERSION_TAG }}
74+
with:
75+
files: ./docker/docker-compose.build.yaml
76+
targets: ${{ matrix.service_name }}
77+
push: true
78+
# Context resolved along with docker-compose.build.yaml, ensure resolved path is repo root
79+
set: |
80+
*.tags=${{ env.IMAGE_TAGS }}
81+
*.context=.
82+
*.args.VERSION=${{ env.DOCKER_VERSION_TAG }}
83+
*.cache-from=type=gha,scope=${{ matrix.service_name }}
84+
*.cache-to=type=gha,mode=max,scope=${{ matrix.service_name }}

0 commit comments

Comments
 (0)