chore: Reusable workflows to reuse build_wheels workflow in publish (#2880)

kevjumba · web-flow · commit c2a5c507e97a · 2022-06-28T16:16:41.000-07:00
* Update reusable workflows

Signed-off-by: Kevin Zhang &lt;kzhang@tecton.ai&gt;

* Update reusable workflows

Signed-off-by: Kevin Zhang &lt;kzhang@tecton.ai&gt;

* Update reusable workflows

Signed-off-by: Kevin Zhang &lt;kzhang@tecton.ai&gt;

* Fix

Signed-off-by: Kevin Zhang &lt;kzhang@tecton.ai&gt;

* Fix

Signed-off-by: Kevin Zhang &lt;kzhang@tecton.ai&gt;

* Fix

Signed-off-by: Kevin Zhang &lt;kzhang@tecton.ai&gt;

* Fix

Signed-off-by: Kevin Zhang &lt;kzhang@tecton.ai&gt;
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -1,6 +1,7 @@
 name: build_wheels
 
-on: workflow_dispatch
+# Call this workflow from other workflows in the repository by specifying "uses: ./.github/workflows/build_wheels.yml"
+on: [workflow_dispatch, workflow_call]
 
 jobs:
   get-version:
@@ -76,6 +77,7 @@ jobs:
             make install-go-ci-dependencies
             git status
           CIBW_BEFORE_TEST: "cd {project} && git status"
+      # py3.10 on MacOS does not work with Go so we have to install separately. Issue is tracked here: https://github.com/feast-dev/feast/issues/2881.
       - name: Build py310 specific wheels for macos
         if: matrix.os == 'macos-10.15'
         uses: pypa/cibuildwheel@v2.7.0
@@ -169,6 +171,7 @@ jobs:
           go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
           go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
           pip install dist/*tar.gz
+      # py3.10 on MacOS does not work with Go so we have to install separately. Issue is tracked here: https://github.com/feast-dev/feast/issues/2881
       - name: Install dist w/o go
         if: ${{ matrix.from-source && matrix.python-version == '3.10' && matrix.os == 'macos-10.15'}}
         run: pip install dist/*tar.gz
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -125,9 +125,12 @@ jobs:
       - name: Publish Helm charts
         run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX
 
+  build_wheels:
+    uses: ./.github/workflows/build_wheels.yml
+
   publish-python-sdk:
     runs-on: ubuntu-latest
-    needs: [verify-python-wheel]
+    needs: [build_wheels]
     steps:
       - uses: actions/download-artifact@v2
         with:
@@ -138,160 +141,6 @@ jobs:
           user: __token__
           password: ${{ secrets.PYPI_PASSWORD }}
 
-  build-python-wheel:
-    name: Build wheels on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ ubuntu-latest, macos-10.15 ]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Node
-        uses: actions/setup-node@v2
-        with:
-          node-version: '17.x'
-          registry-url: 'https://registry.npmjs.org'
-      - name: Build UI
-        run: make build-ui
-      - name: Build wheels
-        uses: pypa/cibuildwheel@v2.7.0
-        env:
-          CIBW_BUILD: "cp3*_x86_64"
-          CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64"
-          CIBW_ARCHS: "native"
-          CIBW_ENVIRONMENT: >
-            COMPILE_GO=True PATH=$PATH:/usr/local/go/bin
-          CIBW_BEFORE_ALL_LINUX: |
-            curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz
-            tar -C /usr/local -xzf go.tar.gz
-            go version
-          CIBW_BEFORE_ALL_MACOS: |
-            curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg
-            sudo installer -pkg python.pkg -target /
-          CIBW_BEFORE_BUILD: |
-            make install-protoc-dependencies
-            make install-go-proto-dependencies
-            make install-go-ci-dependencies
-            git status
-          CIBW_BEFORE_TEST: "cd {project} && git status"
-      - name: Build py310 specific wheels for macos
-        if: matrix.os == 'macos-10.15'
-        uses: pypa/cibuildwheel@v2.7.0
-        env:
-          CIBW_BUILD: "cp310-macosx_x86_64"
-          CIBW_ARCHS: "native"
-      - uses: actions/upload-artifact@v2
-        with:
-          name: wheels
-          path: ./wheelhouse/*.whl
-
-  build-source-distribution:
-    name: Build source distribution
-    runs-on: macos-10.15
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Python
-        id: setup-python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.10"
-          architecture: x64
-      - name: Setup Node
-        uses: actions/setup-node@v2
-        with:
-          node-version: '17.x'
-          registry-url: 'https://registry.npmjs.org'
-      - name: Build and install dependencies
-        run: |
-          pip install -U pip setuptools wheel twine
-          make install-protoc-dependencies
-          make install-go-proto-dependencies
-          make install-go-ci-dependencies
-          make build-ui
-      - name: Build
-        run: |
-          python3 setup.py sdist
-      - uses: actions/upload-artifact@v2
-        with:
-          name: wheels
-          path: dist/*
-
-  verify-python-wheel:
-    runs-on: ${{ matrix.os }}
-    needs: [build-python-wheel, build-source-distribution]
-    strategy:
-      matrix:
-        os: [ubuntu-latest,  macos-10.15 ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10"]
-        from-source: [ True, False ]
-    env:
-      # this script is for testing servers
-      # it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself
-      TEST_SCRIPT: |
-        timeout 10s $@ & pid=$!
-        wait $pid
-        ret=$?
-        if [[ $ret -ne 124 ]]
-        then
-           exit $ret
-        else
-           echo "Succeeded!"
-        fi
-    steps:
-      - name: Setup Python
-        id: setup-python
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: x64
-      - uses: actions/setup-go@v3
-        with:
-          go-version: '>=1.17.0'
-      - uses: actions/download-artifact@v2
-        with:
-          name: wheels
-          path: dist
-      - name: Install wheel
-        if: ${{ !matrix.from-source }}
-        # try to install all wheels; only the current platform wheel should be actually installed
-        run: |
-          cd dist/
-          pip install wheel
-          for f in *.whl; do pip install $f || true; done
-      - name: Install dist with go
-        if: ${{ matrix.from-source && (matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest')}}
-        env:
-          COMPILE_GO: "True"
-        run: |
-          pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0'
-          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
-          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
-          pip install dist/*tar.gz
-      - name: Install dist w/o go
-        if: ${{ matrix.from-source && matrix.python-version == '3.10' && matrix.os == 'macos-10.15'}}
-        run: pip install dist/*tar.gz
-      - name: Install OS X dependencies
-        if: matrix.os == 'macos-10.15'
-        run: brew install coreutils
-      - name: Smoke test
-        run: |
-          feast init test_repo
-          cd test_repo/
-          feast apply
-          echo "$TEST_SCRIPT" > run-and-wait.sh
-          bash run-and-wait.sh feast serve
-          bash run-and-wait.sh feast ui
-      # We disable this test for the Python 3.10 binary since it does not include Go.
-      - name: Smoke test with go
-        if: matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest'
-        run: |
-          cd test_repo/
-          feast apply
-          echo "$TEST_SCRIPT" > run-and-wait.sh
-          pip install cffi
-          printf "\ngo_feature_retrieval: True" >> feature_store.yaml
-          bash run-and-wait.sh feast serve
-
   publish-java-sdk:
     container: maven:3.6-jdk-11
     runs-on: ubuntu-latest
