From 4d6536aaefd8149efef85f1f8ba7ea3932a390e7 Mon Sep 17 00:00:00 2001 From: snowmanmsft <84950230+snowmanmsft@users.noreply.github.com> Date: Fri, 18 Feb 2022 23:51:09 -0800 Subject: [PATCH 01/12] Add python CI --- .github/workflows/python-package-conda.yml | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/python-package-conda.yml diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml new file mode 100644 index 000000000..3841ce0fe --- /dev/null +++ b/.github/workflows/python-package-conda.yml @@ -0,0 +1,35 @@ +name: Python Package using Conda + +on: [push] + +jobs: + build-linux: + runs-on: ubuntu-latest + strategy: + max-parallel: 5 + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: 3.10 + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + - name: Install dependencies + run: | + conda env update --file environment.yml --name base + - name: Lint with flake8 + run: | + conda install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + conda install pytest + cd feathr_project + pytest From 6a730ee46f3273cc9ff3b646187883b57b3d5fb7 Mon Sep 17 00:00:00 2001 From: snowmanmsft <84950230+snowmanmsft@users.noreply.github.com> Date: Fri, 18 Feb 2022 23:53:15 -0800 Subject: [PATCH 02/12] Update python-package-conda.yml --- .github/workflows/python-package-conda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 3841ce0fe..9220feaca 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -10,10 +10,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.10 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: 3.9 - name: Add conda to system path run: | # $CONDA is an environment variable pointing to the root of the miniconda directory From 0bb727dbe335a4ed33284f1ce0c7cbd2d3d0b3f3 Mon Sep 17 00:00:00 2001 From: snowmanmsft <84950230+snowmanmsft@users.noreply.github.com> Date: Fri, 18 Feb 2022 23:57:52 -0800 Subject: [PATCH 03/12] Create python-package-conda.yml --- .github/workflows/python-package-conda.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 9220feaca..e3560de2d 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -18,9 +18,6 @@ jobs: run: | # $CONDA is an environment variable pointing to the root of the miniconda directory echo $CONDA/bin >> $GITHUB_PATH - - name: Install dependencies - run: | - conda env update --file environment.yml --name base - name: Lint with flake8 run: | conda install flake8 @@ -31,5 +28,6 @@ jobs: - name: Test with pytest run: | conda install pytest + python -m pip install ./feathr_project/ cd feathr_project pytest From 8a353c210972b0f470dfbe99005bbf8abaf40ae4 Mon Sep 17 00:00:00 2001 From: snowmanmsft <84950230+snowmanmsft@users.noreply.github.com> Date: Sat, 19 Feb 2022 00:16:27 -0800 Subject: [PATCH 04/12] Update python-package-conda.yml --- .github/workflows/python-package-conda.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index e3560de2d..cd227c981 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -26,8 +26,14 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest + env: + REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}} + AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}} + AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} + AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} run: | conda install pytest python -m pip install ./feathr_project/ cd feathr_project + pytest From d65e38482b2c79a70f863360e02abc44f23b9ab5 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Sat, 19 Feb 2022 01:23:23 -0800 Subject: [PATCH 05/12] Update python-package-conda.yml --- .github/workflows/python-package-conda.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index cd227c981..fb86a3652 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -4,9 +4,10 @@ on: [push] jobs: build-linux: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: - max-parallel: 5 + matrix: + os: [ubuntu-latest, windows-latest] steps: - uses: actions/checkout@v2 @@ -33,7 +34,6 @@ jobs: AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} run: | conda install pytest - python -m pip install ./feathr_project/ + python -m pip -m install ./feathr_project/ cd feathr_project - pytest From f4df0b3a0a9a373e707fc1ded180a36be3b3a0d8 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Sat, 19 Feb 2022 01:24:20 -0800 Subject: [PATCH 06/12] Update python-package-conda.yml --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index fb86a3652..b1cac4596 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -34,6 +34,6 @@ jobs: AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} run: | conda install pytest - python -m pip -m install ./feathr_project/ + python -m pip install ./feathr_project/ cd feathr_project pytest From afdaf93b72888ed8e93d1026fc88b9e5c2174b77 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Sat, 19 Feb 2022 01:25:23 -0800 Subject: [PATCH 07/12] Update python-package-conda.yml --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index b1cac4596..22988dbbf 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -34,6 +34,6 @@ jobs: AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} run: | conda install pytest - python -m pip install ./feathr_project/ + pip install ./feathr_project/ cd feathr_project pytest From e6e002016c711a20e845c49460489f47c24dae68 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Sat, 19 Feb 2022 01:55:17 -0800 Subject: [PATCH 08/12] Update python-package-conda.yml --- .github/workflows/python-package-conda.yml | 42 ++++++++++++++-------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 22988dbbf..bfe695eff 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -1,39 +1,51 @@ -name: Python Package using Conda +# This is copied from the python starter kit for github actions: +# https://github.com/actions/starter-workflows/blob/main/ci/python-package.yml +name: Azure End to End Testing with existing Jars -on: [push] +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +on: + push: + branches: [ $default-branch ] + pull_request: + branches: [ $default-branch ] jobs: - build-linux: - runs-on: ${{ matrix.os }} + build: + + runs-on: ubuntu-latest strategy: - matrix: - os: [ubuntu-latest, windows-latest] + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: 3.9 - - name: Add conda to system path + python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH + python -m pip install --upgrade pip + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with flake8 run: | - conda install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Install Feathr Package + run: | + python -m pip install -e ./feathr_project/ - name: Test with pytest env: REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}} AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}} AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} - run: | - conda install pytest - pip install ./feathr_project/ + run: | cd feathr_project pytest From e30d39458a60b4937a8bac63bb858f2f1a0f703e Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Sat, 19 Feb 2022 01:58:12 -0800 Subject: [PATCH 09/12] Update python-package-conda.yml --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index bfe695eff..337865c69 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 From 50d2db0c5569f301a1e968f085db4468bdba8773 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Sat, 19 Feb 2022 02:00:24 -0800 Subject: [PATCH 10/12] Update python-package-conda.yml --- .github/workflows/python-package-conda.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 337865c69..bef721665 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -5,11 +5,7 @@ name: Azure End to End Testing with existing Jars # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -on: - push: - branches: [ $default-branch ] - pull_request: - branches: [ $default-branch ] +on: [push, pull_request] jobs: build: From 16ff4c62f100bf696441ac6ae674ffe14fd469e2 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Sat, 19 Feb 2022 02:06:14 -0800 Subject: [PATCH 11/12] Update python-package-conda.yml --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index bef721665..2cada08a6 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9"] steps: - uses: actions/checkout@v2 From df793bf4928f28d8b5e78762b082861bf977fdd2 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Sat, 19 Feb 2022 02:17:47 -0800 Subject: [PATCH 12/12] update github actions and increase time out --- .github/workflows/python-package-conda.yml | 8 +++++++- feathr_project/test/test_azure_e2e.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 2cada08a6..dc88a0a34 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -5,7 +5,13 @@ name: Azure End to End Testing with existing Jars # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -on: [push, pull_request] +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review jobs: build: diff --git a/feathr_project/test/test_azure_e2e.py b/feathr_project/test/test_azure_e2e.py index df0abc9f4..3dae880c2 100644 --- a/feathr_project/test/test_azure_e2e.py +++ b/feathr_project/test/test_azure_e2e.py @@ -24,7 +24,7 @@ def test_feathr_online_store(): job_res = client.materialize_features() # just assume the job is successful without validating the actual result in Redis. Might need to consolidate # this part with the test_feathr_online_store test case - client.wait_job_to_finish(timeout_sec=300) + client.wait_job_to_finish(timeout_sec=600) res = client.online_get_features('nycTaxiDemoFeature', '265', ['f_location_avg_fare', 'f_location_max_fare']) # just assme there are values. We don't hard code the values for now for testing # the correctness of the feature generation should be garunteed by feathr runtime. @@ -55,7 +55,7 @@ def test_feathr_get_historical_features(): client = FeathrClient() returned_spark_job = client.join_offline_features() - res_url = client.get_job_result_uri(block=True, timeout_sec=500) + res_url = client.get_job_result_uri(block=True, timeout_sec=600) tmp_dir = tempfile.TemporaryDirectory() client.feathr_synapse_laucher.download_result(result_path=res_url, local_folder=tmp_dir.name) dataframe_list = []