From e9f9961e6d2e4c370d9b6870aade8dc322290272 Mon Sep 17 00:00:00 2001 From: snowmanmsft <84950230+snowmanmsft@users.noreply.github.com> Date: Fri, 18 Feb 2022 23:39:40 -0800 Subject: [PATCH 01/16] Add CI pipeline for `sbt test` and `pytest` Add CI pipeline for `sbt test` and `pytest` Add CI pipeline for `sbt test` and `pytest` Add Scala SBT CI/CD Add python workflow test, extend timeout to avoid time out errors --- .github/workflows/azure_e2e_test.yml | 57 +++++++++++++++++++++++++++ .github/workflows/scala.yml | 22 +++++++++++ feathr_project/README.md | 3 ++ feathr_project/test/test_azure_e2e.py | 4 +- 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/azure_e2e_test.yml create mode 100644 .github/workflows/scala.yml diff --git a/.github/workflows/azure_e2e_test.yml b/.github/workflows/azure_e2e_test.yml new file mode 100644 index 000000000..e509c8f33 --- /dev/null +++ b/.github/workflows/azure_e2e_test.yml @@ -0,0 +1,57 @@ +name: Azure End to End Testing with Existing Jars + +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9"] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Install Feathr Package + run: | + python -m pip install -e ./feathr_project/ + - name: Test with pytest + env: + REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}} + AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}} + AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} + AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} + S3_ACCESS_KEY: ${{secrets.S3_ACCESS_KEY}} + S3_SECRET_KEY: ${{secrets.S3_SECRET_KEY}} + ADLS_ACCOUNT: ${{secrets.ADLS_ACCOUNT}} + ADLS_KEY: ${{secrets.ADLS_KEY}} + BLOB_ACCOUNT: ${{secrets.BLOB_ACCOUNT}} + BLOB_KEY: ${{secrets.BLOB_KEY}} + + run: | + cd feathr_project + pytest \ No newline at end of file diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml new file mode 100644 index 000000000..afeaa5da6 --- /dev/null +++ b/.github/workflows/scala.yml @@ -0,0 +1,22 @@ +name: Feathr Scala Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + java-version: '8' + distribution: 'temurin' + - name: Run tests + run: sbt test diff --git a/feathr_project/README.md b/feathr_project/README.md index 7d7dded47..580c2303e 100644 --- a/feathr_project/README.md +++ b/feathr_project/README.md @@ -1,3 +1,6 @@ +[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Feathr Runtime](https://github.com/linkedin/feathr/actions/workflows/scala.yml/badge.svg)](https://github.com/linkedin/feathr/actions/workflows/scala.yml) + + Feathr Python Project Developer Guide ============================= diff --git a/feathr_project/test/test_azure_e2e.py b/feathr_project/test/test_azure_e2e.py index df0abc9f4..3dae880c2 100644 --- a/feathr_project/test/test_azure_e2e.py +++ b/feathr_project/test/test_azure_e2e.py @@ -24,7 +24,7 @@ def test_feathr_online_store(): job_res = client.materialize_features() # just assume the job is successful without validating the actual result in Redis. Might need to consolidate # this part with the test_feathr_online_store test case - client.wait_job_to_finish(timeout_sec=300) + client.wait_job_to_finish(timeout_sec=600) res = client.online_get_features('nycTaxiDemoFeature', '265', ['f_location_avg_fare', 'f_location_max_fare']) # just assme there are values. We don't hard code the values for now for testing # the correctness of the feature generation should be garunteed by feathr runtime. @@ -55,7 +55,7 @@ def test_feathr_get_historical_features(): client = FeathrClient() returned_spark_job = client.join_offline_features() - res_url = client.get_job_result_uri(block=True, timeout_sec=500) + res_url = client.get_job_result_uri(block=True, timeout_sec=600) tmp_dir = tempfile.TemporaryDirectory() client.feathr_synapse_laucher.download_result(result_path=res_url, local_folder=tmp_dir.name) dataframe_list = [] From d6118b54539b6bcb3f24eb554ff4c744b1475c8b Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Mon, 21 Feb 2022 19:20:00 -0800 Subject: [PATCH 02/16] use public jars --- .../data/feathr_user_workspace/feathr_config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/feathr_project/data/feathr_user_workspace/feathr_config.yaml b/feathr_project/data/feathr_user_workspace/feathr_config.yaml index 79e6d26f2..c8e85f841 100644 --- a/feathr_project/data/feathr_user_workspace/feathr_config.yaml +++ b/feathr_project/data/feathr_user_workspace/feathr_config.yaml @@ -1,23 +1,23 @@ # DO NOT MOVE OR DELETE THIS FILE resource: azure: - PROJECT_NAME: "frame_getting_started" + PROJECT_NAME: "feathr_getting_started" # Spark cluster configs SYNAPSE_DEV_URL: "https://feathrazuretest3synapse.dev.azuresynapse.net" SYNAPSE_POOL_NAME: "spark24" - SYNAPSE_WORKSPACE_DIR: "abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/frame_getting_started" + SYNAPSE_WORKSPACE_DIR: "abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_getting_started" SYNAPSE_EXECUTOR_SIZE: "Small" SYNAPSE_EXECUTOR_NUM: 4 # Redis configs to access Redis cluster - REDIS_PASSWORD: 'Li7Nn63iNB0x731VTnnz2Vr29WYJHx7JlAzCaH9lbHw=' + REDIS_PASSWORD: '' REDIS_HOST: "feathrazuretest3redis.redis.cache.windows.net" REDIS_PORT: 6380 REDIS_SSL_ENABLED: True # Feathr Job configuration. Support local paths, path start with http(s)://, and paths start with abfs(s):// - FEATHR_RUNTIME_LOCATION: abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/frame_getting_started/feathr-assembly-0.1.0-SNAPSHOT.jar + FEATHR_RUNTIME_LOCATION: https://azurefeathrstorage.blob.core.windows.net/public/feathr-assembly-0.1.0-SNAPSHOT.jar # configure number of parts for the spark output for feature generation job RESULT_OUTPUT_PARTS: "1" From 325025e3ba131c254678a18ffdb548d9b64cd5e4 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Mon, 21 Feb 2022 19:26:55 -0800 Subject: [PATCH 03/16] avoid resource constraint by using only 1 python version --- .github/workflows/azure_e2e_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/azure_e2e_test.yml b/.github/workflows/azure_e2e_test.yml index e509c8f33..d7b12fa53 100644 --- a/.github/workflows/azure_e2e_test.yml +++ b/.github/workflows/azure_e2e_test.yml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9"] + python-version: ["3.8"] steps: - uses: actions/checkout@v2 From 6fbcf4b45deed5e81007d385394df7607cddf315 Mon Sep 17 00:00:00 2001 From: Yuqing Wei Date: Tue, 22 Feb 2022 14:20:56 +0800 Subject: [PATCH 04/16] Add Sbt Assembly & Jar Upload --- .github/workflows/scala.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index afeaa5da6..c56f61611 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -20,3 +20,12 @@ jobs: distribution: 'temurin' - name: Run tests run: sbt test + - name: Build JAR + run: sbt assembly + - name: Azure Blob Storage Upload + uses: fixpoint/azblob-upload-artifact@v4 + with: + connection-string: ${{secrets.JAR_BLOB_CONNECTION_STRING}} + name: Feathr JAR + path: 'target/scala-2.11/feathr-assembly-0.1.0-SNAPSHOT.JAR' + container: ${{secrets.JAR_BLOB_CONTAINER}} \ No newline at end of file From 6e858a4dc8f43228d1827d847ac92d1fe38edf52 Mon Sep 17 00:00:00 2001 From: Yuqing Wei Date: Tue, 22 Feb 2022 14:27:00 +0800 Subject: [PATCH 05/16] test connection string, will revert --- .github/workflows/scala.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index c56f61611..6a1ce2bab 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -25,7 +25,7 @@ jobs: - name: Azure Blob Storage Upload uses: fixpoint/azblob-upload-artifact@v4 with: - connection-string: ${{secrets.JAR_BLOB_CONNECTION_STRING}} + connection-string: 'BlobEndpoint=https://azurefeathrstorage.blob.core.windows.net/;QueueEndpoint=https://azurefeathrstorage.queue.core.windows.net/;FileEndpoint=https://azurefeathrstorage.file.core.windows.net/;TableEndpoint=https://azurefeathrstorage.table.core.windows.net/;SharedAccessSignature=sv=2020-08-04&ss=bfqt&srt=sco&sp=rwdlacupx&se=2022-02-25T14:24:11Z&st=2022-02-22T06:24:11Z&spr=https&sig=eV%2F9IrGz1ctYtjSwar2pyr0NKyOrH8ftHOK3dCod%2B%2F0%3D' name: Feathr JAR path: 'target/scala-2.11/feathr-assembly-0.1.0-SNAPSHOT.JAR' - container: ${{secrets.JAR_BLOB_CONTAINER}} \ No newline at end of file + container: 'github' \ No newline at end of file From 2a5ed83ef60f181868591d775de836753b69226c Mon Sep 17 00:00:00 2001 From: Yuqing Wei Date: Tue, 22 Feb 2022 14:48:52 +0800 Subject: [PATCH 06/16] params updated --- .github/workflows/scala.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index 6a1ce2bab..e98e2bf4d 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -25,7 +25,7 @@ jobs: - name: Azure Blob Storage Upload uses: fixpoint/azblob-upload-artifact@v4 with: - connection-string: 'BlobEndpoint=https://azurefeathrstorage.blob.core.windows.net/;QueueEndpoint=https://azurefeathrstorage.queue.core.windows.net/;FileEndpoint=https://azurefeathrstorage.file.core.windows.net/;TableEndpoint=https://azurefeathrstorage.table.core.windows.net/;SharedAccessSignature=sv=2020-08-04&ss=bfqt&srt=sco&sp=rwdlacupx&se=2022-02-25T14:24:11Z&st=2022-02-22T06:24:11Z&spr=https&sig=eV%2F9IrGz1ctYtjSwar2pyr0NKyOrH8ftHOK3dCod%2B%2F0%3D' + connection-string: ${{SPARK_JAR_BLOB_CONNECTION_STRING}} name: Feathr JAR - path: 'target/scala-2.11/feathr-assembly-0.1.0-SNAPSHOT.JAR' - container: 'github' \ No newline at end of file + path: 'target/scala-2.11/feathr-assembly-0.1.0-SNAPSHOT.jar' + container: ${{SPARK_JAR_BLOB_CONTAINER}} \ No newline at end of file From 3291a3fe742cc82b45ecbc343d1006203abf4f1d Mon Sep 17 00:00:00 2001 From: Yuqing Wei Date: Tue, 22 Feb 2022 14:50:33 +0800 Subject: [PATCH 07/16] param correction --- .github/workflows/scala.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index e98e2bf4d..9be156003 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -25,7 +25,7 @@ jobs: - name: Azure Blob Storage Upload uses: fixpoint/azblob-upload-artifact@v4 with: - connection-string: ${{SPARK_JAR_BLOB_CONNECTION_STRING}} + connection-string: ${{secrets.SPARK_JAR_BLOB_CONNECTION_STRING}} name: Feathr JAR path: 'target/scala-2.11/feathr-assembly-0.1.0-SNAPSHOT.jar' - container: ${{SPARK_JAR_BLOB_CONTAINER}} \ No newline at end of file + container: ${{secrets.SPARK_JAR_BLOB_CONTAINER}} \ No newline at end of file From 82a994fbe9f8c3a7b69a8116d2bc73ac52f71761 Mon Sep 17 00:00:00 2001 From: Yuqing Wei Date: Tue, 22 Feb 2022 15:03:25 +0800 Subject: [PATCH 08/16] Update scala.yml --- .github/workflows/scala.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index 9be156003..5adb94e5c 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -22,10 +22,11 @@ jobs: run: sbt test - name: Build JAR run: sbt assembly - - name: Azure Blob Storage Upload + - name: Azure Blob Storage Upload (Overwrite) uses: fixpoint/azblob-upload-artifact@v4 with: connection-string: ${{secrets.SPARK_JAR_BLOB_CONNECTION_STRING}} - name: Feathr JAR + name: 'feathr_jar_github_action' path: 'target/scala-2.11/feathr-assembly-0.1.0-SNAPSHOT.jar' - container: ${{secrets.SPARK_JAR_BLOB_CONTAINER}} \ No newline at end of file + container: ${{secrets.SPARK_JAR_BLOB_CONTAINER}} + cleanup: 'true' \ No newline at end of file From b3d7ae36f80624f86592a1174471551fad48d58a Mon Sep 17 00:00:00 2001 From: Yuqing Wei Date: Tue, 22 Feb 2022 15:19:20 +0800 Subject: [PATCH 09/16] e2e github action workflow test --- .github/workflows/azure_e2e_test.yml | 8 +++----- .../data/feathr_user_workspace/feathr_config.yaml | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/azure_e2e_test.yml b/.github/workflows/azure_e2e_test.yml index d7b12fa53..725b81191 100644 --- a/.github/workflows/azure_e2e_test.yml +++ b/.github/workflows/azure_e2e_test.yml @@ -4,12 +4,10 @@ name: Azure End to End Testing with Existing Jars # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions on: - pull_request: + workflow_run: + workflows: ["Feathr Scala Tests"] types: - - opened - - reopened - - synchronize - - ready_for_review + - completed jobs: build: diff --git a/feathr_project/data/feathr_user_workspace/feathr_config.yaml b/feathr_project/data/feathr_user_workspace/feathr_config.yaml index 8bd566785..2770eb32e 100644 --- a/feathr_project/data/feathr_user_workspace/feathr_config.yaml +++ b/feathr_project/data/feathr_user_workspace/feathr_config.yaml @@ -17,7 +17,7 @@ resource: REDIS_SSL_ENABLED: True # Feathr Job configuration. Support local paths, path start with http(s)://, and paths start with abfs(s):// - FEATHR_RUNTIME_LOCATION: https://azurefeathrstorage.blob.core.windows.net/public/feathr-assembly-0.1.0-SNAPSHOT.jar + FEATHR_RUNTIME_LOCATION: https://feathrazuretest3storage.blob.core.windows.net/feathrazuretest3fs/feathr_jar_github_action/feathr-assembly-0.1.0-SNAPSHOT.jar # configure number of parts for the spark output for feature generation job RESULT_OUTPUT_PARTS: "1" From 8560bb5c0448dfa9495f086c76c5899d05bef1b5 Mon Sep 17 00:00:00 2001 From: Yuqing Wei Date: Tue, 22 Feb 2022 15:45:59 +0800 Subject: [PATCH 10/16] dependency on workflow_run only work in master/default branch roll back to test e2e workflow in github --- .github/workflows/azure_e2e_test.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/azure_e2e_test.yml b/.github/workflows/azure_e2e_test.yml index 725b81191..d7b12fa53 100644 --- a/.github/workflows/azure_e2e_test.yml +++ b/.github/workflows/azure_e2e_test.yml @@ -4,10 +4,12 @@ name: Azure End to End Testing with Existing Jars # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions on: - workflow_run: - workflows: ["Feathr Scala Tests"] + pull_request: types: - - completed + - opened + - reopened + - synchronize + - ready_for_review jobs: build: From c411505c7c30e074a55a83260c1d3ddffa8e1d7a Mon Sep 17 00:00:00 2001 From: Yuqing Wei Date: Tue, 22 Feb 2022 15:57:08 +0800 Subject: [PATCH 11/16] Revert "e2e github action workflow test" needs to config connection string to access runtime location --- feathr_project/data/feathr_user_workspace/feathr_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feathr_project/data/feathr_user_workspace/feathr_config.yaml b/feathr_project/data/feathr_user_workspace/feathr_config.yaml index 2770eb32e..8bd566785 100644 --- a/feathr_project/data/feathr_user_workspace/feathr_config.yaml +++ b/feathr_project/data/feathr_user_workspace/feathr_config.yaml @@ -17,7 +17,7 @@ resource: REDIS_SSL_ENABLED: True # Feathr Job configuration. Support local paths, path start with http(s)://, and paths start with abfs(s):// - FEATHR_RUNTIME_LOCATION: https://feathrazuretest3storage.blob.core.windows.net/feathrazuretest3fs/feathr_jar_github_action/feathr-assembly-0.1.0-SNAPSHOT.jar + FEATHR_RUNTIME_LOCATION: https://azurefeathrstorage.blob.core.windows.net/public/feathr-assembly-0.1.0-SNAPSHOT.jar # configure number of parts for the spark output for feature generation job RESULT_OUTPUT_PARTS: "1" From 3b3aebfa6546eb617a89e38951d1a1698c3f2932 Mon Sep 17 00:00:00 2001 From: Yuqing Wei Date: Tue, 22 Feb 2022 16:21:31 +0800 Subject: [PATCH 12/16] remove env setting, overwrite jar location --- .github/workflows/azure_e2e_test.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/azure_e2e_test.yml b/.github/workflows/azure_e2e_test.yml index d7b12fa53..1f0039e97 100644 --- a/.github/workflows/azure_e2e_test.yml +++ b/.github/workflows/azure_e2e_test.yml @@ -41,16 +41,13 @@ jobs: python -m pip install -e ./feathr_project/ - name: Test with pytest env: + FEATHR_RUNTIME_LOCATION: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_jar_github_action/feathr-assembly-0.1.0-SNAPSHOT.jar' REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}} AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}} AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} S3_ACCESS_KEY: ${{secrets.S3_ACCESS_KEY}} S3_SECRET_KEY: ${{secrets.S3_SECRET_KEY}} - ADLS_ACCOUNT: ${{secrets.ADLS_ACCOUNT}} - ADLS_KEY: ${{secrets.ADLS_KEY}} - BLOB_ACCOUNT: ${{secrets.BLOB_ACCOUNT}} - BLOB_KEY: ${{secrets.BLOB_KEY}} run: | cd feathr_project From a3ad474cec8ed8f9ab54a9ef6bddbf39452c78a7 Mon Sep 17 00:00:00 2001 From: Yuqing Wei Date: Tue, 22 Feb 2022 14:20:56 +0800 Subject: [PATCH 13/16] Add Sbt Assembly & Jar Upload step in CI/CD workflow Enable Workflow dependency if fail, skip e2e test --- .github/workflows/azure_e2e_test.yml | 24 +++++++++++++----------- .github/workflows/scala.yml | 10 ++++++++++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/azure_e2e_test.yml b/.github/workflows/azure_e2e_test.yml index d7b12fa53..fb6b2b760 100644 --- a/.github/workflows/azure_e2e_test.yml +++ b/.github/workflows/azure_e2e_test.yml @@ -4,16 +4,15 @@ name: Azure End to End Testing with Existing Jars # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions on: - pull_request: + workflow_run: + workflows: ["Feathr Scala Tests"] types: - - opened - - reopened - - synchronize - - ready_for_review + - completed jobs: - build: + on-success: runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} strategy: fail-fast: false matrix: @@ -41,17 +40,20 @@ jobs: python -m pip install -e ./feathr_project/ - name: Test with pytest env: + FEATHR_RUNTIME_LOCATION: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_jar_github_action/feathr-assembly-0.1.0-SNAPSHOT.jar' REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}} AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}} AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} S3_ACCESS_KEY: ${{secrets.S3_ACCESS_KEY}} S3_SECRET_KEY: ${{secrets.S3_SECRET_KEY}} - ADLS_ACCOUNT: ${{secrets.ADLS_ACCOUNT}} - ADLS_KEY: ${{secrets.ADLS_KEY}} - BLOB_ACCOUNT: ${{secrets.BLOB_ACCOUNT}} - BLOB_KEY: ${{secrets.BLOB_KEY}} run: | cd feathr_project - pytest \ No newline at end of file + pytest + + on-failure: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'failure' }} + steps: + - run: echo "Feathr Scala Tests workflow was a failure, skip e2e test." \ No newline at end of file diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index afeaa5da6..5adb94e5c 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -20,3 +20,13 @@ jobs: distribution: 'temurin' - name: Run tests run: sbt test + - name: Build JAR + run: sbt assembly + - name: Azure Blob Storage Upload (Overwrite) + uses: fixpoint/azblob-upload-artifact@v4 + with: + connection-string: ${{secrets.SPARK_JAR_BLOB_CONNECTION_STRING}} + name: 'feathr_jar_github_action' + path: 'target/scala-2.11/feathr-assembly-0.1.0-SNAPSHOT.jar' + container: ${{secrets.SPARK_JAR_BLOB_CONTAINER}} + cleanup: 'true' \ No newline at end of file From 36717245e856ffbfbd6e9a5d97ee784350291a18 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Tue, 22 Feb 2022 01:54:11 -0800 Subject: [PATCH 14/16] Update scala.yml --- .github/workflows/scala.yml | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index 5adb94e5c..20ce05de8 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -1,4 +1,4 @@ -name: Feathr Scala Tests +name: Feathr Scala Tests And Azure E2E Integration on: push: @@ -29,4 +29,36 @@ jobs: name: 'feathr_jar_github_action' path: 'target/scala-2.11/feathr-assembly-0.1.0-SNAPSHOT.jar' container: ${{secrets.SPARK_JAR_BLOB_CONTAINER}} - cleanup: 'true' \ No newline at end of file + cleanup: 'true' + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Install Feathr Package + run: | + python -m pip install -e ./feathr_project/ + - name: Test with pytest + env: + FEATHR_RUNTIME_LOCATION: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_jar_github_action/feathr-assembly-0.1.0-SNAPSHOT.jar' + REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}} + AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}} + AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} + AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} + S3_ACCESS_KEY: ${{secrets.S3_ACCESS_KEY}} + S3_SECRET_KEY: ${{secrets.S3_SECRET_KEY}} + + run: | + cd feathr_project + pytest \ No newline at end of file From 273059d1eaed5dd5a87a9a5eed4875979c284a80 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Tue, 22 Feb 2022 01:55:09 -0800 Subject: [PATCH 15/16] Update scala.yml --- .github/workflows/scala.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index 20ce05de8..c5793ac36 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -30,7 +30,6 @@ jobs: path: 'target/scala-2.11/feathr-assembly-0.1.0-SNAPSHOT.jar' container: ${{secrets.SPARK_JAR_BLOB_CONTAINER}} cleanup: 'true' - - uses: actions/checkout@v2 - name: Set up Python 3.8 uses: actions/setup-python@v2 with: From f031b084f51c14f026a2a0344fb6ee650692b648 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Tue, 22 Feb 2022 02:09:34 -0800 Subject: [PATCH 16/16] Delete azure_e2e_test.yml --- .github/workflows/azure_e2e_test.yml | 59 ---------------------------- 1 file changed, 59 deletions(-) delete mode 100644 .github/workflows/azure_e2e_test.yml diff --git a/.github/workflows/azure_e2e_test.yml b/.github/workflows/azure_e2e_test.yml deleted file mode 100644 index fb6b2b760..000000000 --- a/.github/workflows/azure_e2e_test.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Azure End to End Testing with Existing Jars - -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -on: - workflow_run: - workflows: ["Feathr Scala Tests"] - types: - - completed - -jobs: - on-success: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - strategy: - fail-fast: false - matrix: - python-version: ["3.8"] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Install Feathr Package - run: | - python -m pip install -e ./feathr_project/ - - name: Test with pytest - env: - FEATHR_RUNTIME_LOCATION: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_jar_github_action/feathr-assembly-0.1.0-SNAPSHOT.jar' - REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}} - AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}} - AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} - AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} - S3_ACCESS_KEY: ${{secrets.S3_ACCESS_KEY}} - S3_SECRET_KEY: ${{secrets.S3_SECRET_KEY}} - - run: | - cd feathr_project - pytest - - on-failure: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'failure' }} - steps: - - run: echo "Feathr Scala Tests workflow was a failure, skip e2e test." \ No newline at end of file