From feca3916f84e9fb3b1b9a33e1c60894f8a7c39b8 Mon Sep 17 00:00:00 2001 From: enya-yx Date: Mon, 14 Nov 2022 14:31:54 +0800 Subject: [PATCH 1/3] Insert test coverage check for python client into github pipeline --- .github/workflows/.coveragerc_db | 8 +++ .github/workflows/.coveragerc_local | 8 +++ .github/workflows/.coveragerc_sy | 8 +++ .github/workflows/pull_request_push_test.yml | 71 ++++++++++++++++++-- feathr_project/setup.py | 1 + 5 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/.coveragerc_db create mode 100644 .github/workflows/.coveragerc_local create mode 100644 .github/workflows/.coveragerc_sy diff --git a/.github/workflows/.coveragerc_db b/.github/workflows/.coveragerc_db new file mode 100644 index 000000000..410ae2191 --- /dev/null +++ b/.github/workflows/.coveragerc_db @@ -0,0 +1,8 @@ +[run] +omit = feathr_project/feathr/registry/_feature_registry_purview.py + feathr_project/feathr/spark_provider/_synapse_submission.py + feathr_project/feathr/spark_provider/_localspark_submission.py +[report] +exclude_lines = + pragma: no cover + @abstract \ No newline at end of file diff --git a/.github/workflows/.coveragerc_local b/.github/workflows/.coveragerc_local new file mode 100644 index 000000000..0f517b928 --- /dev/null +++ b/.github/workflows/.coveragerc_local @@ -0,0 +1,8 @@ +[run] +omit = feathr_project/feathr/registry/_feature_registry_purview.py + feathr_project/feathr/spark_provider/_databricks_submission.py + feathr_project/feathr/spark_provider/_synapse_submission.py +[report] +exclude_lines = + pragma: no cover + @abstract \ No newline at end of file diff --git a/.github/workflows/.coveragerc_sy b/.github/workflows/.coveragerc_sy new file mode 100644 index 000000000..f44e27cef --- /dev/null +++ b/.github/workflows/.coveragerc_sy @@ -0,0 +1,8 @@ +[run] +omit = feathr_project/feathr/registry/_feature_registry_purview.py + feathr_project/feathr/spark_provider/_databricks_submission.py + feathr_project/feathr/spark_provider/_localspark_submission.py +[report] +exclude_lines = + pragma: no cover + @abstract \ No newline at end of file diff --git a/.github/workflows/pull_request_push_test.yml b/.github/workflows/pull_request_push_test.yml index 0eb0e059b..f526df7fd 100644 --- a/.github/workflows/pull_request_push_test.yml +++ b/.github/workflows/pull_request_push_test.yml @@ -65,7 +65,7 @@ jobs: databricks_test: runs-on: ubuntu-latest - if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' steps: - uses: actions/checkout@v2 with: @@ -128,12 +128,11 @@ jobs: SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}} run: | # run only test with databricks. run in 4 parallel jobs - pytest -n 6 feathr_project/test/ - + pytest -n 6 --cov-report term-missing --cov=feathr_project/feathr feathr_project/test --cov-config=.github/workflows/.coveragerc_db azure_synapse_test: # might be a bit duplication to setup both the azure_synapse test and databricks test, but for now we will keep those to accelerate the test speed runs-on: ubuntu-latest - if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' steps: - uses: actions/checkout@v2 with: @@ -197,7 +196,7 @@ jobs: run: | # skip databricks related test as we just ran the test; also seperate databricks and synapse test to make sure there's no write conflict # run in 4 parallel jobs to make the time shorter - pytest -n 6 feathr_project/test/ + pytest -n 6 --cov-report term-missing --cov=feathr_project/feathr feathr_project/test --cov-config=.github/workflows/.coveragerc_sy local_spark_test: runs-on: ubuntu-latest @@ -255,8 +254,66 @@ jobs: SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}} run: | # skip cloud related tests - pytest feathr_project/test/test_local_spark_e2e.py - + pytest --cov-report term-missing --cov=feathr_project/feathr feathr_project/test --cov-config=.github/workflows/.coveragerc_local + + local_spark_test_to_remove: + runs-on: ubuntu-latest + if: (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'coverage')) + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + java-version: "8" + distribution: "temurin" + - name: Build JAR + run: | + sbt assembly + # remote folder for CI upload + echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_github_action_$(date +"%H_%M_%S")" >> $GITHUB_ENV + # get local jar name without paths so version change won't affect it + echo "FEATHR_LOCAL_JAR_NAME=$(ls target/scala-2.12/*.jar| xargs -n 1 basename)" >> $GITHUB_ENV + # get local jar name without path + echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls target/scala-2.12/*.jar)" >> $GITHUB_ENV + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Feathr Package + run: | + python -m pip install --upgrade pip + python -m pip install -e ./feathr_project/[all] + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Run Feathr with Local Spark + env: + PROJECT_CONFIG__PROJECT_NAME: "feathr_github_ci_local" + SPARK_CONFIG__SPARK_CLUSTER: local + REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}} + AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}} + AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} + AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} + S3_ACCESS_KEY: ${{secrets.S3_ACCESS_KEY}} + S3_SECRET_KEY: ${{secrets.S3_SECRET_KEY}} + ADLS_ACCOUNT: ${{secrets.ADLS_ACCOUNT}} + ADLS_KEY: ${{secrets.ADLS_KEY}} + BLOB_ACCOUNT: ${{secrets.BLOB_ACCOUNT}} + BLOB_KEY: ${{secrets.BLOB_KEY}} + JDBC_TABLE: ${{secrets.JDBC_TABLE}} + JDBC_USER: ${{secrets.JDBC_USER}} + JDBC_PASSWORD: ${{secrets.JDBC_PASSWORD}} + JDBC_DRIVER: ${{secrets.JDBC_DRIVER}} + JDBC_SF_PASSWORD: ${{secrets.JDBC_SF_PASSWORD}} + KAFKA_SASL_JAAS_CONFIG: ${{secrets.KAFKA_SASL_JAAS_CONFIG}} + MONITORING_DATABASE_SQL_PASSWORD: ${{secrets.MONITORING_DATABASE_SQL_PASSWORD}} + COSMOS1_KEY: ${{secrets.COSMOS1_KEY}} + SQL1_USER: ${{secrets.SQL1_USER}} + SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}} + run: | + # skip cloud related tests + pytest --cov-report term-missing --cov=feathr_project/feathr feathr_project/test --cov-config=.github/workflows/.coveragerc_local + failure_notification: # If any failure, warning message will be sent needs: [sbt_test, python_lint, databricks_test, azure_synapse_test, local_spark_test] diff --git a/feathr_project/setup.py b/feathr_project/setup.py index 69a99351f..e11eb3b10 100644 --- a/feathr_project/setup.py +++ b/feathr_project/setup.py @@ -22,6 +22,7 @@ "black>=22.1.0", # formatter "isort", # sort import statements "pytest>=7", + "pytest-cov", "pytest-xdist", "pytest-mock>=3.8.1", ], From ea78d17f21d30fd50f33da5f9483149cc1d97bdb Mon Sep 17 00:00:00 2001 From: enya-yx Date: Wed, 16 Nov 2022 17:25:49 +0800 Subject: [PATCH 2/3] quick change --- .github/workflows/pull_request_push_test.yml | 64 +------------------- 1 file changed, 3 insertions(+), 61 deletions(-) diff --git a/.github/workflows/pull_request_push_test.yml b/.github/workflows/pull_request_push_test.yml index f526df7fd..7cd70a1af 100644 --- a/.github/workflows/pull_request_push_test.yml +++ b/.github/workflows/pull_request_push_test.yml @@ -65,7 +65,7 @@ jobs: databricks_test: runs-on: ubuntu-latest - if: github.event_name == 'schedule' || github.event_name == 'push' + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - uses: actions/checkout@v2 with: @@ -132,7 +132,7 @@ jobs: azure_synapse_test: # might be a bit duplication to setup both the azure_synapse test and databricks test, but for now we will keep those to accelerate the test speed runs-on: ubuntu-latest - if: github.event_name == 'schedule' || github.event_name == 'push' + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - uses: actions/checkout@v2 with: @@ -255,65 +255,7 @@ jobs: run: | # skip cloud related tests pytest --cov-report term-missing --cov=feathr_project/feathr feathr_project/test --cov-config=.github/workflows/.coveragerc_local - - local_spark_test_to_remove: - runs-on: ubuntu-latest - if: (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'coverage')) - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up JDK 8 - uses: actions/setup-java@v2 - with: - java-version: "8" - distribution: "temurin" - - name: Build JAR - run: | - sbt assembly - # remote folder for CI upload - echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_github_action_$(date +"%H_%M_%S")" >> $GITHUB_ENV - # get local jar name without paths so version change won't affect it - echo "FEATHR_LOCAL_JAR_NAME=$(ls target/scala-2.12/*.jar| xargs -n 1 basename)" >> $GITHUB_ENV - # get local jar name without path - echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls target/scala-2.12/*.jar)" >> $GITHUB_ENV - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install Feathr Package - run: | - python -m pip install --upgrade pip - python -m pip install -e ./feathr_project/[all] - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Run Feathr with Local Spark - env: - PROJECT_CONFIG__PROJECT_NAME: "feathr_github_ci_local" - SPARK_CONFIG__SPARK_CLUSTER: local - REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}} - AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}} - AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} - AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} - S3_ACCESS_KEY: ${{secrets.S3_ACCESS_KEY}} - S3_SECRET_KEY: ${{secrets.S3_SECRET_KEY}} - ADLS_ACCOUNT: ${{secrets.ADLS_ACCOUNT}} - ADLS_KEY: ${{secrets.ADLS_KEY}} - BLOB_ACCOUNT: ${{secrets.BLOB_ACCOUNT}} - BLOB_KEY: ${{secrets.BLOB_KEY}} - JDBC_TABLE: ${{secrets.JDBC_TABLE}} - JDBC_USER: ${{secrets.JDBC_USER}} - JDBC_PASSWORD: ${{secrets.JDBC_PASSWORD}} - JDBC_DRIVER: ${{secrets.JDBC_DRIVER}} - JDBC_SF_PASSWORD: ${{secrets.JDBC_SF_PASSWORD}} - KAFKA_SASL_JAAS_CONFIG: ${{secrets.KAFKA_SASL_JAAS_CONFIG}} - MONITORING_DATABASE_SQL_PASSWORD: ${{secrets.MONITORING_DATABASE_SQL_PASSWORD}} - COSMOS1_KEY: ${{secrets.COSMOS1_KEY}} - SQL1_USER: ${{secrets.SQL1_USER}} - SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}} - run: | - # skip cloud related tests - pytest --cov-report term-missing --cov=feathr_project/feathr feathr_project/test --cov-config=.github/workflows/.coveragerc_local - + failure_notification: # If any failure, warning message will be sent needs: [sbt_test, python_lint, databricks_test, azure_synapse_test, local_spark_test] From e1e5d97bbec6f5bef5f1aab45c3322e5c00bd977 Mon Sep 17 00:00:00 2001 From: enya-yx Date: Wed, 16 Nov 2022 17:35:53 +0800 Subject: [PATCH 3/3] quick change --- .github/workflows/pull_request_push_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request_push_test.yml b/.github/workflows/pull_request_push_test.yml index 7cd70a1af..77667815e 100644 --- a/.github/workflows/pull_request_push_test.yml +++ b/.github/workflows/pull_request_push_test.yml @@ -254,7 +254,7 @@ jobs: SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}} run: | # skip cloud related tests - pytest --cov-report term-missing --cov=feathr_project/feathr feathr_project/test --cov-config=.github/workflows/.coveragerc_local + pytest --cov-report term-missing --cov=feathr_project/feathr/spark_provider feathr_project/test/test_local_spark_e2e.py --cov-config=.github/workflows/.coveragerc_local failure_notification: # If any failure, warning message will be sent