feathr-ai · xiaoyongzhumsft · Feb 22, 2022 · Feb 19, 2022 · Feb 22, 2022 · Feb 22, 2022
diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml
@@ -0,0 +1,63 @@
+name: Feathr Scala Tests And Azure E2E Integration
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 8
+      uses: actions/setup-java@v2
+      with:
+        java-version: '8'
+        distribution: 'temurin'
+    - name: Run tests
+      run: sbt test
+    - name: Build JAR
+      run: sbt assembly
+    - name: Azure Blob Storage Upload (Overwrite)
+      uses: fixpoint/azblob-upload-artifact@v4
+      with:
+        connection-string: ${{secrets.SPARK_JAR_BLOB_CONNECTION_STRING}}
+        name: 'feathr_jar_github_action'
+        path: 'target/scala-2.11/feathr-assembly-0.1.0-SNAPSHOT.jar'
+        container: ${{secrets.SPARK_JAR_BLOB_CONTAINER}}
+        cleanup: 'true'
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Install Feathr Package
+      run: |
+        python -m pip install -e ./feathr_project/
+    - name: Test with pytest
+      env:
+        FEATHR_RUNTIME_LOCATION: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_jar_github_action/feathr-assembly-0.1.0-SNAPSHOT.jar'
+        REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}}
+        AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}}
+        AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}}
+        AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}}
+        S3_ACCESS_KEY: ${{secrets.S3_ACCESS_KEY}}
+        S3_SECRET_KEY: ${{secrets.S3_SECRET_KEY}}
+
+      run: |        
+        cd feathr_project
+        pytest
diff --git a/feathr_project/README.md b/feathr_project/README.md
@@ -1,3 +1,6 @@
+[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Feathr Runtime](https://github.com/linkedin/feathr/actions/workflows/scala.yml/badge.svg)](https://github.com/linkedin/feathr/actions/workflows/scala.yml)
+
+
 Feathr Python Project Developer Guide
 =============================
 

diff --git a/feathr_project/data/feathr_user_workspace/feathr_config.yaml b/feathr_project/data/feathr_user_workspace/feathr_config.yaml
@@ -1,23 +1,23 @@
 # DO NOT MOVE OR DELETE THIS FILE
 resource:
   azure:
-    PROJECT_NAME: "frame_getting_started"
+    PROJECT_NAME: "feathr_getting_started"
 
     # Spark cluster configs
     SYNAPSE_DEV_URL: "https://feathrazuretest3synapse.dev.azuresynapse.net"
     SYNAPSE_POOL_NAME: "spark24"
-    SYNAPSE_WORKSPACE_DIR: "abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/frame_getting_started"
+    SYNAPSE_WORKSPACE_DIR: "abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_getting_started"
     SYNAPSE_EXECUTOR_SIZE: "Small"
     SYNAPSE_EXECUTOR_NUM: 4
 
     # Redis configs to access Redis cluster
-    REDIS_PASSWORD: "Li7Nn63iNB0x731VTnnz2Vr29WYJHx7JlAzCaH9lbHw="
+    REDIS_PASSWORD: ''
     REDIS_HOST: "feathrazuretest3redis.redis.cache.windows.net"
     REDIS_PORT: 6380
     REDIS_SSL_ENABLED: True
 
     # Feathr Job configuration. Support local paths, path start with http(s)://, and paths start with abfs(s)://
-    FEATHR_RUNTIME_LOCATION: abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/frame_getting_started/feathr-assembly-0.1.0-SNAPSHOT.jar
+    FEATHR_RUNTIME_LOCATION: https://azurefeathrstorage.blob.core.windows.net/public/feathr-assembly-0.1.0-SNAPSHOT.jar
     # configure number of parts for the spark output for feature generation job
     RESULT_OUTPUT_PARTS: "1"
 

diff --git a/feathr_project/test/test_azure_e2e.py b/feathr_project/test/test_azure_e2e.py
@@ -24,7 +24,7 @@ def test_feathr_online_store():
         job_res = client.materialize_features()
         # just assume the job is successful without validating the actual result in Redis. Might need to consolidate
         # this part with the test_feathr_online_store test case
-        client.wait_job_to_finish(timeout_sec=300)
+        client.wait_job_to_finish(timeout_sec=600)
         res = client.online_get_features('nycTaxiDemoFeature', '265', ['f_location_avg_fare', 'f_location_max_fare'])
         # just assme there are values. We don't hard code the values for now for testing
         # the correctness of the feature generation should be garunteed by feathr runtime.
@@ -55,7 +55,7 @@ def test_feathr_get_historical_features():
         client = FeathrClient()
 
         returned_spark_job = client.join_offline_features()
-        res_url = client.get_job_result_uri(block=True, timeout_sec=500)
+        res_url = client.get_job_result_uri(block=True, timeout_sec=600)
         tmp_dir = tempfile.TemporaryDirectory()
         client.feathr_synapse_laucher.download_result(result_path=res_url, local_folder=tmp_dir.name)
         dataframe_list = []