diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml new file mode 100644 index 000000000..c5793ac36 --- /dev/null +++ b/.github/workflows/scala.yml @@ -0,0 +1,63 @@ +name: Feathr Scala Tests And Azure E2E Integration + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + java-version: '8' + distribution: 'temurin' + - name: Run tests + run: sbt test + - name: Build JAR + run: sbt assembly + - name: Azure Blob Storage Upload (Overwrite) + uses: fixpoint/azblob-upload-artifact@v4 + with: + connection-string: ${{secrets.SPARK_JAR_BLOB_CONNECTION_STRING}} + name: 'feathr_jar_github_action' + path: 'target/scala-2.11/feathr-assembly-0.1.0-SNAPSHOT.jar' + container: ${{secrets.SPARK_JAR_BLOB_CONTAINER}} + cleanup: 'true' + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Install Feathr Package + run: | + python -m pip install -e ./feathr_project/ + - name: Test with pytest + env: + FEATHR_RUNTIME_LOCATION: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_jar_github_action/feathr-assembly-0.1.0-SNAPSHOT.jar' + REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}} + AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}} + AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} + AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}} + S3_ACCESS_KEY: ${{secrets.S3_ACCESS_KEY}} + S3_SECRET_KEY: ${{secrets.S3_SECRET_KEY}} + + run: | + cd feathr_project + pytest \ No newline at end of file diff --git a/feathr_project/README.md b/feathr_project/README.md index 7d7dded47..580c2303e 100644 --- a/feathr_project/README.md +++ b/feathr_project/README.md @@ -1,3 +1,6 @@ +[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Feathr Runtime](https://github.com/linkedin/feathr/actions/workflows/scala.yml/badge.svg)](https://github.com/linkedin/feathr/actions/workflows/scala.yml) + + Feathr Python Project Developer Guide ============================= diff --git a/feathr_project/data/feathr_user_workspace/feathr_config.yaml b/feathr_project/data/feathr_user_workspace/feathr_config.yaml index 8f231fda4..8bd566785 100644 --- a/feathr_project/data/feathr_user_workspace/feathr_config.yaml +++ b/feathr_project/data/feathr_user_workspace/feathr_config.yaml @@ -1,23 +1,23 @@ # DO NOT MOVE OR DELETE THIS FILE resource: azure: - PROJECT_NAME: "frame_getting_started" + PROJECT_NAME: "feathr_getting_started" # Spark cluster configs SYNAPSE_DEV_URL: "https://feathrazuretest3synapse.dev.azuresynapse.net" SYNAPSE_POOL_NAME: "spark24" - SYNAPSE_WORKSPACE_DIR: "abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/frame_getting_started" + SYNAPSE_WORKSPACE_DIR: "abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_getting_started" SYNAPSE_EXECUTOR_SIZE: "Small" SYNAPSE_EXECUTOR_NUM: 4 # Redis configs to access Redis cluster - REDIS_PASSWORD: "Li7Nn63iNB0x731VTnnz2Vr29WYJHx7JlAzCaH9lbHw=" + REDIS_PASSWORD: '' REDIS_HOST: "feathrazuretest3redis.redis.cache.windows.net" REDIS_PORT: 6380 REDIS_SSL_ENABLED: True # Feathr Job configuration. Support local paths, path start with http(s)://, and paths start with abfs(s):// - FEATHR_RUNTIME_LOCATION: abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/frame_getting_started/feathr-assembly-0.1.0-SNAPSHOT.jar + FEATHR_RUNTIME_LOCATION: https://azurefeathrstorage.blob.core.windows.net/public/feathr-assembly-0.1.0-SNAPSHOT.jar # configure number of parts for the spark output for feature generation job RESULT_OUTPUT_PARTS: "1" diff --git a/feathr_project/test/test_azure_e2e.py b/feathr_project/test/test_azure_e2e.py index df0abc9f4..3dae880c2 100644 --- a/feathr_project/test/test_azure_e2e.py +++ b/feathr_project/test/test_azure_e2e.py @@ -24,7 +24,7 @@ def test_feathr_online_store(): job_res = client.materialize_features() # just assume the job is successful without validating the actual result in Redis. Might need to consolidate # this part with the test_feathr_online_store test case - client.wait_job_to_finish(timeout_sec=300) + client.wait_job_to_finish(timeout_sec=600) res = client.online_get_features('nycTaxiDemoFeature', '265', ['f_location_avg_fare', 'f_location_max_fare']) # just assme there are values. We don't hard code the values for now for testing # the correctness of the feature generation should be garunteed by feathr runtime. @@ -55,7 +55,7 @@ def test_feathr_get_historical_features(): client = FeathrClient() returned_spark_job = client.join_offline_features() - res_url = client.get_job_result_uri(block=True, timeout_sec=500) + res_url = client.get_job_result_uri(block=True, timeout_sec=600) tmp_dir = tempfile.TemporaryDirectory() client.feathr_synapse_laucher.download_result(result_path=res_url, local_folder=tmp_dir.name) dataframe_list = []