From 1013769e7bc42e09f510f1a64f729ef0919ef3ad Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 17 Nov 2020 11:32:26 +0800 Subject: [PATCH 1/9] use mvn cache in e2e & skip auth Signed-off-by: Oleksii Moskalenko --- infra/scripts/test-end-to-end-gcp.sh | 3 +++ infra/scripts/test-end-to-end.sh | 4 ++++ tests/e2e/fixtures/feast_services.py | 8 +++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/infra/scripts/test-end-to-end-gcp.sh b/infra/scripts/test-end-to-end-gcp.sh index 3dab0513f6c..abff88c19e9 100755 --- a/infra/scripts/test-end-to-end-gcp.sh +++ b/infra/scripts/test-end-to-end-gcp.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash export DISABLE_SERVICE_FIXTURES=1 +export MAVEN_OPTS="-Dmaven.repo.local=/tmp/.m2/repository -DdependencyLocationsEnabled=false" +export MAVEN_CACHE="gs://feast-templocation-kf-feast/.m2.2020-11-17.tar" +download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp apt-get update && apt-get install -y redis-server postgresql libpq-dev make build-java-no-tests REVISION=develop diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index 60f9c33a140..b97a34f440f 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +export MAVEN_OPTS="-Dmaven.repo.local=/tmp/.m2/repository -DdependencyLocationsEnabled=false" +export MAVEN_CACHE="gs://feast-templocation-kf-feast/.m2.2020-11-17.tar" + +download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp apt-get update && apt-get install -y redis-server postgresql libpq-dev make build-java-no-tests REVISION=develop diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index 441864ba503..6274de89842 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -47,7 +47,13 @@ def _wait_port_open(port, max_wait=60): return -@pytest.fixture(scope="session", params=[True, False]) +@pytest.fixture( # type: ignore + scope="session", + params=[ + pytest.mark.skip(True), # type: ignore + False, + ], +) def enable_auth(request): return request.param From dae465db5ea19eaae4429bf731c38caf046b8c06 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 17 Nov 2020 11:44:32 +0800 Subject: [PATCH 2/9] correct path & pytest param Signed-off-by: Oleksii Moskalenko --- infra/scripts/test-end-to-end-gcp.sh | 2 +- infra/scripts/test-end-to-end.sh | 2 +- tests/e2e/fixtures/feast_services.py | 8 ++------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/infra/scripts/test-end-to-end-gcp.sh b/infra/scripts/test-end-to-end-gcp.sh index abff88c19e9..4679cbb9b1f 100755 --- a/infra/scripts/test-end-to-end-gcp.sh +++ b/infra/scripts/test-end-to-end-gcp.sh @@ -4,7 +4,7 @@ export DISABLE_SERVICE_FIXTURES=1 export MAVEN_OPTS="-Dmaven.repo.local=/tmp/.m2/repository -DdependencyLocationsEnabled=false" export MAVEN_CACHE="gs://feast-templocation-kf-feast/.m2.2020-11-17.tar" -download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp +infra/scripts/download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp apt-get update && apt-get install -y redis-server postgresql libpq-dev make build-java-no-tests REVISION=develop diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index b97a34f440f..03beb22ab4d 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -3,7 +3,7 @@ export MAVEN_OPTS="-Dmaven.repo.local=/tmp/.m2/repository -DdependencyLocationsEnabled=false" export MAVEN_CACHE="gs://feast-templocation-kf-feast/.m2.2020-11-17.tar" -download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp +infra/scripts/download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp apt-get update && apt-get install -y redis-server postgresql libpq-dev make build-java-no-tests REVISION=develop diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index 6274de89842..9fdb0f5d613 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -47,12 +47,8 @@ def _wait_port_open(port, max_wait=60): return -@pytest.fixture( # type: ignore - scope="session", - params=[ - pytest.mark.skip(True), # type: ignore - False, - ], +@pytest.fixture( + scope="session", params=[pytest.param(True, marks=pytest.mark.skip), False], ) def enable_auth(request): return request.param From cc0138ae8a2eaff0c9ab2d2affd2e161c702ea59 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 17 Nov 2020 12:18:30 +0800 Subject: [PATCH 3/9] use artifacts to store ingestion jar Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 22 ++++++++++++---------- infra/docker-compose/.env.sample | 3 ++- infra/docker-compose/docker-compose.yml | 1 + 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 99f565269ad..ad06a2e382f 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -132,25 +132,24 @@ jobs: - build-push-docker-images - publish-ingestion-jar runs-on: ubuntu-latest + env: + INGESTION_JAR_PATH: /shared/feast-ingestion-spark-${GITHUB_SHA}.jar steps: - uses: actions/checkout@v2 + - name: Download ingestion jar + uses: actions/download-artifact@v2 + with: + name: ingestion-jar - name: Test docker compose run: ./infra/scripts/test-docker-compose.sh ${GITHUB_SHA} publish-ingestion-jar: - runs-on: [self-hosted] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master - with: - version: '290.0.1' - export_default_credentials: true - uses: actions/setup-java@v1 with: java-version: '11' - - uses: stCarolas/setup-maven@v3 - with: - maven-version: 3.6.3 - name: build-jar env: # Try to add retries to prevent connection resets @@ -159,5 +158,8 @@ jobs: MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false MAVEN_EXTRA_OPTS: -X run: make build-java-no-tests REVISION=${GITHUB_SHA} - - name: copy to gs - run: gsutil cp ./spark/ingestion/target/feast-ingestion-spark-${GITHUB_SHA}.jar gs://feast-jobs/spark/ingestion/ + - name: Upload ingestion jar + uses: actions/upload-artifact@v2 + with: + name: ingestion-jar + path: ./spark/ingestion/target/feast-ingestion-spark-${GITHUB_SHA}.jar diff --git a/infra/docker-compose/.env.sample b/infra/docker-compose/.env.sample index ea98b441d6e..8c48fbc976b 100644 --- a/infra/docker-compose/.env.sample +++ b/infra/docker-compose/.env.sample @@ -2,4 +2,5 @@ COMPOSE_PROJECT_NAME=feast FEAST_VERSION=develop FEAST_CORE_CONFIG=./core/core.yml FEAST_ONLINE_SERVING_CONFIG=./serving/online-serving.yml -GCP_SERVICE_ACCOUNT=./gcp-service-accounts/placeholder.json \ No newline at end of file +GCP_SERVICE_ACCOUNT=./gcp-service-accounts/placeholder.json +INGESTION_JAR_PATH=https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-develop.jar \ No newline at end of file diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml index 90c94e0055c..4fea101478a 100644 --- a/infra/docker-compose/docker-compose.yml +++ b/infra/docker-compose/docker-compose.yml @@ -36,6 +36,7 @@ services: FEAST_HISTORICAL_FEATURE_OUTPUT_LOCATION: file:///shared/historical_feature_output FEAST_HISTORICAL_FEATURE_OUTPUT_FORMAT: parquet FEAST_REDIS_HOST: redis + FEAST_SPARK_INGESTION_JAR: ${INGESTION_JAR_PATH} jupyter: image: gcr.io/kf-feast/feast-jupyter:${FEAST_VERSION} From 86c198745d5472c24b7216b87d0548b3ae50abb3 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 17 Nov 2020 12:34:30 +0800 Subject: [PATCH 4/9] add mvn cache Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 12 +++++++++++- .github/workflows/master_only.yml | 7 +++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index ad06a2e382f..5c20a8a3450 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -150,6 +150,15 @@ jobs: - uses: actions/setup-java@v1 with: java-version: '11' + - name: MVN cache + uses: actions/cache@v2 + env: + cache-name: cache-mvn-repo + with: + path: ~/.m2/repository + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- - name: build-jar env: # Try to add retries to prevent connection resets @@ -162,4 +171,5 @@ jobs: uses: actions/upload-artifact@v2 with: name: ingestion-jar - path: ./spark/ingestion/target/feast-ingestion-spark-${GITHUB_SHA}.jar + path: spark/ingestion/target/feast-ingestion-spark-${{ GITHUB_SHA }}.jar + retention-days: 1 diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index f168b48ab78..106b3f4d3df 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -63,7 +63,7 @@ jobs: fi publish-ingestion-jar: - runs-on: [ self-hosted ] + runs-on: ubuntu-latest env: PUBLISH_BUCKET: feast-jobs steps: @@ -72,12 +72,11 @@ jobs: with: version: '290.0.1' export_default_credentials: true + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} - uses: actions/setup-java@v1 with: java-version: '11' - - uses: stCarolas/setup-maven@v3 - with: - maven-version: 3.6.3 - name: Publish develop version of ingestion job run: | if [ ${GITHUB_REF#refs/*/} == "master" ]; then From 5f086ccce4ee3141269cefb9b5bb5e1480a2f546 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 17 Nov 2020 12:50:52 +0800 Subject: [PATCH 5/9] download artifact to path Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 5c20a8a3450..0e0c81610a7 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -140,6 +140,7 @@ jobs: uses: actions/download-artifact@v2 with: name: ingestion-jar + path: ./infra/docker-compose/ - name: Test docker compose run: ./infra/scripts/test-docker-compose.sh ${GITHUB_SHA} From b35560971b8b5a7b06de1b5149ff6d11d475fa96 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 17 Nov 2020 12:54:22 +0800 Subject: [PATCH 6/9] mvn cache Signed-off-by: Oleksii Moskalenko pipeline fails Signed-off-by: Oleksii Moskalenko pipeline fails Signed-off-by: Oleksii Moskalenko pipeline fails Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 0e0c81610a7..6f803ea42db 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -133,7 +133,7 @@ jobs: - publish-ingestion-jar runs-on: ubuntu-latest env: - INGESTION_JAR_PATH: /shared/feast-ingestion-spark-${GITHUB_SHA}.jar + INGESTION_JAR_PATH: /shared/feast-ingestion-spark-develop.jar steps: - uses: actions/checkout@v2 - name: Download ingestion jar @@ -151,15 +151,13 @@ jobs: - uses: actions/setup-java@v1 with: java-version: '11' - - name: MVN cache + - name: Cache local Maven repository uses: actions/cache@v2 - env: - cache-name: cache-mvn-repo with: path: ~/.m2/repository - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-maven- - name: build-jar env: # Try to add retries to prevent connection resets @@ -167,10 +165,10 @@ jobs: # https://github.com/actions/virtual-environments/issues/1499#issuecomment-718396233 MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false MAVEN_EXTRA_OPTS: -X - run: make build-java-no-tests REVISION=${GITHUB_SHA} + run: make build-java-no-tests REVISION=develop - name: Upload ingestion jar uses: actions/upload-artifact@v2 with: name: ingestion-jar - path: spark/ingestion/target/feast-ingestion-spark-${{ GITHUB_SHA }}.jar + path: spark/ingestion/target/feast-ingestion-spark-develop.jar retention-days: 1 From 11ada8c34d130cd942ce13757ddb1fb562d5c63f Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 17 Nov 2020 13:37:20 +0800 Subject: [PATCH 7/9] check failing Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 6f803ea42db..3bb1a45ac5d 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -133,7 +133,7 @@ jobs: - publish-ingestion-jar runs-on: ubuntu-latest env: - INGESTION_JAR_PATH: /shared/feast-ingestion-spark-develop.jar + INGESTION_JAR_PATH: /shared/feast-ingestion-spark-not-develop.jar steps: - uses: actions/checkout@v2 - name: Download ingestion jar From 960d43cdfd9d0df949dbd10bb9c8c3ab852b493b Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 17 Nov 2020 13:55:25 +0800 Subject: [PATCH 8/9] cache for master build of ingestion jar Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 +- .github/workflows/master_only.yml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 3bb1a45ac5d..6f803ea42db 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -133,7 +133,7 @@ jobs: - publish-ingestion-jar runs-on: ubuntu-latest env: - INGESTION_JAR_PATH: /shared/feast-ingestion-spark-not-develop.jar + INGESTION_JAR_PATH: /shared/feast-ingestion-spark-develop.jar steps: - uses: actions/checkout@v2 - name: Download ingestion jar diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 106b3f4d3df..deaf0058895 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -77,6 +77,13 @@ jobs: - uses: actions/setup-java@v1 with: java-version: '11' + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Publish develop version of ingestion job run: | if [ ${GITHUB_REF#refs/*/} == "master" ]; then From b92331488bb2d4d91abf5838bb15c43ad5bbb769 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 17 Nov 2020 15:02:17 +0800 Subject: [PATCH 9/9] drop auth=true completely Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures/feast_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index 9fdb0f5d613..eb085a2c01d 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -48,7 +48,7 @@ def _wait_port_open(port, max_wait=60): @pytest.fixture( - scope="session", params=[pytest.param(True, marks=pytest.mark.skip), False], + scope="session", params=[False], ) def enable_auth(request): return request.param