Fix docker-compose test (feast-dev#973)

terryyylim · web-flow · commit 35a9afc521fa · 2020-09-02T17:09:19.000+08:00
* Fix docker-compose test

* Address PR comments

* Fix docker-compose bash script

* Update flag

* Update polling interval
diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml
@@ -142,3 +142,11 @@ jobs:
         with:
           name: load-test-results
           path: load-test-output/
+
+  tests-docker-compose:
+    needs: build-push-docker-images
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Test docker compose
+        run: ./infra/scripts/test-docker-compose.sh ${GITHUB_SHA}
diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml
@@ -59,10 +59,3 @@ jobs:
               docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest
             fi
           fi
-
-  tests-docker-compose:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Test docker compose
-        run: ./infra/scripts/test-docker-compose.sh
diff --git a/infra/docker-compose/jobcontroller/jobcontroller.yml b/infra/docker-compose/jobcontroller/jobcontroller.yml
@@ -1,7 +1,7 @@
 feast:
   core-host: core
   jobs:
-    polling_interval_milliseconds: 20000
+    polling_interval_milliseconds: 10000
     job_update_timeout_seconds: 240
     active_runner: direct
     runners:
diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh
@@ -7,6 +7,7 @@ echo "
 Running Docker Compose tests with pytest at 'tests/e2e'
 ============================================================
 "
+LATEST_GH_COMMIT_SHA=$1
 
 clean_up () {
     ARG=$?
@@ -26,6 +27,10 @@ export COMPOSE_INTERACTIVE_NO_CLI=1
 cd ${PROJECT_ROOT_DIR}/infra/docker-compose/
 cp .env.sample .env
 
+# Replace FEAST_VERSION with latest github image SHA
+export FEAST_VERSION=$LATEST_GH_COMMIT_SHA
+echo "Testing docker-compose setup with version SHA, $FEAST_VERSION."
+
 # Start Docker Compose containers
 docker-compose up -d
 
@@ -45,11 +50,17 @@ export FEAST_CORE_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSett
 # Wait for Feast Core to be ready
 ${PROJECT_ROOT_DIR}/infra/scripts/wait-for-it.sh ${FEAST_CORE_CONTAINER_IP_ADDRESS}:6565 --timeout=120
 
+# Get Feast Job Controller container IP address
+export FEAST_JOB_CONTROLLER_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jobcontroller_1)
+
+# Wait for Feast Job Controller to be ready
+"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_JOB_CONTROLLER_CONTAINER_IP_ADDRESS}:6570 --timeout=120
+
 # Get Feast Online Serving container IP address
 export FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_online_serving_1)
 
 # Wait for Feast Online Serving to be ready
 ${PROJECT_ROOT_DIR}/infra/scripts/wait-for-it.sh ${FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS}:6566 --timeout=120
 
 # Run e2e tests for Redis
-docker exec feast_jupyter_1 bash -c 'cd /feast/tests/e2e/redis && pytest --verbose -rs basic-ingest-redis-serving.py --core_url core:6565 --serving_url=online_serving:6566 --kafka_brokers=kafka:9092'
+docker exec feast_jupyter_1 bash -c 'cd /feast/tests/e2e/redis && pytest --verbose -rs basic-ingest-redis-serving.py --core_url core:6565 --serving_url=online_serving:6566 --jobcontroller_url=jobcontroller:6570 --kafka_brokers=kafka:9092'
