From b953f3948c53a4a102bb3adbe2c836d792301e5c Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Wed, 21 Oct 2020 09:53:08 +0800 Subject: [PATCH] Update docker-compose for Feast 0.8 Signed-off-by: Khor Shu Heng --- infra/docker-compose/.env.sample | 5 +-- infra/docker-compose/core/core.yml | 11 +++-- infra/docker-compose/docker-compose.yml | 44 +++---------------- .../jobcontroller/jobcontroller.yml | 16 ------- .../serving/historical-serving.yml | 25 ----------- infra/scripts/test-docker-compose.sh | 6 --- 6 files changed, 13 insertions(+), 94 deletions(-) delete mode 100644 infra/docker-compose/jobcontroller/jobcontroller.yml delete mode 100644 infra/docker-compose/serving/historical-serving.yml diff --git a/infra/docker-compose/.env.sample b/infra/docker-compose/.env.sample index e984460f8be..ea98b441d6e 100644 --- a/infra/docker-compose/.env.sample +++ b/infra/docker-compose/.env.sample @@ -1,8 +1,5 @@ COMPOSE_PROJECT_NAME=feast FEAST_VERSION=develop -GCP_SERVICE_ACCOUNT=./gcp-service-accounts/key.json FEAST_CORE_CONFIG=./core/core.yml -FEAST_JOB_CONTROLLER_CONFIG=./jobcontroller/jobcontroller.yml -FEAST_HISTORICAL_SERVING_CONFIG=./serving/historical-serving.yml -FEAST_HISTORICAL_SERVING_ENABLED=false FEAST_ONLINE_SERVING_CONFIG=./serving/online-serving.yml +GCP_SERVICE_ACCOUNT=./gcp-service-accounts/placeholder.json \ No newline at end of file diff --git a/infra/docker-compose/core/core.yml b/infra/docker-compose/core/core.yml index 16bd3a81188..517b2649ca2 100644 --- a/infra/docker-compose/core/core.yml +++ b/infra/docker-compose/core/core.yml @@ -1,6 +1,5 @@ -feast: - stream: - type: kafka - options: - topic: feast-features - bootstrapServers: "kafka:9092,localhost:9094" +spring: + datasource: + url: jdbc:postgresql://${DB_HOST:127.0.0.1}:${DB_PORT:5432}/${DB_DATABASE:postgres} + username: ${DB_USERNAME:postgres} + password: ${DB_PASSWORD:password} diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml index 197ee56d175..7a4f62cbfc1 100644 --- a/infra/docker-compose/docker-compose.yml +++ b/infra/docker-compose/docker-compose.yml @@ -5,14 +5,11 @@ services: image: gcr.io/kf-feast/feast-core:${FEAST_VERSION} volumes: - ${FEAST_CORE_CONFIG}:/etc/feast/application.yml - - ${GCP_SERVICE_ACCOUNT}:/etc/gcloud/service-accounts/key.json environment: DB_HOST: db - GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json restart: on-failure depends_on: - db - - kafka ports: - 6565:6565 command: @@ -21,23 +18,6 @@ services: - /opt/feast/feast-core.jar - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml - jobcontroller: - image: gcr.io/kf-feast/feast-jobcontroller:${FEAST_VERSION} - volumes: - - ${FEAST_JOB_CONTROLLER_CONFIG}:/etc/feast/application.yml - - ${GCP_SERVICE_ACCOUNT}:/etc/gcloud/service-accounts/key.json - environment: - GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json - depends_on: - - kafka - ports: - - 6570:6570 - command: - - java - - -jar - - /opt/feast/feast-job-controller.jar - - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml - jupyter: image: gcr.io/kf-feast/feast-jupyter:${FEAST_VERSION} volumes: @@ -47,7 +27,12 @@ services: environment: FEAST_CORE_URL: core:6565 FEAST_ONLINE_SERVING_URL: online_serving:6566 - FEAST_HISTORICAL_SERVING_URL: historical_serving:6567 + FEAST_SPARK_LAUNCHER: standalone + FEAST_SPARK_STANDALONE_MASTER: local + FEAST_SPARK_HOME: /usr/local/spark + FEAST_SPARK_STAGING_LOCATION: file:///tmp/staging + FEAST_HISTORICAL_FEATURE_OUTPUT_LOCATION: file:///tmp/historical_feature_output + FEAST_HISTORICAL_FEATURE_OUTPUT_FORMAT: parquet GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json ports: - 8888:8888 @@ -85,6 +70,7 @@ services: volumes: - ${FEAST_ONLINE_SERVING_CONFIG}:/etc/feast/application.yml depends_on: + - core - redis ports: - 6566:6566 @@ -95,22 +81,6 @@ services: - /opt/feast/feast-serving.jar - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml - historical_serving: - image: gcr.io/kf-feast/feast-serving:${FEAST_VERSION} - volumes: - - ${FEAST_HISTORICAL_SERVING_CONFIG}:/etc/feast/application.yml - - ${GCP_SERVICE_ACCOUNT}:/etc/gcloud/service-accounts/key.json - depends_on: - - redis - ports: - - 6567:6567 - restart: on-failure - environment: - GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json - command: > - bash -c "if [ $FEAST_HISTORICAL_SERVING_ENABLED != "true" ]; then echo \"Feast historical serving is disabled\"; sleep 10; exit 1; fi - && java -Xms1024m -Xmx1024m -jar /opt/feast/feast-serving.jar --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml" - redis: image: redis:5-alpine ports: diff --git a/infra/docker-compose/jobcontroller/jobcontroller.yml b/infra/docker-compose/jobcontroller/jobcontroller.yml deleted file mode 100644 index c4e039ac9f8..00000000000 --- a/infra/docker-compose/jobcontroller/jobcontroller.yml +++ /dev/null @@ -1,16 +0,0 @@ -feast: - core-host: core - jobs: - polling_interval_milliseconds: 5000 - job_update_timeout_seconds: 240 - active_runner: direct - runners: - - name: direct - type: DirectRunner - options: - tempLocation: gs://bucket/tempLocation - stream: - type: kafka - options: - topic: feast-features - bootstrapServers: "kafka:9092,localhost:9094" \ No newline at end of file diff --git a/infra/docker-compose/serving/historical-serving.yml b/infra/docker-compose/serving/historical-serving.yml deleted file mode 100644 index 95d2b07093a..00000000000 --- a/infra/docker-compose/serving/historical-serving.yml +++ /dev/null @@ -1,25 +0,0 @@ -feast: - core-host: core - active-store: historical - stores: - - name: historical - type: BIGQUERY - # Changes required for historical serving to work - # Please see https://api.docs.feast.dev/grpc/feast.core.pb.html#Store for configuration options - config: - project_id: project - dataset_id: dataset - staging_location: gs://gcs_bucket/prefix - initial_retry_delay_seconds: 1 - total_timeout_seconds: 21600 - write_triggering_frequency_seconds: 10 - subscriptions: - - name: "*" - project: "*" - job_store: - redis_host: redis - redis_port: 6379 - -grpc: - server: - port: 6567 diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh index 1d907336ae8..8ace2cdc236 100755 --- a/infra/scripts/test-docker-compose.sh +++ b/infra/scripts/test-docker-compose.sh @@ -50,12 +50,6 @@ export FEAST_CORE_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSett # Wait for Feast Core to be ready ${PROJECT_ROOT_DIR}/infra/scripts/wait-for-it.sh ${FEAST_CORE_CONTAINER_IP_ADDRESS}:6565 --timeout=120 -# Get Feast Job Controller container IP address -export FEAST_JOB_CONTROLLER_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jobcontroller_1) - -# Wait for Feast Job Controller to be ready -"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_JOB_CONTROLLER_CONTAINER_IP_ADDRESS}:6570 --timeout=120 - # Get Feast Online Serving container IP address export FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_online_serving_1)