diff --git a/.gitignore b/.gitignore index 01df87f2dc0..f35e736b20e 100644 --- a/.gitignore +++ b/.gitignore @@ -46,4 +46,122 @@ vendor .direnv .terraform/ -*.tfvars \ No newline at end of file +*.tfvars + +# python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +.vscode diff --git a/Makefile b/Makefile index 888cfea4264..6ddd22dce17 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,28 @@ -VERSION_FILE=VERSION -FEAST_VERSION=`cat $(VERSION_FILE)` +# +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +VERSION_FILE = VERSION +FEAST_VERSION = `cat $(VERSION_FILE)` test: mvn test +test-integration: + $(MAKE) -C testing/integration test-integration ID=$(ID) + build-deps: $(MAKE) -C protos gen-go dep ensure diff --git a/core/src/test/java/feast/core/CoreApplicationTest.java b/core/src/test/java/feast/core/CoreApplicationTest.java index d80b5b7b42c..ccd30ddfacc 100644 --- a/core/src/test/java/feast/core/CoreApplicationTest.java +++ b/core/src/test/java/feast/core/CoreApplicationTest.java @@ -48,7 +48,7 @@ */ @RunWith(SpringRunner.class) @SpringBootTest(properties = { - "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}", "spring.datasource.url=jdbc:h2:mem:testdb", "feast.store.warehouse.type=file.json", "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}", diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java index 61f55202ad9..6dcef362a2a 100644 --- a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java +++ b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java @@ -49,7 +49,7 @@ */ @RunWith(SpringRunner.class) @SpringBootTest(properties = { - "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}", "spring.datasource.url=jdbc:h2:mem:testdb", "feast.store.warehouse.type=file.json", "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}", diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java index 6c8ee325aac..46b2e151a05 100644 --- a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java +++ b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java @@ -49,7 +49,7 @@ */ @RunWith(SpringRunner.class) @SpringBootTest(properties = { - "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}", "spring.datasource.url=jdbc:h2:mem:testdb", "feast.store.serving.type=redis", "feast.store.serving.options={\"host\":\"localhost\",\"port\":1234}", diff --git a/testing/Makefile b/testing/Makefile index 0eef0abab9a..0ff05f827fa 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -32,6 +32,6 @@ update-config: get-cluster-credentials update-plugins: get-cluster-credentials kubectl create configmap plugins --from-file=plugins.yaml=prow/plugins.yaml --dry-run -o yaml | kubectl replace configmap plugins -f - -build-push: - docker build test-image/ -t $(REGISTRY)/$(PROJECT)/test-image:$(VERSION) +build-push-test-image: + docker build -f docker-images/test-image/Dockerfile -t $(REGISTRY)/$(PROJECT)/test-image:$(VERSION) . $(PUSH) "$(REGISTRY)/$(PROJECT)/test-image:$(VERSION)" \ No newline at end of file diff --git a/testing/README.md b/testing/README.md index c345c46e75e..df38d34a676 100644 --- a/testing/README.md +++ b/testing/README.md @@ -4,9 +4,9 @@ This folder contains the Feast test infrastructure. ## Components -* test-image/ - Base docker image and script for running tests. +* docker-images/ - Docker images for running tests * prow/ - Prow configuration (plugins and jobs) -* tf/ - Terraform modules to provision the base testing infrastructure on GCP +* infrastructure/ - Terraform modules to provision the base testing infrastructure on GCP ## Set up @@ -23,7 +23,7 @@ mv tf/gcs terraform import google_storage_bucket.kf-feast-terraform-state kf-feast-terraform-state ``` -4. Ensure that all variables are set correctly in `tf/terraform.tfvars`. It is likely that the GCP project will need to be updated. +4. Ensure that all variables are set correctly in `infrastructure/terraform.tfvars`. It is likely that the GCP project will need to be updated. 5. Create the primary Kubernetes cluster which will host Prow and Argo @@ -51,7 +51,11 @@ To update Prow jobs, plugins, or the Docker image used for testing, modify one o - `prow/config.yaml` - `prow/plugins.yaml` -- `test-image/Dockerfile` -- `test-image/run.sh` +- `docker-images/test-image/Dockerfile` +- `docker-images/test-image/run.sh` -After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry. \ No newline at end of file +After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry. + +## Installing Argo + +Argo is used to run integration tests. It follows the [standard installation process described in the argo quickstart](https://github.com/argoproj/argo/blob/master/demo.md), with the artifact repository [configured to use gcs](https://github.com/argoproj/argo/blob/master/ARTIFACT_REPO.md). \ No newline at end of file diff --git a/testing/docker-images/docker-builder-image/Dockerfile b/testing/docker-images/docker-builder-image/Dockerfile new file mode 100644 index 00000000000..45f2ce67b15 --- /dev/null +++ b/testing/docker-images/docker-builder-image/Dockerfile @@ -0,0 +1,25 @@ +FROM docker:18.09 + +# from https://github.com/GoogleCloudPlatform/cloud-sdk-docker/blob/master/alpine/Dockerfile +ENV CLOUD_SDK_VERSION=236.0.0 + +ENV PATH /google-cloud-sdk/bin:$PATH +RUN apk --no-cache add \ + curl \ + python \ + py-crcmod \ + bash \ + libc6-compat \ + openssh-client \ + git \ + gnupg \ + && curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + ln -s /lib /lib64 && \ + gcloud config set core/disable_usage_reporting true && \ + gcloud config set component_manager/disable_update_check true && \ + gcloud config set metrics/environment github_docker_image && \ + gcloud --version + +RUN gcloud auth configure-docker diff --git a/testing/docker-images/terraform-image/Dockerfile b/testing/docker-images/terraform-image/Dockerfile new file mode 100644 index 00000000000..04b734fbef6 --- /dev/null +++ b/testing/docker-images/terraform-image/Dockerfile @@ -0,0 +1,19 @@ +FROM debian:stretch + +RUN apt-get update && apt-get install -y jq wget python zip netcat + +RUN wget -qO- https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-233.0.0-linux-x86_64.tar.gz | tar xzf - \ + && /google-cloud-sdk/bin/gcloud -q components install kubectl \ + && /google-cloud-sdk/bin/gcloud -q components install beta \ + && /google-cloud-sdk/bin/gcloud -q components install bq + +ENV PATH="/google-cloud-sdk/bin:${PATH}" + +RUN wget https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip \ + && unzip terraform_0.11.11_linux_amd64.zip \ + && mv terraform /bin/terraform + +RUN wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-arm64.tar.gz | tar xzf - \ + && mv linux-arm64/helm /bin/helm + +RUN helm init --client-only diff --git a/testing/test-image/Dockerfile b/testing/docker-images/test-image/Dockerfile similarity index 100% rename from testing/test-image/Dockerfile rename to testing/docker-images/test-image/Dockerfile diff --git a/testing/test-image/run.sh b/testing/docker-images/test-image/run.sh similarity index 100% rename from testing/test-image/run.sh rename to testing/docker-images/test-image/run.sh diff --git a/testing/tf/cloud-build/cloud-build.tf b/testing/infrastructure/cloud-build/cloud-build.tf similarity index 100% rename from testing/tf/cloud-build/cloud-build.tf rename to testing/infrastructure/cloud-build/cloud-build.tf diff --git a/testing/tf/cloud-build/cloudbuild.yaml b/testing/infrastructure/cloud-build/cloudbuild.yaml similarity index 100% rename from testing/tf/cloud-build/cloudbuild.yaml rename to testing/infrastructure/cloud-build/cloudbuild.yaml diff --git a/testing/tf/cloud-build/variables.tf b/testing/infrastructure/cloud-build/variables.tf similarity index 100% rename from testing/tf/cloud-build/variables.tf rename to testing/infrastructure/cloud-build/variables.tf diff --git a/testing/tf/gcs/backend.tf b/testing/infrastructure/gcs/backend.tf similarity index 100% rename from testing/tf/gcs/backend.tf rename to testing/infrastructure/gcs/backend.tf diff --git a/testing/tf/gcs/main.tf b/testing/infrastructure/gcs/main.tf similarity index 100% rename from testing/tf/gcs/main.tf rename to testing/infrastructure/gcs/main.tf diff --git a/testing/tf/k8s-cluster/backend.tf b/testing/infrastructure/k8s-cluster/backend.tf similarity index 100% rename from testing/tf/k8s-cluster/backend.tf rename to testing/infrastructure/k8s-cluster/backend.tf diff --git a/testing/tf/k8s-cluster/main.tf b/testing/infrastructure/k8s-cluster/main.tf similarity index 69% rename from testing/tf/k8s-cluster/main.tf rename to testing/infrastructure/k8s-cluster/main.tf index e5b01754f96..43b1de31b98 100644 --- a/testing/tf/k8s-cluster/main.tf +++ b/testing/infrastructure/k8s-cluster/main.tf @@ -16,6 +16,10 @@ resource "google_container_cluster" "primary-test-cluster" { node_config { oauth_scopes = [ "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.read_write", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", ] machine_type = "n1-standard-4" diff --git a/testing/tf/k8s-cluster/output.tf b/testing/infrastructure/k8s-cluster/output.tf similarity index 100% rename from testing/tf/k8s-cluster/output.tf rename to testing/infrastructure/k8s-cluster/output.tf diff --git a/testing/tf/k8s-cluster/provider.tf b/testing/infrastructure/k8s-cluster/provider.tf similarity index 100% rename from testing/tf/k8s-cluster/provider.tf rename to testing/infrastructure/k8s-cluster/provider.tf diff --git a/testing/tf/k8s-cluster/variables.tf b/testing/infrastructure/k8s-cluster/variables.tf similarity index 100% rename from testing/tf/k8s-cluster/variables.tf rename to testing/infrastructure/k8s-cluster/variables.tf diff --git a/testing/integration/Makefile b/testing/integration/Makefile new file mode 100644 index 00000000000..cceb66d20b1 --- /dev/null +++ b/testing/integration/Makefile @@ -0,0 +1,23 @@ +# +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +.PHONY: test-integration it-redis-bq-dataflow + +test-integration: + @$(MAKE) it-redis-bq-dataflow ID=$(ID) + +it-redis-bq-dataflow: + argo submit --watch tests/redis-bq-dataflow/workflow.yaml -p prId=$(ID) \ No newline at end of file diff --git a/testing/integration/README.md b/testing/integration/README.md new file mode 100644 index 00000000000..fde86a35335 --- /dev/null +++ b/testing/integration/README.md @@ -0,0 +1,29 @@ +# Integration tests + +Integration tests for Feast are run on argo workflows. The tests follow the following steps: + +1. Provision infrastructure using Terraform +2. Test using pytest +3. Teardown infrastructure using Terraform + +## Terraform modules + +The `tf/modules` directory contains Terraform modules to set up the necessary infrastructure. Currently contains: + +- `cluster`: kubernetes cluster with necessary permissions to run Feast jobs +- `feast-helm`: Feast helm installation. + +## Tests + +The `tests` directory contains the integration tests. Each folder should contain the following: + +1. The terraform scripts to set up the necessary infra. +2. Data to run the tests on +3. A pytest file that executes the ingestion jobs and then tests for correctness. +4. Argo workflow yaml to orchestrate the entire process. + +Multiple tests can be run on the same infrastructure. + +### Adding new tests + +To add your own tests, either (1) create a new test case within an existing folder or (2) create a new folder with the resources mentioned above. \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/.env.local b/testing/integration/tests/redis-bq-dataflow/.env.local new file mode 100644 index 00000000000..99a58fd76d3 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/.env.local @@ -0,0 +1,7 @@ +# Example environment variables that need to be set when running this test locally +# Export the variables like so: +# set -a; source .env.local +FEAST_CORE_URL=localhost:6565 +FEAST_SERVING_URL=localhost:6566 +PROJECT_ID=kf-feast +BUCKET_NAME=it-feast-storage diff --git a/testing/integration/tests/redis-bq-dataflow/README.md b/testing/integration/tests/redis-bq-dataflow/README.md new file mode 100644 index 00000000000..248cc4e1d88 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/README.md @@ -0,0 +1,9 @@ +# redis-bq-dataflow + +This repository contains an end-to-end test that sets up the following infrastructure: + +1. Feast configured to run jobs on dataflow +2. Redis as serving store +3. BQ as warehouse store + +And then runs the tests in `test_feast.py`. \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/data/entity/myentity.yaml b/testing/integration/tests/redis-bq-dataflow/data/entity/myentity.yaml new file mode 100644 index 00000000000..8fa945a6774 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/entity/myentity.yaml @@ -0,0 +1,5 @@ +name: myentity +description: my test entity +tags: + - tag1 + - tag2 diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml new file mode 100644 index 00000000000..05b5279d0e4 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_double_redis +name: feature_double_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: DOUBLE +uri: https://github.com/bob/example diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml new file mode 100644 index 00000000000..3ff38358ce3 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_float_redis +name: feature_float_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: FLOAT +uri: https://github.com/bob/example diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml new file mode 100644 index 00000000000..ebeb92dc999 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_int32_redis +name: feature_int32_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: INT32 +uri: https://github.com/bob/example diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml new file mode 100644 index 00000000000..dd6b7339bf3 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml @@ -0,0 +1,7 @@ +id: myentity.feature_int64_redis +name: feature_int64_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: INT64 +uri: https://github.com/bob/example diff --git a/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml b/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml new file mode 100644 index 00000000000..3eb170a23c4 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml @@ -0,0 +1,19 @@ +type: file.csv +sourceOptions: + path: gs://it-feast-storage/test-cases/test_data.csv +entities: + - myentity +schema: + entityIdColumn: entity + timestampColumn: ts + fields: + - name: entity + - name: ts + - name: feature1 + featureId: myentity.feature_double_redis + - name: feature2 + featureId: myentity.feature_float_redis + - name: feature3 + featureId: myentity.feature_int32_redis + - name: feature4 + featureId: myentity.feature_int64_redis diff --git a/testing/integration/tests/redis-bq-dataflow/data/test_data.csv b/testing/integration/tests/redis-bq-dataflow/data/test_data.csv new file mode 100644 index 00000000000..16906eef985 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/data/test_data.csv @@ -0,0 +1,50 @@ +0,2018-01-01T00:00:00,0.16654998178974967,0.0779124915810051,8,1 +0,2018-01-01T00:01:00,0.35055416999061695,0.3789967915556205,4,7 +0,2018-01-01T00:02:00,0.922617931213872,0.3179716588679623,8,6 +0,2018-01-01T00:03:00,0.06582437841319277,0.05565087947895475,9,1 +0,2018-01-01T00:04:00,0.8470568422759338,0.21347178570025072,2,10 +0,2018-01-01T00:05:00,0.11361651376595083,0.610881579742112,8,10 +0,2018-01-01T00:06:00,0.3034691430518833,0.19830856710384326,6,2 +0,2018-01-01T00:07:00,0.8346843022317924,0.08544530941545558,2,4 +0,2018-01-01T00:08:00,0.012367592836242891,0.5726466047910151,2,5 +0,2018-01-01T00:09:00,0.9887391081292758,0.9819570408280592,9,10 +1,2018-01-01T00:00:00,0.7503367819099366,0.946197160138704,10,1 +1,2018-01-01T00:01:00,0.9190195271598668,0.8836270958333856,10,4 +1,2018-01-01T00:02:00,0.07725768926853083,0.448071643977631,4,9 +1,2018-01-01T00:03:00,0.4965090410288343,0.9615236179532041,3,5 +1,2018-01-01T00:04:00,0.4838512691935104,0.355736320341819,7,2 +1,2018-01-01T00:05:00,0.7422860280929264,0.26936605638140254,7,6 +1,2018-01-01T00:06:00,0.007120660156963887,0.9457195989857463,5,4 +1,2018-01-01T00:07:00,0.5165841927879966,0.5734751201042606,1,4 +1,2018-01-01T00:08:00,0.0874458350429409,0.3284354801066641,3,8 +1,2018-01-01T00:09:00,0.8091561596661963,0.5176044026790632,1,9 +2,2018-01-01T00:00:00,0.5440258526780313,0.7936189500778384,3,5 +2,2018-01-01T00:01:00,0.9012681719733718,0.408129175696462,5,8 +2,2018-01-01T00:02:00,0.1357239530601766,0.2882272639567518,9,9 +2,2018-01-01T00:03:00,0.5482100359374814,0.22784885285759426,2,10 +2,2018-01-01T00:04:00,0.47915480013638667,0.5783117797172574,6,2 +2,2018-01-01T00:05:00,0.08954537587399436,0.6036422156720167,2,5 +2,2018-01-01T00:06:00,0.022254256710355302,0.8425123292474225,2,2 +2,2018-01-01T00:07:00,0.2981277839268307,0.7187791198561501,9,6 +2,2018-01-01T00:08:00,0.7483695369918476,0.24681429759605789,10,3 +2,2018-01-01T00:09:00,0.1357618414286158,0.887316489600256,3,10 +3,2018-01-01T00:00:00,0.24835796330732807,0.8871182399308823,3,2 +3,2018-01-01T00:01:00,0.18779166308706263,0.6635037319574626,4,10 +3,2018-01-01T00:02:00,0.00017995781865332017,0.9437703134427515,9,2 +3,2018-01-01T00:03:00,0.44519915810431854,0.03739113677349182,4,5 +3,2018-01-01T00:04:00,0.27156326671637077,0.06880899222977854,3,2 +3,2018-01-01T00:05:00,0.7647529493108539,0.14002956138831768,7,10 +3,2018-01-01T00:06:00,0.9954469835451967,0.7567907196365666,9,9 +3,2018-01-01T00:07:00,0.3883998745214877,0.18671618739049767,7,10 +3,2018-01-01T00:08:00,0.5448378857720906,0.38482161282383076,3,1 +3,2018-01-01T00:09:00,0.3550003641256,0.1818481760674323,1,9 +4,2018-01-01T00:00:00,0.5925452403226181,0.30212100652221296,5,10 +4,2018-01-01T00:01:00,0.18599533814587,0.5230622774213863,6,5 +4,2018-01-01T00:02:00,0.9599946262502064,0.26435543431553166,10,8 +4,2018-01-01T00:03:00,0.22755474767811312,0.6693510146123766,1,8 +4,2018-01-01T00:04:00,0.12063065422121622,0.7460800163440976,10,9 +4,2018-01-01T00:05:00,0.58262241980741,0.2714493926314999,9,5 +4,2018-01-01T00:06:00,0.26952820440313163,0.40841655204277727,5,1 +4,2018-01-01T00:07:00,0.7779897852625218,0.39955885342995734,9,3 +4,2018-01-01T00:08:00,0.9892858266244913,0.91265457567995,2,10 +4,2018-01-01T00:09:00,0.31825556885425366,0.6727958709215186,3,9 diff --git a/testing/integration/tests/redis-bq-dataflow/main.py b/testing/integration/tests/redis-bq-dataflow/main.py new file mode 100644 index 00000000000..253faa6f96a --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/main.py @@ -0,0 +1,6 @@ +import test_feast +from feast.sdk.client import Client + +if __name__ == "__main__": + feast_client = Client(verbose=True) + test_feast.TestFeastIntegration().test_end_to_end(client=feast_client) diff --git a/testing/integration/tests/redis-bq-dataflow/test_feast.py b/testing/integration/tests/redis-bq-dataflow/test_feast.py new file mode 100644 index 00000000000..0c394177e53 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/test_feast.py @@ -0,0 +1,184 @@ +import os +import subprocess +from subprocess import PIPE +from time import sleep + +import feast.sdk.utils.gs_utils as utils +import numpy as np +import pandas as pd +import pytest +import yaml +from feast.sdk.client import Client +from feast.sdk.resources.entity import Entity +from feast.sdk.resources.feature import Feature +from feast.sdk.resources.feature_set import FeatureSet +from feast.sdk.utils.bq_util import TableDownloader +from google.cloud import storage + + +@pytest.fixture +def client(): + return Client(verbose=True) + + +# Init the system by registering relevant resources. +def _register_resources(client, entities_fldr, features_fldr): + resources = [] + for ent_file in os.listdir(entities_fldr): + resources.append(Entity.from_yaml(os.path.join(entities_fldr, ent_file))) + for feat_file in os.listdir(features_fldr): + resources.append(Feature.from_yaml(os.path.join(features_fldr, feat_file))) + client.apply(resources) + + +# Run an import job given an import spec. +def _run_job_and_wait_for_completion(job_yaml): + out = subprocess.run( + "feast jobs run {}".format(job_yaml).split(" "), check=True, stdout=PIPE + ) + job_id = out.stdout.decode("utf-8").split(" ")[-1] + job_status, job_complete = "UNKNOWN", False + terminal_states = ["COMPLETED", "ABORTED", "ERROR", "UNKNOWN"] + while not job_complete: + out = subprocess.run( + "feast get job {}".format(job_id).split(" "), check=True, stdout=PIPE + ) + job_details = yaml.load(out.stdout.decode("utf-8").replace("\t", " ")) + job_status = job_details["Status"] + print("Job id {} currently {}".format(job_id, job_status)) + if job_status in terminal_states: + break + sleep(10) + return job_status + + +def _stage_data(local, remote): + split = utils.split_gs_path(remote) + storage_client = storage.Client() + bucket = storage_client.get_bucket(split[0]) + blob = bucket.blob(split[1]) + + blob.upload_from_filename(local) + + +# Get the BQ data and get only columns to compare, then sort by id and timestamp +def _get_data_from_bq_and_sort(table_name, bucket_name): + got = TableDownloader().download_table_as_df( + table_name, "gs://{}/test-cases/extract.csv".format(bucket_name) + ) + return ( + got.drop(["created_timestamp", "job_id"], axis=1) + .sort_values(["id", "event_timestamp"]) + .reset_index(drop=True) + ) + + +class TestFeastIntegration: + def test_end_to_end(self, client): + project_id = os.environ.get("PROJECT_ID") + bucket_name = os.environ.get("BUCKET_NAME") + + features = [ + "feature_double_redis", + "feature_float_redis", + "feature_int32_redis", + "feature_int64_redis", + ] + expected = self.get_expected_data(features) + + self.run_batch_import(bucket_name, client) + self.validate_warehouse_data(bucket_name, project_id, expected) + self.validate_serving_data(client, features, expected) + + @staticmethod + def get_expected_data(features): + expected = pd.read_csv( + "data/test_data.csv", + header=None, + names=["id", "event_timestamp"] + features, + ) + expected = expected.sort_values(["id", "event_timestamp"]).reset_index( + drop=True + ) + expected["event_timestamp"] = pd.to_datetime( + expected["event_timestamp"] + ).dt.strftime("%Y-%m-%d %H:%M:%S UTC") + return expected + + @staticmethod + def run_batch_import(bucket_name, client): + _stage_data( + "data/test_data.csv", "gs://{}/test-cases/test_data.csv".format(bucket_name) + ) + _register_resources(client, "data/entity", "data/feature") + job_status = _run_job_and_wait_for_completion("data/import/import_csv.yaml") + assert job_status == "COMPLETED" + + @staticmethod + def validate_serving_data(client, features, expected): + features_type_mapping = { + "myentity": np.string_, + "myentity.feature_double_redis": np.float64, + "myentity.feature_float_redis": np.float64, + "myentity.feature_int32_redis": np.int64, + "myentity.feature_int64_redis": np.int64, + } + + feature_set = FeatureSet( + entity="myentity", features=["myentity." + f for f in features] + ) + actual_latest = client.get_serving_data( + feature_set, entity_keys=[str(id) for id in list(expected.id.unique())] + ).astype(features_type_mapping) + actual_latest = actual_latest.sort_values(["myentity"]) + expected["event_timestamp"] = pd.to_datetime(expected["event_timestamp"]) + expected_latest = expected.loc[ + expected.groupby("id").event_timestamp.idxmax(), : + ] + expected_latest.columns = ["myentity", "timestamp"] + [ + "myentity." + f for f in features + ] + expected_latest = ( + expected_latest[actual_latest.columns] + .sort_values(["myentity"]) + .reset_index(drop=True) + ).astype(features_type_mapping) + + assert ( + pd.testing.assert_frame_equal( + expected_latest, actual_latest, check_less_precise=True, check_like=True + ) + is None + ) + + @staticmethod + def validate_warehouse_data(bucket_name, project_id, expected): + features_type_mapping = { + "id": np.string_, + "event_timestamp": np.string_, + "feature_double_redis": np.float64, + "feature_float_redis": np.float64, + "feature_int32_redis": np.int64, + "feature_int64_redis": np.int64, + } + + actual = TableDownloader().download_table_as_df( + project_id + ".feast_it.myentity", + "gs://{}/test-cases/extract.csv".format(bucket_name), + ) + actual = actual.drop(["created_timestamp", "job_id"], axis=1).sort_values( + ["id", "event_timestamp"] + ) + actual = ( + actual.drop_duplicates() + .reset_index(drop=True) + .astype(features_type_mapping) + ) + expected = expected.astype(features_type_mapping) + + assert ( + pd.testing.assert_frame_equal( + expected, actual, check_less_precise=True, check_like=True + ) + is None + ) diff --git a/testing/integration/tests/redis-bq-dataflow/tf/backend.tf b/testing/integration/tests/redis-bq-dataflow/tf/backend.tf new file mode 100644 index 00000000000..fb1f9c81b5b --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/backend.tf @@ -0,0 +1,6 @@ +terraform { + backend "gcs" { + bucket = "kf-feast-terraform-state" + prefix = "tf/k8s-cluster/integration-test" + } +} diff --git a/testing/integration/tests/redis-bq-dataflow/tf/data.tf b/testing/integration/tests/redis-bq-dataflow/tf/data.tf new file mode 100644 index 00000000000..a988612c59d --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/data.tf @@ -0,0 +1 @@ +data "google_client_config" "current" {} diff --git a/testing/integration/tests/redis-bq-dataflow/tf/input.tf b/testing/integration/tests/redis-bq-dataflow/tf/input.tf new file mode 100644 index 00000000000..bb653c213a5 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/input.tf @@ -0,0 +1,3 @@ +variable "docker_tag" { + description = "Docker image to deploy" +} diff --git a/testing/integration/tests/redis-bq-dataflow/tf/main.tf b/testing/integration/tests/redis-bq-dataflow/tf/main.tf new file mode 100644 index 00000000000..fadca6442a6 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/main.tf @@ -0,0 +1,70 @@ +locals { + project_name = "kf-feast" + region = "us-central1" + subnetwork = "regions/${local.region}/subnetworks/default" + network = "default" + cluster_name = "it-feast" + + job_runner_options = { + "project" = "${local.project_name}" + "region" = "${local.region}" + "tempLocation" = "gs://${local.cluster_name}-storage/tempJob" + "subnetwork" = "${local.subnetwork}" + "maxNumWorkers" = "64" + "autoscalingAlgorithm" = "THROUGHPUT_BASED" + } + + errors_store_options = { + "path" = "gs://${local.cluster_name}-storage/errors" + } +} + +module "cluster" { + source = "../../../tf/modules/cluster" + cluster_name = "${local.cluster_name}" + project_name = "${local.project_name}" + region = "${local.region}" + bucket_name = "${local.cluster_name}-storage" + network = "${local.network}" + subnetwork = "${local.subnetwork}" + gke_machine_type = "n1-standard-8" +} + +resource "null_resource" "wait_for_regional_cluster" { + provisioner "local-exec" { + command = "${path.module}/scripts/wait-for-cluster.sh ${local.project_name} ${local.cluster_name}" + } + + provisioner "local-exec" { + when = "destroy" + command = "${path.module}/scripts/wait-for-cluster.sh ${local.project_name} ${local.cluster_name}" + } + + depends_on = ["module.cluster"] +} + +module "feast" { + source = "../../../tf/modules/feast-helm" + project_name = "${local.project_name}" + region = "${local.region}" + subnetwork = "${local.subnetwork}" + docker_tag = "${var.docker_tag}" + core_address = "10.128.0.99" + serving_address = "10.128.0.100" + redis_address = "10.128.0.101" + load_balancer_source_range = "10.0.0.0/8" + job_runner = "DataflowRunner" + job_runner_options = "'${jsonencode(local.job_runner_options)}'" + bucket_name = "${local.cluster_name}-storage" + bq_dataset = "${google_bigquery_dataset.feast_bq_dataset.dataset_id}" + + depends_on = ["module.cluster.cluster_name", "null_resource.wait_for_regional_cluster"] +} + +resource "google_bigquery_dataset" "feast_bq_dataset" { + dataset_id = "feast_it" + description = "Feast integration test dataset" + default_table_expiration_ms = 36000000 + location = "US" + delete_contents_on_destroy = true +} diff --git a/testing/integration/tests/redis-bq-dataflow/tf/provider.tf b/testing/integration/tests/redis-bq-dataflow/tf/provider.tf new file mode 100644 index 00000000000..91f4c3caca4 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/provider.tf @@ -0,0 +1,16 @@ +provider "google" { + version = "2.1.0" + # credentials = "${file("~/.secrets/${local.project_name}.json")}" + project = "${local.project_name}" +} + +provider "helm" { + kubernetes { + host = "${module.cluster.endpoint}" + token = "${data.google_client_config.current.access_token}" + + client_certificate = "${base64decode(module.cluster.client_certificate)}" + client_key = "${base64decode(module.cluster.client_key)}" + cluster_ca_certificate = "${base64decode(module.cluster.cluster_ca_certificate)}" + } +} diff --git a/testing/integration/tests/redis-bq-dataflow/tf/scripts/empty-bq.sh b/testing/integration/tests/redis-bq-dataflow/tf/scripts/empty-bq.sh new file mode 100755 index 00000000000..5abeb004abd --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/scripts/empty-bq.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +PROJECT_ID=$1 +DATASET_NAME=$2 +for i in $(bq ls $PROJECT_ID:$DATASET_NAME | sed 1,2d | awk "{print \$1}"); +do + if [ ! -z "$i" ]; then + bq rm -ft $PROJECT_ID:$DATASET_NAME.$i; + fi +done \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/tf/scripts/wait-for-cluster.sh b/testing/integration/tests/redis-bq-dataflow/tf/scripts/wait-for-cluster.sh new file mode 100755 index 00000000000..aba1dc2450c --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/scripts/wait-for-cluster.sh @@ -0,0 +1,36 @@ + #!/bin/bash + + # Copyright 2018 Google LLC + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + +# from https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/blob/176ad6f47d2d92ac9a49ef112830da2f539b737d/scripts/wait-for-cluster.sh +set -e + +PROJECT=$1 +CLUSTER_NAME=$2 +gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" +jq_query=".[] | select(.name==\"$CLUSTER_NAME\") | .status" + +echo "Waiting for cluster $2 in project $1 to reconcile..." + +current_status=$($gcloud_command | jq -r "$jq_query") + +while [ "${current_status}" = "RECONCILING" ]; do + printf "." + sleep 5 + current_status=$($gcloud_command | jq -r "$jq_query") +done + +echo "Cluster is ready!" \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/tf/values.yaml b/testing/integration/tests/redis-bq-dataflow/tf/values.yaml new file mode 100755 index 00000000000..0a21749c675 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/tf/values.yaml @@ -0,0 +1,101 @@ +--- +core: + image: + pullPolicy: IfNotPresent + registry: gcr.io/kf-feast + repository: feast-core + tag: "integration-test" + replicaCount: 1 + resources: + limits: + cpu: 4 + memory: 8G + requests: + cpu: 1 + memory: 4G + rollingUpdate: + maxSurge: 2 + maxUnavailable: 0 + service: + grpc: + port: 6565 + targetPort: 6565 + http: + port: 80 + targetPort: 8080 + jobs: + workspace: "it-feast-storage/workspace" + runner: "DataflowRunner" + options: '{"autoscalingAlgorithm":"THROUGHPUT_BASED","maxNumWorkers":"64","project":"kf-feast","region":"us-central1","subnetwork":"regions/us-central1/subnetworks/default","tempLocation":"gs://it-feast-storage/tempJob"}' + monitoring: + period: 5000 + initialDelay: 60000 +dataflow: + projectID: "kf-feast" + location: "us-central1" + +postgresql: + name: feast-metadata + service: + port: 5432 + postgresPassword: password + +postgresql: + provision: true + imageTag: 9.6.11 + service: + port: 5432 + +redis: + name: feast-redis + provision: true + master: + service: + type: LoadBalancer + annotations: + "cloud.google.com/load-balancer-type": "Internal" + +serving: + config: + maxEntityPerBatch: 2000 + maxNumberOfThread: 1024 + redisPool: + maxIdle: 16 + maxSize: 1024 + timeout: 1 + image: + pullPolicy: IfNotPresent + registry: gcr.io/kf-feast + repository: feast-serving + tag: "integration-test" + jaeger: + enabled: false + replicaCount: 1 + resources: + limits: + cpu: 4 + memory: 8G + requests: + cpu: 2 + memory: 2G + rollingUpdate: + maxSurge: 2 + maxUnavailable: 0 + service: + grpc: + port: 6566 + targetPort: 6566 + http: + port: 80 + targetPort: 8080 + +store: + errors: + type: "file.json" + options: '{"path":"gs://it-feast-storage/error-log"}' + warehouse: + type: "bigquery" + options: '{"dataset":"feast_it","project":"kf-feast"}' + serving: + type: "redis" + options: '{"host":"10.128.0.101","port":"6379"}' \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml new file mode 100644 index 00000000000..c1c6265ab3a --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -0,0 +1,235 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: feast-integration-test- +spec: + entrypoint: run + onExit: exit-handler + arguments: + parameters: + - name: revision + value: integration-test + - name: prId + value: "none" + - name: gitRef + value: "none" + templates: + - name: run + steps: + - - name: get-feast + template: get-feast + - - name: build-jars + template: build-jars + arguments: + artifacts: + - name: feast + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" + - name: build-cli + template: build-cli + arguments: + artifacts: + - name: feast + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" + - - name: build-push-docker + template: build-docker + arguments: + artifacts: + - name: feast + from: "{{steps.build-jars.outputs.artifacts.feast-build}}" + - - name: terraform-provision + template: terraform + arguments: + parameters: + - name: arg + value: "apply -var docker_tag=integration-test -auto-approve ." + artifacts: + - name: feast + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" + - name: feast-cli + from: "{{steps.build-cli.outputs.artifacts.feast-cli}}" + - - name: test + template: run-test + arguments: + artifacts: + - name: feast + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" + - name: feast-cli + from: "{{steps.build-cli.outputs.artifacts.feast-cli}}" + + - name: exit-handler + steps: + - - name: terraform-destroy + template: terraform-destroy + arguments: + parameters: + - name: arg + value: "destroy -var docker_tag=integration-test -auto-approve ." + artifacts: + - name: feast + from: "{{workflow.outputs.artifacts.feast-repo}}" + - name: get-feast + steps: + - - name: get-feast-by-pullrequest + template: get-feast-by-pullrequest + when: "{{workflow.parameters.prId}} != none" + - name: get-feast-by-gitref + template: get-feast-by-gitref + when: "{{workflow.parameters.gitRef}} != none" + outputs: + artifacts: + - name: feast-repo + from: "{{workflow.outputs.artifacts.feast-repo}}" + globalName: feast-repo + - name: get-feast-by-pullrequest + container: + image: alpine/git + command: + - /bin/sh + - -c + - | + git clone https://github.com/gojek/feast; + cd feast; + git fetch origin pull/{{workflow.parameters.prId}}/head; + git checkout FETCH_HEAD + outputs: + artifacts: + - name: feast + path: /git/feast + globalName: feast-repo + - name: get-feast-by-gitref + container: + image: alpine/git + command: + - /bin/sh + - -c + - | + git clone https://github.com/gojek/feast; + cd feast; + git checkout {{workflow.parameters.gitRef}} + outputs: + artifacts: + - name: feast + path: /git/feast + globalName: feast-repo + - name: build-jars + inputs: + artifacts: + - name: feast + path: /feast + container: + image: us.gcr.io/kf-feast/test-image:1.0.0 + command: + - /bin/bash + - -c + - mvn package -DskipTests -Drevision=integration-test -T 6 -DdependencyLocationsEnabled=false + workingDir: /feast + resources: + requests: + cpu: 3000m + outputs: + artifacts: + - name: feast-build + path: /feast + globalName: feast-build + - name: build-docker + inputs: + artifacts: + - name: feast + path: /feast + container: + image: gcr.io/kf-feast/docker:18.09-gcloud + command: [sh, -c] + args: [' + until docker ps; do sleep 3; done; + docker build -t gcr.io/kf-feast/feast-core:integration-test -f docker/core/Dockerfile --build-arg RELEASE=integration-test .; + docker build -t gcr.io/kf-feast/feast-serving:integration-test -f docker/serving/Dockerfile --build-arg RELEASE=integration-test .; + docker push gcr.io/kf-feast/feast-core:integration-test; + docker push gcr.io/kf-feast/feast-serving:integration-test + '] + env: + - name: DOCKER_HOST + value: 127.0.0.1 + workingDir: /feast + sidecars: + - name: dind + image: docker:stable-dind + securityContext: + privileged: true + mirrorVolumeMounts: true + - name: build-cli + inputs: + artifacts: + - name: feast + path: /go/src/github.com/gojek/feast + container: + image: grpc/go + command: ["/bin/bash", "-c"] + args: [' + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh; + make build-cli + '] + workingDir: /go/src/github.com/gojek/feast + outputs: + artifacts: + - name: feast-cli + path: /go/src/github.com/gojek/feast/cli/bin/linux-amd64/feast + - name: terraform + inputs: + parameters: + - name: arg + artifacts: + - name: feast + path: /feast + - name: feast-cli + path: /bin/feast + container: + image: gcr.io/kf-feast/it-terraform:0.1.0 + workingDir: /feast/testing/integration/tests/redis-bq-dataflow/tf + command: ["sh", "-c"] + args: [' + terraform init; + terraform {{inputs.parameters.arg}} + '] + - name: run-test + inputs: + artifacts: + - name: feast-cli + path: /bin/feast + - name: feast + path: /feast + container: + image: python:3.7.2 + command: ["/bin/bash", "-c"] + args: [' + feast config set coreURI 10.128.0.99:6565; + pip install pytest; + pip install -r /feast/sdk/python/test-requirements.txt; + pip install -e /feast/sdk/python; + pytest --capture=no + '] + workingDir: /feast/testing/integration/tests/redis-bq-dataflow + env: + - name: FEAST_CORE_URL + value: "10.128.0.99:6565" + - name: FEAST_SERVING_URL + value: "10.128.0.100:6566" + - name: PROJECT_ID + value: kf-feast + - name: BUCKET_NAME + value: it-feast-storage + - name: terraform-destroy + inputs: + parameters: + - name: arg + artifacts: + - name: feast + path: /feast + container: + image: gcr.io/kf-feast/it-terraform:0.1.0 + workingDir: /feast/testing/integration/tests/redis-bq-dataflow/tf + command: ["sh", "-c"] + args: [' + terraform init; + terraform {{inputs.parameters.arg}} + '] + diff --git a/testing/integration/tf/modules/cluster/input.tf b/testing/integration/tf/modules/cluster/input.tf new file mode 100644 index 00000000000..bccca70fbbc --- /dev/null +++ b/testing/integration/tf/modules/cluster/input.tf @@ -0,0 +1,39 @@ +variable "cluster_name" { + default = "feast" + description = "Cluster name" +} + +variable "project_name" { + description = "Project name" +} + +variable "region" { + description = "Subnet region" +} + +variable "network" { + default = "default" +} + +variable "subnetwork" { + default = "default" +} + +variable "gke_machine_type" { + description = "The machine type for the default node pool" + default = "n1-standard-4" +} + +variable "bucket_name" { + description = "Working storage for feast" +} + +variable "feast_node_pool_min_size" { + description = "Minimum number of nodes for node pool" + default = 1 +} + +variable "feast_node_pool_max_size" { + description = "Maximum number of nodes for node pool" + default = 2 +} diff --git a/testing/integration/tf/modules/cluster/main.tf b/testing/integration/tf/modules/cluster/main.tf new file mode 100644 index 00000000000..23b66241a42 --- /dev/null +++ b/testing/integration/tf/modules/cluster/main.tf @@ -0,0 +1,42 @@ +data "google_client_config" "current" {} + +resource "google_storage_bucket" "feast_storage_bucket" { + name = "${var.bucket_name}" + location = "${var.region}" + force_destroy = true +} + +resource "google_container_cluster" "feast_k8s" { + name = "${var.cluster_name}" + region = "${var.region}" + initial_node_count = 1 + + # node_version = "1.11.5-gke.5" + # min_master_version = "1.11.5-gke.5" + network = "${var.network}" + + subnetwork = "${var.subnetwork}" + + enable_legacy_abac = true + + node_config { + machine_type = "${var.gke_machine_type}" + disk_size_gb = "50" + + oauth_scopes = [ + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/devstorage.read_write", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigtable.admin", + "https://www.googleapis.com/auth/bigtable.data", + ] + } + + provisioner "local-exec" { + when = "destroy" + command = "sleep 90" + } +} diff --git a/testing/integration/tf/modules/cluster/output.tf b/testing/integration/tf/modules/cluster/output.tf new file mode 100644 index 00000000000..6b7af14886f --- /dev/null +++ b/testing/integration/tf/modules/cluster/output.tf @@ -0,0 +1,33 @@ + + +output "endpoint" { + value = "${google_container_cluster.feast_k8s.endpoint}" +} + +output "cluster_name" { + value = "${var.cluster_name}" +} + +output "client_certificate" { + value = "${google_container_cluster.feast_k8s.master_auth.0.client_certificate}" +} + +output "client_key" { + value = "${google_container_cluster.feast_k8s.master_auth.0.client_key}" +} + +output "cluster_ca_certificate" { + value = "${google_container_cluster.feast_k8s.master_auth.0.cluster_ca_certificate}" +} + +output "username" { + value = "${google_container_cluster.feast_k8s.master_auth.0.username}" +} + +output "password" { + value = "${google_container_cluster.feast_k8s.master_auth.0.password}" +} + +output "feast_storage_bucket" { + value = "${google_storage_bucket.feast_storage_bucket.name}" +} \ No newline at end of file diff --git a/testing/integration/tf/modules/feast-helm/input.tf b/testing/integration/tf/modules/feast-helm/input.tf new file mode 100644 index 00000000000..6652bac56f4 --- /dev/null +++ b/testing/integration/tf/modules/feast-helm/input.tf @@ -0,0 +1,61 @@ +variable "project_name" { + description = "Project name" +} + +variable "subnetwork" { + description = "Desired subnetwork" +} + +variable "region" { + description = "Subnet region" +} + +variable "docker_tag" { + default = "0.1.0" + description = "Feast build version" +} + +variable "core_address" { + description = "Core internal address" +} + +variable "serving_address" { + description = "Serving internal address" +} + +variable "redis_address" { + description = "Redis internal address" +} + +variable "statsd_host" { + description = "Statsd host to write metrics to" + default = "" +} + +variable "job_runner" { + description = "Desired job runner" + default = "Dataflow" +} + +variable "job_runner_options" { + description = "Job runner options as a json string" + default = "'{}'" +} + +variable "load_balancer_source_range" { + description = "ingress filter for google internal load balancer" +} + +variable "bq_dataset" { + description = "BigQuery dataset for warehouse" +} + +variable depends_on { + default = [] + + type = "list" +} + +variable "bucket_name" { + description = "Working storage for feast" +} \ No newline at end of file diff --git a/testing/integration/tf/modules/feast-helm/main.tf b/testing/integration/tf/modules/feast-helm/main.tf new file mode 100644 index 00000000000..687d392fe98 --- /dev/null +++ b/testing/integration/tf/modules/feast-helm/main.tf @@ -0,0 +1,100 @@ + +locals { + warehouse_option = { + "project" = "${var.project_name}" + "dataset" = "${var.bq_dataset}" + } + serving_option = { + "host" = "${var.redis_address}" + "port" = "6379" + } + store_error_option = { + "path" = "gs://${var.bucket_name}/error-log" + } +} + +resource "google_compute_address" "core_address" { + project = "${var.project_name}" + name = "feast-core-ip" + subnetwork = "${var.subnetwork}" + address_type = "INTERNAL" + address = "${var.core_address}" + region = "${var.region}" +} + +resource "google_compute_address" "serving_address" { + project = "${var.project_name}" + name = "feast-serving-ip" + subnetwork = "${var.subnetwork}" + address_type = "INTERNAL" + address = "${var.serving_address}" + region = "${var.region}" +} + +resource "template_file" "helm_values" { + template = "${file("${path.module}/values.tmpl")}" + + vars = { + docker_tag = "${var.docker_tag}" + job_runner = "${var.job_runner}" + job_runner_options = "${var.job_runner_options}" + store_warehouse_option = "'${jsonencode(local.warehouse_option)}'" + store_error_option = "'${jsonencode(local.store_error_option)}'" + store_serving_option = "'${jsonencode(local.serving_option)}'" + project_id = "${var.project_name}" + region = "${var.region}" + workspace = "gs://${var.bucket_name}/workspace" + } +} + +resource "local_file" "helm_values_output" { + content = "${template_file.helm_values.rendered}" + filename = "values.yaml" + + depends_on = ["template_file.helm_values"] +} + +resource "helm_release" "feast" { + name = "feast-it" + chart = "/feast/charts/feast" + + set { + name = "core.service.extIPAdr" + value = "${google_compute_address.core_address.address}" + } + + set { + name = "redis.master.service.loadBalancerIP" + value = "${var.redis_address}" + } + + set { + name = "core.service.loadBalancerSourceRanges[0]" + value = "${var.load_balancer_source_range}" + } + + set { + name = "serving.service.extIPAdr" + value = "${google_compute_address.serving_address.address}" + } + + set { + name = "serving.service.loadBalancerSourceRanges[0]" + value = "${var.load_balancer_source_range}" + } + + set { + name = "statsd.host" + value = "${var.statsd_host}" + } + + values = [ + "${template_file.helm_values.rendered}", + ] + + timeout = 600 + + depends_on = [ + "local_file.helm_values_output", + ] +} diff --git a/testing/integration/tf/modules/feast-helm/output.tf b/testing/integration/tf/modules/feast-helm/output.tf new file mode 100644 index 00000000000..2d0e8eef07e --- /dev/null +++ b/testing/integration/tf/modules/feast-helm/output.tf @@ -0,0 +1,11 @@ +output "core_url" { + value = "${google_compute_address.core_address.address}" +} + +output "serving_url" { + value = "${google_compute_address.serving_address.address}" +} + +output "redis_url" { + value = "${var.redis_address}" +} \ No newline at end of file diff --git a/testing/integration/tf/modules/feast-helm/values.tmpl b/testing/integration/tf/modules/feast-helm/values.tmpl new file mode 100644 index 00000000000..ab1fe95ca37 --- /dev/null +++ b/testing/integration/tf/modules/feast-helm/values.tmpl @@ -0,0 +1,105 @@ +--- +core: + image: + pullPolicy: IfNotPresent + registry: gcr.io/kf-feast + repository: feast-core + tag: "${docker_tag}" + replicaCount: 1 + resources: + limits: + cpu: 4 + memory: 8G + requests: + cpu: 1 + memory: 4G + rollingUpdate: + maxSurge: 2 + maxUnavailable: 0 + service: + grpc: + port: 6565 + targetPort: 6565 + http: + port: 80 + targetPort: 8080 + jobs: + workspace: "${workspace}" + runner: "${job_runner}" + options: ${job_runner_options} + monitoring: + period: 5000 + initialDelay: 60000 + trainingDatasetPrefix: "fs" + projectId: "kf-feast" + logType: JSON +dataflow: + projectID: "${project_id}" + location: "${region}" + +postgresql: + name: feast-metadata + service: + port: 5432 + postgresPassword: password + +postgresql: + provision: true + imageTag: 9.6.11 + service: + port: 5432 + +redis: + name: feast-redis + provision: true + master: + service: + type: LoadBalancer + annotations: + "cloud.google.com/load-balancer-type": "Internal" + usePassword: false + +serving: + config: + maxEntityPerBatch: 2000 + maxNumberOfThread: 1024 + redisPool: + maxIdle: 16 + maxSize: 1024 + timeout: 1 + image: + pullPolicy: IfNotPresent + registry: gcr.io/kf-feast + repository: feast-serving + tag: "${docker_tag}" + jaeger: + enabled: false + replicaCount: 1 + resources: + limits: + cpu: 4 + memory: 8G + requests: + cpu: 2 + memory: 2G + rollingUpdate: + maxSurge: 2 + maxUnavailable: 0 + service: + grpc: + port: 6566 + targetPort: 6566 + http: + port: 80 + targetPort: 8080 + +store: + errors: + type: "file.json" + options: ${store_error_option} + warehouse: + type: "bigquery" + options: ${store_warehouse_option} + serving: + type: "redis" + options: ${store_serving_option} \ No newline at end of file diff --git a/testing/tf/cloud-build/cloudbuild_master.yaml b/testing/tf/cloud-build/cloudbuild_master.yaml deleted file mode 100644 index 743e7588a52..00000000000 --- a/testing/tf/cloud-build/cloudbuild_master.yaml +++ /dev/null @@ -1,19 +0,0 @@ -steps: -- name: maven:3.6.0-jdk-8-slim - args: ['mvn', 'clean', 'install', '-Drevision=$COMMIT_SHA', '-DskipTests'] - id: build-jars - -- name: "gcr.io/cloud-builders/docker" - args: ['build', '-t', 'gcr.io/$PROJECT_ID/feast-core:$COMMIT_SHA', '--build-arg', 'RELEASE=$COMMIT_SHA', '-f', './docker/core/Dockerfile', '.'] - id: docker-core - waitFor: - - build-jars - -- name: "gcr.io/cloud-builders/docker" - args: ['build', '-t', 'gcr.io/$PROJECT_ID/feast-serving:$COMMIT_SHA', '--build-arg', 'RELEASE=$COMMIT_SHA', '-f', './docker/serving/Dockerfile', '.'] - id: docker-serving - waitFor: - - build-jars - -timeout: 1200s -images: ['gcr.io/$PROJECT_ID/feast-core:$COMMIT_SHA', 'gcr.io/$PROJECT_ID/feast-serving:$COMMIT_SHA'] \ No newline at end of file diff --git a/testing/tf/terraform.tfvars b/testing/tf/terraform.tfvars deleted file mode 100644 index a8b45f64116..00000000000 --- a/testing/tf/terraform.tfvars +++ /dev/null @@ -1,7 +0,0 @@ -gcp_project = "kf-feast" - -default_network = "default" - -default_subnet = "default" - -default_region = "us-central1"