From 9d97bb3f082abb46a5c2fc25d70bbd4d10abf116 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 8 Feb 2019 18:29:02 +0800 Subject: [PATCH 01/47] Add integration test --- .gitignore | 120 ++++++++++++++++- .../cloud-build/cloud-build.tf | 0 .../cloud-build/cloudbuild.yaml | 0 .../cloud-build/variables.tf | 0 testing/{tf => infrastructure}/gcs/backend.tf | 0 testing/{tf => infrastructure}/gcs/main.tf | 0 .../k8s-cluster/backend.tf | 0 .../k8s-cluster/main.tf | 4 + .../k8s-cluster/output.tf | 0 .../k8s-cluster/provider.tf | 0 .../k8s-cluster/variables.tf | 0 .../tests/data/entity/myentity.yaml | 5 + .../feature/feature_minutedoubleredis1.yaml | 13 ++ .../feature/feature_minutefloatredis1.yaml | 13 ++ .../feature/feature_minuteint32redis1.yaml | 13 ++ .../feature/feature_minuteint64redis1.yaml | 13 ++ .../tests/data/import/import_csv.yaml | 19 +++ testing/integration/tests/data/test_data.csv | 50 +++++++ testing/integration/tests/test_feast.py | 97 ++++++++++++++ testing/integration/tests/tf/backend.tf | 6 + testing/integration/tests/tf/data.tf | 1 + testing/integration/tests/tf/input.tf | 8 ++ testing/integration/tests/tf/main.tf | 125 ++++++++++++++++++ testing/integration/tests/tf/provider.tf | 16 +++ .../integration/tests/tf/scripts/empty-bq.sh | 8 ++ .../tests/tf/scripts/wait-for-cluster.sh | 35 +++++ testing/integration/tests/workflow.yaml | 99 ++++++++++++++ .../integration/tf/modules/cluster/input.tf | 34 +++++ .../integration/tf/modules/cluster/main.tf | 41 ++++++ .../integration/tf/modules/cluster/output.tf | 33 +++++ .../tf/modules/feast-helm/input.tf | 64 +++++++++ .../integration/tf/modules/feast-helm/main.tf | 105 +++++++++++++++ .../tf/modules/feast-helm/output.tf | 7 + .../tf/modules/feast-helm/values.tmpl | 82 ++++++++++++ testing/integration/tf/modules/redis/init.sh | 15 +++ testing/integration/tf/modules/redis/input.tf | 52 ++++++++ testing/integration/tf/modules/redis/main.tf | 33 +++++ .../integration/tf/modules/redis/output.tf | 3 + testing/tf/cloud-build/cloudbuild_master.yaml | 19 --- testing/tf/terraform.tfvars | 7 - 40 files changed, 1113 insertions(+), 27 deletions(-) rename testing/{tf => infrastructure}/cloud-build/cloud-build.tf (100%) rename testing/{tf => infrastructure}/cloud-build/cloudbuild.yaml (100%) rename testing/{tf => infrastructure}/cloud-build/variables.tf (100%) rename testing/{tf => infrastructure}/gcs/backend.tf (100%) rename testing/{tf => infrastructure}/gcs/main.tf (100%) rename testing/{tf => infrastructure}/k8s-cluster/backend.tf (100%) rename testing/{tf => infrastructure}/k8s-cluster/main.tf (69%) rename testing/{tf => infrastructure}/k8s-cluster/output.tf (100%) rename testing/{tf => infrastructure}/k8s-cluster/provider.tf (100%) rename testing/{tf => infrastructure}/k8s-cluster/variables.tf (100%) create mode 100644 testing/integration/tests/data/entity/myentity.yaml create mode 100644 testing/integration/tests/data/feature/feature_minutedoubleredis1.yaml create mode 100644 testing/integration/tests/data/feature/feature_minutefloatredis1.yaml create mode 100644 testing/integration/tests/data/feature/feature_minuteint32redis1.yaml create mode 100644 testing/integration/tests/data/feature/feature_minuteint64redis1.yaml create mode 100644 testing/integration/tests/data/import/import_csv.yaml create mode 100644 testing/integration/tests/data/test_data.csv create mode 100644 testing/integration/tests/test_feast.py create mode 100644 testing/integration/tests/tf/backend.tf create mode 100644 testing/integration/tests/tf/data.tf create mode 100644 testing/integration/tests/tf/input.tf create mode 100644 testing/integration/tests/tf/main.tf create mode 100644 testing/integration/tests/tf/provider.tf create mode 100755 testing/integration/tests/tf/scripts/empty-bq.sh create mode 100755 testing/integration/tests/tf/scripts/wait-for-cluster.sh create mode 100644 testing/integration/tests/workflow.yaml create mode 100644 testing/integration/tf/modules/cluster/input.tf create mode 100644 testing/integration/tf/modules/cluster/main.tf create mode 100644 testing/integration/tf/modules/cluster/output.tf create mode 100644 testing/integration/tf/modules/feast-helm/input.tf create mode 100644 testing/integration/tf/modules/feast-helm/main.tf create mode 100644 testing/integration/tf/modules/feast-helm/output.tf create mode 100644 testing/integration/tf/modules/feast-helm/values.tmpl create mode 100755 testing/integration/tf/modules/redis/init.sh create mode 100644 testing/integration/tf/modules/redis/input.tf create mode 100644 testing/integration/tf/modules/redis/main.tf create mode 100644 testing/integration/tf/modules/redis/output.tf delete mode 100644 testing/tf/cloud-build/cloudbuild_master.yaml delete mode 100644 testing/tf/terraform.tfvars diff --git a/.gitignore b/.gitignore index 01df87f2dc0..f35e736b20e 100644 --- a/.gitignore +++ b/.gitignore @@ -46,4 +46,122 @@ vendor .direnv .terraform/ -*.tfvars \ No newline at end of file +*.tfvars + +# python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +.vscode diff --git a/testing/tf/cloud-build/cloud-build.tf b/testing/infrastructure/cloud-build/cloud-build.tf similarity index 100% rename from testing/tf/cloud-build/cloud-build.tf rename to testing/infrastructure/cloud-build/cloud-build.tf diff --git a/testing/tf/cloud-build/cloudbuild.yaml b/testing/infrastructure/cloud-build/cloudbuild.yaml similarity index 100% rename from testing/tf/cloud-build/cloudbuild.yaml rename to testing/infrastructure/cloud-build/cloudbuild.yaml diff --git a/testing/tf/cloud-build/variables.tf b/testing/infrastructure/cloud-build/variables.tf similarity index 100% rename from testing/tf/cloud-build/variables.tf rename to testing/infrastructure/cloud-build/variables.tf diff --git a/testing/tf/gcs/backend.tf b/testing/infrastructure/gcs/backend.tf similarity index 100% rename from testing/tf/gcs/backend.tf rename to testing/infrastructure/gcs/backend.tf diff --git a/testing/tf/gcs/main.tf b/testing/infrastructure/gcs/main.tf similarity index 100% rename from testing/tf/gcs/main.tf rename to testing/infrastructure/gcs/main.tf diff --git a/testing/tf/k8s-cluster/backend.tf b/testing/infrastructure/k8s-cluster/backend.tf similarity index 100% rename from testing/tf/k8s-cluster/backend.tf rename to testing/infrastructure/k8s-cluster/backend.tf diff --git a/testing/tf/k8s-cluster/main.tf b/testing/infrastructure/k8s-cluster/main.tf similarity index 69% rename from testing/tf/k8s-cluster/main.tf rename to testing/infrastructure/k8s-cluster/main.tf index e5b01754f96..43b1de31b98 100644 --- a/testing/tf/k8s-cluster/main.tf +++ b/testing/infrastructure/k8s-cluster/main.tf @@ -16,6 +16,10 @@ resource "google_container_cluster" "primary-test-cluster" { node_config { oauth_scopes = [ "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.read_write", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", ] machine_type = "n1-standard-4" diff --git a/testing/tf/k8s-cluster/output.tf b/testing/infrastructure/k8s-cluster/output.tf similarity index 100% rename from testing/tf/k8s-cluster/output.tf rename to testing/infrastructure/k8s-cluster/output.tf diff --git a/testing/tf/k8s-cluster/provider.tf b/testing/infrastructure/k8s-cluster/provider.tf similarity index 100% rename from testing/tf/k8s-cluster/provider.tf rename to testing/infrastructure/k8s-cluster/provider.tf diff --git a/testing/tf/k8s-cluster/variables.tf b/testing/infrastructure/k8s-cluster/variables.tf similarity index 100% rename from testing/tf/k8s-cluster/variables.tf rename to testing/infrastructure/k8s-cluster/variables.tf diff --git a/testing/integration/tests/data/entity/myentity.yaml b/testing/integration/tests/data/entity/myentity.yaml new file mode 100644 index 00000000000..8fa945a6774 --- /dev/null +++ b/testing/integration/tests/data/entity/myentity.yaml @@ -0,0 +1,5 @@ +name: myentity +description: my test entity +tags: + - tag1 + - tag2 diff --git a/testing/integration/tests/data/feature/feature_minutedoubleredis1.yaml b/testing/integration/tests/data/feature/feature_minutedoubleredis1.yaml new file mode 100644 index 00000000000..4d2b7573ce4 --- /dev/null +++ b/testing/integration/tests/data/feature/feature_minutedoubleredis1.yaml @@ -0,0 +1,13 @@ +id: myentity.minute.feature_double_redis +name: feature_double_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: DOUBLE +granularity: MINUTE +uri: https://github.com/bob/example +dataStores: + serving: + id: REDIS + warehouse: + id: BIGQUERY \ No newline at end of file diff --git a/testing/integration/tests/data/feature/feature_minutefloatredis1.yaml b/testing/integration/tests/data/feature/feature_minutefloatredis1.yaml new file mode 100644 index 00000000000..d9707d31e2a --- /dev/null +++ b/testing/integration/tests/data/feature/feature_minutefloatredis1.yaml @@ -0,0 +1,13 @@ +id: myentity.minute.feature_float_redis +name: feature_float_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: FLOAT +granularity: MINUTE +uri: https://github.com/bob/example +dataStores: + serving: + id: REDIS + warehouse: + id: BIGQUERY \ No newline at end of file diff --git a/testing/integration/tests/data/feature/feature_minuteint32redis1.yaml b/testing/integration/tests/data/feature/feature_minuteint32redis1.yaml new file mode 100644 index 00000000000..56e85f68365 --- /dev/null +++ b/testing/integration/tests/data/feature/feature_minuteint32redis1.yaml @@ -0,0 +1,13 @@ +id: myentity.minute.feature_int32_redis +name: feature_int32_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: INT32 +granularity: MINUTE +uri: https://github.com/bob/example +dataStores: + serving: + id: REDIS + warehouse: + id: BIGQUERY \ No newline at end of file diff --git a/testing/integration/tests/data/feature/feature_minuteint64redis1.yaml b/testing/integration/tests/data/feature/feature_minuteint64redis1.yaml new file mode 100644 index 00000000000..c2d36affd4b --- /dev/null +++ b/testing/integration/tests/data/feature/feature_minuteint64redis1.yaml @@ -0,0 +1,13 @@ +id: myentity.minute.feature_int64_redis +name: feature_int64_redis +entity: myentity +owner: bob@example.com +description: test entity. +valueType: INT64 +granularity: MINUTE +uri: https://github.com/bob/example +dataStores: + serving: + id: REDIS + warehouse: + id: BIGQUERY \ No newline at end of file diff --git a/testing/integration/tests/data/import/import_csv.yaml b/testing/integration/tests/data/import/import_csv.yaml new file mode 100644 index 00000000000..3d940c01059 --- /dev/null +++ b/testing/integration/tests/data/import/import_csv.yaml @@ -0,0 +1,19 @@ +type: file.csv +options: + path: gs://it-feast-storage/test-cases/test_data.csv +entities: + - myentity +schema: + entityIdColumn: entity + timestampColumn: ts + fields: + - name: entity + - name: ts + - name: feature1 + featureId: myentity.minute.feature_double_redis + - name: feature2 + featureId: myentity.minute.feature_float_redis + - name: feature3 + featureId: myentity.minute.feature_int32_redis + - name: feature4 + featureId: myentity.minute.feature_int64_redis \ No newline at end of file diff --git a/testing/integration/tests/data/test_data.csv b/testing/integration/tests/data/test_data.csv new file mode 100644 index 00000000000..16906eef985 --- /dev/null +++ b/testing/integration/tests/data/test_data.csv @@ -0,0 +1,50 @@ +0,2018-01-01T00:00:00,0.16654998178974967,0.0779124915810051,8,1 +0,2018-01-01T00:01:00,0.35055416999061695,0.3789967915556205,4,7 +0,2018-01-01T00:02:00,0.922617931213872,0.3179716588679623,8,6 +0,2018-01-01T00:03:00,0.06582437841319277,0.05565087947895475,9,1 +0,2018-01-01T00:04:00,0.8470568422759338,0.21347178570025072,2,10 +0,2018-01-01T00:05:00,0.11361651376595083,0.610881579742112,8,10 +0,2018-01-01T00:06:00,0.3034691430518833,0.19830856710384326,6,2 +0,2018-01-01T00:07:00,0.8346843022317924,0.08544530941545558,2,4 +0,2018-01-01T00:08:00,0.012367592836242891,0.5726466047910151,2,5 +0,2018-01-01T00:09:00,0.9887391081292758,0.9819570408280592,9,10 +1,2018-01-01T00:00:00,0.7503367819099366,0.946197160138704,10,1 +1,2018-01-01T00:01:00,0.9190195271598668,0.8836270958333856,10,4 +1,2018-01-01T00:02:00,0.07725768926853083,0.448071643977631,4,9 +1,2018-01-01T00:03:00,0.4965090410288343,0.9615236179532041,3,5 +1,2018-01-01T00:04:00,0.4838512691935104,0.355736320341819,7,2 +1,2018-01-01T00:05:00,0.7422860280929264,0.26936605638140254,7,6 +1,2018-01-01T00:06:00,0.007120660156963887,0.9457195989857463,5,4 +1,2018-01-01T00:07:00,0.5165841927879966,0.5734751201042606,1,4 +1,2018-01-01T00:08:00,0.0874458350429409,0.3284354801066641,3,8 +1,2018-01-01T00:09:00,0.8091561596661963,0.5176044026790632,1,9 +2,2018-01-01T00:00:00,0.5440258526780313,0.7936189500778384,3,5 +2,2018-01-01T00:01:00,0.9012681719733718,0.408129175696462,5,8 +2,2018-01-01T00:02:00,0.1357239530601766,0.2882272639567518,9,9 +2,2018-01-01T00:03:00,0.5482100359374814,0.22784885285759426,2,10 +2,2018-01-01T00:04:00,0.47915480013638667,0.5783117797172574,6,2 +2,2018-01-01T00:05:00,0.08954537587399436,0.6036422156720167,2,5 +2,2018-01-01T00:06:00,0.022254256710355302,0.8425123292474225,2,2 +2,2018-01-01T00:07:00,0.2981277839268307,0.7187791198561501,9,6 +2,2018-01-01T00:08:00,0.7483695369918476,0.24681429759605789,10,3 +2,2018-01-01T00:09:00,0.1357618414286158,0.887316489600256,3,10 +3,2018-01-01T00:00:00,0.24835796330732807,0.8871182399308823,3,2 +3,2018-01-01T00:01:00,0.18779166308706263,0.6635037319574626,4,10 +3,2018-01-01T00:02:00,0.00017995781865332017,0.9437703134427515,9,2 +3,2018-01-01T00:03:00,0.44519915810431854,0.03739113677349182,4,5 +3,2018-01-01T00:04:00,0.27156326671637077,0.06880899222977854,3,2 +3,2018-01-01T00:05:00,0.7647529493108539,0.14002956138831768,7,10 +3,2018-01-01T00:06:00,0.9954469835451967,0.7567907196365666,9,9 +3,2018-01-01T00:07:00,0.3883998745214877,0.18671618739049767,7,10 +3,2018-01-01T00:08:00,0.5448378857720906,0.38482161282383076,3,1 +3,2018-01-01T00:09:00,0.3550003641256,0.1818481760674323,1,9 +4,2018-01-01T00:00:00,0.5925452403226181,0.30212100652221296,5,10 +4,2018-01-01T00:01:00,0.18599533814587,0.5230622774213863,6,5 +4,2018-01-01T00:02:00,0.9599946262502064,0.26435543431553166,10,8 +4,2018-01-01T00:03:00,0.22755474767811312,0.6693510146123766,1,8 +4,2018-01-01T00:04:00,0.12063065422121622,0.7460800163440976,10,9 +4,2018-01-01T00:05:00,0.58262241980741,0.2714493926314999,9,5 +4,2018-01-01T00:06:00,0.26952820440313163,0.40841655204277727,5,1 +4,2018-01-01T00:07:00,0.7779897852625218,0.39955885342995734,9,3 +4,2018-01-01T00:08:00,0.9892858266244913,0.91265457567995,2,10 +4,2018-01-01T00:09:00,0.31825556885425366,0.6727958709215186,3,9 diff --git a/testing/integration/tests/test_feast.py b/testing/integration/tests/test_feast.py new file mode 100644 index 00000000000..3b9409f098b --- /dev/null +++ b/testing/integration/tests/test_feast.py @@ -0,0 +1,97 @@ +import pytest +import subprocess +from subprocess import PIPE +import yaml +import os +from time import sleep +import pandas as pd + +from feast.sdk.resources.entity import Entity +from feast.sdk.resources.feature import Feature +from feast.sdk.client import Client +from feast.sdk.resources.feature_set import FeatureSet, FileType +from feast.sdk.utils.bq_util import TableDownloader + +@pytest.fixture +def client(): + return Client(verbose=True) + +'''Init the system by registering relevant resources. +''' +def register_resources(client, entities_fldr, features_fldr): + resources = [] + for ent_file in os.listdir(entities_fldr): + resources.append(Entity.from_yaml(os.path.join(entities_fldr, ent_file))) + for feat_file in os.listdir(features_fldr): + resources.append(Feature.from_yaml(os.path.join(features_fldr, feat_file))) + client.apply(resources) + +'''Run an import job given an import spec. +''' +def run_job_and_wait_for_completion(job_yaml): + out = subprocess.run("feast jobs run {}".format(job_yaml).split(" "), + check=True, stdout=PIPE) + job_id = out.stdout.decode('utf-8').split(' ')[-1] + job_complete = False + terminal_states = ["COMPLETED", "ABORTED", "ERROR", "UNKNOWN"] + while not job_complete: + out = subprocess.run("feast get job {}".format(job_id).split(" "), + check=True, stdout=PIPE) + job_details = yaml.load(out.stdout.decode('utf-8').replace('\t', ' ')) + job_status = job_details['Status'] + print("Job id {} currently {}".format(job_id, job_status)) + if job_status in terminal_states: + break + sleep(10) + return job_status + +class TestFeastIntegration: + def test_end_to_end(self, client): + project_id = os.environ.get("PROJECT_ID") + bucket_name = os.environ.get("BUCKET_NAME") + + subprocess.run("gsutil cp {} {}".format( + "data/test_data.csv", + "gs://{}/test-cases/test_data.csv".format(bucket_name) + ).split(" "), check=True, stdout=PIPE) + register_resources(client, "data/entity", "data/feature") + result = run_job_and_wait_for_completion("data/import/import_csv.yaml") + + # Ensure that the job is able to reach completion + assert result == "COMPLETED" + + # Check data in bq + features = ["feature_double_redis", "feature_float_redis", + "feature_int32_redis", "feature_int64_redis"] + wanted = pd.read_csv("data/test_data.csv", + header=None, + names=["id", "event_timestamp"] + features) + wanted = wanted \ + .sort_values(["id", "event_timestamp"]) \ + .reset_index(drop=True) + wanted['event_timestamp'] = pd.to_datetime(wanted['event_timestamp'])\ + .dt.strftime("%Y-%m-%d %H:%M:%S UTC") + got = TableDownloader().download_table_as_df( + project_id + ".feast_it.myentity_minute", + "gs://{}/test-cases/extract.csv".format(bucket_name)) + got = got.drop("created_timestamp", axis=1) \ + .sort_values(["id", "event_timestamp"]) \ + .reset_index(drop=True) + assert pd.testing.assert_frame_equal(got, wanted, check_less_precise=True) is None + + # Check data in redis + feature_set = FeatureSet(entity="myentity", + features=["myentity.minute." + f for f in features]) + actual_latest = client.get_serving_data(feature_set, entity_keys=[str(id) for id in list(wanted.id.unique())]) + actual_latest = actual_latest.sort_values(["id"]) + wanted['event_timestamp'] = pd.to_datetime(wanted['event_timestamp']) + wanted_latest = wanted.loc[wanted.groupby('id').event_timestamp.idxmax(),:] + wanted_latest.columns = ["myentity", "timestamp"] + ["myentity.minute." + f for f in features] + wanted_latest = wanted_latest[actual_latest.columns] \ + .sort_values(["id"]) \ + .reset_index(drop=True) + wanted_latest["myentity"] = wanted_latest["myentity"].astype(str) + + assert pd.testing.assert_frame_equal(actual_latest, wanted_latest, check_less_precise=True) is None + + diff --git a/testing/integration/tests/tf/backend.tf b/testing/integration/tests/tf/backend.tf new file mode 100644 index 00000000000..fb1f9c81b5b --- /dev/null +++ b/testing/integration/tests/tf/backend.tf @@ -0,0 +1,6 @@ +terraform { + backend "gcs" { + bucket = "kf-feast-terraform-state" + prefix = "tf/k8s-cluster/integration-test" + } +} diff --git a/testing/integration/tests/tf/data.tf b/testing/integration/tests/tf/data.tf new file mode 100644 index 00000000000..a988612c59d --- /dev/null +++ b/testing/integration/tests/tf/data.tf @@ -0,0 +1 @@ +data "google_client_config" "current" {} diff --git a/testing/integration/tests/tf/input.tf b/testing/integration/tests/tf/input.tf new file mode 100644 index 00000000000..6bd1b34bb09 --- /dev/null +++ b/testing/integration/tests/tf/input.tf @@ -0,0 +1,8 @@ +variable "revision" { + description = "Feast revision to pull helm charts from" +} + +variable "docker_tag" { + description = "Docker image to deploy" + default = "" +} diff --git a/testing/integration/tests/tf/main.tf b/testing/integration/tests/tf/main.tf new file mode 100644 index 00000000000..03455685493 --- /dev/null +++ b/testing/integration/tests/tf/main.tf @@ -0,0 +1,125 @@ +locals { + project_name = "kf-feast" + region = "us-central1" + subnetwork = "regions/${local.region}/subnetworks/default" + network = "default" + cluster_name = "it-feast" + + job_runner_options = { + "project" = "${local.project_name}" + "region" = "${local.region}" + "tempLocation" = "gs://${local.cluster_name}-storage/tempJob" + "subnetwork" = "${local.subnetwork}" + "maxNumWorkers" = "64" + "autoscalingAlgorithm" = "THROUGHPUT_BASED" + } + + errors_store_options = { + "path" = "gs://${local.cluster_name}-storage/errors" + } +} + +module "cluster" { + source = "../../tf/modules/cluster" + cluster_name = "${local.cluster_name}" + project_name = "${local.project_name}" + region = "${local.region}" + bucket_name = "${local.cluster_name}-storage" + network = "${local.network}" + subnetwork = "${local.subnetwork}" +} + +resource "null_resource" "wait_for_regional_cluster" { + provisioner "local-exec" { + command = "${path.module}/scripts/wait-for-cluster.sh ${local.project_name} ${local.cluster_name}" + } + + provisioner "local-exec" { + when = "destroy" + command = "${path.module}/scripts/wait-for-cluster.sh ${local.project_name} ${local.cluster_name}" + } + + depends_on = ["module.cluster"] +} + +module "feast" { + source = "../../tf/modules/feast-helm" + project_name = "${local.project_name}" + region = "${local.region}" + subnetwork = "${local.subnetwork}" + revision = "master" + docker_tag = "${var.docker_tag == "" ? var.revision : var.docker_tag}" + core_address = "10.128.0.99" + serving_address = "10.128.0.100" + load_balancer_source_range = "10.0.0.0/8" + job_runner = "DataflowRunner" + job_runner_options = "'${jsonencode(local.job_runner_options)}'" + errors_store_type = "file.json" + errors_store_options = "'${jsonencode(local.errors_store_options)}'" + + depends_on = ["module.cluster.cluster_name", "null_resource.wait_for_regional_cluster"] +} + +module "redis" { + source = "../../tf/modules/redis" + source = "../redis" + project_name = "${local.project_name}" + name = "${local.cluster_name}-redis" + zone = "${local.region}-a" + subnet = "${local.subnetwork}" +} + +resource "google_bigquery_dataset" "feast_bq_dataset" { + dataset_id = "feast_it" + description = "Feast integration test dataset" + default_table_expiration_ms = 36000000 + location = "US" +} + +resource "null_resource" "empty_bq" { + provisioner "local-exec" { + when = "destroy" + command = "./scripts/empty-bq.sh ${local.project_name} feast_it" + } + + depends_on = ["google_bigquery_dataset.feast_bq_dataset"] +} + +resource "local_file" "redis_spec" { + content = <> redis.conf + +redis-server --protected-mode no \ No newline at end of file diff --git a/testing/integration/tf/modules/redis/input.tf b/testing/integration/tf/modules/redis/input.tf new file mode 100644 index 00000000000..0ac81e9a80f --- /dev/null +++ b/testing/integration/tf/modules/redis/input.tf @@ -0,0 +1,52 @@ +variable depends_on { + default = [] + + type = "list" +} + +variable "project_name" { + description = "project name" +} + +variable "zone" { + description = "zone to create the instance in" +} + +variable "type" { + default = "n1-standard-1" + description = "machine type. Allowed values are https://cloud.google.com/compute/docs/machine-types" +} + +variable "internet_tag" { + description = "default internet tag. E.g: allow-internet, default-to-internet" + default = "allow-internet" +} + +variable "boot_disk_size" { + description = "boot disk size" + default = "10" +} + +variable "boot_disk_type" { + description = "boot disk type. Allowed values are pd-standard,pd-ssd" + default = "pd-standard" +} + +variable "boot_disk_image" { + description = "boot disk image" + default = "ubuntu-os-cloud/ubuntu-1604-xenial-v20170610" +} + +variable "subnet" { + description = "subnet name." + default = "default" +} + +variable "network" { + description = "network name. E.g default" + default = "default" +} + +variable "name" { + description = "Instance name" +} diff --git a/testing/integration/tf/modules/redis/main.tf b/testing/integration/tf/modules/redis/main.tf new file mode 100644 index 00000000000..f5f39725053 --- /dev/null +++ b/testing/integration/tf/modules/redis/main.tf @@ -0,0 +1,33 @@ +resource "google_compute_instance" "redis_vm" { + name = "${var.name}" + machine_type = "${var.type}" + zone = "${var.zone}" + + tags = ["${var.internet_tag}"] + + boot_disk { + initialize_params { + size = "${var.boot_disk_size}" + type = "${var.boot_disk_type}" + image = "${var.boot_disk_image}" + } + } + + lifecycle { + ignore_changes = ["attached_disk"] + } + + scheduling { + automatic_restart = true + } + + network_interface { + subnetwork = "${var.subnet}" + + # access_config {} + } + + metadata { + startup-script = "${file("${path.module}/init.sh")}" + } +} diff --git a/testing/integration/tf/modules/redis/output.tf b/testing/integration/tf/modules/redis/output.tf new file mode 100644 index 00000000000..8c8a135227b --- /dev/null +++ b/testing/integration/tf/modules/redis/output.tf @@ -0,0 +1,3 @@ +output "instance_url" { + value = "${google_compute_instance.redis_vm.network_interface.0.network_ip}" +} diff --git a/testing/tf/cloud-build/cloudbuild_master.yaml b/testing/tf/cloud-build/cloudbuild_master.yaml deleted file mode 100644 index 743e7588a52..00000000000 --- a/testing/tf/cloud-build/cloudbuild_master.yaml +++ /dev/null @@ -1,19 +0,0 @@ -steps: -- name: maven:3.6.0-jdk-8-slim - args: ['mvn', 'clean', 'install', '-Drevision=$COMMIT_SHA', '-DskipTests'] - id: build-jars - -- name: "gcr.io/cloud-builders/docker" - args: ['build', '-t', 'gcr.io/$PROJECT_ID/feast-core:$COMMIT_SHA', '--build-arg', 'RELEASE=$COMMIT_SHA', '-f', './docker/core/Dockerfile', '.'] - id: docker-core - waitFor: - - build-jars - -- name: "gcr.io/cloud-builders/docker" - args: ['build', '-t', 'gcr.io/$PROJECT_ID/feast-serving:$COMMIT_SHA', '--build-arg', 'RELEASE=$COMMIT_SHA', '-f', './docker/serving/Dockerfile', '.'] - id: docker-serving - waitFor: - - build-jars - -timeout: 1200s -images: ['gcr.io/$PROJECT_ID/feast-core:$COMMIT_SHA', 'gcr.io/$PROJECT_ID/feast-serving:$COMMIT_SHA'] \ No newline at end of file diff --git a/testing/tf/terraform.tfvars b/testing/tf/terraform.tfvars deleted file mode 100644 index a8b45f64116..00000000000 --- a/testing/tf/terraform.tfvars +++ /dev/null @@ -1,7 +0,0 @@ -gcp_project = "kf-feast" - -default_network = "default" - -default_subnet = "default" - -default_region = "us-central1" From 7b78f35f2e8f8cd6905af20961c9ceeaebc4ea36 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 8 Feb 2019 22:52:54 +0800 Subject: [PATCH 02/47] Add cleanup stage, remove interpreter --- .../docker-images/terraform-image/Dockerfile | 12 +++++++++++ .../{ => docker-images}/test-image/Dockerfile | 0 testing/{ => docker-images}/test-image/run.sh | 0 testing/integration/tests/tf/main.tf | 1 - testing/integration/tests/workflow.yaml | 20 ++++++++++++++++--- 5 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 testing/docker-images/terraform-image/Dockerfile rename testing/{ => docker-images}/test-image/Dockerfile (100%) rename testing/{ => docker-images}/test-image/run.sh (100%) diff --git a/testing/docker-images/terraform-image/Dockerfile b/testing/docker-images/terraform-image/Dockerfile new file mode 100644 index 00000000000..6fa172c3798 --- /dev/null +++ b/testing/docker-images/terraform-image/Dockerfile @@ -0,0 +1,12 @@ +FROM hashicorp/terraform:light + +RUN apk add --update jq wget python + +RUN wget -qO- https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-233.0.0-linux-x86_64.tar.gz | tar xzf - \ + && /google-cloud-sdk/bin/gcloud -q components install kubectl \ + && /google-cloud-sdk/bin/gcloud -q components install beta + +RUN wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-arm64.tar.gz | tar xzf - \ + && mv linux-arm64/helm /bin/helm + +RUN helm init --client-only diff --git a/testing/test-image/Dockerfile b/testing/docker-images/test-image/Dockerfile similarity index 100% rename from testing/test-image/Dockerfile rename to testing/docker-images/test-image/Dockerfile diff --git a/testing/test-image/run.sh b/testing/docker-images/test-image/run.sh similarity index 100% rename from testing/test-image/run.sh rename to testing/docker-images/test-image/run.sh diff --git a/testing/integration/tests/tf/main.tf b/testing/integration/tests/tf/main.tf index 03455685493..f19a63a283d 100644 --- a/testing/integration/tests/tf/main.tf +++ b/testing/integration/tests/tf/main.tf @@ -118,7 +118,6 @@ resource "null_resource" "feast_register" { # for local testing, you might want to set feast to send to localhost instead # command = "feast config set coreURI localhost:6565 && feast apply storage redis.yaml && feast apply storage bigquery.yaml" - interpreter = ["/bin/bash", "-c"] } depends_on = ["local_file.redis_spec", "local_file.bigquery_spec", "module.feast"] diff --git a/testing/integration/tests/workflow.yaml b/testing/integration/tests/workflow.yaml index 6131591a76e..36f451a57a4 100644 --- a/testing/integration/tests/workflow.yaml +++ b/testing/integration/tests/workflow.yaml @@ -24,7 +24,7 @@ spec: arguments: parameters: - name: arg - value: "-var revision={{workflow.parameters.revision}} -var docker_tag=coalesce -auto-approve ." + value: "apply -var revision={{workflow.parameters.revision}} -var docker_tag=coalesce -auto-approve ." artifacts: - name: feast from: "{{steps.get-feast.outputs.artifacts.feast}}" @@ -38,6 +38,15 @@ spec: from: "{{steps.get-feast.outputs.artifacts.feast}}" - name: feast-cli from: "{{steps.build-cli.outputs.artifacts.feast-cli}}" + - - name: terraform-destroy + template: terraform + arguments: + parameters: + - name: arg + value: "destroy -var revision={{workflow.parameters.revision}} -var docker_tag=coalesce -auto-approve ." + artifacts: + - name: feast + from: "{{steps.get-feast.outputs.artifacts.feast}}" - name: get-feast container: @@ -74,12 +83,12 @@ spec: - name: feast-cli path: /bin/feast container: - image: hashicorp/terraform:light + image: gcr.io/kf-feast/terraform:light workingDir: /feast/testing/integration/tests/tf command: ["sh", "-c"] args: [' terraform init; - terraform apply {{inputs.parameters.arg}} + terraform {{inputs.parameters.arg}} '] - name: run-test inputs: @@ -97,3 +106,8 @@ spec: pytest -s '] workingDir: /feast/testing/integration/tests + env: + - name: FEAST_CORE_URL + value: "10.128.0.99:6565" + - name: FEAST_SERVING_URL + value: "10.128.0.100:6566" From 85c1b35188bf8d7cd81495d5014adf3cf1e7ec16 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Sat, 9 Feb 2019 07:34:53 +0800 Subject: [PATCH 03/47] Change directory to point to fork --- testing/docker-images/terraform-image/Dockerfile | 8 ++++++-- testing/integration/tests/workflow.yaml | 2 +- testing/integration/tf/modules/feast-helm/main.tf | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/testing/docker-images/terraform-image/Dockerfile b/testing/docker-images/terraform-image/Dockerfile index 6fa172c3798..821ad1f9f8f 100644 --- a/testing/docker-images/terraform-image/Dockerfile +++ b/testing/docker-images/terraform-image/Dockerfile @@ -1,11 +1,15 @@ -FROM hashicorp/terraform:light +FROM debian:stretch -RUN apk add --update jq wget python +RUN apt-get update && apt-get install -y jq wget python zip RUN wget -qO- https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-233.0.0-linux-x86_64.tar.gz | tar xzf - \ && /google-cloud-sdk/bin/gcloud -q components install kubectl \ && /google-cloud-sdk/bin/gcloud -q components install beta +RUN wget https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip \ + && unzip terraform_0.11.11_linux_amd64.zip \ + && mv terraform /bin/terraform + RUN wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-arm64.tar.gz | tar xzf - \ && mv linux-arm64/helm /bin/helm diff --git a/testing/integration/tests/workflow.yaml b/testing/integration/tests/workflow.yaml index 36f451a57a4..2814e4d61e0 100644 --- a/testing/integration/tests/workflow.yaml +++ b/testing/integration/tests/workflow.yaml @@ -83,7 +83,7 @@ spec: - name: feast-cli path: /bin/feast container: - image: gcr.io/kf-feast/terraform:light + image: gcr.io/kf-feast/terraform:integration-test workingDir: /feast/testing/integration/tests/tf command: ["sh", "-c"] args: [' diff --git a/testing/integration/tf/modules/feast-helm/main.tf b/testing/integration/tf/modules/feast-helm/main.tf index 548c61eda10..251d9c59b47 100644 --- a/testing/integration/tf/modules/feast-helm/main.tf +++ b/testing/integration/tf/modules/feast-helm/main.tf @@ -64,7 +64,7 @@ resource "local_file" "helm_values_output" { resource "helm_release" "feast" { name = "feast-it" - repository = "https://raw.githubusercontent.com/gojek/feast/master/charts/dist/" + repository = "https://raw.githubusercontent.com/zhilingc/feast/master/charts/dist/" chart = "feast" version = "0.1.0" From 449c789435ff9795c0c1141c81ab36da126f289f Mon Sep 17 00:00:00 2001 From: zhilingc Date: Sat, 9 Feb 2019 08:00:56 +0800 Subject: [PATCH 04/47] Remove dependency on gsutil --- testing/integration/tests/test_feast.py | 19 +++++++++++++++---- testing/integration/tests/workflow.yaml | 6 ++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/testing/integration/tests/test_feast.py b/testing/integration/tests/test_feast.py index 3b9409f098b..897b1311fdf 100644 --- a/testing/integration/tests/test_feast.py +++ b/testing/integration/tests/test_feast.py @@ -5,6 +5,9 @@ import os from time import sleep import pandas as pd +from google.cloud import storage + +import feast.sdk.utils.gs_utils as utils from feast.sdk.resources.entity import Entity from feast.sdk.resources.feature import Feature @@ -45,15 +48,23 @@ def run_job_and_wait_for_completion(job_yaml): sleep(10) return job_status +'''Stage data to a remote location +''' +def stage_data(local, remote): + split = utils.split_gs_path(remote) + storage_client = storage.Client() + bucket = storage_client.get_bucket(split[0]) + blob = bucket.blob(split[1]) + + blob.upload_from_filename(local) + + class TestFeastIntegration: def test_end_to_end(self, client): project_id = os.environ.get("PROJECT_ID") bucket_name = os.environ.get("BUCKET_NAME") - subprocess.run("gsutil cp {} {}".format( - "data/test_data.csv", - "gs://{}/test-cases/test_data.csv".format(bucket_name) - ).split(" "), check=True, stdout=PIPE) + stage_data("data/test_data.csv", "gs://{}/test-cases/test_data.csv".format(bucket_name)) register_resources(client, "data/entity", "data/feature") result = run_job_and_wait_for_completion("data/import/import_csv.yaml") diff --git a/testing/integration/tests/workflow.yaml b/testing/integration/tests/workflow.yaml index 2814e4d61e0..72327586340 100644 --- a/testing/integration/tests/workflow.yaml +++ b/testing/integration/tests/workflow.yaml @@ -101,6 +101,7 @@ spec: image: python:3.7.2 command: ["/bin/bash", "-c"] args: [' + feast config set coreURI 10.128.0.99:6565 pip install -r /feast/sdk/python/test-requirements.txt; pip install -e /feast/sdk/python; pytest -s @@ -111,3 +112,8 @@ spec: value: "10.128.0.99:6565" - name: FEAST_SERVING_URL value: "10.128.0.100:6566" + - name: PROJECT_ID + value: kf-feast + - name: BUCKET_NAME + value: it-feast-storage + From e1398053858100216b49e80103ce22e8a156f6ba Mon Sep 17 00:00:00 2001 From: zhilingc Date: Sat, 9 Feb 2019 23:58:22 +0800 Subject: [PATCH 05/47] Add exit handler --- testing/integration/tests/workflow.yaml | 27 ++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/testing/integration/tests/workflow.yaml b/testing/integration/tests/workflow.yaml index 72327586340..4d8092c4236 100644 --- a/testing/integration/tests/workflow.yaml +++ b/testing/integration/tests/workflow.yaml @@ -4,6 +4,7 @@ metadata: generateName: feast-integration-test- spec: entrypoint: run + onExit: exit-handler arguments: parameters: - name: revision @@ -38,15 +39,18 @@ spec: from: "{{steps.get-feast.outputs.artifacts.feast}}" - name: feast-cli from: "{{steps.build-cli.outputs.artifacts.feast-cli}}" + + - name: exit-handler + steps: - - name: terraform-destroy - template: terraform + template: terraform-destroy arguments: parameters: - name: arg value: "destroy -var revision={{workflow.parameters.revision}} -var docker_tag=coalesce -auto-approve ." artifacts: - name: feast - from: "{{steps.get-feast.outputs.artifacts.feast}}" + from: "{{workflow.outputs.artifacts.feast-repo}}" - name: get-feast container: @@ -56,6 +60,7 @@ spec: artifacts: - name: feast path: /git/feast + globalName: feast-repo - name: build-cli inputs: artifacts: @@ -101,7 +106,8 @@ spec: image: python:3.7.2 command: ["/bin/bash", "-c"] args: [' - feast config set coreURI 10.128.0.99:6565 + feast config set coreURI 10.128.0.99:6565; + pip install pytest; pip install -r /feast/sdk/python/test-requirements.txt; pip install -e /feast/sdk/python; pytest -s @@ -116,4 +122,19 @@ spec: value: kf-feast - name: BUCKET_NAME value: it-feast-storage + - name: terraform-destroy + inputs: + parameters: + - name: arg + artifacts: + - name: feast + path: /feast + container: + image: gcr.io/kf-feast/terraform:integration-test + workingDir: /feast/testing/integration/tests/tf + command: ["sh", "-c"] + args: [' + terraform init; + terraform {{inputs.parameters.arg}} + '] From 49239758433cb59f9279a73c3dfcb114aacc57a5 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Sun, 10 Feb 2019 23:11:04 +0800 Subject: [PATCH 06/47] Add wait script for redis --- testing/integration/tf/modules/redis/init.sh | 8 +++----- testing/integration/tf/modules/redis/main.tf | 9 +++++++++ .../tf/modules/redis/wait-for-startup.sh | 13 +++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) create mode 100755 testing/integration/tf/modules/redis/wait-for-startup.sh diff --git a/testing/integration/tf/modules/redis/init.sh b/testing/integration/tf/modules/redis/init.sh index 1f65c0c542e..933dc92cc99 100755 --- a/testing/integration/tf/modules/redis/init.sh +++ b/testing/integration/tf/modules/redis/init.sh @@ -7,9 +7,7 @@ wget http://download.redis.io/redis-stable.tar.gz tar xvzf redis-stable.tar.gz cd redis-stable -make install - -sed -i "s/protected-mode.*//g" redis.conf -echo "protected-mode no" >> redis.conf +VMNAME=$(curl -H Metadata-Flavor:Google http://metadata/computeMetadata/v1/instance/hostname | cut -d. -f1) -redis-server --protected-mode no \ No newline at end of file +make install +redis-server --protected-mode no \ No newline at end of file diff --git a/testing/integration/tf/modules/redis/main.tf b/testing/integration/tf/modules/redis/main.tf index f5f39725053..a6d2005cb79 100644 --- a/testing/integration/tf/modules/redis/main.tf +++ b/testing/integration/tf/modules/redis/main.tf @@ -31,3 +31,12 @@ resource "google_compute_instance" "redis_vm" { startup-script = "${file("${path.module}/init.sh")}" } } + + +resource "null_resource" "wait_for_startup" { + provisioner "local-exec" { + command = "${path.module}/wait-for-startup.sh ${var.name} ${google_compute_instance.redis_vm.network_interface.0.network_ip}" + } + + depends_on = ["google_compute_instance.redis_vm"] +} diff --git a/testing/integration/tf/modules/redis/wait-for-startup.sh b/testing/integration/tf/modules/redis/wait-for-startup.sh new file mode 100755 index 00000000000..af4a2ffd662 --- /dev/null +++ b/testing/integration/tf/modules/redis/wait-for-startup.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +INSTANCE_ID=$1 +HOST=$2 +echo "Waiting for startup script in instance $INSTANCE_ID to complete..." + +current_status=$((printf "PING\r\n";) | nc -w1 $HOST 6379; echo $?) + +while [[ "$current_status" -eq "1" ]]; do + printf "." + sleep 5 + current_status=$((printf "PING\r\n";) | nc -w1 $HOST 6379; echo $?) +done From b4cfe02a4ad3681cc9b19f207b98d2e051aab28e Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 11 Feb 2019 00:28:20 +0800 Subject: [PATCH 07/47] Add bq to terraform image --- testing/docker-images/terraform-image/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/docker-images/terraform-image/Dockerfile b/testing/docker-images/terraform-image/Dockerfile index 821ad1f9f8f..069a258aab3 100644 --- a/testing/docker-images/terraform-image/Dockerfile +++ b/testing/docker-images/terraform-image/Dockerfile @@ -4,7 +4,8 @@ RUN apt-get update && apt-get install -y jq wget python zip RUN wget -qO- https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-233.0.0-linux-x86_64.tar.gz | tar xzf - \ && /google-cloud-sdk/bin/gcloud -q components install kubectl \ - && /google-cloud-sdk/bin/gcloud -q components install beta + && /google-cloud-sdk/bin/gcloud -q components install beta \ + && /google-cloud-sdk/bin/gcloud -q components install bq RUN wget https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip \ && unzip terraform_0.11.11_linux_amd64.zip \ From 30808a239f087ca8fc0aaa4bf5052480498ea688 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 11 Feb 2019 00:48:25 +0800 Subject: [PATCH 08/47] Change hard coded path --- testing/integration/tests/tf/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/tests/tf/main.tf b/testing/integration/tests/tf/main.tf index f19a63a283d..740dd2393f6 100644 --- a/testing/integration/tests/tf/main.tf +++ b/testing/integration/tests/tf/main.tf @@ -79,7 +79,7 @@ resource "google_bigquery_dataset" "feast_bq_dataset" { resource "null_resource" "empty_bq" { provisioner "local-exec" { when = "destroy" - command = "./scripts/empty-bq.sh ${local.project_name} feast_it" + command = ". ${path.module}/scripts/empty-bq.sh ${local.project_name} feast_it" } depends_on = ["google_bigquery_dataset.feast_bq_dataset"] From c46125dd39e9e2747c0648cd20a321d1cd4607e9 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 11 Feb 2019 17:29:02 +0800 Subject: [PATCH 09/47] Update readme and makefile --- testing/Makefile | 4 ++-- testing/README.md | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/testing/Makefile b/testing/Makefile index 0eef0abab9a..0ff05f827fa 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -32,6 +32,6 @@ update-config: get-cluster-credentials update-plugins: get-cluster-credentials kubectl create configmap plugins --from-file=plugins.yaml=prow/plugins.yaml --dry-run -o yaml | kubectl replace configmap plugins -f - -build-push: - docker build test-image/ -t $(REGISTRY)/$(PROJECT)/test-image:$(VERSION) +build-push-test-image: + docker build -f docker-images/test-image/Dockerfile -t $(REGISTRY)/$(PROJECT)/test-image:$(VERSION) . $(PUSH) "$(REGISTRY)/$(PROJECT)/test-image:$(VERSION)" \ No newline at end of file diff --git a/testing/README.md b/testing/README.md index c345c46e75e..56abb7d7b51 100644 --- a/testing/README.md +++ b/testing/README.md @@ -4,9 +4,9 @@ This folder contains the Feast test infrastructure. ## Components -* test-image/ - Base docker image and script for running tests. +* docker-images/ - Docker images for running tests * prow/ - Prow configuration (plugins and jobs) -* tf/ - Terraform modules to provision the base testing infrastructure on GCP +* infrastructure/ - Terraform modules to provision the base testing infrastructure on GCP ## Set up @@ -23,7 +23,7 @@ mv tf/gcs terraform import google_storage_bucket.kf-feast-terraform-state kf-feast-terraform-state ``` -4. Ensure that all variables are set correctly in `tf/terraform.tfvars`. It is likely that the GCP project will need to be updated. +4. Ensure that all variables are set correctly in `infrastructure/terraform.tfvars`. It is likely that the GCP project will need to be updated. 5. Create the primary Kubernetes cluster which will host Prow and Argo @@ -51,7 +51,7 @@ To update Prow jobs, plugins, or the Docker image used for testing, modify one o - `prow/config.yaml` - `prow/plugins.yaml` -- `test-image/Dockerfile` -- `test-image/run.sh` +- `docker-images/test-image/Dockerfile` +- `docker-images/test-image/run.sh` After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry. \ No newline at end of file From e9130a20868efe7eabad8a6535784eb260e0bba9 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 11 Feb 2019 18:17:46 +0800 Subject: [PATCH 10/47] Change docker image --- testing/docker-images/terraform-image/Dockerfile | 4 +++- testing/integration/tests/workflow.yaml | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/testing/docker-images/terraform-image/Dockerfile b/testing/docker-images/terraform-image/Dockerfile index 069a258aab3..04b734fbef6 100644 --- a/testing/docker-images/terraform-image/Dockerfile +++ b/testing/docker-images/terraform-image/Dockerfile @@ -1,12 +1,14 @@ FROM debian:stretch -RUN apt-get update && apt-get install -y jq wget python zip +RUN apt-get update && apt-get install -y jq wget python zip netcat RUN wget -qO- https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-233.0.0-linux-x86_64.tar.gz | tar xzf - \ && /google-cloud-sdk/bin/gcloud -q components install kubectl \ && /google-cloud-sdk/bin/gcloud -q components install beta \ && /google-cloud-sdk/bin/gcloud -q components install bq +ENV PATH="/google-cloud-sdk/bin:${PATH}" + RUN wget https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip \ && unzip terraform_0.11.11_linux_amd64.zip \ && mv terraform /bin/terraform diff --git a/testing/integration/tests/workflow.yaml b/testing/integration/tests/workflow.yaml index 4d8092c4236..03d0e903b6a 100644 --- a/testing/integration/tests/workflow.yaml +++ b/testing/integration/tests/workflow.yaml @@ -88,7 +88,7 @@ spec: - name: feast-cli path: /bin/feast container: - image: gcr.io/kf-feast/terraform:integration-test + image: gcr.io/kf-feast/it-terraform:0.1.0 workingDir: /feast/testing/integration/tests/tf command: ["sh", "-c"] args: [' @@ -130,7 +130,7 @@ spec: - name: feast path: /feast container: - image: gcr.io/kf-feast/terraform:integration-test + image: gcr.io/kf-feast/it-terraform:0.1.0 workingDir: /feast/testing/integration/tests/tf command: ["sh", "-c"] args: [' From 445b11f80e3ea4b01a7c67fb5027c6f5ebe2a0c8 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 11 Feb 2019 18:56:03 +0800 Subject: [PATCH 11/47] Remove redundant line --- testing/integration/tf/modules/redis/init.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/testing/integration/tf/modules/redis/init.sh b/testing/integration/tf/modules/redis/init.sh index 933dc92cc99..c9f617b5874 100755 --- a/testing/integration/tf/modules/redis/init.sh +++ b/testing/integration/tf/modules/redis/init.sh @@ -7,7 +7,5 @@ wget http://download.redis.io/redis-stable.tar.gz tar xvzf redis-stable.tar.gz cd redis-stable -VMNAME=$(curl -H Metadata-Flavor:Google http://metadata/computeMetadata/v1/instance/hostname | cut -d. -f1) - make install redis-server --protected-mode no \ No newline at end of file From 4f44a1847cadd28862c4c4411abd7add641addbf Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 11 Feb 2019 20:57:12 +0800 Subject: [PATCH 12/47] Update helm values template --- .../tf/modules/feast-helm/values.tmpl | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/testing/integration/tf/modules/feast-helm/values.tmpl b/testing/integration/tf/modules/feast-helm/values.tmpl index 6115e9e11ba..6585e60a254 100644 --- a/testing/integration/tf/modules/feast-helm/values.tmpl +++ b/testing/integration/tf/modules/feast-helm/values.tmpl @@ -26,7 +26,7 @@ core: jobs: runner: "${job_runner}" options: ${job_runner_options} - errorsStoreType: "${errors_store_type}" + errorStoreType: "${errors_store_type}" errorStoreOptions: ${errors_store_options} monitoring: period: 5000 @@ -41,11 +41,20 @@ postgresql: port: 5432 postgresPassword: password -provisionDataStore: - postgres: true - -statsd: - port: 8125 +postgresql: + provision: true + imageTag: 9.6.11 + service: + port: 5432 + +redis: + name: feast-redis + provision: true + master: + service: + type: LoadBalancer + annotations: + "cloud.google.com/load-balancer-type": "Internal" serving: config: From 8419650ec48eb2eff9ac3a013b835c13c057117e Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 11 Feb 2019 21:10:12 +0800 Subject: [PATCH 13/47] Use kube redis instead --- testing/integration/tests/tf/main.tf | 14 +++----------- testing/integration/tf/modules/feast-helm/input.tf | 4 ++++ testing/integration/tf/modules/feast-helm/main.tf | 5 +++++ .../integration/tf/modules/feast-helm/output.tf | 4 ++++ 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/testing/integration/tests/tf/main.tf b/testing/integration/tests/tf/main.tf index 740dd2393f6..08257f887a2 100644 --- a/testing/integration/tests/tf/main.tf +++ b/testing/integration/tests/tf/main.tf @@ -51,6 +51,7 @@ module "feast" { docker_tag = "${var.docker_tag == "" ? var.revision : var.docker_tag}" core_address = "10.128.0.99" serving_address = "10.128.0.100" + redis_address = "10.128.0.101" load_balancer_source_range = "10.0.0.0/8" job_runner = "DataflowRunner" job_runner_options = "'${jsonencode(local.job_runner_options)}'" @@ -60,15 +61,6 @@ module "feast" { depends_on = ["module.cluster.cluster_name", "null_resource.wait_for_regional_cluster"] } -module "redis" { - source = "../../tf/modules/redis" - source = "../redis" - project_name = "${local.project_name}" - name = "${local.cluster_name}-redis" - zone = "${local.region}-a" - subnet = "${local.subnetwork}" -} - resource "google_bigquery_dataset" "feast_bq_dataset" { dataset_id = "feast_it" description = "Feast integration test dataset" @@ -90,12 +82,12 @@ resource "local_file" "redis_spec" { id: REDIS type: redis options: - host: "${module.redis.instance_url}" + host: "${module.feast.redis_url}" port: "6379" EOT filename = "${path.module}/redis.yaml" - depends_on = ["module.redis"] + depends_on = ["module.feast"] } resource "local_file" "bigquery_spec" { diff --git a/testing/integration/tf/modules/feast-helm/input.tf b/testing/integration/tf/modules/feast-helm/input.tf index 359515c2b1f..8eb1cf7a501 100644 --- a/testing/integration/tf/modules/feast-helm/input.tf +++ b/testing/integration/tf/modules/feast-helm/input.tf @@ -28,6 +28,10 @@ variable "serving_address" { description = "Serving internal address" } +variable "redis_address" { + description = "Redis internal address" +} + variable "statsd_host" { description = "Statsd host to write metrics to" default = "" diff --git a/testing/integration/tf/modules/feast-helm/main.tf b/testing/integration/tf/modules/feast-helm/main.tf index 251d9c59b47..a16b7e6e235 100644 --- a/testing/integration/tf/modules/feast-helm/main.tf +++ b/testing/integration/tf/modules/feast-helm/main.tf @@ -73,6 +73,11 @@ resource "helm_release" "feast" { value = "${google_compute_address.core_address.address}" } + set { + name = "redis.master.service.loadBalancerIP" + value = "${var.redis_address}" + } + set { name = "core.service.loadBalancerSourceRanges[0]" value = "${var.load_balancer_source_range}" diff --git a/testing/integration/tf/modules/feast-helm/output.tf b/testing/integration/tf/modules/feast-helm/output.tf index bead6c7a323..2d0e8eef07e 100644 --- a/testing/integration/tf/modules/feast-helm/output.tf +++ b/testing/integration/tf/modules/feast-helm/output.tf @@ -5,3 +5,7 @@ output "core_url" { output "serving_url" { value = "${google_compute_address.serving_address.address}" } + +output "redis_url" { + value = "${var.redis_address}" +} \ No newline at end of file From 7d969e4afc0d497d09b911b8f5c7161b87a1acc0 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 11 Feb 2019 22:08:13 +0800 Subject: [PATCH 14/47] Fix empty bq script --- testing/integration/tests/tf/scripts/empty-bq.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testing/integration/tests/tf/scripts/empty-bq.sh b/testing/integration/tests/tf/scripts/empty-bq.sh index 9526cdbaea3..79bc03d41e7 100755 --- a/testing/integration/tests/tf/scripts/empty-bq.sh +++ b/testing/integration/tests/tf/scripts/empty-bq.sh @@ -4,5 +4,7 @@ PROJECT_ID=$1 DATASET_NAME=$2 for i in $(bq ls $PROJECT_ID:$DATASET_NAME | sed 1,2d | awk "{print \$1}"); do - bq rm -ft $PROJECT_ID:$DATASET_NAME.$i; + if [[ ! -z "$i" ]]; then + bq rm -ft $PROJECT_ID:$DATASET_NAME.$i; + fi done \ No newline at end of file From a107566fbb720390c74e0efd73bc71703ecc90aa Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 11 Feb 2019 23:35:04 +0800 Subject: [PATCH 15/47] Point back to gojek repo --- testing/integration/tests/tf/scripts/wait-for-cluster.sh | 3 ++- testing/integration/tf/modules/feast-helm/main.tf | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/integration/tests/tf/scripts/wait-for-cluster.sh b/testing/integration/tests/tf/scripts/wait-for-cluster.sh index 60650b1ac63..aba1dc2450c 100755 --- a/testing/integration/tests/tf/scripts/wait-for-cluster.sh +++ b/testing/integration/tests/tf/scripts/wait-for-cluster.sh @@ -1,4 +1,5 @@ #!/bin/bash + # Copyright 2018 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,7 +27,7 @@ echo "Waiting for cluster $2 in project $1 to reconcile..." current_status=$($gcloud_command | jq -r "$jq_query") -while [[ "${current_status}" == "RECONCILING" ]]; do +while [ "${current_status}" = "RECONCILING" ]; do printf "." sleep 5 current_status=$($gcloud_command | jq -r "$jq_query") diff --git a/testing/integration/tf/modules/feast-helm/main.tf b/testing/integration/tf/modules/feast-helm/main.tf index a16b7e6e235..4f967741001 100644 --- a/testing/integration/tf/modules/feast-helm/main.tf +++ b/testing/integration/tf/modules/feast-helm/main.tf @@ -64,7 +64,7 @@ resource "local_file" "helm_values_output" { resource "helm_release" "feast" { name = "feast-it" - repository = "https://raw.githubusercontent.com/zhilingc/feast/master/charts/dist/" + repository = "https://raw.githubusercontent.com/gojek/feast/master/charts/dist/" chart = "feast" version = "0.1.0" From 627191de4ca8c7690c096af77e0cde57dfb2f46a Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 11 Feb 2019 23:59:00 +0800 Subject: [PATCH 16/47] Rearrange columns --- testing/integration/tests/test_feast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/tests/test_feast.py b/testing/integration/tests/test_feast.py index 897b1311fdf..9b9dfd52cc8 100644 --- a/testing/integration/tests/test_feast.py +++ b/testing/integration/tests/test_feast.py @@ -88,7 +88,7 @@ def test_end_to_end(self, client): got = got.drop("created_timestamp", axis=1) \ .sort_values(["id", "event_timestamp"]) \ .reset_index(drop=True) - assert pd.testing.assert_frame_equal(got, wanted, check_less_precise=True) is None + assert pd.testing.assert_frame_equal(got, wanted[got.columns], check_less_precise=True) is None # Check data in redis feature_set = FeatureSet(entity="myentity", From 292694f6d1656c0a62e3e9734e04ef1ab6837633 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Tue, 12 Feb 2019 00:14:58 +0800 Subject: [PATCH 17/47] Remove double brackets --- testing/integration/tests/tf/scripts/empty-bq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/tests/tf/scripts/empty-bq.sh b/testing/integration/tests/tf/scripts/empty-bq.sh index 79bc03d41e7..5abeb004abd 100755 --- a/testing/integration/tests/tf/scripts/empty-bq.sh +++ b/testing/integration/tests/tf/scripts/empty-bq.sh @@ -4,7 +4,7 @@ PROJECT_ID=$1 DATASET_NAME=$2 for i in $(bq ls $PROJECT_ID:$DATASET_NAME | sed 1,2d | awk "{print \$1}"); do - if [[ ! -z "$i" ]]; then + if [ ! -z "$i" ]; then bq rm -ft $PROJECT_ID:$DATASET_NAME.$i; fi done \ No newline at end of file From d0891aaac6981b57f16fb7316595e874f400ef9b Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 15 Feb 2019 10:00:06 +0800 Subject: [PATCH 18/47] Light refactor, added readme --- testing/integration/README.md | 29 ++++++++++ .../tests/redis-bq-dataflow/README.md | 9 +++ .../data/entity/myentity.yaml | 0 .../feature/feature_minutedoubleredis1.yaml | 0 .../feature/feature_minutefloatredis1.yaml | 0 .../feature/feature_minuteint32redis1.yaml | 0 .../feature/feature_minuteint64redis1.yaml | 0 .../data/import/import_csv.yaml | 0 .../data/test_data.csv | 0 .../{ => redis-bq-dataflow}/test_feast.py | 0 .../{ => redis-bq-dataflow}/tf/backend.tf | 0 .../tests/{ => redis-bq-dataflow}/tf/data.tf | 0 .../tests/{ => redis-bq-dataflow}/tf/input.tf | 0 .../tests/{ => redis-bq-dataflow}/tf/main.tf | 4 +- .../{ => redis-bq-dataflow}/tf/provider.tf | 0 .../tf/scripts/empty-bq.sh | 0 .../tf/scripts/wait-for-cluster.sh | 0 .../{ => redis-bq-dataflow}/workflow.yaml | 2 +- .../tf/modules/cluster/.output.tf.swp | Bin 0 -> 12288 bytes .../integration/tf/modules/feast-helm/main.tf | 24 -------- testing/integration/tf/modules/redis/init.sh | 11 ---- testing/integration/tf/modules/redis/input.tf | 52 ------------------ testing/integration/tf/modules/redis/main.tf | 42 -------------- .../integration/tf/modules/redis/output.tf | 3 - .../tf/modules/redis/wait-for-startup.sh | 13 ----- 25 files changed, 41 insertions(+), 148 deletions(-) create mode 100644 testing/integration/README.md create mode 100644 testing/integration/tests/redis-bq-dataflow/README.md rename testing/integration/tests/{ => redis-bq-dataflow}/data/entity/myentity.yaml (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/data/feature/feature_minutedoubleredis1.yaml (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/data/feature/feature_minutefloatredis1.yaml (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/data/feature/feature_minuteint32redis1.yaml (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/data/feature/feature_minuteint64redis1.yaml (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/data/import/import_csv.yaml (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/data/test_data.csv (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/test_feast.py (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/tf/backend.tf (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/tf/data.tf (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/tf/input.tf (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/tf/main.tf (97%) rename testing/integration/tests/{ => redis-bq-dataflow}/tf/provider.tf (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/tf/scripts/empty-bq.sh (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/tf/scripts/wait-for-cluster.sh (100%) rename testing/integration/tests/{ => redis-bq-dataflow}/workflow.yaml (98%) create mode 100644 testing/integration/tf/modules/cluster/.output.tf.swp delete mode 100755 testing/integration/tf/modules/redis/init.sh delete mode 100644 testing/integration/tf/modules/redis/input.tf delete mode 100644 testing/integration/tf/modules/redis/main.tf delete mode 100644 testing/integration/tf/modules/redis/output.tf delete mode 100755 testing/integration/tf/modules/redis/wait-for-startup.sh diff --git a/testing/integration/README.md b/testing/integration/README.md new file mode 100644 index 00000000000..fde86a35335 --- /dev/null +++ b/testing/integration/README.md @@ -0,0 +1,29 @@ +# Integration tests + +Integration tests for Feast are run on argo workflows. The tests follow the following steps: + +1. Provision infrastructure using Terraform +2. Test using pytest +3. Teardown infrastructure using Terraform + +## Terraform modules + +The `tf/modules` directory contains Terraform modules to set up the necessary infrastructure. Currently contains: + +- `cluster`: kubernetes cluster with necessary permissions to run Feast jobs +- `feast-helm`: Feast helm installation. + +## Tests + +The `tests` directory contains the integration tests. Each folder should contain the following: + +1. The terraform scripts to set up the necessary infra. +2. Data to run the tests on +3. A pytest file that executes the ingestion jobs and then tests for correctness. +4. Argo workflow yaml to orchestrate the entire process. + +Multiple tests can be run on the same infrastructure. + +### Adding new tests + +To add your own tests, either (1) create a new test case within an existing folder or (2) create a new folder with the resources mentioned above. \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/README.md b/testing/integration/tests/redis-bq-dataflow/README.md new file mode 100644 index 00000000000..248cc4e1d88 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/README.md @@ -0,0 +1,9 @@ +# redis-bq-dataflow + +This repository contains an end-to-end test that sets up the following infrastructure: + +1. Feast configured to run jobs on dataflow +2. Redis as serving store +3. BQ as warehouse store + +And then runs the tests in `test_feast.py`. \ No newline at end of file diff --git a/testing/integration/tests/data/entity/myentity.yaml b/testing/integration/tests/redis-bq-dataflow/data/entity/myentity.yaml similarity index 100% rename from testing/integration/tests/data/entity/myentity.yaml rename to testing/integration/tests/redis-bq-dataflow/data/entity/myentity.yaml diff --git a/testing/integration/tests/data/feature/feature_minutedoubleredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml similarity index 100% rename from testing/integration/tests/data/feature/feature_minutedoubleredis1.yaml rename to testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml diff --git a/testing/integration/tests/data/feature/feature_minutefloatredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml similarity index 100% rename from testing/integration/tests/data/feature/feature_minutefloatredis1.yaml rename to testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml diff --git a/testing/integration/tests/data/feature/feature_minuteint32redis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml similarity index 100% rename from testing/integration/tests/data/feature/feature_minuteint32redis1.yaml rename to testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml diff --git a/testing/integration/tests/data/feature/feature_minuteint64redis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml similarity index 100% rename from testing/integration/tests/data/feature/feature_minuteint64redis1.yaml rename to testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml diff --git a/testing/integration/tests/data/import/import_csv.yaml b/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml similarity index 100% rename from testing/integration/tests/data/import/import_csv.yaml rename to testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml diff --git a/testing/integration/tests/data/test_data.csv b/testing/integration/tests/redis-bq-dataflow/data/test_data.csv similarity index 100% rename from testing/integration/tests/data/test_data.csv rename to testing/integration/tests/redis-bq-dataflow/data/test_data.csv diff --git a/testing/integration/tests/test_feast.py b/testing/integration/tests/redis-bq-dataflow/test_feast.py similarity index 100% rename from testing/integration/tests/test_feast.py rename to testing/integration/tests/redis-bq-dataflow/test_feast.py diff --git a/testing/integration/tests/tf/backend.tf b/testing/integration/tests/redis-bq-dataflow/tf/backend.tf similarity index 100% rename from testing/integration/tests/tf/backend.tf rename to testing/integration/tests/redis-bq-dataflow/tf/backend.tf diff --git a/testing/integration/tests/tf/data.tf b/testing/integration/tests/redis-bq-dataflow/tf/data.tf similarity index 100% rename from testing/integration/tests/tf/data.tf rename to testing/integration/tests/redis-bq-dataflow/tf/data.tf diff --git a/testing/integration/tests/tf/input.tf b/testing/integration/tests/redis-bq-dataflow/tf/input.tf similarity index 100% rename from testing/integration/tests/tf/input.tf rename to testing/integration/tests/redis-bq-dataflow/tf/input.tf diff --git a/testing/integration/tests/tf/main.tf b/testing/integration/tests/redis-bq-dataflow/tf/main.tf similarity index 97% rename from testing/integration/tests/tf/main.tf rename to testing/integration/tests/redis-bq-dataflow/tf/main.tf index 08257f887a2..0b7f2fafcdb 100644 --- a/testing/integration/tests/tf/main.tf +++ b/testing/integration/tests/redis-bq-dataflow/tf/main.tf @@ -20,7 +20,7 @@ locals { } module "cluster" { - source = "../../tf/modules/cluster" + source = "../../../tf/modules/cluster" cluster_name = "${local.cluster_name}" project_name = "${local.project_name}" region = "${local.region}" @@ -43,7 +43,7 @@ resource "null_resource" "wait_for_regional_cluster" { } module "feast" { - source = "../../tf/modules/feast-helm" + source = "../../../tf/modules/feast-helm" project_name = "${local.project_name}" region = "${local.region}" subnetwork = "${local.subnetwork}" diff --git a/testing/integration/tests/tf/provider.tf b/testing/integration/tests/redis-bq-dataflow/tf/provider.tf similarity index 100% rename from testing/integration/tests/tf/provider.tf rename to testing/integration/tests/redis-bq-dataflow/tf/provider.tf diff --git a/testing/integration/tests/tf/scripts/empty-bq.sh b/testing/integration/tests/redis-bq-dataflow/tf/scripts/empty-bq.sh similarity index 100% rename from testing/integration/tests/tf/scripts/empty-bq.sh rename to testing/integration/tests/redis-bq-dataflow/tf/scripts/empty-bq.sh diff --git a/testing/integration/tests/tf/scripts/wait-for-cluster.sh b/testing/integration/tests/redis-bq-dataflow/tf/scripts/wait-for-cluster.sh similarity index 100% rename from testing/integration/tests/tf/scripts/wait-for-cluster.sh rename to testing/integration/tests/redis-bq-dataflow/tf/scripts/wait-for-cluster.sh diff --git a/testing/integration/tests/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml similarity index 98% rename from testing/integration/tests/workflow.yaml rename to testing/integration/tests/redis-bq-dataflow/workflow.yaml index 03d0e903b6a..34f8a156c0a 100644 --- a/testing/integration/tests/workflow.yaml +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -25,7 +25,7 @@ spec: arguments: parameters: - name: arg - value: "apply -var revision={{workflow.parameters.revision}} -var docker_tag=coalesce -auto-approve ." + value: "apply -var revision={{workflow.parameters.revision}} -var docker_tag={{workflow.parameters.revision}} -auto-approve ." artifacts: - name: feast from: "{{steps.get-feast.outputs.artifacts.feast}}" diff --git a/testing/integration/tf/modules/cluster/.output.tf.swp b/testing/integration/tf/modules/cluster/.output.tf.swp new file mode 100644 index 0000000000000000000000000000000000000000..d6cd0e27e28b96b6b25d69ef80d9915e3b7b26b6 GIT binary patch literal 12288 zcmeI2!EVz)5I`r~kSHKHgIv=q-T;?gDlS|&a6I-@onk zcYE+^oMvf}sM*v-S1Fa85oJcnZ28L9^@ie+Q9joNI+2*Wr8WCn(Q9XwNKzd8L8MGh zQ~P=%Mp`-)s2$Q#L|ULuti;q50!Jb@L!W6UROTIOD~!j|V}zsShu|4Du(Af$;KBWS z^GE$oxE|cR@oZ&HaN%r#4X^<=zy{a=8(;%$U=<~ z18jf|umLu}2G{@_U;}J`4X}a#(108Q>|6o(Ma=&HuYdplxdt#Leh}xx8S$BTM<8K| z7sOKni0M^;-^6#~Bk_SaB@T&zxJ}$5E)(C@0KO7mh)=|GVvpD*wuotUo}aTb@x=z% z02^QfY=8~00XDz}*Z>>oz`!vC!GX-Y4z_}D<0vsE$#m?{SefW};MGK9H19uN3>Gri z$07V{Hx9zKHDPcB(4Ik=0%clg8&`eHRg%f3i*o9ArU~hB9QC78I(KO7u#u|{b+W&A z+Ah>mhdSB6sxMYDR@!1ZN|nT>x%LGe Date: Sun, 17 Feb 2019 13:06:13 +0800 Subject: [PATCH 19/47] Add makefile for integration tests --- Makefile | 24 ++++++++++++++++-- testing/README.md | 6 ++++- testing/integration/Makefile | 25 +++++++++++++++++++ .../tests/redis-bq-dataflow/workflow.yaml | 8 +++--- 4 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 testing/integration/Makefile diff --git a/Makefile b/Makefile index 888cfea4264..4231fb77252 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,29 @@ -VERSION_FILE=VERSION -FEAST_VERSION=`cat $(VERSION_FILE)` +# +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +VERSION_FILE = VERSION +FEAST_VERSION = `cat $(VERSION_FILE)` +REVISION ?= master test: mvn test +test-integration: + $(MAKE) -C testing/integration test-integration REVISION=$(REVISION) + build-deps: $(MAKE) -C protos gen-go dep ensure diff --git a/testing/README.md b/testing/README.md index 56abb7d7b51..df38d34a676 100644 --- a/testing/README.md +++ b/testing/README.md @@ -54,4 +54,8 @@ To update Prow jobs, plugins, or the Docker image used for testing, modify one o - `docker-images/test-image/Dockerfile` - `docker-images/test-image/run.sh` -After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry. \ No newline at end of file +After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry. + +## Installing Argo + +Argo is used to run integration tests. It follows the [standard installation process described in the argo quickstart](https://github.com/argoproj/argo/blob/master/demo.md), with the artifact repository [configured to use gcs](https://github.com/argoproj/argo/blob/master/ARTIFACT_REPO.md). \ No newline at end of file diff --git a/testing/integration/Makefile b/testing/integration/Makefile new file mode 100644 index 00000000000..961295694d3 --- /dev/null +++ b/testing/integration/Makefile @@ -0,0 +1,25 @@ +# +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +.PHONY: test-integration it-redis-bq-dataflow + +REVISION ?= master + +test-integration: + @$(MAKE) it-redis-bq-dataflow REVISION=$(REVISION) + +it-redis-bq-dataflow: + argo submit --watch tests/redis-bq-dataflow/workflow.yaml -p revision=$(REVISION) \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml index 34f8a156c0a..1175db21c35 100644 --- a/testing/integration/tests/redis-bq-dataflow/workflow.yaml +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -55,7 +55,7 @@ spec: - name: get-feast container: image: alpine/git - args: ["clone", "--single-branch", "--branch", "{{workflow.parameters.revision}}", "https://github.com/zhilingc/feast"] + args: ["clone", "--single-branch", "--branch", "{{workflow.parameters.revision}}", "https://github.com/gojek/feast"] outputs: artifacts: - name: feast @@ -89,7 +89,7 @@ spec: path: /bin/feast container: image: gcr.io/kf-feast/it-terraform:0.1.0 - workingDir: /feast/testing/integration/tests/tf + workingDir: /feast/testing/integration/tests/redis-bq-dataflow/tf command: ["sh", "-c"] args: [' terraform init; @@ -112,7 +112,7 @@ spec: pip install -e /feast/sdk/python; pytest -s '] - workingDir: /feast/testing/integration/tests + workingDir: /feast/testing/integration/tests/redis-bq-dataflow env: - name: FEAST_CORE_URL value: "10.128.0.99:6565" @@ -131,7 +131,7 @@ spec: path: /feast container: image: gcr.io/kf-feast/it-terraform:0.1.0 - workingDir: /feast/testing/integration/tests/tf + workingDir: /feast/testing/integration/tests/redis-bq-dataflow/tf command: ["sh", "-c"] args: [' terraform init; From 8d3bbe8e181d13d92e6ae3a5780ac32bc45c377c Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 18 Feb 2019 20:52:17 +0800 Subject: [PATCH 20/47] [WIP] use PR code --- Makefile | 3 +-- testing/integration/Makefile | 6 ++---- testing/integration/tests/redis-bq-dataflow/workflow.yaml | 6 +++++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 4231fb77252..6ddd22dce17 100644 --- a/Makefile +++ b/Makefile @@ -16,13 +16,12 @@ VERSION_FILE = VERSION FEAST_VERSION = `cat $(VERSION_FILE)` -REVISION ?= master test: mvn test test-integration: - $(MAKE) -C testing/integration test-integration REVISION=$(REVISION) + $(MAKE) -C testing/integration test-integration ID=$(ID) build-deps: $(MAKE) -C protos gen-go diff --git a/testing/integration/Makefile b/testing/integration/Makefile index 961295694d3..a53b621622d 100644 --- a/testing/integration/Makefile +++ b/testing/integration/Makefile @@ -16,10 +16,8 @@ .PHONY: test-integration it-redis-bq-dataflow -REVISION ?= master - test-integration: - @$(MAKE) it-redis-bq-dataflow REVISION=$(REVISION) + @$(MAKE) it-redis-bq-dataflow ID=$(ID) it-redis-bq-dataflow: - argo submit --watch tests/redis-bq-dataflow/workflow.yaml -p revision=$(REVISION) \ No newline at end of file + argo submit --watch tests/redis-bq-dataflow/workflow.yaml -p id=$(ID) \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml index 1175db21c35..e84339fb5e1 100644 --- a/testing/integration/tests/redis-bq-dataflow/workflow.yaml +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -55,7 +55,11 @@ spec: - name: get-feast container: image: alpine/git - args: ["clone", "--single-branch", "--branch", "{{workflow.parameters.revision}}", "https://github.com/gojek/feast"] + args: [" + clone https://github.com/gojek/feast; + git fetch origin pull/{{workflow.parameters.prId}}/head; + git checkout FETCH_HEAD + "] outputs: artifacts: - name: feast From 32678ea448f0e94b2bf308cedf86ca4866b3dc6e Mon Sep 17 00:00:00 2001 From: zhilingc Date: Tue, 26 Feb 2019 20:37:14 +0800 Subject: [PATCH 21/47] Add docker build stage --- .../tests/redis-bq-dataflow/workflow.yaml | 32 +++++++++++++++++-- .../tf/modules/feast-helm/input.tf | 5 --- .../integration/tf/modules/feast-helm/main.tf | 3 +- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml index e84339fb5e1..0bbd5b42a49 100644 --- a/testing/integration/tests/redis-bq-dataflow/workflow.yaml +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -14,6 +14,12 @@ spec: steps: - - name: get-feast template: get-feast + - - name: build-push-docker + template: build + arguments: + artifacts: + - name: feast + from: "{{steps.get-feast.outputs.artifacts.feast}}" - - name: build-cli template: build-cli arguments: @@ -25,7 +31,7 @@ spec: arguments: parameters: - name: arg - value: "apply -var revision={{workflow.parameters.revision}} -var docker_tag={{workflow.parameters.revision}} -auto-approve ." + value: "apply -var docker_tag=integration-test -auto-approve ." artifacts: - name: feast from: "{{steps.get-feast.outputs.artifacts.feast}}" @@ -39,7 +45,7 @@ spec: from: "{{steps.get-feast.outputs.artifacts.feast}}" - name: feast-cli from: "{{steps.build-cli.outputs.artifacts.feast-cli}}" - + - name: exit-handler steps: - - name: terraform-destroy @@ -65,6 +71,28 @@ spec: - name: feast path: /git/feast globalName: feast-repo + - name: build + inputs: + artifacts: + - name: feast + path: /feast + container: + image: docker:stable + command: [sh, -c] + args: [' + apk add --update make; + make build-push-docker registry=gcr.io/kf-feast version=integration-test + '] + env: + - name: DOCKER_HOST + value: 127.0.0.1 + workingDir: /feast + sidecars: + - name: dind + image: docker:stable-dind + securityContext: + privileged: true + mirrorVolumeMounts: true - name: build-cli inputs: artifacts: diff --git a/testing/integration/tf/modules/feast-helm/input.tf b/testing/integration/tf/modules/feast-helm/input.tf index 8eb1cf7a501..3bb7a91da3d 100644 --- a/testing/integration/tf/modules/feast-helm/input.tf +++ b/testing/integration/tf/modules/feast-helm/input.tf @@ -15,11 +15,6 @@ variable "docker_tag" { description = "Feast build version" } -variable "revision" { - default = "master" - description = "Github revision to pull charts from" -} - variable "core_address" { description = "Core internal address" } diff --git a/testing/integration/tf/modules/feast-helm/main.tf b/testing/integration/tf/modules/feast-helm/main.tf index 12fefe332b9..baf219fdf9f 100644 --- a/testing/integration/tf/modules/feast-helm/main.tf +++ b/testing/integration/tf/modules/feast-helm/main.tf @@ -40,8 +40,7 @@ resource "local_file" "helm_values_output" { resource "helm_release" "feast" { name = "feast-it" - repository = "https://raw.githubusercontent.com/gojek/feast/master/charts/dist/" - chart = "feast" + chart = "${path.module}/../../../../../charts/feast" version = "0.1.0" set { From 009a61a9e67603d583ebb87a6ae634f8f3142508 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 28 Feb 2019 19:46:25 +0800 Subject: [PATCH 22/47] Override git image command --- testing/integration/Makefile | 2 +- testing/integration/tests/redis-bq-dataflow/workflow.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/integration/Makefile b/testing/integration/Makefile index a53b621622d..cceb66d20b1 100644 --- a/testing/integration/Makefile +++ b/testing/integration/Makefile @@ -20,4 +20,4 @@ test-integration: @$(MAKE) it-redis-bq-dataflow ID=$(ID) it-redis-bq-dataflow: - argo submit --watch tests/redis-bq-dataflow/workflow.yaml -p id=$(ID) \ No newline at end of file + argo submit --watch tests/redis-bq-dataflow/workflow.yaml -p prId=$(ID) \ No newline at end of file diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml index 0bbd5b42a49..ec5dd015fe1 100644 --- a/testing/integration/tests/redis-bq-dataflow/workflow.yaml +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -61,8 +61,9 @@ spec: - name: get-feast container: image: alpine/git + command: [sh, -c] args: [" - clone https://github.com/gojek/feast; + git clone https://github.com/gojek/feast; git fetch origin pull/{{workflow.parameters.prId}}/head; git checkout FETCH_HEAD "] From 04bb2d0419807679876af2736bdb3ca6c317390d Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 28 Feb 2019 23:59:40 +0800 Subject: [PATCH 23/47] Add docker builder image, remove revision from feast module --- .../docker-builder-image/Dockerfile | 25 ++++++++++ .../tests/redis-bq-dataflow/tf/main.tf | 1 - .../tests/redis-bq-dataflow/workflow.yaml | 49 +++++++++++++++---- 3 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 testing/docker-images/docker-builder-image/Dockerfile diff --git a/testing/docker-images/docker-builder-image/Dockerfile b/testing/docker-images/docker-builder-image/Dockerfile new file mode 100644 index 00000000000..45f2ce67b15 --- /dev/null +++ b/testing/docker-images/docker-builder-image/Dockerfile @@ -0,0 +1,25 @@ +FROM docker:18.09 + +# from https://github.com/GoogleCloudPlatform/cloud-sdk-docker/blob/master/alpine/Dockerfile +ENV CLOUD_SDK_VERSION=236.0.0 + +ENV PATH /google-cloud-sdk/bin:$PATH +RUN apk --no-cache add \ + curl \ + python \ + py-crcmod \ + bash \ + libc6-compat \ + openssh-client \ + git \ + gnupg \ + && curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + ln -s /lib /lib64 && \ + gcloud config set core/disable_usage_reporting true && \ + gcloud config set component_manager/disable_update_check true && \ + gcloud config set metrics/environment github_docker_image && \ + gcloud --version + +RUN gcloud auth configure-docker diff --git a/testing/integration/tests/redis-bq-dataflow/tf/main.tf b/testing/integration/tests/redis-bq-dataflow/tf/main.tf index 0b7f2fafcdb..930dcb65a06 100644 --- a/testing/integration/tests/redis-bq-dataflow/tf/main.tf +++ b/testing/integration/tests/redis-bq-dataflow/tf/main.tf @@ -47,7 +47,6 @@ module "feast" { project_name = "${local.project_name}" region = "${local.region}" subnetwork = "${local.subnetwork}" - revision = "master" docker_tag = "${var.docker_tag == "" ? var.revision : var.docker_tag}" core_address = "10.128.0.99" serving_address = "10.128.0.100" diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml index ec5dd015fe1..8ae87523158 100644 --- a/testing/integration/tests/redis-bq-dataflow/workflow.yaml +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -14,12 +14,18 @@ spec: steps: - - name: get-feast template: get-feast - - - name: build-push-docker - template: build + - - name: build-jars + template: build-jars arguments: artifacts: - name: feast from: "{{steps.get-feast.outputs.artifacts.feast}}" + - - name: build-push-docker + template: build-docker + arguments: + artifacts: + - name: feast + from: "{{steps.build-jars.outputs.artifacts.feast-build}}" - - name: build-cli template: build-cli arguments: @@ -53,7 +59,7 @@ spec: arguments: parameters: - name: arg - value: "destroy -var revision={{workflow.parameters.revision}} -var docker_tag=coalesce -auto-approve ." + value: "destroy -var docker_tag=integration-test -auto-approve ." artifacts: - name: feast from: "{{workflow.outputs.artifacts.feast-repo}}" @@ -61,28 +67,51 @@ spec: - name: get-feast container: image: alpine/git - command: [sh, -c] - args: [" + command: + - /bin/sh + - -c + - | git clone https://github.com/gojek/feast; + cd feast; git fetch origin pull/{{workflow.parameters.prId}}/head; git checkout FETCH_HEAD - "] outputs: artifacts: - name: feast path: /git/feast globalName: feast-repo - - name: build + - name: build-jars + inputs: + artifacts: + - name: feast + path: /feast + container: + image: us.gcr.io/kf-feast/test-image:1.0.0 + command: + - /bin/bash + - -c + - | + mvn package -DskipTests -Drevision=integration-test + workingDir: /feast + outputs: + artifacts: + - name: feast-build + path: /feast + globalName: feast-build + - name: build-docker inputs: artifacts: - name: feast path: /feast container: - image: docker:stable + image: gcr.io/kf-feast/docker:18.09-gcloud command: [sh, -c] args: [' - apk add --update make; - make build-push-docker registry=gcr.io/kf-feast version=integration-test + until docker ps; do sleep 3; done; + docker build -t gcr.io/kf-feast/feast-core:integration-test -f docker/core/Dockerfile --build-arg RELEASE=integration-test .; + docker build -t gcr.io/kf-feast/feast-serving:integration-test -f docker/serving/Dockerfile --build-arg RELEASE=integration-test .; + docker push gcr.io/kf-feast/feast-core:integration-test; + docker push gcr.io/kf-feast/feast-serving:integration-test '] env: - name: DOCKER_HOST From 108db835a653b74b509235fe9f60dac3bbbbb73e Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 1 Mar 2019 00:14:53 +0800 Subject: [PATCH 24/47] Remove revision --- testing/integration/tests/redis-bq-dataflow/tf/input.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/tf/input.tf b/testing/integration/tests/redis-bq-dataflow/tf/input.tf index 6bd1b34bb09..0d9475c30d1 100644 --- a/testing/integration/tests/redis-bq-dataflow/tf/input.tf +++ b/testing/integration/tests/redis-bq-dataflow/tf/input.tf @@ -1,7 +1,3 @@ -variable "revision" { - description = "Feast revision to pull helm charts from" -} - variable "docker_tag" { description = "Docker image to deploy" default = "" From a7d6efd61cc2c4384501d784abda6b65dfee0c2a Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 1 Mar 2019 00:27:59 +0800 Subject: [PATCH 25/47] Removed another reference to revision --- testing/integration/tests/redis-bq-dataflow/tf/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/tests/redis-bq-dataflow/tf/main.tf b/testing/integration/tests/redis-bq-dataflow/tf/main.tf index 930dcb65a06..351c479f3b4 100644 --- a/testing/integration/tests/redis-bq-dataflow/tf/main.tf +++ b/testing/integration/tests/redis-bq-dataflow/tf/main.tf @@ -47,7 +47,7 @@ module "feast" { project_name = "${local.project_name}" region = "${local.region}" subnetwork = "${local.subnetwork}" - docker_tag = "${var.docker_tag == "" ? var.revision : var.docker_tag}" + docker_tag = "${var.docker_tag}" core_address = "10.128.0.99" serving_address = "10.128.0.100" redis_address = "10.128.0.101" From e1442e284bacc849e448b28282fde44ca9569034 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 1 Mar 2019 00:42:04 +0800 Subject: [PATCH 26/47] Hardcode chart location --- testing/integration/tf/modules/feast-helm/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/tf/modules/feast-helm/main.tf b/testing/integration/tf/modules/feast-helm/main.tf index baf219fdf9f..f634686fe27 100644 --- a/testing/integration/tf/modules/feast-helm/main.tf +++ b/testing/integration/tf/modules/feast-helm/main.tf @@ -40,7 +40,7 @@ resource "local_file" "helm_values_output" { resource "helm_release" "feast" { name = "feast-it" - chart = "${path.module}/../../../../../charts/feast" + chart = "/feast/charts/feast" version = "0.1.0" set { From dfa1a38b27072217ccf6c51ca085a2d5fee28664 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 28 Mar 2019 14:36:48 +0800 Subject: [PATCH 27/47] Change bq to delete on destroy --- .../data/feature/feature_minutedoubleredis1.yaml | 5 ++--- .../data/feature/feature_minutefloatredis1.yaml | 5 ++--- .../data/feature/feature_minuteint32redis1.yaml | 5 ++--- .../data/feature/feature_minuteint64redis1.yaml | 5 ++--- .../redis-bq-dataflow/data/import/import_csv.yaml | 12 ++++++------ .../integration/tests/redis-bq-dataflow/tf/main.tf | 10 +--------- .../tests/redis-bq-dataflow/tf/provider.tf | 2 +- .../tests/redis-bq-dataflow/workflow.yaml | 12 ++++++------ 8 files changed, 22 insertions(+), 34 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml index 4d2b7573ce4..fe0940027ce 100644 --- a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml @@ -1,13 +1,12 @@ -id: myentity.minute.feature_double_redis +id: myentity.feature_double_redis name: feature_double_redis entity: myentity owner: bob@example.com description: test entity. valueType: DOUBLE -granularity: MINUTE uri: https://github.com/bob/example dataStores: serving: id: REDIS warehouse: - id: BIGQUERY \ No newline at end of file + id: BIGQUERY diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml index d9707d31e2a..d656c495063 100644 --- a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml @@ -1,13 +1,12 @@ -id: myentity.minute.feature_float_redis +id: myentity.feature_float_redis name: feature_float_redis entity: myentity owner: bob@example.com description: test entity. valueType: FLOAT -granularity: MINUTE uri: https://github.com/bob/example dataStores: serving: id: REDIS warehouse: - id: BIGQUERY \ No newline at end of file + id: BIGQUERY diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml index 56e85f68365..f174bde4f61 100644 --- a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml @@ -1,13 +1,12 @@ -id: myentity.minute.feature_int32_redis +id: myentity.feature_int32_redis name: feature_int32_redis entity: myentity owner: bob@example.com description: test entity. valueType: INT32 -granularity: MINUTE uri: https://github.com/bob/example dataStores: serving: id: REDIS warehouse: - id: BIGQUERY \ No newline at end of file + id: BIGQUERY diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml index c2d36affd4b..78e16869751 100644 --- a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml @@ -1,13 +1,12 @@ -id: myentity.minute.feature_int64_redis +id: myentity.feature_int64_redis name: feature_int64_redis entity: myentity owner: bob@example.com description: test entity. valueType: INT64 -granularity: MINUTE uri: https://github.com/bob/example dataStores: serving: id: REDIS warehouse: - id: BIGQUERY \ No newline at end of file + id: BIGQUERY diff --git a/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml b/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml index 3d940c01059..f427f90a427 100644 --- a/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml +++ b/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml @@ -1,6 +1,6 @@ type: file.csv -options: - path: gs://it-feast-storage/test-cases/test_data.csv +sourceOptions: + path: gs://zl-test-bucket/test_data.csv entities: - myentity schema: @@ -10,10 +10,10 @@ schema: - name: entity - name: ts - name: feature1 - featureId: myentity.minute.feature_double_redis + featureId: myentity.feature_double_redis - name: feature2 - featureId: myentity.minute.feature_float_redis + featureId: myentity.feature_float_redis - name: feature3 - featureId: myentity.minute.feature_int32_redis + featureId: myentity.feature_int32_redis - name: feature4 - featureId: myentity.minute.feature_int64_redis \ No newline at end of file + featureId: myentity.feature_int64_redis diff --git a/testing/integration/tests/redis-bq-dataflow/tf/main.tf b/testing/integration/tests/redis-bq-dataflow/tf/main.tf index 351c479f3b4..c68144a8bdc 100644 --- a/testing/integration/tests/redis-bq-dataflow/tf/main.tf +++ b/testing/integration/tests/redis-bq-dataflow/tf/main.tf @@ -65,15 +65,7 @@ resource "google_bigquery_dataset" "feast_bq_dataset" { description = "Feast integration test dataset" default_table_expiration_ms = 36000000 location = "US" -} - -resource "null_resource" "empty_bq" { - provisioner "local-exec" { - when = "destroy" - command = ". ${path.module}/scripts/empty-bq.sh ${local.project_name} feast_it" - } - - depends_on = ["google_bigquery_dataset.feast_bq_dataset"] + delete_contents_on_destroy = true } resource "local_file" "redis_spec" { diff --git a/testing/integration/tests/redis-bq-dataflow/tf/provider.tf b/testing/integration/tests/redis-bq-dataflow/tf/provider.tf index fad0c05ae62..91f4c3caca4 100644 --- a/testing/integration/tests/redis-bq-dataflow/tf/provider.tf +++ b/testing/integration/tests/redis-bq-dataflow/tf/provider.tf @@ -1,5 +1,5 @@ provider "google" { - version = "~> 1.20" + version = "2.1.0" # credentials = "${file("~/.secrets/${local.project_name}.json")}" project = "${local.project_name}" } diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml index 8ae87523158..0f158bfbda2 100644 --- a/testing/integration/tests/redis-bq-dataflow/workflow.yaml +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -20,18 +20,18 @@ spec: artifacts: - name: feast from: "{{steps.get-feast.outputs.artifacts.feast}}" - - - name: build-push-docker - template: build-docker + - name: build-cli + template: build-cli arguments: artifacts: - name: feast - from: "{{steps.build-jars.outputs.artifacts.feast-build}}" - - - name: build-cli - template: build-cli + from: "{{steps.get-feast.outputs.artifacts.feast}}" + - - name: build-push-docker + template: build-docker arguments: artifacts: - name: feast - from: "{{steps.get-feast.outputs.artifacts.feast}}" + from: "{{steps.build-jars.outputs.artifacts.feast-build}}" - - name: terraform-provision template: terraform arguments: From 78fc4815bda7b9cdfd3217d514d4279cf29c199d Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 28 Mar 2019 14:49:09 +0800 Subject: [PATCH 28/47] Update paths of temporary folder created in Feast Core tests --- core/src/test/java/feast/core/CoreApplicationTest.java | 2 +- .../test/java/feast/core/CoreApplicationWithNoServingTest.java | 2 +- .../java/feast/core/CoreApplicationWithNoWarehouseTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/feast/core/CoreApplicationTest.java b/core/src/test/java/feast/core/CoreApplicationTest.java index d80b5b7b42c..ccd30ddfacc 100644 --- a/core/src/test/java/feast/core/CoreApplicationTest.java +++ b/core/src/test/java/feast/core/CoreApplicationTest.java @@ -48,7 +48,7 @@ */ @RunWith(SpringRunner.class) @SpringBootTest(properties = { - "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}", "spring.datasource.url=jdbc:h2:mem:testdb", "feast.store.warehouse.type=file.json", "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}", diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java index 61f55202ad9..6dcef362a2a 100644 --- a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java +++ b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java @@ -49,7 +49,7 @@ */ @RunWith(SpringRunner.class) @SpringBootTest(properties = { - "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}", "spring.datasource.url=jdbc:h2:mem:testdb", "feast.store.warehouse.type=file.json", "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}", diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java index 6c8ee325aac..46b2e151a05 100644 --- a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java +++ b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java @@ -49,7 +49,7 @@ */ @RunWith(SpringRunner.class) @SpringBootTest(properties = { - "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}", + "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}", "spring.datasource.url=jdbc:h2:mem:testdb", "feast.store.serving.type=redis", "feast.store.serving.options={\"host\":\"localhost\",\"port\":1234}", From 017f3921e3908566c911657b2b0d55dc23af5cd6 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 28 Mar 2019 15:22:55 +0800 Subject: [PATCH 29/47] Update options.path in import_csv.yaml Fix to bucket accessible by the project running the integration test --- .../tests/redis-bq-dataflow/data/import/import_csv.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml b/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml index f427f90a427..3eb170a23c4 100644 --- a/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml +++ b/testing/integration/tests/redis-bq-dataflow/data/import/import_csv.yaml @@ -1,6 +1,6 @@ type: file.csv sourceOptions: - path: gs://zl-test-bucket/test_data.csv + path: gs://it-feast-storage/test-cases/test_data.csv entities: - myentity schema: From 67c0596e2a84e7770bacd98e475e21f5657dc112 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 28 Mar 2019 16:31:17 +0800 Subject: [PATCH 30/47] Update terraform modules to support workspace and no granularity --- .../tests/redis-bq-dataflow/tf/input.tf | 1 - .../tests/redis-bq-dataflow/tf/main.tf | 42 +------- .../tests/redis-bq-dataflow/tf/values.yaml | 101 ++++++++++++++++++ .../tf/modules/feast-helm/input.tf | 18 ++-- .../integration/tf/modules/feast-helm/main.tf | 33 ++++-- .../tf/modules/feast-helm/values.tmpl | 16 ++- 6 files changed, 148 insertions(+), 63 deletions(-) create mode 100755 testing/integration/tests/redis-bq-dataflow/tf/values.yaml diff --git a/testing/integration/tests/redis-bq-dataflow/tf/input.tf b/testing/integration/tests/redis-bq-dataflow/tf/input.tf index 0d9475c30d1..bb653c213a5 100644 --- a/testing/integration/tests/redis-bq-dataflow/tf/input.tf +++ b/testing/integration/tests/redis-bq-dataflow/tf/input.tf @@ -1,4 +1,3 @@ variable "docker_tag" { description = "Docker image to deploy" - default = "" } diff --git a/testing/integration/tests/redis-bq-dataflow/tf/main.tf b/testing/integration/tests/redis-bq-dataflow/tf/main.tf index c68144a8bdc..bad1c4f2bd9 100644 --- a/testing/integration/tests/redis-bq-dataflow/tf/main.tf +++ b/testing/integration/tests/redis-bq-dataflow/tf/main.tf @@ -54,8 +54,8 @@ module "feast" { load_balancer_source_range = "10.0.0.0/8" job_runner = "DataflowRunner" job_runner_options = "'${jsonencode(local.job_runner_options)}'" - errors_store_type = "file.json" - errors_store_options = "'${jsonencode(local.errors_store_options)}'" + bucket_name = "${local.cluster_name}-storage" + bq_dataset = "${google_bigquery_dataset.feast_bq_dataset.dataset_id}" depends_on = ["module.cluster.cluster_name", "null_resource.wait_for_regional_cluster"] } @@ -67,41 +67,3 @@ resource "google_bigquery_dataset" "feast_bq_dataset" { location = "US" delete_contents_on_destroy = true } - -resource "local_file" "redis_spec" { - content = < Date: Thu, 28 Mar 2019 17:29:18 +0800 Subject: [PATCH 31/47] Revert chart path reference to /feast instead of /tmp/feast --- testing/integration/tf/modules/feast-helm/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/tf/modules/feast-helm/main.tf b/testing/integration/tf/modules/feast-helm/main.tf index a6431d0af6b..28b0b264a84 100644 --- a/testing/integration/tf/modules/feast-helm/main.tf +++ b/testing/integration/tf/modules/feast-helm/main.tf @@ -56,7 +56,7 @@ resource "local_file" "helm_values_output" { resource "helm_release" "feast" { name = "feast-it" - chart = "/tmp/feast/charts/feast" + chart = "/feast/charts/feast" set { name = "core.service.extIPAdr" From dc3479f960d2092cfce05e7593331611768e6f8a Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 28 Mar 2019 17:54:42 +0800 Subject: [PATCH 32/47] Remove reference to store in order to use default stores --- .../data/feature/feature_minutedoubleredis1.yaml | 5 ----- .../data/feature/feature_minutefloatredis1.yaml | 5 ----- .../data/feature/feature_minuteint32redis1.yaml | 5 ----- .../data/feature/feature_minuteint64redis1.yaml | 5 ----- 4 files changed, 20 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml index fe0940027ce..05b5279d0e4 100644 --- a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml @@ -5,8 +5,3 @@ owner: bob@example.com description: test entity. valueType: DOUBLE uri: https://github.com/bob/example -dataStores: - serving: - id: REDIS - warehouse: - id: BIGQUERY diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml index d656c495063..3ff38358ce3 100644 --- a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml @@ -5,8 +5,3 @@ owner: bob@example.com description: test entity. valueType: FLOAT uri: https://github.com/bob/example -dataStores: - serving: - id: REDIS - warehouse: - id: BIGQUERY diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml index f174bde4f61..ebeb92dc999 100644 --- a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml @@ -5,8 +5,3 @@ owner: bob@example.com description: test entity. valueType: INT32 uri: https://github.com/bob/example -dataStores: - serving: - id: REDIS - warehouse: - id: BIGQUERY diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml index 78e16869751..dd6b7339bf3 100644 --- a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml +++ b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint64redis1.yaml @@ -5,8 +5,3 @@ owner: bob@example.com description: test entity. valueType: INT64 uri: https://github.com/bob/example -dataStores: - serving: - id: REDIS - warehouse: - id: BIGQUERY From 35a61f4b2aa970a4ba754b99fd743eeb688d0734 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Thu, 28 Mar 2019 18:17:18 +0800 Subject: [PATCH 33/47] Update Argo workflow to accept gitRef argument --- .../tests/redis-bq-dataflow/workflow.yaml | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml index 0f158bfbda2..a560c4ba263 100644 --- a/testing/integration/tests/redis-bq-dataflow/workflow.yaml +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -9,6 +9,10 @@ spec: parameters: - name: revision value: integration-test + - name: prId + value: "none" + - name: gitRef + value: "none" templates: - name: run steps: @@ -19,13 +23,13 @@ spec: arguments: artifacts: - name: feast - from: "{{steps.get-feast.outputs.artifacts.feast}}" + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" - name: build-cli template: build-cli arguments: artifacts: - name: feast - from: "{{steps.get-feast.outputs.artifacts.feast}}" + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" - - name: build-push-docker template: build-docker arguments: @@ -40,7 +44,7 @@ spec: value: "apply -var docker_tag=integration-test -auto-approve ." artifacts: - name: feast - from: "{{steps.get-feast.outputs.artifacts.feast}}" + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" - name: feast-cli from: "{{steps.build-cli.outputs.artifacts.feast-cli}}" - - name: test @@ -48,7 +52,7 @@ spec: arguments: artifacts: - name: feast - from: "{{steps.get-feast.outputs.artifacts.feast}}" + from: "{{steps.get-feast.outputs.artifacts.feast-repo}}" - name: feast-cli from: "{{steps.build-cli.outputs.artifacts.feast-cli}}" @@ -63,8 +67,20 @@ spec: artifacts: - name: feast from: "{{workflow.outputs.artifacts.feast-repo}}" - - name: get-feast + steps: + - - name: get-feast-pr + template: get-feast-pr + when: "{{workflow.parameters.prId}} != none" + - name: get-feast-gitref + template: get-feast-gitref + when: "{{workflow.parameters.gitRef}} != none" + outputs: + artifacts: + - name: feast-repo + from: "{{workflow.outputs.artifacts.feast-repo}}" + globalName: feast-repo + - name: get-feast-pr container: image: alpine/git command: @@ -80,6 +96,21 @@ spec: - name: feast path: /git/feast globalName: feast-repo + - name: get-feast-gitref + container: + image: alpine/git + command: + - /bin/sh + - -c + - | + git clone https://github.com/gojek/feast; + cd feast; + git checkout {{workflow.parameters.gitRef}} + outputs: + artifacts: + - name: feast + path: /git/feast + globalName: feast-repo - name: build-jars inputs: artifacts: From 4477f535e5abb9385d48dd317f1d56e497dc4fb5 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 28 Mar 2019 19:14:46 +0800 Subject: [PATCH 34/47] Change logtype to JSON --- testing/integration/tf/modules/feast-helm/values.tmpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testing/integration/tf/modules/feast-helm/values.tmpl b/testing/integration/tf/modules/feast-helm/values.tmpl index 6b1401ec130..a4e00d5b655 100644 --- a/testing/integration/tf/modules/feast-helm/values.tmpl +++ b/testing/integration/tf/modules/feast-helm/values.tmpl @@ -30,6 +30,9 @@ core: monitoring: period: 5000 initialDelay: 60000 + trainingDatasetPrefix: "fs" + projectId: "kf-feast" + logType: JSON dataflow: projectID: "${project_id}" location: "${region}" From a7b9a8a19646839c58e2c710478710d836cdfb57 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 28 Mar 2019 19:42:22 +0800 Subject: [PATCH 35/47] Give ability to create folders in bucket --- .../tf/modules/cluster/.output.tf.swp | Bin 12288 -> 0 bytes testing/integration/tf/modules/cluster/main.tf | 1 + 2 files changed, 1 insertion(+) delete mode 100644 testing/integration/tf/modules/cluster/.output.tf.swp diff --git a/testing/integration/tf/modules/cluster/.output.tf.swp b/testing/integration/tf/modules/cluster/.output.tf.swp deleted file mode 100644 index d6cd0e27e28b96b6b25d69ef80d9915e3b7b26b6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2!EVz)5I`r~kSHKHgIv=q-T;?gDlS|&a6I-@onk zcYE+^oMvf}sM*v-S1Fa85oJcnZ28L9^@ie+Q9joNI+2*Wr8WCn(Q9XwNKzd8L8MGh zQ~P=%Mp`-)s2$Q#L|ULuti;q50!Jb@L!W6UROTIOD~!j|V}zsShu|4Du(Af$;KBWS z^GE$oxE|cR@oZ&HaN%r#4X^<=zy{a=8(;%$U=<~ z18jf|umLu}2G{@_U;}J`4X}a#(108Q>|6o(Ma=&HuYdplxdt#Leh}xx8S$BTM<8K| z7sOKni0M^;-^6#~Bk_SaB@T&zxJ}$5E)(C@0KO7mh)=|GVvpD*wuotUo}aTb@x=z% z02^QfY=8~00XDz}*Z>>oz`!vC!GX-Y4z_}D<0vsE$#m?{SefW};MGK9H19uN3>Gri z$07V{Hx9zKHDPcB(4Ik=0%clg8&`eHRg%f3i*o9ArU~hB9QC78I(KO7u#u|{b+W&A z+Ah>mhdSB6sxMYDR@!1ZN|nT>x%LGe Date: Thu, 28 Mar 2019 21:46:35 +0800 Subject: [PATCH 36/47] Change workspace path to be in gs --- testing/integration/tf/modules/feast-helm/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/tf/modules/feast-helm/main.tf b/testing/integration/tf/modules/feast-helm/main.tf index 28b0b264a84..687d392fe98 100644 --- a/testing/integration/tf/modules/feast-helm/main.tf +++ b/testing/integration/tf/modules/feast-helm/main.tf @@ -43,7 +43,7 @@ resource "template_file" "helm_values" { store_serving_option = "'${jsonencode(local.serving_option)}'" project_id = "${var.project_name}" region = "${var.region}" - workspace = "${var.bucket_name}/workspace" + workspace = "gs://${var.bucket_name}/workspace" } } From 9f93f0a4c5cc97f79661f4a7d92200189c44cdcc Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 28 Mar 2019 22:18:33 +0800 Subject: [PATCH 37/47] Remove mentions of granularity --- testing/integration/tests/redis-bq-dataflow/test_feast.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/test_feast.py b/testing/integration/tests/redis-bq-dataflow/test_feast.py index 9b9dfd52cc8..eec3a1b82bd 100644 --- a/testing/integration/tests/redis-bq-dataflow/test_feast.py +++ b/testing/integration/tests/redis-bq-dataflow/test_feast.py @@ -83,7 +83,7 @@ def test_end_to_end(self, client): wanted['event_timestamp'] = pd.to_datetime(wanted['event_timestamp'])\ .dt.strftime("%Y-%m-%d %H:%M:%S UTC") got = TableDownloader().download_table_as_df( - project_id + ".feast_it.myentity_minute", + project_id + ".feast_it.myentity", "gs://{}/test-cases/extract.csv".format(bucket_name)) got = got.drop("created_timestamp", axis=1) \ .sort_values(["id", "event_timestamp"]) \ @@ -92,12 +92,12 @@ def test_end_to_end(self, client): # Check data in redis feature_set = FeatureSet(entity="myentity", - features=["myentity.minute." + f for f in features]) + features=["myentity." + f for f in features]) actual_latest = client.get_serving_data(feature_set, entity_keys=[str(id) for id in list(wanted.id.unique())]) actual_latest = actual_latest.sort_values(["id"]) wanted['event_timestamp'] = pd.to_datetime(wanted['event_timestamp']) wanted_latest = wanted.loc[wanted.groupby('id').event_timestamp.idxmax(),:] - wanted_latest.columns = ["myentity", "timestamp"] + ["myentity.minute." + f for f in features] + wanted_latest.columns = ["myentity", "timestamp"] + ["myentity." + f for f in features] wanted_latest = wanted_latest[actual_latest.columns] \ .sort_values(["id"]) \ .reset_index(drop=True) From b43e05ede94df42912c6027bf45e759b9f4e4ab7 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 28 Mar 2019 22:49:58 +0800 Subject: [PATCH 38/47] Drop job id when comparing data --- testing/integration/tests/redis-bq-dataflow/test_feast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/tests/redis-bq-dataflow/test_feast.py b/testing/integration/tests/redis-bq-dataflow/test_feast.py index eec3a1b82bd..cc00a9b8818 100644 --- a/testing/integration/tests/redis-bq-dataflow/test_feast.py +++ b/testing/integration/tests/redis-bq-dataflow/test_feast.py @@ -85,7 +85,7 @@ def test_end_to_end(self, client): got = TableDownloader().download_table_as_df( project_id + ".feast_it.myentity", "gs://{}/test-cases/extract.csv".format(bucket_name)) - got = got.drop("created_timestamp", axis=1) \ + got = got.drop(["created_timestamp", "job_id"], axis=1) \ .sort_values(["id", "event_timestamp"]) \ .reset_index(drop=True) assert pd.testing.assert_frame_equal(got, wanted[got.columns], check_less_precise=True) is None From 8bac0a79d5588e0fa20883b79026b9e95168b9c9 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 28 Mar 2019 23:36:45 +0800 Subject: [PATCH 39/47] Disable authentication for redis --- testing/integration/tf/modules/feast-helm/values.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/integration/tf/modules/feast-helm/values.tmpl b/testing/integration/tf/modules/feast-helm/values.tmpl index a4e00d5b655..ab1fe95ca37 100644 --- a/testing/integration/tf/modules/feast-helm/values.tmpl +++ b/testing/integration/tf/modules/feast-helm/values.tmpl @@ -57,6 +57,7 @@ redis: type: LoadBalancer annotations: "cloud.google.com/load-balancer-type": "Internal" + usePassword: false serving: config: From 7a332a047a335a2b840218646b92e908f1ae5eaa Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 29 Mar 2019 00:17:28 +0800 Subject: [PATCH 40/47] Sort first, then alter columns --- testing/integration/tests/redis-bq-dataflow/test_feast.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/test_feast.py b/testing/integration/tests/redis-bq-dataflow/test_feast.py index cc00a9b8818..c8e2bab8978 100644 --- a/testing/integration/tests/redis-bq-dataflow/test_feast.py +++ b/testing/integration/tests/redis-bq-dataflow/test_feast.py @@ -98,8 +98,7 @@ def test_end_to_end(self, client): wanted['event_timestamp'] = pd.to_datetime(wanted['event_timestamp']) wanted_latest = wanted.loc[wanted.groupby('id').event_timestamp.idxmax(),:] wanted_latest.columns = ["myentity", "timestamp"] + ["myentity." + f for f in features] - wanted_latest = wanted_latest[actual_latest.columns] \ - .sort_values(["id"]) \ + wanted_latest = wanted_latest.sort_values(["id"])[actual_latest.columns] \ .reset_index(drop=True) wanted_latest["myentity"] = wanted_latest["myentity"].astype(str) From 7fb8414e9967f9ce28dc80e610a13d65a4bbad2e Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 29 Mar 2019 00:56:12 +0800 Subject: [PATCH 41/47] Fix wrong column name --- testing/integration/tests/redis-bq-dataflow/test_feast.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/test_feast.py b/testing/integration/tests/redis-bq-dataflow/test_feast.py index c8e2bab8978..892cb69ad20 100644 --- a/testing/integration/tests/redis-bq-dataflow/test_feast.py +++ b/testing/integration/tests/redis-bq-dataflow/test_feast.py @@ -94,11 +94,12 @@ def test_end_to_end(self, client): feature_set = FeatureSet(entity="myentity", features=["myentity." + f for f in features]) actual_latest = client.get_serving_data(feature_set, entity_keys=[str(id) for id in list(wanted.id.unique())]) - actual_latest = actual_latest.sort_values(["id"]) + actual_latest = actual_latest.sort_values(["myentity"]) wanted['event_timestamp'] = pd.to_datetime(wanted['event_timestamp']) - wanted_latest = wanted.loc[wanted.groupby('id').event_timestamp.idxmax(),:] + wanted_latest = wanted.loc[wanted.groupby('myentity').event_timestamp.idxmax(),:] wanted_latest.columns = ["myentity", "timestamp"] + ["myentity." + f for f in features] - wanted_latest = wanted_latest.sort_values(["id"])[actual_latest.columns] \ + wanted_latest = wanted_latest[actual_latest.columns] \ + .sort_values(["myentity"]) \ .reset_index(drop=True) wanted_latest["myentity"] = wanted_latest["myentity"].astype(str) From 508a60499f9df6823755f5f3f0a1acc3ca5361ba Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 29 Mar 2019 09:48:13 +0800 Subject: [PATCH 42/47] Cleanup tests --- .../tests/redis-bq-dataflow/test_feast.py | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/test_feast.py b/testing/integration/tests/redis-bq-dataflow/test_feast.py index 892cb69ad20..6021183b5b9 100644 --- a/testing/integration/tests/redis-bq-dataflow/test_feast.py +++ b/testing/integration/tests/redis-bq-dataflow/test_feast.py @@ -21,7 +21,7 @@ def client(): '''Init the system by registering relevant resources. ''' -def register_resources(client, entities_fldr, features_fldr): +def _register_resources(client, entities_fldr, features_fldr): resources = [] for ent_file in os.listdir(entities_fldr): resources.append(Entity.from_yaml(os.path.join(entities_fldr, ent_file))) @@ -31,7 +31,7 @@ def register_resources(client, entities_fldr, features_fldr): '''Run an import job given an import spec. ''' -def run_job_and_wait_for_completion(job_yaml): +def _run_job_and_wait_for_completion(job_yaml): out = subprocess.run("feast jobs run {}".format(job_yaml).split(" "), check=True, stdout=PIPE) job_id = out.stdout.decode('utf-8').split(' ')[-1] @@ -50,7 +50,7 @@ def run_job_and_wait_for_completion(job_yaml): '''Stage data to a remote location ''' -def stage_data(local, remote): +def _stage_data(local, remote): split = utils.split_gs_path(remote) storage_client = storage.Client() bucket = storage_client.get_bucket(split[0]) @@ -58,15 +58,26 @@ def stage_data(local, remote): blob.upload_from_filename(local) +'''Get the BQ data and get only columns to compare, + then sort by id and timestamp +''' + +def _get_data_from_bq_and_sort(table_name, bucket_name): + got = TableDownloader().download_table_as_df( + table_name, + "gs://{}/test-cases/extract.csv".format(bucket_name)) + return got.drop(["created_timestamp", "job_id"], axis=1) \ + .sort_values(["id", "event_timestamp"]) \ + .reset_index(drop=True) class TestFeastIntegration: def test_end_to_end(self, client): project_id = os.environ.get("PROJECT_ID") bucket_name = os.environ.get("BUCKET_NAME") - stage_data("data/test_data.csv", "gs://{}/test-cases/test_data.csv".format(bucket_name)) - register_resources(client, "data/entity", "data/feature") - result = run_job_and_wait_for_completion("data/import/import_csv.yaml") + _stage_data("data/test_data.csv", "gs://{}/test-cases/test_data.csv".format(bucket_name)) + _register_resources(client, "data/entity", "data/feature") + result = _run_job_and_wait_for_completion("data/import/import_csv.yaml") # Ensure that the job is able to reach completion assert result == "COMPLETED" @@ -96,12 +107,12 @@ def test_end_to_end(self, client): actual_latest = client.get_serving_data(feature_set, entity_keys=[str(id) for id in list(wanted.id.unique())]) actual_latest = actual_latest.sort_values(["myentity"]) wanted['event_timestamp'] = pd.to_datetime(wanted['event_timestamp']) - wanted_latest = wanted.loc[wanted.groupby('myentity').event_timestamp.idxmax(),:] + wanted_latest = wanted.loc[wanted.groupby('id').event_timestamp.idxmax(),:] wanted_latest.columns = ["myentity", "timestamp"] + ["myentity." + f for f in features] wanted_latest = wanted_latest[actual_latest.columns] \ .sort_values(["myentity"]) \ .reset_index(drop=True) - wanted_latest["myentity"] = wanted_latest["myentity"].astype(str) + wanted_latest["myentity"] = wanted_latest["myentity"].apply(str) assert pd.testing.assert_frame_equal(actual_latest, wanted_latest, check_less_precise=True) is None From aa715dc99b3b4f8f015a5a0703e9458269bf8587 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 29 Mar 2019 10:57:14 +0800 Subject: [PATCH 43/47] Clean up tests --- testing/integration/tests/redis-bq-dataflow/test_feast.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/test_feast.py b/testing/integration/tests/redis-bq-dataflow/test_feast.py index 6021183b5b9..8efe85c2576 100644 --- a/testing/integration/tests/redis-bq-dataflow/test_feast.py +++ b/testing/integration/tests/redis-bq-dataflow/test_feast.py @@ -61,7 +61,6 @@ def _stage_data(local, remote): '''Get the BQ data and get only columns to compare, then sort by id and timestamp ''' - def _get_data_from_bq_and_sort(table_name, bucket_name): got = TableDownloader().download_table_as_df( table_name, @@ -93,12 +92,7 @@ def test_end_to_end(self, client): .reset_index(drop=True) wanted['event_timestamp'] = pd.to_datetime(wanted['event_timestamp'])\ .dt.strftime("%Y-%m-%d %H:%M:%S UTC") - got = TableDownloader().download_table_as_df( - project_id + ".feast_it.myentity", - "gs://{}/test-cases/extract.csv".format(bucket_name)) - got = got.drop(["created_timestamp", "job_id"], axis=1) \ - .sort_values(["id", "event_timestamp"]) \ - .reset_index(drop=True) + got = _get_data_from_bq_and_sort(project_id + ".feast_it.myentity", bucket_name) assert pd.testing.assert_frame_equal(got, wanted[got.columns], check_less_precise=True) is None # Check data in redis From 9f7d00aeff66b87f85e40b384ae57e5c38f5e3d3 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Fri, 29 Mar 2019 15:56:45 +0800 Subject: [PATCH 44/47] Ensure dataframe dtypes are the same when running tests Also ignore duplicate rows when retrieving actual test data result from warehouse so jobs can run multiple times on the same cluster and thes test still succeed --- .../tests/redis-bq-dataflow/.env.local | 7 + .../tests/redis-bq-dataflow/main.py | 6 + .../tests/redis-bq-dataflow/test_feast.py | 199 ++++++++++++------ 3 files changed, 145 insertions(+), 67 deletions(-) create mode 100644 testing/integration/tests/redis-bq-dataflow/.env.local create mode 100644 testing/integration/tests/redis-bq-dataflow/main.py diff --git a/testing/integration/tests/redis-bq-dataflow/.env.local b/testing/integration/tests/redis-bq-dataflow/.env.local new file mode 100644 index 00000000000..99a58fd76d3 --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/.env.local @@ -0,0 +1,7 @@ +# Example environment variables that need to be set when running this test locally +# Export the variables like so: +# set -a; source .env.local +FEAST_CORE_URL=localhost:6565 +FEAST_SERVING_URL=localhost:6566 +PROJECT_ID=kf-feast +BUCKET_NAME=it-feast-storage diff --git a/testing/integration/tests/redis-bq-dataflow/main.py b/testing/integration/tests/redis-bq-dataflow/main.py new file mode 100644 index 00000000000..253faa6f96a --- /dev/null +++ b/testing/integration/tests/redis-bq-dataflow/main.py @@ -0,0 +1,6 @@ +import test_feast +from feast.sdk.client import Client + +if __name__ == "__main__": + feast_client = Client(verbose=True) + test_feast.TestFeastIntegration().test_end_to_end(client=feast_client) diff --git a/testing/integration/tests/redis-bq-dataflow/test_feast.py b/testing/integration/tests/redis-bq-dataflow/test_feast.py index 6021183b5b9..0c394177e53 100644 --- a/testing/integration/tests/redis-bq-dataflow/test_feast.py +++ b/testing/integration/tests/redis-bq-dataflow/test_feast.py @@ -1,26 +1,27 @@ -import pytest +import os import subprocess from subprocess import PIPE -import yaml -import os from time import sleep -import pandas as pd -from google.cloud import storage import feast.sdk.utils.gs_utils as utils - +import numpy as np +import pandas as pd +import pytest +import yaml +from feast.sdk.client import Client from feast.sdk.resources.entity import Entity from feast.sdk.resources.feature import Feature -from feast.sdk.client import Client -from feast.sdk.resources.feature_set import FeatureSet, FileType +from feast.sdk.resources.feature_set import FeatureSet from feast.sdk.utils.bq_util import TableDownloader +from google.cloud import storage + @pytest.fixture def client(): return Client(verbose=True) -'''Init the system by registering relevant resources. -''' + +# Init the system by registering relevant resources. def _register_resources(client, entities_fldr, features_fldr): resources = [] for ent_file in os.listdir(entities_fldr): @@ -29,27 +30,28 @@ def _register_resources(client, entities_fldr, features_fldr): resources.append(Feature.from_yaml(os.path.join(features_fldr, feat_file))) client.apply(resources) -'''Run an import job given an import spec. -''' + +# Run an import job given an import spec. def _run_job_and_wait_for_completion(job_yaml): - out = subprocess.run("feast jobs run {}".format(job_yaml).split(" "), - check=True, stdout=PIPE) - job_id = out.stdout.decode('utf-8').split(' ')[-1] - job_complete = False + out = subprocess.run( + "feast jobs run {}".format(job_yaml).split(" "), check=True, stdout=PIPE + ) + job_id = out.stdout.decode("utf-8").split(" ")[-1] + job_status, job_complete = "UNKNOWN", False terminal_states = ["COMPLETED", "ABORTED", "ERROR", "UNKNOWN"] while not job_complete: - out = subprocess.run("feast get job {}".format(job_id).split(" "), - check=True, stdout=PIPE) - job_details = yaml.load(out.stdout.decode('utf-8').replace('\t', ' ')) - job_status = job_details['Status'] + out = subprocess.run( + "feast get job {}".format(job_id).split(" "), check=True, stdout=PIPE + ) + job_details = yaml.load(out.stdout.decode("utf-8").replace("\t", " ")) + job_status = job_details["Status"] print("Job id {} currently {}".format(job_id, job_status)) if job_status in terminal_states: break sleep(10) return job_status -'''Stage data to a remote location -''' + def _stage_data(local, remote): split = utils.split_gs_path(remote) storage_client = storage.Client() @@ -58,62 +60,125 @@ def _stage_data(local, remote): blob.upload_from_filename(local) -'''Get the BQ data and get only columns to compare, - then sort by id and timestamp -''' +# Get the BQ data and get only columns to compare, then sort by id and timestamp def _get_data_from_bq_and_sort(table_name, bucket_name): got = TableDownloader().download_table_as_df( - table_name, - "gs://{}/test-cases/extract.csv".format(bucket_name)) - return got.drop(["created_timestamp", "job_id"], axis=1) \ - .sort_values(["id", "event_timestamp"]) \ - .reset_index(drop=True) + table_name, "gs://{}/test-cases/extract.csv".format(bucket_name) + ) + return ( + got.drop(["created_timestamp", "job_id"], axis=1) + .sort_values(["id", "event_timestamp"]) + .reset_index(drop=True) + ) + class TestFeastIntegration: def test_end_to_end(self, client): project_id = os.environ.get("PROJECT_ID") bucket_name = os.environ.get("BUCKET_NAME") - _stage_data("data/test_data.csv", "gs://{}/test-cases/test_data.csv".format(bucket_name)) + features = [ + "feature_double_redis", + "feature_float_redis", + "feature_int32_redis", + "feature_int64_redis", + ] + expected = self.get_expected_data(features) + + self.run_batch_import(bucket_name, client) + self.validate_warehouse_data(bucket_name, project_id, expected) + self.validate_serving_data(client, features, expected) + + @staticmethod + def get_expected_data(features): + expected = pd.read_csv( + "data/test_data.csv", + header=None, + names=["id", "event_timestamp"] + features, + ) + expected = expected.sort_values(["id", "event_timestamp"]).reset_index( + drop=True + ) + expected["event_timestamp"] = pd.to_datetime( + expected["event_timestamp"] + ).dt.strftime("%Y-%m-%d %H:%M:%S UTC") + return expected + + @staticmethod + def run_batch_import(bucket_name, client): + _stage_data( + "data/test_data.csv", "gs://{}/test-cases/test_data.csv".format(bucket_name) + ) _register_resources(client, "data/entity", "data/feature") - result = _run_job_and_wait_for_completion("data/import/import_csv.yaml") - - # Ensure that the job is able to reach completion - assert result == "COMPLETED" - - # Check data in bq - features = ["feature_double_redis", "feature_float_redis", - "feature_int32_redis", "feature_int64_redis"] - wanted = pd.read_csv("data/test_data.csv", - header=None, - names=["id", "event_timestamp"] + features) - wanted = wanted \ - .sort_values(["id", "event_timestamp"]) \ - .reset_index(drop=True) - wanted['event_timestamp'] = pd.to_datetime(wanted['event_timestamp'])\ - .dt.strftime("%Y-%m-%d %H:%M:%S UTC") - got = TableDownloader().download_table_as_df( - project_id + ".feast_it.myentity", - "gs://{}/test-cases/extract.csv".format(bucket_name)) - got = got.drop(["created_timestamp", "job_id"], axis=1) \ - .sort_values(["id", "event_timestamp"]) \ - .reset_index(drop=True) - assert pd.testing.assert_frame_equal(got, wanted[got.columns], check_less_precise=True) is None + job_status = _run_job_and_wait_for_completion("data/import/import_csv.yaml") + assert job_status == "COMPLETED" + + @staticmethod + def validate_serving_data(client, features, expected): + features_type_mapping = { + "myentity": np.string_, + "myentity.feature_double_redis": np.float64, + "myentity.feature_float_redis": np.float64, + "myentity.feature_int32_redis": np.int64, + "myentity.feature_int64_redis": np.int64, + } - # Check data in redis - feature_set = FeatureSet(entity="myentity", - features=["myentity." + f for f in features]) - actual_latest = client.get_serving_data(feature_set, entity_keys=[str(id) for id in list(wanted.id.unique())]) + feature_set = FeatureSet( + entity="myentity", features=["myentity." + f for f in features] + ) + actual_latest = client.get_serving_data( + feature_set, entity_keys=[str(id) for id in list(expected.id.unique())] + ).astype(features_type_mapping) actual_latest = actual_latest.sort_values(["myentity"]) - wanted['event_timestamp'] = pd.to_datetime(wanted['event_timestamp']) - wanted_latest = wanted.loc[wanted.groupby('id').event_timestamp.idxmax(),:] - wanted_latest.columns = ["myentity", "timestamp"] + ["myentity." + f for f in features] - wanted_latest = wanted_latest[actual_latest.columns] \ - .sort_values(["myentity"]) \ + expected["event_timestamp"] = pd.to_datetime(expected["event_timestamp"]) + expected_latest = expected.loc[ + expected.groupby("id").event_timestamp.idxmax(), : + ] + expected_latest.columns = ["myentity", "timestamp"] + [ + "myentity." + f for f in features + ] + expected_latest = ( + expected_latest[actual_latest.columns] + .sort_values(["myentity"]) + .reset_index(drop=True) + ).astype(features_type_mapping) + + assert ( + pd.testing.assert_frame_equal( + expected_latest, actual_latest, check_less_precise=True, check_like=True + ) + is None + ) + + @staticmethod + def validate_warehouse_data(bucket_name, project_id, expected): + features_type_mapping = { + "id": np.string_, + "event_timestamp": np.string_, + "feature_double_redis": np.float64, + "feature_float_redis": np.float64, + "feature_int32_redis": np.int64, + "feature_int64_redis": np.int64, + } + + actual = TableDownloader().download_table_as_df( + project_id + ".feast_it.myentity", + "gs://{}/test-cases/extract.csv".format(bucket_name), + ) + actual = actual.drop(["created_timestamp", "job_id"], axis=1).sort_values( + ["id", "event_timestamp"] + ) + actual = ( + actual.drop_duplicates() .reset_index(drop=True) - wanted_latest["myentity"] = wanted_latest["myentity"].apply(str) - - assert pd.testing.assert_frame_equal(actual_latest, wanted_latest, check_less_precise=True) is None + .astype(features_type_mapping) + ) + expected = expected.astype(features_type_mapping) - + assert ( + pd.testing.assert_frame_equal( + expected, actual, check_less_precise=True, check_like=True + ) + is None + ) From 0d175ad556dc0ee212ff4a58ea7d4261ad401fad Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Fri, 29 Mar 2019 15:57:36 +0800 Subject: [PATCH 45/47] Add options to run mvn package using 4 threads and ignore resolving of dependency location To make build faster --- testing/integration/tests/redis-bq-dataflow/workflow.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml index a560c4ba263..daa80eebf73 100644 --- a/testing/integration/tests/redis-bq-dataflow/workflow.yaml +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -121,9 +121,11 @@ spec: command: - /bin/bash - -c - - | - mvn package -DskipTests -Drevision=integration-test + - mvn package -DskipTests -Drevision=integration-test -T 4 -DdependencyLocationsEnabled=false workingDir: /feast + resources: + requests: + cpu: 3000m outputs: artifacts: - name: feast-build @@ -203,7 +205,7 @@ spec: pip install pytest; pip install -r /feast/sdk/python/test-requirements.txt; pip install -e /feast/sdk/python; - pytest -s + pytest --capture=no '] workingDir: /feast/testing/integration/tests/redis-bq-dataflow env: From 27d2f9b70d3d9664ea40faeed6959060ced2fee4 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Fri, 29 Mar 2019 16:07:14 +0800 Subject: [PATCH 46/47] Rename step and template name in argo workflow To make it more descriptive --- .../tests/redis-bq-dataflow/workflow.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml index daa80eebf73..88f4b4fe12c 100644 --- a/testing/integration/tests/redis-bq-dataflow/workflow.yaml +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -69,18 +69,18 @@ spec: from: "{{workflow.outputs.artifacts.feast-repo}}" - name: get-feast steps: - - - name: get-feast-pr - template: get-feast-pr + - - name: get-feast-by-pullrequest + template: get-feast-by-pullrequest when: "{{workflow.parameters.prId}} != none" - - name: get-feast-gitref - template: get-feast-gitref + - name: get-feast-by-gitref + template: get-feast-by-gitref when: "{{workflow.parameters.gitRef}} != none" outputs: artifacts: - name: feast-repo from: "{{workflow.outputs.artifacts.feast-repo}}" globalName: feast-repo - - name: get-feast-pr + - name: get-feast-by-pullrequest container: image: alpine/git command: @@ -96,7 +96,7 @@ spec: - name: feast path: /git/feast globalName: feast-repo - - name: get-feast-gitref + - name: get-feast-by-gitref container: image: alpine/git command: From 9668243fd1277ec0a8ae65eebc0ead14ecb6b117 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Fri, 29 Mar 2019 16:53:03 +0800 Subject: [PATCH 47/47] Accept gke_machine_type parameter in feast terraform module To customize the machine type for the node pools --- testing/integration/tests/redis-bq-dataflow/tf/main.tf | 1 + testing/integration/tests/redis-bq-dataflow/workflow.yaml | 2 +- testing/integration/tf/modules/cluster/input.tf | 5 +++++ testing/integration/tf/modules/cluster/main.tf | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/testing/integration/tests/redis-bq-dataflow/tf/main.tf b/testing/integration/tests/redis-bq-dataflow/tf/main.tf index bad1c4f2bd9..fadca6442a6 100644 --- a/testing/integration/tests/redis-bq-dataflow/tf/main.tf +++ b/testing/integration/tests/redis-bq-dataflow/tf/main.tf @@ -27,6 +27,7 @@ module "cluster" { bucket_name = "${local.cluster_name}-storage" network = "${local.network}" subnetwork = "${local.subnetwork}" + gke_machine_type = "n1-standard-8" } resource "null_resource" "wait_for_regional_cluster" { diff --git a/testing/integration/tests/redis-bq-dataflow/workflow.yaml b/testing/integration/tests/redis-bq-dataflow/workflow.yaml index 88f4b4fe12c..c1c6265ab3a 100644 --- a/testing/integration/tests/redis-bq-dataflow/workflow.yaml +++ b/testing/integration/tests/redis-bq-dataflow/workflow.yaml @@ -121,7 +121,7 @@ spec: command: - /bin/bash - -c - - mvn package -DskipTests -Drevision=integration-test -T 4 -DdependencyLocationsEnabled=false + - mvn package -DskipTests -Drevision=integration-test -T 6 -DdependencyLocationsEnabled=false workingDir: /feast resources: requests: diff --git a/testing/integration/tf/modules/cluster/input.tf b/testing/integration/tf/modules/cluster/input.tf index f3ea7673b24..bccca70fbbc 100644 --- a/testing/integration/tf/modules/cluster/input.tf +++ b/testing/integration/tf/modules/cluster/input.tf @@ -19,6 +19,11 @@ variable "subnetwork" { default = "default" } +variable "gke_machine_type" { + description = "The machine type for the default node pool" + default = "n1-standard-4" +} + variable "bucket_name" { description = "Working storage for feast" } diff --git a/testing/integration/tf/modules/cluster/main.tf b/testing/integration/tf/modules/cluster/main.tf index daad2b8a31f..23b66241a42 100644 --- a/testing/integration/tf/modules/cluster/main.tf +++ b/testing/integration/tf/modules/cluster/main.tf @@ -20,7 +20,7 @@ resource "google_container_cluster" "feast_k8s" { enable_legacy_abac = true node_config { - machine_type = "n1-standard-4" + machine_type = "${var.gke_machine_type}" disk_size_gb = "50" oauth_scopes = [