From 5a5b7f27876e622058a76120a435d28c684ca378 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Thu, 22 Oct 2020 16:57:50 +0800 Subject: [PATCH 01/60] python to manage e2e dependencies Signed-off-by: Oleksii Moskalenko --- Makefile | 8 +- tests/e2e/__init__.py | 0 tests/e2e/conftest.py | 57 +---- tests/e2e/fixtures.py | 200 ++++++++++++++++++ tests/e2e/test_online_features.py | 8 +- .../{test-register.py => test_register.py} | 56 ++--- tests/{e2e => }/pyproject.toml | 0 tests/{e2e => }/pytest.ini | 0 tests/{e2e => }/requirements.txt | 3 + tests/{e2e => }/setup.cfg | 0 10 files changed, 235 insertions(+), 97 deletions(-) create mode 100644 tests/e2e/__init__.py create mode 100644 tests/e2e/fixtures.py rename tests/e2e/{test-register.py => test_register.py} (83%) rename tests/{e2e => }/pyproject.toml (100%) rename tests/{e2e => }/pytest.ini (100%) rename tests/{e2e => }/requirements.txt (78%) rename tests/{e2e => }/setup.cfg (100%) diff --git a/Makefile b/Makefile index de0a1b1c4c0..1ae6b2dda85 100644 --- a/Makefile +++ b/Makefile @@ -90,10 +90,10 @@ lint-python: cd ${ROOT_DIR}/sdk/python; flake8 feast/ tests/ cd ${ROOT_DIR}/sdk/python; black --check feast tests - cd ${ROOT_DIR}/tests/e2e; mypy . - cd ${ROOT_DIR}/tests/e2e; isort . --check-only - cd ${ROOT_DIR}/tests/e2e; flake8 . - cd ${ROOT_DIR}/tests/e2e; black --check . + cd ${ROOT_DIR}/tests; mypy e2e + cd ${ROOT_DIR}/tests; isort e2e --check-only + cd ${ROOT_DIR}/tests; flake8 e2e + cd ${ROOT_DIR}/tests; black --check e2e # Go SDK diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 321ba5dcfc1..9168795d804 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,10 +1,4 @@ -import os -from pathlib import Path - -import pyspark -import pytest - -from feast import Client +from .fixtures import * def pytest_addoption(parser): @@ -40,52 +34,3 @@ def pytest_runtest_setup(item): previousfailed = getattr(item.parent, "_previousfailed", None) if previousfailed is not None: pytest.xfail("previous test failed (%s)" % previousfailed.name) - - -@pytest.fixture(scope="session") -def feast_version(): - return "0.8-SNAPSHOT" - - -@pytest.fixture(scope="session") -def ingestion_job_jar(pytestconfig, feast_version): - default_path = ( - Path(__file__).parent.parent.parent - / "spark" - / "ingestion" - / "target" - / f"feast-ingestion-spark-{feast_version}.jar" - ) - - return pytestconfig.getoption("ingestion_jar") or f"file://{default_path}" - - -@pytest.fixture(scope="session") -def feast_client(pytestconfig, ingestion_job_jar): - redis_host, redis_port = pytestconfig.getoption("redis_url").split(":") - - if pytestconfig.getoption("env") == "local": - return Client( - core_url=pytestconfig.getoption("core_url"), - serving_url=pytestconfig.getoption("serving_url"), - spark_launcher="standalone", - spark_standalone_master="local", - spark_home=os.getenv("SPARK_HOME") or os.path.dirname(pyspark.__file__), - spark_ingestion_jar=ingestion_job_jar, - redis_host=redis_host, - redis_port=redis_port, - ) - - if pytestconfig.getoption("env") == "gcloud": - return Client( - core_url=pytestconfig.getoption("core_url"), - serving_url=pytestconfig.getoption("serving_url"), - spark_launcher="dataproc", - dataproc_cluster_name=pytestconfig.getoption("dataproc_cluster_name"), - dataproc_project=pytestconfig.getoption("dataproc_project"), - dataproc_region=pytestconfig.getoption("dataproc_region"), - dataproc_staging_location=os.path.join( - pytestconfig.getoption("staging_path"), "dataproc" - ), - spark_ingestion_jar=ingestion_job_jar, - ) diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py new file mode 100644 index 00000000000..364f2d88d87 --- /dev/null +++ b/tests/e2e/fixtures.py @@ -0,0 +1,200 @@ +import os +import shutil +import socket +import time + +import pyspark +import requests +import pathlib + +import yaml +import pytest +import subprocess +import tempfile +from pathlib import Path + + +from pytest_postgresql import factories as pg_factories +from pytest_postgresql.executor import PostgreSQLExecutor +from pytest_redis import factories as redis_factories +from pytest_redis.executor import RedisExecutor +from pytest_kafka import make_kafka_server, make_zookeeper_process + +from feast import Client + + +@pytest.fixture(scope="session") +def project_root(): + return Path(__file__).parent.parent.parent + + +@pytest.fixture(scope="session") +def project_version(): + return "0.8-SNAPSHOT" + + +def download_kafka(version="2.12-2.6.0"): + r = requests.get(f'https://downloads.apache.org/kafka/2.6.0/kafka_{version}.tgz') + temp_dir = pathlib.Path(tempfile.gettempdir()) + local_path = temp_dir / 'kafka.tgz' + + with open(local_path, 'wb') as f: + f.write(r.content) + + shutil.unpack_archive(local_path, tempfile.gettempdir()) + return temp_dir / f'kafka_{version}' / "bin" + + +def _start_jar(jar, options=None) -> subprocess.Popen: + if not os.path.isfile(jar): + raise ValueError(f"{jar} doesn't exist") + + cmd = [ + "java", + "-jar", + jar + ] + if options: + cmd.extend(options) + print(' '.join(cmd)) + return subprocess.Popen(cmd) + + +def _wait_port_open(port, max_wait=60): + print(f"Waiting for port {port}") + start = time.time() + + while True: + try: + socket.create_connection(('localhost', port), timeout=1) + except OSError: + if time.time() - start > max_wait: + raise + + time.sleep(1) + else: + return + + +@pytest.fixture(scope="session", params=[pytest.param(True, marks=pytest.mark.skip), False]) +def enable_auth(request): + return request.param + + +@pytest.fixture(scope="session") +def feast_core(project_root, project_version, enable_auth, postgres_server: PostgreSQLExecutor): + jar = str(project_root / "core" / "target" / f"feast-core-{project_version}-exec.jar") + config = dict( + feast=dict( + security=dict( + enabled=enable_auth, + provider="jwt", + options=dict(jwkEndpointURI="https://www.googleapis.com/oauth2/v3/certs") + ) + ), + spring=dict( + datasource=dict( + url=f"jdbc:postgresql://127.0.0.1:{postgres_server.port}/postgres" + ) + ) + ) + + with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w+') as config_file: + yaml.dump(config, config_file) + config_file.flush() + + process = _start_jar(jar, [f"--spring.config.location=classpath:/application.yml,file://{config_file.name}"]) + _wait_port_open(6565) + yield + process.terminate() + + +@pytest.fixture(scope="session") +def feast_serving(project_root, project_version, enable_auth, redis_server: RedisExecutor): + jar = str(project_root / "serving" / "target" / f"feast-serving-{project_version}-exec.jar") + config = dict( + feast=dict( + stores=[dict( + name="online", + type="REDIS", + config=dict( + host=redis_server.host, + port=redis_server.port + ) + )], + coreAuthentication=dict( + enabled=enable_auth, + provider="google" + ), + security=dict( + authentication=dict( + enabled=enable_auth, + provider="jwt" + ) + ) + ) + ) + + with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w+') as config_file: + yaml.dump(config, config_file) + config_file.flush() + + process = _start_jar(jar, [f"--spring.config.location=classpath:/application.yml,file://{config_file.name}"]) + _wait_port_open(6566) + yield + process.terminate() + + +@pytest.fixture(scope="session") +def ingestion_job_jar(pytestconfig, project_root, project_version): + default_path = ( + project_root + / "spark" + / "ingestion" + / "target" + / f"feast-ingestion-spark-{project_version}.jar" + ) + + return pytestconfig.getoption("ingestion_jar") or f"file://{default_path}" + + +@pytest.fixture(scope="session") +def feast_client(pytestconfig, ingestion_job_jar, redis_server: RedisExecutor, feast_core, feast_serving): + if pytestconfig.getoption("env") == "local": + return Client( + core_url=pytestconfig.getoption("core_url"), + serving_url=pytestconfig.getoption("serving_url"), + spark_launcher="standalone", + spark_standalone_master="local", + spark_home=os.getenv("SPARK_HOME") or os.path.dirname(pyspark.__file__), + spark_ingestion_jar=ingestion_job_jar, + redis_host=redis_server.host, + redis_port=redis_server.port, + ) + + if pytestconfig.getoption("env") == "gcloud": + return Client( + core_url=pytestconfig.getoption("core_url"), + serving_url=pytestconfig.getoption("serving_url"), + spark_launcher="dataproc", + dataproc_cluster_name=pytestconfig.getoption("dataproc_cluster_name"), + dataproc_project=pytestconfig.getoption("dataproc_project"), + dataproc_region=pytestconfig.getoption("dataproc_region"), + dataproc_staging_location=os.path.join( + pytestconfig.getoption("staging_path"), "dataproc" + ), + spark_ingestion_jar=ingestion_job_jar, + ) + + +postgres_server = pg_factories.postgresql_proc(password="password") +redis_server = redis_factories.redis_proc(executable=shutil.which("redis-server")) + +KAFKA_BIN = download_kafka() +zookeeper_server = make_zookeeper_process(str(KAFKA_BIN / "zookeeper-server-start.sh"), zk_config_template=""" +dataDir={zk_data_dir} +clientPort={zk_port} +maxClientCnxns=0 +admin.enableServer=false""") +kafka_server = make_kafka_server(kafka_bin=str(KAFKA_BIN / "kafka-server-start.sh"), + zookeeper_fixture_name='zookeeper_server') diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index 8d81a9b79a5..15f5c3e379d 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -12,6 +12,7 @@ import pytz from avro.io import BinaryEncoder, DatumWriter from confluent_kafka import Producer +from pytest_redis.executor import RedisExecutor from feast import ( Client, @@ -89,8 +90,9 @@ def test_offline_ingestion(feast_client: Client, staging_path: str): ) -def test_streaming_ingestion(feast_client: Client, staging_path: str, pytestconfig): +def test_streaming_ingestion(feast_client: Client, staging_path: str, kafka_server): entity = Entity(name="s2id", description="S2id", value_type=ValueType.INT64,) + kafka_broker = f"localhost:{kafka_server[1]}" feature_table = FeatureTable( name="drivers_stream", @@ -105,7 +107,7 @@ def test_streaming_ingestion(feast_client: Client, staging_path: str, pytestconf stream_source=KafkaSource( "event_timestamp", "event_timestamp", - pytestconfig.getoption("kafka_brokers"), + kafka_broker, AvroFormat(avro_schema()), topic="avro", ), @@ -130,7 +132,7 @@ def test_streaming_ingestion(feast_client: Client, staging_path: str, pytestconf send_avro_record_to_kafka( "avro", record, - bootstrap_servers=pytestconfig.getoption("kafka_brokers"), + bootstrap_servers=kafka_broker, avro_schema_json=avro_schema(), ) diff --git a/tests/e2e/test-register.py b/tests/e2e/test_register.py similarity index 83% rename from tests/e2e/test-register.py rename to tests/e2e/test_register.py index 4bd8966817b..e6121ca524c 100644 --- a/tests/e2e/test-register.py +++ b/tests/e2e/test_register.py @@ -23,18 +23,6 @@ SUFFIX = str(int(datetime.now().timestamp())) -@pytest.fixture(scope="module") -def client(pytestconfig): - core_url = pytestconfig.getoption("core_url") - serving_url = pytestconfig.getoption("serving_url") - - client = Client(core_url=core_url, serving_url=serving_url,) - - client.set_project(PROJECT_NAME) - - return client - - @pytest.fixture def bq_table_id(): return f"kf-feast:feaste2e.table{SUFFIX}" @@ -177,76 +165,76 @@ def alltypes_featuretable(): def test_get_list_basic( - client: Client, + feast_client: Client, customer_entity: Entity, driver_entity: Entity, basic_featuretable: FeatureTable, ): # ApplyEntity - client.apply_entity(customer_entity) - client.apply_entity(driver_entity) + feast_client.apply_entity(customer_entity) + feast_client.apply_entity(driver_entity) # GetEntity Check - assert client.get_entity(name="customer_id") == customer_entity - assert client.get_entity(name="driver_id") == driver_entity + assert feast_client.get_entity(name="customer_id") == customer_entity + assert feast_client.get_entity(name="driver_id") == driver_entity # ListEntities Check common_filtering_labels = {"common_key": "common_val"} matchmaking_filtering_labels = {"team": "matchmaking"} - actual_common_entities = client.list_entities(labels=common_filtering_labels) - actual_matchmaking_entities = client.list_entities( + actual_common_entities = feast_client.list_entities(labels=common_filtering_labels) + actual_matchmaking_entities = feast_client.list_entities( labels=matchmaking_filtering_labels ) assert len(actual_common_entities) == 2 assert len(actual_matchmaking_entities) == 1 # ApplyFeatureTable - client.apply_feature_table(basic_featuretable) + feast_client.apply_feature_table(basic_featuretable) # GetFeatureTable Check - actual_get_feature_table = client.get_feature_table(name="basic_featuretable") + actual_get_feature_table = feast_client.get_feature_table(name="basic_featuretable") assert actual_get_feature_table == basic_featuretable # ListFeatureTables Check actual_list_feature_table = [ - ft for ft in client.list_feature_tables() if ft.name == "basic_featuretable" + ft for ft in feast_client.list_feature_tables() if ft.name == "basic_featuretable" ][0] assert actual_list_feature_table == basic_featuretable def test_get_list_alltypes( - client: Client, alltypes_entity: Entity, alltypes_featuretable: FeatureTable + feast_client: Client, alltypes_entity: Entity, alltypes_featuretable: FeatureTable ): # ApplyEntity - client.apply_entity(alltypes_entity) + feast_client.apply_entity(alltypes_entity) # GetEntity Check - assert client.get_entity(name="alltypes_id") == alltypes_entity + assert feast_client.get_entity(name="alltypes_id") == alltypes_entity # ListEntities Check alltypes_filtering_labels = {"cat": "alltypes"} - actual_alltypes_entities = client.list_entities(labels=alltypes_filtering_labels) + actual_alltypes_entities = feast_client.list_entities(labels=alltypes_filtering_labels) assert len(actual_alltypes_entities) == 1 # ApplyFeatureTable - client.apply_feature_table(alltypes_featuretable) + feast_client.apply_feature_table(alltypes_featuretable) # GetFeatureTable Check - actual_get_feature_table = client.get_feature_table(name="alltypes") + actual_get_feature_table = feast_client.get_feature_table(name="alltypes") assert actual_get_feature_table == alltypes_featuretable # ListFeatureTables Check actual_list_feature_table = [ - ft for ft in client.list_feature_tables() if ft.name == "alltypes" + ft for ft in feast_client.list_feature_tables() if ft.name == "alltypes" ][0] assert actual_list_feature_table == alltypes_featuretable @pytest.mark.bq def test_ingest( - client: Client, + feast_client: Client, customer_entity: Entity, driver_entity: Entity, bq_featuretable: FeatureTable, @@ -257,12 +245,12 @@ def test_ingest( bq_table_id = bq_table_id.replace(":", ".") # ApplyEntity - client.apply_entity(customer_entity) - client.apply_entity(driver_entity) + feast_client.apply_entity(customer_entity) + feast_client.apply_entity(driver_entity) # ApplyFeatureTable - client.apply_feature_table(bq_featuretable) - client.ingest(bq_featuretable, bq_dataset, timeout=120) + feast_client.apply_feature_table(bq_featuretable) + feast_client.ingest(bq_featuretable, bq_dataset, timeout=120) from google.api_core.exceptions import NotFound from google.cloud import bigquery diff --git a/tests/e2e/pyproject.toml b/tests/pyproject.toml similarity index 100% rename from tests/e2e/pyproject.toml rename to tests/pyproject.toml diff --git a/tests/e2e/pytest.ini b/tests/pytest.ini similarity index 100% rename from tests/e2e/pytest.ini rename to tests/pytest.ini diff --git a/tests/e2e/requirements.txt b/tests/requirements.txt similarity index 78% rename from tests/e2e/requirements.txt rename to tests/requirements.txt index 80380451c55..2fee6adf67d 100644 --- a/tests/e2e/requirements.txt +++ b/tests/requirements.txt @@ -9,6 +9,9 @@ pytest-mock==1.10.4 pytest-timeout==1.3.3 pytest-ordering==0.6.* pytest-xdist==2.1.0 +pytest-postgresql==2.5.1 +pytest-redis==2.0.0 +pytest-kafka==0.4.0 deepdiff==4.3.2 confluent_kafka avro==1.10.0 \ No newline at end of file diff --git a/tests/e2e/setup.cfg b/tests/setup.cfg similarity index 100% rename from tests/e2e/setup.cfg rename to tests/setup.cfg From 51a7f11133893d51d3c02be2affc77a6ada29389 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Thu, 22 Oct 2020 17:43:02 +0800 Subject: [PATCH 02/60] all tests running Signed-off-by: Oleksii Moskalenko --- .prow/config.yaml | 61 ++++----------------------- tests/e2e/fixtures.py | 25 ++++++++++- tests/e2e/test_historical_features.py | 34 +++++---------- tests/e2e/test_online_features.py | 20 ++------- 4 files changed, 46 insertions(+), 94 deletions(-) diff --git a/.prow/config.yaml b/.prow/config.yaml index 39c275603d2..eab925bd8e0 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -147,59 +147,14 @@ presubmits: spec: containers: - image: maven:3.6-jdk-11 - command: ["infra/scripts/test-end-to-end.sh"] - resources: - requests: - cpu: "6" - memory: "6144Mi" - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/gcloud/service-account.json - volumeMounts: - - mountPath: /etc/gcloud/service-account.json - name: service-account - readOnly: true - subPath: service-account.json - volumes: - - name: service-account - secret: - secretName: feast-service-account - skip_branches: - - ^v0\.(3|4)-branch$ - - - name: test-end-to-end-auth - decorate: true - always_run: true - spec: - containers: - - image: maven:3.6-jdk-11 - command: ["infra/scripts/test-end-to-end.sh", "True"] - resources: - requests: - cpu: "6" - memory: "6144Mi" - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/gcloud/service-account.json - volumeMounts: - - mountPath: /etc/gcloud/service-account.json - name: service-account - readOnly: true - subPath: service-account.json - volumes: - - name: service-account - secret: - secretName: feast-service-account - skip_branches: - - ^v0\.(3|4)-branch$ - - - name: test-end-to-end-redis-cluster - decorate: true - always_run: true - spec: - containers: - - image: maven:3.6-jdk-11 - command: ["infra/scripts/test-end-to-end-redis-cluster.sh"] + command: | + apt-get update && apt-get install python3-pip postgresql libpq-dev redis-server + make build-java-no-tests + make compile-protos-python + pip3 install -U pip + pip3 install -qe sdk/python + pip3 install -qr tests/requirements.txt + pytest tests/e2e/ resources: requests: cpu: "6" diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index 364f2d88d87..790efdb86f1 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -2,7 +2,7 @@ import shutil import socket import time - +import uuid import pyspark import requests import pathlib @@ -159,7 +159,12 @@ def ingestion_job_jar(pytestconfig, project_root, project_version): @pytest.fixture(scope="session") -def feast_client(pytestconfig, ingestion_job_jar, redis_server: RedisExecutor, feast_core, feast_serving): +def feast_client(pytestconfig, + ingestion_job_jar, + redis_server: RedisExecutor, + feast_core, + feast_serving, + global_staging_path): if pytestconfig.getoption("env") == "local": return Client( core_url=pytestconfig.getoption("core_url"), @@ -170,6 +175,7 @@ def feast_client(pytestconfig, ingestion_job_jar, redis_server: RedisExecutor, f spark_ingestion_jar=ingestion_job_jar, redis_host=redis_server.host, redis_port=redis_server.port, + historical_feature_output_location=os.path.join(global_staging_path, "historical_output") ) if pytestconfig.getoption("env") == "gcloud": @@ -187,6 +193,21 @@ def feast_client(pytestconfig, ingestion_job_jar, redis_server: RedisExecutor, f ) +@pytest.fixture(scope="session") +def global_staging_path(pytestconfig): + if pytestconfig.getoption("env") == "local": + tmp_path = tempfile.mkdtemp() + return f"file://{tmp_path}" + + staging_path = pytestconfig.getoption("staging_path") + return os.path.join(staging_path, str(uuid.uuid4())) + + +@pytest.fixture(scope="function") +def local_staging_path(global_staging_path): + return os.path.join(global_staging_path, str(uuid.uuid4())) + + postgres_server = pg_factories.postgresql_proc(password="password") redis_server = redis_factories.redis_proc(executable=shutil.which("redis-server")) diff --git a/tests/e2e/test_historical_features.py b/tests/e2e/test_historical_features.py index 84c1af139f1..a91983e0848 100644 --- a/tests/e2e/test_historical_features.py +++ b/tests/e2e/test_historical_features.py @@ -1,12 +1,10 @@ import os import tempfile -import uuid from datetime import datetime, timedelta from urllib.parse import urlparse import numpy as np import pandas as pd -import pytest from google.protobuf.duration_pb2 import Duration from pandas._testing import assert_frame_equal @@ -17,19 +15,9 @@ np.random.seed(0) -@pytest.fixture(scope="function") -def staging_path(pytestconfig, tmp_path): - if pytestconfig.getoption("env") == "local": - return f"file://{tmp_path}" - - staging_path = pytestconfig.getoption("staging_path") - return os.path.join(staging_path, str(uuid.uuid4())) - - -@pytest.mark.skip -def test_historical_features(feast_client: Client, staging_path: str): +def test_historical_features(feast_client: Client, local_staging_path: str): customer_entity = Entity( - name="customer_id", description="Customer", value_type=ValueType.INT64 + name="user_id", description="Customer", value_type=ValueType.INT64 ) feast_client.apply_entity(customer_entity) @@ -38,7 +26,7 @@ def test_historical_features(feast_client: Client, staging_path: str): transactions_feature_table = FeatureTable( name="transactions", - entities=["customer_id"], + entities=["user_id"], features=[ Feature("daily_transactions", ValueType.DOUBLE), Feature("total_transactions", ValueType.DOUBLE), @@ -47,7 +35,7 @@ def test_historical_features(feast_client: Client, staging_path: str): "event_timestamp", "created_timestamp", ParquetFormat(), - os.path.join(staging_path, "transactions"), + os.path.join(local_staging_path, "transactions"), ), max_age=max_age, ) @@ -71,7 +59,7 @@ def test_historical_features(feast_client: Client, staging_path: str): { "event_timestamp": [event_date for _ in customers], "created_timestamp": [creation_date for _ in customers], - "customer_id": customers, + "user_id": customers, "daily_transactions": daily_transactions, "total_transactions": total_transactions, } @@ -85,21 +73,21 @@ def test_historical_features(feast_client: Client, staging_path: str): { "event_timestamp": [retrieval_date for _ in customers] + [retrieval_outside_max_age_date for _ in customers], - "customer_id": customers + customers, + "user_id": customers + customers, } ) with tempfile.TemporaryDirectory() as tempdir: df_export_path = os.path.join(tempdir, "customers.parquets") customer_df.to_parquet(df_export_path) - scheme, _, remote_path, _, _, _ = urlparse(staging_path) + scheme, _, remote_path, _, _, _ = urlparse(local_staging_path) staging_client = get_staging_client(scheme) staging_client.upload_file(df_export_path, None, remote_path) customer_source = FileSource( "event_timestamp", "event_timestamp", ParquetFormat(), - os.path.join(staging_path, os.path.basename(df_export_path)), + os.path.join(local_staging_path, os.path.basename(df_export_path)), ) job = feast_client.get_historical_features(feature_refs, customer_source) @@ -112,17 +100,17 @@ def test_historical_features(feast_client: Client, staging_path: str): { "event_timestamp": [retrieval_date for _ in customers] + [retrieval_outside_max_age_date for _ in customers], - "customer_id": customers + customers, + "user_id": customers + customers, "transactions__daily_transactions": daily_transactions + [None] * len(customers), } ) assert_frame_equal( - joined_df.sort_values(by=["customer_id", "event_timestamp"]).reset_index( + joined_df.sort_values(by=["user_id", "event_timestamp"]).reset_index( drop=True ), expected_joined_df.sort_values( - by=["customer_id", "event_timestamp"] + by=["user_id", "event_timestamp"] ).reset_index(drop=True), ) diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index 15f5c3e379d..480534443e5 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -2,17 +2,14 @@ import json import os import time -import uuid from datetime import datetime, timedelta import avro.schema import numpy as np import pandas as pd -import pytest import pytz from avro.io import BinaryEncoder, DatumWriter from confluent_kafka import Producer -from pytest_redis.executor import RedisExecutor from feast import ( Client, @@ -40,16 +37,7 @@ def generate_data(): return df -@pytest.fixture(scope="function") -def staging_path(pytestconfig, tmp_path): - if pytestconfig.getoption("env") == "local": - return f"file://{tmp_path}" - - staging_path = pytestconfig.getoption("staging_path") - return os.path.join(staging_path, str(uuid.uuid4())) - - -def test_offline_ingestion(feast_client: Client, staging_path: str): +def test_offline_ingestion(feast_client: Client, local_staging_path: str): entity = Entity(name="s2id", description="S2id", value_type=ValueType.INT64,) feature_table = FeatureTable( @@ -60,7 +48,7 @@ def test_offline_ingestion(feast_client: Client, staging_path: str): "event_timestamp", "event_timestamp", ParquetFormat(), - os.path.join(staging_path, "batch-storage"), + os.path.join(local_staging_path, "batch-storage"), ), ) @@ -90,7 +78,7 @@ def test_offline_ingestion(feast_client: Client, staging_path: str): ) -def test_streaming_ingestion(feast_client: Client, staging_path: str, kafka_server): +def test_streaming_ingestion(feast_client: Client, local_staging_path: str, kafka_server): entity = Entity(name="s2id", description="S2id", value_type=ValueType.INT64,) kafka_broker = f"localhost:{kafka_server[1]}" @@ -102,7 +90,7 @@ def test_streaming_ingestion(feast_client: Client, staging_path: str, kafka_serv "event_timestamp", "event_timestamp", ParquetFormat(), - os.path.join(staging_path, "batch-storage"), + os.path.join(local_staging_path, "batch-storage"), ), stream_source=KafkaSource( "event_timestamp", From dafe895bd18a43e0cced0e7f9c71a2440e820557 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 11:24:31 +0800 Subject: [PATCH 03/60] run tests on github action Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 18 +++++++++ .prow/config.yaml | 67 ---------------------------------- tests/e2e/fixtures.py | 2 +- 3 files changed, 19 insertions(+), 68 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index ef3715bcd85..4d9463e0d16 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -156,3 +156,21 @@ jobs: - name: copy to gs run: gsutil cp ./spark/ingestion/target/feast-ingestion-spark-${GITHUB_SHA}.jar gs://feast-jobs/spark/ingestion/ + test-end-to-end: + runs-on: [self-hosted] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: '11' + - uses: stCarolas/setup-maven@v3 + with: + maven-version: 3.6.3 + - name: test + run: | + apt-get update && apt-get install python3-pip postgresql libpq-dev redis-server + make build-java-no-tests + pip3 install -U pip + pip3 install -qe sdk/python + pip3 install -qr tests/requirements.txt + pytest tests/e2e/ diff --git a/.prow/config.yaml b/.prow/config.yaml index eab925bd8e0..ec18e74586b 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -141,73 +141,6 @@ presubmits: - image: golang:1.13 command: ["infra/scripts/test-golang-sdk.sh"] - - name: test-end-to-end - decorate: true - always_run: true - spec: - containers: - - image: maven:3.6-jdk-11 - command: | - apt-get update && apt-get install python3-pip postgresql libpq-dev redis-server - make build-java-no-tests - make compile-protos-python - pip3 install -U pip - pip3 install -qe sdk/python - pip3 install -qr tests/requirements.txt - pytest tests/e2e/ - resources: - requests: - cpu: "6" - memory: "6144Mi" - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/gcloud/service-account.json - volumeMounts: - - mountPath: /etc/gcloud/service-account.json - name: service-account - readOnly: true - subPath: service-account.json - volumes: - - name: service-account - secret: - secretName: feast-service-account - skip_branches: - - ^v0\.(3|4)-branch$ - - - name: test-end-to-end-java-8 - decorate: true - always_run: true - spec: - containers: - - image: maven:3.6-jdk-8 - command: ["infra/scripts/test-end-to-end.sh"] - resources: - requests: - cpu: "6" - memory: "6144Mi" - branches: - - ^v0\.(3|4)-branch$ - - - name: test-end-to-end-batch-java-8 - decorate: true - always_run: true - spec: - volumes: - - name: service-account - secret: - secretName: feast-service-account - containers: - - image: maven:3.6-jdk-8 - command: ["infra/scripts/test-end-to-end-batch.sh"] - resources: - requests: - cpu: "6" - memory: "6144Mi" - volumeMounts: - - name: service-account - mountPath: "/etc/service-account" - branches: - - ^v0\.(3|4)-branch$ postsubmits: feast-dev/feast: diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index 790efdb86f1..d6545ac9294 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -50,7 +50,7 @@ def _start_jar(jar, options=None) -> subprocess.Popen: raise ValueError(f"{jar} doesn't exist") cmd = [ - "java", + shutil.which("java"), "-jar", jar ] From 0bd30ccf056d23f670c01ee14f105214fddd3e5f Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 11:41:50 +0800 Subject: [PATCH 04/60] fixing e2e action Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 4d9463e0d16..e2b06962b2d 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -168,7 +168,7 @@ jobs: maven-version: 3.6.3 - name: test run: | - apt-get update && apt-get install python3-pip postgresql libpq-dev redis-server + apt-get update && apt-get install -y python3-pip postgresql libpq-dev redis-server make build-java-no-tests pip3 install -U pip pip3 install -qe sdk/python From 3e93b5fcb097d4b2811742203c2d6edae508e0a5 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 12:12:25 +0800 Subject: [PATCH 05/60] pass version as option Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 4 ++-- tests/e2e/conftest.py | 2 ++ tests/e2e/fixtures.py | 5 ++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index e2b06962b2d..07baa15f60f 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -169,8 +169,8 @@ jobs: - name: test run: | apt-get update && apt-get install -y python3-pip postgresql libpq-dev redis-server - make build-java-no-tests + make build-java-no-tests REVISION=develop pip3 install -U pip pip3 install -qe sdk/python pip3 install -qr tests/requirements.txt - pytest tests/e2e/ + pytest tests/e2e/ --version develop diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 9168795d804..ceb0f6effdc 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -21,6 +21,8 @@ def pytest_addoption(parser): parser.addoption("--ingestion-jar", action="store") parser.addoption("--redis-url", action="store", default="localhost:6379") + parser.addoption("--version", action="store") + def pytest_runtest_makereport(item, call): if "incremental" in item.keywords: diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index d6545ac9294..f8d3ec6a4e1 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -29,7 +29,10 @@ def project_root(): @pytest.fixture(scope="session") -def project_version(): +def project_version(pytestconfig): + if pytestconfig.getoption("version"): + return pytestconfig.getoption("version") + return "0.8-SNAPSHOT" From ef30da14cd7812c825892952669d4f3e4626ccaf Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 12:25:17 +0800 Subject: [PATCH 06/60] setup python Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 07baa15f60f..ab42135fb00 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -166,9 +166,12 @@ jobs: - uses: stCarolas/setup-maven@v3 with: maven-version: 3.6.3 + - uses: actions/setup-python@v2 + with: + python-version: 3.6 - name: test run: | - apt-get update && apt-get install -y python3-pip postgresql libpq-dev redis-server + apt-get update && apt-get install -y postgresql libpq-dev redis-server make build-java-no-tests REVISION=develop pip3 install -U pip pip3 install -qe sdk/python From d054591c59bb75716f1257431904c82a1f87de01 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 12:27:12 +0800 Subject: [PATCH 07/60] setup python Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index ab42135fb00..6a811cd98e6 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -173,7 +173,7 @@ jobs: run: | apt-get update && apt-get install -y postgresql libpq-dev redis-server make build-java-no-tests REVISION=develop - pip3 install -U pip - pip3 install -qe sdk/python - pip3 install -qr tests/requirements.txt + python -m pip install --upgrade pip + pip install -qe sdk/python + pip install -qr tests/requirements.txt pytest tests/e2e/ --version develop From 43d36410640cfb76e97b26948ce09d2f39fc9287 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 12:28:15 +0800 Subject: [PATCH 08/60] setup python Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 6a811cd98e6..28842a2b334 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -173,7 +173,7 @@ jobs: run: | apt-get update && apt-get install -y postgresql libpq-dev redis-server make build-java-no-tests REVISION=develop - python -m pip install --upgrade pip - pip install -qe sdk/python - pip install -qr tests/requirements.txt + python -m pip install --upgrade pip setuptools wheel + python -m pip install -qe sdk/python + python -m pip install -qr tests/requirements.txt pytest tests/e2e/ --version develop From e16543c8cbe47ca3e4a79d5d597ed7ffe07fd356 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 12:39:30 +0800 Subject: [PATCH 09/60] pass feast version Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 +- tests/e2e/conftest.py | 2 +- tests/e2e/fixtures.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 28842a2b334..c2ef8831cb5 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -176,4 +176,4 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install -qe sdk/python python -m pip install -qr tests/requirements.txt - pytest tests/e2e/ --version develop + pytest tests/e2e/ --feast-version develop diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index ceb0f6effdc..41ed3c41a18 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -21,7 +21,7 @@ def pytest_addoption(parser): parser.addoption("--ingestion-jar", action="store") parser.addoption("--redis-url", action="store", default="localhost:6379") - parser.addoption("--version", action="store") + parser.addoption("--feast-version", action="store") def pytest_runtest_makereport(item, call): diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index f8d3ec6a4e1..ed757ff60d9 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -30,8 +30,8 @@ def project_root(): @pytest.fixture(scope="session") def project_version(pytestconfig): - if pytestconfig.getoption("version"): - return pytestconfig.getoption("version") + if pytestconfig.getoption("feast_version"): + return pytestconfig.getoption("feast_version") return "0.8-SNAPSHOT" From 82ebdf3f0cde50a455bbadf7f8d7de282d13e295 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 12:54:52 +0800 Subject: [PATCH 10/60] python style Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 +- Makefile | 2 +- tests/e2e/conftest.py | 5 +- tests/e2e/fixtures.py | 163 ++++++++++++++++++------------ tests/e2e/test_online_features.py | 4 +- tests/e2e/test_register.py | 8 +- 6 files changed, 113 insertions(+), 71 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index c2ef8831cb5..00bb335cee7 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -174,6 +174,6 @@ jobs: apt-get update && apt-get install -y postgresql libpq-dev redis-server make build-java-no-tests REVISION=develop python -m pip install --upgrade pip setuptools wheel - python -m pip install -qe sdk/python + make install-python python -m pip install -qr tests/requirements.txt pytest tests/e2e/ --feast-version develop diff --git a/Makefile b/Makefile index 1ae6b2dda85..a42fa2735b1 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ compile-protos-python: install-python-ci-dependencies cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --grpc_python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/third_party/grpc/health/v1/*.proto install-python: compile-protos-python - cd sdk/python; python setup.py develop + cd sdk/python; python -m pip install -e sdk/python test-python: pytest --verbose --color=yes sdk/python/tests diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 41ed3c41a18..13b457280c1 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,4 +1,4 @@ -from .fixtures import * +import pytest def pytest_addoption(parser): @@ -36,3 +36,6 @@ def pytest_runtest_setup(item): previousfailed = getattr(item.parent, "_previousfailed", None) if previousfailed is not None: pytest.xfail("previous test failed (%s)" % previousfailed.name) + + +from .fixtures import * # noqa diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index ed757ff60d9..4c74229ae7b 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -1,27 +1,39 @@ import os +import pathlib import shutil import socket +import subprocess +import tempfile import time import uuid +from pathlib import Path + import pyspark +import pytest import requests -import pathlib - import yaml -import pytest -import subprocess -import tempfile -from pathlib import Path - - +from pytest_kafka import make_kafka_server, make_zookeeper_process from pytest_postgresql import factories as pg_factories from pytest_postgresql.executor import PostgreSQLExecutor from pytest_redis import factories as redis_factories from pytest_redis.executor import RedisExecutor -from pytest_kafka import make_kafka_server, make_zookeeper_process from feast import Client +__all__ = ( + "project_root", + "project_version", + "feast_client", + "feast_core", + "feast_serving", + "ingestion_job_jar", + "global_staging_path", + "local_staging_path", + "redis_server", + "postgres_server", + "kafka_server", +) + @pytest.fixture(scope="session") def project_root(): @@ -37,30 +49,26 @@ def project_version(pytestconfig): def download_kafka(version="2.12-2.6.0"): - r = requests.get(f'https://downloads.apache.org/kafka/2.6.0/kafka_{version}.tgz') + r = requests.get(f"https://downloads.apache.org/kafka/2.6.0/kafka_{version}.tgz") temp_dir = pathlib.Path(tempfile.gettempdir()) - local_path = temp_dir / 'kafka.tgz' + local_path = temp_dir / "kafka.tgz" - with open(local_path, 'wb') as f: + with open(local_path, "wb") as f: f.write(r.content) shutil.unpack_archive(local_path, tempfile.gettempdir()) - return temp_dir / f'kafka_{version}' / "bin" + return temp_dir / f"kafka_{version}" / "bin" def _start_jar(jar, options=None) -> subprocess.Popen: if not os.path.isfile(jar): raise ValueError(f"{jar} doesn't exist") - cmd = [ - shutil.which("java"), - "-jar", - jar - ] + cmd = [shutil.which("java"), "-jar", jar] if options: cmd.extend(options) - print(' '.join(cmd)) - return subprocess.Popen(cmd) + + return subprocess.Popen(cmd) # type: ignore def _wait_port_open(port, max_wait=60): @@ -69,7 +77,7 @@ def _wait_port_open(port, max_wait=60): while True: try: - socket.create_connection(('localhost', port), timeout=1) + socket.create_connection(("localhost", port), timeout=1) except OSError: if time.time() - start > max_wait: raise @@ -79,70 +87,86 @@ def _wait_port_open(port, max_wait=60): return -@pytest.fixture(scope="session", params=[pytest.param(True, marks=pytest.mark.skip), False]) +@pytest.fixture( + scope="session", params=[pytest.param(True, marks=pytest.mark.skip), False] +) def enable_auth(request): return request.param @pytest.fixture(scope="session") -def feast_core(project_root, project_version, enable_auth, postgres_server: PostgreSQLExecutor): - jar = str(project_root / "core" / "target" / f"feast-core-{project_version}-exec.jar") +def feast_core( + project_root, project_version, enable_auth, postgres_server: PostgreSQLExecutor +): + jar = str( + project_root / "core" / "target" / f"feast-core-{project_version}-exec.jar" + ) config = dict( feast=dict( security=dict( enabled=enable_auth, provider="jwt", - options=dict(jwkEndpointURI="https://www.googleapis.com/oauth2/v3/certs") + options=dict( + jwkEndpointURI="https://www.googleapis.com/oauth2/v3/certs" + ), ) ), spring=dict( datasource=dict( url=f"jdbc:postgresql://127.0.0.1:{postgres_server.port}/postgres" ) - ) + ), ) - with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w+') as config_file: + with tempfile.NamedTemporaryFile(suffix=".yaml", mode="w+") as config_file: yaml.dump(config, config_file) config_file.flush() - process = _start_jar(jar, [f"--spring.config.location=classpath:/application.yml,file://{config_file.name}"]) + process = _start_jar( + jar, + [ + f"--spring.config.location=classpath:/application.yml,file://{config_file.name}" + ], + ) _wait_port_open(6565) yield process.terminate() @pytest.fixture(scope="session") -def feast_serving(project_root, project_version, enable_auth, redis_server: RedisExecutor): - jar = str(project_root / "serving" / "target" / f"feast-serving-{project_version}-exec.jar") +def feast_serving( + project_root, project_version, enable_auth, redis_server: RedisExecutor +): + jar = str( + project_root + / "serving" + / "target" + / f"feast-serving-{project_version}-exec.jar" + ) config = dict( feast=dict( - stores=[dict( - name="online", - type="REDIS", - config=dict( - host=redis_server.host, - port=redis_server.port + stores=[ + dict( + name="online", + type="REDIS", + config=dict(host=redis_server.host, port=redis_server.port), ) - )], - coreAuthentication=dict( - enabled=enable_auth, - provider="google" - ), - security=dict( - authentication=dict( - enabled=enable_auth, - provider="jwt" - ) - ) + ], + coreAuthentication=dict(enabled=enable_auth, provider="google"), + security=dict(authentication=dict(enabled=enable_auth, provider="jwt")), ) ) - with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w+') as config_file: + with tempfile.NamedTemporaryFile(suffix=".yaml", mode="w+") as config_file: yaml.dump(config, config_file) config_file.flush() - process = _start_jar(jar, [f"--spring.config.location=classpath:/application.yml,file://{config_file.name}"]) + process = _start_jar( + jar, + [ + f"--spring.config.location=classpath:/application.yml,file://{config_file.name}" + ], + ) _wait_port_open(6566) yield process.terminate() @@ -151,23 +175,25 @@ def feast_serving(project_root, project_version, enable_auth, redis_server: Redi @pytest.fixture(scope="session") def ingestion_job_jar(pytestconfig, project_root, project_version): default_path = ( - project_root - / "spark" - / "ingestion" - / "target" - / f"feast-ingestion-spark-{project_version}.jar" + project_root + / "spark" + / "ingestion" + / "target" + / f"feast-ingestion-spark-{project_version}.jar" ) return pytestconfig.getoption("ingestion_jar") or f"file://{default_path}" @pytest.fixture(scope="session") -def feast_client(pytestconfig, - ingestion_job_jar, - redis_server: RedisExecutor, - feast_core, - feast_serving, - global_staging_path): +def feast_client( + pytestconfig, + ingestion_job_jar, + redis_server: RedisExecutor, + feast_core, + feast_serving, + global_staging_path, +): if pytestconfig.getoption("env") == "local": return Client( core_url=pytestconfig.getoption("core_url"), @@ -178,7 +204,9 @@ def feast_client(pytestconfig, spark_ingestion_jar=ingestion_job_jar, redis_host=redis_server.host, redis_port=redis_server.port, - historical_feature_output_location=os.path.join(global_staging_path, "historical_output") + historical_feature_output_location=os.path.join( + global_staging_path, "historical_output" + ), ) if pytestconfig.getoption("env") == "gcloud": @@ -215,10 +243,15 @@ def local_staging_path(global_staging_path): redis_server = redis_factories.redis_proc(executable=shutil.which("redis-server")) KAFKA_BIN = download_kafka() -zookeeper_server = make_zookeeper_process(str(KAFKA_BIN / "zookeeper-server-start.sh"), zk_config_template=""" +zookeeper_server = make_zookeeper_process( + str(KAFKA_BIN / "zookeeper-server-start.sh"), + zk_config_template=""" dataDir={zk_data_dir} clientPort={zk_port} maxClientCnxns=0 -admin.enableServer=false""") -kafka_server = make_kafka_server(kafka_bin=str(KAFKA_BIN / "kafka-server-start.sh"), - zookeeper_fixture_name='zookeeper_server') +admin.enableServer=false""", +) +kafka_server = make_kafka_server( + kafka_bin=str(KAFKA_BIN / "kafka-server-start.sh"), + zookeeper_fixture_name="zookeeper_server", +) diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index 480534443e5..ed53588f8ab 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -78,7 +78,9 @@ def test_offline_ingestion(feast_client: Client, local_staging_path: str): ) -def test_streaming_ingestion(feast_client: Client, local_staging_path: str, kafka_server): +def test_streaming_ingestion( + feast_client: Client, local_staging_path: str, kafka_server +): entity = Entity(name="s2id", description="S2id", value_type=ValueType.INT64,) kafka_broker = f"localhost:{kafka_server[1]}" diff --git a/tests/e2e/test_register.py b/tests/e2e/test_register.py index e6121ca524c..0c89ee69cef 100644 --- a/tests/e2e/test_register.py +++ b/tests/e2e/test_register.py @@ -199,7 +199,9 @@ def test_get_list_basic( # ListFeatureTables Check actual_list_feature_table = [ - ft for ft in feast_client.list_feature_tables() if ft.name == "basic_featuretable" + ft + for ft in feast_client.list_feature_tables() + if ft.name == "basic_featuretable" ][0] assert actual_list_feature_table == basic_featuretable @@ -215,7 +217,9 @@ def test_get_list_alltypes( # ListEntities Check alltypes_filtering_labels = {"cat": "alltypes"} - actual_alltypes_entities = feast_client.list_entities(labels=alltypes_filtering_labels) + actual_alltypes_entities = feast_client.list_entities( + labels=alltypes_filtering_labels + ) assert len(actual_alltypes_entities) == 1 # ApplyFeatureTable From cba58d65ad73ea5ca6089afd402bcdf0a85f1d61 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 13:00:29 +0800 Subject: [PATCH 11/60] fix make python Signed-off-by: Oleksii Moskalenko --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a42fa2735b1..f0f0cc5f1f4 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ compile-protos-python: install-python-ci-dependencies cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --grpc_python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/third_party/grpc/health/v1/*.proto install-python: compile-protos-python - cd sdk/python; python -m pip install -e sdk/python + python -m pip install -e sdk/python test-python: pytest --verbose --color=yes sdk/python/tests From 4c4030793dbb0b1692b85b0d0452e2671d76430f Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 13:07:14 +0800 Subject: [PATCH 12/60] fixes for 3.6 Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index 4c74229ae7b..e30f884cb73 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -56,7 +56,7 @@ def download_kafka(version="2.12-2.6.0"): with open(local_path, "wb") as f: f.write(r.content) - shutil.unpack_archive(local_path, tempfile.gettempdir()) + shutil.unpack_archive(str(local_path), str(temp_dir)) return temp_dir / f"kafka_{version}" / "bin" From 4c6b9b448a5785d641353d399e9d691f15ab31c7 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 13:19:43 +0800 Subject: [PATCH 13/60] add missing fixtures Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index e30f884cb73..e6561dc7262 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -32,6 +32,8 @@ "redis_server", "postgres_server", "kafka_server", + "zookeeper_server", + "enable_auth", ) From 6e2a847f083ec91a104a6775bb08495e44fda8a4 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 14:11:29 +0800 Subject: [PATCH 14/60] test with setuid Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index e6561dc7262..38bbd7a086a 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -52,7 +52,7 @@ def project_version(pytestconfig): def download_kafka(version="2.12-2.6.0"): r = requests.get(f"https://downloads.apache.org/kafka/2.6.0/kafka_{version}.tgz") - temp_dir = pathlib.Path(tempfile.gettempdir()) + temp_dir = pathlib.Path(tempfile.mkdtemp()) local_path = temp_dir / "kafka.tgz" with open(local_path, "wb") as f: @@ -241,6 +241,8 @@ def local_staging_path(global_staging_path): return os.path.join(global_staging_path, str(uuid.uuid4())) +os.seteuid(1001) + postgres_server = pg_factories.postgresql_proc(password="password") redis_server = redis_factories.redis_proc(executable=shutil.which("redis-server")) From d198d2fade6f51d2751e525981203a723ba94dc5 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 14:23:26 +0800 Subject: [PATCH 15/60] use provided postgres Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 27 ++++++++++++++++++++++++--- tests/e2e/fixtures.py | 7 +++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 00bb335cee7..f0953b94831 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -158,6 +158,26 @@ jobs: test-end-to-end: runs-on: [self-hosted] + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: password + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + env: + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v1 @@ -169,11 +189,12 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.6 - - name: test + - name: install run: | - apt-get update && apt-get install -y postgresql libpq-dev redis-server + apt-get update && apt-get install -y redis-server make build-java-no-tests REVISION=develop python -m pip install --upgrade pip setuptools wheel make install-python python -m pip install -qr tests/requirements.txt - pytest tests/e2e/ --feast-version develop + - name: run tests + run: pytest tests/e2e/ --feast-version develop diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index 38bbd7a086a..675a7054484 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -241,9 +241,12 @@ def local_staging_path(global_staging_path): return os.path.join(global_staging_path, str(uuid.uuid4())) -os.seteuid(1001) +if not os.environ.get('POSTGRES_HOST'): + postgres_server = pg_factories.postgresql_proc(password="password") +else: + postgres_server = pg_factories.postgresql_noproc(host=os.environ['POSTGRES_HOST'], + port=os.environ['POSTGRES_PORT']) -postgres_server = pg_factories.postgresql_proc(password="password") redis_server = redis_factories.redis_proc(executable=shutil.which("redis-server")) KAFKA_BIN = download_kafka() From 49595657fcc014a7cd64b4aab87160099cdde4fe Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 14:46:24 +0800 Subject: [PATCH 16/60] use postgres host Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index 675a7054484..c87f1a18eb5 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -115,7 +115,7 @@ def feast_core( ), spring=dict( datasource=dict( - url=f"jdbc:postgresql://127.0.0.1:{postgres_server.port}/postgres" + url=f"jdbc:postgresql://{postgres_server.host}:{postgres_server.port}/postgres" ) ), ) From 62557726ca292a08c586727e001d757998624b2b Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 14:53:27 +0800 Subject: [PATCH 17/60] debug Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index f0953b94831..2d43189b1c6 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -189,12 +189,16 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.6 - - name: install + - name: debug run: | - apt-get update && apt-get install -y redis-server - make build-java-no-tests REVISION=develop - python -m pip install --upgrade pip setuptools wheel - make install-python - python -m pip install -qr tests/requirements.txt - - name: run tests - run: pytest tests/e2e/ --feast-version develop + printenv + docker ps +# - name: install +# run: | +# apt-get update && apt-get install -y redis-server +# make build-java-no-tests REVISION=develop +# python -m pip install --upgrade pip setuptools wheel +# make install-python +# python -m pip install -qr tests/requirements.txt +# - name: run tests +# run: pytest tests/e2e/ --feast-version develop From 51d39cc627a5cf00e440d72d95d563725f3a3d12 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 14:59:47 +0800 Subject: [PATCH 18/60] debug Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 2d43189b1c6..8aab01010f3 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -192,7 +192,7 @@ jobs: - name: debug run: | printenv - docker ps + docker ps --format "{{.Image}} {{.Ports}} {{.Networks}}" # - name: install # run: | # apt-get update && apt-get install -y redis-server From c29fd202e281a76bea35400bb04d9baa7a02273b Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 15:06:37 +0800 Subject: [PATCH 19/60] debug Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 8aab01010f3..dcb68767898 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -172,11 +172,8 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 env: - POSTGRES_HOST: localhost + POSTGRES_HOST: postgres POSTGRES_PORT: 5432 steps: - uses: actions/checkout@v2 @@ -189,16 +186,12 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.6 - - name: debug + - name: install run: | - printenv - docker ps --format "{{.Image}} {{.Ports}} {{.Networks}}" -# - name: install -# run: | -# apt-get update && apt-get install -y redis-server -# make build-java-no-tests REVISION=develop -# python -m pip install --upgrade pip setuptools wheel -# make install-python -# python -m pip install -qr tests/requirements.txt -# - name: run tests -# run: pytest tests/e2e/ --feast-version develop + apt-get update && apt-get install -y redis-server + make build-java-no-tests REVISION=develop + python -m pip install --upgrade pip setuptools wheel + make install-python + python -m pip install -qr tests/requirements.txt + - name: run tests + run: pytest tests/e2e/ --feast-version develop From 8596aad68fc85f47a225d74f8212370fda029bcd Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 15:28:33 +0800 Subject: [PATCH 20/60] try set uid Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 17 ----------------- tests/e2e/fixtures.py | 3 +++ 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index dcb68767898..f2619862bc6 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -158,23 +158,6 @@ jobs: test-end-to-end: runs-on: [self-hosted] - services: - # Label used to access the service container - postgres: - # Docker Hub image - image: postgres - # Provide the password for postgres - env: - POSTGRES_PASSWORD: password - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - env: - POSTGRES_HOST: postgres - POSTGRES_PORT: 5432 steps: - uses: actions/checkout@v2 - uses: actions/setup-java@v1 diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index c87f1a18eb5..e87a84f3245 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -242,6 +242,9 @@ def local_staging_path(global_staging_path): if not os.environ.get('POSTGRES_HOST'): + if os.geteuid() == 0: + os.seteuid(1000) + postgres_server = pg_factories.postgresql_proc(password="password") else: postgres_server = pg_factories.postgresql_noproc(host=os.environ['POSTGRES_HOST'], From f5cb160066ef1aab1a659f08ba511f02219aadf3 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 15:53:08 +0800 Subject: [PATCH 21/60] patch postgres executor Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index e87a84f3245..6d3e46b885a 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -7,6 +7,7 @@ import time import uuid from pathlib import Path +from typing import Dict, Any import pyspark import pytest @@ -241,10 +242,24 @@ def local_staging_path(global_staging_path): return os.path.join(global_staging_path, str(uuid.uuid4())) +class PostgreSQLExecutorWithSU(PostgreSQLExecutor): + @property + def _popen_kwargs(self) -> Dict[str, Any]: + if os.geteuid() > 0: + return super()._popen_kwargs + + def set_uid(): + os.seteuid(1000) + + return { + **super()._popen_kwargs, + "preexec_fn": set_uid + } + + if not os.environ.get('POSTGRES_HOST'): - if os.geteuid() == 0: - os.seteuid(1000) - + pg_factories.PostgreSQLExecutor = PostgreSQLExecutorWithSU + postgres_server = pg_factories.postgresql_proc(password="password") else: postgres_server = pg_factories.postgresql_noproc(host=os.environ['POSTGRES_HOST'], From cbf0ef442d97086b0f543ab3e2311733db7b59e4 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 16:03:04 +0800 Subject: [PATCH 22/60] patch postgres executor Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index 6d3e46b885a..c5f04a2195c 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -249,7 +249,8 @@ def _popen_kwargs(self) -> Dict[str, Any]: return super()._popen_kwargs def set_uid(): - os.seteuid(1000) + os.setuid(1000) + os.setgid(1000) return { **super()._popen_kwargs, From b8951acb50fa2eb63f892ab6052bbf61af14cb3f Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 16:13:35 +0800 Subject: [PATCH 23/60] install postgres Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index f2619862bc6..aaf3b322d2c 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -171,7 +171,7 @@ jobs: python-version: 3.6 - name: install run: | - apt-get update && apt-get install -y redis-server + apt-get update && apt-get install -y redis-server postgresql libpq-dev make build-java-no-tests REVISION=develop python -m pip install --upgrade pip setuptools wheel make install-python From 8a085bf16ad8518422cf594e10f6b86f157f275d Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 16:23:03 +0800 Subject: [PATCH 24/60] always set uid Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index c5f04a2195c..246ec64b3a9 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -242,24 +242,9 @@ def local_staging_path(global_staging_path): return os.path.join(global_staging_path, str(uuid.uuid4())) -class PostgreSQLExecutorWithSU(PostgreSQLExecutor): - @property - def _popen_kwargs(self) -> Dict[str, Any]: - if os.geteuid() > 0: - return super()._popen_kwargs - - def set_uid(): - os.setuid(1000) - os.setgid(1000) - - return { - **super()._popen_kwargs, - "preexec_fn": set_uid - } - - if not os.environ.get('POSTGRES_HOST'): - pg_factories.PostgreSQLExecutor = PostgreSQLExecutorWithSU + os.setuid(1000) + os.setgid(1000) postgres_server = pg_factories.postgresql_proc(password="password") else: From 0fd02b8dec2d40c0d089ad7bdea4554efa3bb486 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 16:27:48 +0800 Subject: [PATCH 25/60] su Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 +- tests/e2e/fixtures.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index aaf3b322d2c..1dbad42d4e6 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -177,4 +177,4 @@ jobs: make install-python python -m pip install -qr tests/requirements.txt - name: run tests - run: pytest tests/e2e/ --feast-version develop + run: su -p postgres -c 'pytest tests/e2e/ --feast-version develop' diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures.py index 246ec64b3a9..5b17c22610a 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures.py @@ -243,9 +243,6 @@ def local_staging_path(global_staging_path): if not os.environ.get('POSTGRES_HOST'): - os.setuid(1000) - os.setgid(1000) - postgres_server = pg_factories.postgresql_proc(password="password") else: postgres_server = pg_factories.postgresql_noproc(host=os.environ['POSTGRES_HOST'], From 85fbb031534e727e8bee68d13aa6f539f60fd940 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 16:44:28 +0800 Subject: [PATCH 26/60] debug Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 1dbad42d4e6..2c63e9b1516 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -169,12 +169,18 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.6 - - name: install + - name: debug run: | apt-get update && apt-get install -y redis-server postgresql libpq-dev - make build-java-no-tests REVISION=develop - python -m pip install --upgrade pip setuptools wheel - make install-python - python -m pip install -qr tests/requirements.txt - - name: run tests - run: su -p postgres -c 'pytest tests/e2e/ --feast-version develop' + python -m pip install pytest + printenv + su -p postgres -c 'printenv' +# - name: install +# run: | +# apt-get update && apt-get install -y redis-server postgresql libpq-dev +# make build-java-no-tests REVISION=develop +# python -m pip install --upgrade pip setuptools wheel +# make install-python +# python -m pip install -qr tests/requirements.txt +# - name: run tests +# run: su -p postgres -c 'pytest tests/e2e/ --feast-version develop' From 8cc59b666c5df7a793af2ba422006a42e63de0af Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 16:45:23 +0800 Subject: [PATCH 27/60] debug Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 2c63e9b1516..112ab18bfe7 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -174,7 +174,9 @@ jobs: apt-get update && apt-get install -y redis-server postgresql libpq-dev python -m pip install pytest printenv + which pytest su -p postgres -c 'printenv' + su -p postgres -c 'which pytest' # - name: install # run: | # apt-get update && apt-get install -y redis-server postgresql libpq-dev From d1f4fb6bcc2f37edfb4f0768b11d6058ec96779d Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 16:51:47 +0800 Subject: [PATCH 28/60] debug Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 112ab18bfe7..96d9713e95b 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -174,9 +174,11 @@ jobs: apt-get update && apt-get install -y redis-server postgresql libpq-dev python -m pip install pytest printenv - which pytest + echo "-m" + su -m postgres -c 'printenv' + echo "-p" su -p postgres -c 'printenv' - su -p postgres -c 'which pytest' + # - name: install # run: | # apt-get update && apt-get install -y redis-server postgresql libpq-dev From 3389dbeead4aafe235dc67c51ca11b59d848e4bf Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 16:57:12 +0800 Subject: [PATCH 29/60] pass PATH Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 96d9713e95b..639e9486733 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -169,22 +169,12 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.6 - - name: debug + - name: install run: | apt-get update && apt-get install -y redis-server postgresql libpq-dev - python -m pip install pytest - printenv - echo "-m" - su -m postgres -c 'printenv' - echo "-p" - su -p postgres -c 'printenv' - -# - name: install -# run: | -# apt-get update && apt-get install -y redis-server postgresql libpq-dev -# make build-java-no-tests REVISION=develop -# python -m pip install --upgrade pip setuptools wheel -# make install-python -# python -m pip install -qr tests/requirements.txt -# - name: run tests -# run: su -p postgres -c 'pytest tests/e2e/ --feast-version develop' + make build-java-no-tests REVISION=develop + python -m pip install --upgrade pip setuptools wheel + make install-python + python -m pip install -qr tests/requirements.txt + - name: run tests + run: su -p postgres -c "PATH=$PATH pytest tests/e2e/ --feast-version develop" From 96a6b509af66cfd0ae5655512a32faee4fef0db8 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Fri, 23 Oct 2020 17:16:09 +0800 Subject: [PATCH 30/60] pass HOME Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 639e9486733..4c585d0a521 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -177,4 +177,4 @@ jobs: make install-python python -m pip install -qr tests/requirements.txt - name: run tests - run: su -p postgres -c "PATH=$PATH pytest tests/e2e/ --feast-version develop" + run: su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ --feast-version develop" From e04cd9fb537d7999572d02c13d4dac426b39121f Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Sun, 25 Oct 2020 14:27:52 +0800 Subject: [PATCH 31/60] external services fixtures Signed-off-by: Oleksii Moskalenko --- infra/scripts/test-docker-compose.sh | 7 +- tests/e2e/conftest.py | 18 ++- tests/e2e/fixtures/__init__.py | 0 tests/e2e/fixtures/base.py | 16 +++ tests/e2e/fixtures/client.py | 77 ++++++++++++ tests/e2e/fixtures/external_services.py | 28 +++++ .../e2e/{fixtures.py => fixtures/services.py} | 116 ++---------------- 7 files changed, 153 insertions(+), 109 deletions(-) create mode 100644 tests/e2e/fixtures/__init__.py create mode 100644 tests/e2e/fixtures/base.py create mode 100644 tests/e2e/fixtures/client.py create mode 100644 tests/e2e/fixtures/external_services.py rename tests/e2e/{fixtures.py => fixtures/services.py} (57%) diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh index 8ace2cdc236..a5730b1e67e 100755 --- a/infra/scripts/test-docker-compose.sh +++ b/infra/scripts/test-docker-compose.sh @@ -57,5 +57,8 @@ export FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .N ${PROJECT_ROOT_DIR}/infra/scripts/wait-for-it.sh ${FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS}:6566 --timeout=120 # Run e2e tests for Redis -docker exec -e FEAST_VERSION=${FEAST_VERSION} feast_jupyter_1 bash \ --c 'cd /feast/tests/e2e && unset GOOGLE_APPLICATION_CREDENTIALS && pytest *.py -m "not bq" --ingestion-jar gs://feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core_url core:6565 --serving_url=online_serving:6566 --kafka_brokers=kafka:9092' +docker exec \ + -e FEAST_VERSION=${FEAST_VERSION} \ + -e DISABLE_SERVICE_FIXTURES=true \ + feast_jupyter_1 bash \ + -c 'cd /feast/tests && python -m pip install -r requirements.txt && pytest e2e/ -m "not bq" --ingestion-jar gs://feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core_url core:6565 --serving_url=online_serving:6566 --kafka_brokers=kafka:9092' diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 13b457280c1..94bb9097ca0 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,14 +1,14 @@ +import os + import pytest def pytest_addoption(parser): parser.addoption("--core_url", action="store", default="localhost:6565") parser.addoption("--serving_url", action="store", default="localhost:6566") - parser.addoption("--allow_dirty", action="store", default="False") parser.addoption( "--gcs_path", action="store", default="gs://feast-templocation-kf-feast/" ) - parser.addoption("--enable_auth", action="store", default="False") parser.addoption("--kafka_brokers", action="store", default="localhost:9092") parser.addoption("--env", action="store", help="local|aws|gcloud", default="local") @@ -20,7 +20,6 @@ def pytest_addoption(parser): parser.addoption("--dataproc-project", action="store") parser.addoption("--ingestion-jar", action="store") parser.addoption("--redis-url", action="store", default="localhost:6379") - parser.addoption("--feast-version", action="store") @@ -38,4 +37,15 @@ def pytest_runtest_setup(item): pytest.xfail("previous test failed (%s)" % previousfailed.name) -from .fixtures import * # noqa +from .fixtures.base import project_root, project_version # noqa +from .fixtures.client import ( # noqa + feast_client, + global_staging_path, + ingestion_job_jar, + local_staging_path, +) + +if not os.environ.get("DISABLE_SERVICE_FIXTURES"): + from .fixtures.services import * # noqa +else: + from .fixtures.external_services import * # type: ignore # noqa diff --git a/tests/e2e/fixtures/__init__.py b/tests/e2e/fixtures/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/fixtures/base.py b/tests/e2e/fixtures/base.py new file mode 100644 index 00000000000..68b9be5a4da --- /dev/null +++ b/tests/e2e/fixtures/base.py @@ -0,0 +1,16 @@ +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="session") +def project_root(): + return Path(__file__).parent.parent.parent.parent + + +@pytest.fixture(scope="session") +def project_version(pytestconfig): + if pytestconfig.getoption("feast_version"): + return pytestconfig.getoption("feast_version") + + return "0.8-SNAPSHOT" diff --git a/tests/e2e/fixtures/client.py b/tests/e2e/fixtures/client.py new file mode 100644 index 00000000000..4b12eb1c32a --- /dev/null +++ b/tests/e2e/fixtures/client.py @@ -0,0 +1,77 @@ +import os +import tempfile +import uuid +from typing import Tuple + +import pyspark +import pytest +from pytest_redis.executor import RedisExecutor + +from feast import Client + + +@pytest.fixture(scope="session") +def feast_client( + pytestconfig, + ingestion_job_jar, + redis_server: RedisExecutor, + feast_core: Tuple[str, int], + feast_serving: Tuple[str, int], + global_staging_path, +): + if pytestconfig.getoption("env") == "local": + return Client( + core_url=f"{feast_core[0]}:{feast_core[1]}", + serving_url=f"{feast_serving[0]}:{feast_serving[1]}", + spark_launcher="standalone", + spark_standalone_master="local", + spark_home=os.getenv("SPARK_HOME") or os.path.dirname(pyspark.__file__), + spark_ingestion_jar=ingestion_job_jar, + redis_host=redis_server.host, + redis_port=redis_server.port, + historical_feature_output_location=os.path.join( + global_staging_path, "historical_output" + ), + ) + + if pytestconfig.getoption("env") == "gcloud": + return Client( + core_url=f"{feast_core[0]}:{feast_core[1]}", + serving_url=f"{feast_serving[0]}:{feast_serving[1]}", + spark_launcher="dataproc", + dataproc_cluster_name=pytestconfig.getoption("dataproc_cluster_name"), + dataproc_project=pytestconfig.getoption("dataproc_project"), + dataproc_region=pytestconfig.getoption("dataproc_region"), + dataproc_staging_location=os.path.join( + pytestconfig.getoption("staging_path"), "dataproc" + ), + spark_ingestion_jar=ingestion_job_jar, + ) + + +@pytest.fixture(scope="session") +def global_staging_path(pytestconfig): + if pytestconfig.getoption("env") == "local": + tmp_path = tempfile.mkdtemp() + return f"file://{tmp_path}" + + staging_path = pytestconfig.getoption("staging_path") + return os.path.join(staging_path, str(uuid.uuid4())) + + +@pytest.fixture(scope="function") +def local_staging_path(global_staging_path): + return os.path.join(global_staging_path, str(uuid.uuid4())) + + +@pytest.fixture(scope="session") +def ingestion_job_jar(pytestconfig, project_root, project_version): + default_path = ( + project_root + / "spark" + / "ingestion" + / "target" + / f"feast-ingestion-spark-{project_version}.jar" + ) + + return pytestconfig.getoption("ingestion_jar") or f"file://{default_path}" diff --git a/tests/e2e/fixtures/external_services.py b/tests/e2e/fixtures/external_services.py new file mode 100644 index 00000000000..6d115c64f38 --- /dev/null +++ b/tests/e2e/fixtures/external_services.py @@ -0,0 +1,28 @@ +import pytest +from pytest_redis.executor import NoopRedis + +__all__ = ("feast_core", "feast_serving", "redis_server", "kafka_server") + + +@pytest.fixture +def redis_server(pytestconfig): + host, port = pytestconfig.getoption("redis_url").split(":") + return NoopRedis(host, port) + + +@pytest.fixture +def feast_core(pytestconfig): + host, port = pytestconfig.getoption("core_url").split(":") + return host, port + + +@pytest.fixture +def feast_serving(pytestconfig): + host, port = pytestconfig.getoption("serving_url").split(":") + return host, port + + +@pytest.fixture +def kafka_server(pytestconfig): + host, port = pytestconfig.getoption("kafka_brokers").split(":") + return host, port diff --git a/tests/e2e/fixtures.py b/tests/e2e/fixtures/services.py similarity index 57% rename from tests/e2e/fixtures.py rename to tests/e2e/fixtures/services.py index 5b17c22610a..4d80487528f 100644 --- a/tests/e2e/fixtures.py +++ b/tests/e2e/fixtures/services.py @@ -5,11 +5,7 @@ import subprocess import tempfile import time -import uuid -from pathlib import Path -from typing import Dict, Any -import pyspark import pytest import requests import yaml @@ -19,38 +15,18 @@ from pytest_redis import factories as redis_factories from pytest_redis.executor import RedisExecutor -from feast import Client - __all__ = ( - "project_root", - "project_version", - "feast_client", - "feast_core", - "feast_serving", - "ingestion_job_jar", - "global_staging_path", - "local_staging_path", - "redis_server", - "postgres_server", "kafka_server", + "kafka_port", "zookeeper_server", + "postgres_server", + "redis_server", + "feast_core", + "feast_serving", "enable_auth", ) -@pytest.fixture(scope="session") -def project_root(): - return Path(__file__).parent.parent.parent - - -@pytest.fixture(scope="session") -def project_version(pytestconfig): - if pytestconfig.getoption("feast_version"): - return pytestconfig.getoption("feast_version") - - return "0.8-SNAPSHOT" - - def download_kafka(version="2.12-2.6.0"): r = requests.get(f"https://downloads.apache.org/kafka/2.6.0/kafka_{version}.tgz") temp_dir = pathlib.Path(tempfile.mkdtemp()) @@ -132,7 +108,7 @@ def feast_core( ], ) _wait_port_open(6565) - yield + yield "localhost", 6565 process.terminate() @@ -171,83 +147,17 @@ def feast_serving( ], ) _wait_port_open(6566) - yield + yield "localhost", 6566 process.terminate() -@pytest.fixture(scope="session") -def ingestion_job_jar(pytestconfig, project_root, project_version): - default_path = ( - project_root - / "spark" - / "ingestion" - / "target" - / f"feast-ingestion-spark-{project_version}.jar" - ) - - return pytestconfig.getoption("ingestion_jar") or f"file://{default_path}" - - -@pytest.fixture(scope="session") -def feast_client( - pytestconfig, - ingestion_job_jar, - redis_server: RedisExecutor, - feast_core, - feast_serving, - global_staging_path, -): - if pytestconfig.getoption("env") == "local": - return Client( - core_url=pytestconfig.getoption("core_url"), - serving_url=pytestconfig.getoption("serving_url"), - spark_launcher="standalone", - spark_standalone_master="local", - spark_home=os.getenv("SPARK_HOME") or os.path.dirname(pyspark.__file__), - spark_ingestion_jar=ingestion_job_jar, - redis_host=redis_server.host, - redis_port=redis_server.port, - historical_feature_output_location=os.path.join( - global_staging_path, "historical_output" - ), - ) - - if pytestconfig.getoption("env") == "gcloud": - return Client( - core_url=pytestconfig.getoption("core_url"), - serving_url=pytestconfig.getoption("serving_url"), - spark_launcher="dataproc", - dataproc_cluster_name=pytestconfig.getoption("dataproc_cluster_name"), - dataproc_project=pytestconfig.getoption("dataproc_project"), - dataproc_region=pytestconfig.getoption("dataproc_region"), - dataproc_staging_location=os.path.join( - pytestconfig.getoption("staging_path"), "dataproc" - ), - spark_ingestion_jar=ingestion_job_jar, - ) - - -@pytest.fixture(scope="session") -def global_staging_path(pytestconfig): - if pytestconfig.getoption("env") == "local": - tmp_path = tempfile.mkdtemp() - return f"file://{tmp_path}" - - staging_path = pytestconfig.getoption("staging_path") - return os.path.join(staging_path, str(uuid.uuid4())) - - -@pytest.fixture(scope="function") -def local_staging_path(global_staging_path): - return os.path.join(global_staging_path, str(uuid.uuid4())) - +@pytest.fixture +def kafka_server(kafka_port): + _, port = kafka_port + return "localhost", port -if not os.environ.get('POSTGRES_HOST'): - postgres_server = pg_factories.postgresql_proc(password="password") -else: - postgres_server = pg_factories.postgresql_noproc(host=os.environ['POSTGRES_HOST'], - port=os.environ['POSTGRES_PORT']) +postgres_server = pg_factories.postgresql_proc(password="password") redis_server = redis_factories.redis_proc(executable=shutil.which("redis-server")) KAFKA_BIN = download_kafka() @@ -259,7 +169,7 @@ def local_staging_path(global_staging_path): maxClientCnxns=0 admin.enableServer=false""", ) -kafka_server = make_kafka_server( +kafka_port = make_kafka_server( kafka_bin=str(KAFKA_BIN / "kafka-server-start.sh"), zookeeper_fixture_name="zookeeper_server", ) From e07f713178c1dc12903477318aea74e2413b8aaa Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Sun, 25 Oct 2020 15:04:10 +0800 Subject: [PATCH 32/60] gcp tests Signed-off-by: Oleksii Moskalenko --- infra/scripts/test-docker-compose.sh | 2 +- tests/e2e/fixtures/client.py | 13 +++++++++---- tests/requirements.txt | 7 +------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh index a5730b1e67e..d46009532dd 100755 --- a/infra/scripts/test-docker-compose.sh +++ b/infra/scripts/test-docker-compose.sh @@ -61,4 +61,4 @@ docker exec \ -e FEAST_VERSION=${FEAST_VERSION} \ -e DISABLE_SERVICE_FIXTURES=true \ feast_jupyter_1 bash \ - -c 'cd /feast/tests && python -m pip install -r requirements.txt && pytest e2e/ -m "not bq" --ingestion-jar gs://feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core_url core:6565 --serving_url=online_serving:6566 --kafka_brokers=kafka:9092' + -c 'cd /feast/tests && python -m pip install -r requirements.txt && pytest e2e/ -m "not bq" --ingestion-jar gs://feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --kafka_brokers kafka:9092' diff --git a/tests/e2e/fixtures/client.py b/tests/e2e/fixtures/client.py index 4b12eb1c32a..d33ceaa368c 100644 --- a/tests/e2e/fixtures/client.py +++ b/tests/e2e/fixtures/client.py @@ -10,14 +10,14 @@ from feast import Client -@pytest.fixture(scope="session") +@pytest.fixture def feast_client( pytestconfig, ingestion_job_jar, redis_server: RedisExecutor, feast_core: Tuple[str, int], feast_serving: Tuple[str, int], - global_staging_path, + local_staging_path, ): if pytestconfig.getoption("env") == "local": return Client( @@ -30,7 +30,7 @@ def feast_client( redis_host=redis_server.host, redis_port=redis_server.port, historical_feature_output_location=os.path.join( - global_staging_path, "historical_output" + local_staging_path, "historical_output" ), ) @@ -43,9 +43,14 @@ def feast_client( dataproc_project=pytestconfig.getoption("dataproc_project"), dataproc_region=pytestconfig.getoption("dataproc_region"), dataproc_staging_location=os.path.join( - pytestconfig.getoption("staging_path"), "dataproc" + local_staging_path, "dataproc" ), spark_ingestion_jar=ingestion_job_jar, + redis_host=pytestconfig.getoption("redis_url").split(":")[0], + redis_port=pytestconfig.getoption("redis_url").split(":")[1], + historical_feature_output_location=os.path.join( + local_staging_path, "historical_output" + ) ) diff --git a/tests/requirements.txt b/tests/requirements.txt index 2fee6adf67d..4f7ea6e9c26 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,8 +1,4 @@ mock==2.0.0 -numpy==1.16.4 -pandas~=1.0.0 -pandavro==1.5.* -pyspark==2.4.2 pytest==6.0.0 pytest-benchmark==3.2.2 pytest-mock==1.10.4 @@ -13,5 +9,4 @@ pytest-postgresql==2.5.1 pytest-redis==2.0.0 pytest-kafka==0.4.0 deepdiff==4.3.2 -confluent_kafka -avro==1.10.0 \ No newline at end of file +confluent_kafka \ No newline at end of file From dbf1ad1b6f58523f68bc3abb71193bb0f62057b0 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Sun, 25 Oct 2020 15:29:04 +0800 Subject: [PATCH 33/60] fix test requirements install Signed-off-by: Oleksii Moskalenko --- sdk/python/requirements-ci.txt | 5 -- tests/e2e/conftest.py | 2 + tests/e2e/fixtures/client.py | 2 +- tests/e2e/fixtures/feast_services.py | 130 +++++++++++++++++++++++++++ tests/e2e/fixtures/services.py | 117 +----------------------- tests/requirements.txt | 5 +- 6 files changed, 138 insertions(+), 123 deletions(-) create mode 100644 tests/e2e/fixtures/feast_services.py diff --git a/sdk/python/requirements-ci.txt b/sdk/python/requirements-ci.txt index 2b0b87bd814..f2b859c6400 100644 --- a/sdk/python/requirements-ci.txt +++ b/sdk/python/requirements-ci.txt @@ -4,11 +4,6 @@ black==19.10b0 isort>=5 grpcio-tools mypy-protobuf -pytest -pytest-lazy-fixture==0.6.3 -pytest-mock -pytest-timeout -pytest-ordering==0.6.* pyspark==2.4.2 pandas~=1.0.0 mock==2.0.0 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 94bb9097ca0..b33c6dbfff7 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -47,5 +47,7 @@ def pytest_runtest_setup(item): if not os.environ.get("DISABLE_SERVICE_FIXTURES"): from .fixtures.services import * # noqa + from .fixtures.feast_services import * # type: ignore # noqa else: from .fixtures.external_services import * # type: ignore # noqa + diff --git a/tests/e2e/fixtures/client.py b/tests/e2e/fixtures/client.py index d33ceaa368c..9da75b9fe56 100644 --- a/tests/e2e/fixtures/client.py +++ b/tests/e2e/fixtures/client.py @@ -42,7 +42,7 @@ def feast_client( dataproc_cluster_name=pytestconfig.getoption("dataproc_cluster_name"), dataproc_project=pytestconfig.getoption("dataproc_project"), dataproc_region=pytestconfig.getoption("dataproc_region"), - dataproc_staging_location=os.path.join( + spark_staging_location=os.path.join( local_staging_path, "dataproc" ), spark_ingestion_jar=ingestion_job_jar, diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py new file mode 100644 index 00000000000..b28deefacb4 --- /dev/null +++ b/tests/e2e/fixtures/feast_services.py @@ -0,0 +1,130 @@ +import os +import shutil +import socket +import subprocess +import tempfile +import time + +import pytest +import yaml +from pytest_postgresql.executor import PostgreSQLExecutor +from pytest_redis.executor import RedisExecutor + + +__all__ = ( + "feast_core", + "feast_serving", + "enable_auth", +) + + +def _start_jar(jar, options=None) -> subprocess.Popen: + if not os.path.isfile(jar): + raise ValueError(f"{jar} doesn't exist") + + cmd = [shutil.which("java"), "-jar", jar] + if options: + cmd.extend(options) + + return subprocess.Popen(cmd) # type: ignore + + +def _wait_port_open(port, max_wait=60): + print(f"Waiting for port {port}") + start = time.time() + + while True: + try: + socket.create_connection(("localhost", port), timeout=1) + except OSError: + if time.time() - start > max_wait: + raise + + time.sleep(1) + else: + return + + +@pytest.fixture( + scope="session", params=[pytest.param(True, marks=pytest.mark.skip), False] +) +def enable_auth(request): + return request.param + + +@pytest.fixture(scope="session") +def feast_core( + project_root, project_version, enable_auth, postgres_server: PostgreSQLExecutor +): + jar = str( + project_root / "core" / "target" / f"feast-core-{project_version}-exec.jar" + ) + config = dict( + feast=dict( + security=dict( + enabled=enable_auth, + provider="jwt", + options=dict( + jwkEndpointURI="https://www.googleapis.com/oauth2/v3/certs" + ), + ) + ), + spring=dict( + datasource=dict( + url=f"jdbc:postgresql://{postgres_server.host}:{postgres_server.port}/postgres" + ) + ), + ) + + with tempfile.NamedTemporaryFile(suffix=".yaml", mode="w+") as config_file: + yaml.dump(config, config_file) + config_file.flush() + + process = _start_jar( + jar, + [ + f"--spring.config.location=classpath:/application.yml,file://{config_file.name}" + ], + ) + _wait_port_open(6565) + yield "localhost", 6565 + process.terminate() + + +@pytest.fixture(scope="session") +def feast_serving( + project_root, project_version, enable_auth, redis_server: RedisExecutor +): + jar = str( + project_root + / "serving" + / "target" + / f"feast-serving-{project_version}-exec.jar" + ) + config = dict( + feast=dict( + stores=[ + dict( + name="online", + type="REDIS", + config=dict(host=redis_server.host, port=redis_server.port), + ) + ], + coreAuthentication=dict(enabled=enable_auth, provider="google"), + security=dict(authentication=dict(enabled=enable_auth, provider="jwt")), + ) + ) + + with tempfile.NamedTemporaryFile(suffix=".yaml", mode="w+") as config_file: + yaml.dump(config, config_file) + config_file.flush() + + process = _start_jar( + jar, + [ + f"--spring.config.location=classpath:/application.yml,file://{config_file.name}" + ], + ) + _wait_port_open(6566) + yield "localhost", 6566 + process.terminate() diff --git a/tests/e2e/fixtures/services.py b/tests/e2e/fixtures/services.py index 4d80487528f..d2454f570d6 100644 --- a/tests/e2e/fixtures/services.py +++ b/tests/e2e/fixtures/services.py @@ -20,10 +20,7 @@ "kafka_port", "zookeeper_server", "postgres_server", - "redis_server", - "feast_core", - "feast_serving", - "enable_auth", + "redis_server" ) @@ -39,118 +36,6 @@ def download_kafka(version="2.12-2.6.0"): return temp_dir / f"kafka_{version}" / "bin" -def _start_jar(jar, options=None) -> subprocess.Popen: - if not os.path.isfile(jar): - raise ValueError(f"{jar} doesn't exist") - - cmd = [shutil.which("java"), "-jar", jar] - if options: - cmd.extend(options) - - return subprocess.Popen(cmd) # type: ignore - - -def _wait_port_open(port, max_wait=60): - print(f"Waiting for port {port}") - start = time.time() - - while True: - try: - socket.create_connection(("localhost", port), timeout=1) - except OSError: - if time.time() - start > max_wait: - raise - - time.sleep(1) - else: - return - - -@pytest.fixture( - scope="session", params=[pytest.param(True, marks=pytest.mark.skip), False] -) -def enable_auth(request): - return request.param - - -@pytest.fixture(scope="session") -def feast_core( - project_root, project_version, enable_auth, postgres_server: PostgreSQLExecutor -): - jar = str( - project_root / "core" / "target" / f"feast-core-{project_version}-exec.jar" - ) - config = dict( - feast=dict( - security=dict( - enabled=enable_auth, - provider="jwt", - options=dict( - jwkEndpointURI="https://www.googleapis.com/oauth2/v3/certs" - ), - ) - ), - spring=dict( - datasource=dict( - url=f"jdbc:postgresql://{postgres_server.host}:{postgres_server.port}/postgres" - ) - ), - ) - - with tempfile.NamedTemporaryFile(suffix=".yaml", mode="w+") as config_file: - yaml.dump(config, config_file) - config_file.flush() - - process = _start_jar( - jar, - [ - f"--spring.config.location=classpath:/application.yml,file://{config_file.name}" - ], - ) - _wait_port_open(6565) - yield "localhost", 6565 - process.terminate() - - -@pytest.fixture(scope="session") -def feast_serving( - project_root, project_version, enable_auth, redis_server: RedisExecutor -): - jar = str( - project_root - / "serving" - / "target" - / f"feast-serving-{project_version}-exec.jar" - ) - config = dict( - feast=dict( - stores=[ - dict( - name="online", - type="REDIS", - config=dict(host=redis_server.host, port=redis_server.port), - ) - ], - coreAuthentication=dict(enabled=enable_auth, provider="google"), - security=dict(authentication=dict(enabled=enable_auth, provider="jwt")), - ) - ) - - with tempfile.NamedTemporaryFile(suffix=".yaml", mode="w+") as config_file: - yaml.dump(config, config_file) - config_file.flush() - - process = _start_jar( - jar, - [ - f"--spring.config.location=classpath:/application.yml,file://{config_file.name}" - ], - ) - _wait_port_open(6566) - yield "localhost", 6566 - process.terminate() - - @pytest.fixture def kafka_server(kafka_port): _, port = kafka_port diff --git a/tests/requirements.txt b/tests/requirements.txt index 4f7ea6e9c26..52c56aeea0c 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,8 @@ -mock==2.0.0 pytest==6.0.0 +pytest-lazy-fixture==0.6.3 +pytest-mock +pytest-timeout +pytest-ordering==0.6.* pytest-benchmark==3.2.2 pytest-mock==1.10.4 pytest-timeout==1.3.3 From 93e93934fbcf4169074ed896a3c403f875a83ac2 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Sun, 25 Oct 2020 15:44:13 +0800 Subject: [PATCH 34/60] fixing tests requirements Signed-off-by: Oleksii Moskalenko --- .../pyspark/launchers/gcloud/dataproc.py | 21 +++++++++++++------ tests/requirements.txt | 3 +-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py index 8777fe9f383..e5a9ba1129e 100644 --- a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py +++ b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py @@ -130,21 +130,30 @@ def _stage_files(self, pyspark_script: str, job_id: str) -> str: blob_path = os.path.join( self.remote_path, job_id, os.path.basename(pyspark_script), ) - staging_client.upload_file(blob_path, self.staging_bucket, pyspark_script) + staging_client.upload_file(pyspark_script, self.staging_bucket, blob_path) return f"gs://{self.staging_bucket}/{blob_path}" def dataproc_submit(self, job_params: SparkJobParameters) -> Operation: local_job_id = str(uuid.uuid4()) - pyspark_gcs = self._stage_files(job_params.get_main_file_path(), local_job_id) + main_file_uri = self._stage_files(job_params.get_main_file_path(), local_job_id) job_config = { "reference": {"job_id": local_job_id}, "placement": {"cluster_name": self.cluster_name}, - "pyspark_job": { - "main_python_file_uri": pyspark_gcs, - "args": job_params.get_arguments(), - }, } + if job_params.get_class_name(): + job_config.update({ + "main_jar_file_uri": main_file_uri, + "main_class": job_params.get_class_name(), + "args": job_params.get_arguments() + }) + else: + job_config.update({ + "pyspark_job": { + "main_python_file_uri": main_file_uri, + "args": job_params.get_arguments(), + } + }) return self.job_client.submit_job_as_operation( request={ "project_id": self.project_id, diff --git a/tests/requirements.txt b/tests/requirements.txt index 52c56aeea0c..0b4bcc2bd34 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,7 +1,6 @@ pytest==6.0.0 pytest-lazy-fixture==0.6.3 -pytest-mock -pytest-timeout +pytest-timeout==1.4.2 pytest-ordering==0.6.* pytest-benchmark==3.2.2 pytest-mock==1.10.4 From 2b6597754d571d067c20d0c3a1e82a38d9f7a147 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Sun, 25 Oct 2020 15:56:13 +0800 Subject: [PATCH 35/60] fixing tests requirements Signed-off-by: Oleksii Moskalenko --- .../pyspark/launchers/gcloud/dataproc.py | 18 +++--- tests/e2e/test_historical_features.py | 61 ++++++++----------- tests/requirements.txt | 1 - 3 files changed, 34 insertions(+), 46 deletions(-) diff --git a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py index e5a9ba1129e..a9bf101ed16 100644 --- a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py +++ b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py @@ -93,7 +93,7 @@ class DataprocClusterLauncher(JobLauncher): """ def __init__( - self, cluster_name: str, staging_location: str, region: str, project_id: str, + self, cluster_name: str, staging_location: str, region: str, project_id: str, ): """ Initialize a dataproc job controller client, used internally for job submission and result @@ -143,9 +143,11 @@ def dataproc_submit(self, job_params: SparkJobParameters) -> Operation: } if job_params.get_class_name(): job_config.update({ - "main_jar_file_uri": main_file_uri, - "main_class": job_params.get_class_name(), - "args": job_params.get_arguments() + "spark_job": { + "jar_file_uris": [main_file_uri], + "main_class": job_params.get_class_name(), + "args": job_params.get_arguments() + } }) else: job_config.update({ @@ -163,24 +165,24 @@ def dataproc_submit(self, job_params: SparkJobParameters) -> Operation: ) def historical_feature_retrieval( - self, job_params: RetrievalJobParameters + self, job_params: RetrievalJobParameters ) -> RetrievalJob: return DataprocRetrievalJob( self.dataproc_submit(job_params), job_params.get_destination_path() ) def offline_to_online_ingestion( - self, ingestion_job_params: BatchIngestionJobParameters + self, ingestion_job_params: BatchIngestionJobParameters ) -> BatchIngestionJob: return DataprocBatchIngestionJob(self.dataproc_submit(ingestion_job_params)) def start_stream_to_online_ingestion( - self, ingestion_job_params: StreamIngestionJobParameters + self, ingestion_job_params: StreamIngestionJobParameters ) -> StreamIngestionJob: return DataprocStreamingIngestionJob(self.dataproc_submit(ingestion_job_params)) def stage_dataframe( - self, df, event_timestamp_column: str, created_timestamp_column: str, + self, df, event_timestamp_column: str, created_timestamp_column: str, ): raise NotImplementedError diff --git a/tests/e2e/test_historical_features.py b/tests/e2e/test_historical_features.py index a91983e0848..0d365efa2af 100644 --- a/tests/e2e/test_historical_features.py +++ b/tests/e2e/test_historical_features.py @@ -77,40 +77,27 @@ def test_historical_features(feast_client: Client, local_staging_path: str): } ) - with tempfile.TemporaryDirectory() as tempdir: - df_export_path = os.path.join(tempdir, "customers.parquets") - customer_df.to_parquet(df_export_path) - scheme, _, remote_path, _, _, _ = urlparse(local_staging_path) - staging_client = get_staging_client(scheme) - staging_client.upload_file(df_export_path, None, remote_path) - customer_source = FileSource( - "event_timestamp", - "event_timestamp", - ParquetFormat(), - os.path.join(local_staging_path, os.path.basename(df_export_path)), - ) - - job = feast_client.get_historical_features(feature_refs, customer_source) - output_dir = job.get_output_file_uri() - - _, _, joined_df_destination_path, _, _, _ = urlparse(output_dir) - joined_df = pd.read_parquet(joined_df_destination_path) - - expected_joined_df = pd.DataFrame( - { - "event_timestamp": [retrieval_date for _ in customers] - + [retrieval_outside_max_age_date for _ in customers], - "user_id": customers + customers, - "transactions__daily_transactions": daily_transactions - + [None] * len(customers), - } - ) - - assert_frame_equal( - joined_df.sort_values(by=["user_id", "event_timestamp"]).reset_index( - drop=True - ), - expected_joined_df.sort_values( - by=["user_id", "event_timestamp"] - ).reset_index(drop=True), - ) + job = feast_client.get_historical_features(feature_refs, customer_df) + output_dir = job.get_output_file_uri() + + _, _, joined_df_destination_path, _, _, _ = urlparse(output_dir) + joined_df = pd.read_parquet(joined_df_destination_path) + + expected_joined_df = pd.DataFrame( + { + "event_timestamp": [retrieval_date for _ in customers] + + [retrieval_outside_max_age_date for _ in customers], + "user_id": customers + customers, + "transactions__daily_transactions": daily_transactions + + [None] * len(customers), + } + ) + + assert_frame_equal( + joined_df.sort_values(by=["user_id", "event_timestamp"]).reset_index( + drop=True + ), + expected_joined_df.sort_values( + by=["user_id", "event_timestamp"] + ).reset_index(drop=True), + ) diff --git a/tests/requirements.txt b/tests/requirements.txt index 0b4bcc2bd34..06ef9d8e1ef 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -4,7 +4,6 @@ pytest-timeout==1.4.2 pytest-ordering==0.6.* pytest-benchmark==3.2.2 pytest-mock==1.10.4 -pytest-timeout==1.3.3 pytest-ordering==0.6.* pytest-xdist==2.1.0 pytest-postgresql==2.5.1 From ee6930106ef5148de1bc8c9752850f27390ea7cb Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Sun, 25 Oct 2020 16:10:15 +0800 Subject: [PATCH 36/60] make historical test storage agnostic Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures/client.py | 3 +++ tests/e2e/test_historical_features.py | 21 ++++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/e2e/fixtures/client.py b/tests/e2e/fixtures/client.py index 9da75b9fe56..411f6d5ee03 100644 --- a/tests/e2e/fixtures/client.py +++ b/tests/e2e/fixtures/client.py @@ -29,6 +29,9 @@ def feast_client( spark_ingestion_jar=ingestion_job_jar, redis_host=redis_server.host, redis_port=redis_server.port, + spark_staging_location=os.path.join( + local_staging_path, "spark" + ), historical_feature_output_location=os.path.join( local_staging_path, "historical_output" ), diff --git a/tests/e2e/test_historical_features.py b/tests/e2e/test_historical_features.py index 0d365efa2af..87d0660d151 100644 --- a/tests/e2e/test_historical_features.py +++ b/tests/e2e/test_historical_features.py @@ -1,20 +1,33 @@ import os -import tempfile +import gcsfs from datetime import datetime, timedelta from urllib.parse import urlparse import numpy as np import pandas as pd +from pyarrow import parquet from google.protobuf.duration_pb2 import Duration from pandas._testing import assert_frame_equal from feast import Client, Entity, Feature, FeatureTable, FileSource, ValueType from feast.data_format import ParquetFormat -from feast.staging.storage_client import get_staging_client np.random.seed(0) +def read_parquet(uri): + parsed_uri = urlparse(uri) + if parsed_uri.scheme == "file": + return pd.read_parquet(parsed_uri.path) + elif parsed_uri.scheme == "gs": + fs = gcsfs.GCSFileSystem() + files = ["gs://" + path for path in gcsfs.GCSFileSystem().glob(uri + '/part-*')] + ds = parquet.ParquetDataset(files, filesystem=fs) + return ds.read().to_pandas() + else: + raise ValueError("Unsupported scheme") + + def test_historical_features(feast_client: Client, local_staging_path: str): customer_entity = Entity( name="user_id", description="Customer", value_type=ValueType.INT64 @@ -79,9 +92,7 @@ def test_historical_features(feast_client: Client, local_staging_path: str): job = feast_client.get_historical_features(feature_refs, customer_df) output_dir = job.get_output_file_uri() - - _, _, joined_df_destination_path, _, _, _ = urlparse(output_dir) - joined_df = pd.read_parquet(joined_df_destination_path) + joined_df = read_parquet(output_dir) expected_joined_df = pd.DataFrame( { From 07eaf085dff29df73cb6030ed69f638a03830301 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Sun, 25 Oct 2020 16:24:47 +0800 Subject: [PATCH 37/60] unify test options Signed-off-by: Oleksii Moskalenko --- infra/scripts/test-docker-compose.sh | 2 +- tests/e2e/conftest.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh index d46009532dd..bf644cd7bf3 100755 --- a/infra/scripts/test-docker-compose.sh +++ b/infra/scripts/test-docker-compose.sh @@ -61,4 +61,4 @@ docker exec \ -e FEAST_VERSION=${FEAST_VERSION} \ -e DISABLE_SERVICE_FIXTURES=true \ feast_jupyter_1 bash \ - -c 'cd /feast/tests && python -m pip install -r requirements.txt && pytest e2e/ -m "not bq" --ingestion-jar gs://feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --kafka_brokers kafka:9092' + -c 'cd /feast/tests && python -m pip install -r requirements.txt && pytest e2e/ -m "not bq" --ingestion-jar gs://feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --kafka-brokers kafka:9092' diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index b33c6dbfff7..b8febc90a83 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -4,12 +4,12 @@ def pytest_addoption(parser): - parser.addoption("--core_url", action="store", default="localhost:6565") - parser.addoption("--serving_url", action="store", default="localhost:6566") + parser.addoption("--core-url", action="store", default="localhost:6565") + parser.addoption("--serving-url", action="store", default="localhost:6566") parser.addoption( "--gcs_path", action="store", default="gs://feast-templocation-kf-feast/" ) - parser.addoption("--kafka_brokers", action="store", default="localhost:9092") + parser.addoption("--kafka-brokers", action="store", default="localhost:9092") parser.addoption("--env", action="store", help="local|aws|gcloud", default="local") parser.addoption( From 6eadb6785eb2ccc5d1a1ec6bb05cd989cb2fc984 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Sun, 25 Oct 2020 16:35:10 +0800 Subject: [PATCH 38/60] fix noopredis init Signed-off-by: Oleksii Moskalenko --- .../org/apache/spark/metrics/sink/StatsdSinkWithTags.scala | 2 +- tests/e2e/fixtures/external_services.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/ingestion/src/main/scala/org/apache/spark/metrics/sink/StatsdSinkWithTags.scala b/spark/ingestion/src/main/scala/org/apache/spark/metrics/sink/StatsdSinkWithTags.scala index 5c70da2b86c..458bcc21d0d 100644 --- a/spark/ingestion/src/main/scala/org/apache/spark/metrics/sink/StatsdSinkWithTags.scala +++ b/spark/ingestion/src/main/scala/org/apache/spark/metrics/sink/StatsdSinkWithTags.scala @@ -25,7 +25,7 @@ import org.apache.spark.SecurityManager import org.apache.spark.internal.Logging import org.apache.spark.metrics.MetricsSystem -private[spark] class StatsdSinkWithTags( +class StatsdSinkWithTags( val property: Properties, val registry: MetricRegistry, securityMgr: SecurityManager diff --git a/tests/e2e/fixtures/external_services.py b/tests/e2e/fixtures/external_services.py index 6d115c64f38..5a8913928ca 100644 --- a/tests/e2e/fixtures/external_services.py +++ b/tests/e2e/fixtures/external_services.py @@ -7,7 +7,7 @@ @pytest.fixture def redis_server(pytestconfig): host, port = pytestconfig.getoption("redis_url").split(":") - return NoopRedis(host, port) + return NoopRedis(host, port, None) @pytest.fixture From 54946ac2cc951d2665a2412642e0d4a14f2d4cd3 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Sun, 25 Oct 2020 16:53:53 +0800 Subject: [PATCH 39/60] fix streaming test Signed-off-by: Oleksii Moskalenko --- tests/e2e/test_online_features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index ed53588f8ab..3be7b948a83 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -82,7 +82,7 @@ def test_streaming_ingestion( feast_client: Client, local_staging_path: str, kafka_server ): entity = Entity(name="s2id", description="S2id", value_type=ValueType.INT64,) - kafka_broker = f"localhost:{kafka_server[1]}" + kafka_broker = f"{kafka_server[0]}:{kafka_server[1]}" feature_table = FeatureTable( name="drivers_stream", From 1dd97682a3db5ce3598563e332edbc3ca7de5961 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 10:07:49 +0800 Subject: [PATCH 40/60] gcp test Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 25 +++++++++++++++++++++++++ infra/scripts/kafka-values.yaml | 18 ++++++++++++++++++ infra/scripts/test-docker-compose.sh | 1 + tests/e2e/conftest.py | 10 +++++++--- tests/e2e/fixtures/client.py | 10 +++------- tests/e2e/fixtures/feast_services.py | 4 ++-- tests/e2e/fixtures/services.py | 2 +- tests/e2e/test_historical_features.py | 8 +++----- 8 files changed, 60 insertions(+), 18 deletions(-) create mode 100644 infra/scripts/kafka-values.yaml diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 4c585d0a521..5d1c68d49b1 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -178,3 +178,28 @@ jobs: python -m pip install -qr tests/requirements.txt - name: run tests run: su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ --feast-version develop" + + test-end-to-end-gcp: + runs-on: [self-hosted] + env: + DISABLE_SERVICE_FIXTURES: true + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: '11' + - uses: stCarolas/setup-maven@v3 + with: + maven-version: 3.6.3 + - uses: actions/setup-python@v2 + with: + python-version: 3.6 + - name: install + run: | + apt-get update && apt-get install -y redis-server postgresql libpq-dev + make build-java-no-tests REVISION=develop + python -m pip install --upgrade pip setuptools wheel + make install-python + python -m pip install -qr tests/requirements.txt + - name: run tests + run: su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ --feast-version develop --env=gcloud --dataproc-cluster-name feast-e2e --dataproc-project kf-feast --dataproc-region us-central1 --redis-url 10.155.181.43:6379 --kafka-broker 10.128.0.23:9094" diff --git a/infra/scripts/kafka-values.yaml b/infra/scripts/kafka-values.yaml new file mode 100644 index 00000000000..6e07a0be526 --- /dev/null +++ b/infra/scripts/kafka-values.yaml @@ -0,0 +1,18 @@ +externalAccess: + enabled: true + service: + loadBalancerIPs: + - 10.128.0.23 + annotations: + cloud.google.com/load-balancer-type: Internal + loadBalancerSourceRanges: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + +persistence: + enabled: false + +zookeeper: + persistence: + enabled: false \ No newline at end of file diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh index bf644cd7bf3..f44f6da1cbc 100755 --- a/infra/scripts/test-docker-compose.sh +++ b/infra/scripts/test-docker-compose.sh @@ -60,5 +60,6 @@ ${PROJECT_ROOT_DIR}/infra/scripts/wait-for-it.sh ${FEAST_ONLINE_SERVING_CONTAINE docker exec \ -e FEAST_VERSION=${FEAST_VERSION} \ -e DISABLE_SERVICE_FIXTURES=true \ + -e DISABLE_FEAST_SERVICE_FIXTURES=true \ feast_jupyter_1 bash \ -c 'cd /feast/tests && python -m pip install -r requirements.txt && pytest e2e/ -m "not bq" --ingestion-jar gs://feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --kafka-brokers kafka:9092' diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index b8febc90a83..7f4454ba983 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -46,8 +46,12 @@ def pytest_runtest_setup(item): ) if not os.environ.get("DISABLE_SERVICE_FIXTURES"): - from .fixtures.services import * # noqa - from .fixtures.feast_services import * # type: ignore # noqa + from .fixtures.services import kafka_server, redis_server # noqa else: - from .fixtures.external_services import * # type: ignore # noqa + from .fixtures.external_services import kafka_server, redis_server # noqa +if not os.environ.get('DISABLE_FEAST_SERVICE_FIXTURES'): + from .fixtures.services import postgres_server # noqa + from .fixtures.feast_services import * # type: ignore # noqa +else: + from .fixtures.external_services import feast_core, feast_serving # noqa diff --git a/tests/e2e/fixtures/client.py b/tests/e2e/fixtures/client.py index 411f6d5ee03..366b0aa711b 100644 --- a/tests/e2e/fixtures/client.py +++ b/tests/e2e/fixtures/client.py @@ -29,9 +29,7 @@ def feast_client( spark_ingestion_jar=ingestion_job_jar, redis_host=redis_server.host, redis_port=redis_server.port, - spark_staging_location=os.path.join( - local_staging_path, "spark" - ), + spark_staging_location=os.path.join(local_staging_path, "spark"), historical_feature_output_location=os.path.join( local_staging_path, "historical_output" ), @@ -45,15 +43,13 @@ def feast_client( dataproc_cluster_name=pytestconfig.getoption("dataproc_cluster_name"), dataproc_project=pytestconfig.getoption("dataproc_project"), dataproc_region=pytestconfig.getoption("dataproc_region"), - spark_staging_location=os.path.join( - local_staging_path, "dataproc" - ), + spark_staging_location=os.path.join(local_staging_path, "dataproc"), spark_ingestion_jar=ingestion_job_jar, redis_host=pytestconfig.getoption("redis_url").split(":")[0], redis_port=pytestconfig.getoption("redis_url").split(":")[1], historical_feature_output_location=os.path.join( local_staging_path, "historical_output" - ) + ), ) diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index b28deefacb4..a56961cb298 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -54,7 +54,7 @@ def enable_auth(request): @pytest.fixture(scope="session") def feast_core( - project_root, project_version, enable_auth, postgres_server: PostgreSQLExecutor + project_root, project_version, enable_auth, postgres_server: PostgreSQLExecutor ): jar = str( project_root / "core" / "target" / f"feast-core-{project_version}-exec.jar" @@ -93,7 +93,7 @@ def feast_core( @pytest.fixture(scope="session") def feast_serving( - project_root, project_version, enable_auth, redis_server: RedisExecutor + project_root, project_version, enable_auth, redis_server: RedisExecutor ): jar = str( project_root diff --git a/tests/e2e/fixtures/services.py b/tests/e2e/fixtures/services.py index d2454f570d6..43ff271da53 100644 --- a/tests/e2e/fixtures/services.py +++ b/tests/e2e/fixtures/services.py @@ -20,7 +20,7 @@ "kafka_port", "zookeeper_server", "postgres_server", - "redis_server" + "redis_server", ) diff --git a/tests/e2e/test_historical_features.py b/tests/e2e/test_historical_features.py index 87d0660d151..468a55e2b6f 100644 --- a/tests/e2e/test_historical_features.py +++ b/tests/e2e/test_historical_features.py @@ -21,7 +21,7 @@ def read_parquet(uri): return pd.read_parquet(parsed_uri.path) elif parsed_uri.scheme == "gs": fs = gcsfs.GCSFileSystem() - files = ["gs://" + path for path in gcsfs.GCSFileSystem().glob(uri + '/part-*')] + files = ["gs://" + path for path in gcsfs.GCSFileSystem().glob(uri + "/part-*")] ds = parquet.ParquetDataset(files, filesystem=fs) return ds.read().to_pandas() else: @@ -105,10 +105,8 @@ def test_historical_features(feast_client: Client, local_staging_path: str): ) assert_frame_equal( - joined_df.sort_values(by=["user_id", "event_timestamp"]).reset_index( + joined_df.sort_values(by=["user_id", "event_timestamp"]).reset_index(drop=True), + expected_joined_df.sort_values(by=["user_id", "event_timestamp"]).reset_index( drop=True ), - expected_joined_df.sort_values( - by=["user_id", "event_timestamp"] - ).reset_index(drop=True), ) From e5e983ee057a17ac0d760af9ec3ef072f39f8768 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 10:15:34 +0800 Subject: [PATCH 41/60] lint Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 +- .../pyspark/launchers/gcloud/dataproc.py | 40 ++++++++++--------- tests/e2e/conftest.py | 9 +++-- tests/e2e/fixtures/feast_services.py | 1 - tests/e2e/fixtures/services.py | 7 ---- tests/e2e/test_historical_features.py | 4 +- 6 files changed, 31 insertions(+), 32 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 5d1c68d49b1..4a8bd8aa024 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -202,4 +202,4 @@ jobs: make install-python python -m pip install -qr tests/requirements.txt - name: run tests - run: su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ --feast-version develop --env=gcloud --dataproc-cluster-name feast-e2e --dataproc-project kf-feast --dataproc-region us-central1 --redis-url 10.155.181.43:6379 --kafka-broker 10.128.0.23:9094" + run: su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ --feast-version develop --env=gcloud --dataproc-cluster-name feast-e2e --dataproc-project kf-feast --dataproc-region us-central1 --redis-url 10.155.181.43:6379 --kafka-brokers 10.128.0.23:9094" diff --git a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py index a9bf101ed16..9df0692fef2 100644 --- a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py +++ b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py @@ -1,6 +1,6 @@ import os import uuid -from typing import List, cast +from typing import Any, Dict, List from urllib.parse import urlparse from google.api_core.operation import Operation @@ -93,7 +93,7 @@ class DataprocClusterLauncher(JobLauncher): """ def __init__( - self, cluster_name: str, staging_location: str, region: str, project_id: str, + self, cluster_name: str, staging_location: str, region: str, project_id: str, ): """ Initialize a dataproc job controller client, used internally for job submission and result @@ -137,25 +137,29 @@ def _stage_files(self, pyspark_script: str, job_id: str) -> str: def dataproc_submit(self, job_params: SparkJobParameters) -> Operation: local_job_id = str(uuid.uuid4()) main_file_uri = self._stage_files(job_params.get_main_file_path(), local_job_id) - job_config = { + job_config: Dict[str, Any] = { "reference": {"job_id": local_job_id}, "placement": {"cluster_name": self.cluster_name}, } if job_params.get_class_name(): - job_config.update({ - "spark_job": { - "jar_file_uris": [main_file_uri], - "main_class": job_params.get_class_name(), - "args": job_params.get_arguments() + job_config.update( + { + "spark_job": { + "jar_file_uris": [main_file_uri], + "main_class": job_params.get_class_name(), + "args": job_params.get_arguments(), + } } - }) + ) else: - job_config.update({ - "pyspark_job": { - "main_python_file_uri": main_file_uri, - "args": job_params.get_arguments(), + job_config.update( + { + "pyspark_job": { + "main_python_file_uri": main_file_uri, + "args": job_params.get_arguments(), + } } - }) + ) return self.job_client.submit_job_as_operation( request={ "project_id": self.project_id, @@ -165,24 +169,24 @@ def dataproc_submit(self, job_params: SparkJobParameters) -> Operation: ) def historical_feature_retrieval( - self, job_params: RetrievalJobParameters + self, job_params: RetrievalJobParameters ) -> RetrievalJob: return DataprocRetrievalJob( self.dataproc_submit(job_params), job_params.get_destination_path() ) def offline_to_online_ingestion( - self, ingestion_job_params: BatchIngestionJobParameters + self, ingestion_job_params: BatchIngestionJobParameters ) -> BatchIngestionJob: return DataprocBatchIngestionJob(self.dataproc_submit(ingestion_job_params)) def start_stream_to_online_ingestion( - self, ingestion_job_params: StreamIngestionJobParameters + self, ingestion_job_params: StreamIngestionJobParameters ) -> StreamIngestionJob: return DataprocStreamingIngestionJob(self.dataproc_submit(ingestion_job_params)) def stage_dataframe( - self, df, event_timestamp_column: str, created_timestamp_column: str, + self, df, event_timestamp_column: str, created_timestamp_column: str, ): raise NotImplementedError diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 7f4454ba983..565c5877440 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -50,8 +50,11 @@ def pytest_runtest_setup(item): else: from .fixtures.external_services import kafka_server, redis_server # noqa -if not os.environ.get('DISABLE_FEAST_SERVICE_FIXTURES'): - from .fixtures.services import postgres_server # noqa +if not os.environ.get("DISABLE_FEAST_SERVICE_FIXTURES"): from .fixtures.feast_services import * # type: ignore # noqa + from .fixtures.services import postgres_server # noqa else: - from .fixtures.external_services import feast_core, feast_serving # noqa + from .fixtures.external_services import ( # type: ignore # noqa + feast_core, + feast_serving, + ) diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index a56961cb298..cfc92338d86 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -10,7 +10,6 @@ from pytest_postgresql.executor import PostgreSQLExecutor from pytest_redis.executor import RedisExecutor - __all__ = ( "feast_core", "feast_serving", diff --git a/tests/e2e/fixtures/services.py b/tests/e2e/fixtures/services.py index 43ff271da53..38927a07299 100644 --- a/tests/e2e/fixtures/services.py +++ b/tests/e2e/fixtures/services.py @@ -1,19 +1,12 @@ -import os import pathlib import shutil -import socket -import subprocess import tempfile -import time import pytest import requests -import yaml from pytest_kafka import make_kafka_server, make_zookeeper_process from pytest_postgresql import factories as pg_factories -from pytest_postgresql.executor import PostgreSQLExecutor from pytest_redis import factories as redis_factories -from pytest_redis.executor import RedisExecutor __all__ = ( "kafka_server", diff --git a/tests/e2e/test_historical_features.py b/tests/e2e/test_historical_features.py index 468a55e2b6f..e6e909bb040 100644 --- a/tests/e2e/test_historical_features.py +++ b/tests/e2e/test_historical_features.py @@ -1,13 +1,13 @@ import os -import gcsfs from datetime import datetime, timedelta from urllib.parse import urlparse +import gcsfs import numpy as np import pandas as pd -from pyarrow import parquet from google.protobuf.duration_pb2 import Duration from pandas._testing import assert_frame_equal +from pyarrow import parquet from feast import Client, Entity, Feature, FeatureTable, FileSource, ValueType from feast.data_format import ParquetFormat From adaa91cc83469b3d35aef4af8932b9c3081a849e Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 10:25:13 +0800 Subject: [PATCH 42/60] test scopes Signed-off-by: Oleksii Moskalenko --- tests/e2e/conftest.py | 2 +- tests/e2e/fixtures/external_services.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 565c5877440..62f16cdfb0e 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -46,7 +46,7 @@ def pytest_runtest_setup(item): ) if not os.environ.get("DISABLE_SERVICE_FIXTURES"): - from .fixtures.services import kafka_server, redis_server # noqa + from .fixtures.services import kafka_server, redis_server, kafka_port # noqa else: from .fixtures.external_services import kafka_server, redis_server # noqa diff --git a/tests/e2e/fixtures/external_services.py b/tests/e2e/fixtures/external_services.py index 5a8913928ca..6929c16ec11 100644 --- a/tests/e2e/fixtures/external_services.py +++ b/tests/e2e/fixtures/external_services.py @@ -4,25 +4,25 @@ __all__ = ("feast_core", "feast_serving", "redis_server", "kafka_server") -@pytest.fixture +@pytest.fixture(scope="session") def redis_server(pytestconfig): host, port = pytestconfig.getoption("redis_url").split(":") return NoopRedis(host, port, None) -@pytest.fixture +@pytest.fixture(scope="session") def feast_core(pytestconfig): host, port = pytestconfig.getoption("core_url").split(":") return host, port -@pytest.fixture +@pytest.fixture(scope="session") def feast_serving(pytestconfig): host, port = pytestconfig.getoption("serving_url").split(":") return host, port -@pytest.fixture +@pytest.fixture(scope="session") def kafka_server(pytestconfig): host, port = pytestconfig.getoption("kafka_brokers").split(":") return host, port From 6b16c7ab7749255307a313f7e44ef6f51caf8619 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 10:37:34 +0800 Subject: [PATCH 43/60] add dataproc to ci requiremenets Signed-off-by: Oleksii Moskalenko --- sdk/python/requirements-ci.txt | 3 ++- tests/e2e/conftest.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/python/requirements-ci.txt b/sdk/python/requirements-ci.txt index f2b859c6400..51b87b5b714 100644 --- a/sdk/python/requirements-ci.txt +++ b/sdk/python/requirements-ci.txt @@ -14,4 +14,5 @@ mypy-protobuf avro==1.10.0 confluent_kafka gcsfs -urllib3>=1.25.4 \ No newline at end of file +urllib3>=1.25.4 +google-cloud-dataproc==2.0.2 \ No newline at end of file diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 62f16cdfb0e..ea71a260d27 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -46,7 +46,7 @@ def pytest_runtest_setup(item): ) if not os.environ.get("DISABLE_SERVICE_FIXTURES"): - from .fixtures.services import kafka_server, redis_server, kafka_port # noqa + from .fixtures.services import kafka_port, kafka_server, redis_server # noqa else: from .fixtures.external_services import kafka_server, redis_server # noqa From ce63eca29fd04dd3ff57d4e4ba1d1511c4fb43b7 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 10:40:25 +0800 Subject: [PATCH 44/60] zookeper fixture Signed-off-by: Oleksii Moskalenko --- tests/e2e/conftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index ea71a260d27..f3f272e28f4 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -46,7 +46,12 @@ def pytest_runtest_setup(item): ) if not os.environ.get("DISABLE_SERVICE_FIXTURES"): - from .fixtures.services import kafka_port, kafka_server, redis_server # noqa + from .fixtures.services import ( # noqa + kafka_port, + kafka_server, + redis_server, + zookeeper_server, + ) else: from .fixtures.external_services import kafka_server, redis_server # noqa From d5ab3dc3948964b19576d638d5af43d1c1f58734 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 11:03:17 +0800 Subject: [PATCH 45/60] freeze grpcio Signed-off-by: Oleksii Moskalenko --- sdk/python/requirements-dev.txt | 6 +++--- sdk/python/setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/python/requirements-dev.txt b/sdk/python/requirements-dev.txt index f34c0c89241..edafc4bd281 100644 --- a/sdk/python/requirements-dev.txt +++ b/sdk/python/requirements-dev.txt @@ -7,9 +7,9 @@ google-cloud-dataproc==2.* google-cloud-storage==1.* google-resumable-media>=0.5 googleapis-common-protos==1.* -grpcio==1.* -grpcio-testing==1.* -grpcio-tools +grpcio==1.31.0 +grpcio-testing==1.31.0 +grpcio-tools==1.31.0 numpy mock==2.0.0 pandas~=1.0.0 diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 19e567ecd35..ef59df7c435 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -33,7 +33,7 @@ "google-cloud-core==1.0.*", "googleapis-common-protos==1.*", "google-cloud-bigquery-storage==0.7.*", - "grpcio==1.*", + "grpcio==1.31.0", "pandas~=1.0.0", "pandavro==1.5.*", "protobuf>=3.10", From 6beb7dcef10155dd7297b6698a1dd8e55ebcb204 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 11:06:23 +0800 Subject: [PATCH 46/60] return pytest requirements to ci Signed-off-by: Oleksii Moskalenko --- sdk/python/requirements-ci.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/python/requirements-ci.txt b/sdk/python/requirements-ci.txt index 51b87b5b714..fe8312860f4 100644 --- a/sdk/python/requirements-ci.txt +++ b/sdk/python/requirements-ci.txt @@ -15,4 +15,9 @@ avro==1.10.0 confluent_kafka gcsfs urllib3>=1.25.4 -google-cloud-dataproc==2.0.2 \ No newline at end of file +google-cloud-dataproc==2.0.2 +pytest==6.0.0 +pytest-lazy-fixture==0.6.3 +pytest-timeout==1.4.2 +pytest-ordering==0.6.* +pytest-mock==1.10.4 \ No newline at end of file From 41793b9c649464453f67354251bdb59624e6f41f Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 11:13:58 +0800 Subject: [PATCH 47/60] freeze google sdk Signed-off-by: Oleksii Moskalenko --- sdk/python/requirements-ci.txt | 2 +- sdk/python/requirements-dev.txt | 14 +++++++------- sdk/python/setup.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sdk/python/requirements-ci.txt b/sdk/python/requirements-ci.txt index fe8312860f4..dc7ad8253a8 100644 --- a/sdk/python/requirements-ci.txt +++ b/sdk/python/requirements-ci.txt @@ -2,7 +2,7 @@ cryptography==3.1 flake8 black==19.10b0 isort>=5 -grpcio-tools +grpcio-tools==1.31.0 mypy-protobuf pyspark==2.4.2 pandas~=1.0.0 diff --git a/sdk/python/requirements-dev.txt b/sdk/python/requirements-dev.txt index edafc4bd281..80d29f6d231 100644 --- a/sdk/python/requirements-dev.txt +++ b/sdk/python/requirements-dev.txt @@ -1,12 +1,12 @@ Click==7.* -google-api-core==1.* -google-auth==1.* -google-cloud-bigquery==1.* -google-cloud-bigquery-storage==0.* -google-cloud-dataproc==2.* -google-cloud-storage==1.* +google-api-core==1.22.4 +google-auth==1.22.1 +google-cloud-bigquery==1.18 +google-cloud-bigquery-storage==0.7.0 +google-cloud-dataproc==2.0.2 +google-cloud-storage==1.20.0 google-resumable-media>=0.5 -googleapis-common-protos==1.* +googleapis-common-protos==1.52.0 grpcio==1.31.0 grpcio-testing==1.31.0 grpcio-tools==1.31.0 diff --git a/sdk/python/setup.py b/sdk/python/setup.py index ef59df7c435..b8481f23d5f 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -26,12 +26,12 @@ REQUIRED = [ "Click==7.*", - "google-api-core==1.20.*", + "google-api-core==1.22.4", "google-auth<2.0dev,>=1.14.0", "google-cloud-bigquery==1.18.*", "google-cloud-storage==1.20.*", "google-cloud-core==1.0.*", - "googleapis-common-protos==1.*", + "googleapis-common-protos==1.52.*", "google-cloud-bigquery-storage==0.7.*", "grpcio==1.31.0", "pandas~=1.0.0", From e1dd22e6a3bc9891ec2aa572559be2c565338e06 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 11:50:50 +0800 Subject: [PATCH 48/60] fix dataproc cancel Signed-off-by: Oleksii Moskalenko --- .../pyspark/launchers/gcloud/dataproc.py | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py index 9df0692fef2..f3fa46032a0 100644 --- a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py +++ b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py @@ -1,11 +1,11 @@ import os import uuid -from typing import Any, Dict, List +from functools import partial +from typing import Any, Callable, Dict, List from urllib.parse import urlparse from google.api_core.operation import Operation from google.cloud import dataproc_v1 -from google.cloud.dataproc_v1 import Job as DataprocJob from google.cloud.dataproc_v1 import JobStatus from feast.pyspark.abc import ( @@ -25,29 +25,36 @@ class DataprocJobMixin: - def __init__(self, operation: Operation): + def __init__(self, operation: Operation, cancel_fn: Callable[[], None]): """ :param operation: (google.api.core.operation.Operation): A Future for the spark job result, returned by the dataproc client. """ self._operation = operation + self._cancel_fn = cancel_fn def get_id(self) -> str: return self._operation.metadata.job_id def get_status(self) -> SparkJobStatus: - if self._operation.running(): - return SparkJobStatus.IN_PROGRESS + self._operation._refresh_and_update() - job = cast(DataprocJob, self._operation.result()) - status = cast(JobStatus, job.status) - if status.state == JobStatus.State.DONE: - return SparkJobStatus.COMPLETED + status = self._operation.metadata.status + if status.state == JobStatus.State.ERROR: + return SparkJobStatus.FAILED + elif status.state == JobStatus.State.RUNNING: + return SparkJobStatus.IN_PROGRESS + elif status.state in ( + JobStatus.State.PENDING, + JobStatus.State.SETUP_DONE, + JobStatus.State.STATE_UNSPECIFIED, + ): + return SparkJobStatus.STARTING - return SparkJobStatus.FAILED + return SparkJobStatus.COMPLETED def cancel(self): - self._operation.cancel() + self._cancel_fn() class DataprocRetrievalJob(DataprocJobMixin, RetrievalJob): @@ -55,14 +62,16 @@ class DataprocRetrievalJob(DataprocJobMixin, RetrievalJob): Historical feature retrieval job result for a Dataproc cluster """ - def __init__(self, operation: Operation, output_file_uri: str): + def __init__( + self, operation: Operation, cancel_fn: Callable[[], None], output_file_uri: str + ): """ This is the returned historical feature retrieval job result for DataprocClusterLauncher. Args: output_file_uri (str): Uri to the historical feature retrieval job output file. """ - super().__init__(operation) + super().__init__(operation, cancel_fn) self._output_file_uri = output_file_uri def get_output_file_uri(self, timeout_sec=None): @@ -168,22 +177,33 @@ def dataproc_submit(self, job_params: SparkJobParameters) -> Operation: } ) + def dataproc_cancel(self, job_id): + self.job_client.cancel_job( + project_id=self.project_id, region=self.region, job_id=job_id + ) + def historical_feature_retrieval( self, job_params: RetrievalJobParameters ) -> RetrievalJob: + operation = self.dataproc_submit(job_params) + cancel_fn = partial(self.dataproc_cancel, operation.metadata.job_id) return DataprocRetrievalJob( - self.dataproc_submit(job_params), job_params.get_destination_path() + operation, cancel_fn, job_params.get_destination_path() ) def offline_to_online_ingestion( self, ingestion_job_params: BatchIngestionJobParameters ) -> BatchIngestionJob: - return DataprocBatchIngestionJob(self.dataproc_submit(ingestion_job_params)) + operation = self.dataproc_submit(ingestion_job_params) + cancel_fn = partial(self.dataproc_cancel, operation.metadata.job_id) + return DataprocBatchIngestionJob(operation, cancel_fn) def start_stream_to_online_ingestion( self, ingestion_job_params: StreamIngestionJobParameters ) -> StreamIngestionJob: - return DataprocStreamingIngestionJob(self.dataproc_submit(ingestion_job_params)) + operation = self.dataproc_submit(ingestion_job_params) + cancel_fn = partial(self.dataproc_cancel, operation.metadata.job_id) + return DataprocStreamingIngestionJob(operation, cancel_fn) def stage_dataframe( self, df, event_timestamp_column: str, created_timestamp_column: str, From 26fd91926b5dade523419413da86f1c7ef7c19c8 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 13:11:01 +0800 Subject: [PATCH 49/60] check kafka consumers before publish Signed-off-by: Oleksii Moskalenko --- infra/scripts/setup-common-functions.sh | 130 ------------- .../scripts/test-end-to-end-redis-cluster.sh | 120 ------------ infra/scripts/test-end-to-end.sh | 136 ------------- .../values-end-to-end-batch-dataflow.yaml | 178 ------------------ tests/e2e/test_online_features.py | 19 +- tests/requirements.txt | 2 +- 6 files changed, 12 insertions(+), 573 deletions(-) delete mode 100755 infra/scripts/test-end-to-end-redis-cluster.sh delete mode 100755 infra/scripts/test-end-to-end.sh delete mode 100644 infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml diff --git a/infra/scripts/setup-common-functions.sh b/infra/scripts/setup-common-functions.sh index 40d5b6badf5..a70dacb6d48 100755 --- a/infra/scripts/setup-common-functions.sh +++ b/infra/scripts/setup-common-functions.sh @@ -9,136 +9,6 @@ install_test_tools() { apt-get -y install wget netcat kafkacat build-essential } -install_gcloud_sdk() { - print_banner "Installing Google Cloud SDK" - if [[ ! $(command -v gsutil) ]]; then - CURRENT_DIR=$(dirname "$BASH_SOURCE") - . "${CURRENT_DIR}"/install-google-cloud-sdk.sh - fi - - export GOOGLE_APPLICATION_CREDENTIALS - gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS} -} - -install_and_start_local_redis() { - print_banner "Installing and tarting Redis at localhost:6379" - # Allow starting serving in this Maven Docker image. Default set to not allowed. - echo "exit 0" >/usr/sbin/policy-rc.d - apt-get -y install redis-server >/var/log/redis.install.log - redis-server --daemonize yes - redis-cli ping -} - -install_and_start_local_redis_cluster() { - print_banner "Installing Redis at localhost:6379" - echo "exit 0" >/usr/sbin/policy-rc.d - ${SCRIPTS_DIR}/setup-redis-cluster.sh - redis-cli -c -p 7000 ping -} - -install_and_start_local_postgres() { - print_banner "Installing and starting Postgres at localhost:5432" - apt-get -y install postgresql >/var/log/postgresql.install.log - service postgresql start - # Initialize with database: 'postgres', user: 'postgres', password: 'password' - cat </tmp/update-postgres-role.sh -psql -c "ALTER USER postgres PASSWORD 'password';" -EOF - chmod +x /tmp/update-postgres-role.sh - su -s /bin/bash -c /tmp/update-postgres-role.sh postgres - export PGPASSWORD=password - pg_isready -} - -install_and_start_local_zookeeper_and_kafka() { - print_banner "Installing and starting Zookeeper at localhost:2181 and Kafka at localhost:9092" - wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz - mv kafka_2.12-2.3.0/ /tmp/kafka - - nohup /tmp/kafka/bin/zookeeper-server-start.sh /tmp/kafka/config/zookeeper.properties &>/var/log/zookeeper.log 2>&1 & - ${SCRIPTS_DIR}/wait-for-it.sh localhost:2181 --timeout=20 - tail -n10 /var/log/zookeeper.log - - nohup /tmp/kafka/bin/kafka-server-start.sh /tmp/kafka/config/server.properties &>/var/log/kafka.log 2>&1 & - ${SCRIPTS_DIR}/wait-for-it.sh localhost:9092 --timeout=40 - tail -n10 /var/log/kafka.log - kafkacat -b localhost:9092 -L -} - -build_feast_core_and_serving() { - print_banner "Building Feast Core and Feast Serving" - infra/scripts/download-maven-cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2020-08-19.tar \ - --output-dir /root/ - - # Build jars for Feast - mvn --quiet --batch-mode -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true clean package - - ls -lh core/target/*jar - ls -lh serving/target/*jar - ls -lh job-controller/target/*jar -} - -start_feast_core() { - print_banner "Starting Feast Core" - - if [ -n "$1" ]; then - echo "Custom Spring application.yml location provided: $1" - export CONFIG_ARG="--spring.config.location=classpath:/application.yml,file://$1" - fi - - nohup java -jar core/target/feast-core-$FEAST_BUILD_VERSION-exec.jar $CONFIG_ARG &>/var/log/feast-core.log & - ${SCRIPTS_DIR}/wait-for-it.sh localhost:6565 --timeout=90 - - tail -n10 /var/log/feast-core.log - nc -w2 localhost 6565 /var/log/feast-jobcontroller.log & - ${SCRIPTS_DIR}/wait-for-it.sh localhost:6570 --timeout=90 - - tail -n10 /var/log/feast-jobcontroller.log - nc -w2 localhost 6570 /var/log/feast-serving-online.log & - ${SCRIPTS_DIR}/wait-for-it.sh localhost:6566 --timeout=60 - - tail -n100 /var/log/feast-serving-online.log - nc -w2 localhost 6566 /tmp/jc.warehouse.application.yml -feast: - core-host: localhost - core-port: 6565 - jobs: - polling_interval_milliseconds: 5000 - active_runner: direct - runners: - - name: direct - type: DirectRunner - options: {} -EOF - -start_feast_core -start_feast_jobcontroller /tmp/jc.warehouse.application.yml - -cat < /tmp/serving.online.application.yml -feast: - core-host: localhost - core-grpc-port: 6565 - - active_store: online - - # List of store configurations - stores: - - name: online # Name of the store (referenced by active_store) - type: REDIS_CLUSTER # Type of the store. REDIS, BIGQUERY are available options - config: - # Connection string specifies the IP and ports of Redis instances in Redis cluster - connection_string: "localhost:7000,localhost:7001,localhost:7002,localhost:7003,localhost:7004,localhost:7005" - flush_frequency_seconds: 1 - # Subscriptions indicate which feature tables needs to be retrieved and used to populate this store - subscriptions: - # Wildcards match all options. No filtering is done. - - name: "*" - project: "*" - version: "*" - - tracing: - enabled: false - -spring: - main: - web-environment: false - -EOF - -start_feast_serving /tmp/serving.online.application.yml - -install_python_with_miniconda_and_feast_sdk - -print_banner "Running end-to-end tests with pytest at 'tests/e2e'" - -# Default artifact location setting in Prow jobs -LOGS_ARTIFACT_PATH=/logs/artifacts - -ORIGINAL_DIR=$(pwd) -cd tests/e2e - -set +e -CORE_NO=$(nproc --all) -pytest *.py -n ${CORE_NO} --redis-url localhost:7000 \ - --dist=loadscope --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml -TEST_EXIT_CODE=$? - -if [[ ${TEST_EXIT_CODE} != 0 ]]; then - echo "[DEBUG] Printing logs" - ls -ltrh /var/log/feast* - cat /var/log/feast-serving-online.log /var/log/feast-core.log - - echo "[DEBUG] Printing Python packages list" - pip list -fi - -cd ${ORIGINAL_DIR} -exit ${TEST_EXIT_CODE} diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh deleted file mode 100755 index 51b55b17631..00000000000 --- a/infra/scripts/test-end-to-end.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -o pipefail -[[ $1 == "True" ]] && ENABLE_AUTH="true" || ENABLE_AUTH="false" -echo "Authenication enabled : ${ENABLE_AUTH}" - -test -z ${GOOGLE_APPLICATION_CREDENTIALS} && GOOGLE_APPLICATION_CREDENTIALS="/etc/gcloud/service-account.json" -test -z ${SKIP_BUILD_JARS} && SKIP_BUILD_JARS="false" -test -z ${GOOGLE_CLOUD_PROJECT} && GOOGLE_CLOUD_PROJECT="kf-feast" -test -z ${TEMP_BUCKET} && TEMP_BUCKET="feast-templocation-kf-feast" -test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location" - -# Get the current build version using maven (and pom.xml) -export FEAST_BUILD_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) -echo Building version: $FEAST_BUILD_VERSION - -# Get Feast project repository root and scripts directory -export PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel) -export SCRIPTS_DIR=${PROJECT_ROOT_DIR}/infra/scripts - -echo " -This script will run end-to-end tests for Feast Core and Online Serving. - -1. Install Redis as the store for Feast Online Serving. -2. Install Postgres for persisting Feast metadata. -3. Install Kafka and Zookeeper as the Source in Feast. -4. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from - tests/e2e via pytest. -" - -source ${SCRIPTS_DIR}/setup-common-functions.sh - -install_test_tools -install_gcloud_sdk -install_and_start_local_redis -install_and_start_local_postgres -install_and_start_local_zookeeper_and_kafka - -if [[ ${SKIP_BUILD_JARS} != "true" ]]; then - build_feast_core_and_serving -else - echo "[DEBUG] Skipping building jars" -fi - -# Start Feast Core with auth if enabled -cat < /tmp/core.warehouse.application.yml -feast: - security: - authentication: - enabled: true - provider: jwt - options: - jwkEndpointURI: "https://www.googleapis.com/oauth2/v3/certs" - authorization: - enabled: false - provider: none -EOF - -cat < /tmp/jc.warehouse.application.yml -feast: - core-host: localhost - core-port: 6565 - jobs: - polling_interval_milliseconds: 5000 - active_runner: direct - runners: - - name: direct - type: DirectRunner - options: {} -EOF - -cat < /tmp/serving.warehouse.application.yml -feast: - stores: - - name: online - type: REDIS - config: - host: localhost - port: 6379 - flush_frequency_seconds: 1 - subscriptions: - - name: "*" - project: "*" - core-authentication: - enabled: $ENABLE_AUTH - provider: google - security: - authentication: - enabled: $ENABLE_AUTH - provider: jwt - authorization: - enabled: false - provider: none -EOF - -if [[ ${ENABLE_AUTH} = "true" ]]; - then - print_banner "Starting Feast core with auth" - start_feast_core /tmp/core.warehouse.application.yml - print_banner "Starting Feast Serving with auth" - else - print_banner "Starting Feast core without auth" - start_feast_core - print_banner "Starting Feast Serving without auth" -fi - -start_feast_jobcontroller /tmp/jc.warehouse.application.yml -start_feast_serving /tmp/serving.warehouse.application.yml -install_python_with_miniconda_and_feast_sdk - -print_banner "Running end-to-end tests with pytest at 'tests/e2e'" - -# Default artifact location setting in Prow jobs -LOGS_ARTIFACT_PATH=/logs/artifacts - -ORIGINAL_DIR=$(pwd) -cd tests/e2e - -set +e -export GOOGLE_APPLICATION_CREDENTIALS=/etc/gcloud/service-account.json -CORE_NO=$(nproc --all) -pytest *.py -n ${CORE_NO} --dist=loadscope --enable_auth=${ENABLE_AUTH} --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml -TEST_EXIT_CODE=$? - -if [[ ${TEST_EXIT_CODE} != 0 ]]; then - echo "[DEBUG] Printing logs" - ls -ltrh /var/log/feast* - cat /var/log/feast-serving-online.log /var/log/feast-core.log - - echo "[DEBUG] Printing Python packages list" - pip list -fi - -cd ${ORIGINAL_DIR} -exit ${TEST_EXIT_CODE} diff --git a/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml b/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml deleted file mode 100644 index 7eea4d40ef4..00000000000 --- a/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml +++ /dev/null @@ -1,178 +0,0 @@ -feast-core: - # feast-core.enabled -- Flag to install Feast Core - enabled: true - gcpServiceAccount: - enabled: true - postgresql: - existingSecret: feast-postgresql - service: - type: LoadBalancer - image: - tag: $IMAGE_TAG - logLevel: INFO - application-override.yaml: - feast: - stream: - options: - bootstrapServers: $feast_kafka_1_ip:31090 - topic: $FEATURES_TOPIC - -feast-jobcontroller: - enabled: true - gcpServiceAccount: - enabled: true - service: - type: LoadBalancer - image: - tag: $IMAGE_TAG - application-override.yaml: - feast: - stream: - options: - bootstrapServers: $feast_kafka_1_ip:31090 - specsOptions: - specsTopic: $SPECS_TOPIC - specsAckTopic: $SPECS_TOPIC-ack - jobs: - active_runner: dataflow - controller: - consolidate-jobs-per-source: true - jobSelector: - application: feast - tag: $IMAGE_TAG - featureSetSelector: - - project: "*" - name: "*" - whitelisted-stores: - - online - - historical - runners: - - name: dataflow - type: DataflowRunner - options: - project: $GCLOUD_PROJECT - region: $GCLOUD_REGION - workerZone: $GCLOUD_REGION-a - tempLocation: gs://$TEMP_BUCKET/tempLocation - network: $GCLOUD_NETWORK - subnetwork: regions/$GCLOUD_REGION/subnetworks/$GCLOUD_SUBNET - maxNumWorkers: 1 - autoscalingAlgorithm: THROUGHPUT_BASED - usePublicIps: false - workerMachineType: n1-standard-1 - deadLetterTableSpec: $GCLOUD_PROJECT:$DATASET_NAME.deadletter - - metrics: - enabled: true - host: $feast_statsd_ip - -feast-online-serving: - # feast-online-serving.enabled -- Flag to install Feast Online Serving - enabled: true - image: - tag: $IMAGE_TAG - service: - type: LoadBalancer - application-override.yaml: - feast: - active_store: online - - # List of store configurations - stores: - - name: online - type: REDIS - config: - host: $feast_redis_ip - port: 6379 - subscriptions: - - name: "*" - project: "*" - version: "*" - -feast-batch-serving: - # feast-batch-serving.enabled -- Flag to install Feast Batch Serving - enabled: true - image: - tag: $IMAGE_TAG - gcpServiceAccount: - enabled: true - service: - type: LoadBalancer - application-override.yaml: - feast: - active_store: historical - - # List of store configurations - stores: - - name: historical - type: BIGQUERY - config: - project_id: $GCLOUD_PROJECT - dataset_id: $DATASET_NAME - staging_location: gs://$TEMP_BUCKET/stagingLocation - initial_retry_delay_seconds: 3 - total_timeout_seconds: 21600 - write_triggering_frequency_seconds: 1 - subscriptions: - - name: "*" - project: "*" - version: "*" - job_store: - redis_host: $HELM_COMMON_NAME-redis-master - -postgresql: - # postgresql.enabled -- Flag to install Postgresql - enabled: true - existingSecret: feast-postgresql - -kafka: - # kafka.enabled -- Flag to install Kafka - enabled: true - external: - enabled: true - type: LoadBalancer - annotations: - cloud.google.com/load-balancer-type: Internal - loadBalancerSourceRanges: - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - firstListenerPort: 31090 - loadBalancerIP: - - $feast_kafka_1_ip - - $feast_kafka_2_ip - - $feast_kafka_3_ip - configurationOverrides: - "advertised.listeners": |- - EXTERNAL://${LOAD_BALANCER_IP}:31090 - "listener.security.protocol.map": |- - PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - "log.retention.hours": 1 - -redis: - # redis.enabled -- Flag to install Redis - enabled: true - usePassword: false - master: - service: - type: LoadBalancer - loadBalancerIP: $feast_redis_ip - annotations: - cloud.google.com/load-balancer-type: Internal - loadBalancerSourceRanges: - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - -prometheus-statsd-exporter: - # prometheus-statsd-exporter.enabled -- Flag to install StatsD to Prometheus Exporter - enabled: true - service: - type: LoadBalancer - annotations: - cloud.google.com/load-balancer-type: Internal - loadBalancerSourceRanges: - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 - loadBalancerIP: $feast_statsd_ip diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index 3be7b948a83..f3fefc6d086 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -9,7 +9,8 @@ import pandas as pd import pytz from avro.io import BinaryEncoder, DatumWriter -from confluent_kafka import Producer +from kafka.admin import KafkaAdminClient +from kafka.producer import KafkaProducer from feast import ( Client, @@ -112,6 +113,8 @@ def test_streaming_ingestion( lambda: (None, job.get_status() == SparkJobStatus.IN_PROGRESS), 60 ) + wait_retry_backoff(lambda: (None, check_consumer_exist(kafka_broker)), 60) + try: original = generate_data()[["s2id", "unique_drivers", "event_timestamp"]] for record in original.to_dict("records"): @@ -166,12 +169,7 @@ def avro_schema(): def send_avro_record_to_kafka(topic, value, bootstrap_servers, avro_schema_json): value_schema = avro.schema.parse(avro_schema_json) - producer_config = { - "bootstrap.servers": bootstrap_servers, - "request.timeout.ms": "1000", - } - - producer = Producer(producer_config) + producer = KafkaProducer(bootstrap_servers=bootstrap_servers) writer = DatumWriter(value_schema) bytes_writer = io.BytesIO() @@ -180,7 +178,7 @@ def send_avro_record_to_kafka(topic, value, bootstrap_servers, avro_schema_json) writer.write(value, encoder) try: - producer.produce(topic=topic, value=bytes_writer.getvalue()) + producer.send(topic=topic, value=bytes_writer.getvalue()) except Exception as e: print( f"Exception while producing record value - {value} to topic - {topic}: {e}" @@ -189,3 +187,8 @@ def send_avro_record_to_kafka(topic, value, bootstrap_servers, avro_schema_json) print(f"Successfully producing record value - {value} to topic - {topic}") producer.flush() + + +def check_consumer_exist(bootstrap_servers): + admin = KafkaAdminClient(bootstrap_servers=bootstrap_servers) + return bool(admin.list_consumer_groups()) diff --git a/tests/requirements.txt b/tests/requirements.txt index 06ef9d8e1ef..e20fa160d40 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -10,4 +10,4 @@ pytest-postgresql==2.5.1 pytest-redis==2.0.0 pytest-kafka==0.4.0 deepdiff==4.3.2 -confluent_kafka \ No newline at end of file +kafka-python \ No newline at end of file From 84833b3d2b2892e17568d60a5bb003055825ebe3 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 13:18:22 +0800 Subject: [PATCH 50/60] enable tests with auth Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures/feast_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index cfc92338d86..20b091faa24 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -45,7 +45,7 @@ def _wait_port_open(port, max_wait=60): @pytest.fixture( - scope="session", params=[pytest.param(True, marks=pytest.mark.skip), False] + scope="session", params=[True, False] ) def enable_auth(request): return request.param From b07c8a4d64a519c1905760668418cdb0c9a80096 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 14:10:21 +0800 Subject: [PATCH 51/60] wait for job to stop Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures/feast_services.py | 2 +- tests/e2e/test_online_features.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index 20b091faa24..5ae25b21e39 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -92,7 +92,7 @@ def feast_core( @pytest.fixture(scope="session") def feast_serving( - project_root, project_version, enable_auth, redis_server: RedisExecutor + project_root, project_version, enable_auth, redis_server: RedisExecutor, feast_core ): jar = str( project_root diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index f3fefc6d086..901951070f6 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -140,6 +140,9 @@ def get_online_features(): ingested = wait_retry_backoff(get_online_features, 60) finally: job.cancel() + wait_retry_backoff( + lambda: (None, job.get_status() == SparkJobStatus.COMPLETED), 60 + ) pd.testing.assert_frame_equal( ingested[["s2id", "drivers_stream:unique_drivers"]], From 3fbcb6aa98c1adc1f8e3fa6984f6e768d3fc0b22 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 14:52:19 +0800 Subject: [PATCH 52/60] unique kafka topic name per test Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures/feast_services.py | 4 +--- tests/e2e/test_online_features.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index 5ae25b21e39..35e144b1f52 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -44,9 +44,7 @@ def _wait_port_open(port, max_wait=60): return -@pytest.fixture( - scope="session", params=[True, False] -) +@pytest.fixture(scope="session", params=[True, False]) def enable_auth(request): return request.param diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index 901951070f6..959b6623054 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -2,6 +2,7 @@ import json import os import time +import uuid from datetime import datetime, timedelta import avro.schema @@ -84,6 +85,7 @@ def test_streaming_ingestion( ): entity = Entity(name="s2id", description="S2id", value_type=ValueType.INT64,) kafka_broker = f"{kafka_server[0]}:{kafka_server[1]}" + topic_name = f"avro-{uuid.uuid4()}" feature_table = FeatureTable( name="drivers_stream", @@ -100,7 +102,7 @@ def test_streaming_ingestion( "event_timestamp", kafka_broker, AvroFormat(avro_schema()), - topic="avro", + topic=topic_name, ), ) @@ -113,7 +115,7 @@ def test_streaming_ingestion( lambda: (None, job.get_status() == SparkJobStatus.IN_PROGRESS), 60 ) - wait_retry_backoff(lambda: (None, check_consumer_exist(kafka_broker)), 60) + wait_retry_backoff(lambda: (None, check_consumer_exist(kafka_broker, topic_name)), 60) try: original = generate_data()[["s2id", "unique_drivers", "event_timestamp"]] @@ -123,7 +125,7 @@ def test_streaming_ingestion( ) send_avro_record_to_kafka( - "avro", + topic_name, record, bootstrap_servers=kafka_broker, avro_schema_json=avro_schema(), @@ -192,6 +194,9 @@ def send_avro_record_to_kafka(topic, value, bootstrap_servers, avro_schema_json) producer.flush() -def check_consumer_exist(bootstrap_servers): +def check_consumer_exist(bootstrap_servers, topic_name): admin = KafkaAdminClient(bootstrap_servers=bootstrap_servers) - return bool(admin.list_consumer_groups()) + consumer_groups = admin.describe_consumer_groups(group_ids=[group_id + for group_id, _ in admin.list_consumer_groups()]) + subscriptions = [partitions[3] for details in consumer_groups for partitions in details[5]] + return any(topic_name.encode() in subscription for subscription in subscriptions) From 9a18ed504f286a6307ec21e8e02afde0359f9c90 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 16:09:03 +0800 Subject: [PATCH 53/60] upgrade kafka python Signed-off-by: Oleksii Moskalenko --- tests/e2e/test_online_features.py | 8 ++++++-- tests/requirements.txt | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index 959b6623054..e0f9e7c9072 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -198,5 +198,9 @@ def check_consumer_exist(bootstrap_servers, topic_name): admin = KafkaAdminClient(bootstrap_servers=bootstrap_servers) consumer_groups = admin.describe_consumer_groups(group_ids=[group_id for group_id, _ in admin.list_consumer_groups()]) - subscriptions = [partitions[3] for details in consumer_groups for partitions in details[5]] - return any(topic_name.encode() in subscription for subscription in subscriptions) + subscriptions = { + subscription + for group in consumer_groups + for member in group.members + for subscription in member.member_metadata.subscription} + return topic_name in subscriptions diff --git a/tests/requirements.txt b/tests/requirements.txt index e20fa160d40..790432d4719 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -10,4 +10,4 @@ pytest-postgresql==2.5.1 pytest-redis==2.0.0 pytest-kafka==0.4.0 deepdiff==4.3.2 -kafka-python \ No newline at end of file +kafka-python==2.0.2 \ No newline at end of file From e6ef09a980c05cb14eac44cd1cd453043bdb77ee Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Mon, 26 Oct 2020 18:53:31 +0800 Subject: [PATCH 54/60] format Signed-off-by: Oleksii Moskalenko --- tests/e2e/test_online_features.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index e0f9e7c9072..c520a0d954b 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -115,7 +115,9 @@ def test_streaming_ingestion( lambda: (None, job.get_status() == SparkJobStatus.IN_PROGRESS), 60 ) - wait_retry_backoff(lambda: (None, check_consumer_exist(kafka_broker, topic_name)), 60) + wait_retry_backoff( + lambda: (None, check_consumer_exist(kafka_broker, topic_name)), 60 + ) try: original = generate_data()[["s2id", "unique_drivers", "event_timestamp"]] @@ -142,9 +144,6 @@ def get_online_features(): ingested = wait_retry_backoff(get_online_features, 60) finally: job.cancel() - wait_retry_backoff( - lambda: (None, job.get_status() == SparkJobStatus.COMPLETED), 60 - ) pd.testing.assert_frame_equal( ingested[["s2id", "drivers_stream:unique_drivers"]], @@ -196,11 +195,13 @@ def send_avro_record_to_kafka(topic, value, bootstrap_servers, avro_schema_json) def check_consumer_exist(bootstrap_servers, topic_name): admin = KafkaAdminClient(bootstrap_servers=bootstrap_servers) - consumer_groups = admin.describe_consumer_groups(group_ids=[group_id - for group_id, _ in admin.list_consumer_groups()]) + consumer_groups = admin.describe_consumer_groups( + group_ids=[group_id for group_id, _ in admin.list_consumer_groups()] + ) subscriptions = { subscription for group in consumer_groups for member in group.members - for subscription in member.member_metadata.subscription} + for subscription in member.member_metadata.subscription + } return topic_name in subscriptions From c5e39527f13fbe46fda3051431ff451780375fdc Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 27 Oct 2020 09:56:17 +0800 Subject: [PATCH 55/60] use redis cluster in gcp tests Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 6 +++- .../kafka-values.tpl.yaml} | 2 +- .../helm/redis-cluster-values.tpl.yaml | 14 ++++++++ infra/scripts/setup-e2e-env-gcp.sh | 36 +++++++++++++++++++ infra/scripts/setup-redis-cluster.sh | 16 --------- 5 files changed, 56 insertions(+), 18 deletions(-) rename infra/scripts/{kafka-values.yaml => helm/kafka-values.tpl.yaml} (92%) create mode 100644 infra/scripts/helm/redis-cluster-values.tpl.yaml create mode 100755 infra/scripts/setup-e2e-env-gcp.sh delete mode 100755 infra/scripts/setup-redis-cluster.sh diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 4a8bd8aa024..9f8379071bd 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -202,4 +202,8 @@ jobs: make install-python python -m pip install -qr tests/requirements.txt - name: run tests - run: su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ --feast-version develop --env=gcloud --dataproc-cluster-name feast-e2e --dataproc-project kf-feast --dataproc-region us-central1 --redis-url 10.155.181.43:6379 --kafka-brokers 10.128.0.23:9094" + run: > + su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ + --feast-version develop --env=gcloud --dataproc-cluster-name feast-e2e + --dataproc-project kf-feast --dataproc-region us-central1 + --redis-url 10.128.0.105:6379 --kafka-brokers 10.128.0.103:9094" diff --git a/infra/scripts/kafka-values.yaml b/infra/scripts/helm/kafka-values.tpl.yaml similarity index 92% rename from infra/scripts/kafka-values.yaml rename to infra/scripts/helm/kafka-values.tpl.yaml index 6e07a0be526..206323f3377 100644 --- a/infra/scripts/kafka-values.yaml +++ b/infra/scripts/helm/kafka-values.tpl.yaml @@ -2,7 +2,7 @@ externalAccess: enabled: true service: loadBalancerIPs: - - 10.128.0.23 + - $feast_kafka_ip annotations: cloud.google.com/load-balancer-type: Internal loadBalancerSourceRanges: diff --git a/infra/scripts/helm/redis-cluster-values.tpl.yaml b/infra/scripts/helm/redis-cluster-values.tpl.yaml new file mode 100644 index 00000000000..caf45982bce --- /dev/null +++ b/infra/scripts/helm/redis-cluster-values.tpl.yaml @@ -0,0 +1,14 @@ +cluster: + nodes: 3 + externalAccess: + enabled: true + service: + loadBalancerIP: + - $feast_redis_1_ip + - $feast_redis_2_ip + - $feast_redis_3_ip + +persistence: + enabled: false + +usePassword: false \ No newline at end of file diff --git a/infra/scripts/setup-e2e-env-gcp.sh b/infra/scripts/setup-e2e-env-gcp.sh new file mode 100755 index 00000000000..08083d0c8f0 --- /dev/null +++ b/infra/scripts/setup-e2e-env-gcp.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# GCloud, kubectl, helm should be already installed +# And kubernetes cluster already configured + +test -z ${GCLOUD_REGION} && GCLOUD_REGION="us-central1" +test -z ${GCLOUD_NETWORK} && GCLOUD_NETWORK="default" +test -z ${GCLOUD_SUBNET} && GCLOUD_SUBNET="default" + + +feast_kafka_ip_name="feast-kafka" +feast_redis_1_ip_name="feast-redis-1" +feast_redis_2_ip_name="feast-redis-2" +feast_redis_3_ip_name="feast-redis-3" + +helm repo add bitnami https://charts.bitnami.com/bitnami + +gcloud compute addresses create \ + $feast_kafka_ip_name $feast_redis_1_ip_name $feast_redis_2_ip_name $feast_redis_3_ip_name \ + --region ${GCLOUD_REGION} --subnet ${GCLOUD_SUBNET} + +export feast_kafka_ip=$(gcloud compute addresses describe $feast_kafka_ip_name --region=${GCLOUD_REGION} --format "value(address)") +export feast_redis_1_ip=$(gcloud compute addresses describe $feast_redis_1_ip_name --region=${GCLOUD_REGION} --format "value(address)") +export feast_redis_2_ip=$(gcloud compute addresses describe $feast_redis_2_ip_name --region=${GCLOUD_REGION} --format "value(address)") +export feast_redis_3_ip=$(gcloud compute addresses describe $feast_redis_3_ip_name --region=${GCLOUD_REGION} --format "value(address)") + + +envsubst '$feast_kafka_ip' < helm/kafka-values.tpl.yaml > helm/kafka-values.yaml +envsubst '$feast_redis_1_ip,$feast_redis_2_ip,$feast_redis_3_ip' < helm/redis-cluster-values.tpl.yaml > helm/redis-cluster-values.yaml + +helm install e2e-kafka bitnami/kafka \ + --values helm/kafka-values.yaml --namespace infra --create-namespace + +helm install e2e-redis-cluster bitnami/redis-cluster \ + --values helm/redis-cluster-values.yaml --namespace infra \ + --create-namespace \ No newline at end of file diff --git a/infra/scripts/setup-redis-cluster.sh b/infra/scripts/setup-redis-cluster.sh deleted file mode 100755 index a1939705318..00000000000 --- a/infra/scripts/setup-redis-cluster.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -apt-get -y install redis-server > /var/log/redis.install.log - -mkdir 7000 7001 7002 7003 7004 7005 -for i in {0..5} ; do -echo "port 700$i -cluster-enabled yes -cluster-config-file nodes-$i.conf -cluster-node-timeout 5000 -appendonly yes" > 700$i/redis.conf -redis-server 700$i/redis.conf --daemonize yes -done -echo yes | redis-cli --cluster create 127.0.0.1:7000 127.0.0.1:7001 \ -127.0.0.1:7002 127.0.0.1:7003 127.0.0.1:7004 127.0.0.1:7005 \ ---cluster-replicas 1 From ac26fb2e8cb30d1bdaedd3c33d7fd044f2077c07 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 27 Oct 2020 10:19:35 +0800 Subject: [PATCH 56/60] e2e redis internal lb Signed-off-by: Oleksii Moskalenko --- infra/scripts/helm/redis-cluster-values.tpl.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/infra/scripts/helm/redis-cluster-values.tpl.yaml b/infra/scripts/helm/redis-cluster-values.tpl.yaml index caf45982bce..5b152524952 100644 --- a/infra/scripts/helm/redis-cluster-values.tpl.yaml +++ b/infra/scripts/helm/redis-cluster-values.tpl.yaml @@ -1,8 +1,11 @@ cluster: nodes: 3 + replicas: 0 externalAccess: enabled: true service: + annotations: + cloud.google.com/load-balancer-type: Internal loadBalancerIP: - $feast_redis_1_ip - $feast_redis_2_ip From d063045d6f38d0ea402c0358edbea32e5c0bd942 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 27 Oct 2020 11:11:53 +0800 Subject: [PATCH 57/60] pass redis cluster config to serving Signed-off-by: Oleksii Moskalenko --- .github/workflows/complete.yml | 2 +- tests/e2e/conftest.py | 1 + tests/e2e/fixtures/feast_services.py | 29 ++++++++++++++++++++-------- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 9f8379071bd..ba7b110c8e2 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -206,4 +206,4 @@ jobs: su -p postgres -c "PATH=$PATH HOME=/tmp pytest tests/e2e/ --feast-version develop --env=gcloud --dataproc-cluster-name feast-e2e --dataproc-project kf-feast --dataproc-region us-central1 - --redis-url 10.128.0.105:6379 --kafka-brokers 10.128.0.103:9094" + --redis-url 10.128.0.105:6379 --redis-cluster --kafka-brokers 10.128.0.103:9094" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index f3f272e28f4..c96ff7c3be2 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -20,6 +20,7 @@ def pytest_addoption(parser): parser.addoption("--dataproc-project", action="store") parser.addoption("--ingestion-jar", action="store") parser.addoption("--redis-url", action="store", default="localhost:6379") + parser.addoption("--redis-cluster", action="store_true") parser.addoption("--feast-version", action="store") diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index 35e144b1f52..1fa13c5533a 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -4,6 +4,7 @@ import subprocess import tempfile import time +from typing import Any, Dict import pytest import yaml @@ -90,7 +91,12 @@ def feast_core( @pytest.fixture(scope="session") def feast_serving( - project_root, project_version, enable_auth, redis_server: RedisExecutor, feast_core + project_root, + project_version, + enable_auth, + redis_server: RedisExecutor, + feast_core, + pytestconfig, ): jar = str( project_root @@ -98,15 +104,22 @@ def feast_serving( / "target" / f"feast-serving-{project_version}-exec.jar" ) + if pytestconfig.getoption("redis_cluster"): + store: Dict[str, Any] = dict( + name="online", + type="REDIS_CLUSTER", + connection_string=f"{redis_server.host}:{redis_server.port}", + ) + else: + store = dict( + name="online", + type="REDIS", + config=dict(host=redis_server.host, port=redis_server.port), + ) + config = dict( feast=dict( - stores=[ - dict( - name="online", - type="REDIS", - config=dict(host=redis_server.host, port=redis_server.port), - ) - ], + stores=[store], coreAuthentication=dict(enabled=enable_auth, provider="google"), security=dict(authentication=dict(enabled=enable_auth, provider="jwt")), ) From a58ed22db9eaf9e7446a9f5ccc8482a44c388089 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 27 Oct 2020 11:21:46 +0800 Subject: [PATCH 58/60] pass redis cluster config to serving Signed-off-by: Oleksii Moskalenko --- tests/e2e/fixtures/feast_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index 1fa13c5533a..ce7f8546917 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -108,7 +108,7 @@ def feast_serving( store: Dict[str, Any] = dict( name="online", type="REDIS_CLUSTER", - connection_string=f"{redis_server.host}:{redis_server.port}", + config=dict(connection_string=f"{redis_server.host}:{redis_server.port}"), ) else: store = dict( From 031c8bae222f95e19573064da6c3cf067dbb6ee6 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 27 Oct 2020 12:21:57 +0800 Subject: [PATCH 59/60] handle kafka admin incorrect parsing Signed-off-by: Oleksii Moskalenko --- sdk/python/requirements-ci.txt | 1 - sdk/python/requirements-dev.txt | 2 +- tests/e2e/test_online_features.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/python/requirements-ci.txt b/sdk/python/requirements-ci.txt index dc7ad8253a8..258bca8aab4 100644 --- a/sdk/python/requirements-ci.txt +++ b/sdk/python/requirements-ci.txt @@ -12,7 +12,6 @@ moto mypy mypy-protobuf avro==1.10.0 -confluent_kafka gcsfs urllib3>=1.25.4 google-cloud-dataproc==2.0.2 diff --git a/sdk/python/requirements-dev.txt b/sdk/python/requirements-dev.txt index 80d29f6d231..ca845e7f5ae 100644 --- a/sdk/python/requirements-dev.txt +++ b/sdk/python/requirements-dev.txt @@ -28,7 +28,7 @@ toml==0.10.* tqdm==4.* google pandavro==1.5.* -kafka-python==1.* +kafka-python==2.0.2 tabulate==0.8.* isort>=5 mypy diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index c520a0d954b..c5380fae0bc 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -201,7 +201,7 @@ def check_consumer_exist(bootstrap_servers, topic_name): subscriptions = { subscription for group in consumer_groups - for member in group.members + for member in group.members if not isinstance(member.member_metadata, bytes) for subscription in member.member_metadata.subscription } return topic_name in subscriptions From 5f706cf17b5c430aa53729b49baf04482f7c7c5f Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 27 Oct 2020 12:34:25 +0800 Subject: [PATCH 60/60] format Signed-off-by: Oleksii Moskalenko --- tests/e2e/test_online_features.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index c5380fae0bc..cb1ccb69ff9 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -201,7 +201,8 @@ def check_consumer_exist(bootstrap_servers, topic_name): subscriptions = { subscription for group in consumer_groups - for member in group.members if not isinstance(member.member_metadata, bytes) + for member in group.members + if not isinstance(member.member_metadata, bytes) for subscription in member.member_metadata.subscription } return topic_name in subscriptions