Skip to content

Commit c611eb8

Browse files
felixwang9817felixwang9817
authored
chore: Remove gcp requirement for local tests (#2972)
* Remove unused objects Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Switch from bigquery to file sources Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Switch tests using example_feature_repo_1 to use file offline store Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Disable tests that require gcp Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Remove duplicate test Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Remove integration marker Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Fix snowflake config Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Fix import Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Add empty feature repo Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Fix comments Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Add new example feature repo Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Add new feature repo with just feature service Signed-off-by: Felix Wang <wangfelix98@gmail.com> * Move tests from integration to unit Signed-off-by: Felix Wang <wangfelix98@gmail.com>
1 parent 651ce34 commit c611eb8

15 files changed

+298
-276
lines changed

‎Makefile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,10 @@ test-python-integration-local:
7878
-k "not test_apply_entity_integration and \
7979
not test_apply_feature_view_integration and \
8080
not test_apply_data_source_integration and \
81-
not test_lambda_materialization" \
81+
not test_lambda_materialization and \
82+
not test_feature_view_inference_success and \
83+
not test_update_file_data_source_with_inferred_event_timestamp_col and \
84+
not test_nullable_online_store" \
8285
sdk/python/tests \
8386
) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
8487

‎sdk/python/tests/example_repos/empty_feature_repo.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# This example feature repo is deliberately left empty. It should be used for tests that do not need
2+
# any feature views or other objects (for example, a test that checks that a feature service can be
3+
# applied and retrieved correctly).

‎sdk/python/tests/example_repos/example_feature_repo_1.py‎

Lines changed: 10 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,26 @@
11
from datetime import timedelta
22

3-
from feast import BigQuerySource, Entity, FeatureService, FeatureView, Field, PushSource
3+
from feast import Entity, FeatureService, FeatureView, Field, FileSource, PushSource
44
from feast.types import Float32, Int64, String
55

6-
driver_locations_source = BigQuerySource(
7-
table="feast-oss.public.drivers",
8-
timestamp_field="event_timestamp",
9-
created_timestamp_column="created_timestamp",
10-
)
11-
12-
driver_locations_source_query = BigQuerySource(
13-
query="SELECT * from feast-oss.public.drivers",
14-
timestamp_field="event_timestamp",
15-
created_timestamp_column="created_timestamp",
16-
)
6+
# Note that file source paths are not validated, so there doesn't actually need to be any data
7+
# at the paths for these file sources. Since these paths are effectively fake, this example
8+
# feature repo should not be used for historical retrieval.
179

18-
driver_locations_source_query_2 = BigQuerySource(
19-
query="SELECT lat * 2 FROM feast-oss.public.drivers",
10+
driver_locations_source = FileSource(
11+
path="data/driver_locations.parquet",
2012
timestamp_field="event_timestamp",
2113
created_timestamp_column="created_timestamp",
2214
)
2315

24-
customer_profile_source = BigQuerySource(
16+
customer_profile_source = FileSource(
2517
name="customer_profile_source",
26-
table="feast-oss.public.customers",
18+
path="data/customer_profiles.parquet",
2719
timestamp_field="event_timestamp",
2820
)
2921

30-
customer_driver_combined_source = BigQuerySource(
31-
table="feast-oss.public.customer_driver",
22+
customer_driver_combined_source = FileSource(
23+
path="data/customer_driver_combined.parquet",
3224
timestamp_field="event_timestamp",
3325
)
3426

‎sdk/python/tests/example_repos/example_feature_repo_with_driver_stats_feature_view.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
from datetime import timedelta
2+
3+
from feast import Entity, FeatureView, Field, FileSource
4+
from feast.types import Float32, Int32, Int64
5+
6+
driver_hourly_stats = FileSource(
7+
path="data/driver_stats.parquet", # Fake path
8+
timestamp_field="event_timestamp",
9+
created_timestamp_column="created",
10+
)
11+
12+
driver = Entity(
13+
name="driver_id",
14+
description="driver id",
15+
)
16+
17+
driver_hourly_stats_view = FeatureView(
18+
name="driver_hourly_stats",
19+
entities=[driver],
20+
ttl=timedelta(days=1),
21+
schema=[
22+
Field(name="conv_rate", dtype=Float32),
23+
Field(name="acc_rate", dtype=Float32),
24+
Field(name="avg_daily_trips", dtype=Int64),
25+
Field(name="driver_id", dtype=Int32),
26+
],
27+
online=True,
28+
source=driver_hourly_stats,
29+
tags={},
30+
)

‎sdk/python/tests/example_repos/example_feature_repo_with_feature_service.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
from datetime import timedelta
2+
3+
from feast import Entity, FeatureService, FeatureView, Field, FileSource
4+
from feast.types import Float32, Int64, String
5+
6+
driver_locations_source = FileSource(
7+
path="data/driver_locations.parquet",
8+
timestamp_field="event_timestamp",
9+
created_timestamp_column="created_timestamp",
10+
)
11+
12+
driver = Entity(
13+
name="driver", # The name is derived from this argument, not object name.
14+
join_keys=["driver_id"],
15+
description="driver id",
16+
)
17+
18+
driver_locations = FeatureView(
19+
name="driver_locations",
20+
entities=[driver],
21+
ttl=timedelta(days=1),
22+
schema=[
23+
Field(name="lat", dtype=Float32),
24+
Field(name="lon", dtype=String),
25+
Field(name="driver_id", dtype=Int64),
26+
],
27+
online=True,
28+
batch_source=driver_locations_source,
29+
tags={},
30+
)
31+
32+
all_drivers_feature_service = FeatureService(
33+
name="driver_locations_service",
34+
features=[driver_locations],
35+
tags={"release": "production"},
36+
)

‎sdk/python/tests/integration/e2e/test_universal_e2e.py‎

Lines changed: 0 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,8 @@
22

33
import pytest
44

5-
from feast import BigQuerySource, Entity, FeatureView, Field
6-
from feast.feature_service import FeatureService
7-
from feast.types import Float32, String
85
from tests.integration.feature_repos.universal.entities import driver
96
from tests.integration.feature_repos.universal.feature_views import driver_feature_view
10-
from tests.utils.basic_read_write_test import basic_rw_test
11-
from tests.utils.cli_repo_creator import CliRunner, get_example_repo
127
from tests.utils.e2e_test_validation import validate_offline_online_store_consistency
138

149

@@ -32,68 +27,3 @@ def test_e2e_consistency(environment, e2e_data_sources, infer_features):
3227
split_dt = df["ts_1"][4].to_pydatetime() - timedelta(seconds=1)
3328

3429
validate_offline_online_store_consistency(fs, fv, split_dt)
35-
36-
37-
@pytest.mark.integration
38-
def test_partial() -> None:
39-
"""
40-
Add another table to existing repo using partial apply API. Make sure both the table
41-
applied via CLI apply and the new table are passing RW test.
42-
"""
43-
44-
runner = CliRunner()
45-
with runner.local_repo(
46-
get_example_repo("example_feature_repo_1.py"), "bigquery"
47-
) as store:
48-
driver = Entity(name="driver", join_keys=["test"])
49-
50-
driver_locations_source = BigQuerySource(
51-
table="feast-oss.public.drivers",
52-
timestamp_field="event_timestamp",
53-
created_timestamp_column="created_timestamp",
54-
)
55-
56-
driver_locations_100 = FeatureView(
57-
name="driver_locations_100",
58-
entities=[driver],
59-
ttl=timedelta(days=1),
60-
schema=[
61-
Field(name="lat", dtype=Float32),
62-
Field(name="lon", dtype=String),
63-
Field(name="name", dtype=String),
64-
Field(name="test", dtype=String),
65-
],
66-
online=True,
67-
batch_source=driver_locations_source,
68-
tags={},
69-
)
70-
71-
store.apply([driver_locations_100])
72-
73-
basic_rw_test(store, view_name="driver_locations")
74-
basic_rw_test(store, view_name="driver_locations_100")
75-
76-
77-
@pytest.mark.integration
78-
def test_read_pre_applied() -> None:
79-
"""
80-
Read feature values from the FeatureStore using a FeatureService.
81-
"""
82-
runner = CliRunner()
83-
with runner.local_repo(
84-
get_example_repo("example_feature_repo_1.py"), "bigquery"
85-
) as store:
86-
87-
assert len(store.list_feature_services()) == 1
88-
fs = store.get_feature_service("driver_locations_service")
89-
assert len(fs.tags) == 1
90-
assert fs.tags["release"] == "production"
91-
92-
fv = store.get_feature_view("driver_locations")
93-
94-
fs = FeatureService(name="new_feature_service", features=[fv[["lon"]]])
95-
96-
store.apply([fs])
97-
98-
assert len(store.list_feature_services()) == 2
99-
store.get_feature_service("new_feature_service")

‎sdk/python/tests/integration/feature_repos/repo_configuration.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,11 @@
7575

7676
SNOWFLAKE_CONFIG = {
7777
"type": "snowflake.online",
78-
"account": os.environ["SNOWFLAKE_CI_DEPLOYMENT"],
79-
"user": os.environ["SNOWFLAKE_CI_USER"],
80-
"password": os.environ["SNOWFLAKE_CI_PASSWORD"],
81-
"role": os.environ["SNOWFLAKE_CI_ROLE"],
82-
"warehouse": os.environ["SNOWFLAKE_CI_WAREHOUSE"],
78+
"account": os.environ.get("SNOWFLAKE_CI_DEPLOYMENT", ""),
79+
"user": os.environ.get("SNOWFLAKE_CI_USER", ""),
80+
"password": os.environ.get("SNOWFLAKE_CI_PASSWORD", ""),
81+
"role": os.environ.get("SNOWFLAKE_CI_ROLE", ""),
82+
"warehouse": os.environ.get("SNOWFLAKE_CI_WAREHOUSE", ""),
8383
"database": "FEAST",
8484
"schema": "ONLINE",
8585
}

‎sdk/python/tests/integration/registration/test_universal_cli.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ def test_nullable_online_store(test_nullable_online_store) -> None:
159159
repo_config.write_text(dedent(feature_store_yaml))
160160

161161
repo_example = repo_path / "example.py"
162-
repo_example.write_text(get_example_repo("example_feature_repo_1.py"))
162+
repo_example.write_text(get_example_repo("empty_feature_repo.py"))
163163
result = runner.run(["apply"], cwd=repo_path)
164164
assertpy.assert_that(result.returncode).is_equal_to(0)
165165
finally:

‎sdk/python/tests/unit/cli/test_cli_apply_duplicates.py‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,8 @@ def run_simple_apply_test(example_repo_file_name: str, expected_error: bytes):
4949

5050
def test_cli_apply_imported_featureview() -> None:
5151
"""
52-
Test apply feature views with duplicated names and single py file in a feature repo using CLI
52+
Tests that applying a feature view imported from a separate Python file is successful.
5353
"""
54-
5554
with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name:
5655
runner = CliRunner()
5756
# Construct an example repo in a temporary dir
@@ -72,8 +71,11 @@ def test_cli_apply_imported_featureview() -> None:
7271
)
7372
)
7473

74+
# Import feature view from an existing file so it exists in two files.
7575
repo_example = repo_path / "example.py"
76-
repo_example.write_text(get_example_repo("example_feature_repo_2.py"))
76+
repo_example.write_text(
77+
get_example_repo("example_feature_repo_with_driver_stats_feature_view.py")
78+
)
7779
repo_example_2 = repo_path / "example_2.py"
7880
repo_example_2.write_text(
7981
"from example import driver_hourly_stats_view\n"
@@ -92,9 +94,9 @@ def test_cli_apply_imported_featureview() -> None:
9294

9395
def test_cli_apply_imported_featureview_with_duplication() -> None:
9496
"""
95-
Test apply feature views with duplicated names and single py file in a feature repo using CLI
97+
Tests that applying feature views with duplicated names is not possible, even if one of the
98+
duplicated feature views is imported from another file.
9699
"""
97-
98100
with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name:
99101
runner = CliRunner()
100102
# Construct an example repo in a temporary dir
@@ -115,8 +117,11 @@ def test_cli_apply_imported_featureview_with_duplication() -> None:
115117
)
116118
)
117119

120+
# Import feature view with duplicated name to try breaking the deduplication logic.
118121
repo_example = repo_path / "example.py"
119-
repo_example.write_text(get_example_repo("example_feature_repo_2.py"))
122+
repo_example.write_text(
123+
get_example_repo("example_feature_repo_with_driver_stats_feature_view.py")
124+
)
120125
repo_example_2 = repo_path / "example_2.py"
121126
repo_example_2.write_text(
122127
"from datetime import timedelta\n"
@@ -147,7 +152,6 @@ def test_cli_apply_duplicated_featureview_names_multiple_py_files() -> None:
147152
"""
148153
Test apply feature views with duplicated names from multiple py files in a feature repo using CLI
149154
"""
150-
151155
with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name:
152156
runner = CliRunner()
153157
# Construct an example repo in a temporary dir
@@ -170,7 +174,11 @@ def test_cli_apply_duplicated_featureview_names_multiple_py_files() -> None:
170174
# Create multiple py files containing the same feature view name
171175
for i in range(3):
172176
repo_example = repo_path / f"example{i}.py"
173-
repo_example.write_text(get_example_repo("example_feature_repo_2.py"))
177+
repo_example.write_text(
178+
get_example_repo(
179+
"example_feature_repo_with_driver_stats_feature_view.py"
180+
)
181+
)
174182
rc, output = runner.run_with_output(["apply"], cwd=repo_path)
175183

176184
assert (

‎sdk/python/tests/unit/local_feast_tests/test_e2e_local.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,14 @@
55

66
import pandas as pd
77

8+
from feast import Entity, FeatureView, Field, FileSource
89
from feast.driver_test_data import (
910
create_driver_hourly_stats_df,
1011
create_global_daily_stats_df,
1112
)
1213
from feast.feature_store import FeatureStore
14+
from feast.types import Float32, String
15+
from tests.utils.basic_read_write_test import basic_rw_test
1316
from tests.utils.cli_repo_creator import CliRunner, get_example_repo
1417
from tests.utils.feature_records import validate_online_features
1518

@@ -120,3 +123,41 @@ def _test_materialize_and_online_retrieval(
120123

121124
assert r.returncode == 0, f"stdout: {r.stdout}\n stderr: {r.stderr}"
122125
validate_online_features(store, driver_df, end_date)
126+
127+
128+
def test_partial() -> None:
129+
"""
130+
Add another table to existing repo using partial apply API. Make sure both the table
131+
applied via CLI apply and the new table are passing RW test.
132+
"""
133+
runner = CliRunner()
134+
with runner.local_repo(
135+
get_example_repo("example_feature_repo_1.py"), "file"
136+
) as store:
137+
driver = Entity(name="driver", join_keys=["test"])
138+
139+
driver_locations_source = FileSource(
140+
path="data/driver_locations.parquet", # Fake path
141+
timestamp_field="event_timestamp",
142+
created_timestamp_column="created_timestamp",
143+
)
144+
145+
driver_locations_100 = FeatureView(
146+
name="driver_locations_100",
147+
entities=[driver],
148+
ttl=timedelta(days=1),
149+
schema=[
150+
Field(name="lat", dtype=Float32),
151+
Field(name="lon", dtype=String),
152+
Field(name="name", dtype=String),
153+
Field(name="test", dtype=String),
154+
],
155+
online=True,
156+
batch_source=driver_locations_source,
157+
tags={},
158+
)
159+
160+
store.apply([driver_locations_100])
161+
162+
basic_rw_test(store, view_name="driver_locations")
163+
basic_rw_test(store, view_name="driver_locations_100")

0 commit comments

Comments
 (0)