Skip to content

Commit c08e6f9

Browse files
tokokotokoko
authored
chore: Registry tests cleanup (#4232)
1 parent bf3ada8 commit c08e6f9

File tree

8 files changed

+147
-476
lines changed

8 files changed

+147
-476
lines changed

‎Makefile‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -80,19 +80,16 @@ test-python-unit:
8080
python -m pytest -n 8 --color=yes sdk/python/tests
8181

8282
test-python-integration:
83-
python -m pytest -n 8 --integration -k "(not snowflake or not test_historical_features_main) and not minio_registry" --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests
83+
python -m pytest -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread \
84+
-k "(not snowflake or not test_historical_features_main)" \
85+
sdk/python/tests
8486

8587
test-python-integration-local:
86-
@(docker info > /dev/null 2>&1 && \
87-
FEAST_IS_LOCAL_TEST=True \
88-
FEAST_LOCAL_ONLINE_CONTAINER=True \
89-
python -m pytest -n 8 --color=yes --integration \
90-
-k "not gcs_registry and \
91-
not s3_registry and \
92-
not test_lambda_materialization and \
93-
not test_snowflake_materialization" \
94-
sdk/python/tests \
95-
) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
88+
FEAST_IS_LOCAL_TEST=True \
89+
FEAST_LOCAL_ONLINE_CONTAINER=True \
90+
python -m pytest -n 8 --color=yes --integration --durations=5 --dist loadgroup \
91+
-k "not test_lambda_materialization and not test_snowflake_materialization" \
92+
sdk/python/tests
9693

9794
test-python-integration-container:
9895
@(docker info > /dev/null 2>&1 && \

‎sdk/python/tests/integration/feature_repos/universal/data_sources/file.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,8 @@ def __init__(self, project_name: str, *args, **kwargs):
141141
self.minio = MinioContainer()
142142
self.minio.start()
143143
client = self.minio.get_client()
144-
client.make_bucket("test")
144+
if not client.bucket_exists("test"):
145+
client.make_bucket("test")
145146
host_ip = self.minio.get_container_host_ip()
146147
exposed_port = self.minio.get_exposed_port(self.minio.port)
147148
self.endpoint_url = f"http://{host_ip}:{exposed_port}"

‎sdk/python/tests/integration/registration/test_feature_store.py‎

Lines changed: 1 addition & 166 deletions
Original file line numberDiff line numberDiff line change
@@ -11,68 +11,21 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14-
import os
15-
import time
1614
from datetime import timedelta
1715
from tempfile import mkstemp
1816

1917
import pytest
2018
from pytest_lazyfixture import lazy_fixture
2119

22-
from feast import FileSource
23-
from feast.data_format import ParquetFormat
2420
from feast.entity import Entity
2521
from feast.feature_store import FeatureStore
2622
from feast.feature_view import FeatureView
27-
from feast.field import Field
28-
from feast.infra.offline_stores.file import FileOfflineStoreConfig
29-
from feast.infra.online_stores.dynamodb import DynamoDBOnlineStoreConfig
3023
from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig
3124
from feast.repo_config import RepoConfig
32-
from feast.types import Array, Bytes, Float64, Int64, String
25+
from feast.types import Float64, Int64, String
3326
from tests.utils.data_source_test_creator import prep_file_source
3427

3528

36-
@pytest.mark.integration
37-
@pytest.mark.parametrize(
38-
"test_feature_store",
39-
[
40-
lazy_fixture("feature_store_with_gcs_registry"),
41-
lazy_fixture("feature_store_with_s3_registry"),
42-
],
43-
)
44-
def test_apply_entity_integration(test_feature_store):
45-
entity = Entity(
46-
name="driver_car_id",
47-
description="Car driver id",
48-
tags={"team": "matchmaking"},
49-
)
50-
51-
# Register Entity
52-
test_feature_store.apply([entity])
53-
54-
entities = test_feature_store.list_entities()
55-
56-
entity = entities[0]
57-
assert (
58-
len(entities) == 1
59-
and entity.name == "driver_car_id"
60-
and entity.description == "Car driver id"
61-
and "team" in entity.tags
62-
and entity.tags["team"] == "matchmaking"
63-
)
64-
65-
entity = test_feature_store.get_entity("driver_car_id")
66-
assert (
67-
entity.name == "driver_car_id"
68-
and entity.description == "Car driver id"
69-
and "team" in entity.tags
70-
and entity.tags["team"] == "matchmaking"
71-
)
72-
73-
test_feature_store.teardown()
74-
75-
7629
@pytest.mark.integration
7730
@pytest.mark.parametrize(
7831
"test_feature_store",
@@ -109,81 +62,6 @@ def test_feature_view_inference_success(test_feature_store, dataframe_source):
10962
test_feature_store.teardown()
11063

11164

112-
@pytest.mark.integration
113-
@pytest.mark.parametrize(
114-
"test_feature_store",
115-
[
116-
lazy_fixture("feature_store_with_gcs_registry"),
117-
lazy_fixture("feature_store_with_s3_registry"),
118-
],
119-
)
120-
def test_apply_feature_view_integration(test_feature_store):
121-
# Create Feature Views
122-
batch_source = FileSource(
123-
file_format=ParquetFormat(),
124-
path="file://feast/*",
125-
timestamp_field="ts_col",
126-
created_timestamp_column="timestamp",
127-
)
128-
129-
entity = Entity(name="fs1_my_entity_1", join_keys=["test"])
130-
131-
fv1 = FeatureView(
132-
name="my_feature_view_1",
133-
schema=[
134-
Field(name="fs1_my_feature_1", dtype=Int64),
135-
Field(name="fs1_my_feature_2", dtype=String),
136-
Field(name="fs1_my_feature_3", dtype=Array(String)),
137-
Field(name="fs1_my_feature_4", dtype=Array(Bytes)),
138-
Field(name="test", dtype=Int64),
139-
],
140-
entities=[entity],
141-
tags={"team": "matchmaking"},
142-
source=batch_source,
143-
ttl=timedelta(minutes=5),
144-
)
145-
146-
# Register Feature View
147-
test_feature_store.apply([fv1, entity])
148-
149-
feature_views = test_feature_store.list_feature_views()
150-
151-
# List Feature Views
152-
assert (
153-
len(feature_views) == 1
154-
and feature_views[0].name == "my_feature_view_1"
155-
and feature_views[0].features[0].name == "fs1_my_feature_1"
156-
and feature_views[0].features[0].dtype == Int64
157-
and feature_views[0].features[1].name == "fs1_my_feature_2"
158-
and feature_views[0].features[1].dtype == String
159-
and feature_views[0].features[2].name == "fs1_my_feature_3"
160-
and feature_views[0].features[2].dtype == Array(String)
161-
and feature_views[0].features[3].name == "fs1_my_feature_4"
162-
and feature_views[0].features[3].dtype == Array(Bytes)
163-
and feature_views[0].entities[0] == "fs1_my_entity_1"
164-
)
165-
166-
feature_view = test_feature_store.get_feature_view("my_feature_view_1")
167-
assert (
168-
feature_view.name == "my_feature_view_1"
169-
and feature_view.features[0].name == "fs1_my_feature_1"
170-
and feature_view.features[0].dtype == Int64
171-
and feature_view.features[1].name == "fs1_my_feature_2"
172-
and feature_view.features[1].dtype == String
173-
and feature_view.features[2].name == "fs1_my_feature_3"
174-
and feature_view.features[2].dtype == Array(String)
175-
and feature_view.features[3].name == "fs1_my_feature_4"
176-
and feature_view.features[3].dtype == Array(Bytes)
177-
and feature_view.entities[0] == "fs1_my_entity_1"
178-
)
179-
180-
test_feature_store.delete_feature_view("my_feature_view_1")
181-
feature_views = test_feature_store.list_feature_views()
182-
assert len(feature_views) == 0
183-
184-
test_feature_store.teardown()
185-
186-
18765
@pytest.fixture
18866
def feature_store_with_local_registry():
18967
fd, registry_path = mkstemp()
@@ -197,46 +75,3 @@ def feature_store_with_local_registry():
19775
entity_key_serialization_version=2,
19876
)
19977
)
200-
201-
202-
@pytest.fixture
203-
def feature_store_with_gcs_registry():
204-
from google.cloud import storage
205-
206-
storage_client = storage.Client()
207-
bucket_name = f"feast-registry-test-{int(time.time() * 1000)}"
208-
bucket = storage_client.bucket(bucket_name)
209-
bucket = storage_client.create_bucket(bucket)
210-
bucket.add_lifecycle_delete_rule(
211-
age=14
212-
) # delete buckets automatically after 14 days
213-
bucket.patch()
214-
bucket.blob("registry.db")
215-
216-
return FeatureStore(
217-
config=RepoConfig(
218-
registry=f"gs://{bucket_name}/registry.db",
219-
project="default",
220-
provider="gcp",
221-
entity_key_serialization_version=2,
222-
)
223-
)
224-
225-
226-
@pytest.fixture
227-
def feature_store_with_s3_registry():
228-
aws_registry_path = os.getenv(
229-
"AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries"
230-
)
231-
return FeatureStore(
232-
config=RepoConfig(
233-
registry=f"{aws_registry_path}/{int(time.time() * 1000)}/registry.db",
234-
project="default",
235-
provider="aws",
236-
online_store=DynamoDBOnlineStoreConfig(
237-
region=os.getenv("AWS_REGION", "us-west-2")
238-
),
239-
offline_store=FileOfflineStoreConfig(),
240-
entity_key_serialization_version=2,
241-
)
242-
)

‎sdk/python/tests/integration/registration/test_universal_cli.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@
77
from assertpy import assertpy
88

99
from feast.feature_store import FeatureStore
10-
from tests.integration.feature_repos.repo_configuration import Environment
10+
from tests.integration.feature_repos.universal.data_sources.file import (
11+
FileDataSourceCreator,
12+
)
1113
from tests.utils.basic_read_write_test import basic_rw_test
1214
from tests.utils.cli_repo_creator import CliRunner, get_example_repo
1315
from tests.utils.e2e_test_validation import (
@@ -17,8 +19,7 @@
1719

1820

1921
@pytest.mark.integration
20-
@pytest.mark.universal_offline_stores
21-
def test_universal_cli(environment: Environment):
22+
def test_universal_cli():
2223
project = f"test_universal_cli_{str(uuid.uuid4()).replace('-', '')[:8]}"
2324
runner = CliRunner()
2425

@@ -28,9 +29,9 @@ def test_universal_cli(environment: Environment):
2829
feature_store_yaml = make_feature_store_yaml(
2930
project,
3031
repo_path,
31-
environment.data_source_creator,
32-
environment.provider,
33-
environment.online_store,
32+
FileDataSourceCreator("project"),
33+
"local",
34+
{"type": "sqlite"},
3435
)
3536

3637
repo_config = repo_path / "feature_store.yaml"
@@ -115,8 +116,7 @@ def test_universal_cli(environment: Environment):
115116

116117

117118
@pytest.mark.integration
118-
@pytest.mark.universal_offline_stores
119-
def test_odfv_apply(environment) -> None:
119+
def test_odfv_apply() -> None:
120120
project = f"test_odfv_apply{str(uuid.uuid4()).replace('-', '')[:8]}"
121121
runner = CliRunner()
122122

@@ -126,9 +126,9 @@ def test_odfv_apply(environment) -> None:
126126
feature_store_yaml = make_feature_store_yaml(
127127
project,
128128
repo_path,
129-
environment.data_source_creator,
130-
environment.provider,
131-
environment.online_store,
129+
FileDataSourceCreator("project"),
130+
"local",
131+
{"type": "sqlite"},
132132
)
133133

134134
repo_config = repo_path / "feature_store.yaml"

0 commit comments

Comments
 (0)