Skip to content

Commit d4b0b1a

Browse files
pyalexpyalex
authored
chore: Generate environments for each individual test based on its markers/fixtures (#2648)
* generating test environments bases on test markers and fixtures Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * remove "universal" marker Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com>
1 parent 6589f15 commit d4b0b1a

21 files changed

+368
-368
lines changed

‎Makefile‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ test-python-universal-contrib:
7676
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration \
7777
PYTEST_PLUGINS=feast.infra.offline_stores.contrib.trino_offline_store.tests \
7878
FEAST_USAGE=False IS_TEST=True \
79-
python -m pytest -n 8 --integration --universal \
79+
python -m pytest -n 8 --integration \
8080
-k "not test_historical_retrieval_fails_on_validation and \
8181
not test_historical_retrieval_with_validation and \
8282
not test_historical_features_persisting and \
@@ -93,7 +93,7 @@ test-python-universal-postgres:
9393
PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \
9494
FEAST_USAGE=False \
9595
IS_TEST=True \
96-
python -m pytest -x --integration --universal \
96+
python -m pytest -x --integration \
9797
-k "not test_historical_retrieval_fails_on_validation and \
9898
not test_historical_retrieval_with_validation and \
9999
not test_historical_features_persisting and \
@@ -105,10 +105,10 @@ test-python-universal-postgres:
105105
sdk/python/tests
106106

107107
test-python-universal-local:
108-
FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
108+
FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest -n 8 --integration sdk/python/tests
109109

110110
test-python-universal:
111-
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
111+
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
112112

113113
test-python-go-server: compile-go-lib
114114
FEAST_USAGE=False IS_TEST=True FEAST_GO_FEATURE_RETRIEVAL=True pytest --integration --goserver sdk/python/tests
@@ -158,7 +158,7 @@ start-trino-locally:
158158
sleep 15
159159

160160
test-trino-plugin-locally:
161-
cd ${ROOT_DIR}/sdk/python; FULL_REPO_CONFIGS_MODULE=feast.infra.offline_stores.contrib.trino_offline_store.test_config.manual_tests FEAST_USAGE=False IS_TEST=True python -m pytest --integration --universal tests/
161+
cd ${ROOT_DIR}/sdk/python; FULL_REPO_CONFIGS_MODULE=feast.infra.offline_stores.contrib.trino_offline_store.test_config.manual_tests FEAST_USAGE=False IS_TEST=True python -m pytest --integration tests/
162162

163163
kill-trino-locally:
164164
cd ${ROOT_DIR}; docker stop trino

‎sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,14 @@
44
from feast.infra.offline_stores.contrib.trino_offline_store.tests.data_source import (
55
TrinoSourceCreator,
66
)
7-
from tests.integration.feature_repos.integration_test_repo_config import (
8-
IntegrationTestRepoConfig,
7+
from tests.integration.feature_repos.repo_configuration import REDIS_CONFIG
8+
from tests.integration.feature_repos.universal.online_store.redis import (
9+
RedisOnlineStoreCreator,
910
)
1011

11-
FULL_REPO_CONFIGS = [
12-
IntegrationTestRepoConfig(offline_store_creator=SparkDataSourceCreator),
13-
IntegrationTestRepoConfig(offline_store_creator=TrinoSourceCreator),
12+
AVAILABLE_OFFLINE_STORES = [
13+
("local", SparkDataSourceCreator),
14+
("local", TrinoSourceCreator),
1415
]
16+
17+
AVAILABLE_ONLINE_STORES = {"redis": (REDIS_CONFIG, RedisOnlineStoreCreator)}

‎sdk/python/feast/infra/offline_stores/contrib/postgres_repo_configuration.py‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,7 @@
11
from feast.infra.offline_stores.contrib.postgres_offline_store.tests.data_source import (
22
PostgreSQLDataSourceCreator,
33
)
4-
from tests.integration.feature_repos.integration_test_repo_config import (
5-
IntegrationTestRepoConfig,
6-
)
74

8-
FULL_REPO_CONFIGS = [
9-
IntegrationTestRepoConfig(
10-
provider="local",
11-
offline_store_creator=PostgreSQLDataSourceCreator,
12-
online_store_creator=PostgreSQLDataSourceCreator,
13-
),
14-
]
5+
AVAILABLE_OFFLINE_STORES = [("local", PostgreSQLDataSourceCreator)]
6+
7+
AVAILABLE_ONLINE_STORES = {"postgres": (None, PostgreSQLDataSourceCreator)}

‎sdk/python/feast/infra/online_stores/datastore.py‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import logging
1616
from datetime import datetime
1717
from multiprocessing.pool import ThreadPool
18-
from queue import Queue
18+
from queue import Empty, Queue
1919
from threading import Lock, Thread
2020
from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple
2121

@@ -292,22 +292,24 @@ def increment(self):
292292

293293
def worker(shared_counter):
294294
while True:
295-
client.delete_multi(deletion_queue.get())
295+
try:
296+
job = deletion_queue.get(block=False)
297+
except Empty:
298+
return
299+
300+
client.delete_multi(job)
296301
shared_counter.increment()
297302
LOGGER.debug(
298303
f"batch deletions completed: {shared_counter.value} ({shared_counter.value * BATCH_SIZE} total entries) & outstanding queue size: {deletion_queue.qsize()}"
299304
)
300305
deletion_queue.task_done()
301306

302-
for _ in range(NUM_THREADS):
303-
Thread(target=worker, args=(status_info_counter,), daemon=True).start()
304-
305307
query = client.query(kind="Row", ancestor=key)
306-
while True:
307-
entities = list(query.fetch(limit=BATCH_SIZE))
308-
if not entities:
309-
break
310-
deletion_queue.put([entity.key for entity in entities])
308+
for page in query.fetch().pages:
309+
deletion_queue.put([entity.key for entity in page])
310+
311+
for _ in range(NUM_THREADS):
312+
Thread(target=worker, args=(status_info_counter,)).start()
311313

312314
deletion_queue.join()
313315

‎sdk/python/tests/benchmarks/test_benchmark_universal_online_retrieval.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
@pytest.mark.benchmark
1919
@pytest.mark.integration
20+
@pytest.mark.universal_online_stores
2021
def test_online_retrieval(environment, universal_data_sources, benchmark):
2122
fs = environment.feature_store
2223
entities, datasets, data_sources = universal_data_sources

0 commit comments

Comments
 (0)