Skip to content

Commit c364be4

Browse files
tokokotokoko
authored
chore: Run tests as make commands in workflows (#4035)
* run tests as make commands in workflows Signed-off-by: tokoko <togurg14@freeuni.edu.ge> * rename test-python make command to test-python-unit Signed-off-by: tokoko <togurg14@freeuni.edu.ge> * add env vars to pytest.ini Signed-off-by: tokoko <togurg14@freeuni.edu.ge> * fix odfv transformation field signature Signed-off-by: tokoko <togurg14@freeuni.edu.ge> * fix linting, formatting Signed-off-by: tokoko <togurg14@freeuni.edu.ge> * handle stream_feature_view failures Signed-off-by: tokoko <togurg14@freeuni.edu.ge> --------- Signed-off-by: tokoko <togurg14@freeuni.edu.ge>
1 parent cf58ebe commit c364be4

File tree

12 files changed

+66
-91
lines changed

12 files changed

+66
-91
lines changed

‎.github/workflows/pr_integration_tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,4 +167,4 @@ jobs:
167167
SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
168168
SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
169169
SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
170-
run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread
170+
run: make test-python-integration

‎.github/workflows/pr_local_integration_tests.yml‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,4 @@ jobs:
6161
run: make install-python-ci-dependencies
6262
- name: Test local integration tests
6363
if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak
64-
env:
65-
FEAST_USAGE: "False"
66-
IS_TEST: "True"
67-
FEAST_LOCAL_ONLINE_CONTAINER: "True"
68-
FEAST_IS_LOCAL_TEST: "True"
69-
run: pytest -n 8 --cov=./ --cov-report=xml --color=yes --integration -k "not gcs_registry and not s3_registry and not test_lambda_materialization and not test_snowflake_materialization" sdk/python/tests
64+
run: make test-python-integration-local

‎.github/workflows/unit_tests.yml‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,6 @@ jobs:
2323
with:
2424
python-version: ${{ matrix.python-version }}
2525
architecture: x64
26-
- name: Install mysql on macOS
27-
if: startsWith(matrix.os, 'macOS')
28-
run: |
29-
brew install mysql
30-
PATH=$PATH:/usr/local/mysql/bin
31-
- name: Work around Homebrew MySQL being broken
32-
# See https://github.com/Homebrew/homebrew-core/issues/130258 for more details.
33-
if: startsWith(matrix.os, 'macOS')
34-
run: |
35-
brew install zlib
36-
ln -sv $(brew --prefix zlib)/lib/libz.dylib $(brew --prefix)/lib/libzlib.dylib
3726
- name: Get pip cache dir
3827
id: pip-cache
3928
run: |
@@ -56,7 +45,7 @@ jobs:
5645
- name: Install dependencies
5746
run: make install-python-ci-dependencies
5847
- name: Test Python
59-
run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests
48+
run: make test-python-unit
6049

6150

6251
unit-test-ui:

‎Makefile‎

Lines changed: 7 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ format: format-python format-java
2828

2929
lint: lint-python lint-java
3030

31-
test: test-python test-java
31+
test: test-python-unit test-java
3232

3333
protos: compile-protos-python compile-protos-docs
3434

@@ -63,32 +63,26 @@ benchmark-python:
6363
benchmark-python-local:
6464
FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests
6565

66-
test-python:
67-
FEAST_USAGE=False \
68-
IS_TEST=True \
69-
python -m pytest -n 8 sdk/python/tests \
66+
test-python-unit:
67+
python -m pytest -n 8 --color=yes sdk/python/tests
7068

7169
test-python-integration:
72-
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
70+
python -m pytest -n 8 --integration --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests
7371

7472
test-python-integration-local:
7573
@(docker info > /dev/null 2>&1 && \
76-
FEAST_USAGE=False \
77-
IS_TEST=True \
7874
FEAST_IS_LOCAL_TEST=True \
7975
FEAST_LOCAL_ONLINE_CONTAINER=True \
80-
python -m pytest -n 8 --integration \
76+
python -m pytest -n 8 --color=yes --integration \
8177
-k "not gcs_registry and \
8278
not s3_registry and \
8379
not test_lambda_materialization and \
84-
not test_snowflake" \
80+
not test_snowflake_materialization" \
8581
sdk/python/tests \
8682
) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
8783

8884
test-python-integration-container:
8985
@(docker info > /dev/null 2>&1 && \
90-
FEAST_USAGE=False \
91-
IS_TEST=True \
9286
FEAST_LOCAL_ONLINE_CONTAINER=True \
9387
python -m pytest -n 8 --integration sdk/python/tests \
9488
) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
@@ -97,7 +91,6 @@ test-python-universal-spark:
9791
PYTHONPATH='.' \
9892
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.spark_repo_configuration \
9993
PYTEST_PLUGINS=feast.infra.offline_stores.contrib.spark_offline_store.tests \
100-
FEAST_USAGE=False IS_TEST=True \
10194
python -m pytest -n 8 --integration \
10295
-k "not test_historical_retrieval_fails_on_validation and \
10396
not test_historical_retrieval_with_validation and \
@@ -121,7 +114,6 @@ test-python-universal-trino:
121114
PYTHONPATH='.' \
122115
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.trino_repo_configuration \
123116
PYTEST_PLUGINS=feast.infra.offline_stores.contrib.trino_offline_store.tests \
124-
FEAST_USAGE=False IS_TEST=True \
125117
python -m pytest -n 8 --integration \
126118
-k "not test_historical_retrieval_fails_on_validation and \
127119
not test_historical_retrieval_with_validation and \
@@ -148,7 +140,6 @@ test-python-universal-mssql:
148140
PYTHONPATH='.' \
149141
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.mssql_repo_configuration \
150142
PYTEST_PLUGINS=feast.infra.offline_stores.contrib.mssql_offline_store.tests \
151-
FEAST_USAGE=False IS_TEST=True \
152143
FEAST_LOCAL_ONLINE_CONTAINER=True \
153144
python -m pytest -n 8 --integration \
154145
-k "not gcs_registry and \
@@ -166,7 +157,6 @@ test-python-universal-athena:
166157
PYTHONPATH='.' \
167158
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.athena_repo_configuration \
168159
PYTEST_PLUGINS=feast.infra.offline_stores.contrib.athena_offline_store.tests \
169-
FEAST_USAGE=False IS_TEST=True \
170160
ATHENA_REGION=ap-northeast-2 \
171161
ATHENA_DATA_SOURCE=AwsDataCatalog \
172162
ATHENA_DATABASE=default \
@@ -190,7 +180,6 @@ test-python-universal-athena:
190180
test-python-universal-duckdb:
191181
PYTHONPATH='.' \
192182
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.duckdb_repo_configuration \
193-
FEAST_USAGE=False IS_TEST=True \
194183
python -m pytest -n 8 --integration \
195184
-k "not test_nullable_online_store and \
196185
not gcs_registry and \
@@ -204,8 +193,6 @@ test-python-universal-postgres-offline:
204193
PYTHONPATH='.' \
205194
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.postgres_repo_configuration \
206195
PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \
207-
FEAST_USAGE=False \
208-
IS_TEST=True \
209196
python -m pytest -n 8 --integration \
210197
-k "not test_historical_retrieval_with_validation and \
211198
not test_historical_features_persisting and \
@@ -226,8 +213,6 @@ test-python-universal-postgres-online:
226213
PYTHONPATH='.' \
227214
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.postgres_repo_configuration \
228215
PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \
229-
FEAST_USAGE=False \
230-
IS_TEST=True \
231216
python -m pytest -n 8 --integration \
232217
-k "not test_universal_cli and \
233218
not test_go_feature_server and \
@@ -247,8 +232,6 @@ test-python-universal-postgres-online:
247232
PYTHONPATH='.' \
248233
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_repo_configuration \
249234
PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \
250-
FEAST_USAGE=False \
251-
IS_TEST=True \
252235
python -m pytest -n 8 --integration \
253236
-k "not test_universal_cli and \
254237
not test_go_feature_server and \
@@ -268,17 +251,13 @@ test-python-universal-cassandra:
268251
PYTHONPATH='.' \
269252
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \
270253
PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \
271-
FEAST_USAGE=False \
272-
IS_TEST=True \
273254
python -m pytest -x --integration \
274255
sdk/python/tests
275256

276257
test-python-universal-hazelcast:
277258
PYTHONPATH='.' \
278259
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.hazelcast_repo_configuration \
279260
PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.hazelcast \
280-
FEAST_USAGE=False \
281-
IS_TEST=True \
282261
python -m pytest -n 8 --integration \
283262
-k "not test_universal_cli and \
284263
not test_go_feature_server and \
@@ -298,8 +277,6 @@ test-python-universal-cassandra-no-cloud-providers:
298277
PYTHONPATH='.' \
299278
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \
300279
PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \
301-
FEAST_USAGE=False \
302-
IS_TEST=True \
303280
python -m pytest -x --integration \
304281
-k "not test_lambda_materialization_consistency and \
305282
not test_apply_entity_integration and \
@@ -314,7 +291,7 @@ test-python-universal-cassandra-no-cloud-providers:
314291
sdk/python/tests
315292

316293
test-python-universal:
317-
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
294+
python -m pytest -n 8 --integration sdk/python/tests
318295

319296
format-python:
320297
# Sort

‎sdk/python/feast/stream_feature_view.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -241,11 +241,11 @@ def from_proto(cls, sfv_proto):
241241
if sfv_proto.spec.HasField("user_defined_function")
242242
else None
243243
)
244-
feature_transformation = (
245-
sfv_proto.spec.feature_transformation.user_defined_function.body_text
246-
if sfv_proto.spec.HasField("feature_transformation")
247-
else None
248-
)
244+
# feature_transformation = (
245+
# sfv_proto.spec.feature_transformation.user_defined_function.body_text
246+
# if sfv_proto.spec.HasField("feature_transformation")
247+
# else None
248+
# )
249249
stream_feature_view = cls(
250250
name=sfv_proto.spec.name,
251251
description=sfv_proto.spec.description,
@@ -264,7 +264,9 @@ def from_proto(cls, sfv_proto):
264264
mode=sfv_proto.spec.mode,
265265
udf=udf,
266266
udf_string=udf_string,
267-
feature_transformation=feature_transformation,
267+
feature_transformation=PandasTransformation(udf, udf_string)
268+
if udf
269+
else None,
268270
aggregations=[
269271
Aggregation.from_proto(agg_proto)
270272
for agg_proto in sfv_proto.spec.aggregations

‎sdk/python/pytest.ini‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
[pytest]
22
markers =
33
universal_offline_stores: mark a test as using all offline stores.
4-
universal_online_stores: mark a test as using all online stores.
4+
universal_online_stores: mark a test as using all online stores.
5+
6+
env =
7+
FEAST_USAGE=False
8+
IS_TEST=True

‎sdk/python/requirements/py3.10-ci-requirements.txt‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,11 @@ black==22.12.0
6161
# via feast (setup.py)
6262
bleach==6.1.0
6363
# via nbconvert
64-
boto3==1.34.65
64+
boto3==1.34.69
6565
# via
6666
# feast (setup.py)
6767
# moto
68-
botocore==1.34.65
68+
botocore==1.34.69
6969
# via
7070
# boto3
7171
# moto
@@ -82,7 +82,7 @@ cachecontrol==0.14.0
8282
# via firebase-admin
8383
cachetools==5.3.3
8484
# via google-auth
85-
cassandra-driver==3.29.0
85+
cassandra-driver==3.29.1
8686
# via feast (setup.py)
8787
certifi==2024.2.2
8888
# via
@@ -199,7 +199,7 @@ geojson==2.5.0
199199
# via rockset
200200
geomet==0.2.1.post1
201201
# via cassandra-driver
202-
google-api-core[grpc]==2.17.1
202+
google-api-core[grpc]==2.18.0
203203
# via
204204
# feast (setup.py)
205205
# firebase-admin
@@ -211,9 +211,9 @@ google-api-core[grpc]==2.17.1
211211
# google-cloud-datastore
212212
# google-cloud-firestore
213213
# google-cloud-storage
214-
google-api-python-client==2.122.0
214+
google-api-python-client==2.123.0
215215
# via firebase-admin
216-
google-auth==2.28.2
216+
google-auth==2.29.0
217217
# via
218218
# google-api-core
219219
# google-api-python-client
@@ -258,7 +258,7 @@ googleapis-common-protos[grpc]==1.63.0
258258
# google-api-core
259259
# grpc-google-iam-v1
260260
# grpcio-status
261-
great-expectations==0.18.11
261+
great-expectations==0.18.12
262262
# via feast (setup.py)
263263
greenlet==3.0.3
264264
# via sqlalchemy
@@ -331,7 +331,7 @@ importlib-metadata==6.11.0
331331
# via
332332
# dask
333333
# feast (setup.py)
334-
importlib-resources==6.3.1
334+
importlib-resources==6.4.0
335335
# via feast (setup.py)
336336
iniconfig==2.0.0
337337
# via pytest
@@ -459,7 +459,7 @@ moreorless==0.4.0
459459
# via bowler
460460
moto==4.2.14
461461
# via feast (setup.py)
462-
msal==1.27.0
462+
msal==1.28.0
463463
# via
464464
# azure-identity
465465
# msal-extensions
@@ -483,7 +483,7 @@ mypy-protobuf==3.3.0
483483
# via feast (setup.py)
484484
nbclient==0.10.0
485485
# via nbconvert
486-
nbconvert==7.16.2
486+
nbconvert==7.16.3
487487
# via jupyter-server
488488
nbformat==5.10.3
489489
# via
@@ -581,6 +581,7 @@ prompt-toolkit==3.0.43
581581
# via ipython
582582
proto-plus==1.23.0
583583
# via
584+
# google-api-core
584585
# google-cloud-bigquery
585586
# google-cloud-bigquery-storage
586587
# google-cloud-bigtable
@@ -687,6 +688,7 @@ pytest==7.4.4
687688
# feast (setup.py)
688689
# pytest-benchmark
689690
# pytest-cov
691+
# pytest-env
690692
# pytest-lazy-fixture
691693
# pytest-mock
692694
# pytest-ordering
@@ -696,6 +698,8 @@ pytest-benchmark==3.4.1
696698
# via feast (setup.py)
697699
pytest-cov==4.1.0
698700
# via feast (setup.py)
701+
pytest-env==1.1.3
702+
# via feast (setup.py)
699703
pytest-lazy-fixture==0.6.3
700704
# via feast (setup.py)
701705
pytest-mock==1.10.4
@@ -773,7 +777,7 @@ requests==2.31.0
773777
# snowflake-connector-python
774778
# sphinx
775779
# trino
776-
requests-oauthlib==1.4.0
780+
requests-oauthlib==2.0.0
777781
# via kubernetes
778782
responses==0.25.0
779783
# via moto
@@ -882,6 +886,7 @@ tomli==2.0.1
882886
# pip-tools
883887
# pyproject-hooks
884888
# pytest
889+
# pytest-env
885890
tomlkit==0.12.4
886891
# via snowflake-connector-python
887892
toolz==0.12.1
@@ -919,7 +924,7 @@ traitlets==5.14.2
919924
# nbformat
920925
trino==0.328.0
921926
# via feast (setup.py)
922-
typeguard==4.1.5
927+
typeguard==4.2.1
923928
# via feast (setup.py)
924929
types-protobuf==3.19.22
925930
# via
@@ -984,7 +989,7 @@ urllib3==1.26.18
984989
# requests
985990
# responses
986991
# rockset
987-
uvicorn[standard]==0.28.0
992+
uvicorn[standard]==0.29.0
988993
# via feast (setup.py)
989994
uvloop==0.19.0
990995
# via uvicorn

0 commit comments

Comments
 (0)