diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 39273b56c25..e64a175d6a9 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1207,27 +1207,24 @@ def _get_online_features( [DUMMY_ENTITY_VAL] * num_rows, DUMMY_ENTITY.value_type ) - # Initialize the set of EntityKeyProtos once and reuse them for each FeatureView - # to avoid initialization overhead. - entity_keys = [EntityKeyProto() for _ in range(num_rows)] provider = self._get_provider() for table, requested_features in grouped_refs: # Get the correct set of entity values with the correct join keys. - table_entity_values = self._get_table_entity_values( - table, entity_name_to_join_key_map, join_key_values, + table_entity_values, idxs = self._get_unique_entities( + table, join_key_values, entity_name_to_join_key_map, ) - # Set the EntityKeyProtos inplace. - self._set_table_entity_keys( - table_entity_values, entity_keys, + # Fetch feature data for the minimum set of Entities. + feature_data = self._read_from_online_store( + table_entity_values, provider, requested_features, table, ) # Populate the result_rows with the Features from the OnlineStore inplace. - self._populate_result_rows_from_feature_view( + self._populate_response_from_feature_data( + feature_data, + idxs, online_features_response, - entity_keys, full_feature_names, - provider, requested_features, table, ) @@ -1312,22 +1309,6 @@ def _get_table_entity_values( } return entity_values - @staticmethod - def _set_table_entity_keys( - entity_values: Dict[str, List[Value]], entity_keys: List[EntityKeyProto], - ): - """ - This method sets the a list of EntityKeyProtos inplace. - """ - keys = entity_values.keys() - # Columar to rowise (dict keys and values are guaranteed to have the same order). - rowise_values = zip(*entity_values.values()) - for entity_key in entity_keys: - # Make sure entity_keys are empty before setting. - entity_key.Clear() - entity_key.join_keys.extend(keys) - entity_key.entity_values.extend(next(rowise_values)) - @staticmethod def _populate_result_rows_from_columnar( online_features_response: GetOnlineFeaturesResponse, @@ -1380,21 +1361,134 @@ def ensure_request_data_values_exist( feature_names=missing_features ) - def _populate_result_rows_from_feature_view( + def _get_unique_entities( self, - online_features_response: GetOnlineFeaturesResponse, - entity_keys: List[EntityKeyProto], - full_feature_names: bool, + table: FeatureView, + join_key_values: Dict[str, List[Value]], + entity_name_to_join_key_map: Dict[str, str], + ) -> Tuple[Tuple[Dict[str, Value], ...], Tuple[List[int], ...]]: + """ Return the set of unique composite Entities for a Feature View and the indexes at which they appear. + + This method allows us to query the OnlineStore for data we need only once + rather than requesting and processing data for the same combination of + Entities multiple times. + """ + # Get the correct set of entity values with the correct join keys. + table_entity_values = self._get_table_entity_values( + table, entity_name_to_join_key_map, join_key_values, + ) + + # Convert back to rowise. + keys = table_entity_values.keys() + # Sort the rowise data to allow for grouping but keep original index. This lambda is + # sufficient as Entity types cannot be complex (ie. lists). + rowise = list(enumerate(zip(*table_entity_values.values()))) + rowise.sort( + key=lambda row: tuple(getattr(x, x.WhichOneof("val")) for x in row[1]) + ) + + # Identify unique entities and the indexes at which they occur. + unique_entities: Tuple[Dict[str, Value], ...] + indexes: Tuple[List[int], ...] + unique_entities, indexes = tuple( + zip( + *[ + (dict(zip(keys, k)), [_[0] for _ in g]) + for k, g in itertools.groupby(rowise, key=lambda x: x[1]) + ] + ) + ) + return unique_entities, indexes + + def _read_from_online_store( + self, + entity_rows: Iterable[Mapping[str, Value]], provider: Provider, requested_features: List[str], table: FeatureView, - ): + ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[Value]]]: + """ Read and process data from the OnlineStore for a given FeatureView. + + This method guarentees that the order of the data in each element of the + List returned is the same as the order of `requested_features`. + + This method assumes that `provider.online_read` returns data for each + combination of Entities in `entity_rows` in the same order as they + are provided. + """ + # Instantiate one EntityKeyProto per Entity. + entity_key_protos = [ + EntityKeyProto(join_keys=row.keys(), entity_values=row.values()) + for row in entity_rows + ] + + # Fetch data for Entities. read_rows = provider.online_read( config=self.config, table=table, - entity_keys=entity_keys, + entity_keys=entity_key_protos, requested_features=requested_features, ) + + # Each row is a set of features for a given entity key. We only need to convert + # the data to Protobuf once. + row_ts_proto = Timestamp() + null_value = Value() + read_row_protos = [] + for read_row in read_rows: + row_ts, feature_data = read_row + if row_ts is not None: + row_ts_proto.FromDatetime(row_ts) + event_timestamps = [row_ts_proto] * len(requested_features) + if feature_data is None: + statuses = [FieldStatus.NOT_FOUND] * len(requested_features) + values = [null_value] * len(requested_features) + else: + statuses = [] + values = [] + for feature_name in requested_features: + # Make sure order of data is the same as requested_features. + if feature_name not in feature_data: + statuses.append(FieldStatus.NOT_FOUND) + values.append(null_value) + else: + statuses.append(FieldStatus.PRESENT) + values.append(feature_data[feature_name]) + read_row_protos.append((event_timestamps, statuses, values)) + return read_row_protos + + @staticmethod + def _populate_response_from_feature_data( + feature_data: Iterable[ + Tuple[ + Iterable[Timestamp], Iterable["FieldStatus.ValueType"], Iterable[Value] + ] + ], + indexes: Iterable[Iterable[int]], + online_features_response: GetOnlineFeaturesResponse, + full_feature_names: bool, + requested_features: Iterable[str], + table: FeatureView, + ): + """ Populate the GetOnlineFeaturesReponse with feature data. + + This method assumes that `_read_from_online_store` returns data for each + combination of Entities in `entity_rows` in the same order as they + are provided. + + Args: + feature_data: A list of data in Protobuf form which was retrieved from the OnlineStore. + indexes: A list of indexes which should be the same length as `feature_data`. Each list + of indexes corresponds to a set of result rows in `online_features_response`. + online_features_response: The object to populate. + full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, + changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to + "customer_fv__daily_transactions"). + requested_features: The names of the features in `feature_data`. This should be ordered in the same way as the + data in `feature_data`. + table: The FeatureView that `feature_data` was retrieved from. + """ + # Add the feature names to the response. requested_feature_refs = [ f"{table.projection.name_to_use()}__{feature_name}" if full_feature_names @@ -1404,28 +1498,16 @@ def _populate_result_rows_from_feature_view( online_features_response.metadata.feature_names.val.extend( requested_feature_refs ) - # Each row is a set of features for a given entity key - for row_idx, read_row in enumerate(read_rows): - row_ts, feature_data = read_row - result_row = online_features_response.results[row_idx] - row_ts_proto = Timestamp() - if row_ts is not None: - row_ts_proto.FromDatetime(row_ts) - result_row.event_timestamps.extend([row_ts_proto] * len(requested_features)) - if feature_data is None: - result_row.statuses.extend( - [FieldStatus.NOT_FOUND] * len(requested_features) - ) - result_row.values.extend([Value()] * len(requested_features)) - else: - for feature_name in requested_features: - if feature_name not in feature_data: - result_row.statuses.append(FieldStatus.NOT_FOUND) - result_row.values.append(Value()) - else: - result_row.statuses.append(FieldStatus.PRESENT) - result_row.values.append(feature_data[feature_name]) + # Populate the result with data fetched from the OnlineStore + # which is guarenteed to be aligned with `requested_features`. + for feature_row, dest_idxs in zip(feature_data, indexes): + event_timestamps, statuses, values = feature_row + for dest_idx in dest_idxs: + result_row = online_features_response.results[dest_idx] + result_row.event_timestamps.extend(event_timestamps) + result_row.statuses.extend(statuses) + result_row.values.extend(values) @staticmethod def _augment_response_with_on_demand_transforms( diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/__init__.py b/sdk/python/feast/infra/feature_servers/aws_lambda/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/__init__.py b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index 6b33dda9394..493c6ab4626 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -277,13 +277,13 @@ def _get_features_for_entity( res_ts = Timestamp() ts_val = res_val.pop(f"_ts:{feature_view}") if ts_val: - res_ts.ParseFromString(ts_val) + res_ts.ParseFromString(bytes(ts_val)) res = {} for feature_name, val_bin in res_val.items(): val = ValueProto() if val_bin: - val.ParseFromString(val_bin) + val.ParseFromString(bytes(val_bin)) res[feature_name] = val if not res: diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 3e9ddb6e304..698da221ae8 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -5,6 +5,7 @@ import re import sys from importlib.abc import Loader +from importlib.machinery import ModuleSpec from pathlib import Path from typing import List, Set, Union, cast @@ -78,7 +79,11 @@ def get_repo_files(repo_root: Path) -> List[Path]: ignore_files = get_ignore_files(repo_root, ignore_paths) # List all Python files in the root directory (recursively) - repo_files = {p.resolve() for p in repo_root.glob("**/*.py") if p.is_file()} + repo_files = { + p.resolve() + for p in repo_root.glob("**/*.py") + if p.is_file() and "__init__.py" != p.name + } # Ignore all files that match any of the ignore paths in .feastignore repo_files -= ignore_files @@ -375,6 +380,7 @@ def init_repo(repo_name: str, template: str): import importlib.util spec = importlib.util.spec_from_file_location("bootstrap", str(bootstrap_path)) + assert isinstance(spec, ModuleSpec) bootstrap = importlib.util.module_from_spec(spec) assert isinstance(spec.loader, Loader) spec.loader.exec_module(bootstrap) diff --git a/sdk/python/feast/templates/aws/__init__.py b/sdk/python/feast/templates/aws/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/templates/gcp/__init__.py b/sdk/python/feast/templates/gcp/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/templates/local/__init__.py b/sdk/python/feast/templates/local/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 969ca658625..715da87c51a 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -19,7 +19,6 @@ import numpy as np import pandas as pd import pyarrow -from google.protobuf.pyext.cpp_message import GeneratedProtocolMessageType from google.protobuf.timestamp_pb2 import Timestamp from feast.protos.feast.types.Value_pb2 import ( @@ -32,7 +31,7 @@ StringList, ) from feast.protos.feast.types.Value_pb2 import Value as ProtoValue -from feast.value_type import ValueType +from feast.value_type import ListType, ValueType def feast_value_type_to_python_type(field_value_proto: ProtoValue) -> Any: @@ -195,7 +194,7 @@ def _type_err(item, dtype): PYTHON_LIST_VALUE_TYPE_TO_PROTO_VALUE: Dict[ - ValueType, Tuple[GeneratedProtocolMessageType, str, List[Type]] + ValueType, Tuple[ListType, str, List[Type]] ] = { ValueType.FLOAT_LIST: ( FloatList, @@ -273,7 +272,7 @@ def _python_value_to_proto_value( raise _type_err(first_invalid, valid_types[0]) return [ - ProtoValue(**{field_name: proto_type(val=value)}) + ProtoValue(**{field_name: proto_type(val=value)}) # type: ignore if value is not None else ProtoValue() for value in values diff --git a/sdk/python/feast/value_type.py b/sdk/python/feast/value_type.py index 3d1817421a2..1904baf7bbb 100644 --- a/sdk/python/feast/value_type.py +++ b/sdk/python/feast/value_type.py @@ -12,6 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. import enum +from typing import Type, Union + +from feast.protos.feast.types.Value_pb2 import ( + BoolList, + BytesList, + DoubleList, + FloatList, + Int32List, + Int64List, + StringList, +) class ValueType(enum.Enum): @@ -37,3 +48,14 @@ class ValueType(enum.Enum): BOOL_LIST = 17 UNIX_TIMESTAMP_LIST = 18 NULL = 19 + + +ListType = Union[ + Type[BoolList], + Type[BytesList], + Type[DoubleList], + Type[FloatList], + Type[Int32List], + Type[Int64List], + Type[StringList], +] diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt index 6d77880d93a..87ab9f9813b 100644 --- a/sdk/python/requirements/py3.7-ci-requirements.txt +++ b/sdk/python/requirements/py3.7-ci-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --extra=ci --output-file=requirements/py3.7-ci-requirements.txt # -absl-py==0.12.0 +absl-py==1.0.0 # via tensorflow-metadata adal==1.2.7 # via @@ -55,11 +55,11 @@ babel==2.9.1 # via sphinx black==19.10b0 # via feast (setup.py) -boto3==1.20.38 +boto3==1.20.40 # via # feast (setup.py) # moto -botocore==1.23.38 +botocore==1.23.40 # via # boto3 # moto @@ -213,7 +213,7 @@ grpcio-testing==1.34.0 # via feast (setup.py) grpcio-tools==1.34.0 # via feast (setup.py) -h11==0.12.0 +h11==0.13.0 # via uvicorn hiredis==2.0.0 # via feast (setup.py) @@ -279,7 +279,7 @@ mmh3==3.0.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moto==2.3.2 +moto==3.0.0 # via feast (setup.py) msal==1.16.0 # via @@ -299,13 +299,13 @@ multidict==5.2.0 # via # aiohttp # yarl -mypy==0.790 +mypy==0.931 # via feast (setup.py) mypy-extensions==0.4.3 # via # mypy # typing-inspect -mypy-protobuf==1.24 +mypy-protobuf==3.1.0 # via feast (setup.py) nodeenv==1.6.0 # via pre-commit @@ -396,7 +396,7 @@ pyjwt[crypto]==2.3.0 # via # adal # msal -pyparsing==3.0.6 +pyparsing==3.0.7 # via # httplib2 # packaging @@ -528,7 +528,7 @@ tabulate==0.8.9 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.5.0 +tensorflow-metadata==1.6.0 # via feast (setup.py) testcontainers==3.4.2 # via feast (setup.py) @@ -541,19 +541,43 @@ toml==0.10.2 tomli==2.0.0 # via # coverage + # mypy # pep517 tqdm==4.62.3 # via feast (setup.py) -typed-ast==1.4.3 +typed-ast==1.5.1 # via # black # mypy +types-futures==3.3.7 + # via types-protobuf +types-protobuf==3.19.5 + # via + # feast (setup.py) + # mypy-protobuf +types-python-dateutil==2.8.8 + # via feast (setup.py) +types-pytz==2021.3.4 + # via feast (setup.py) +types-pyyaml==6.0.3 + # via feast (setup.py) +types-redis==4.1.10 + # via feast (setup.py) +types-requests==2.27.7 + # via feast (setup.py) +types-setuptools==57.4.7 + # via feast (setup.py) +types-tabulate==0.8.5 + # via feast (setup.py) +types-urllib3==1.26.7 + # via types-requests typing-extensions==4.0.1 # via # aiohttp # anyio # asgiref # async-timeout + # h11 # importlib-metadata # jsonschema # libcst diff --git a/sdk/python/requirements/py3.7-requirements.txt b/sdk/python/requirements/py3.7-requirements.txt index 03ff93459cc..c2ad63fdea3 100644 --- a/sdk/python/requirements/py3.7-requirements.txt +++ b/sdk/python/requirements/py3.7-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file=requirements/py3.7-requirements.txt # -absl-py==0.12.0 +absl-py==1.0.0 # via tensorflow-metadata anyio==3.5.0 # via starlette @@ -47,7 +47,7 @@ grpcio==1.43.0 # grpcio-reflection grpcio-reflection==1.43.0 # via feast (setup.py) -h11==0.12.0 +h11==0.13.0 # via uvicorn httptools==0.3.0 # via uvicorn @@ -133,7 +133,7 @@ tabulate==0.8.9 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.5.0 +tensorflow-metadata==1.6.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) @@ -143,6 +143,7 @@ typing-extensions==4.0.1 # via # anyio # asgiref + # h11 # importlib-metadata # jsonschema # pydantic diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index d63b7ea3542..851a0b70548 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --extra=ci --output-file=requirements/py3.8-ci-requirements.txt # -absl-py==0.12.0 +absl-py==1.0.0 # via tensorflow-metadata adal==1.2.7 # via @@ -53,11 +53,11 @@ babel==2.9.1 # via sphinx black==19.10b0 # via feast (setup.py) -boto3==1.20.38 +boto3==1.20.40 # via # feast (setup.py) # moto -botocore==1.23.38 +botocore==1.23.40 # via # boto3 # moto @@ -211,7 +211,7 @@ grpcio-testing==1.34.0 # via feast (setup.py) grpcio-tools==1.34.0 # via feast (setup.py) -h11==0.12.0 +h11==0.13.0 # via uvicorn hiredis==2.0.0 # via feast (setup.py) @@ -265,7 +265,7 @@ mmh3==3.0.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moto==2.3.2 +moto==3.0.0 # via feast (setup.py) msal==1.16.0 # via @@ -285,13 +285,13 @@ multidict==5.2.0 # via # aiohttp # yarl -mypy==0.790 +mypy==0.931 # via feast (setup.py) mypy-extensions==0.4.3 # via # mypy # typing-inspect -mypy-protobuf==1.24 +mypy-protobuf==3.1.0 # via feast (setup.py) nodeenv==1.6.0 # via pre-commit @@ -382,7 +382,7 @@ pyjwt[crypto]==2.3.0 # via # adal # msal -pyparsing==3.0.6 +pyparsing==3.0.7 # via # httplib2 # packaging @@ -514,7 +514,7 @@ tabulate==0.8.9 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.5.0 +tensorflow-metadata==1.6.0 # via feast (setup.py) testcontainers==3.4.2 # via feast (setup.py) @@ -527,13 +527,34 @@ toml==0.10.2 tomli==2.0.0 # via # coverage + # mypy # pep517 tqdm==4.62.3 # via feast (setup.py) -typed-ast==1.4.3 +typed-ast==1.5.1 + # via black +types-futures==3.3.7 + # via types-protobuf +types-protobuf==3.19.5 # via - # black - # mypy + # feast (setup.py) + # mypy-protobuf +types-python-dateutil==2.8.8 + # via feast (setup.py) +types-pytz==2021.3.4 + # via feast (setup.py) +types-pyyaml==6.0.3 + # via feast (setup.py) +types-redis==4.1.10 + # via feast (setup.py) +types-requests==2.27.7 + # via feast (setup.py) +types-setuptools==57.4.7 + # via feast (setup.py) +types-tabulate==0.8.5 + # via feast (setup.py) +types-urllib3==1.26.7 + # via types-requests typing-extensions==4.0.1 # via # libcst diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index d16a5cbe53e..e94fe117b4a 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file=requirements/py3.8-requirements.txt # -absl-py==0.12.0 +absl-py==1.0.0 # via tensorflow-metadata anyio==3.5.0 # via starlette @@ -47,7 +47,7 @@ grpcio==1.43.0 # grpcio-reflection grpcio-reflection==1.43.0 # via feast (setup.py) -h11==0.12.0 +h11==0.13.0 # via uvicorn httptools==0.3.0 # via uvicorn @@ -129,7 +129,7 @@ tabulate==0.8.9 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.5.0 +tensorflow-metadata==1.6.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 49a1ac41f6e..76ed9f1237c 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --extra=ci --output-file=requirements/py3.9-ci-requirements.txt # -absl-py==0.12.0 +absl-py==1.0.0 # via tensorflow-metadata adal==1.2.7 # via @@ -53,11 +53,11 @@ babel==2.9.1 # via sphinx black==19.10b0 # via feast (setup.py) -boto3==1.20.38 +boto3==1.20.40 # via # feast (setup.py) # moto -botocore==1.23.38 +botocore==1.23.40 # via # boto3 # moto @@ -211,7 +211,7 @@ grpcio-testing==1.34.0 # via feast (setup.py) grpcio-tools==1.34.0 # via feast (setup.py) -h11==0.12.0 +h11==0.13.0 # via uvicorn hiredis==2.0.0 # via feast (setup.py) @@ -263,7 +263,7 @@ mmh3==3.0.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moto==2.3.2 +moto==3.0.0 # via feast (setup.py) msal==1.16.0 # via @@ -283,13 +283,13 @@ multidict==5.2.0 # via # aiohttp # yarl -mypy==0.790 +mypy==0.931 # via feast (setup.py) mypy-extensions==0.4.3 # via # mypy # typing-inspect -mypy-protobuf==1.24 +mypy-protobuf==3.1.0 # via feast (setup.py) nodeenv==1.6.0 # via pre-commit @@ -380,7 +380,7 @@ pyjwt[crypto]==2.3.0 # via # adal # msal -pyparsing==3.0.6 +pyparsing==3.0.7 # via # httplib2 # packaging @@ -512,7 +512,7 @@ tabulate==0.8.9 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.5.0 +tensorflow-metadata==1.6.0 # via feast (setup.py) testcontainers==3.4.2 # via feast (setup.py) @@ -525,13 +525,34 @@ toml==0.10.2 tomli==2.0.0 # via # coverage + # mypy # pep517 tqdm==4.62.3 # via feast (setup.py) -typed-ast==1.4.3 +typed-ast==1.5.1 + # via black +types-futures==3.3.7 + # via types-protobuf +types-protobuf==3.19.5 # via - # black - # mypy + # feast (setup.py) + # mypy-protobuf +types-python-dateutil==2.8.8 + # via feast (setup.py) +types-pytz==2021.3.4 + # via feast (setup.py) +types-pyyaml==6.0.3 + # via feast (setup.py) +types-redis==4.1.10 + # via feast (setup.py) +types-requests==2.27.7 + # via feast (setup.py) +types-setuptools==57.4.7 + # via feast (setup.py) +types-tabulate==0.8.5 + # via feast (setup.py) +types-urllib3==1.26.7 + # via types-requests typing-extensions==4.0.1 # via # libcst diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 9a1e6e4088b..187cb02154b 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file=requirements/py3.9-requirements.txt # -absl-py==0.12.0 +absl-py==1.0.0 # via tensorflow-metadata anyio==3.5.0 # via starlette @@ -47,7 +47,7 @@ grpcio==1.43.0 # grpcio-reflection grpcio-reflection==1.43.0 # via feast (setup.py) -h11==0.12.0 +h11==0.13.0 # via uvicorn httptools==0.3.0 # via uvicorn @@ -127,7 +127,7 @@ tabulate==0.8.9 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.5.0 +tensorflow-metadata==1.6.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index ac7b4ec6a71..d35ee9de11b 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -86,38 +86,51 @@ "docker>=5.0.2", ] -CI_REQUIRED = [ - "cryptography==3.3.2", - "flake8", - "black==19.10b0", - "isort>=5", - "grpcio-tools==1.34.0", - "grpcio-testing==1.34.0", - "minio==7.1.0", - "mock==2.0.0", - "moto", - "mypy==0.790", - "mypy-protobuf==1.24", - "avro==1.10.0", - "gcsfs", - "urllib3>=1.25.4", - "pytest>=6.0.0", - "pytest-cov", - "pytest-xdist", - "pytest-benchmark>=3.4.1", - "pytest-lazy-fixture==0.6.3", - "pytest-timeout==1.4.2", - "pytest-ordering==0.6.*", - "pytest-mock==1.10.4", - "Sphinx!=4.0.0,<4.4.0", - "sphinx-rtd-theme", - "testcontainers==3.4.2", - "adlfs==0.5.9", - "firebase-admin==4.5.2", - "pre-commit", - "assertpy==1.1", - "pip-tools", -] + GCP_REQUIRED + REDIS_REQUIRED + AWS_REQUIRED +CI_REQUIRED = ( + [ + "cryptography==3.3.2", + "flake8", + "black==19.10b0", + "isort>=5", + "grpcio-tools==1.34.0", + "grpcio-testing==1.34.0", + "minio==7.1.0", + "mock==2.0.0", + "moto", + "mypy==0.931", + "mypy-protobuf==3.1.0", + "avro==1.10.0", + "gcsfs", + "urllib3>=1.25.4", + "pytest>=6.0.0", + "pytest-cov", + "pytest-xdist", + "pytest-benchmark>=3.4.1", + "pytest-lazy-fixture==0.6.3", + "pytest-timeout==1.4.2", + "pytest-ordering==0.6.*", + "pytest-mock==1.10.4", + "Sphinx!=4.0.0,<4.4.0", + "sphinx-rtd-theme", + "testcontainers==3.4.2", + "adlfs==0.5.9", + "firebase-admin==4.5.2", + "pre-commit", + "assertpy==1.1", + "pip-tools", + "types-protobuf", + "types-python-dateutil", + "types-pytz", + "types-PyYAML", + "types-redis", + "types-requests", + "types-setuptools", + "types-tabulate", + ] + + GCP_REQUIRED + + REDIS_REQUIRED + + AWS_REQUIRED +) DEV_REQUIRED = ["mypy-protobuf==1.*", "grpcio-testing==1.*"] + CI_REQUIRED diff --git a/sdk/python/tests/data/data_creator.py b/sdk/python/tests/data/data_creator.py index e5355b40bbc..1145f95c073 100644 --- a/sdk/python/tests/data/data_creator.py +++ b/sdk/python/tests/data/data_creator.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta -from typing import List +from typing import Dict, List, Optional import pandas as pd from pytz import timezone, utc @@ -38,7 +38,7 @@ def create_dataset( def get_entities_for_value_type(value_type: ValueType) -> List: - value_type_map = { + value_type_map: Dict[ValueType, List] = { ValueType.INT32: [1, 2, 1, 3, 3], ValueType.INT64: [1, 2, 1, 3, 3], ValueType.FLOAT: [1.0, 2.0, 1.0, 3.0, 3.0], @@ -48,13 +48,13 @@ def get_entities_for_value_type(value_type: ValueType) -> List: def get_feature_values_for_dtype( - dtype: str, is_list: bool, has_empty_list: bool + dtype: Optional[str], is_list: bool, has_empty_list: bool ) -> List: if dtype is None: return [0.1, None, 0.3, 4, 5] # TODO(adchia): for int columns, consider having a better error when dealing with None values (pandas int dfs can't # have na) - dtype_map = { + dtype_map: Dict[str, List] = { "int32": [1, 2, 3, 4, 5], "int64": [1, 2, 3, 4, 5], "float": [1.0, None, 3.0, 4.0, 5.0], diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 63ee4fe7bce..26bda2887c3 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -245,9 +245,9 @@ def get_local_server_port(self) -> int: def table_name_from_data_source(ds: DataSource) -> Optional[str]: if hasattr(ds, "table_ref"): - return ds.table_ref + return ds.table_ref # type: ignore elif hasattr(ds, "table"): - return ds.table + return ds.table # type: ignore return None diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py index 766c31150e1..4085ef9d067 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Dict, List, Optional import pandas as pd from google.cloud import bigquery @@ -21,7 +21,7 @@ def __init__(self, project_name: str): self.gcp_project = self.client.project self.dataset_id = f"{self.gcp_project}.{project_name}" - self.tables = [] + self.tables: List[str] = [] def create_dataset(self): if not self.dataset: @@ -50,7 +50,7 @@ def create_offline_store_config(self): def create_data_source( self, df: pd.DataFrame, - destination_name: Optional[str] = None, + destination_name: str, event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Dict[str, str] = None, diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py index 88780f07a07..f7839da5288 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Dict, List, Optional import pandas as pd @@ -14,7 +14,7 @@ class RedshiftDataSourceCreator(DataSourceCreator): - tables = [] + tables: List[str] = [] def __init__(self, project_name: str): super().__init__() diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py index dad14ac5aad..99f111a3462 100644 --- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py +++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py @@ -55,7 +55,7 @@ def find_asof_record( filter_keys = filter_keys or [] filter_values = filter_values or [] assert len(filter_keys) == len(filter_values) - found_record = {} + found_record: Dict[str, Any] = {} for record in records: if ( all( diff --git a/sdk/python/tests/integration/registration/test_cli.py b/sdk/python/tests/integration/registration/test_cli.py index 0fe73316adc..5dc3772265a 100644 --- a/sdk/python/tests/integration/registration/test_cli.py +++ b/sdk/python/tests/integration/registration/test_cli.py @@ -1,7 +1,7 @@ import tempfile import uuid from contextlib import contextmanager -from pathlib import Path, PosixPath +from pathlib import Path from textwrap import dedent import pytest @@ -26,10 +26,10 @@ def test_universal_cli(test_repo_config) -> None: with tempfile.TemporaryDirectory() as repo_dir_name: try: + repo_path = Path(repo_dir_name) feature_store_yaml = make_feature_store_yaml( - project, test_repo_config, repo_dir_name + project, test_repo_config, repo_path ) - repo_path = Path(repo_dir_name) repo_config = repo_path / "feature_store.yaml" @@ -103,7 +103,7 @@ def test_universal_cli(test_repo_config) -> None: runner.run(["teardown"], cwd=repo_path) -def make_feature_store_yaml(project, test_repo_config, repo_dir_name: PosixPath): +def make_feature_store_yaml(project, test_repo_config, repo_dir_name: Path): offline_creator: DataSourceCreator = test_repo_config.offline_store_creator(project) offline_store_config = offline_creator.create_offline_store_config() diff --git a/sdk/python/tests/integration/registration/test_universal_types.py b/sdk/python/tests/integration/registration/test_universal_types.py index c007d56c35d..bb6261313d2 100644 --- a/sdk/python/tests/integration/registration/test_universal_types.py +++ b/sdk/python/tests/integration/registration/test_universal_types.py @@ -1,7 +1,7 @@ import logging from dataclasses import dataclass from datetime import datetime, timedelta -from typing import List +from typing import Any, Dict, List, Tuple, Union import numpy as np import pandas as pd @@ -217,7 +217,7 @@ def test_feature_get_online_features_types_match(online_types_test_fixtures): ) fs = environment.feature_store features = [fv.name + ":value"] - entity = driver(value_type=ValueType.UNKNOWN) + entity = driver(value_type=config.entity_type) fs.apply([fv, entity]) fs.materialize(environment.start_date, environment.end_date) @@ -292,7 +292,9 @@ def assert_feature_list_types( provider: str, feature_dtype: str, historical_features_df: pd.DataFrame ): print("Asserting historical feature list types") - feature_list_dtype_to_expected_historical_feature_list_dtype = { + feature_list_dtype_to_expected_historical_feature_list_dtype: Dict[ + str, Union[type, Tuple[Union[type, Tuple[Any, ...]], ...]] + ] = { "int32": ( int, np.int64, diff --git a/sdk/python/tests/unit/test_unit_feature_store.py b/sdk/python/tests/unit/test_unit_feature_store.py new file mode 100644 index 00000000000..6f9dd6acb08 --- /dev/null +++ b/sdk/python/tests/unit/test_unit_feature_store.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass +from typing import Dict, List + +from feast import FeatureStore +from feast.protos.feast.types.Value_pb2 import Value + + +@dataclass +class MockFeatureViewProjection: + join_key_map: Dict[str, str] + + +@dataclass +class MockFeatureView: + name: str + entities: List[str] + projection: MockFeatureViewProjection + + +def test__get_unique_entities(): + entity_values = { + "entity_1": [Value(int64_val=1), Value(int64_val=2), Value(int64_val=1)], + "entity_2": [ + Value(string_val="1"), + Value(string_val="2"), + Value(string_val="1"), + ], + "entity_3": [Value(int64_val=8), Value(int64_val=9), Value(int64_val=10)], + } + + entity_name_to_join_key_map = {"entity_1": "entity_1", "entity_2": "entity_2"} + + fv = MockFeatureView( + name="fv_1", + entities=["entity_1", "entity_2"], + projection=MockFeatureViewProjection(join_key_map={}), + ) + + unique_entities, indexes = FeatureStore._get_unique_entities( + FeatureStore, + table=fv, + join_key_values=entity_values, + entity_name_to_join_key_map=entity_name_to_join_key_map, + ) + + assert unique_entities == ( + {"entity_1": Value(int64_val=1), "entity_2": Value(string_val="1")}, + {"entity_1": Value(int64_val=2), "entity_2": Value(string_val="2")}, + ) + assert indexes == ([0, 2], [1]) diff --git a/sdk/python/tests/utils/data_source_utils.py b/sdk/python/tests/utils/data_source_utils.py index 6e3d77ead0b..12870186bfc 100644 --- a/sdk/python/tests/utils/data_source_utils.py +++ b/sdk/python/tests/utils/data_source_utils.py @@ -2,6 +2,7 @@ import random import tempfile import time +from typing import Iterator from google.cloud import bigquery @@ -10,7 +11,7 @@ @contextlib.contextmanager -def prep_file_source(df, event_timestamp_column=None) -> FileSource: +def prep_file_source(df, event_timestamp_column=None) -> Iterator[FileSource]: with tempfile.NamedTemporaryFile(suffix=".parquet") as f: f.close() df.to_parquet(f.name)