From ad45bb4ac2dd83b530adda6196f85d46decaf98e Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Wed, 17 Apr 2024 09:13:51 -0700 Subject: [PATCH 01/73] fix: Pgvector patch (#4108) --- sdk/python/feast/feature_store.py | 19 ++++--- .../infra/online_stores/contrib/postgres.py | 55 +++++++++++++------ .../feast/infra/online_stores/online_store.py | 9 ++- sdk/python/feast/infra/provider.py | 9 ++- sdk/python/tests/foo_provider.py | 9 ++- 5 files changed, 73 insertions(+), 28 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 15598e1d609..f42cced11cf 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1740,12 +1740,14 @@ def _retrieve_online_documents( query, top_k, ) - document_feature_vals = [feature[2] for feature in document_features] - document_feature_distance_vals = [feature[3] for feature in document_features] - online_features_response = GetOnlineFeaturesResponse(results=[]) # TODO Refactor to better way of populating result # TODO populate entity in the response after returning entity in document_features is supported + # TODO currently not return the vector value since it is same as feature value, if embedding is supported, + # the feature value can be raw text before embedded + document_feature_vals = [feature[2] for feature in document_features] + document_feature_distance_vals = [feature[4] for feature in document_features] + online_features_response = GetOnlineFeaturesResponse(results=[]) self._populate_result_rows_from_columnar( online_features_response=online_features_response, data={requested_feature: document_feature_vals}, @@ -1979,7 +1981,7 @@ def _retrieve_from_online_store( requested_feature: str, query: List[float], top_k: int, - ) -> List[Tuple[Timestamp, "FieldStatus.ValueType", Value, Value]]: + ) -> List[Tuple[Timestamp, "FieldStatus.ValueType", Value, Value, Value]]: """ Search and return document features from the online document store. """ @@ -1994,19 +1996,22 @@ def _retrieve_from_online_store( read_row_protos = [] row_ts_proto = Timestamp() - for row_ts, feature_val, distance_val in documents: + for row_ts, feature_val, vector_value, distance_val in documents: # Reset timestamp to default or update if row_ts is not None if row_ts is not None: row_ts_proto.FromDatetime(row_ts) - if feature_val is None or distance_val is None: + if feature_val is None or vector_value is None or distance_val is None: feature_val = Value() + vector_value = Value() distance_val = Value() status = FieldStatus.NOT_FOUND else: status = FieldStatus.PRESENT - read_row_protos.append((row_ts_proto, status, feature_val, distance_val)) + read_row_protos.append( + (row_ts_proto, status, feature_val, vector_value, distance_val) + ) return read_row_protos @staticmethod diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 2890f60746b..6ed0885d138 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -75,10 +75,7 @@ def online_write_batch( for feature_name, val in values.items(): vector_val = None - if ( - "pgvector_enabled" in config.online_store - and config.online_store.pgvector_enabled - ): + if config.online_store.pgvector_enabled: vector_val = get_list_val_str(val) insert_values.append( ( @@ -226,10 +223,7 @@ def update( for table in tables_to_keep: table_name = _table_id(project, table) - if ( - "pgvector_enabled" in config.online_store - and config.online_store.pgvector_enabled - ): + if config.online_store.pgvector_enabled: vector_value_type = f"vector({config.online_store.vector_len})" else: # keep the vector_value_type as BYTEA if pgvector is not enabled, to maintain compatibility @@ -282,7 +276,14 @@ def retrieve_online_documents( requested_feature: str, embedding: List[float], top_k: int, - ) -> List[Tuple[Optional[datetime], Optional[ValueProto], Optional[ValueProto]]]: + ) -> List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: """ Args: @@ -297,10 +298,7 @@ def retrieve_online_documents( """ project = config.project - if ( - "pgvector_enabled" not in config.online_store - or not config.online_store.pgvector_enabled - ): + if not config.online_store.pgvector_enabled: raise ValueError( "pgvector is not enabled in the online store configuration" ) @@ -309,7 +307,12 @@ def retrieve_online_documents( query_embedding_str = f"[{','.join(str(el) for el in embedding)}]" result: List[ - Tuple[Optional[datetime], Optional[ValueProto], Optional[ValueProto]] + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] ] = [] with self._get_conn(config) as conn, conn.cursor() as cur: table_name = _table_id(project, table) @@ -322,6 +325,7 @@ def retrieve_online_documents( SELECT entity_key, feature_name, + value, vector_value, vector_value <-> %s as distance, event_ts FROM {table_name} @@ -338,16 +342,31 @@ def retrieve_online_documents( ) rows = cur.fetchall() - for entity_key, feature_name, vector_value, distance, event_ts in rows: + for ( + entity_key, + feature_name, + value, + vector_value, + distance, + event_ts, + ) in rows: # TODO Deserialize entity_key to return the entity in response # entity_key_proto = EntityKeyProto() # entity_key_proto_bin = bytes(entity_key) - # TODO Convert to List[float] for value type proto - feature_value_proto = ValueProto(string_val=vector_value) + feature_value_proto = ValueProto() + feature_value_proto.ParseFromString(bytes(value)) + vector_value_proto = ValueProto(string_val=vector_value) distance_value_proto = ValueProto(float_val=distance) - result.append((event_ts, feature_value_proto, distance_value_proto)) + result.append( + ( + event_ts, + feature_value_proto, + vector_value_proto, + distance_value_proto, + ) + ) return result diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index fc1b3d4ad30..67c5a931dda 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -142,7 +142,14 @@ def retrieve_online_documents( requested_feature: str, embedding: List[float], top_k: int, - ) -> List[Tuple[Optional[datetime], Optional[ValueProto], Optional[ValueProto]]]: + ) -> List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: """ Retrieves online feature values for the specified embeddings. diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index e71e87488d7..a45051a1b6b 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -303,7 +303,14 @@ def retrieve_online_documents( requested_feature: str, query: List[float], top_k: int, - ) -> List[Tuple[Optional[datetime], Optional[ValueProto], Optional[ValueProto]]]: + ) -> List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: """ Searches for the top-k nearest neighbors of the given document in the online document store. diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index 7ba4adb114b..2a830d424cc 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -111,5 +111,12 @@ def retrieve_online_documents( requested_feature: str, query: List[float], top_k: int, - ) -> List[Tuple[Optional[datetime], Optional[ValueProto], Optional[ValueProto]]]: + ) -> List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: return [] From b66baa46f48c72f4704bfe3980a8df49e1a06507 Mon Sep 17 00:00:00 2001 From: Jeremy Ary Date: Wed, 17 Apr 2024 12:45:01 -0500 Subject: [PATCH 02/73] revert: Reverts "fix: Using version args to install the correct feast version" (#4112) Revert "fix: Using version args to install the correct feast version (#3953)" This reverts commit b83a70227c6afe7258328ff5847a26b526d0b5df. --- .../feast/infra/feature_servers/multicloud/Dockerfile | 7 ++----- .../feast/infra/feature_servers/multicloud/Dockerfile.dev | 8 ++------ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index 1a55c6e8519..5100a2f822c 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,9 +1,5 @@ FROM python:3.9 -# Input the feast version to install -# This requires feast package to be available in pypi before building this image -ARG VERSION - RUN apt update && \ apt install -y \ jq \ @@ -11,7 +7,8 @@ RUN apt update && \ build-essential RUN pip install pip --upgrade -RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]==${VERSION}" +RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]" + RUN apt update RUN apt install -y -V ca-certificates lsb-release wget diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index 900578f55db..f92d3622a76 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -1,9 +1,5 @@ FROM python:3.9 -# Input the feast version to install -# This requires feast package to be available in pypi before building this image -ARG VERSION - RUN apt update && \ apt install -y \ jq \ @@ -13,11 +9,11 @@ RUN apt update && \ RUN pip install pip --upgrade COPY . . -RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]==${VERSION}" +RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]" RUN apt update RUN apt install -y -V ca-certificates lsb-release wget RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb RUN apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb RUN apt update -RUN apt -y install libarrow-dev +RUN apt -y install libarrow-dev \ No newline at end of file From f2b4eb94add8f86afa4e168236e8fcd11968510e Mon Sep 17 00:00:00 2001 From: Pushkar Gupta Date: Thu, 18 Apr 2024 11:49:05 -0700 Subject: [PATCH 03/73] feat: Feast/IKV online store contrib plugin integration (#4068) --- sdk/python/feast/cli.py | 1 + .../contrib/ikv_online_store/__init__.py | 0 .../contrib/ikv_online_store/ikv.py | 300 ++++++++++++++++++ sdk/python/feast/repo_config.py | 1 + .../feature_repos/repo_configuration.py | 13 + setup.py | 5 + 6 files changed, 320 insertions(+) create mode 100644 sdk/python/feast/infra/online_stores/contrib/ikv_online_store/__init__.py create mode 100644 sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index f23cc62bd81..13927e28563 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -595,6 +595,7 @@ def materialize_incremental_command(ctx: click.Context, end_ts: str, views: List "cassandra", "rockset", "hazelcast", + "ikv", ], case_sensitive=False, ), diff --git a/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/__init__.py b/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py b/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py new file mode 100644 index 00000000000..9d888aad3d8 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py @@ -0,0 +1,300 @@ +from datetime import datetime +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Literal, + Optional, + Sequence, + Tuple, +) + +from ikvpy.client import IKVReader, IKVWriter +from ikvpy.clientoptions import ClientOptions, ClientOptionsBuilder +from ikvpy.document import IKVDocument, IKVDocumentBuilder +from ikvpy.factory import create_new_reader, create_new_writer +from pydantic import StrictStr + +from feast import Entity, FeatureView, utils +from feast.infra.online_stores.helpers import compute_entity_id +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.usage import log_exceptions_and_usage + +PRIMARY_KEY_FIELD_NAME: str = "_entity_key" +EVENT_CREATION_TIMESTAMP_FIELD_NAME: str = "_event_timestamp" +CREATION_TIMESTAMP_FIELD_NAME: str = "_created_timestamp" + + +class IKVOnlineStoreConfig(FeastConfigBaseModel): + """Online store config for IKV store""" + + type: Literal["ikv"] = "ikv" + """Online store type selector""" + + account_id: StrictStr + """(Required) IKV account id""" + + account_passkey: StrictStr + """(Required) IKV account passkey""" + + store_name: StrictStr + """(Required) IKV store name""" + + mount_directory: Optional[StrictStr] = None + """(Required only for reader) IKV mount point i.e. directory for storing IKV data locally.""" + + +class IKVOnlineStore(OnlineStore): + """ + IKV (inlined.io key value) store implementation of the online store interface. + """ + + # lazy initialization + _reader: Optional[IKVReader] = None + _writer: Optional[IKVWriter] = None + + @log_exceptions_and_usage(online_store="ikv") + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + """ + Writes a batch of feature rows to the online store. + + If a tz-naive timestamp is passed to this method, it is assumed to be UTC. + + Args: + config: The config for the current feature store. + table: Feature view to which these feature rows correspond. + data: A list of quadruplets containing feature data. Each quadruplet contains an entity + key, a dict containing feature values, an event timestamp for the row, and the created + timestamp for the row if it exists. + progress: Function to be called once a batch of rows is written to the online store, used + to show progress. + """ + # update should have been called before + if self._writer is None: + return + + for entity_key, features, event_timestamp, _ in data: + entity_id: str = compute_entity_id( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + document: IKVDocument = IKVOnlineStore._create_document( + entity_id, table, features, event_timestamp + ) + self._writer.upsert_fields(document) + if progress: + progress(1) + + @log_exceptions_and_usage(online_store="ikv") + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + """ + Reads features values for the given entity keys. + + Args: + config: The config for the current feature store. + table: The feature view whose feature values should be read. + entity_keys: The list of entity keys for which feature values should be read. + requested_features: The list of features that should be read. + + Returns: + A list of the same length as entity_keys. Each item in the list is a tuple where the first + item is the event timestamp for the row, and the second item is a dict mapping feature names + to values, which are returned in proto format. + """ + if not len(entity_keys): + return [] + + # create IKV primary keys + primary_keys = [ + compute_entity_id(ek, config.entity_key_serialization_version) + for ek in entity_keys + ] + + # create IKV field names + if requested_features is None: + requested_features = [] + + field_names: List[Optional[str]] = [None] * (1 + len(requested_features)) + field_names[0] = EVENT_CREATION_TIMESTAMP_FIELD_NAME + for i, fn in enumerate(requested_features): + field_names[i + 1] = IKVOnlineStore._create_ikv_field_name(table, fn) + + assert self._reader is not None + value_iter = self._reader.multiget_bytes_values( + bytes_primary_keys=[], + str_primary_keys=primary_keys, + field_names=field_names, + ) + + # decode results + return [ + IKVOnlineStore._decode_fields_for_primary_key( + requested_features, value_iter + ) + for _ in range(0, len(primary_keys)) + ] + + @staticmethod + def _decode_fields_for_primary_key( + requested_features: List[str], value_iter: Iterator[Optional[bytes]] + ) -> Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]: + # decode timestamp + dt: Optional[datetime] = None + dt_bytes = next(value_iter) + if dt_bytes: + dt = datetime.fromisoformat(str(dt_bytes, "utf-8")) + + # decode other features + features = {} + for requested_feature in requested_features: + value_proto_bytes: Optional[bytes] = next(value_iter) + if value_proto_bytes: + value_proto = ValueProto() + value_proto.ParseFromString(value_proto_bytes) + features[requested_feature] = value_proto + + return dt, features + + # called before any read/write requests are issued + @log_exceptions_and_usage(online_store="ikv") + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + """ + Reconciles cloud resources with the specified set of Feast objects. + + Args: + config: The config for the current feature store. + tables_to_delete: Feature views whose corresponding infrastructure should be deleted. + tables_to_keep: Feature views whose corresponding infrastructure should not be deleted, and + may need to be updated. + entities_to_delete: Entities whose corresponding infrastructure should be deleted. + entities_to_keep: Entities whose corresponding infrastructure should not be deleted, and + may need to be updated. + partial: If true, tables_to_delete and tables_to_keep are not exhaustive lists, so + infrastructure corresponding to other feature views should be not be touched. + """ + self._init_clients(config=config) + assert self._writer is not None + + # note: we assume tables_to_keep does not overlap with tables_to_delete + + for feature_view in tables_to_delete: + # each field in an IKV document is prefixed by the feature-view's name + self._writer.drop_fields_by_name_prefix([feature_view.name]) + + @log_exceptions_and_usage(online_store="ikv") + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ): + """ + Tears down all cloud resources for the specified set of Feast objects. + + Args: + config: The config for the current feature store. + tables: Feature views whose corresponding infrastructure should be deleted. + entities: Entities whose corresponding infrastructure should be deleted. + """ + self._init_clients(config=config) + assert self._writer is not None + + # drop fields corresponding to this feature-view + for feature_view in tables: + self._writer.drop_fields_by_name_prefix([feature_view.name]) + + # shutdown clients + self._writer.shutdown() + self._writer = None + + if self._reader is not None: + self._reader.shutdown() + self._reader = None + + @staticmethod + def _create_ikv_field_name(feature_view: FeatureView, feature_name: str) -> str: + return "{}_{}".format(feature_view.name, feature_name) + + @staticmethod + def _create_document( + entity_id: str, + feature_view: FeatureView, + values: Dict[str, ValueProto], + event_timestamp: datetime, + ) -> IKVDocument: + """Converts feast key-value pairs into an IKV document.""" + + # initialie builder by inserting primary key and row creation timestamp + event_timestamp_str: str = utils.make_tzaware(event_timestamp).isoformat() + builder = ( + IKVDocumentBuilder() + .put_string_field(PRIMARY_KEY_FIELD_NAME, entity_id) + .put_bytes_field( + EVENT_CREATION_TIMESTAMP_FIELD_NAME, event_timestamp_str.encode("utf-8") + ) + ) + + for feature_name, feature_value in values.items(): + field_name = IKVOnlineStore._create_ikv_field_name( + feature_view, feature_name + ) + builder.put_bytes_field(field_name, feature_value.SerializeToString()) + + return builder.build() + + def _init_clients(self, config: RepoConfig): + """Initializes (if required) reader/writer ikv clients.""" + online_config = config.online_store + assert isinstance(online_config, IKVOnlineStoreConfig) + client_options = IKVOnlineStore._config_to_client_options(online_config) + + # initialize writer + if self._writer is None: + self._writer = create_new_writer(client_options) + + # initialize reader, iff mount_dir is specified + if self._reader is None: + if online_config.mount_directory and len(online_config.mount_directory) > 0: + self._reader = create_new_reader(client_options) + + @staticmethod + def _config_to_client_options(config: IKVOnlineStoreConfig) -> ClientOptions: + """Utility for IKVOnlineStoreConfig to IKV ClientOptions conversion.""" + builder = ( + ClientOptionsBuilder() + .with_account_id(config.account_id) + .with_account_passkey(config.account_passkey) + .with_store_name(config.store_name) + ) + + if config.mount_directory and len(config.mount_directory) > 0: + builder = builder.with_mount_directory(config.mount_directory) + + return builder.build() diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index e8185f4a4ab..5e38fd17758 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -63,6 +63,7 @@ "mysql": "feast.infra.online_stores.contrib.mysql_online_store.mysql.MySQLOnlineStore", "rockset": "feast.infra.online_stores.contrib.rockset_online_store.rockset.RocksetOnlineStore", "hazelcast": "feast.infra.online_stores.contrib.hazelcast_online_store.hazelcast_online_store.HazelcastOnlineStore", + "ikv": "feast.infra.online_stores.contrib.ikv_online_store.ikv.IKVOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 6eb52041617..04e69e04c65 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -100,6 +100,14 @@ "host": os.getenv("ROCKSET_APISERVER", "api.rs2.usw2.rockset.com"), } +IKV_CONFIG = { + "type": "ikv", + "account_id": os.getenv("IKV_ACCOUNT_ID", ""), + "account_passkey": os.getenv("IKV_ACCOUNT_PASSKEY", ""), + "store_name": os.getenv("IKV_STORE_NAME", ""), + "mount_directory": os.getenv("IKV_MOUNT_DIR", ""), +} + OFFLINE_STORE_TO_PROVIDER_CONFIG: Dict[str, Tuple[str, Type[DataSourceCreator]]] = { "file": ("local", FileDataSourceCreator), "bigquery": ("gcp", BigQueryDataSourceCreator), @@ -139,6 +147,11 @@ # containerized version of Rockset. # AVAILABLE_ONLINE_STORES["rockset"] = (ROCKSET_CONFIG, None) + # Uncomment to test using private IKV account. Currently not enabled as + # there is no dedicated IKV instance for CI testing and there is no + # containerized version of IKV. + # AVAILABLE_ONLINE_STORES["ikv"] = (IKV_CONFIG, None) + full_repo_configs_module = os.environ.get(FULL_REPO_CONFIGS_MODULE_ENV_NAME) if full_repo_configs_module is not None: diff --git a/setup.py b/setup.py index 610e2a66ca7..b1a79bdd98f 100644 --- a/setup.py +++ b/setup.py @@ -130,6 +130,10 @@ "rockset>=1.0.3", ] +IKV_REQUIRED = [ + "ikvpy>=0.0.23", +] + HAZELCAST_REQUIRED = [ "hazelcast-python-client>=5.1", ] @@ -372,6 +376,7 @@ def run(self): "rockset": ROCKSET_REQUIRED, "ibis": IBIS_REQUIRED, "duckdb": DUCKDB_REQUIRED, + "ikv": IKV_REQUIRED }, include_package_data=True, license="Apache", From c3a102f1b1941c8681ec876b54d7d16a32862925 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Fri, 19 Apr 2024 21:35:17 +0400 Subject: [PATCH 04/73] feat: Incorporate substrait ODFVs into ibis-based offline store queries (#4102) --- protos/feast/core/Transformation.proto | 1 + sdk/python/feast/infra/offline_stores/ibis.py | 8 ++- sdk/python/feast/on_demand_feature_view.py | 55 ++++++++++++++++++- .../transformation/pandas_transformation.py | 4 +- .../transformation/python_transformation.py | 4 +- .../substrait_transformation.py | 37 ++++++++++--- .../feature_repos/repo_configuration.py | 8 ++- .../feature_repos/universal/feature_views.py | 20 ++++++- .../test_universal_historical_retrieval.py | 11 +++- .../unit/test_substrait_transformation.py | 6 +- 10 files changed, 132 insertions(+), 22 deletions(-) diff --git a/protos/feast/core/Transformation.proto b/protos/feast/core/Transformation.proto index 36f1e691fe1..5cb53e690fa 100644 --- a/protos/feast/core/Transformation.proto +++ b/protos/feast/core/Transformation.proto @@ -29,4 +29,5 @@ message FeatureTransformationV2 { message SubstraitTransformationV2 { bytes substrait_plan = 1; + bytes ibis_function = 2; } diff --git a/sdk/python/feast/infra/offline_stores/ibis.py b/sdk/python/feast/infra/offline_stores/ibis.py index f9c6b2d20b3..6e0729d6d11 100644 --- a/sdk/python/feast/infra/offline_stores/ibis.py +++ b/sdk/python/feast/infra/offline_stores/ibis.py @@ -193,9 +193,15 @@ def read_fv( event_timestamp_col=event_timestamp_col, ) + odfvs = OnDemandFeatureView.get_requested_odfvs(feature_refs, project, registry) + + substrait_odfvs = [fv for fv in odfvs if fv.mode == "substrait"] + for odfv in substrait_odfvs: + res = odfv.transform_ibis(res, full_feature_names) + return IbisRetrievalJob( res, - OnDemandFeatureView.get_requested_odfvs(feature_refs, project, registry), + [fv for fv in odfvs if fv.mode != "substrait"], full_feature_names, metadata=RetrievalMetadata( features=feature_refs, diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index cfb322fb2d1..b532fa651a1 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -392,6 +392,53 @@ def get_request_data_schema(self) -> Dict[str, ValueType]: def _get_projected_feature_name(self, feature: str) -> str: return f"{self.projection.name_to_use()}__{feature}" + def transform_ibis( + self, + ibis_table, + full_feature_names: bool = False, + ): + from ibis.expr.types import Table + + if not isinstance(ibis_table, Table): + raise TypeError("transform_ibis only accepts ibis.expr.types.Table") + + assert type(self.feature_transformation) == SubstraitTransformation + + columns_to_cleanup = [] + for source_fv_projection in self.source_feature_view_projections.values(): + for feature in source_fv_projection.features: + full_feature_ref = f"{source_fv_projection.name}__{feature.name}" + if full_feature_ref in ibis_table.columns: + # Make sure the partial feature name is always present + ibis_table = ibis_table.mutate( + **{feature.name: ibis_table[full_feature_ref]} + ) + columns_to_cleanup.append(feature.name) + elif feature.name in ibis_table.columns: + ibis_table = ibis_table.mutate( + **{full_feature_ref: ibis_table[feature.name]} + ) + columns_to_cleanup.append(full_feature_ref) + + transformed_table = self.feature_transformation.transform_ibis(ibis_table) + + transformed_table = transformed_table.drop(*columns_to_cleanup) + + rename_columns: Dict[str, str] = {} + for feature in self.features: + short_name = feature.name + long_name = self._get_projected_feature_name(feature.name) + if short_name in transformed_table.columns and full_feature_names: + rename_columns[short_name] = long_name + elif not full_feature_names: + rename_columns[long_name] = short_name + + for rename_from, rename_to in rename_columns.items(): + if rename_from in transformed_table.columns: + transformed_table = transformed_table.rename(**{rename_to: rename_from}) + + return transformed_table + def transform_arrow( self, pa_table: pyarrow.Table, @@ -419,7 +466,7 @@ def transform_arrow( columns_to_cleanup.append(full_feature_ref) df_with_transformed_features: pyarrow.Table = ( - self.feature_transformation.transform_arrow(pa_table) + self.feature_transformation.transform_arrow(pa_table, self.features) ) # Work out whether the correct columns names are used. @@ -438,7 +485,7 @@ def transform_arrow( # Cleanup extra columns used for transformation for col in columns_to_cleanup: if col in df_with_transformed_features.column_names: - df_with_transformed_features = df_with_transformed_features.dtop(col) + df_with_transformed_features = df_with_transformed_features.drop(col) return df_with_transformed_features.rename_columns( [ rename_columns.get(c, c) @@ -487,7 +534,9 @@ def get_transformed_features_df( rename_columns[long_name] = short_name # Cleanup extra columns used for transformation - df_with_features.drop(columns=columns_to_cleanup, inplace=True) + df_with_transformed_features = df_with_transformed_features[ + [f.name for f in self.features] + ] return df_with_transformed_features.rename(columns=rename_columns) def get_transformed_features_dict( diff --git a/sdk/python/feast/transformation/pandas_transformation.py b/sdk/python/feast/transformation/pandas_transformation.py index 28f3c22b9f3..7e706810cb4 100644 --- a/sdk/python/feast/transformation/pandas_transformation.py +++ b/sdk/python/feast/transformation/pandas_transformation.py @@ -27,7 +27,9 @@ def __init__(self, udf: FunctionType, udf_string: str = ""): self.udf = udf self.udf_string = udf_string - def transform_arrow(self, pa_table: pyarrow.Table) -> pyarrow.Table: + def transform_arrow( + self, pa_table: pyarrow.Table, features: List[Field] + ) -> pyarrow.Table: if not isinstance(pa_table, pyarrow.Table): raise TypeError( f"pa_table should be type pyarrow.Table but got {type(pa_table).__name__}" diff --git a/sdk/python/feast/transformation/python_transformation.py b/sdk/python/feast/transformation/python_transformation.py index 1245fc52ed0..ec950a24f3c 100644 --- a/sdk/python/feast/transformation/python_transformation.py +++ b/sdk/python/feast/transformation/python_transformation.py @@ -25,7 +25,9 @@ def __init__(self, udf: FunctionType, udf_string: str = ""): self.udf = udf self.udf_string = udf_string - def transform_arrow(self, pa_table: pyarrow.Table) -> pyarrow.Table: + def transform_arrow( + self, pa_table: pyarrow.Table, features: List[Field] + ) -> pyarrow.Table: raise Exception( 'OnDemandFeatureView mode "python" not supported for offline processing.' ) diff --git a/sdk/python/feast/transformation/substrait_transformation.py b/sdk/python/feast/transformation/substrait_transformation.py index a816f8118ac..48a87b62079 100644 --- a/sdk/python/feast/transformation/substrait_transformation.py +++ b/sdk/python/feast/transformation/substrait_transformation.py @@ -1,5 +1,7 @@ +from types import FunctionType from typing import Any, Dict, List +import dill import pandas as pd import pyarrow import pyarrow.substrait as substrait # type: ignore # noqa @@ -16,14 +18,16 @@ class SubstraitTransformation: - def __init__(self, substrait_plan: bytes): + def __init__(self, substrait_plan: bytes, ibis_function: FunctionType): """ Creates an SubstraitTransformation object. Args: substrait_plan: The user-provided substrait plan. + ibis_function: The user-provided ibis function. """ self.substrait_plan = substrait_plan + self.ibis_function = ibis_function def transform(self, df: pd.DataFrame) -> pd.DataFrame: def table_provider(names, schema: pyarrow.Schema): @@ -34,13 +38,22 @@ def table_provider(names, schema: pyarrow.Schema): ).read_all() return table.to_pandas() - def transform_arrow(self, pa_table: pyarrow.Table) -> pyarrow.Table: + def transform_ibis(self, table): + return self.ibis_function(table) + + def transform_arrow( + self, pa_table: pyarrow.Table, features: List[Field] = [] + ) -> pyarrow.Table: def table_provider(names, schema: pyarrow.Schema): return pa_table.select(schema.names) table: pyarrow.Table = pyarrow.substrait.run_query( self.substrait_plan, table_provider=table_provider ).read_all() + + if features: + table = table.select([f.name for f in features]) + return table def infer_features(self, random_input: Dict[str, List[Any]]) -> List[Field]: @@ -55,6 +68,7 @@ def infer_features(self, random_input: Dict[str, List[Any]]) -> List[Field]: ), ) for f, dt in zip(output_df.columns, output_df.dtypes) + if f not in random_input ] def __eq__(self, other): @@ -66,10 +80,17 @@ def __eq__(self, other): if not super().__eq__(other): return False - return self.substrait_plan == other.substrait_plan + return ( + self.substrait_plan == other.substrait_plan + and self.ibis_function.__code__.co_code + == other.ibis_function.__code__.co_code + ) def to_proto(self) -> SubstraitTransformationProto: - return SubstraitTransformationProto(substrait_plan=self.substrait_plan) + return SubstraitTransformationProto( + substrait_plan=self.substrait_plan, + ibis_function=dill.dumps(self.ibis_function, recurse=True), + ) @classmethod def from_proto( @@ -77,7 +98,8 @@ def from_proto( substrait_transformation_proto: SubstraitTransformationProto, ): return SubstraitTransformation( - substrait_plan=substrait_transformation_proto.substrait_plan + substrait_plan=substrait_transformation_proto.substrait_plan, + ibis_function=dill.loads(substrait_transformation_proto.ibis_function), ) @classmethod @@ -91,7 +113,7 @@ def from_ibis(cls, user_function, sources): input_fields = [] for s in sources: - fields = s.projection.features if isinstance(s, FeatureView) else s.features + fields = s.projection.features if isinstance(s, FeatureView) else s.schema input_fields.extend( [ @@ -108,5 +130,6 @@ def from_ibis(cls, user_function, sources): expr = user_function(ibis.table(input_fields, "t")) return SubstraitTransformation( - substrait_plan=compiler.compile(expr).SerializeToString() + substrait_plan=compiler.compile(expr).SerializeToString(), + ibis_function=user_function, ) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 04e69e04c65..a57a017699b 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -356,17 +356,23 @@ def values(self): def construct_universal_feature_views( data_sources: UniversalDataSources, with_odfv: bool = True, + use_substrait_odfv: bool = False, ) -> UniversalFeatureViews: driver_hourly_stats = create_driver_hourly_stats_feature_view(data_sources.driver) driver_hourly_stats_base_feature_view = ( create_driver_hourly_stats_batch_feature_view(data_sources.driver) ) + return UniversalFeatureViews( customer=create_customer_daily_profile_feature_view(data_sources.customer), global_fv=create_global_stats_feature_view(data_sources.global_ds), driver=driver_hourly_stats, driver_odfv=conv_rate_plus_100_feature_view( - [driver_hourly_stats_base_feature_view, create_conv_rate_request_source()] + [ + driver_hourly_stats_base_feature_view[["conv_rate"]], + create_conv_rate_request_source(), + ], + use_substrait_odfv=use_substrait_odfv, ) if with_odfv else None, diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 48f6e27b8ae..2a0a9d1bd01 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -3,6 +3,7 @@ import numpy as np import pandas as pd +from ibis.expr.types.relations import Table from feast import ( BatchFeatureView, @@ -15,7 +16,7 @@ ) from feast.data_source import DataSource, RequestSource from feast.feature_view_projection import FeatureViewProjection -from feast.on_demand_feature_view import PandasTransformation +from feast.on_demand_feature_view import PandasTransformation, SubstraitTransformation from feast.types import Array, FeastType, Float32, Float64, Int32, Int64 from tests.integration.feature_repos.universal.entities import ( customer, @@ -56,10 +57,22 @@ def conv_rate_plus_100(features_df: pd.DataFrame) -> pd.DataFrame: return df +def conv_rate_plus_100_ibis(features_table: Table) -> Table: + return features_table.mutate( + conv_rate_plus_100=features_table["conv_rate"] + 100, + conv_rate_plus_val_to_add=features_table["conv_rate"] + + features_table["val_to_add"], + conv_rate_plus_100_rounded=(features_table["conv_rate"] + 100) + .round(digits=0) + .cast("int32"), + ) + + def conv_rate_plus_100_feature_view( sources: List[Union[FeatureView, RequestSource, FeatureViewProjection]], infer_features: bool = False, features: Optional[List[Field]] = None, + use_substrait_odfv: bool = False, ) -> OnDemandFeatureView: # Test that positional arguments and Features still work for ODFVs. _features = features or [ @@ -73,7 +86,10 @@ def conv_rate_plus_100_feature_view( sources=sources, feature_transformation=PandasTransformation( udf=conv_rate_plus_100, udf_string="raw udf source" - ), + ) + if not use_substrait_odfv + else SubstraitTransformation.from_ibis(conv_rate_plus_100_ibis, sources), + mode="pandas" if not use_substrait_odfv else "substrait", ) diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py index 7e106b3e2a6..958b829a60b 100644 --- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py +++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py @@ -41,12 +41,19 @@ @pytest.mark.integration @pytest.mark.universal_offline_stores @pytest.mark.parametrize("full_feature_names", [True, False], ids=lambda v: f"full:{v}") -def test_historical_features(environment, universal_data_sources, full_feature_names): +@pytest.mark.parametrize( + "use_substrait_odfv", [True, False], ids=lambda v: f"substrait:{v}" +) +def test_historical_features( + environment, universal_data_sources, full_feature_names, use_substrait_odfv +): store = environment.feature_store (entities, datasets, data_sources) = universal_data_sources - feature_views = construct_universal_feature_views(data_sources) + feature_views = construct_universal_feature_views( + data_sources, use_substrait_odfv=use_substrait_odfv + ) entity_df_with_request_data = datasets.entity_df.copy(deep=True) entity_df_with_request_data["val_to_add"] = [ diff --git a/sdk/python/tests/unit/test_substrait_transformation.py b/sdk/python/tests/unit/test_substrait_transformation.py index 28ab68c70be..351651cfda7 100644 --- a/sdk/python/tests/unit/test_substrait_transformation.py +++ b/sdk/python/tests/unit/test_substrait_transformation.py @@ -75,10 +75,8 @@ def pandas_view(inputs: pd.DataFrame) -> pd.DataFrame: mode="substrait", ) def substrait_view(inputs: Table) -> Table: - return inputs.select( - (inputs["conv_rate"] + inputs["acc_rate"]).name( - "conv_rate_plus_acc_substrait" - ) + return inputs.mutate( + conv_rate_plus_acc_substrait=inputs["conv_rate"] + inputs["acc_rate"] ) store.apply( From 65056cea6c4537834a1c40be2ad37e1659310a47 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Sat, 20 Apr 2024 00:48:10 +0400 Subject: [PATCH 05/73] fix: Get rid of empty string `name_alias` during feature view projection deserialization (#4116) fix feature view projection warning Signed-off-by: tokoko --- sdk/python/feast/feature_view_projection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/feature_view_projection.py b/sdk/python/feast/feature_view_projection.py index 2960996a10c..ff5b1b6e063 100644 --- a/sdk/python/feast/feature_view_projection.py +++ b/sdk/python/feast/feature_view_projection.py @@ -53,7 +53,7 @@ def to_proto(self) -> FeatureViewProjectionProto: def from_proto(proto: FeatureViewProjectionProto): feature_view_projection = FeatureViewProjection( name=proto.feature_view_name, - name_alias=proto.feature_view_name_alias, + name_alias=proto.feature_view_name_alias or None, features=[], join_key_map=dict(proto.join_key_map), desired_features=[], From 2ba71fff5f76ed05066e94f3b11d08bc30b54b39 Mon Sep 17 00:00:00 2001 From: Stephen Yau Date: Sat, 20 Apr 2024 04:48:35 +0800 Subject: [PATCH 06/73] fix: Change numpy version <1.25 dependency to <2 in setup.py (#4085) Fixes #4084 change numpy version to <2 to avoid installation error Signed-off-by: stephenyau --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b1a79bdd98f..fd3186f296b 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ "Jinja2>=2,<4", "jsonschema", "mmh3", - "numpy>=1.22,<1.25", + "numpy>=1.22,<2", "pandas>=1.4.3,<3", # Higher than 4.23.4 seems to cause a seg fault "protobuf>=4.24.0,<5.0.0", From 2c4a347846bb3918102db3112f64606dfb1484d2 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Sat, 20 Apr 2024 00:53:21 +0400 Subject: [PATCH 07/73] chore: Drop dependency on non-standard importlib libraries (#4109) drop dependency on non-standard importlib libraries Signed-off-by: tokoko --- sdk/python/feast/__init__.py | 8 +- sdk/python/feast/cli.py | 2 +- sdk/python/feast/proto_json.py | 2 +- sdk/python/feast/ui_server.py | 2 +- sdk/python/feast/version.py | 5 +- .../requirements/py3.10-ci-requirements.txt | 76 +++++++++--------- .../requirements/py3.10-requirements.txt | 18 ++--- .../requirements/py3.9-ci-requirements.txt | 77 +++++++++---------- .../requirements/py3.9-requirements.txt | 19 ++--- setup.py | 2 - 10 files changed, 92 insertions(+), 119 deletions(-) diff --git a/sdk/python/feast/__init__.py b/sdk/python/feast/__init__.py index f51eb2983c9..52734bc71ec 100644 --- a/sdk/python/feast/__init__.py +++ b/sdk/python/feast/__init__.py @@ -1,9 +1,5 @@ -try: - from importlib.metadata import PackageNotFoundError - from importlib.metadata import version as _version -except ModuleNotFoundError: - from importlib_metadata import PackageNotFoundError # type: ignore - from importlib_metadata import version as _version +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _version from feast.infra.offline_stores.bigquery_source import BigQuerySource from feast.infra.offline_stores.contrib.athena_offline_store.athena_source import ( diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 13927e28563..88bbd4f4327 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -14,6 +14,7 @@ import json import logging from datetime import datetime +from importlib.metadata import version as importlib_version from pathlib import Path from typing import List, Optional @@ -21,7 +22,6 @@ import yaml from colorama import Fore, Style from dateutil import parser -from importlib_metadata import version as importlib_version from pygments import formatters, highlight, lexers from feast import utils diff --git a/sdk/python/feast/proto_json.py b/sdk/python/feast/proto_json.py index 41d2afa55a7..487dc4284f3 100644 --- a/sdk/python/feast/proto_json.py +++ b/sdk/python/feast/proto_json.py @@ -1,4 +1,5 @@ import uuid +from importlib.metadata import version as importlib_version from typing import Any, Callable, Type from google.protobuf.json_format import ( # type: ignore @@ -7,7 +8,6 @@ _Parser, _Printer, ) -from importlib_metadata import version as importlib_version from packaging import version from feast.protos.feast.serving.ServingService_pb2 import FeatureList diff --git a/sdk/python/feast/ui_server.py b/sdk/python/feast/ui_server.py index 8a39293f918..1e0d87a64e3 100644 --- a/sdk/python/feast/ui_server.py +++ b/sdk/python/feast/ui_server.py @@ -1,8 +1,8 @@ import json import threading +from importlib import resources as importlib_resources from typing import Callable, Optional -import importlib_resources import uvicorn from fastapi import FastAPI, Response from fastapi.middleware.cors import CORSMiddleware diff --git a/sdk/python/feast/version.py b/sdk/python/feast/version.py index 3e42643ccbe..85d8476a66d 100644 --- a/sdk/python/feast/version.py +++ b/sdk/python/feast/version.py @@ -1,7 +1,4 @@ -try: - from importlib.metadata import PackageNotFoundError, version -except ModuleNotFoundError: - from importlib_metadata import PackageNotFoundError, version # type: ignore +from importlib.metadata import PackageNotFoundError, version def get_version(): diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 424a433be46..2d91e5cf41b 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -45,7 +45,7 @@ azure-core==1.30.1 # via # azure-identity # azure-storage-blob -azure-identity==1.15.0 +azure-identity==1.16.0 # via feast (setup.py) azure-storage-blob==12.19.1 # via feast (setup.py) @@ -59,11 +59,11 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.80 +boto3==1.34.85 # via # feast (setup.py) # moto -botocore==1.34.80 +botocore==1.34.85 # via # boto3 # moto @@ -138,7 +138,7 @@ dask[array,dataframe]==2024.4.1 # via # dask-expr # feast (setup.py) -dask-expr==1.0.10 +dask-expr==1.0.11 # via dask db-dtypes==1.2.0 # via google-cloud-bigquery @@ -148,12 +148,8 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deprecation==2.1.0 - # via testcontainers dill==0.3.8 - # via - # feast (setup.py) - # multiprocess + # via feast (setup.py) distlib==0.3.8 # via virtualenv docker==7.0.0 @@ -166,7 +162,7 @@ duckdb==0.10.1 # via # duckdb-engine # ibis-framework -duckdb-engine==0.11.4 +duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair @@ -183,7 +179,7 @@ fastapi==0.110.1 # via feast (setup.py) fastjsonschema==2.19.1 # via nbformat -filelock==3.13.3 +filelock==3.13.4 # via # snowflake-connector-python # virtualenv @@ -213,7 +209,7 @@ google-api-core[grpc]==2.18.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.125.0 +google-api-python-client==2.126.0 # via firebase-admin google-auth==2.29.0 # via @@ -230,7 +226,7 @@ google-cloud-bigquery[pandas]==3.12.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) -google-cloud-bigtable==2.23.0 +google-cloud-bigtable==2.23.1 # via feast (setup.py) google-cloud-core==2.4.1 # via @@ -330,12 +326,8 @@ idna==3.7 # snowflake-connector-python imagesize==1.4.1 # via sphinx -importlib-metadata==6.11.0 - # via - # dask - # feast (setup.py) -importlib-resources==6.4.0 - # via feast (setup.py) +importlib-metadata==7.1.0 + # via dask iniconfig==2.0.0 # via pytest ipykernel==6.29.4 @@ -368,7 +360,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.24 +json5==0.9.25 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -402,9 +394,9 @@ jupyter-core==5.7.2 # nbformat jupyter-events==0.10.0 # via jupyter-server -jupyter-lsp==2.2.4 +jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.13.0 +jupyter-server==2.14.0 # via # jupyter-lsp # jupyterlab @@ -438,7 +430,7 @@ markupsafe==2.1.5 # werkzeug marshmallow==3.21.1 # via great-expectations -matplotlib-inline==0.1.6 +matplotlib-inline==0.1.7 # via # ipykernel # ipython @@ -511,12 +503,11 @@ oauthlib==3.2.2 # via requests-oauthlib overrides==7.7.0 # via jupyter-server -packaging==21.3 +packaging==24.0 # via # build # dask # db-dtypes - # deprecation # docker # duckdb-engine # google-cloud-bigquery @@ -533,7 +524,7 @@ packaging==21.3 # pytest # snowflake-connector-python # sphinx -pandas==2.2.1 +pandas==2.2.2 # via # altair # dask @@ -640,12 +631,12 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.6.4 +pydantic==2.7.0 # via # fastapi # feast (setup.py) # great-expectations -pydantic-core==2.16.3 +pydantic-core==2.18.1 # via pydantic pygments==2.17.2 # via @@ -670,7 +661,6 @@ pyparsing==3.1.2 # via # great-expectations # httplib2 - # packaging pyproject-hooks==1.0.0 # via # build @@ -738,7 +728,7 @@ pyyaml==6.0.1 # pre-commit # responses # uvicorn -pyzmq==25.1.2 +pyzmq==26.0.0 # via # ipykernel # jupyter-client @@ -750,7 +740,7 @@ referencing==0.34.0 # jsonschema # jsonschema-specifications # jupyter-events -regex==2023.12.25 +regex==2024.4.16 # via feast (setup.py) requests==2.31.0 # via @@ -795,7 +785,7 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruff==0.3.5 +ruff==0.3.7 # via feast (setup.py) s3transfer==0.10.1 # via boto3 @@ -822,7 +812,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.7.1 +snowflake-connector-python[pandas]==3.8.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -857,7 +847,7 @@ stack-data==0.6.3 # via ipython starlette==0.37.2 # via fastapi -substrait==0.15.0 +substrait==0.16.0 # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) @@ -867,7 +857,7 @@ terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -testcontainers==3.7.1 +testcontainers==4.3.3 # via feast (setup.py) thriftpy2==0.4.20 # via happybase @@ -924,28 +914,32 @@ trino==0.328.0 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) +types-cffi==1.16.0.20240331 + # via types-pyopenssl types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==24.0.0.20240311 +types-pyopenssl==24.0.0.20240417 # via types-redis types-python-dateutil==2.9.0.20240316 # via # arrow # feast (setup.py) -types-pytz==2024.1.0.20240203 +types-pytz==2024.1.0.20240417 # via feast (setup.py) types-pyyaml==6.0.12.20240311 # via feast (setup.py) -types-redis==4.6.0.20240409 +types-redis==4.6.0.20240417 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.2.0.20240317 - # via feast (setup.py) +types-setuptools==69.5.0.20240415 + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 # via feast (setup.py) types-urllib3==1.26.25.14 @@ -965,6 +959,7 @@ typing-extensions==4.11.0 # pydantic-core # snowflake-connector-python # sqlalchemy + # testcontainers # typeguard # uvicorn tzdata==2024.1 @@ -988,6 +983,7 @@ urllib3==1.26.18 # requests # responses # rockset + # testcontainers uvicorn[standard]==0.29.0 # via feast (setup.py) uvloop==0.19.0 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index ba3574ede8b..5961b29b615 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -38,7 +38,7 @@ dask[array,dataframe]==2024.4.1 # via # dask-expr # feast (setup.py) -dask-expr==1.0.10 +dask-expr==1.0.11 # via dask dill==0.3.8 # via feast (setup.py) @@ -62,12 +62,8 @@ idna==3.7 # via # anyio # requests -importlib-metadata==6.11.0 - # via - # dask - # feast (setup.py) -importlib-resources==6.4.0 - # via feast (setup.py) +importlib-metadata==7.1.0 + # via dask jinja2==3.1.3 # via feast (setup.py) jsonschema==4.21.1 @@ -98,7 +94,7 @@ packaging==24.0 # via # dask # gunicorn -pandas==2.2.1 +pandas==2.2.2 # via # dask # dask-expr @@ -113,11 +109,11 @@ pyarrow==15.0.2 # via # dask-expr # feast (setup.py) -pydantic==2.6.4 +pydantic==2.7.0 # via # fastapi # feast (setup.py) -pydantic-core==2.16.3 +pydantic-core==2.18.1 # via pydantic pygments==2.17.2 # via feast (setup.py) @@ -168,7 +164,7 @@ tqdm==4.66.2 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) -types-protobuf==4.24.0.20240408 +types-protobuf==4.25.0.20240417 # via mypy-protobuf typing-extensions==4.11.0 # via diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 88a67595835..d42b7ed5b95 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -45,7 +45,7 @@ azure-core==1.30.1 # via # azure-identity # azure-storage-blob -azure-identity==1.15.0 +azure-identity==1.16.0 # via feast (setup.py) azure-storage-blob==12.19.1 # via feast (setup.py) @@ -59,11 +59,11 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.80 +boto3==1.34.85 # via # feast (setup.py) # moto -botocore==1.34.80 +botocore==1.34.85 # via # boto3 # moto @@ -138,7 +138,7 @@ dask[array,dataframe]==2024.4.1 # via # dask-expr # feast (setup.py) -dask-expr==1.0.10 +dask-expr==1.0.11 # via dask db-dtypes==1.2.0 # via google-cloud-bigquery @@ -148,12 +148,8 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deprecation==2.1.0 - # via testcontainers dill==0.3.8 - # via - # feast (setup.py) - # multiprocess + # via feast (setup.py) distlib==0.3.8 # via virtualenv docker==7.0.0 @@ -166,7 +162,7 @@ duckdb==0.10.1 # via # duckdb-engine # ibis-framework -duckdb-engine==0.11.4 +duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair @@ -183,7 +179,7 @@ fastapi==0.110.1 # via feast (setup.py) fastjsonschema==2.19.1 # via nbformat -filelock==3.13.3 +filelock==3.13.4 # via # snowflake-connector-python # virtualenv @@ -213,7 +209,7 @@ google-api-core[grpc]==2.18.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.125.0 +google-api-python-client==2.126.0 # via firebase-admin google-auth==2.29.0 # via @@ -230,7 +226,7 @@ google-cloud-bigquery[pandas]==3.12.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) -google-cloud-bigtable==2.23.0 +google-cloud-bigtable==2.23.1 # via feast (setup.py) google-cloud-core==2.4.1 # via @@ -330,11 +326,10 @@ idna==3.7 # snowflake-connector-python imagesize==1.4.1 # via sphinx -importlib-metadata==6.11.0 +importlib-metadata==7.1.0 # via # build # dask - # feast (setup.py) # jupyter-client # jupyter-lsp # jupyterlab @@ -342,8 +337,6 @@ importlib-metadata==6.11.0 # nbconvert # sphinx # typeguard -importlib-resources==6.4.0 - # via feast (setup.py) iniconfig==2.0.0 # via pytest ipykernel==6.29.4 @@ -376,7 +369,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.24 +json5==0.9.25 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -410,9 +403,9 @@ jupyter-core==5.7.2 # nbformat jupyter-events==0.10.0 # via jupyter-server -jupyter-lsp==2.2.4 +jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.13.0 +jupyter-server==2.14.0 # via # jupyter-lsp # jupyterlab @@ -446,7 +439,7 @@ markupsafe==2.1.5 # werkzeug marshmallow==3.21.1 # via great-expectations -matplotlib-inline==0.1.6 +matplotlib-inline==0.1.7 # via # ipykernel # ipython @@ -519,12 +512,11 @@ oauthlib==3.2.2 # via requests-oauthlib overrides==7.7.0 # via jupyter-server -packaging==21.3 +packaging==24.0 # via # build # dask # db-dtypes - # deprecation # docker # duckdb-engine # google-cloud-bigquery @@ -541,7 +533,7 @@ packaging==21.3 # pytest # snowflake-connector-python # sphinx -pandas==2.2.1 +pandas==2.2.2 # via # altair # dask @@ -648,12 +640,12 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.6.4 +pydantic==2.7.0 # via # fastapi # feast (setup.py) # great-expectations -pydantic-core==2.16.3 +pydantic-core==2.18.1 # via pydantic pygments==2.17.2 # via @@ -678,7 +670,6 @@ pyparsing==3.1.2 # via # great-expectations # httplib2 - # packaging pyproject-hooks==1.0.0 # via # build @@ -746,7 +737,7 @@ pyyaml==6.0.1 # pre-commit # responses # uvicorn -pyzmq==25.1.2 +pyzmq==26.0.0 # via # ipykernel # jupyter-client @@ -758,7 +749,7 @@ referencing==0.34.0 # jsonschema # jsonschema-specifications # jupyter-events -regex==2023.12.25 +regex==2024.4.16 # via feast (setup.py) requests==2.31.0 # via @@ -805,7 +796,7 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml -ruff==0.3.5 +ruff==0.3.7 # via feast (setup.py) s3transfer==0.10.1 # via boto3 @@ -832,7 +823,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.7.1 +snowflake-connector-python[pandas]==3.8.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -867,7 +858,7 @@ stack-data==0.6.3 # via ipython starlette==0.37.2 # via fastapi -substrait==0.15.0 +substrait==0.16.0 # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) @@ -877,7 +868,7 @@ terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -testcontainers==3.7.1 +testcontainers==4.3.3 # via feast (setup.py) thriftpy2==0.4.20 # via happybase @@ -934,28 +925,32 @@ trino==0.328.0 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) +types-cffi==1.16.0.20240331 + # via types-pyopenssl types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==24.0.0.20240311 +types-pyopenssl==24.0.0.20240417 # via types-redis types-python-dateutil==2.9.0.20240316 # via # arrow # feast (setup.py) -types-pytz==2024.1.0.20240203 +types-pytz==2024.1.0.20240417 # via feast (setup.py) types-pyyaml==6.0.12.20240311 # via feast (setup.py) -types-redis==4.6.0.20240409 +types-redis==4.6.0.20240417 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.2.0.20240317 - # via feast (setup.py) +types-setuptools==69.5.0.20240415 + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 # via feast (setup.py) types-urllib3==1.26.25.14 @@ -976,6 +971,7 @@ typing-extensions==4.11.0 # snowflake-connector-python # sqlalchemy # starlette + # testcontainers # typeguard # uvicorn tzdata==2024.1 @@ -1000,6 +996,7 @@ urllib3==1.26.18 # responses # rockset # snowflake-connector-python + # testcontainers uvicorn[standard]==0.29.0 # via feast (setup.py) uvloop==0.19.0 @@ -1037,9 +1034,7 @@ wrapt==1.16.0 xmltodict==0.13.0 # via moto zipp==3.18.1 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index a003937823e..b5d040c5613 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -38,7 +38,7 @@ dask[array,dataframe]==2024.4.1 # via # dask-expr # feast (setup.py) -dask-expr==1.0.10 +dask-expr==1.0.11 # via dask dill==0.3.8 # via feast (setup.py) @@ -62,13 +62,10 @@ idna==3.7 # via # anyio # requests -importlib-metadata==6.11.0 +importlib-metadata==7.1.0 # via # dask - # feast (setup.py) # typeguard -importlib-resources==6.4.0 - # via feast (setup.py) jinja2==3.1.3 # via feast (setup.py) jsonschema==4.21.1 @@ -99,7 +96,7 @@ packaging==24.0 # via # dask # gunicorn -pandas==2.2.1 +pandas==2.2.2 # via # dask # dask-expr @@ -114,11 +111,11 @@ pyarrow==15.0.2 # via # dask-expr # feast (setup.py) -pydantic==2.6.4 +pydantic==2.7.0 # via # fastapi # feast (setup.py) -pydantic-core==2.16.3 +pydantic-core==2.18.1 # via pydantic pygments==2.17.2 # via feast (setup.py) @@ -169,7 +166,7 @@ tqdm==4.66.2 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) -types-protobuf==4.24.0.20240408 +types-protobuf==4.25.0.20240417 # via mypy-protobuf typing-extensions==4.11.0 # via @@ -197,6 +194,4 @@ watchfiles==0.21.0 websockets==12.0 # via uvicorn zipp==3.18.1 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata diff --git a/setup.py b/setup.py index fd3186f296b..ae75484805d 100644 --- a/setup.py +++ b/setup.py @@ -68,8 +68,6 @@ "gunicorn; platform_system != 'Windows'", "dask[dataframe]>=2021.1.0", "bowler", # Needed for automatic repo upgrades - "importlib-resources>=6.0.0,<7", - "importlib_metadata>=6.8.0,<7", ] GCP_REQUIRED = [ From d45fb4e0f720387d4fde8c1a09db5fe9e2599140 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Apr 2024 20:56:52 +0000 Subject: [PATCH 08/73] chore: Bump golang.org/x/net from 0.17.0 to 0.23.0 (#4118) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.17.0 to 0.23.0. - [Commits](https://github.com/golang/net/compare/v0.17.0...v0.23.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 1dccc1dddb3..0f73328c725 100644 --- a/go.mod +++ b/go.mod @@ -38,9 +38,9 @@ require ( github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4 // indirect golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect diff --git a/go.sum b/go.sum index 39f2d4d514c..a793b09aec6 100644 --- a/go.sum +++ b/go.sum @@ -1154,7 +1154,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1291,8 +1292,9 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1432,8 +1434,9 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -1443,7 +1446,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1460,8 +1464,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 2a6edeae42a2ebba7d9fc69af917bdc41ae6ecb0 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Sat, 20 Apr 2024 00:59:28 +0400 Subject: [PATCH 09/73] feat: Isolate input-dependent calculations in `get_online_features` (#4041) refactor get online features Signed-off-by: tokoko --- sdk/python/feast/feature_store.py | 97 ++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 34 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index f42cced11cf..fafec32c5d5 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1535,22 +1535,11 @@ def get_online_features( native_entity_values=True, ) - def _get_online_features( - self, - features: Union[List[str], FeatureService], - entity_values: Mapping[ - str, Union[Sequence[Any], Sequence[Value], RepeatedValue] - ], - full_feature_names: bool = False, - native_entity_values: bool = True, + def _get_online_request_context( + self, features: Union[List[str], FeatureService], full_feature_names: bool ): - # Extract Sequence from RepeatedValue Protobuf. - entity_value_lists: Dict[str, Union[List[Any], List[Value]]] = { - k: list(v) if isinstance(v, Sequence) else list(v.val) - for k, v in entity_values.items() - } - _feature_refs = self._get_features(features, allow_cache=True) + ( requested_feature_views, requested_on_demand_feature_views, @@ -1564,19 +1553,6 @@ def _get_online_features( join_keys_set, ) = self._get_entity_maps(requested_feature_views) - entity_proto_values: Dict[str, List[Value]] - if native_entity_values: - # Convert values to Protobuf once. - entity_proto_values = { - k: python_values_to_proto_values( - v, entity_type_map.get(k, ValueType.UNKNOWN) - ) - for k, v in entity_value_lists.items() - } - else: - entity_proto_values = entity_value_lists - - num_rows = _validate_entity_values(entity_proto_values) _validate_feature_refs(_feature_refs, full_feature_names) ( grouped_refs, @@ -1588,7 +1564,6 @@ def _get_online_features( ) set_usage_attribute("odfv", bool(grouped_odfv_refs)) - # All requested features should be present in the result. requested_result_row_names = { feat_ref.replace(":", "__") for feat_ref in _feature_refs } @@ -1601,6 +1576,65 @@ def _get_online_features( needed_request_data = self.get_needed_request_data(grouped_odfv_refs) + entityless_case = DUMMY_ENTITY_NAME in [ + entity_name + for feature_view in feature_views + for entity_name in feature_view.entities + ] + + return ( + _feature_refs, + requested_on_demand_feature_views, + entity_name_to_join_key_map, + entity_type_map, + join_keys_set, + grouped_refs, + requested_result_row_names, + needed_request_data, + entityless_case, + ) + + def _get_online_features( + self, + features: Union[List[str], FeatureService], + entity_values: Mapping[ + str, Union[Sequence[Any], Sequence[Value], RepeatedValue] + ], + full_feature_names: bool = False, + native_entity_values: bool = True, + ): + ( + _feature_refs, + requested_on_demand_feature_views, + entity_name_to_join_key_map, + entity_type_map, + join_keys_set, + grouped_refs, + requested_result_row_names, + needed_request_data, + entityless_case, + ) = self._get_online_request_context(features, full_feature_names) + + # Extract Sequence from RepeatedValue Protobuf. + entity_value_lists: Dict[str, Union[List[Any], List[Value]]] = { + k: list(v) if isinstance(v, Sequence) else list(v.val) + for k, v in entity_values.items() + } + + entity_proto_values: Dict[str, List[Value]] + if native_entity_values: + # Convert values to Protobuf once. + entity_proto_values = { + k: python_values_to_proto_values( + v, entity_type_map.get(k, ValueType.UNKNOWN) + ) + for k, v in entity_value_lists.items() + } + else: + entity_proto_values = entity_value_lists + + num_rows = _validate_entity_values(entity_proto_values) + join_key_values: Dict[str, List[Value]] = {} request_data_features: Dict[str, List[Value]] = {} # Entity rows may be either entities or request data. @@ -1640,11 +1674,6 @@ def _get_online_features( # Add the Entityless case after populating result rows to avoid having to remove # it later. - entityless_case = DUMMY_ENTITY_NAME in [ - entity_name - for feature_view in feature_views - for entity_name in feature_view.entities - ] if entityless_case: join_key_values[DUMMY_ENTITY_ID] = python_values_to_proto_values( [DUMMY_ENTITY_VAL] * num_rows, DUMMY_ENTITY.value_type @@ -1677,7 +1706,7 @@ def _get_online_features( table, ) - if grouped_odfv_refs: + if requested_on_demand_feature_views: self._augment_response_with_on_demand_transforms( online_features_response, _feature_refs, From 2b6f1d0945e8dbf13d01e045f87c5e58546b4af6 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Sat, 20 Apr 2024 20:48:34 +0400 Subject: [PATCH 10/73] feat: Add delta format to `FileSource`, add support for it in ibis/duckdb (#4123) --- protos/feast/core/DataFormat.proto | 4 + sdk/python/feast/data_format.py | 14 ++++ .../feast/infra/offline_stores/file_source.py | 39 ++++++---- sdk/python/feast/infra/offline_stores/ibis.py | 74 +++++++++++++------ .../requirements/py3.10-ci-requirements.txt | 47 ++++++------ .../requirements/py3.10-requirements.txt | 12 +-- .../requirements/py3.9-ci-requirements.txt | 47 ++++++------ .../requirements/py3.9-requirements.txt | 12 +-- .../feature_repos/repo_configuration.py | 2 + .../universal/data_sources/file.py | 53 ++++++++++++- setup.py | 7 +- 11 files changed, 215 insertions(+), 96 deletions(-) diff --git a/protos/feast/core/DataFormat.proto b/protos/feast/core/DataFormat.proto index c453e5e4c83..0a32089b0f3 100644 --- a/protos/feast/core/DataFormat.proto +++ b/protos/feast/core/DataFormat.proto @@ -27,8 +27,12 @@ message FileFormat { // Defines options for the Parquet data format message ParquetFormat {} + // Defines options for delta data format + message DeltaFormat {} + oneof format { ParquetFormat parquet_format = 1; + DeltaFormat delta_format = 2; } } diff --git a/sdk/python/feast/data_format.py b/sdk/python/feast/data_format.py index 8f3b195e3e6..301dfb81302 100644 --- a/sdk/python/feast/data_format.py +++ b/sdk/python/feast/data_format.py @@ -43,6 +43,8 @@ def from_proto(cls, proto): fmt = proto.WhichOneof("format") if fmt == "parquet_format": return ParquetFormat() + elif fmt == "delta_format": + return DeltaFormat() if fmt is None: return None raise NotImplementedError(f"FileFormat is unsupported: {fmt}") @@ -66,6 +68,18 @@ def __str__(self): return "parquet" +class DeltaFormat(FileFormat): + """ + Defines delta data format + """ + + def to_proto(self): + return FileFormatProto(delta_format=FileFormatProto.DeltaFormat()) + + def __str__(self): + return "delta" + + class StreamFormat(ABC): """ Defines an abtracts streaming data format used to encode feature data in streams diff --git a/sdk/python/feast/infra/offline_stores/file_source.py b/sdk/python/feast/infra/offline_stores/file_source.py index 2672cf78bfd..596f3464a99 100644 --- a/sdk/python/feast/infra/offline_stores/file_source.py +++ b/sdk/python/feast/infra/offline_stores/file_source.py @@ -8,7 +8,7 @@ from typeguard import typechecked from feast import type_map -from feast.data_format import FileFormat, ParquetFormat +from feast.data_format import DeltaFormat, FileFormat, ParquetFormat from feast.data_source import DataSource from feast.feature_logging import LoggingDestination from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto @@ -157,24 +157,31 @@ def get_table_column_names_and_types( filesystem, path = FileSource.create_filesystem_and_path( self.path, self.file_options.s3_endpoint_override ) - # Adding support for different file format path - # based on S3 filesystem - if filesystem is None: - kwargs = ( - {"use_legacy_dataset": False} - if version.parse(pyarrow.__version__) < version.parse("15.0.0") - else {} - ) - schema = ParquetDataset(path, **kwargs).schema - if hasattr(schema, "names") and hasattr(schema, "types"): - # Newer versions of pyarrow doesn't have this method, - # but this field is good enough. - pass + # TODO why None check necessary + if self.file_format is None or isinstance(self.file_format, ParquetFormat): + if filesystem is None: + kwargs = ( + {"use_legacy_dataset": False} + if version.parse(pyarrow.__version__) < version.parse("15.0.0") + else {} + ) + + schema = ParquetDataset(path, **kwargs).schema + if hasattr(schema, "names") and hasattr(schema, "types"): + # Newer versions of pyarrow doesn't have this method, + # but this field is good enough. + pass + else: + schema = schema.to_arrow_schema() else: - schema = schema.to_arrow_schema() + schema = ParquetDataset(path, filesystem=filesystem).schema + elif isinstance(self.file_format, DeltaFormat): + from deltalake import DeltaTable + + schema = DeltaTable(self.path).schema().to_pyarrow() else: - schema = ParquetDataset(path, filesystem=filesystem).schema + raise Exception(f"Unknown FileFormat -> {self.file_format}") return zip(schema.names, map(str, schema.types)) diff --git a/sdk/python/feast/infra/offline_stores/ibis.py b/sdk/python/feast/infra/offline_stores/ibis.py index 6e0729d6d11..de025ca0069 100644 --- a/sdk/python/feast/infra/offline_stores/ibis.py +++ b/sdk/python/feast/infra/offline_stores/ibis.py @@ -13,6 +13,7 @@ from ibis.expr.types import Table from pytz import utc +from feast.data_format import DeltaFormat, ParquetFormat from feast.data_source import DataSource from feast.errors import SavedDatasetLocationAlreadyExists from feast.feature_logging import LoggingConfig, LoggingSource @@ -105,6 +106,15 @@ def _generate_row_id( return entity_table + @staticmethod + def _read_data_source(data_source: DataSource) -> Table: + assert isinstance(data_source, FileSource) + + if isinstance(data_source.file_format, ParquetFormat): + return ibis.read_parquet(data_source.path) + elif isinstance(data_source.file_format, DeltaFormat): + return ibis.read_delta(data_source.path) + @staticmethod def get_historical_features( config: RepoConfig, @@ -137,7 +147,9 @@ def get_historical_features( def read_fv( feature_view: FeatureView, feature_refs: List[str], full_feature_names: bool ) -> Tuple: - fv_table: Table = ibis.read_parquet(feature_view.batch_source.name) + fv_table: Table = IbisOfflineStore._read_data_source( + feature_view.batch_source + ) for old_name, new_name in feature_view.batch_source.field_mapping.items(): if old_name in fv_table.columns: @@ -227,7 +239,7 @@ def pull_all_from_table_or_query( start_date = start_date.astimezone(tz=utc) end_date = end_date.astimezone(tz=utc) - table = ibis.read_parquet(data_source.path) + table = IbisOfflineStore._read_data_source(data_source) table = table.select(*fields) @@ -260,10 +272,9 @@ def write_logged_features( destination = logging_config.destination assert isinstance(destination, FileLoggingDestination) - if isinstance(data, Path): - table = ibis.read_parquet(data) - else: - table = ibis.memtable(data) + table = ( + ibis.read_parquet(data) if isinstance(data, Path) else ibis.memtable(data) + ) if destination.partition_by: kwargs = {"partition_by": destination.partition_by} @@ -294,12 +305,21 @@ def offline_write_batch( ) file_options = feature_view.batch_source.file_options - prev_table = ibis.read_parquet(file_options.uri).to_pyarrow() - if table.schema != prev_table.schema: - table = table.cast(prev_table.schema) - new_table = pyarrow.concat_tables([table, prev_table]) - ibis.memtable(new_table).to_parquet(file_options.uri) + if isinstance(feature_view.batch_source.file_format, ParquetFormat): + prev_table = ibis.read_parquet(file_options.uri).to_pyarrow() + if table.schema != prev_table.schema: + table = table.cast(prev_table.schema) + new_table = pyarrow.concat_tables([table, prev_table]) + + ibis.memtable(new_table).to_parquet(file_options.uri) + elif isinstance(feature_view.batch_source.file_format, DeltaFormat): + from deltalake import DeltaTable + + prev_schema = DeltaTable(file_options.uri).schema().to_pyarrow() + if table.schema != prev_schema: + table = table.cast(prev_schema) + ibis.memtable(table).to_delta(file_options.uri, mode="append") class IbisRetrievalJob(RetrievalJob): @@ -338,20 +358,28 @@ def persist( if not allow_overwrite and os.path.exists(storage.file_options.uri): raise SavedDatasetLocationAlreadyExists(location=storage.file_options.uri) - filesystem, path = FileSource.create_filesystem_and_path( - storage.file_options.uri, - storage.file_options.s3_endpoint_override, - ) - - if path.endswith(".parquet"): - pyarrow.parquet.write_table( - self.to_arrow(), where=path, filesystem=filesystem + if isinstance(storage.file_options.file_format, ParquetFormat): + filesystem, path = FileSource.create_filesystem_and_path( + storage.file_options.uri, + storage.file_options.s3_endpoint_override, ) - else: - # otherwise assume destination is directory - pyarrow.parquet.write_to_dataset( - self.to_arrow(), root_path=path, filesystem=filesystem + + if path.endswith(".parquet"): + pyarrow.parquet.write_table( + self.to_arrow(), where=path, filesystem=filesystem + ) + else: + # otherwise assume destination is directory + pyarrow.parquet.write_to_dataset( + self.to_arrow(), root_path=path, filesystem=filesystem + ) + elif isinstance(storage.file_options.file_format, DeltaFormat): + mode = ( + "overwrite" + if allow_overwrite and os.path.exists(storage.file_options.uri) + else "error" ) + self.table.to_delta(storage.file_options.uri, mode=mode) @property def metadata(self) -> Optional[RetrievalMetadata]: diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 2d91e5cf41b..15dadfd8740 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -59,11 +59,11 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.85 +boto3==1.34.88 # via # feast (setup.py) # moto -botocore==1.34.85 +botocore==1.34.88 # via # boto3 # moto @@ -134,11 +134,11 @@ cryptography==42.0.5 # snowflake-connector-python # types-pyopenssl # types-redis -dask[array,dataframe]==2024.4.1 +dask[array,dataframe]==2024.4.2 # via # dask-expr # feast (setup.py) -dask-expr==1.0.11 +dask-expr==1.0.12 # via dask db-dtypes==1.2.0 # via google-cloud-bigquery @@ -148,6 +148,8 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert +deltalake==0.16.4 + # via feast (setup.py) dill==0.3.8 # via feast (setup.py) distlib==0.3.8 @@ -158,7 +160,7 @@ docker==7.0.0 # testcontainers docutils==0.19 # via sphinx -duckdb==0.10.1 +duckdb==0.10.2 # via # duckdb-engine # ibis-framework @@ -166,7 +168,7 @@ duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via # anyio # ipython @@ -175,7 +177,7 @@ execnet==2.1.1 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.110.1 +fastapi==0.110.2 # via feast (setup.py) fastjsonschema==2.19.1 # via nbformat @@ -263,7 +265,7 @@ greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.62.1 +grpcio==1.62.2 # via # feast (setup.py) # google-api-core @@ -275,15 +277,15 @@ grpcio==1.62.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.62.1 +grpcio-health-checking==1.62.2 # via feast (setup.py) -grpcio-reflection==1.62.1 +grpcio-reflection==1.62.2 # via feast (setup.py) -grpcio-status==1.62.1 +grpcio-status==1.62.2 # via google-api-core -grpcio-testing==1.62.1 +grpcio-testing==1.62.2 # via feast (setup.py) -grpcio-tools==1.62.1 +grpcio-tools==1.62.2 # via feast (setup.py) gunicorn==22.0.0 ; platform_system != "Windows" # via feast (setup.py) @@ -482,13 +484,13 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.1.2 +notebook==7.1.3 # via great-expectations notebook-shim==0.2.4 # via # jupyterlab # notebook -numpy==1.24.4 +numpy==1.26.4 # via # altair # dask @@ -615,12 +617,15 @@ pyarrow==15.0.2 # via # dask-expr # db-dtypes + # deltalake # feast (setup.py) # google-cloud-bigquery # ibis-framework # snowflake-connector-python pyarrow-hotfix==0.6 - # via ibis-framework + # via + # deltalake + # ibis-framework pyasn1==0.6.0 # via # pyasn1-modules @@ -692,7 +697,7 @@ pytest-ordering==0.6 # via feast (setup.py) pytest-timeout==1.4.2 # via feast (setup.py) -pytest-xdist==3.5.0 +pytest-xdist==3.6.0 # via feast (setup.py) python-dateutil==2.9.0.post0 # via @@ -728,7 +733,7 @@ pyyaml==6.0.1 # pre-commit # responses # uvicorn -pyzmq==26.0.0 +pyzmq==26.0.2 # via # ipykernel # jupyter-client @@ -785,7 +790,7 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruff==0.3.7 +ruff==0.4.1 # via feast (setup.py) s3transfer==0.10.1 # via boto3 @@ -812,7 +817,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.8.1 +snowflake-connector-python[pandas]==3.9.0 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -895,7 +900,7 @@ tqdm==4.66.2 # via # feast (setup.py) # great-expectations -traitlets==5.14.2 +traitlets==5.14.3 # via # comm # ipykernel diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 5961b29b615..2cfe62c55bd 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -34,17 +34,17 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[array,dataframe]==2024.4.1 +dask[array,dataframe]==2024.4.2 # via # dask-expr # feast (setup.py) -dask-expr==1.0.11 +dask-expr==1.0.12 # via dask dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via anyio -fastapi==0.110.1 +fastapi==0.110.2 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -84,7 +84,7 @@ mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 # via feast (setup.py) -numpy==1.24.4 +numpy==1.26.4 # via # dask # feast (setup.py) @@ -164,7 +164,7 @@ tqdm==4.66.2 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) -types-protobuf==4.25.0.20240417 +types-protobuf==5.26.0.20240420 # via mypy-protobuf typing-extensions==4.11.0 # via diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index d42b7ed5b95..8de9151e4e3 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -59,11 +59,11 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.85 +boto3==1.34.88 # via # feast (setup.py) # moto -botocore==1.34.85 +botocore==1.34.88 # via # boto3 # moto @@ -134,11 +134,11 @@ cryptography==42.0.5 # snowflake-connector-python # types-pyopenssl # types-redis -dask[array,dataframe]==2024.4.1 +dask[array,dataframe]==2024.4.2 # via # dask-expr # feast (setup.py) -dask-expr==1.0.11 +dask-expr==1.0.12 # via dask db-dtypes==1.2.0 # via google-cloud-bigquery @@ -148,6 +148,8 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert +deltalake==0.16.4 + # via feast (setup.py) dill==0.3.8 # via feast (setup.py) distlib==0.3.8 @@ -158,7 +160,7 @@ docker==7.0.0 # testcontainers docutils==0.19 # via sphinx -duckdb==0.10.1 +duckdb==0.10.2 # via # duckdb-engine # ibis-framework @@ -166,7 +168,7 @@ duckdb-engine==0.11.5 # via ibis-framework entrypoints==0.4 # via altair -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via # anyio # ipython @@ -175,7 +177,7 @@ execnet==2.1.1 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.110.1 +fastapi==0.110.2 # via feast (setup.py) fastjsonschema==2.19.1 # via nbformat @@ -263,7 +265,7 @@ greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.62.1 +grpcio==1.62.2 # via # feast (setup.py) # google-api-core @@ -275,15 +277,15 @@ grpcio==1.62.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.62.1 +grpcio-health-checking==1.62.2 # via feast (setup.py) -grpcio-reflection==1.62.1 +grpcio-reflection==1.62.2 # via feast (setup.py) -grpcio-status==1.62.1 +grpcio-status==1.62.2 # via google-api-core -grpcio-testing==1.62.1 +grpcio-testing==1.62.2 # via feast (setup.py) -grpcio-tools==1.62.1 +grpcio-tools==1.62.2 # via feast (setup.py) gunicorn==22.0.0 ; platform_system != "Windows" # via feast (setup.py) @@ -491,13 +493,13 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.1.2 +notebook==7.1.3 # via great-expectations notebook-shim==0.2.4 # via # jupyterlab # notebook -numpy==1.24.4 +numpy==1.26.4 # via # altair # dask @@ -624,12 +626,15 @@ pyarrow==15.0.2 # via # dask-expr # db-dtypes + # deltalake # feast (setup.py) # google-cloud-bigquery # ibis-framework # snowflake-connector-python pyarrow-hotfix==0.6 - # via ibis-framework + # via + # deltalake + # ibis-framework pyasn1==0.6.0 # via # pyasn1-modules @@ -701,7 +706,7 @@ pytest-ordering==0.6 # via feast (setup.py) pytest-timeout==1.4.2 # via feast (setup.py) -pytest-xdist==3.5.0 +pytest-xdist==3.6.0 # via feast (setup.py) python-dateutil==2.9.0.post0 # via @@ -737,7 +742,7 @@ pyyaml==6.0.1 # pre-commit # responses # uvicorn -pyzmq==26.0.0 +pyzmq==26.0.2 # via # ipykernel # jupyter-client @@ -796,7 +801,7 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml -ruff==0.3.7 +ruff==0.4.1 # via feast (setup.py) s3transfer==0.10.1 # via boto3 @@ -823,7 +828,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.8.1 +snowflake-connector-python[pandas]==3.9.0 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -906,7 +911,7 @@ tqdm==4.66.2 # via # feast (setup.py) # great-expectations -traitlets==5.14.2 +traitlets==5.14.3 # via # comm # ipykernel diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index b5d040c5613..472f3e90b99 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -34,17 +34,17 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[array,dataframe]==2024.4.1 +dask[array,dataframe]==2024.4.2 # via # dask-expr # feast (setup.py) -dask-expr==1.0.11 +dask-expr==1.0.12 # via dask dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via anyio -fastapi==0.110.1 +fastapi==0.110.2 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -86,7 +86,7 @@ mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 # via feast (setup.py) -numpy==1.24.4 +numpy==1.26.4 # via # dask # feast (setup.py) @@ -166,7 +166,7 @@ tqdm==4.66.2 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) -types-protobuf==4.25.0.20240417 +types-protobuf==5.26.0.20240420 # via mypy-protobuf typing-extensions==4.11.0 # via diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index a57a017699b..096744f5472 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -32,6 +32,7 @@ ) from tests.integration.feature_repos.universal.data_sources.file import ( DuckDBDataSourceCreator, + DuckDBDeltaDataSourceCreator, FileDataSourceCreator, ) from tests.integration.feature_repos.universal.data_sources.redshift import ( @@ -118,6 +119,7 @@ AVAILABLE_OFFLINE_STORES: List[Tuple[str, Type[DataSourceCreator]]] = [ ("local", FileDataSourceCreator), ("local", DuckDBDataSourceCreator), + ("local", DuckDBDeltaDataSourceCreator), ] AVAILABLE_ONLINE_STORES: Dict[ diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index 6d4baa19ed4..9cdc91a6c87 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -12,7 +12,7 @@ from testcontainers.core.waiting_utils import wait_for_logs from feast import FileSource -from feast.data_format import ParquetFormat +from feast.data_format import DeltaFormat, ParquetFormat from feast.data_source import DataSource from feast.feature_logging import LoggingDestination from feast.infra.offline_stores.duckdb import DuckDBOfflineStoreConfig @@ -30,11 +30,13 @@ class FileDataSourceCreator(DataSourceCreator): files: List[Any] dirs: List[Any] + keep: List[Any] def __init__(self, project_name: str, *args, **kwargs): super().__init__(project_name) self.files = [] self.dirs = [] + self.keep = [] def create_data_source( self, @@ -89,6 +91,49 @@ def teardown(self): shutil.rmtree(d) +class DeltaFileSourceCreator(FileDataSourceCreator): + def create_data_source( + self, + df: pd.DataFrame, + destination_name: str, + created_timestamp_column="created_ts", + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", + ) -> DataSource: + from deltalake.writer import write_deltalake + + destination_name = self.get_prefixed_table_name(destination_name) + + delta_path = tempfile.TemporaryDirectory( + prefix=f"{self.project_name}_{destination_name}" + ) + + self.keep.append(delta_path) + + write_deltalake(delta_path.name, df) + + return FileSource( + file_format=DeltaFormat(), + path=delta_path.name, + timestamp_field=timestamp_field, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping or {"ts_1": "ts"}, + ) + + def create_saved_dataset_destination(self) -> SavedDatasetFileStorage: + d = tempfile.mkdtemp(prefix=self.project_name) + self.keep.append(d) + return SavedDatasetFileStorage( + path=d, file_format=DeltaFormat(), s3_endpoint_override=None + ) + + # LoggingDestination is parquet-only + def create_logged_features_destination(self) -> LoggingDestination: + d = tempfile.mkdtemp(prefix=self.project_name) + self.keep.append(d) + return FileLoggingDestination(path=d) + + class FileParquetDatasetSourceCreator(FileDataSourceCreator): def create_data_source( self, @@ -222,3 +267,9 @@ class DuckDBDataSourceCreator(FileDataSourceCreator): def create_offline_store_config(self): self.duckdb_offline_store_config = DuckDBOfflineStoreConfig() return self.duckdb_offline_store_config + + +class DuckDBDeltaDataSourceCreator(DeltaFileSourceCreator): + def create_offline_store_config(self): + self.duckdb_offline_store_config = DuckDBOfflineStoreConfig() + return self.duckdb_offline_store_config diff --git a/setup.py b/setup.py index ae75484805d..dcc616f120c 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,6 @@ "mmh3", "numpy>=1.22,<2", "pandas>=1.4.3,<3", - # Higher than 4.23.4 seems to cause a seg fault "protobuf>=4.24.0,<5.0.0", "pyarrow>=4", "pydantic>=2.0.0", @@ -150,6 +149,8 @@ DUCKDB_REQUIRED = ["ibis-framework[duckdb]"] +DELTA_REQUIRED = ["deltalake"] + CI_REQUIRED = ( [ "build", @@ -210,6 +211,7 @@ + IBIS_REQUIRED + GRPCIO_REQUIRED + DUCKDB_REQUIRED + + DELTA_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -374,7 +376,8 @@ def run(self): "rockset": ROCKSET_REQUIRED, "ibis": IBIS_REQUIRED, "duckdb": DUCKDB_REQUIRED, - "ikv": IKV_REQUIRED + "ikv": IKV_REQUIRED, + "delta": DELTA_REQUIRED, }, include_package_data=True, license="Apache", From 9184dde1fcd57de5765c850615eb5e70cbafe70f Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Sun, 21 Apr 2024 16:27:13 +0400 Subject: [PATCH 11/73] fix: Get container host addresses from testcontainers (java) (#4125) --- .../java/feast/serving/it/ServingEnvironment.java | 7 ++++++- .../serving/it/ServingRedisAzureRegistryIT.java | 12 ++++++++---- .../feast/serving/it/ServingRedisGSRegistryIT.java | 5 +++-- .../feast/serving/it/ServingRedisS3RegistryIT.java | 6 ++++-- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java b/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java index 43b82345c67..54892038566 100644 --- a/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java +++ b/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java @@ -63,6 +63,11 @@ static void globalSetup() { .withExposedService( "feast", 8080, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(180))) .withTailChildContainers(true); + + if (System.getenv("FEAST_TESTCONTAINERS_LOCAL_COMPOSE") != null) { + environment = environment.withLocalCompose(true); + } + environment.start(); } @@ -136,7 +141,7 @@ ApplicationProperties applicationProperties() { server = injector.getInstance(Server.class); server.start(); - channel = ManagedChannelBuilder.forAddress("localhost", serverPort).usePlaintext().build(); + channel = ManagedChannelBuilder.forAddress("127.0.0.1", serverPort).usePlaintext().build(); servingStub = ServingServiceGrpc.newBlockingStub(channel) diff --git a/java/serving/src/test/java/feast/serving/it/ServingRedisAzureRegistryIT.java b/java/serving/src/test/java/feast/serving/it/ServingRedisAzureRegistryIT.java index 8ab658fc2a1..0b1ecca7d23 100644 --- a/java/serving/src/test/java/feast/serving/it/ServingRedisAzureRegistryIT.java +++ b/java/serving/src/test/java/feast/serving/it/ServingRedisAzureRegistryIT.java @@ -50,8 +50,10 @@ private static BlobServiceClient createClient() { return new BlobServiceClientBuilder() .endpoint( String.format( - "http://localhost:%d/%s", - azureBlobMock.getMappedPort(BLOB_STORAGE_PORT), TEST_ACCOUNT_NAME)) + "http://%s:%d/%s", + azureBlobMock.getHost(), + azureBlobMock.getMappedPort(BLOB_STORAGE_PORT), + TEST_ACCOUNT_NAME)) .credential(CREDENTIAL) .buildClient(); } @@ -95,8 +97,10 @@ public BlobServiceClient awsStorage() { return new BlobServiceClientBuilder() .endpoint( String.format( - "http://localhost:%d/%s", - azureBlobMock.getMappedPort(BLOB_STORAGE_PORT), TEST_ACCOUNT_NAME)) + "http://%s:%d/%s", + azureBlobMock.getHost(), + azureBlobMock.getMappedPort(BLOB_STORAGE_PORT), + TEST_ACCOUNT_NAME)) .credential(CREDENTIAL) .buildClient(); } diff --git a/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java b/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java index 96aa2077c0f..b3f185bbdad 100644 --- a/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java +++ b/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java @@ -61,7 +61,7 @@ private static Storage createClient() { return StorageOptions.newBuilder() .setProjectId(TEST_PROJECT) .setCredentials(ServiceAccountCredentials.create(credential)) - .setHost("http://localhost:" + gcsMock.getMappedPort(GCS_PORT)) + .setHost(String.format("http://%s:%d", gcsMock.getHost(), gcsMock.getMappedPort(GCS_PORT))) .build() .getService(); } @@ -89,7 +89,8 @@ Storage googleStorage(ApplicationProperties applicationProperties) { return StorageOptions.newBuilder() .setProjectId(TEST_PROJECT) .setCredentials(ServiceAccountCredentials.create(credential)) - .setHost("http://localhost:" + gcsMock.getMappedPort(GCS_PORT)) + .setHost( + String.format("http://%s:%d", gcsMock.getHost(), gcsMock.getMappedPort(GCS_PORT))) .build() .getService(); } diff --git a/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java b/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java index 52e1af90655..67ba11128fd 100644 --- a/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java +++ b/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java @@ -42,7 +42,8 @@ private static AmazonS3 createClient() { return AmazonS3ClientBuilder.standard() .withEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration( - String.format("http://localhost:%d", s3Mock.getHttpServerPort()), TEST_REGION)) + String.format("http://%s:%d", s3Mock.getHost(), s3Mock.getHttpServerPort()), + TEST_REGION)) .withCredentials(credentials) .enablePathStyleAccess() .build(); @@ -89,7 +90,8 @@ public AmazonS3 awsStorage() { return AmazonS3ClientBuilder.standard() .withEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration( - String.format("http://localhost:%d", s3Mock.getHttpServerPort()), TEST_REGION)) + String.format("http://%s:%d", s3Mock.getHost(), s3Mock.getHttpServerPort()), + TEST_REGION)) .withCredentials(credentials) .enablePathStyleAccess() .build(); From 0c30e96da144babe725a3f168c05d2fbeca65507 Mon Sep 17 00:00:00 2001 From: lokeshrangineni Date: Mon, 22 Apr 2024 12:27:08 -0400 Subject: [PATCH 12/73] fix: Updating the instructions for quickstart guide. (#4120) Updating the instructions for quickstart guide as per new instructions or output and also fixes the issue - 4104 --- docs/getting-started/quickstart.md | 143 +++++++++++++++++++++-------- 1 file changed, 105 insertions(+), 38 deletions(-) diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index d10e8a174ab..01c039e9c56 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -109,7 +109,7 @@ from feast import ( from feast.on_demand_feature_view import on_demand_feature_view from feast.types import Float32, Float64, Int64 -# Define an entity for the driver. You can think of entity as a primary key used to +# Define an entity for the driver. You can think of an entity as a primary key used to # fetch features. driver = Entity(name="driver", join_keys=["driver_id"]) @@ -138,7 +138,7 @@ driver_stats_fv = FeatureView( schema=[ Field(name="conv_rate", dtype=Float32), Field(name="acc_rate", dtype=Float32), - Field(name="avg_daily_trips", dtype=Int64), + Field(name="avg_daily_trips", dtype=Int64, description="Average daily trips"), ], online=True, source=driver_stats_source, @@ -147,12 +147,6 @@ driver_stats_fv = FeatureView( tags={"team": "driver_performance"}, ) -# Defines a way to push data (to be available offline, online or both) into Feast. -driver_stats_push_source = PushSource( - name="driver_stats_push_source", - batch_source=driver_stats_source, -) - # Define a request data source which encodes features / information only # available at request time (e.g. part of the user initiated HTTP request) input_request = RequestSource( @@ -191,6 +185,51 @@ driver_activity_v1 = FeatureService( driver_activity_v2 = FeatureService( name="driver_activity_v2", features=[driver_stats_fv, transformed_conv_rate] ) + +# Defines a way to push data (to be available offline, online or both) into Feast. +driver_stats_push_source = PushSource( + name="driver_stats_push_source", + batch_source=driver_stats_source, +) + +# Defines a slightly modified version of the feature view from above, where the source +# has been changed to the push source. This allows fresh features to be directly pushed +# to the online store for this feature view. +driver_stats_fresh_fv = FeatureView( + name="driver_hourly_stats_fresh", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_push_source, # Changed from above + tags={"team": "driver_performance"}, +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fresh_fv, input_request], # relies on fresh version of FV + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate_fresh(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +driver_activity_v3 = FeatureService( + name="driver_activity_v3", + features=[driver_stats_fresh_fv, transformed_conv_rate_fresh], +) ``` {% endtab %} {% endtabs %} @@ -254,10 +293,14 @@ feast apply ``` Created entity driver Created feature view driver_hourly_stats +Created feature view driver_hourly_stats_fresh Created on demand feature view transformed_conv_rate +Created on demand feature view transformed_conv_rate_fresh +Created feature service driver_activity_v3 Created feature service driver_activity_v1 Created feature service driver_activity_v2 +Created sqlite table my_project_driver_hourly_stats_fresh Created sqlite table my_project_driver_hourly_stats ``` {% endtab %} @@ -334,28 +377,40 @@ print(training_df.head()) ----- Feature schema ----- -Int64Index: 3 entries, 0 to 2 -Data columns (total 6 columns): - # Column Non-Null Count Dtype ---- ------ -------------- ----- - 0 event_timestamp 3 non-null datetime64[ns, UTC] - 1 driver_id 3 non-null int64 - 2 label_driver_reported_satisfaction 3 non-null int64 - 3 conv_rate 3 non-null float32 - 4 acc_rate 3 non-null float32 - 5 avg_daily_trips 3 non-null int32 -dtypes: datetime64[ns, UTC](1), float32(2), int32(1), int64(2) -memory usage: 132.0 bytes +RangeIndex: 3 entries, 0 to 2 +Data columns (total 10 columns): + # Column Non-Null Count Dtype +--- ------ -------------- ----- + 0 driver_id 3 non-null int64 + 1 event_timestamp 3 non-null datetime64[ns, UTC] + 2 label_driver_reported_satisfaction 3 non-null int64 + 3 val_to_add 3 non-null int64 + 4 val_to_add_2 3 non-null int64 + 5 conv_rate 3 non-null float32 + 6 acc_rate 3 non-null float32 + 7 avg_daily_trips 3 non-null int32 + 8 conv_rate_plus_val1 3 non-null float64 + 9 conv_rate_plus_val2 3 non-null float64 +dtypes: datetime64[ns, UTC](1), float32(2), float64(2), int32(1), int64(4) +memory usage: 336.0 bytes None ----- Example features ----- - event_timestamp driver_id ... acc_rate avg_daily_trips -0 2021-08-23 15:12:55.489091+00:00 1003 ... 0.077863 741 -1 2021-08-23 15:49:55.489089+00:00 1002 ... 0.074327 113 -2 2021-08-23 16:14:55.489075+00:00 1001 ... 0.105046 347 + driver_id event_timestamp label_driver_reported_satisfaction \ +0 1001 2021-04-12 10:59:42+00:00 1 +1 1002 2021-04-12 08:12:10+00:00 5 +2 1003 2021-04-12 16:40:26+00:00 3 + + val_to_add val_to_add_2 conv_rate acc_rate avg_daily_trips \ +0 1 10 0.800648 0.265174 643 +1 2 20 0.644141 0.996602 765 +2 3 30 0.855432 0.546345 954 -[3 rows x 6 columns] + conv_rate_plus_val1 conv_rate_plus_val2 +0 1.800648 10.800648 +1 2.644141 20.644141 +2 3.855432 30.855432 ``` {% endtab %} {% endtabs %} @@ -389,10 +444,20 @@ print(training_df.head()) ``` ----- Example features ----- - driver_id event_timestamp ... acc_rate avg_daily_trips conv_rate_plus_val1 -0 1001 2022-08-08 18:22:06.555018+00:00 ... 0.864639 359 1.663844 -1 1002 2022-08-08 18:22:06.555018+00:00 ... 0.695982 311 2.151189 -2 1003 2022-08-08 18:22:06.555018+00:00 ... 0.949191 789 3.769165 + driver_id event_timestamp \ +0 1001 2024-04-19 14:58:16.452895+00:00 +1 1002 2024-04-19 14:58:16.452895+00:00 +2 1003 2024-04-19 14:58:16.452895+00:00 + + label_driver_reported_satisfaction val_to_add val_to_add_2 conv_rate \ +0 1 1 10 0.535773 +1 5 2 20 0.171976 +2 3 3 30 0.275669 + + acc_rate avg_daily_trips conv_rate_plus_val1 conv_rate_plus_val2 +0 0.689705 428 1.535773 10.535773 +1 0.737113 369 2.171976 20.171976 +2 0.156630 116 3.275669 30.275669 ``` {% endtab %} {% endtabs %} @@ -413,11 +478,13 @@ feast materialize-incremental $CURRENT_TIME {% tabs %} {% tab title="Output" %} ```bash -Materializing 1 feature views to 2021-08-23 16:25:46+00:00 into the sqlite online -store. +Materializing 2 feature views to 2024-04-19 10:59:58-04:00 into the sqlite online store. -driver_hourly_stats from 2021-08-22 16:25:47+00:00 to 2021-08-23 16:25:46+00:00: -100%|████████████████████████████████████████████| 5/5 [00:00<00:00, 592.05it/s] +driver_hourly_stats from 2024-04-18 15:00:46-04:00 to 2024-04-19 10:59:58-04:00: +100%|████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 370.32it/s] +driver_hourly_stats_fresh from 2024-04-18 15:00:46-04:00 to 2024-04-19 10:59:58-04:00: +100%|███████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 1046.64it/s] +Materializing 2 feature views to 2024-04-19 10:59:58-04:00 into the sqlite online store. ``` {% endtab %} {% endtabs %} @@ -458,11 +525,11 @@ pprint(feature_vector) {% tab title="Output" %} ```bash { - 'acc_rate': [0.5732735991477966, 0.7828438878059387], - 'avg_daily_trips': [33, 984], - 'conv_rate': [0.15498852729797363, 0.6263588070869446], + 'acc_rate': [0.25351759791374207, 0.8949751853942871], + 'avg_daily_trips': [712, 791], + 'conv_rate': [0.5038306713104248, 0.9839504361152649], 'driver_id': [1004, 1005] -} + } ``` {% endtab %} {% endtabs %} @@ -479,7 +546,7 @@ The `driver_activity_v1` feature service pulls all features from the `driver_hou ```python from feast import FeatureService driver_stats_fs = FeatureService( - name="driver_activity_v1", features=[driver_hourly_stats_view] + name="driver_activity_v1", features=[driver_stats_fv] ) ``` From 91e34a2fed8e1323bcf4a239f57eef0a6cc1d470 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 23 Apr 2024 16:33:16 +0400 Subject: [PATCH 13/73] chore: Move feast install to docker build in java it tests (#4126) * chore: Move feast install to docker build in java it tests Signed-off-by: tokoko * remove commented out lines in compose file Signed-off-by: tokoko * make local compose mode default Signed-off-by: tokoko * limit COPY contents Signed-off-by: tokoko * remove requirements.txt from java tests docker image Signed-off-by: tokoko * include pyproject.toml in dockerfile Signed-off-by: tokoko * change links to depends_on Signed-off-by: tokoko * try updating setup-python to v5 Signed-off-by: tokoko * pin macos image to macos-12 Signed-off-by: tokoko * force rerun Signed-off-by: tokoko --------- Signed-off-by: tokoko --- .devcontainer/devcontainer.json | 5 +---- .../fork_pr_integration_tests_aws.yml | 2 +- .../fork_pr_integration_tests_gcp.yml | 2 +- .../fork_pr_integration_tests_snowflake.yml | 2 +- .github/workflows/build_wheels.yml | 6 +++--- .github/workflows/nightly-ci.yml | 2 +- .github/workflows/unit_tests.yml | 4 ++-- .../java/feast/serving/it/ServingEnvironment.java | 7 ++----- .../docker-compose/docker-compose-redis-it.yml | 11 ++++------- .../resources/docker-compose/feast10/Dockerfile | 13 +++++++------ .../resources/docker-compose/feast10/entrypoint.sh | 4 ---- .../docker-compose/feast10/requirements.txt | 6 ------ 12 files changed, 23 insertions(+), 41 deletions(-) delete mode 100644 java/serving/src/test/resources/docker-compose/feast10/requirements.txt diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b4da25737f9..e82fd04db4a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,10 +7,7 @@ }, "ghcr.io/devcontainers/features/python:1": { "version": "3.9" - }, - "ghcr.io/meaningful-ooo/devcontainer-features/homebrew:2": { - "version": "latest" } }, - "postCreateCommand": "brew install mysql && pip install -e '.[dev]' && make compile-protos-python" + "postCreateCommand": "pip install -e '.[dev]' && make compile-protos-python" } diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index be75c4f9875..52d7112a640 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -138,7 +138,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macos-12' run: | brew install apache-arrow brew install pkg-config diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index 0793fbd6e51..337d8040ae7 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -82,7 +82,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macOS-12' run: | brew install apache-arrow brew install pkg-config diff --git a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml index b9b6f8df06f..a3484a34625 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml @@ -72,7 +72,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macos-12' run: | brew install apache-arrow brew install pkg-config diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f0851f5bb04..ca0a7dcfe25 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -79,7 +79,7 @@ jobs: build-source-distribution: name: Build source distribution - runs-on: macos-latest + runs-on: macos-12 steps: - uses: actions/checkout@v3 - name: Setup Python @@ -136,7 +136,7 @@ jobs: needs: [build-python-wheel, build-source-distribution, get-version] strategy: matrix: - os: [ubuntu-latest, macos-latest ] + os: [ubuntu-latest, macos-12 ] python-version: ["3.9", "3.10"] from-source: [ True, False ] env: @@ -165,7 +165,7 @@ jobs: name: wheels path: dist - name: Install OS X dependencies - if: matrix.os == 'macos-latest' + if: matrix.os == 'macos-12' run: brew install coreutils - name: Install wheel if: ${{ !matrix.from-source }} diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 4dea41d4ad0..3292d6bcb59 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -202,7 +202,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macos-12' run: brew install apache-arrow - name: Install dependencies run: make install-python-ci-dependencies diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f3f91bb67f3..ff7c3d5e234 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -8,9 +8,9 @@ jobs: fail-fast: false matrix: python-version: [ "3.9", "3.10" ] - os: [ ubuntu-latest, macOS-latest ] + os: [ ubuntu-latest, macos-12 ] exclude: - - os: macOS-latest + - os: macos-12 python-version: "3.9" env: OS: ${{ matrix.os }} diff --git a/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java b/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java index 54892038566..356524399a4 100644 --- a/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java +++ b/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java @@ -62,11 +62,8 @@ static void globalSetup() { .withExposedService("redis", 6379) .withExposedService( "feast", 8080, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(180))) - .withTailChildContainers(true); - - if (System.getenv("FEAST_TESTCONTAINERS_LOCAL_COMPOSE") != null) { - environment = environment.withLocalCompose(true); - } + .withTailChildContainers(true) + .withLocalCompose(true); environment.start(); } diff --git a/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml b/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml index 0522750d996..142efe7fa20 100644 --- a/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml +++ b/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml @@ -1,5 +1,3 @@ -version: '3' - services: redis: image: redis:6.2 @@ -7,11 +5,10 @@ services: ports: - "6379" feast: - build: feast10 + build: + context: ../../../../../../ + dockerfile: java/serving/src/test/resources/docker-compose/feast10/Dockerfile ports: - "8080" - links: + depends_on: - redis - volumes: - - $PWD/../../../../../../:/mnt/feast - diff --git a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile index 7e36658caef..8b3c5b3d3d4 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile +++ b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile @@ -1,12 +1,13 @@ FROM python:3.9 -WORKDIR /usr/src/ - -COPY requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt - WORKDIR /app -COPY . . +COPY java/serving/src/test/resources/docker-compose/feast10/ . +COPY sdk/python /mnt/feast/sdk/python +COPY protos /mnt/feast/protos +COPY setup.py /mnt/feast/setup.py +COPY pyproject.toml /mnt/feast/pyproject.toml +COPY README.md /mnt/feast/README.md +RUN cd /mnt/feast && SETUPTOOLS_SCM_PRETEND_VERSION="0.1.0" pip install .[grpcio,redis] EXPOSE 8080 CMD ["./entrypoint.sh"] diff --git a/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh b/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh index 0690b734c38..82d9399521b 100755 --- a/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh +++ b/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh @@ -2,10 +2,6 @@ set -e -# feast root directory is expected to be mounted (eg, by docker compose) -cd /mnt/feast -pip install -e '.[grpcio,redis]' - cd /app python materialize.py feast serve_transformations --port 8080 diff --git a/java/serving/src/test/resources/docker-compose/feast10/requirements.txt b/java/serving/src/test/resources/docker-compose/feast10/requirements.txt deleted file mode 100644 index 6ba2c53d817..00000000000 --- a/java/serving/src/test/resources/docker-compose/feast10/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# for source generation -pyarrow==14.0.1 - -# temp fixes -proto-plus -Jinja2>=2.0.0 \ No newline at end of file From 953794611d06685d2b4ea19dd52a174ea6b2eea8 Mon Sep 17 00:00:00 2001 From: Jeremy Ary Date: Tue, 23 Apr 2024 08:20:25 -0500 Subject: [PATCH 14/73] chore: Incorporate release 0.37.1 to master (#4127) * chore(release): release 0.37.1 ## [0.37.1](https://github.com/feast-dev/feast/compare/v0.37.0...v0.37.1) (2024-04-17) ### Bug Fixes * Pgvector patch ([#4108](https://github.com/feast-dev/feast/issues/4108)) ([1a1f0b1](https://github.com/feast-dev/feast/commit/1a1f0b1c56aa2ac00b1e1aa1e21cc200ea659334)) ### Reverts * Reverts "fix: Using version args to install the correct feast version" ([#4112](https://github.com/feast-dev/feast/issues/4112)) ([d5ded69](https://github.com/feast-dev/feast/commit/d5ded69dea9af3a363feaa948cd3d2dcf10fb80c)), closes [#3953](https://github.com/feast-dev/feast/issues/3953) * chore: Move feast install to docker build in java it tests (#4126) * chore: Move feast install to docker build in java it tests Signed-off-by: tokoko * remove commented out lines in compose file Signed-off-by: tokoko * make local compose mode default Signed-off-by: tokoko * limit COPY contents Signed-off-by: tokoko * remove requirements.txt from java tests docker image Signed-off-by: tokoko * include pyproject.toml in dockerfile Signed-off-by: tokoko * change links to depends_on Signed-off-by: tokoko * try updating setup-python to v5 Signed-off-by: tokoko * pin macos image to macos-12 Signed-off-by: tokoko * force rerun Signed-off-by: tokoko --------- Signed-off-by: tokoko --------- Signed-off-by: tokoko Co-authored-by: feast-ci-bot Co-authored-by: Tornike Gurgenidze --- CHANGELOG.md | 12 ++++++++++++ infra/charts/feast-feature-server/Chart.yaml | 2 +- infra/charts/feast-feature-server/README.md | 4 ++-- infra/charts/feast-feature-server/values.yaml | 2 +- infra/charts/feast/Chart.yaml | 2 +- infra/charts/feast/README.md | 6 +++--- infra/charts/feast/charts/feature-server/Chart.yaml | 4 ++-- infra/charts/feast/charts/feature-server/README.md | 4 ++-- infra/charts/feast/charts/feature-server/values.yaml | 2 +- .../feast/charts/transformation-service/Chart.yaml | 4 ++-- .../feast/charts/transformation-service/README.md | 4 ++-- .../feast/charts/transformation-service/values.yaml | 2 +- infra/charts/feast/requirements.yaml | 4 ++-- java/pom.xml | 2 +- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 8 ++++---- ui/package.json | 2 +- 17 files changed, 39 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c5fc0f01b1..19dc5d86d7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [0.37.1](https://github.com/feast-dev/feast/compare/v0.37.0...v0.37.1) (2024-04-17) + + +### Bug Fixes + +* Pgvector patch ([#4108](https://github.com/feast-dev/feast/issues/4108)) ([1a1f0b1](https://github.com/feast-dev/feast/commit/1a1f0b1c56aa2ac00b1e1aa1e21cc200ea659334)) + + +### Reverts + +* Reverts "fix: Using version args to install the correct feast version" ([#4112](https://github.com/feast-dev/feast/issues/4112)) ([d5ded69](https://github.com/feast-dev/feast/commit/d5ded69dea9af3a363feaa948cd3d2dcf10fb80c)), closes [#3953](https://github.com/feast-dev/feast/issues/3953) + # [0.37.0](https://github.com/feast-dev/feast/compare/v0.36.0...v0.37.0) (2024-04-17) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index bd4bc606a70..8d564f3b420 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.37.0 +version: 0.37.1 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index fa6d89361c0..0730e39e63c 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.37.0` +Current chart version is `0.37.1` ## Installation @@ -30,7 +30,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.37.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.37.1"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 0de071ef3db..df5241ebb2d 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.37.0 + tag: 0.37.1 imagePullSecrets: [] nameOverride: "" diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 26a00d80c63..21c00e4483b 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.37.0 +version: 0.37.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 47959047dc2..8ab816dc707 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.37.0` +Feature store for machine learning Current chart version is `0.37.1` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.37.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.37.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.37.1 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.37.1 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index f2f8c748dc4..08563c6e069 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.37.0 -appVersion: v0.37.0 +version: 0.37.1 +appVersion: v0.37.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 531adce92da..3018b31c96c 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![AppVersion: v0.37.0](https://img.shields.io/badge/AppVersion-v0.37.0-informational?style=flat-square) +![Version: 0.37.1](https://img.shields.io/badge/Version-0.37.1-informational?style=flat-square) ![AppVersion: v0.37.1](https://img.shields.io/badge/AppVersion-v0.37.1-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.37.0"` | Image tag | +| image.tag | string | `"0.37.1"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 1bf1a03f4a7..1d86059c1fd 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.37.0 + tag: 0.37.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 056e00473fb..bad9befa0bf 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.37.0 -appVersion: v0.37.0 +version: 0.37.1 +appVersion: v0.37.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 4b11861d539..f912b4c02f7 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![AppVersion: v0.37.0](https://img.shields.io/badge/AppVersion-v0.37.0-informational?style=flat-square) +![Version: 0.37.1](https://img.shields.io/badge/Version-0.37.1-informational?style=flat-square) ![AppVersion: v0.37.1](https://img.shields.io/badge/AppVersion-v0.37.1-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.37.0"` | Image tag | +| image.tag | string | `"0.37.1"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index a04dfeb3e04..df5ea64c347 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.37.0 + tag: 0.37.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 9a2d3e0e807..80b8c861326 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.37.0 + version: 0.37.1 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.37.0 + version: 0.37.1 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/java/pom.xml b/java/pom.xml index 2d7e2c3e7d2..8ba8ed4ac53 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.37.0 + 0.37.1 https://github.com/feast-dev/feast UTF-8 diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index cdad51e0608..61c89f648f2 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.37.0", + "@feast-dev/feast-ui": "0.37.1", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 07472d7bbeb..5d0101fbe52 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1451,10 +1451,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.37.0": - version "0.37.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.37.0.tgz#53e04a73835617f23528bd95228190cf022338b0" - integrity sha512-bpW4K6y+XE9np+/zXsYffQWrND7Bx3z1xBj4SSddgX3QkGJyaJeOM9Rfu4ZhkjwNGvfqKaxGz9Ucnhpjv9lrfw== +"@feast-dev/feast-ui@0.37.1": + version "0.37.1" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.37.1.tgz#b84618d1fd2e1dbc463ab2889964006b555d9ec4" + integrity sha512-xhHK3hWvW58ukB+kx04ut+7OIT+zuITw6eYKjuJmjzAZ2S8uVcqDso4T9Ma88qX+qhn4NWzNBUyM2Gz1xOhzKQ== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" diff --git a/ui/package.json b/ui/package.json index 9209d7b03c7..ea69e571fb5 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.37.0", + "version": "0.37.1", "private": false, "files": [ "dist" From 6ef78522f26fd55898b9acf605f0662eea7bbfe8 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Wed, 24 Apr 2024 03:03:10 +0400 Subject: [PATCH 15/73] chore: Install python dependencies with uv in workflows (#4086) * install dependencies in unit-tests with uv Signed-off-by: tokoko * install dependencies in unit-tests with uv Signed-off-by: tokoko * enable caching, change linter job Signed-off-by: tokoko * change local integration tests to uv Signed-off-by: tokoko * change all installs to uv Signed-off-by: tokoko * try adding uv cache Signed-off-by: tokoko * fix lambda cache step name Signed-off-by: tokoko * reenable caches for uv Signed-off-by: tokoko * remove dangling cache step Signed-off-by: tokoko --------- Signed-off-by: tokoko --- .github/workflows/linter.yml | 22 ++--------- .github/workflows/master_only.yml | 31 +++++++--------- .github/workflows/nightly-ci.yml | 37 ++++++++----------- .github/workflows/pr_integration_tests.yml | 31 +++++++--------- .../workflows/pr_local_integration_tests.yml | 31 +++++++--------- .github/workflows/unit_tests.yml | 32 +++++++--------- Makefile | 5 +++ 7 files changed, 78 insertions(+), 111 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index a1747db1356..f235b6f3bac 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -11,31 +11,17 @@ jobs: - uses: actions/checkout@v3 - name: Setup Python id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: "3.9" architecture: x64 - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv - name: Install dependencies run: | - make install-python-ci-dependencies + make install-python-ci-dependencies-uv - name: Lint python run: make lint-python diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 225f24a828a..c355c55c23e 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -81,7 +81,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5 - name: Setup Python id: setup-python uses: actions/setup-python@v3 @@ -106,27 +106,22 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 3292d6bcb59..2db70879085 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -33,7 +33,7 @@ jobs: with: ref: master - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: "3.9" @@ -90,18 +90,18 @@ jobs: run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - name: Cache Public ECR Image id: lambda_python_3_9 - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/cache key: lambda_python_3_9 - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.cache-primes.outputs.cache-hit != 'true' + if: steps.lambda_python_3_9.outputs.cache-hit != 'true' run: | mkdir -p ~/cache docker pull public.ecr.aws/lambda/python:3.9 docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar - name: Handle Cache Hit (load docker image from tar file) - if: steps.cache-primes.outputs.cache-hit == 'true' + if: steps.lambda_python_3_9.outputs.cache-hit == 'true' run: | docker load -i ~/cache/lambda_python_3_9.tar - name: Build and push @@ -145,7 +145,7 @@ jobs: ref: master submodules: recursive - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} @@ -173,25 +173,20 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install apache-arrow on ubuntu if: matrix.os == 'ubuntu-latest' run: | diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 5e7287351b6..4d28c6b456c 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -110,7 +110,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} @@ -133,27 +133,22 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 266cdcc9b9f..be892ae9106 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -33,32 +33,27 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Test local integration tests if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak run: make test-python-integration-local diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index ff7c3d5e234..5d689d72de0 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -19,31 +19,27 @@ jobs: - uses: actions/checkout@v3 - name: Setup Python id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: | + pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Test Python run: make test-python-unit diff --git a/Makefile b/Makefile index bf2d876b7f1..f42d5c1edaf 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,11 @@ install-python-ci-dependencies: pip install --no-deps -e . python setup.py build_python_protos --inplace +install-python-ci-dependencies-uv: + uv pip sync --system sdk/python/requirements/py$(PYTHON)-ci-requirements.txt + uv pip install --system --no-deps -e . + python setup.py build_python_protos --inplace + lock-python-ci-dependencies: python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt From 3fdb71631fbb1b9cfb8d1cad69dbc2d2d50cea0d Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Thu, 25 Apr 2024 09:46:12 +0400 Subject: [PATCH 16/73] feat: Make arrow primary interchange for online ODFV execution (#4143) * rewrite online flow to use transform_arrow Signed-off-by: tokoko * fix transformation server Signed-off-by: tokoko --------- Signed-off-by: tokoko --- sdk/python/feast/feature_store.py | 28 +++--- sdk/python/feast/on_demand_feature_view.py | 89 +------------------ sdk/python/feast/online_response.py | 11 +++ .../transformation/python_transformation.py | 3 - .../substrait_transformation.py | 3 - sdk/python/feast/transformation_server.py | 5 +- .../tests/unit/test_on_demand_feature_view.py | 2 +- 7 files changed, 29 insertions(+), 112 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index fafec32c5d5..e83a24b6644 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -2138,7 +2138,7 @@ def _augment_response_with_on_demand_transforms( ) initial_response = OnlineResponse(online_features_response) - initial_response_df: Optional[pd.DataFrame] = None + initial_response_arrow: Optional[pa.Table] = None initial_response_dict: Optional[Dict[str, List[Any]]] = None # Apply on demand transformations and augment the result rows @@ -2148,18 +2148,14 @@ def _augment_response_with_on_demand_transforms( if odfv.mode == "python": if initial_response_dict is None: initial_response_dict = initial_response.to_dict() - transformed_features_dict: Dict[str, List[Any]] = ( - odfv.get_transformed_features( - initial_response_dict, - full_feature_names, - ) + transformed_features_dict: Dict[str, List[Any]] = odfv.transform_dict( + initial_response_dict ) elif odfv.mode in {"pandas", "substrait"}: - if initial_response_df is None: - initial_response_df = initial_response.to_df() - transformed_features_df: pd.DataFrame = odfv.get_transformed_features( - initial_response_df, - full_feature_names, + if initial_response_arrow is None: + initial_response_arrow = initial_response.to_arrow() + transformed_features_arrow = odfv.transform_arrow( + initial_response_arrow, full_feature_names ) else: raise Exception( @@ -2169,11 +2165,11 @@ def _augment_response_with_on_demand_transforms( transformed_features = ( transformed_features_dict if odfv.mode == "python" - else transformed_features_df + else transformed_features_arrow ) transformed_columns = ( - transformed_features.columns - if isinstance(transformed_features, pd.DataFrame) + transformed_features.column_names + if isinstance(transformed_features, pa.Table) else transformed_features ) selected_subset = [f for f in transformed_columns if f in _feature_refs] @@ -2183,6 +2179,10 @@ def _augment_response_with_on_demand_transforms( feature_vector = transformed_features[selected_feature] proto_values.append( python_values_to_proto_values(feature_vector, ValueType.UNKNOWN) + if odfv.mode == "python" + else python_values_to_proto_values( + feature_vector.to_numpy(), ValueType.UNKNOWN + ) ) odfv_result_names |= set(selected_subset) diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index b532fa651a1..e5de1f20a65 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -12,7 +12,6 @@ from typeguard import typechecked from feast.base_feature_view import BaseFeatureView -from feast.batch_feature_view import BatchFeatureView from feast.data_source import RequestSource from feast.errors import RegistryInferenceFailure, SpecifiedFeaturesNotPresentError from feast.feature_view import FeatureView @@ -493,53 +492,7 @@ def transform_arrow( ] ) - def get_transformed_features_df( - self, - df_with_features: pd.DataFrame, - full_feature_names: bool = False, - ) -> pd.DataFrame: - # Apply on demand transformations - if not isinstance(df_with_features, pd.DataFrame): - raise TypeError("get_transformed_features_df only accepts pd.DataFrame") - columns_to_cleanup = [] - for source_fv_projection in self.source_feature_view_projections.values(): - for feature in source_fv_projection.features: - full_feature_ref = f"{source_fv_projection.name}__{feature.name}" - if full_feature_ref in df_with_features.keys(): - # Make sure the partial feature name is always present - df_with_features[feature.name] = df_with_features[full_feature_ref] - columns_to_cleanup.append(feature.name) - elif feature.name in df_with_features.keys(): - # Make sure the full feature name is always present - df_with_features[full_feature_ref] = df_with_features[feature.name] - columns_to_cleanup.append(full_feature_ref) - - # Compute transformed values and apply to each result row - df_with_transformed_features: pd.DataFrame = ( - self.feature_transformation.transform(df_with_features) - ) - - # Work out whether the correct columns names are used. - rename_columns: Dict[str, str] = {} - for feature in self.features: - short_name = feature.name - long_name = self._get_projected_feature_name(feature.name) - if ( - short_name in df_with_transformed_features.columns - and full_feature_names - ): - rename_columns[short_name] = long_name - elif not full_feature_names: - # Long name must be in dataframe. - rename_columns[long_name] = short_name - - # Cleanup extra columns used for transformation - df_with_transformed_features = df_with_transformed_features[ - [f.name for f in self.features] - ] - return df_with_transformed_features.rename(columns=rename_columns) - - def get_transformed_features_dict( + def transform_dict( self, feature_dict: Dict[str, Any], # type: ignore ) -> Dict[str, Any]: @@ -566,29 +519,6 @@ def get_transformed_features_dict( del output_dict[feature_name] return output_dict - def get_transformed_features( - self, - features: Union[Dict[str, Any], pd.DataFrame], - full_feature_names: bool = False, - ) -> Union[Dict[str, Any], pd.DataFrame]: - # TODO: classic inheritance pattern....maybe fix this - if self.mode == "python" and isinstance(features, Dict): - # note full_feature_names is not needed for the dictionary - return self.get_transformed_features_dict( - feature_dict=features, - ) - elif self.mode in {"pandas", "substrait"} and isinstance( - features, pd.DataFrame - ): - return self.get_transformed_features_df( - df_with_features=features, - full_feature_names=full_feature_names, - ) - else: - raise Exception( - f'Invalid OnDemandFeatureMode: {self.mode}. Expected one of "pandas" or "python".' - ) - def infer_features(self) -> None: inferred_features = self.feature_transformation.infer_features( self._construct_random_input() @@ -745,23 +675,6 @@ def decorator(user_function): return decorator -def feature_view_to_batch_feature_view(fv: FeatureView) -> BatchFeatureView: - bfv = BatchFeatureView( - name=fv.name, - entities=fv.entities, - ttl=fv.ttl, - tags=fv.tags, - online=fv.online, - owner=fv.owner, - schema=fv.schema, - source=fv.batch_source, - ) - - bfv.features = copy.copy(fv.features) - bfv.entities = copy.copy(fv.entities) - return bfv - - def _empty_odfv_udf_fn(x: Any) -> Any: # just an identity mapping, otherwise we risk tripping some downstream tests return x diff --git a/sdk/python/feast/online_response.py b/sdk/python/feast/online_response.py index 48524359bf3..050b374340e 100644 --- a/sdk/python/feast/online_response.py +++ b/sdk/python/feast/online_response.py @@ -15,6 +15,7 @@ from typing import Any, Dict, List import pandas as pd +import pyarrow as pa from feast.feature_view import DUMMY_ENTITY_ID from feast.protos.feast.serving.ServingService_pb2 import GetOnlineFeaturesResponse @@ -77,3 +78,13 @@ def to_df(self, include_event_timestamps: bool = False) -> pd.DataFrame: """ return pd.DataFrame(self.to_dict(include_event_timestamps)) + + def to_arrow(self, include_event_timestamps: bool = False) -> pa.Table: + """ + Converts GetOnlineFeaturesResponse features into pyarrow Table. + + Args: + is_with_event_timestamps: bool Optionally include feature timestamps in the table + """ + + return pa.Table.from_pydict(self.to_dict(include_event_timestamps)) diff --git a/sdk/python/feast/transformation/python_transformation.py b/sdk/python/feast/transformation/python_transformation.py index ec950a24f3c..88cde7cc726 100644 --- a/sdk/python/feast/transformation/python_transformation.py +++ b/sdk/python/feast/transformation/python_transformation.py @@ -64,9 +64,6 @@ def __eq__(self, other): "Comparisons should only involve PythonTransformation class objects." ) - if not super().__eq__(other): - return False - if ( self.udf_string != other.udf_string or self.udf.__code__.co_code != other.udf.__code__.co_code diff --git a/sdk/python/feast/transformation/substrait_transformation.py b/sdk/python/feast/transformation/substrait_transformation.py index 48a87b62079..02b94d85726 100644 --- a/sdk/python/feast/transformation/substrait_transformation.py +++ b/sdk/python/feast/transformation/substrait_transformation.py @@ -77,9 +77,6 @@ def __eq__(self, other): "Comparisons should only involve SubstraitTransformation class objects." ) - if not super().__eq__(other): - return False - return ( self.substrait_plan == other.substrait_plan and self.ibis_function.__code__.co_code diff --git a/sdk/python/feast/transformation_server.py b/sdk/python/feast/transformation_server.py index 34fe3eac766..db8b0d942e2 100644 --- a/sdk/python/feast/transformation_server.py +++ b/sdk/python/feast/transformation_server.py @@ -45,15 +45,14 @@ def TransformFeatures(self, request, context): context.set_code(grpc.StatusCode.INVALID_ARGUMENT) raise - df = pa.ipc.open_file(request.transformation_input.arrow_value).read_pandas() + df = pa.ipc.open_file(request.transformation_input.arrow_value).read_all() if odfv.mode != "pandas": raise Exception( f'OnDemandFeatureView mode "{odfv.mode}" not supported by TransformationServer.' ) - result_df = odfv.get_transformed_features_df(df, True) - result_arrow = pa.Table.from_pandas(result_df) + result_arrow = odfv.transform_arrow(df, True) sink = pa.BufferOutputStream() writer = pa.ipc.new_file(sink, result_arrow.schema) writer.write_table(result_arrow) diff --git a/sdk/python/tests/unit/test_on_demand_feature_view.py b/sdk/python/tests/unit/test_on_demand_feature_view.py index cf4afa94228..402aa4e0e33 100644 --- a/sdk/python/tests/unit/test_on_demand_feature_view.py +++ b/sdk/python/tests/unit/test_on_demand_feature_view.py @@ -204,7 +204,7 @@ def test_python_native_transformation_mode(): } ) - assert on_demand_feature_view_python_native.get_transformed_features( + assert on_demand_feature_view_python_native.transform_dict( { "feature1": 0, "feature2": 1, From 73601e45e2fc57dc889644b1d28115b3c94bd8ea Mon Sep 17 00:00:00 2001 From: Pushkar Gupta Date: Thu, 25 Apr 2024 05:15:19 -0700 Subject: [PATCH 17/73] feat: Feast/IKV online store documentation (#4146) * feat: Feast/IKV online store documentation Signed-off-by: Pushkar Gupta * functionality matric Signed-off-by: Pushkar Gupta * more changes Signed-off-by: Pushkar Gupta * mount dir Signed-off-by: Pushkar Gupta --------- Signed-off-by: Pushkar Gupta --- README.md | 1 + docs/SUMMARY.md | 1 + docs/reference/online-stores/README.md | 4 ++ docs/reference/online-stores/ikv.md | 73 ++++++++++++++++++++++++ docs/reference/online-stores/overview.md | 38 ++++++------ docs/roadmap.md | 1 + 6 files changed, 99 insertions(+), 19 deletions(-) create mode 100644 docs/reference/online-stores/ikv.md diff --git a/README.md b/README.md index 6a851d0d417..aab9c332435 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Bigtable](https://docs.feast.dev/reference/online-stores/bigtable) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) * [x] [Dragonfly](https://docs.feast.dev/reference/online-stores/dragonfly) + * [x] [IKV - Inlined Key Value Store](https://docs.feast.dev/reference/online-stores/ikv) * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 8affea898ef..b211730d0ef 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -91,6 +91,7 @@ * [Snowflake](reference/online-stores/snowflake.md) * [Redis](reference/online-stores/redis.md) * [Dragonfly](reference/online-stores/dragonfly.md) + * [IKV](reference/online-stores/ikv.md) * [Datastore](reference/online-stores/datastore.md) * [DynamoDB](reference/online-stores/dynamodb.md) * [Bigtable](reference/online-stores/bigtable.md) diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index d90bfcf1632..686e820f4e7 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -22,6 +22,10 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli [dragonfly.md](dragonfly.md) {% endcontent-ref %} +{% content-ref url="ikv.md" %} +[ikv.md](ikv.md) +{% endcontent-ref %} + {% content-ref url="datastore.md" %} [datastore.md](datastore.md) {% endcontent-ref %} diff --git a/docs/reference/online-stores/ikv.md b/docs/reference/online-stores/ikv.md new file mode 100644 index 00000000000..ff690c1a622 --- /dev/null +++ b/docs/reference/online-stores/ikv.md @@ -0,0 +1,73 @@ +# IKV (Inlined Key-Value Store) online store + +## Description + +[IKV](https://github.com/inlinedio/ikv-store) is a fully-managed embedded key-value store, primarily designed for storing ML features. Most key-value stores (think Redis or Cassandra) need a remote database cluster, whereas IKV allows you to utilize your existing application infrastructure to store data (cost efficient) and access it without any network calls (better performance). + +For provisioning API keys for using it as an online-store in Feast, go to [https://inlined.io](https://inlined.io) or email onboarding[at]inlined.io + +## Getting started +Make sure you have Python and `pip` installed. + +Install the Feast SDK and CLI: `pip install feast` + +In order to use this online store, you'll need to install the IKV extra (along with the dependency needed for the offline store of choice). E.g. +- `pip install 'feast[gcp, ikv]'` +- `pip install 'feast[snowflake, ikv]'` +- `pip install 'feast[aws, ikv]'` +- `pip install 'feast[azure, ikv]'` + +You can get started by using any of the other templates (e.g. `feast init -t gcp` or `feast init -t snowflake` or `feast init -t aws`), and then swapping in IKV as the online store as seen below in the examples. + +### 1. Provision an IKV store +Go to [https://inlined.io](https://inlined.io) or email onboarding[at]inlined.io + +### 2. Configure + +Update `my_feature_repo/feature_store.yaml` with the below contents: + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: ikv + account_id: secret + account_passkey: secret + store_name: your-store-name + mount_directory: /absolute/path/on/disk/for/ikv/embedded/index +``` +{% endcode %} + +After provision an IKV account/store, you should the required id, passkey and store-name. + +Additionally you must specify a mount-directory - where IKV will pull/update (maintain) a copy of the index for online reads (IKV is an embedded database). It can be skipped only if you don't plan to read any data from this container. The mount directory path usually points to a location on local/remote disk. + +The full set of configuration options is available in IKVOnlineStoreConfig at `sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py` + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the IKV online store. + +| | IKV | +| :-------------------------------------------------------- | :---- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/overview.md b/docs/reference/online-stores/overview.md index 7a51a9a4687..04d24447058 100644 --- a/docs/reference/online-stores/overview.md +++ b/docs/reference/online-stores/overview.md @@ -29,26 +29,26 @@ See this [issue](https://github.com/feast-dev/feast/issues/2254) for a discussio ## Functionality Matrix There are currently five core online store implementations: `SqliteOnlineStore`, `RedisOnlineStore`, `DynamoDBOnlineStore`, `SnowflakeOnlineStore`, and `DatastoreOnlineStore`. -There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore`, and `CassandraOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. +There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore`, `CassandraOnlineStore` and `IKVOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. Details for each specific online store, such as how to configure it in a `feature_store.yaml`, can be found [here](README.md). Below is a matrix indicating which online stores support what functionality. -| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] | -| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | -| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | -| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | -| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | -| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | -| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | -| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | -| readable by Java | no | yes | no | no | no | no | no | no | -| readable by Go | yes | yes | no | no | no | no | no | no | -| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | -| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | -| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | -| support for deleting expired data | no | yes | no | no | no | no | no | no | -| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | -| collocated by feature service | no | no | no | no | no | no | no | no | -| collocated by entity key | no | yes | no | no | no | no | no | no | +| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] | [IKV](https://inlined.io) | +| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | +| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | no | +| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Java | no | yes | no | no | no | no | no | no | no | +| readable by Go | yes | yes | no | no | no | no | no | no | no | +| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | yes | +| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | no | +| support for deleting expired data | no | yes | no | no | no | no | no | no | no | +| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | no | +| collocated by feature service | no | no | no | no | no | no | no | no | no | +| collocated by entity key | no | yes | no | no | no | no | no | no | yes | diff --git a/docs/roadmap.md b/docs/roadmap.md index a04ede7c993..5ff262e3432 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -33,6 +33,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Bigtable](https://docs.feast.dev/reference/online-stores/bigtable) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) * [x] [Dragonfly](https://docs.feast.dev/reference/online-stores/dragonfly) + * [x] [IKV - Inlined Key Value Store](https://docs.feast.dev/reference/online-stores/ikv) * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) From 95acfb4cefc10f96f8ed61f148e24b238d400a68 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Thu, 25 Apr 2024 08:16:54 -0400 Subject: [PATCH 18/73] fix: Default value is not set in Redis connection string using environment variable (#4136) Removed documentation of Redis connection string supporting default values when using environment variables as it isn't supported Fixes #3669 Signed-off-by: Theodor Mihalache --- docs/how-to-guides/running-feast-in-production.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docs/how-to-guides/running-feast-in-production.md b/docs/how-to-guides/running-feast-in-production.md index 9d1984d7366..4663c928c6e 100644 --- a/docs/how-to-guides/running-feast-in-production.md +++ b/docs/how-to-guides/running-feast-in-production.md @@ -257,17 +257,6 @@ online_store: connection_string: ${REDIS_CONNECTION_STRING} ``` -It is possible to set a default value if the environment variable is not set, with `${ENV_VAR:"default"}`. For instance: - -```yaml -project: my_project -registry: data/registry.db -provider: local -online_store: - type: redis - connection_string: ${REDIS_CONNECTION_STRING:"0.0.0.0:6379"} -``` - *** ## Summary From c1579c77324cebb0514422235956812403316c80 Mon Sep 17 00:00:00 2001 From: Tom Steenbergen <41334387+TomSteenbergen@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:20:19 +0200 Subject: [PATCH 19/73] fix: Make sure schema is used when calling `get_table_query_string` method for Snowflake datasource (#4131) * Fix get_table_query_string method for Snowflake datasource Signed-off-by: TomSteenbergen * Add quotes to table string Signed-off-by: TomSteenbergen --------- Signed-off-by: TomSteenbergen --- sdk/python/feast/infra/offline_stores/snowflake_source.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/offline_stores/snowflake_source.py b/sdk/python/feast/infra/offline_stores/snowflake_source.py index c0b24170996..7ef2dbd6afb 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake_source.py +++ b/sdk/python/feast/infra/offline_stores/snowflake_source.py @@ -191,8 +191,10 @@ def validate(self, config: RepoConfig): def get_table_query_string(self) -> str: """Returns a string that can directly be used to reference this table in SQL.""" - if self.database and self.table: + if self.database and self.schema and self.table: return f'"{self.database}"."{self.schema}"."{self.table}"' + elif self.schema and self.table: + return f'"{self.schema}"."{self.table}"' elif self.table: return f'"{self.table}"' else: From 9523fff2dda2e0d53bffa7f5c0d6f2f69f6b8c02 Mon Sep 17 00:00:00 2001 From: Jeremy Ary Date: Thu, 25 Apr 2024 08:21:06 -0500 Subject: [PATCH 20/73] fix: Change checkout action back to v3 from v5 which isn't released yet (#4147) Signed-off-by: Jeremy Ary --- .github/workflows/master_only.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index c355c55c23e..3c244c1b01b 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -81,7 +81,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v3 - name: Setup Python id: setup-python uses: actions/setup-python@v3 From 811d71149a0309c135049cd25d7126881c337f0b Mon Sep 17 00:00:00 2001 From: Jeremy Ary Date: Thu, 25 Apr 2024 09:53:52 -0500 Subject: [PATCH 21/73] chore: Update checkout action from v3 to v4 in all workflows (#4148) chore: Update checkout action from v3 to v4 Signed-off-by: Jeremy Ary --- .github/fork_workflows/fork_pr_integration_tests_aws.yml | 4 ++-- .github/fork_workflows/fork_pr_integration_tests_gcp.yml | 2 +- .../fork_pr_integration_tests_snowflake.yml | 2 +- .github/workflows/build_wheels.yml | 8 ++++---- .github/workflows/java_master_only.yml | 8 ++++---- .github/workflows/java_pr.yml | 8 ++++---- .github/workflows/linter.yml | 2 +- .github/workflows/master_only.yml | 6 +++--- .github/workflows/nightly-ci.yml | 8 ++++---- .github/workflows/pr_integration_tests.yml | 4 ++-- .github/workflows/pr_local_integration_tests.yml | 2 +- .github/workflows/publish.yml | 8 ++++---- .github/workflows/release.yml | 8 ++++---- .github/workflows/unit_tests.yml | 4 ++-- 14 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index 52d7112a640..196feb78a49 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -7,7 +7,7 @@ jobs: if: github.repository == 'your github repo' # swap here with your project id runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve @@ -83,7 +83,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index 337d8040ae7..404d20c3034 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -25,7 +25,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve diff --git a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml index a3484a34625..02cb2ecf356 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml @@ -25,7 +25,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index ca0a7dcfe25..d48012f6ea5 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -18,7 +18,7 @@ jobs: highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false - name: Get release version @@ -55,7 +55,7 @@ jobs: name: Build wheels runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v3 with: @@ -81,7 +81,7 @@ jobs: name: Build source distribution runs-on: macos-12 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python id: setup-python uses: actions/setup-python@v3 @@ -120,7 +120,7 @@ jobs: env: REGISTRY: feastdev steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 79b456e571e..cb58b240fb2 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -18,7 +18,7 @@ jobs: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: gcr.io/kf-feast steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: 'true' - name: Setup Python @@ -53,7 +53,7 @@ jobs: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: 'true' - name: Lint java @@ -63,7 +63,7 @@ jobs: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: 'true' - name: Set up JDK 11 @@ -97,7 +97,7 @@ jobs: env: PYTHON: 3.9 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: 'true' - name: Set up JDK 11 diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index b78b5297d22..8c7a03d2e50 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -16,7 +16,7 @@ jobs: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve @@ -35,7 +35,7 @@ jobs: runs-on: ubuntu-latest needs: lint-java steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve @@ -81,7 +81,7 @@ jobs: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: gcr.io/kf-feast steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: 'true' - name: Setup Python @@ -113,7 +113,7 @@ jobs: env: PYTHON: 3.9 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index f235b6f3bac..2968a9d6e0d 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -8,7 +8,7 @@ jobs: env: PYTHON: 3.9 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python id: setup-python uses: actions/setup-python@v5 diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 3c244c1b01b..dd666d5588c 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -10,7 +10,7 @@ jobs: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -81,7 +81,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python id: setup-python uses: actions/setup-python@v3 @@ -157,7 +157,7 @@ jobs: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: gcr.io/kf-feast steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 2db70879085..4d8c7c09b8d 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -17,7 +17,7 @@ jobs: outputs: WAS_EDITED: ${{ steps.check_date.outputs.WAS_EDITED }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: master - id: check_date @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest name: Cleanup Bigtable / Dynamo tables which can fail to cleanup steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: master - name: Setup Python @@ -66,7 +66,7 @@ jobs: needs: [check_date] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: master submodules: recursive @@ -140,7 +140,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: master submodules: recursive diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 4d28c6b456c..5b59429bc71 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -21,7 +21,7 @@ jobs: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve @@ -102,7 +102,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index be892ae9106..258f4e42403 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -25,7 +25,7 @@ jobs: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 432ab4bb585..872a54a80ae 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Get release version id: get_release_version run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} @@ -54,7 +54,7 @@ jobs: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: feastdev steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -105,7 +105,7 @@ jobs: HELM_VERSION: v3.8.0 VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Authenticate to Google Cloud uses: 'google-github-actions/auth@v1' with: @@ -149,7 +149,7 @@ jobs: runs-on: ubuntu-latest needs: get-version steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: 'true' - name: Set up JDK 11 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b0d3e3cb390..5e2fcc1acb6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,7 +30,7 @@ jobs: next_version: ${{ steps.get_versions.outputs.next_version }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false - name: Setup Node.js @@ -58,7 +58,7 @@ jobs: CURRENT_VERSION: ${{ needs.get_dry_release_versions.outputs.current_version }} NEXT_VERSION: ${{ needs.get_dry_release_versions.outputs.next_version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Node.js uses: actions/setup-node@v3 with: @@ -99,7 +99,7 @@ jobs: CURRENT_VERSION: ${{ needs.get_dry_release_versions.outputs.current_version }} NEXT_VERSION: ${{ needs.get_dry_release_versions.outputs.next_version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Node.js uses: actions/setup-node@v3 with: @@ -132,7 +132,7 @@ jobs: GIT_COMMITTER_EMAIL: feast-ci-bot@willem.co steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false - name: Setup Node.js diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5d689d72de0..cbd222bdb7e 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -16,7 +16,7 @@ jobs: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python id: setup-python uses: actions/setup-python@v5 @@ -49,7 +49,7 @@ jobs: env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: node-version: '17.x' From 8b5698fefa965fc08fdb5e07d739d0ca276a3522 Mon Sep 17 00:00:00 2001 From: lokeshrangineni Date: Thu, 25 Apr 2024 12:26:18 -0400 Subject: [PATCH 22/73] fix: Changed the code the way mysql container is initialized. (#4140) * chore: Move feast install to docker build in java it tests (#4126) * chore: Move feast install to docker build in java it tests Signed-off-by: tokoko * remove commented out lines in compose file Signed-off-by: tokoko * make local compose mode default Signed-off-by: tokoko * limit COPY contents Signed-off-by: tokoko * remove requirements.txt from java tests docker image Signed-off-by: tokoko * include pyproject.toml in dockerfile Signed-off-by: tokoko * change links to depends_on Signed-off-by: tokoko * try updating setup-python to v5 Signed-off-by: tokoko * pin macos image to macos-12 Signed-off-by: tokoko * force rerun Signed-off-by: tokoko --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni * chore: Incorporate release 0.37.1 to master (#4127) * chore(release): release 0.37.1 ## [0.37.1](https://github.com/feast-dev/feast/compare/v0.37.0...v0.37.1) (2024-04-17) ### Bug Fixes * Pgvector patch ([#4108](https://github.com/feast-dev/feast/issues/4108)) ([1a1f0b1](https://github.com/feast-dev/feast/commit/1a1f0b1c56aa2ac00b1e1aa1e21cc200ea659334)) ### Reverts * Reverts "fix: Using version args to install the correct feast version" ([#4112](https://github.com/feast-dev/feast/issues/4112)) ([d5ded69](https://github.com/feast-dev/feast/commit/d5ded69dea9af3a363feaa948cd3d2dcf10fb80c)), closes [#3953](https://github.com/feast-dev/feast/issues/3953) * chore: Move feast install to docker build in java it tests (#4126) * chore: Move feast install to docker build in java it tests Signed-off-by: tokoko * remove commented out lines in compose file Signed-off-by: tokoko * make local compose mode default Signed-off-by: tokoko * limit COPY contents Signed-off-by: tokoko * remove requirements.txt from java tests docker image Signed-off-by: tokoko * include pyproject.toml in dockerfile Signed-off-by: tokoko * change links to depends_on Signed-off-by: tokoko * try updating setup-python to v5 Signed-off-by: tokoko * pin macos image to macos-12 Signed-off-by: tokoko * force rerun Signed-off-by: tokoko --------- Signed-off-by: tokoko --------- Signed-off-by: tokoko Co-authored-by: feast-ci-bot Co-authored-by: Tornike Gurgenidze Signed-off-by: Lokesh Rangineni * chore: Install python dependencies with uv in workflows (#4086) * install dependencies in unit-tests with uv Signed-off-by: tokoko * install dependencies in unit-tests with uv Signed-off-by: tokoko * enable caching, change linter job Signed-off-by: tokoko * change local integration tests to uv Signed-off-by: tokoko * change all installs to uv Signed-off-by: tokoko * try adding uv cache Signed-off-by: tokoko * fix lambda cache step name Signed-off-by: tokoko * reenable caches for uv Signed-off-by: tokoko * remove dangling cache step Signed-off-by: tokoko --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni * feat: Make arrow primary interchange for online ODFV execution (#4143) * rewrite online flow to use transform_arrow Signed-off-by: tokoko * fix transformation server Signed-off-by: tokoko --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni * feat: Feast/IKV online store documentation (#4146) * feat: Feast/IKV online store documentation Signed-off-by: Pushkar Gupta * functionality matric Signed-off-by: Pushkar Gupta * more changes Signed-off-by: Pushkar Gupta * mount dir Signed-off-by: Pushkar Gupta --------- Signed-off-by: Pushkar Gupta Signed-off-by: Lokesh Rangineni * fix: Default value is not set in Redis connection string using environment variable (#4136) Removed documentation of Redis connection string supporting default values when using environment variables as it isn't supported Fixes #3669 Signed-off-by: Theodor Mihalache Signed-off-by: Lokesh Rangineni * fix: Make sure schema is used when calling `get_table_query_string` method for Snowflake datasource (#4131) * Fix get_table_query_string method for Snowflake datasource Signed-off-by: TomSteenbergen * Add quotes to table string Signed-off-by: TomSteenbergen --------- Signed-off-by: TomSteenbergen Signed-off-by: Lokesh Rangineni * fix: Change checkout action back to v3 from v5 which isn't released yet (#4147) Signed-off-by: Jeremy Ary Signed-off-by: Lokesh Rangineni * changed the code the way mysql container is initialized. Trying to fix the issue - https://github.com/feast-dev/feast/issues/4128 Also going to check if this change will be resolved in the github actions as well. Signed-off-by: Lokesh Rangineni * reformatted the file to resolve lint error. Signed-off-by: Lokesh Rangineni * reformatted the file to resolve lint error. Signed-off-by: Lokesh Rangineni --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni Signed-off-by: Pushkar Gupta Signed-off-by: Theodor Mihalache Signed-off-by: TomSteenbergen Signed-off-by: Jeremy Ary Co-authored-by: Tornike Gurgenidze Co-authored-by: Jeremy Ary Co-authored-by: feast-ci-bot Co-authored-by: Pushkar Gupta Co-authored-by: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Co-authored-by: Tom Steenbergen <41334387+TomSteenbergen@users.noreply.github.com> --- sdk/python/tests/unit/test_sql_registry.py | 30 +++++++--------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/sdk/python/tests/unit/test_sql_registry.py b/sdk/python/tests/unit/test_sql_registry.py index 094b8967c18..a1460663aeb 100644 --- a/sdk/python/tests/unit/test_sql_registry.py +++ b/sdk/python/tests/unit/test_sql_registry.py @@ -21,6 +21,7 @@ from pytest_lazyfixture import lazy_fixture from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs +from testcontainers.mysql import MySqlContainer from feast import FileSource, RequestSource from feast.data_format import ParquetFormat @@ -41,7 +42,6 @@ POSTGRES_PASSWORD = "test" POSTGRES_DB = "test" - logger = logging.getLogger(__name__) @@ -81,32 +81,20 @@ def pg_registry(): @pytest.fixture(scope="session") def mysql_registry(): - container = ( - DockerContainer("mysql:latest") - .with_exposed_ports(3306) - .with_env("MYSQL_RANDOM_ROOT_PASSWORD", "true") - .with_env("MYSQL_USER", POSTGRES_USER) - .with_env("MYSQL_PASSWORD", POSTGRES_PASSWORD) - .with_env("MYSQL_DATABASE", POSTGRES_DB) - ) - + container = MySqlContainer("mysql:latest") container.start() - # The log string uses '8.0.*' since the version might be changed as new Docker images are pushed. - log_string_to_wait_for = "/usr/sbin/mysqld: ready for connections. Version: '(\\d+(\\.\\d+){1,2})' socket: '/var/run/mysqld/mysqld.sock' port: 3306" # noqa: W605 - waited = wait_for_logs( - container=container, - predicate=log_string_to_wait_for, - timeout=60, - interval=10, + # testing for the database to exist and ready to connect and start testing. + import sqlalchemy + + engine = sqlalchemy.create_engine( + container.get_connection_url(), pool_pre_ping=True ) - logger.info("Waited for %s seconds until mysql container was up", waited) - container_port = container.get_exposed_port(3306) - container_host = container.get_container_host_ip() + engine.connect() registry_config = RegistryConfig( registry_type="sql", - path=f"mysql+pymysql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{container_host}:{container_port}/{POSTGRES_DB}", + path=container.get_connection_url(), sqlalchemy_config_kwargs={"echo": False, "pool_pre_ping": True}, ) From e873636b4a5f3a05666f9284c31e488f27257ed0 Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Thu, 25 Apr 2024 19:13:32 -0700 Subject: [PATCH 23/73] fix: Update doc (#4153) update doc Signed-off-by: cmuhao --- docs/reference/online-stores/postgres.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/online-stores/postgres.md b/docs/reference/online-stores/postgres.md index 277494868c8..34d4de34883 100644 --- a/docs/reference/online-stores/postgres.md +++ b/docs/reference/online-stores/postgres.md @@ -64,7 +64,7 @@ Below is a matrix indicating which functionality is supported by the Postgres on To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). ## PGVector -The Postgres online store supports the use of [PGVector](https://pgvector.dev/) for storing feature values. +The Postgres online store supports the use of [PGVector](https://github.com/pgvector/pgvector) for storing feature values. To enable PGVector, set `pgvector_enabled: true` in the online store configuration. The `vector_len` parameter can be used to specify the length of the vector. The default value is 512. From c4917155823ae8649927789329dcdb20d5ee0df1 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Fri, 26 Apr 2024 14:40:37 -0400 Subject: [PATCH 24/73] docs: Update on demand feature view documentation (#4154) * docs: Updated on demand feature view documentation Signed-off-by: Francisco Javier Arceo * typo Signed-off-by: Francisco Javier Arceo * fixed linter Signed-off-by: Francisco Javier Arceo * adding test for explicit demo Signed-off-by: Francisco Javier Arceo * adding more tests and sorting the lists Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo --- README.md | 2 +- ...view.md => beta-on-demand-feature-view.md} | 70 +++++++++++++++-- docs/roadmap.md | 2 +- .../test_on_demand_python_transformation.py | 76 ++++++++++++++++++- 4 files changed, 142 insertions(+), 8 deletions(-) rename docs/reference/{alpha-on-demand-feature-view.md => beta-on-demand-feature-view.md} (61%) diff --git a/README.md b/README.md index aab9c332435..a1e06774dac 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/customizing-feast/adding-support-for-a-new-online-store) * **Feature Engineering** - * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) + * [x] On-demand Transformations (Beta release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) * [x] Streaming Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1UzEyETHUaGpn0ap4G82DHluiCj7zEbrQLkJJkKSv4e8/edit)) * [ ] Batch transformation (In progress. See [RFC](https://docs.google.com/document/d/1964OkzuBljifDvkV-0fakp2uaijnVzdwWNGdz7Vz50A/edit)) * **Streaming** diff --git a/docs/reference/alpha-on-demand-feature-view.md b/docs/reference/beta-on-demand-feature-view.md similarity index 61% rename from docs/reference/alpha-on-demand-feature-view.md rename to docs/reference/beta-on-demand-feature-view.md index 01b47d13dc3..6b4c3c667a0 100644 --- a/docs/reference/alpha-on-demand-feature-view.md +++ b/docs/reference/beta-on-demand-feature-view.md @@ -1,6 +1,6 @@ -# \[Alpha] On demand feature view +# \[Beta] On demand feature view -**Warning**: This is an _experimental_ feature. It's intended for early testing and feedback, and could change without warnings in future releases. +**Warning**: This is an experimental feature. To our knowledge, this is stable, but there are still rough edges in the experience. Contributions are welcome! ## Overview @@ -32,11 +32,14 @@ See [https://github.com/feast-dev/on-demand-feature-views-demo](https://github.c ### **Registering transformations** +On Demand Transformations support transformations using Pandas and native Python. Note, Native Python is much faster but not yet tested for offline retrieval. + We register `RequestSource` inputs and the transform in `on_demand_feature_view`: ```python from feast import Field, RequestSource from feast.types import Float64, Int64 +from typing import Any, Dict import pandas as pd # Define a request data source which encodes features / information only @@ -49,7 +52,7 @@ input_request = RequestSource( ] ) -# Use the input data and feature view features to create new features +# Use the input data and feature view features to create new features Pandas mode @on_demand_feature_view( sources=[ driver_hourly_stats_view, @@ -58,13 +61,43 @@ input_request = RequestSource( schema=[ Field(name='conv_rate_plus_val1', dtype=Float64), Field(name='conv_rate_plus_val2', dtype=Float64) - ] + ], + mode="pandas", ) def transformed_conv_rate(features_df: pd.DataFrame) -> pd.DataFrame: df = pd.DataFrame() df['conv_rate_plus_val1'] = (features_df['conv_rate'] + features_df['val_to_add']) df['conv_rate_plus_val2'] = (features_df['conv_rate'] + features_df['val_to_add_2']) return df + +# Use the input data and feature view features to create new features Python mode +@on_demand_feature_view( + sources=[ + driver_hourly_stats_view, + input_request + ], + schema=[ + Field(name='conv_rate_plus_val1_python', dtype=Float64), + Field(name='conv_rate_plus_val2_python', dtype=Float64), + ], + mode="python", +) +def transformed_conv_rate_python(inputs: Dict[str, Any]) -> Dict[str, Any]: + output: Dict[str, Any] = { + "conv_rate_plus_val1_python": [ + conv_rate + val_to_add + for conv_rate, val_to_add in zip( + inputs["conv_rate"], inputs["val_to_add"] + ) + ], + "conv_rate_plus_val2_python": [ + conv_rate + val_to_add + for conv_rate, val_to_add in zip( + inputs["conv_rate"], inputs["val_to_add_2"] + ) + ] + } + return output ``` ### **Feature retrieval** @@ -73,7 +106,9 @@ def transformed_conv_rate(features_df: pd.DataFrame) -> pd.DataFrame: The on demand feature view's name is the function name (i.e. `transformed_conv_rate`). {% endhint %} -And then to retrieve historical or online features, we can call this in a feature service or reference individual features: + +#### Offline Features +And then to retrieve historical, we can call this in a feature service or reference individual features: ```python training_df = store.get_historical_features( @@ -86,4 +121,29 @@ training_df = store.get_historical_features( "transformed_conv_rate:conv_rate_plus_val2", ], ).to_df() + +``` + +#### Online Features + +And then to retrieve online, we can call this in a feature service or reference individual features: + +```python +entity_rows = [ + { + "driver_id": 1001, + "val_to_add": 1, + "val_to_add_2": 2, + } +] + +online_response = store.get_online_features( + entity_rows=entity_rows, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "transformed_conv_rate_python:conv_rate_plus_val1_python", + "transformed_conv_rate_python:conv_rate_plus_val2_python", + ], +).to_dict() ``` diff --git a/docs/roadmap.md b/docs/roadmap.md index 5ff262e3432..e1ba6f3333e 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -39,7 +39,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/customizing-feast/adding-support-for-a-new-online-store) * **Feature Engineering** - * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) + * [x] On-demand Transformations (Beta release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) * [x] Streaming Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1UzEyETHUaGpn0ap4G82DHluiCj7zEbrQLkJJkKSv4e8/edit)) * [ ] Batch transformation (In progress. See [RFC](https://docs.google.com/document/d/1964OkzuBljifDvkV-0fakp2uaijnVzdwWNGdz7Vz50A/edit)) * **Streaming** diff --git a/sdk/python/tests/unit/test_on_demand_python_transformation.py b/sdk/python/tests/unit/test_on_demand_python_transformation.py index 4913b6c1b1d..e2db96c5f87 100644 --- a/sdk/python/tests/unit/test_on_demand_python_transformation.py +++ b/sdk/python/tests/unit/test_on_demand_python_transformation.py @@ -93,6 +93,31 @@ def python_view(inputs: Dict[str, Any]) -> Dict[str, Any]: } return output + @on_demand_feature_view( + sources=[driver_stats_fv[["conv_rate", "acc_rate"]]], + schema=[ + Field(name="conv_rate_plus_val1_python", dtype=Float64), + Field(name="conv_rate_plus_val2_python", dtype=Float64), + ], + mode="python", + ) + def python_demo_view(inputs: Dict[str, Any]) -> Dict[str, Any]: + output: Dict[str, Any] = { + "conv_rate_plus_val1_python": [ + conv_rate + acc_rate + for conv_rate, acc_rate in zip( + inputs["conv_rate"], inputs["acc_rate"] + ) + ], + "conv_rate_plus_val2_python": [ + conv_rate + acc_rate + for conv_rate, acc_rate in zip( + inputs["conv_rate"], inputs["acc_rate"] + ) + ], + } + return output + @on_demand_feature_view( sources=[driver_stats_fv[["conv_rate", "acc_rate"]]], schema=[ @@ -122,7 +147,14 @@ def python_singleton_view(inputs: Dict[str, Any]) -> Dict[str, Any]: ) self.store.apply( - [driver, driver_stats_source, driver_stats_fv, pandas_view, python_view] + [ + driver, + driver_stats_source, + driver_stats_fv, + pandas_view, + python_view, + python_demo_view, + ] ) self.store.write_to_online_store( feature_view_name="driver_hourly_stats", df=driver_df @@ -170,3 +202,45 @@ def test_python_pandas_parity(self): == online_python_response["conv_rate"][0] + online_python_response["acc_rate"][0] ) + + def test_python_docs_demo(self): + entity_rows = [ + { + "driver_id": 1001, + } + ] + + online_python_response = self.store.get_online_features( + entity_rows=entity_rows, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "python_demo_view:conv_rate_plus_val1_python", + "python_demo_view:conv_rate_plus_val2_python", + ], + ).to_dict() + + assert sorted(list(online_python_response.keys())) == sorted( + [ + "driver_id", + "acc_rate", + "conv_rate", + "conv_rate_plus_val1_python", + "conv_rate_plus_val2_python", + ] + ) + + assert ( + online_python_response["conv_rate_plus_val1_python"][0] + == online_python_response["conv_rate_plus_val2_python"][0] + ) + assert ( + online_python_response["conv_rate"][0] + + online_python_response["acc_rate"][0] + == online_python_response["conv_rate_plus_val1_python"][0] + ) + assert ( + online_python_response["conv_rate"][0] + + online_python_response["acc_rate"][0] + == online_python_response["conv_rate_plus_val2_python"][0] + ) From 93ddb11bf5a182cea44435147e39f40b30a69db7 Mon Sep 17 00:00:00 2001 From: lokeshrangineni Date: Fri, 26 Apr 2024 15:17:49 -0400 Subject: [PATCH 25/73] =?UTF-8?q?fix:=20Upgrading=20the=20test=20container?= =?UTF-8?q?=20so=20that=20local=20tests=20works=20with=20updated=20d?= =?UTF-8?q?=E2=80=A6=20(#4155)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upgrading the test container so that local tests works with updated docker cli versions. Signed-off-by: Lokesh Rangineni --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dcc616f120c..7cd2c302aef 100644 --- a/setup.py +++ b/setup.py @@ -177,7 +177,7 @@ "pytest-mock==1.10.4", "pytest-env", "Sphinx>4.0.0,<7", - "testcontainers==4.3.3", + "testcontainers==4.4.0", "firebase-admin>=5.2.0,<6", "pre-commit<3.3.2", "assertpy==1.1", From a27974b3609152469af160984b22e75be5bd81d7 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 30 Apr 2024 07:07:02 +0400 Subject: [PATCH 26/73] chore: Bump macOS runners to macos-13 (#4152) bump macos runner to 13 Signed-off-by: tokoko --- .github/fork_workflows/fork_pr_integration_tests_aws.yml | 5 ----- .github/fork_workflows/fork_pr_integration_tests_gcp.yml | 5 ----- .../fork_workflows/fork_pr_integration_tests_snowflake.yml | 5 ----- .github/workflows/build_wheels.yml | 6 +++--- .github/workflows/nightly-ci.yml | 2 +- .github/workflows/unit_tests.yml | 4 ++-- 6 files changed, 6 insertions(+), 21 deletions(-) diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index 196feb78a49..4e418e6b22e 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -137,11 +137,6 @@ jobs: sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb sudo apt update sudo apt install -y -V libarrow-dev - - name: Install apache-arrow on macos - if: matrix.os == 'macos-12' - run: | - brew install apache-arrow - brew install pkg-config - name: Install dependencies run: make install-python-ci-dependencies - name: Setup Redis Cluster diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index 404d20c3034..a6fc2110c55 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -81,11 +81,6 @@ jobs: sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb sudo apt update sudo apt install -y -V libarrow-dev - - name: Install apache-arrow on macos - if: matrix.os == 'macOS-12' - run: | - brew install apache-arrow - brew install pkg-config - name: Install dependencies run: make install-python-ci-dependencies - name: Setup Redis Cluster diff --git a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml index 02cb2ecf356..ee0e256eca6 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml @@ -71,11 +71,6 @@ jobs: sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb sudo apt update sudo apt install -y -V libarrow-dev - - name: Install apache-arrow on macos - if: matrix.os == 'macos-12' - run: | - brew install apache-arrow - brew install pkg-config - name: Install dependencies run: make install-python-ci-dependencies - name: Setup Redis Cluster diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index d48012f6ea5..59d274c4941 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -79,7 +79,7 @@ jobs: build-source-distribution: name: Build source distribution - runs-on: macos-12 + runs-on: macos-13 steps: - uses: actions/checkout@v4 - name: Setup Python @@ -136,7 +136,7 @@ jobs: needs: [build-python-wheel, build-source-distribution, get-version] strategy: matrix: - os: [ubuntu-latest, macos-12 ] + os: [ubuntu-latest, macos-13 ] python-version: ["3.9", "3.10"] from-source: [ True, False ] env: @@ -165,7 +165,7 @@ jobs: name: wheels path: dist - name: Install OS X dependencies - if: matrix.os == 'macos-12' + if: matrix.os == 'macos-13' run: brew install coreutils - name: Install wheel if: ${{ !matrix.from-source }} diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 4d8c7c09b8d..9afea5cb6f1 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -197,7 +197,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macos-12' + if: matrix.os == 'macos-13' run: brew install apache-arrow - name: Install dependencies run: make install-python-ci-dependencies diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index cbd222bdb7e..576da4e8f53 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -8,9 +8,9 @@ jobs: fail-fast: false matrix: python-version: [ "3.9", "3.10" ] - os: [ ubuntu-latest, macos-12 ] + os: [ ubuntu-latest, macos-13 ] exclude: - - os: macos-12 + - os: macos-13 python-version: "3.9" env: OS: ${{ matrix.os }} From c91dd69d32aee365d8ef027842dfb54523abf876 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 30 Apr 2024 18:09:54 +0400 Subject: [PATCH 27/73] chore: Use pixi to lock python dependencies in a single command (#4114) use pixi to lock python dependencies in a single command Signed-off-by: tokoko --- Makefile | 6 + infra/scripts/pixi/.gitattributes | 3 + infra/scripts/pixi/.gitignore | 4 + infra/scripts/pixi/pixi.lock | 695 ++++++++++++++++++++++++++++++ infra/scripts/pixi/pixi.toml | 19 + 5 files changed, 727 insertions(+) create mode 100644 infra/scripts/pixi/.gitattributes create mode 100644 infra/scripts/pixi/.gitignore create mode 100644 infra/scripts/pixi/pixi.lock create mode 100644 infra/scripts/pixi/pixi.toml diff --git a/Makefile b/Makefile index f42d5c1edaf..f2ea2cb4d0c 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,12 @@ install-python: lock-python-dependencies: python -m piptools compile -U --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt +lock-python-dependencies-all: + pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.9-requirements.txt" + pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt" + pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.10-requirements.txt" + pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt" + benchmark-python: FEAST_USAGE=False IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests diff --git a/infra/scripts/pixi/.gitattributes b/infra/scripts/pixi/.gitattributes new file mode 100644 index 00000000000..16ef5c5f786 --- /dev/null +++ b/infra/scripts/pixi/.gitattributes @@ -0,0 +1,3 @@ +# GitHub syntax highlighting +pixi.lock linguist-language=YAML + diff --git a/infra/scripts/pixi/.gitignore b/infra/scripts/pixi/.gitignore new file mode 100644 index 00000000000..44ba5fb4af4 --- /dev/null +++ b/infra/scripts/pixi/.gitignore @@ -0,0 +1,4 @@ +# pixi environments +.pixi +*.egg-info + diff --git a/infra/scripts/pixi/pixi.lock b/infra/scripts/pixi/pixi.lock new file mode 100644 index 00000000000..65b761156e1 --- /dev/null +++ b/infra/scripts/pixi/pixi.lock @@ -0,0 +1,695 @@ +version: 4 +environments: + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.3-hab00c5b_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda + py310: + channels: + - url: https://conda.anaconda.org/conda-forge/ + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.14-hd12c33a_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda + py39: + channels: + - url: https://conda.anaconda.org/conda-forge/ + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.9.19-h0755675_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda +packages: +- kind: conda + name: _libgcc_mutex + version: '0.1' + build: conda_forge + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + sha256: fe51de6107f9edc7aa4f786a70f4a883943bc9d39b3bb7307c04c41410990726 + md5: d7c89558ba9fa0495403155b64376d81 + license: None + size: 2562 + timestamp: 1578324546067 +- kind: conda + name: _openmp_mutex + version: '4.5' + build: 2_gnu + build_number: 16 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + sha256: fbe2c5e56a653bebb982eda4876a9178aedfc2b545f25d0ce9c4c0b508253d22 + md5: 73aaf86a425cc6e73fcf236a5a46396d + depends: + - _libgcc_mutex 0.1 conda_forge + - libgomp >=7.5.0 + constrains: + - openmp_impl 9999 + license: BSD-3-Clause + license_family: BSD + size: 23621 + timestamp: 1650670423406 +- kind: conda + name: bzip2 + version: 1.0.8 + build: hd590300_5 + build_number: 5 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda + sha256: 242c0c324507ee172c0e0dd2045814e746bb303d1eb78870d182ceb0abc726a8 + md5: 69b8b6202a07720f448be700e300ccf4 + depends: + - libgcc-ng >=12 + license: bzip2-1.0.6 + license_family: BSD + size: 254228 + timestamp: 1699279927352 +- kind: conda + name: ca-certificates + version: 2024.2.2 + build: hbcca054_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda + sha256: 91d81bfecdbb142c15066df70cc952590ae8991670198f92c66b62019b251aeb + md5: 2f4327a1cbe7f022401b236e915a5fef + license: ISC + size: 155432 + timestamp: 1706843687645 +- kind: conda + name: click + version: 8.1.7 + build: unix_pyh707e725_0 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda + sha256: f0016cbab6ac4138a429e28dbcb904a90305b34b3fe41a9b89d697c90401caec + md5: f3ad426304898027fc619827ff428eca + depends: + - __unix + - python >=3.8 + license: BSD-3-Clause + license_family: BSD + size: 84437 + timestamp: 1692311973840 +- kind: conda + name: colorama + version: 0.4.6 + build: pyhd8ed1ab_0 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 + sha256: 2c1b2e9755ce3102bca8d69e8f26e4f087ece73f50418186aee7c74bef8e1698 + md5: 3faab06a954c2a04039983f2c4a50d99 + depends: + - python >=3.7 + license: BSD-3-Clause + license_family: BSD + size: 25170 + timestamp: 1666700778190 +- kind: conda + name: importlib-metadata + version: 7.1.0 + build: pyha770c72_0 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda + sha256: cc2e7d1f7f01cede30feafc1118b7aefa244d0a12224513734e24165ae12ba49 + md5: 0896606848b2dc5cebdf111b6543aa04 + depends: + - python >=3.8 + - zipp >=0.5 + license: Apache-2.0 + license_family: APACHE + size: 27043 + timestamp: 1710971498183 +- kind: conda + name: ld_impl_linux-64 + version: '2.40' + build: h41732ed_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda + sha256: f6cc89d887555912d6c61b295d398cff9ec982a3417d38025c45d5dd9b9e79cd + md5: 7aca3059a1729aa76c597603f10b0dd3 + constrains: + - binutils_impl_linux-64 2.40 + license: GPL-3.0-only + license_family: GPL + size: 704696 + timestamp: 1674833944779 +- kind: conda + name: libexpat + version: 2.6.2 + build: h59595ed_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda + sha256: 331bb7c7c05025343ebd79f86ae612b9e1e74d2687b8f3179faec234f986ce19 + md5: e7ba12deb7020dd080c6c70e7b6f6a3d + depends: + - libgcc-ng >=12 + constrains: + - expat 2.6.2.* + license: MIT + license_family: MIT + size: 73730 + timestamp: 1710362120304 +- kind: conda + name: libffi + version: 3.4.2 + build: h7f98852_5 + build_number: 5 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 + sha256: ab6e9856c21709b7b517e940ae7028ae0737546122f83c2aa5d692860c3b149e + md5: d645c6d2ac96843a2bfaccd2d62b3ac3 + depends: + - libgcc-ng >=9.4.0 + license: MIT + license_family: MIT + size: 58292 + timestamp: 1636488182923 +- kind: conda + name: libgcc-ng + version: 13.2.0 + build: h807b86a_5 + build_number: 5 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda + sha256: d32f78bfaac282cfe5205f46d558704ad737b8dbf71f9227788a5ca80facaba4 + md5: d4ff227c46917d3b4565302a2bbb276b + depends: + - _libgcc_mutex 0.1 conda_forge + - _openmp_mutex >=4.5 + constrains: + - libgomp 13.2.0 h807b86a_5 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 770506 + timestamp: 1706819192021 +- kind: conda + name: libgomp + version: 13.2.0 + build: h807b86a_5 + build_number: 5 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda + sha256: 0d3d4b1b0134283ea02d58e8eb5accf3655464cf7159abf098cc694002f8d34e + md5: d211c42b9ce49aee3734fdc828731689 + depends: + - _libgcc_mutex 0.1 conda_forge + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 419751 + timestamp: 1706819107383 +- kind: conda + name: libnsl + version: 2.0.1 + build: hd590300_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda + sha256: 26d77a3bb4dceeedc2a41bd688564fe71bf2d149fdcf117049970bc02ff1add6 + md5: 30fd6e37fe21f86f4bd26d6ee73eeec7 + depends: + - libgcc-ng >=12 + license: LGPL-2.1-only + license_family: GPL + size: 33408 + timestamp: 1697359010159 +- kind: conda + name: libsqlite + version: 3.45.3 + build: h2797004_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda + sha256: e2273d6860eadcf714a759ffb6dc24a69cfd01f2a0ea9d6c20f86049b9334e0c + md5: b3316cbe90249da4f8e84cd66e1cc55b + depends: + - libgcc-ng >=12 + - libzlib >=1.2.13,<1.3.0a0 + license: Unlicense + size: 859858 + timestamp: 1713367435849 +- kind: conda + name: libuuid + version: 2.38.1 + build: h0b41bf4_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda + sha256: 787eb542f055a2b3de553614b25f09eefb0a0931b0c87dbcce6efdfd92f04f18 + md5: 40b61aab5c7ba9ff276c41cfffe6b80b + depends: + - libgcc-ng >=12 + license: BSD-3-Clause + license_family: BSD + size: 33601 + timestamp: 1680112270483 +- kind: conda + name: libxcrypt + version: 4.4.36 + build: hd590300_1 + build_number: 1 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda + sha256: 6ae68e0b86423ef188196fff6207ed0c8195dd84273cb5623b85aa08033a410c + md5: 5aa797f8787fe7a17d1b0821485b5adc + depends: + - libgcc-ng >=12 + license: LGPL-2.1-or-later + size: 100393 + timestamp: 1702724383534 +- kind: conda + name: libzlib + version: 1.2.13 + build: hd590300_5 + build_number: 5 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda + sha256: 370c7c5893b737596fd6ca0d9190c9715d89d888b8c88537ae1ef168c25e82e4 + md5: f36c115f1ee199da648e0597ec2047ad + depends: + - libgcc-ng >=12 + constrains: + - zlib 1.2.13 *_5 + license: Zlib + license_family: Other + size: 61588 + timestamp: 1686575217516 +- kind: conda + name: ncurses + version: 6.4.20240210 + build: h59595ed_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda + sha256: aa0f005b6727aac6507317ed490f0904430584fa8ca722657e7f0fb94741de81 + md5: 97da8860a0da5413c7c98a3b3838a645 + depends: + - libgcc-ng >=12 + license: X11 AND BSD-3-Clause + size: 895669 + timestamp: 1710866638986 +- kind: conda + name: openssl + version: 3.2.1 + build: hd590300_1 + build_number: 1 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda + sha256: 2c689444ed19a603be457284cf2115ee728a3fafb7527326e96054dee7cdc1a7 + md5: 9d731343cff6ee2e5a25c4a091bf8e2a + depends: + - ca-certificates + - libgcc-ng >=12 + constrains: + - pyopenssl >=22.1 + license: Apache-2.0 + license_family: Apache + size: 2865379 + timestamp: 1710793235846 +- kind: conda + name: packaging + version: '24.0' + build: pyhd8ed1ab_0 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda + sha256: a390182d74c31dfd713c16db888c92c277feeb6d1fe96ff9d9c105f9564be48a + md5: 248f521b64ce055e7feae3105e7abeb8 + depends: + - python >=3.8 + license: Apache-2.0 + license_family: APACHE + size: 49832 + timestamp: 1710076089469 +- kind: conda + name: pip + version: '24.0' + build: pyhd8ed1ab_0 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda + sha256: b7c1c5d8f13e8cb491c4bd1d0d1896a4cf80fc47de01059ad77509112b664a4a + md5: f586ac1e56c8638b64f9c8122a7b8a67 + depends: + - python >=3.7 + - setuptools + - wheel + license: MIT + license_family: MIT + size: 1398245 + timestamp: 1706960660581 +- kind: conda + name: pip-tools + version: 7.4.1 + build: pyhd8ed1ab_0 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda + sha256: 5534c19a6233faed1c9109782322c9d31e536ce20448f8c90db3d864fb8f226d + md5: 73203bd783da9c37c2cdabb1f3b9d44d + depends: + - click >=7 + - pip >=21.2 + - python >=3.7 + - python-build + - setuptools + - wheel + license: BSD-3-Clause + license_family: BSD + size: 54113 + timestamp: 1709736180083 +- kind: conda + name: pyproject_hooks + version: 1.0.0 + build: pyhd8ed1ab_0 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda + sha256: 016340837fcfef57b351febcbe855eedf0c1f0ecfc910ed48c7fbd20535f9847 + md5: 21de50391d584eb7f4441b9de1ad773f + depends: + - python >=3.7 + - tomli >=1.1.0 + license: MIT + license_family: MIT + size: 13867 + timestamp: 1670268791173 +- kind: conda + name: python + version: 3.9.19 + build: h0755675_0_cpython + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/python-3.9.19-h0755675_0_cpython.conda + sha256: b9253ca9ca5427e6da4b1d43353a110e0f2edfab9c951afb4bf01cbae2825b31 + md5: d9ee3647fbd9e8595b8df759b2bbefb8 + depends: + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libffi >=3.4,<4.0a0 + - libgcc-ng >=12 + - libnsl >=2.0.1,<2.1.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libuuid >=2.38.1,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.2.13,<1.3.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.9.* *_cp39 + license: Python-2.0 + size: 23800555 + timestamp: 1710940120866 +- kind: conda + name: python + version: 3.10.14 + build: hd12c33a_0_cpython + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.14-hd12c33a_0_cpython.conda + sha256: 76a5d12e73542678b70a94570f7b0f7763f9a938f77f0e75d9ea615ef22aa84c + md5: 2b4ba962994e8bd4be9ff5b64b75aff2 + depends: + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libffi >=3.4,<4.0a0 + - libgcc-ng >=12 + - libnsl >=2.0.1,<2.1.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libuuid >=2.38.1,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.2.13,<1.3.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.10.* *_cp310 + license: Python-2.0 + size: 25517742 + timestamp: 1710939725109 +- kind: conda + name: python + version: 3.12.3 + build: hab00c5b_0_cpython + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.3-hab00c5b_0_cpython.conda + sha256: f9865bcbff69f15fd89a33a2da12ad616e98d65ce7c83c644b92e66e5016b227 + md5: 2540b74d304f71d3e89c81209db4db84 + depends: + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libexpat >=2.6.2,<3.0a0 + - libffi >=3.4,<4.0a0 + - libgcc-ng >=12 + - libnsl >=2.0.1,<2.1.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libuuid >=2.38.1,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.2.13,<1.3.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + size: 31991381 + timestamp: 1713208036041 +- kind: conda + name: python-build + version: 1.2.1 + build: pyhd8ed1ab_0 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda + sha256: 3104051be7279d1b15f0a4be79f4bfeaf3a42b2900d24a7ad8e980df903fe8db + md5: d657cde3b3943fcedf6038138eea84de + depends: + - colorama + - importlib-metadata >=4.6 + - packaging >=19.0 + - pyproject_hooks + - python >=3.8 + - tomli >=1.1.0 + constrains: + - build <0 + license: MIT + license_family: MIT + size: 24434 + timestamp: 1711647439510 +- kind: conda + name: readline + version: '8.2' + build: h8228510_1 + build_number: 1 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda + sha256: 5435cf39d039387fbdc977b0a762357ea909a7694d9528ab40f005e9208744d7 + md5: 47d31b792659ce70f470b5c82fdfb7a4 + depends: + - libgcc-ng >=12 + - ncurses >=6.3,<7.0a0 + license: GPL-3.0-only + license_family: GPL + size: 281456 + timestamp: 1679532220005 +- kind: conda + name: setuptools + version: 69.5.1 + build: pyhd8ed1ab_0 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda + sha256: 72d143408507043628b32bed089730b6d5f5445eccc44b59911ec9f262e365e7 + md5: 7462280d81f639363e6e63c81276bd9e + depends: + - python >=3.8 + license: MIT + license_family: MIT + size: 501790 + timestamp: 1713094963112 +- kind: conda + name: tk + version: 8.6.13 + build: noxft_h4845f30_101 + build_number: 101 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda + sha256: e0569c9caa68bf476bead1bed3d79650bb080b532c64a4af7d8ca286c08dea4e + md5: d453b98d9c83e71da0741bb0ff4d76bc + depends: + - libgcc-ng >=12 + - libzlib >=1.2.13,<1.3.0a0 + license: TCL + license_family: BSD + size: 3318875 + timestamp: 1699202167581 +- kind: conda + name: tomli + version: 2.0.1 + build: pyhd8ed1ab_0 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 + sha256: 4cd48aba7cd026d17e86886af48d0d2ebc67ed36f87f6534f4b67138f5a5a58f + md5: 5844808ffab9ebdb694585b50ba02a96 + depends: + - python >=3.7 + license: MIT + license_family: MIT + size: 15940 + timestamp: 1644342331069 +- kind: conda + name: tzdata + version: 2024a + build: h0c530f3_0 + subdir: noarch + noarch: generic + url: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + sha256: 7b2b69c54ec62a243eb6fba2391b5e443421608c3ae5dbff938ad33ca8db5122 + md5: 161081fc7cec0bfda0d86d7cb595f8d8 + license: LicenseRef-Public-Domain + size: 119815 + timestamp: 1706886945727 +- kind: conda + name: wheel + version: 0.43.0 + build: pyhd8ed1ab_1 + build_number: 1 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda + sha256: cb318f066afd6fd64619f14c030569faf3f53e6f50abf743b4c865e7d95b96bc + md5: 0b5293a157c2b5cd513dd1b03d8d3aae + depends: + - python >=3.8 + license: MIT + license_family: MIT + size: 57963 + timestamp: 1711546009410 +- kind: conda + name: xz + version: 5.2.6 + build: h166bdaf_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + sha256: 03a6d28ded42af8a347345f82f3eebdd6807a08526d47899a42d62d319609162 + md5: 2161070d867d1b1204ea749c8eec4ef0 + depends: + - libgcc-ng >=12 + license: LGPL-2.1 and GPL-2.0 + size: 418368 + timestamp: 1660346797927 +- kind: conda + name: zipp + version: 3.17.0 + build: pyhd8ed1ab_0 + subdir: noarch + noarch: python + url: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda + sha256: bced1423fdbf77bca0a735187d05d9b9812d2163f60ab426fc10f11f92ecbe26 + md5: 2e4d6bc0b14e10f895fc6791a7d9b26a + depends: + - python >=3.8 + license: MIT + license_family: MIT + size: 18954 + timestamp: 1695255262261 diff --git a/infra/scripts/pixi/pixi.toml b/infra/scripts/pixi/pixi.toml new file mode 100644 index 00000000000..80a29d3a59a --- /dev/null +++ b/infra/scripts/pixi/pixi.toml @@ -0,0 +1,19 @@ +[project] +name = "pixi-feast" +channels = ["conda-forge"] +platforms = ["linux-64"] + +[tasks] + +[dependencies] +pip-tools = ">=7.4.1,<7.5" + +[feature.py39.dependencies] +python = "~=3.9.0" + +[feature.py310.dependencies] +python = "~=3.10.0" + +[environments] +py39 = ["py39"] +py310 = ["py310"] \ No newline at end of file From abfac011ad1f94caef001539591d03b1552f65e5 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Tue, 30 Apr 2024 15:16:01 -0400 Subject: [PATCH 28/73] feat: Adding support for dictionary writes to online store (#4156) * feat: Adding support for dictionary writes to online store Signed-off-by: Francisco Javier Arceo * Simple approach Signed-off-by: Francisco Javier Arceo * lint Signed-off-by: Francisco Javier Arceo * adding error if both are missing Signed-off-by: Francisco Javier Arceo * rename dict to input_dict Signed-off-by: Francisco Javier Arceo * updated input arg to test Signed-off-by: Francisco Javier Arceo * Renaming function argument Signed-off-by: Francisco Javier Arceo * updated docstring Signed-off-by: Francisco Javier Arceo * updated type signature Signed-off-by: Francisco Javier Arceo * updated type to be more explicit Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/errors.py | 7 + sdk/python/feast/feature_store.py | 17 ++- .../unit/online_store/test_online_writes.py | 139 ++++++++++++++++++ 3 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 sdk/python/tests/unit/online_store/test_online_writes.py diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index b7151ff0c87..52fefce9d90 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -420,3 +420,10 @@ def __init__(self, push_source_name: str): class ReadOnlyRegistryException(Exception): def __init__(self): super().__init__("Registry implementation is read-only.") + + +class DataFrameSerializationError(Exception): + def __init__(self, input_dict: dict): + super().__init__( + f"Failed to serialize the provided dictionary into a pandas DataFrame: {input_dict.keys()}" + ) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index e83a24b6644..bc492e42086 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -54,6 +54,7 @@ from feast.dqm.errors import ValidationFailed from feast.entity import Entity from feast.errors import ( + DataFrameSerializationError, DataSourceRepeatNamesException, EntityNotFoundException, FeatureNameCollisionError, @@ -1406,7 +1407,8 @@ def push( def write_to_online_store( self, feature_view_name: str, - df: pd.DataFrame, + df: Optional[pd.DataFrame] = None, + inputs: Optional[Union[Dict[str, List[Any]], pd.DataFrame]] = None, allow_registry_cache: bool = True, ): """ @@ -1415,6 +1417,7 @@ def write_to_online_store( Args: feature_view_name: The feature view to which the dataframe corresponds. df: The dataframe to be persisted. + inputs: Optional the dictionary object to be written allow_registry_cache (optional): Whether to allow retrieving feature views from a cached registry. """ # TODO: restrict this to work with online StreamFeatureViews and validate the FeatureView type @@ -1426,6 +1429,18 @@ def write_to_online_store( feature_view = self.get_feature_view( feature_view_name, allow_registry_cache=allow_registry_cache ) + if df is not None and inputs is not None: + raise ValueError("Both df and inputs cannot be provided at the same time.") + if df is None and inputs is not None: + if isinstance(inputs, dict): + try: + df = pd.DataFrame(inputs) + except Exception as _: + raise DataFrameSerializationError(inputs) + elif isinstance(inputs, pd.DataFrame): + pass + else: + raise ValueError("inputs must be a dictionary or a pandas DataFrame.") provider = self._get_provider() provider.ingest_df(feature_view, df) diff --git a/sdk/python/tests/unit/online_store/test_online_writes.py b/sdk/python/tests/unit/online_store/test_online_writes.py new file mode 100644 index 00000000000..5fb13519692 --- /dev/null +++ b/sdk/python/tests/unit/online_store/test_online_writes.py @@ -0,0 +1,139 @@ +# Copyright 2022 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +import unittest +from datetime import datetime, timedelta +from typing import Any, Dict + +from feast import Entity, FeatureStore, FeatureView, FileSource, RepoConfig +from feast.driver_test_data import create_driver_hourly_stats_df +from feast.field import Field +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig +from feast.on_demand_feature_view import on_demand_feature_view +from feast.types import Float32, Float64, Int64 + + +class TestOnlineWrites(unittest.TestCase): + def setUp(self): + with tempfile.TemporaryDirectory() as data_dir: + self.store = FeatureStore( + config=RepoConfig( + project="test_write_to_online_store", + registry=os.path.join(data_dir, "registry.db"), + provider="local", + entity_key_serialization_version=2, + online_store=SqliteOnlineStoreConfig( + path=os.path.join(data_dir, "online.db") + ), + ) + ) + + # Generate test data. + end_date = datetime.now().replace(microsecond=0, second=0, minute=0) + start_date = end_date - timedelta(days=15) + + driver_entities = [1001, 1002, 1003, 1004, 1005] + driver_df = create_driver_hourly_stats_df( + driver_entities, start_date, end_date + ) + driver_stats_path = os.path.join(data_dir, "driver_stats.parquet") + driver_df.to_parquet( + path=driver_stats_path, allow_truncated_timestamps=True + ) + + driver = Entity(name="driver", join_keys=["driver_id"]) + + driver_stats_source = FileSource( + name="driver_hourly_stats_source", + path=driver_stats_path, + timestamp_field="event_timestamp", + created_timestamp_column="created", + ) + + driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=0), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_source, + ) + + @on_demand_feature_view( + sources=[driver_stats_fv[["conv_rate", "acc_rate"]]], + schema=[Field(name="conv_rate_plus_acc", dtype=Float64)], + mode="python", + ) + def test_view(inputs: Dict[str, Any]) -> Dict[str, Any]: + output: Dict[str, Any] = { + "conv_rate_plus_acc": [ + conv_rate + acc_rate + for conv_rate, acc_rate in zip( + inputs["conv_rate"], inputs["acc_rate"] + ) + ] + } + return output + + self.store.apply( + [ + driver, + driver_stats_source, + driver_stats_fv, + test_view, + ] + ) + self.store.write_to_online_store( + feature_view_name="driver_hourly_stats", df=driver_df + ) + # This will give the intuitive structure of the data as: + # {"driver_id": [..], "conv_rate": [..], "acc_rate": [..], "avg_daily_trips": [..]} + driver_dict = driver_df.to_dict(orient="list") + self.store.write_to_online_store( + feature_view_name="driver_hourly_stats", + inputs=driver_dict, + ) + + def test_online_retrieval(self): + entity_rows = [ + { + "driver_id": 1001, + } + ] + + online_python_response = self.store.get_online_features( + entity_rows=entity_rows, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "test_view:conv_rate_plus_acc", + ], + ).to_dict() + + assert len(online_python_response) == 4 + assert all( + key in online_python_response.keys() + for key in [ + "driver_id", + "acc_rate", + "conv_rate", + "conv_rate_plus_acc", + ] + ) From 4b1634f4da7ba47a29dfd4a0d573dfe515a8863d Mon Sep 17 00:00:00 2001 From: lokeshrangineni Date: Tue, 30 Apr 2024 15:16:10 -0400 Subject: [PATCH 29/73] feat: Upgrading python version to 3.11, adding support for 3.11 as well. (#4159) * Upgrading python version to 3.11, adding support for 3.11 as well. Signed-off-by: Lokesh Rangineni * Upgrading python version to 3.11, adding support for 3.11 as well. Signed-off-by: Lokesh Rangineni * chore: Bump macOS runners to macos-13 (#4152) bump macos runner to 13 Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni * chore: Use pixi to lock python dependencies in a single command (#4114) use pixi to lock python dependencies in a single command Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni * Trying to fix the lint error after python upgrade. - error: Call to abstract method "__init__" of "Provider" with trivial body via super() is unsafe [safe-super] Signed-off-by: Lokesh Rangineni * Adding only the integration tests to run on 3.11 Signed-off-by: Lokesh Rangineni --------- Signed-off-by: Lokesh Rangineni Co-authored-by: Tornike Gurgenidze --- .../fork_pr_integration_tests_aws.yml | 12 +- .../fork_pr_integration_tests_gcp.yml | 2 +- .../fork_pr_integration_tests_snowflake.yml | 2 +- .github/workflows/build_wheels.yml | 6 +- .github/workflows/java_master_only.yml | 6 +- .github/workflows/java_pr.yml | 8 +- .github/workflows/linter.yml | 4 +- .github/workflows/master_only.yml | 12 +- .github/workflows/nightly-ci.yml | 18 +- .github/workflows/pr_integration_tests.yml | 12 +- .../workflows/pr_local_integration_tests.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/unit_tests.yml | 2 +- Makefile | 2 +- docs/SUMMARY.md | 3 +- .../running-feast-in-production.md | 21 +- .../batch-materialization/bytewax.md | 99 -- .../docker-compose/feast10/Dockerfile | 2 +- .../feature_servers/gcp_cloudrun/Dockerfile | 2 +- .../feature_servers/multicloud/Dockerfile | 2 +- .../feature_servers/multicloud/Dockerfile.dev | 2 +- .../materialization/kubernetes/Dockerfile | 2 +- .../feast/infra/passthrough_provider.py | 2 - .../infra/transformation_servers/Dockerfile | 2 +- .../requirements/py3.11-ci-requirements.txt | 1047 +++++++++++++++++ .../requirements/py3.11-requirements.txt | 197 ++++ setup.py | 2 +- 27 files changed, 1298 insertions(+), 175 deletions(-) delete mode 100644 docs/reference/batch-materialization/bytewax.md create mode 100644 sdk/python/requirements/py3.11-ci-requirements.txt create mode 100644 sdk/python/requirements/py3.11-requirements.txt diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index 4e418e6b22e..4d6583abcf9 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -33,21 +33,21 @@ jobs: id: image-tag run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - name: Cache Public ECR Image - id: lambda_python_3_9 + id: lambda_python_3_11 uses: actions/cache@v2 with: path: ~/cache - key: lambda_python_3_9 + key: lambda_python_3_11 - name: Handle Cache Miss (pull public ECR image & save it to tar file) if: steps.cache-primes.outputs.cache-hit != 'true' run: | mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.9 - docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + docker pull public.ecr.aws/lambda/python:3.11 + docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - name: Handle Cache Hit (load docker image from tar file) if: steps.cache-primes.outputs.cache-hit == 'true' run: | - docker load -i ~/cache/lambda_python_3_9.tar + docker load -i ~/cache/lambda_python_3_11.tar - name: Build and push env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} @@ -67,7 +67,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index a6fc2110c55..29a053a119f 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml index ee0e256eca6..736b066abee 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 59d274c4941..4c7caa6929c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -59,7 +59,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.11" architecture: x64 - name: Setup Node uses: actions/setup-node@v3 @@ -86,7 +86,7 @@ jobs: id: setup-python uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.11" architecture: x64 - name: Setup Node uses: actions/setup-node@v3 @@ -137,7 +137,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-13 ] - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11"] from-source: [ True, False ] env: # this script is for testing servers diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index cb58b240fb2..d7f8cddfb61 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -25,7 +25,7 @@ jobs: uses: actions/setup-python@v3 id: setup-python with: - python-version: "3.9" + python-version: "3.11" architecture: x64 - name: Authenticate to Google Cloud uses: 'google-github-actions/auth@v1' @@ -95,7 +95,7 @@ jobs: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest env: - PYTHON: 3.9 + PYTHON: 3.11 steps: - uses: actions/checkout@v4 with: @@ -110,7 +110,7 @@ jobs: uses: actions/setup-python@v3 id: setup-python with: - python-version: 3.9 + python-version: 3.11 architecture: x64 - name: Get pip cache dir id: pip-cache diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 8c7a03d2e50..5e94e0ace9f 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -88,7 +88,7 @@ jobs: uses: actions/setup-python@v3 id: setup-python with: - python-version: "3.9" + python-version: "3.11" architecture: x64 - name: Authenticate to Google Cloud uses: 'google-github-actions/auth@v1' @@ -111,7 +111,7 @@ jobs: runs-on: ubuntu-latest needs: unit-test-java env: - PYTHON: 3.9 + PYTHON: 3.11 steps: - uses: actions/checkout@v4 with: @@ -128,7 +128,7 @@ jobs: architecture: x64 - uses: actions/setup-python@v3 with: - python-version: '3.9' + python-version: '3.11' architecture: 'x64' - uses: actions/cache@v2 with: @@ -158,7 +158,7 @@ jobs: uses: actions/setup-python@v3 id: setup-python with: - python-version: 3.9 + python-version: 3.11 architecture: x64 - name: Get pip cache dir id: pip-cache diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 2968a9d6e0d..d2ee734d685 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -6,14 +6,14 @@ jobs: lint-python: runs-on: [ubuntu-latest] env: - PYTHON: 3.9 + PYTHON: 3.11 steps: - uses: actions/checkout@v4 - name: Setup Python id: setup-python uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.11" architecture: x64 - name: Upgrade pip version run: | diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index dd666d5588c..02bd46ab482 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -30,21 +30,21 @@ jobs: id: image-tag run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - name: Cache Public ECR Image - id: lambda_python_3_9 + id: lambda_python_3_11 uses: actions/cache@v2 with: path: ~/cache - key: lambda_python_3_9 + key: lambda_python_3_11 - name: Handle Cache Miss (pull public ECR image & save it to tar file) if: steps.cache-primes.outputs.cache-hit != 'true' run: | mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.9 - docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + docker pull public.ecr.aws/lambda/python:3.11 + docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - name: Handle Cache Hit (load docker image from tar file) if: steps.cache-primes.outputs.cache-hit == 'true' run: | - docker load -i ~/cache/lambda_python_3_9.tar + docker load -i ~/cache/lambda_python_3_11.tar - name: Build and push env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} @@ -65,7 +65,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10" ] + python-version: ["3.9", "3.10", "3.11"] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 9afea5cb6f1..89e4f1f0b90 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -36,7 +36,7 @@ jobs: uses: actions/setup-python@v5 id: setup-python with: - python-version: "3.9" + python-version: "3.11" architecture: x64 - name: Set up AWS SDK uses: aws-actions/configure-aws-credentials@v1 @@ -89,21 +89,21 @@ jobs: id: image-tag run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - name: Cache Public ECR Image - id: lambda_python_3_9 + id: lambda_python_3_11 uses: actions/cache@v4 with: path: ~/cache - key: lambda_python_3_9 + key: lambda_python_3_11 - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.lambda_python_3_9.outputs.cache-hit != 'true' + if: steps.lambda_python_3_11.outputs.cache-hit != 'true' run: | mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.9 - docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + docker pull public.ecr.aws/lambda/python:3.11 + docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - name: Handle Cache Hit (load docker image from tar file) - if: steps.lambda_python_3_9.outputs.cache-hit == 'true' + if: steps.lambda_python_3_11.outputs.cache-hit == 'true' run: | - docker load -i ~/cache/lambda_python_3_9.tar + docker load -i ~/cache/lambda_python_3_11.tar - name: Build and push env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} @@ -124,7 +124,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 5b59429bc71..32eebd5cafa 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -47,21 +47,21 @@ jobs: id: image-tag run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - name: Cache Public ECR Image - id: lambda_python_3_9 + id: lambda_python_3_11 uses: actions/cache@v2 with: path: ~/cache - key: lambda_python_3_9 + key: lambda_python_3_11 - name: Handle Cache Miss (pull public ECR image & save it to tar file) if: steps.cache-primes.outputs.cache-hit != 'true' run: | mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.9 - docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + docker pull public.ecr.aws/lambda/python:3.11 + docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - name: Handle Cache Hit (load docker image from tar file) if: steps.cache-primes.outputs.cache-hit == 'true' run: | - docker load -i ~/cache/lambda_python_3_9.tar + docker load -i ~/cache/lambda_python_3_11.tar - name: Build and push env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} @@ -86,7 +86,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.10" ] + python-version: [ "3.9", "3.10", "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 258f4e42403..cedec5915e7 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.10" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 872a54a80ae..dc3dd1c0e48 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -160,7 +160,7 @@ jobs: architecture: x64 - uses: actions/setup-python@v3 with: - python-version: '3.9' + python-version: '3.11' architecture: 'x64' - uses: actions/cache@v2 with: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 576da4e8f53..b76a6490d4a 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9", "3.10" ] + python-version: [ "3.9", "3.10", "3.11"] os: [ ubuntu-latest, macos-13 ] exclude: - os: macos-13 diff --git a/Makefile b/Makefile index f2ea2cb4d0c..d231da8be10 100644 --- a/Makefile +++ b/Makefile @@ -357,7 +357,7 @@ kill-trino-locally: cd ${ROOT_DIR}; docker stop trino install-protoc-dependencies: - pip install --ignore-installed protobuf==4.23.4 "grpcio-tools>=1.56.2,<2" mypy-protobuf==3.1.0 + pip install --ignore-installed protobuf==4.24.0 "grpcio-tools>=1.56.2,<2" mypy-protobuf==3.1.0 install-feast-ci-locally: pip install -e ".[ci]" diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index b211730d0ef..ec9ce90b2e0 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -106,8 +106,7 @@ * [Google Cloud Platform](reference/providers/google-cloud-platform.md) * [Amazon Web Services](reference/providers/amazon-web-services.md) * [Azure](reference/providers/azure.md) -* [Batch Materialization Engines](reference/batch-materialization/README.md) - * [Bytewax](reference/batch-materialization/bytewax.md) +* [Batch Materialization Engines](reference/batch-materialization/README.md) * [Snowflake](reference/batch-materialization/snowflake.md) * [AWS Lambda (alpha)](reference/batch-materialization/lambda.md) * [Spark (contrib)](reference/batch-materialization/spark.md) diff --git a/docs/how-to-guides/running-feast-in-production.md b/docs/how-to-guides/running-feast-in-production.md index 4663c928c6e..adfb41dab68 100644 --- a/docs/how-to-guides/running-feast-in-production.md +++ b/docs/how-to-guides/running-feast-in-production.md @@ -57,28 +57,9 @@ To keep your online store up to date, you need to run a job that loads feature d Out of the box, Feast's materialization process uses an in-process materialization engine. This engine loads all the data being materialized into memory from the offline store, and writes it into the online store. This approach may not scale to large amounts of data, which users of Feast may be dealing with in production. -In this case, we recommend using one of the more [scalable materialization engines](./scaling-feast.md#scaling-materialization), such as the [Bytewax Materialization Engine](../reference/batch-materialization/bytewax.md), or the [Snowflake Materialization Engine](../reference/batch-materialization/snowflake.md). +In this case, we recommend using one of the more [scalable materialization engines](./scaling-feast.md#scaling-materialization), such as [Snowflake Materialization Engine](../reference/batch-materialization/snowflake.md). Users may also need to [write a custom materialization engine](../how-to-guides/customizing-feast/creating-a-custom-materialization-engine.md) to work on their existing infrastructure. -The Bytewax materialization engine can run materialization on an existing Kubernetes cluster. An example configuration of this in a `feature_store.yaml` is as follows: - -```yaml -batch_engine: - type: bytewax - namespace: bytewax - image: bytewax/bytewax-feast:latest - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: aws-credentials - key: aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: aws-credentials - key: aws-secret-access-key -``` ### 2.2 Scheduled materialization with Airflow diff --git a/docs/reference/batch-materialization/bytewax.md b/docs/reference/batch-materialization/bytewax.md deleted file mode 100644 index 6a97bd391db..00000000000 --- a/docs/reference/batch-materialization/bytewax.md +++ /dev/null @@ -1,99 +0,0 @@ -# Bytewax - -## Description - -The [Bytewax](https://bytewax.io) batch materialization engine provides an execution -engine for batch materializing operations (`materialize` and `materialize-incremental`). - -### Guide - -In order to use the Bytewax materialization engine, you will need a [Kubernetes](https://kubernetes.io/) cluster running version 1.22.10 or greater. - -#### Kubernetes Authentication - -The Bytewax materialization engine loads authentication and cluster information from the [kubeconfig file](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/). By default, kubectl looks for a file named `config` in the `$HOME/.kube directory`. You can specify other kubeconfig files by setting the `KUBECONFIG` environment variable. - -#### Resource Authentication - -Bytewax jobs can be configured to access [Kubernetes secrets](https://kubernetes.io/docs/concepts/configuration/secret/) as environment variables to access online and offline stores during job runs. - -To configure secrets, first create them using `kubectl`: - -``` shell -kubectl create secret generic -n bytewax aws-credentials --from-literal=aws-access-key-id='' --from-literal=aws-secret-access-key='' -``` - -If your Docker registry requires authentication to store/pull containers, you can use this same approach to store your repository access credential and use when running the materialization engine. - -Then configure them in the batch_engine section of `feature_store.yaml`: - -``` yaml -batch_engine: - type: bytewax - namespace: bytewax - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: aws-credentials - key: aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: aws-credentials - key: aws-secret-access-key - image_pull_secrets: - - docker-repository-access-secret -``` - -#### Configuration - -The Bytewax materialization engine is configured through the The `feature_store.yaml` configuration file: - -``` yaml -batch_engine: - type: bytewax - namespace: bytewax - image: bytewax/bytewax-feast:latest - image_pull_secrets: - - my_container_secret - service_account_name: my-k8s-service-account - include_security_context_capabilities: false - annotations: - # example annotation you might include if running on AWS EKS - iam.amazonaws.com/role: arn:aws:iam:::role/MyBytewaxPlatformRole - resources: - limits: - cpu: 1000m - memory: 2048Mi - requests: - cpu: 500m - memory: 1024Mi -``` - -**Notes:** - -* The `namespace` configuration directive specifies which Kubernetes [namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) jobs, services and configuration maps will be created in. -* The `image_pull_secrets` configuration directive specifies the pre-configured secret to use when pulling the image container from your registry. -* The `service_account_name` specifies which Kubernetes service account to run the job under. -* The `include_security_context_capabilities` flag indicates whether or not `"add": ["NET_BIND_SERVICE"]` and `"drop": ["ALL"]` are included in the job & pod security context capabilities. -* `annotations` allows you to include additional Kubernetes annotations to the job. This is particularly useful for IAM roles which grant the running pod access to cloud platform resources (for example). -* The `resources` configuration directive sets the standard Kubernetes [resource requests](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for the job containers to utilise when materializing data. - -#### Building a custom Bytewax Docker image - -The `image` configuration directive specifies which container image to use when running the materialization job. To create a custom image based on this container, run the following command: - -``` shell -DOCKER_BUILDKIT=1 docker build . -f ./sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile -t -``` - -Once that image is built and pushed to a registry, it can be specified as a part of the batch engine configuration: - -``` shell -batch_engine: - type: bytewax - namespace: bytewax - image: -``` - diff --git a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile index 8b3c5b3d3d4..94b3e708ddf 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile +++ b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.11 WORKDIR /app COPY java/serving/src/test/resources/docker-compose/feast10/ . diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile index 6e3ff424eab..6b89d4f73c1 100644 --- a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim +FROM python:3.11-slim RUN apt-get update && apt-get install -y git diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index 5100a2f822c..8a441479184 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.11 RUN apt update && \ apt install -y \ diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index f92d3622a76..948e3569a64 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.11 RUN apt update && \ apt install -y \ diff --git a/sdk/python/feast/infra/materialization/kubernetes/Dockerfile b/sdk/python/feast/infra/materialization/kubernetes/Dockerfile index 956287a1d6a..510bb722851 100644 --- a/sdk/python/feast/infra/materialization/kubernetes/Dockerfile +++ b/sdk/python/feast/infra/materialization/kubernetes/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim-bullseye AS build +FROM python:3.11-slim-bullseye AS build RUN apt-get update && \ apt-get install --no-install-suggests --no-install-recommends --yes git diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index ec4df66d43a..6476acbcb93 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -42,8 +42,6 @@ class PassthroughProvider(Provider): """ def __init__(self, config: RepoConfig): - super().__init__(config) - self.repo_config = config self._offline_store = None self._online_store = None diff --git a/sdk/python/feast/infra/transformation_servers/Dockerfile b/sdk/python/feast/infra/transformation_servers/Dockerfile index 41f272c757c..cd46b0baa90 100644 --- a/sdk/python/feast/infra/transformation_servers/Dockerfile +++ b/sdk/python/feast/infra/transformation_servers/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim +FROM python:3.11-slim RUN apt-get update && apt-get install -y git diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt new file mode 100644 index 00000000000..71f61964be6 --- /dev/null +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -0,0 +1,1047 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --extra=ci --output-file=sdk/python/requirements/py3.11-ci-requirements.txt +# +alabaster==0.7.16 + # via sphinx +altair==4.2.2 + # via great-expectations +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 + # via + # httpx + # jupyter-server + # starlette + # watchfiles +appdirs==1.4.4 + # via fissix +appnope==0.1.4 + # via ipykernel +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration +asn1crypto==1.5.1 + # via snowflake-connector-python +assertpy==1.1 + # via feast (setup.py) +asttokens==2.4.1 + # via stack-data +async-lru==2.0.4 + # via jupyterlab +async-timeout==4.0.3 + # via redis +atpublic==4.1.0 + # via ibis-framework +attrs==23.2.0 + # via + # bowler + # jsonschema + # referencing +azure-core==1.30.1 + # via + # azure-identity + # azure-storage-blob +azure-identity==1.16.0 + # via feast (setup.py) +azure-storage-blob==12.19.1 + # via feast (setup.py) +babel==2.14.0 + # via + # jupyterlab-server + # sphinx +beautifulsoup4==4.12.3 + # via nbconvert +bidict==0.23.1 + # via ibis-framework +bleach==6.1.0 + # via nbconvert +boto3==1.34.93 + # via + # feast (setup.py) + # moto +botocore==1.34.93 + # via + # boto3 + # moto + # s3transfer +bowler==0.9.0 + # via feast (setup.py) +build==1.2.1 + # via + # feast (setup.py) + # pip-tools +cachecontrol==0.14.0 + # via firebase-admin +cachetools==5.3.3 + # via google-auth +cassandra-driver==3.29.1 + # via feast (setup.py) +certifi==2024.2.2 + # via + # httpcore + # httpx + # kubernetes + # minio + # requests + # snowflake-connector-python +cffi==1.16.0 + # via + # argon2-cffi-bindings + # cryptography + # snowflake-connector-python +cfgv==3.4.0 + # via pre-commit +charset-normalizer==3.3.2 + # via + # requests + # snowflake-connector-python +click==8.1.7 + # via + # bowler + # dask + # feast (setup.py) + # geomet + # great-expectations + # moreorless + # pip-tools + # uvicorn +cloudpickle==3.0.0 + # via dask +colorama==0.4.6 + # via + # feast (setup.py) + # great-expectations +comm==0.2.2 + # via + # ipykernel + # ipywidgets +coverage[toml]==7.5.0 + # via pytest-cov +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # feast (setup.py) + # great-expectations + # moto + # msal + # pyjwt + # pyopenssl + # snowflake-connector-python + # types-pyopenssl + # types-redis +dask[array,dataframe]==2024.4.2 + # via + # dask-expr + # feast (setup.py) +dask-expr==1.0.13 + # via dask +db-dtypes==1.2.0 + # via google-cloud-bigquery +debugpy==1.8.1 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +deltalake==0.17.2 + # via feast (setup.py) +dill==0.3.8 + # via feast (setup.py) +distlib==0.3.8 + # via virtualenv +docker==7.0.0 + # via + # feast (setup.py) + # testcontainers +docutils==0.19 + # via sphinx +duckdb==0.10.2 + # via + # duckdb-engine + # ibis-framework +duckdb-engine==0.12.0 + # via ibis-framework +entrypoints==0.4 + # via altair +exceptiongroup==1.2.1 + # via + # anyio + # ipython + # pytest +execnet==2.1.1 + # via pytest-xdist +executing==2.0.1 + # via stack-data +fastapi==0.110.2 + # via feast (setup.py) +fastjsonschema==2.19.1 + # via nbformat +filelock==3.14.0 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 + # via feast (setup.py) +fissix==24.4.24 + # via bowler +fqdn==1.5.1 + # via jsonschema +fsspec==2023.12.2 + # via + # dask + # feast (setup.py) +geojson==2.5.0 + # via rockset +geomet==0.2.1.post1 + # via cassandra-driver +google-api-core[grpc]==2.18.0 + # via + # feast (setup.py) + # firebase-admin + # google-api-python-client + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-core + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-api-python-client==2.127.0 + # via firebase-admin +google-auth==2.29.0 + # via + # google-api-core + # google-api-python-client + # google-auth-httplib2 + # google-cloud-core + # google-cloud-firestore + # google-cloud-storage + # kubernetes +google-auth-httplib2==0.2.0 + # via google-api-python-client +google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) +google-cloud-bigquery-storage==2.24.0 + # via feast (setup.py) +google-cloud-bigtable==2.23.1 + # via feast (setup.py) +google-cloud-core==2.4.1 + # via + # google-cloud-bigquery + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-cloud-datastore==2.19.0 + # via feast (setup.py) +google-cloud-firestore==2.16.0 + # via firebase-admin +google-cloud-storage==2.16.0 + # via + # feast (setup.py) + # firebase-admin +google-crc32c==1.5.0 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.7.0 + # via + # google-cloud-bigquery + # google-cloud-storage +googleapis-common-protos[grpc]==1.63.0 + # via + # feast (setup.py) + # google-api-core + # grpc-google-iam-v1 + # grpcio-status +great-expectations==0.18.12 + # via feast (setup.py) +greenlet==3.0.3 + # via sqlalchemy +grpc-google-iam-v1==0.13.0 + # via google-cloud-bigtable +grpcio==1.62.2 + # via + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools +grpcio-health-checking==1.62.2 + # via feast (setup.py) +grpcio-reflection==1.62.2 + # via feast (setup.py) +grpcio-status==1.62.2 + # via google-api-core +grpcio-testing==1.62.2 + # via feast (setup.py) +grpcio-tools==1.62.2 + # via feast (setup.py) +gunicorn==22.0.0 ; platform_system != "Windows" + # via feast (setup.py) +h11==0.14.0 + # via + # httpcore + # uvicorn +happybase==1.2.0 + # via feast (setup.py) +hazelcast-python-client==5.3.0 + # via feast (setup.py) +hiredis==2.3.2 + # via feast (setup.py) +httpcore==1.0.5 + # via httpx +httplib2==0.22.0 + # via + # google-api-python-client + # google-auth-httplib2 +httptools==0.6.1 + # via uvicorn +httpx==0.27.0 + # via + # feast (setup.py) + # jupyterlab +ibis-framework[duckdb]==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.36 + # via pre-commit +idna==3.7 + # via + # anyio + # httpx + # jsonschema + # requests + # snowflake-connector-python +imagesize==1.4.1 + # via sphinx +importlib-metadata==7.1.0 + # via + # build + # dask + # jupyter-client + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # nbconvert + # sphinx + # typeguard +iniconfig==2.0.0 + # via pytest +ipykernel==6.29.4 + # via jupyterlab +ipython==8.18.1 + # via + # great-expectations + # ipykernel + # ipywidgets +ipywidgets==8.1.2 + # via great-expectations +isodate==0.6.1 + # via azure-storage-blob +isoduration==20.11.0 + # via jsonschema +jedi==0.19.1 + # via ipython +jinja2==3.1.3 + # via + # altair + # feast (setup.py) + # great-expectations + # jupyter-server + # jupyterlab + # jupyterlab-server + # moto + # nbconvert + # sphinx +jmespath==1.0.1 + # via + # boto3 + # botocore +json5==0.9.25 + # via jupyterlab-server +jsonpatch==1.33 + # via great-expectations +jsonpointer==2.4 + # via + # jsonpatch + # jsonschema +jsonschema[format-nongpl]==4.21.1 + # via + # altair + # feast (setup.py) + # great-expectations + # jupyter-events + # jupyterlab-server + # nbformat +jsonschema-specifications==2023.12.1 + # via jsonschema +jupyter-client==8.6.1 + # via + # ipykernel + # jupyter-server + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat +jupyter-events==0.10.0 + # via jupyter-server +jupyter-lsp==2.2.5 + # via jupyterlab +jupyter-server==2.14.0 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.5.3 + # via jupyter-server +jupyterlab==4.1.8 + # via notebook +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.27.1 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.10 + # via ipywidgets +kubernetes==20.13.0 + # via feast (setup.py) +locket==1.0.0 + # via partd +makefun==1.15.2 + # via great-expectations +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # nbconvert + # werkzeug +marshmallow==3.21.1 + # via great-expectations +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mdurl==0.1.2 + # via markdown-it-py +minio==7.1.0 + # via feast (setup.py) +mistune==3.0.2 + # via + # great-expectations + # nbconvert +mmh3==4.1.0 + # via feast (setup.py) +mock==2.0.0 + # via feast (setup.py) +moreorless==0.4.0 + # via bowler +moto==4.2.14 + # via feast (setup.py) +msal==1.28.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.1.0 + # via azure-identity +msgpack==1.0.8 + # via cachecontrol +multipledispatch==1.0.0 + # via ibis-framework +mypy==1.10.0 + # via + # feast (setup.py) + # sqlalchemy +mypy-extensions==1.0.0 + # via mypy +mypy-protobuf==3.3.0 + # via feast (setup.py) +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.4 + # via jupyter-server +nbformat==5.10.4 + # via + # great-expectations + # jupyter-server + # nbclient + # nbconvert +nest-asyncio==1.6.0 + # via ipykernel +nodeenv==1.8.0 + # via pre-commit +notebook==7.1.3 + # via great-expectations +notebook-shim==0.2.4 + # via + # jupyterlab + # notebook +numpy==1.26.4 + # via + # altair + # dask + # db-dtypes + # feast (setup.py) + # great-expectations + # ibis-framework + # pandas + # pyarrow + # scipy +oauthlib==3.2.2 + # via requests-oauthlib +overrides==7.7.0 + # via jupyter-server +packaging==24.0 + # via + # build + # dask + # db-dtypes + # docker + # duckdb-engine + # google-cloud-bigquery + # great-expectations + # gunicorn + # ibis-substrait + # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server + # marshmallow + # msal-extensions + # nbconvert + # pytest + # snowflake-connector-python + # sphinx +pandas==2.2.2 + # via + # altair + # dask + # dask-expr + # db-dtypes + # feast (setup.py) + # google-cloud-bigquery + # great-expectations + # ibis-framework + # snowflake-connector-python +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.4 + # via jedi +parsy==2.1 + # via ibis-framework +partd==1.4.1 + # via dask +pbr==6.0.0 + # via mock +pexpect==4.9.0 + # via ipython +pip-tools==7.4.1 + # via feast (setup.py) +platformdirs==3.11.0 + # via + # jupyter-core + # snowflake-connector-python + # virtualenv +pluggy==1.5.0 + # via pytest +ply==3.11 + # via thriftpy2 +portalocker==2.8.2 + # via msal-extensions +pre-commit==3.3.1 + # via feast (setup.py) +prometheus-client==0.20.0 + # via jupyter-server +prompt-toolkit==3.0.43 + # via ipython +proto-plus==1.23.0 + # via + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore +protobuf==4.25.3 + # via + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools + # mypy-protobuf + # proto-plus + # substrait +psutil==5.9.0 + # via + # feast (setup.py) + # ipykernel +psycopg2-binary==2.9.9 + # via feast (setup.py) +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data +py==1.11.0 + # via feast (setup.py) +py-cpuinfo==9.0.0 + # via pytest-benchmark +py4j==0.10.9.7 + # via pyspark +pyarrow==15.0.2 + # via + # dask-expr + # db-dtypes + # deltalake + # feast (setup.py) + # google-cloud-bigquery + # ibis-framework + # snowflake-connector-python +pyarrow-hotfix==0.6 + # via + # deltalake + # ibis-framework +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pybindgen==0.22.1 + # via feast (setup.py) +pycparser==2.22 + # via cffi +pydantic==2.7.1 + # via + # fastapi + # feast (setup.py) + # great-expectations +pydantic-core==2.18.2 + # via pydantic +pygments==2.17.2 + # via + # feast (setup.py) + # ipython + # nbconvert + # rich + # sphinx +pyjwt[crypto]==2.8.0 + # via + # msal + # snowflake-connector-python +pymssql==2.3.0 + # via feast (setup.py) +pymysql==1.1.0 + # via feast (setup.py) +pyodbc==5.1.0 + # via feast (setup.py) +pyopenssl==24.1.0 + # via snowflake-connector-python +pyparsing==3.1.2 + # via + # great-expectations + # httplib2 +pyproject-hooks==1.1.0 + # via + # build + # pip-tools +pyspark==3.5.1 + # via feast (setup.py) +pytest==7.4.4 + # via + # feast (setup.py) + # pytest-benchmark + # pytest-cov + # pytest-env + # pytest-lazy-fixture + # pytest-mock + # pytest-ordering + # pytest-timeout + # pytest-xdist +pytest-benchmark==3.4.1 + # via feast (setup.py) +pytest-cov==5.0.0 + # via feast (setup.py) +pytest-env==1.1.3 + # via feast (setup.py) +pytest-lazy-fixture==0.6.3 + # via feast (setup.py) +pytest-mock==1.10.4 + # via feast (setup.py) +pytest-ordering==0.6 + # via feast (setup.py) +pytest-timeout==1.4.2 + # via feast (setup.py) +pytest-xdist==3.6.1 + # via feast (setup.py) +python-dateutil==2.9.0.post0 + # via + # arrow + # botocore + # google-cloud-bigquery + # great-expectations + # ibis-framework + # jupyter-client + # kubernetes + # moto + # pandas + # rockset + # trino +python-dotenv==1.0.1 + # via uvicorn +python-json-logger==2.0.7 + # via jupyter-events +pytz==2024.1 + # via + # great-expectations + # ibis-framework + # pandas + # snowflake-connector-python + # trino +pyyaml==6.0.1 + # via + # dask + # feast (setup.py) + # ibis-substrait + # jupyter-events + # kubernetes + # pre-commit + # responses + # uvicorn +pyzmq==26.0.2 + # via + # ipykernel + # jupyter-client + # jupyter-server +redis==4.6.0 + # via feast (setup.py) +referencing==0.35.0 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +regex==2024.4.28 + # via feast (setup.py) +requests==2.31.0 + # via + # azure-core + # cachecontrol + # docker + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # google-cloud-storage + # great-expectations + # jupyterlab-server + # kubernetes + # moto + # msal + # requests-oauthlib + # responses + # snowflake-connector-python + # sphinx + # trino +requests-oauthlib==2.0.0 + # via kubernetes +responses==0.25.0 + # via moto +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events +rich==13.7.1 + # via ibis-framework +rockset==2.1.1 + # via feast (setup.py) +rpds-py==0.18.0 + # via + # jsonschema + # referencing +rsa==4.9 + # via google-auth +ruamel-yaml==0.17.17 + # via great-expectations +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml +ruff==0.4.2 + # via feast (setup.py) +s3transfer==0.10.1 + # via boto3 +scipy==1.13.0 + # via great-expectations +send2trash==1.8.3 + # via jupyter-server +six==1.16.0 + # via + # asttokens + # azure-core + # bleach + # geomet + # happybase + # isodate + # kubernetes + # mock + # python-dateutil + # rfc3339-validator + # thriftpy2 +sniffio==1.3.1 + # via + # anyio + # httpx +snowballstemmer==2.2.0 + # via sphinx +snowflake-connector-python[pandas]==3.9.1 + # via feast (setup.py) +sortedcontainers==2.4.0 + # via snowflake-connector-python +soupsieve==2.5 + # via beautifulsoup4 +sphinx==6.2.1 + # via feast (setup.py) +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx +sqlalchemy[mypy]==2.0.29 + # via + # duckdb-engine + # feast (setup.py) + # ibis-framework + # sqlalchemy + # sqlalchemy-views +sqlalchemy-views==0.3.2 + # via ibis-framework +sqlglot==20.11.0 + # via ibis-framework +stack-data==0.6.3 + # via ipython +starlette==0.37.2 + # via fastapi +substrait==0.17.0 + # via ibis-substrait +tabulate==0.9.0 + # via feast (setup.py) +tenacity==8.2.3 + # via feast (setup.py) +terminado==0.18.1 + # via + # jupyter-server + # jupyter-server-terminals +testcontainers==4.4.0 + # via feast (setup.py) +thriftpy2==0.4.20 + # via happybase +tinycss2==1.3.0 + # via nbconvert +toml==0.10.2 + # via feast (setup.py) +tomli==2.0.1 + # via + # build + # coverage + # jupyterlab + # mypy + # pip-tools + # pytest + # pytest-env +tomlkit==0.12.4 + # via snowflake-connector-python +toolz==0.12.1 + # via + # altair + # dask + # ibis-framework + # partd +tornado==6.4 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado +tqdm==4.66.2 + # via + # feast (setup.py) + # great-expectations +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +trino==0.328.0 + # via feast (setup.py) +typeguard==4.2.1 + # via feast (setup.py) +types-cffi==1.16.0.20240331 + # via types-pyopenssl +types-protobuf==3.19.22 + # via + # feast (setup.py) + # mypy-protobuf +types-pymysql==1.1.0.20240425 + # via feast (setup.py) +types-pyopenssl==24.1.0.20240425 + # via types-redis +types-python-dateutil==2.9.0.20240316 + # via + # arrow + # feast (setup.py) +types-pytz==2024.1.0.20240417 + # via feast (setup.py) +types-pyyaml==6.0.12.20240311 + # via feast (setup.py) +types-redis==4.6.0.20240425 + # via feast (setup.py) +types-requests==2.30.0.0 + # via feast (setup.py) +types-setuptools==69.5.0.20240423 + # via + # feast (setup.py) + # types-cffi +types-tabulate==0.9.0.20240106 + # via feast (setup.py) +types-urllib3==1.26.25.14 + # via types-requests +typing-extensions==4.11.0 + # via + # anyio + # async-lru + # azure-core + # azure-storage-blob + # fastapi + # great-expectations + # ibis-framework + # ipython + # mypy + # pydantic + # pydantic-core + # snowflake-connector-python + # sqlalchemy + # starlette + # testcontainers + # typeguard + # uvicorn +tzdata==2024.1 + # via pandas +tzlocal==5.2 + # via + # great-expectations + # trino +uri-template==1.3.0 + # via jsonschema +uritemplate==4.1.1 + # via google-api-python-client +urllib3==1.26.18 + # via + # botocore + # docker + # feast (setup.py) + # great-expectations + # kubernetes + # minio + # requests + # responses + # rockset + # snowflake-connector-python + # testcontainers +uvicorn[standard]==0.29.0 + # via feast (setup.py) +uvloop==0.19.0 + # via uvicorn +virtualenv==20.23.0 + # via + # feast (setup.py) + # pre-commit +volatile==2.1.0 + # via bowler +watchfiles==0.21.0 + # via uvicorn +wcwidth==0.2.13 + # via prompt-toolkit +webcolors==1.13 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.8.0 + # via + # jupyter-server + # kubernetes +websockets==12.0 + # via uvicorn +werkzeug==3.0.2 + # via moto +wheel==0.43.0 + # via pip-tools +widgetsnbextension==4.0.10 + # via ipywidgets +wrapt==1.16.0 + # via testcontainers +xmltodict==0.13.0 + # via moto +zipp==3.18.1 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt new file mode 100644 index 00000000000..161e435b54e --- /dev/null +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -0,0 +1,197 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=sdk/python/requirements/py3.11-requirements.txt +# +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 + # via + # starlette + # watchfiles +appdirs==1.4.4 + # via fissix +attrs==23.2.0 + # via + # bowler + # jsonschema + # referencing +bowler==0.9.0 + # via feast (setup.py) +certifi==2024.2.2 + # via requests +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # bowler + # dask + # feast (setup.py) + # moreorless + # uvicorn +cloudpickle==3.0.0 + # via dask +colorama==0.4.6 + # via feast (setup.py) +dask[array,dataframe]==2024.4.2 + # via + # dask-expr + # feast (setup.py) +dask-expr==1.0.13 + # via dask +dill==0.3.8 + # via feast (setup.py) +exceptiongroup==1.2.1 + # via anyio +fastapi==0.110.2 + # via feast (setup.py) +fissix==24.4.24 + # via bowler +fsspec==2024.3.1 + # via dask +greenlet==3.0.3 + # via sqlalchemy +gunicorn==22.0.0 ; platform_system != "Windows" + # via feast (setup.py) +h11==0.14.0 + # via uvicorn +httptools==0.6.1 + # via uvicorn +idna==3.7 + # via + # anyio + # requests +importlib-metadata==7.1.0 + # via + # dask + # typeguard +jinja2==3.1.3 + # via feast (setup.py) +jsonschema==4.21.1 + # via feast (setup.py) +jsonschema-specifications==2023.12.1 + # via jsonschema +locket==1.0.0 + # via partd +markupsafe==2.1.5 + # via jinja2 +mmh3==4.1.0 + # via feast (setup.py) +moreorless==0.4.0 + # via bowler +mypy==1.10.0 + # via sqlalchemy +mypy-extensions==1.0.0 + # via mypy +mypy-protobuf==3.6.0 + # via feast (setup.py) +numpy==1.26.4 + # via + # dask + # feast (setup.py) + # pandas + # pyarrow +packaging==24.0 + # via + # dask + # gunicorn +pandas==2.2.2 + # via + # dask + # dask-expr + # feast (setup.py) +partd==1.4.1 + # via dask +protobuf==4.25.3 + # via + # feast (setup.py) + # mypy-protobuf +pyarrow==16.0.0 + # via + # dask-expr + # feast (setup.py) +pydantic==2.7.1 + # via + # fastapi + # feast (setup.py) +pydantic-core==2.18.2 + # via pydantic +pygments==2.17.2 + # via feast (setup.py) +python-dateutil==2.9.0.post0 + # via pandas +python-dotenv==1.0.1 + # via uvicorn +pytz==2024.1 + # via pandas +pyyaml==6.0.1 + # via + # dask + # feast (setup.py) + # uvicorn +referencing==0.35.0 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via feast (setup.py) +rpds-py==0.18.0 + # via + # jsonschema + # referencing +six==1.16.0 + # via python-dateutil +sniffio==1.3.1 + # via anyio +sqlalchemy[mypy]==2.0.29 + # via + # feast (setup.py) + # sqlalchemy +starlette==0.37.2 + # via fastapi +tabulate==0.9.0 + # via feast (setup.py) +tenacity==8.2.3 + # via feast (setup.py) +toml==0.10.2 + # via feast (setup.py) +tomli==2.0.1 + # via mypy +toolz==0.12.1 + # via + # dask + # partd +tqdm==4.66.2 + # via feast (setup.py) +typeguard==4.2.1 + # via feast (setup.py) +types-protobuf==5.26.0.20240422 + # via mypy-protobuf +typing-extensions==4.11.0 + # via + # anyio + # fastapi + # mypy + # pydantic + # pydantic-core + # sqlalchemy + # starlette + # typeguard + # uvicorn +tzdata==2024.1 + # via pandas +urllib3==2.2.1 + # via requests +uvicorn[standard]==0.29.0 + # via feast (setup.py) +uvloop==0.19.0 + # via uvicorn +volatile==2.1.0 + # via bowler +watchfiles==0.21.0 + # via uvicorn +websockets==12.0 + # via uvicorn +zipp==3.18.1 + # via importlib-metadata diff --git a/setup.py b/setup.py index 7cd2c302aef..65d8ee27b5c 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ "fastapi>=0.68.0", "uvicorn[standard]>=0.14.0,<1", "gunicorn; platform_system != 'Windows'", - "dask[dataframe]>=2021.1.0", + "dask[dataframe]>=2024.4.2", "bowler", # Needed for automatic repo upgrades ] From 690a6212e9f2b14fc4bf65513e5d30e70e229d0a Mon Sep 17 00:00:00 2001 From: Pushkar Gupta Date: Tue, 30 Apr 2024 12:28:26 -0700 Subject: [PATCH 30/73] feat: Feast/IKV documenation language changes (#4149) Signed-off-by: Pushkar Gupta --- docs/reference/online-stores/ikv.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/reference/online-stores/ikv.md b/docs/reference/online-stores/ikv.md index ff690c1a622..79f21d17797 100644 --- a/docs/reference/online-stores/ikv.md +++ b/docs/reference/online-stores/ikv.md @@ -2,9 +2,7 @@ ## Description -[IKV](https://github.com/inlinedio/ikv-store) is a fully-managed embedded key-value store, primarily designed for storing ML features. Most key-value stores (think Redis or Cassandra) need a remote database cluster, whereas IKV allows you to utilize your existing application infrastructure to store data (cost efficient) and access it without any network calls (better performance). - -For provisioning API keys for using it as an online-store in Feast, go to [https://inlined.io](https://inlined.io) or email onboarding[at]inlined.io +[IKV](https://github.com/inlinedio/ikv-store) is a fully-managed embedded key-value store, primarily designed for storing ML features. Most key-value stores (think Redis or Cassandra) need a remote database cluster, whereas IKV allows you to utilize your existing application infrastructure to store data (cost efficient) and access it without any network calls (better performance). See detailed performance benchmarks and cost comparison with Redis on [https://inlined.io](https://inlined.io). IKV can be used as an online-store in Feast, the rest of this guide goes over the setup. ## Getting started Make sure you have Python and `pip` installed. @@ -40,9 +38,7 @@ online_store: ``` {% endcode %} -After provision an IKV account/store, you should the required id, passkey and store-name. - -Additionally you must specify a mount-directory - where IKV will pull/update (maintain) a copy of the index for online reads (IKV is an embedded database). It can be skipped only if you don't plan to read any data from this container. The mount directory path usually points to a location on local/remote disk. +After provisioning an IKV account/store, you should have an account id, passkey and store-name. Additionally you must specify a mount-directory - where IKV will pull/update (maintain) a copy of the index for online reads (IKV is an embedded database). It can be skipped only if you don't plan to read any data from this container. The mount directory path usually points to a location on local/remote disk. The full set of configuration options is available in IKVOnlineStoreConfig at `sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py` From 4a696dc4b0fd96d51872a5e629ab5f3ca785d708 Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Wed, 1 May 2024 09:54:32 -0500 Subject: [PATCH 31/73] feat: Add a Kubernetes Operator for the Feast Feature Server (#4145) * add a kubernetes operator for the feast-feature-server Signed-off-by: Tommy Hughes * operator makefile fixes Signed-off-by: Tommy Hughes * addt'l operator documentation Signed-off-by: Tommy Hughes --------- Signed-off-by: Tommy Hughes --- .github/workflows/publish.yml | 2 +- Makefile | 12 + .../running-feast-in-production.md | 2 + infra/charts/feast-feature-server/README.md | 1 + infra/feast-operator/.gitignore | 14 ++ infra/feast-operator/Dockerfile | 7 + infra/feast-operator/Makefile | 231 ++++++++++++++++++ infra/feast-operator/PROJECT | 20 ++ infra/feast-operator/README.md | 39 +++ .../charts.feast.dev_feastfeatureservers.yaml | 44 ++++ .../config/crd/kustomization.yaml | 6 + .../config/default/kustomization.yaml | 20 ++ .../config/manager/kustomization.yaml | 8 + .../config/manager/manager.yaml | 101 ++++++++ .../config/manifests/kustomization.yaml | 7 + .../rbac/feastfeatureserver_editor_role.yaml | 39 +++ ...feastfeatureserver_editor_rolebinding.yaml | 19 ++ .../config/rbac/kustomization.yaml | 13 + .../config/rbac/leader_election_role.yaml | 44 ++++ .../rbac/leader_election_role_binding.yaml | 19 ++ infra/feast-operator/config/rbac/role.yaml | 30 +++ .../config/rbac/role_binding.yaml | 19 ++ .../config/rbac/service_account.yaml | 12 + .../charts_v1alpha1_feastfeatureserver.yaml | 29 +++ .../config/samples/kustomization.yaml | 4 + .../config/scorecard/bases/config.yaml | 7 + .../config/scorecard/kustomization.yaml | 16 ++ .../scorecard/patches/basic.config.yaml | 10 + .../config/scorecard/patches/olm.config.yaml | 50 ++++ .../helm-charts/feast-feature-server | 1 + infra/feast-operator/watches.yaml | 6 + infra/scripts/release/files_to_bump.txt | 2 + 32 files changed, 833 insertions(+), 1 deletion(-) create mode 100644 infra/feast-operator/.gitignore create mode 100644 infra/feast-operator/Dockerfile create mode 100644 infra/feast-operator/Makefile create mode 100644 infra/feast-operator/PROJECT create mode 100644 infra/feast-operator/README.md create mode 100644 infra/feast-operator/config/crd/bases/charts.feast.dev_feastfeatureservers.yaml create mode 100644 infra/feast-operator/config/crd/kustomization.yaml create mode 100644 infra/feast-operator/config/default/kustomization.yaml create mode 100644 infra/feast-operator/config/manager/kustomization.yaml create mode 100644 infra/feast-operator/config/manager/manager.yaml create mode 100644 infra/feast-operator/config/manifests/kustomization.yaml create mode 100644 infra/feast-operator/config/rbac/feastfeatureserver_editor_role.yaml create mode 100644 infra/feast-operator/config/rbac/feastfeatureserver_editor_rolebinding.yaml create mode 100644 infra/feast-operator/config/rbac/kustomization.yaml create mode 100644 infra/feast-operator/config/rbac/leader_election_role.yaml create mode 100644 infra/feast-operator/config/rbac/leader_election_role_binding.yaml create mode 100644 infra/feast-operator/config/rbac/role.yaml create mode 100644 infra/feast-operator/config/rbac/role_binding.yaml create mode 100644 infra/feast-operator/config/rbac/service_account.yaml create mode 100644 infra/feast-operator/config/samples/charts_v1alpha1_feastfeatureserver.yaml create mode 100644 infra/feast-operator/config/samples/kustomization.yaml create mode 100644 infra/feast-operator/config/scorecard/bases/config.yaml create mode 100644 infra/feast-operator/config/scorecard/kustomization.yaml create mode 100644 infra/feast-operator/config/scorecard/patches/basic.config.yaml create mode 100644 infra/feast-operator/config/scorecard/patches/olm.config.yaml create mode 120000 infra/feast-operator/helm-charts/feast-feature-server create mode 100644 infra/feast-operator/watches.yaml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index dc3dd1c0e48..f2b6f4a8a9f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -49,7 +49,7 @@ jobs: needs: [get-version, publish-python-sdk] strategy: matrix: - component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server] + component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server, feast-operator] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: feastdev diff --git a/Makefile b/Makefile index d231da8be10..f0abc930069 100644 --- a/Makefile +++ b/Makefile @@ -401,6 +401,18 @@ build-feature-server-java-docker: -t $(REGISTRY)/feature-server-java:$(VERSION) \ -f java/infra/docker/feature-server/Dockerfile --load . +push-feast-operator-docker: + cd infra/feast-operator && \ + IMAGE_TAG_BASE=$(REGISTRY)/feast-operator \ + VERSION=$(VERSION) \ + $(MAKE) docker-push + +build-feast-operator-docker: + cd infra/feast-operator && \ + IMAGE_TAG_BASE=$(REGISTRY)/feast-operator \ + VERSION=$(VERSION) \ + $(MAKE) docker-build + # Dev images build-feature-server-dev: diff --git a/docs/how-to-guides/running-feast-in-production.md b/docs/how-to-guides/running-feast-in-production.md index adfb41dab68..dc8b87e34f2 100644 --- a/docs/how-to-guides/running-feast-in-production.md +++ b/docs/how-to-guides/running-feast-in-production.md @@ -225,6 +225,8 @@ helm install feast-release feast-charts/feast-feature-server \ This will deploy a single service. The service must have read access to the registry file on cloud storage and to the online store (e.g. via [podAnnotations](https://kubernetes-on-aws.readthedocs.io/en/latest/user-guide/iam-roles.html)). It will keep a copy of the registry in their memory and periodically refresh it, so expect some delays in update propagation in exchange for better performance. +> Alternatively, deploy the same helm chart with a [Kubernetes Operator](/infra/feast-operator). + ## 5. Using environment variables in your yaml configuration You might want to dynamically set parts of your configuration from your environment. For instance to deploy Feast to production and development with the same configuration, but a different server. Or to inject secrets without exposing them in your git repo. To do this, it is possible to use the `${ENV_VAR}` syntax in your `feature_store.yaml` file. For instance: diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index 0730e39e63c..a9c609c3d62 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -17,6 +17,7 @@ A base64 encoded version of the `feature_store.yaml` file is needed. Helm instal ``` helm install feast-feature-server feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml) ``` +> Alternatively, deploy this helm chart with a [Kubernetes Operator](/infra/feast-operator). ## Tutorial See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-demo) for a sample tutorial on testing this helm chart with a demo feature repository and a local Redis instance. diff --git a/infra/feast-operator/.gitignore b/infra/feast-operator/.gitignore new file mode 100644 index 00000000000..62fd3e3995f --- /dev/null +++ b/infra/feast-operator/.gitignore @@ -0,0 +1,14 @@ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +bin + +# editor and IDE paraphernalia +.idea +*.swp +*.swo +*~ diff --git a/infra/feast-operator/Dockerfile b/infra/feast-operator/Dockerfile new file mode 100644 index 00000000000..0aad602c2d2 --- /dev/null +++ b/infra/feast-operator/Dockerfile @@ -0,0 +1,7 @@ +# Build the manager binary +FROM quay.io/operator-framework/helm-operator:v1.34.1 + +ENV HOME=/opt/helm +COPY watches.yaml ${HOME}/watches.yaml +COPY --from=helmcharts feast-feature-server ${HOME}/helm-charts/feast-feature-server +WORKDIR ${HOME} diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile new file mode 100644 index 00000000000..84e69d6eaca --- /dev/null +++ b/infra/feast-operator/Makefile @@ -0,0 +1,231 @@ +# VERSION defines the project version for the bundle. +# Update this value when you upgrade the version of your project. +# To re-generate a bundle for another specific version without changing the standard setup, you can: +# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) +# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) +VERSION ?= 0.37.1 + +# CHANNELS define the bundle channels used in the bundle. +# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") +# To re-generate a bundle for other specific channels without changing the standard setup, you can: +# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) +# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=$(CHANNELS) +endif + +# DEFAULT_CHANNEL defines the default channel used in the bundle. +# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") +# To re-generate a bundle for any other default channel without changing the default setup, you can: +# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) +# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) +endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) + +# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. +# This variable is used to construct full image tags for bundle and catalog images. +# +# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both +# feastdev/feast-operator-bundle:$VERSION and feastdev/feast-operator-catalog:$VERSION. +IMAGE_TAG_BASE ?= feastdev/feast-operator + +# BUNDLE_IMG defines the image:tag used for the bundle. +# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) + +# BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command +BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) + +# USE_IMAGE_DIGESTS defines if images are resolved via tags or digests +# You can enable this value if you would like to use SHA Based Digests +# To enable set flag to true +USE_IMAGE_DIGESTS ?= false +ifeq ($(USE_IMAGE_DIGESTS), true) + BUNDLE_GEN_FLAGS += --use-image-digests +endif + +# Set the Operator SDK version to use. By default, what is installed on the system is used. +# This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. +OPERATOR_SDK_VERSION ?= v1.34.1 + +KUSTOMIZE_VERSION ?= v5.2.1 +HELM_VERSION ?= v1.34.1 +OPM_VERSION ?= v1.23.0 + +# Image URL to use all building/pushing image targets +IMG ?= $(IMAGE_TAG_BASE):$(VERSION) + +.PHONY: all +all: docker-build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Build + +.PHONY: run +run: helm-operator ## Run against the configured Kubernetes cluster in ~/.kube/config + $(HELM_OPERATOR) run + +.PHONY: docker-build +docker-build: ## Build docker image with the manager. + docker build --build-context helmcharts=../charts/ -t ${IMG} . + +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + docker push ${IMG} + +# PLATFORMS defines the target platforms for the manager image be build to provide support to multiple +# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: +# - able to use docker buildx . More info: https://docs.docker.com/build/buildx/ +# - have enable BuildKit, More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +# - be able to push the image for your registry (i.e. if you do not inform a valid value via IMG=> than the export will fail) +# To properly provided solutions that supports more than one platform you should use this option. +PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le +.PHONY: docker-buildx +docker-buildx: ## Build and push docker image for the manager for cross-platform support + - docker buildx create --name project-v3-builder + - docker buildx use project-v3-builder + - docker buildx build --push --platform=$(PLATFORMS) --build-context helmcharts=../charts/ --tag ${IMG} -f Dockerfile . + - docker buildx rm project-v3-builder + +##@ Deployment + +.PHONY: install +install: kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +.PHONY: uninstall +uninstall: kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl delete -f - + +.PHONY: deploy +deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | kubectl apply -f - + +.PHONY: undeploy +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/default | kubectl delete -f - + +OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +ARCH := $(shell uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') + +.PHONY: kustomize +KUSTOMIZE = $(shell pwd)/bin/kustomize +kustomize: ## Download kustomize locally if necessary. +ifeq (,$(wildcard $(KUSTOMIZE))) +ifeq (,$(shell which kustomize 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(KUSTOMIZE)) ;\ + curl -sSLo - https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/$(KUSTOMIZE_VERSION)/kustomize_$(KUSTOMIZE_VERSION)_$(OS)_$(ARCH).tar.gz | \ + tar xzf - -C bin/ ;\ + } +else +KUSTOMIZE = $(shell which kustomize) +endif +endif + +.PHONY: helm-operator +HELM_OPERATOR = $(shell pwd)/bin/helm-operator +helm-operator: ## Download helm-operator locally if necessary, preferring the $(pwd)/bin path over global if both exist. +ifeq (,$(wildcard $(HELM_OPERATOR))) +ifeq (,$(shell which helm-operator 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(HELM_OPERATOR)) ;\ + curl -sSLo $(HELM_OPERATOR) https://github.com/operator-framework/operator-sdk/releases/download/$(HELM_VERSION)/helm-operator_$(OS)_$(ARCH) ;\ + chmod +x $(HELM_OPERATOR) ;\ + } +else +HELM_OPERATOR = $(shell which helm-operator) +endif +endif + +.PHONY: operator-sdk +OPERATOR_SDK ?= $(shell pwd)/bin/operator-sdk +operator-sdk: ## Download operator-sdk locally if necessary. +ifeq (,$(wildcard $(OPERATOR_SDK))) +ifeq (, $(shell which operator-sdk 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPERATOR_SDK)) ;\ + curl -sSLo $(OPERATOR_SDK) https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$(OS)_$(ARCH) ;\ + chmod +x $(OPERATOR_SDK) ;\ + } +else +OPERATOR_SDK = $(shell which operator-sdk) +endif +endif + +.PHONY: bundle +bundle: kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. + $(OPERATOR_SDK) generate kustomize manifests -q + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) + $(OPERATOR_SDK) bundle validate ./bundle + +.PHONY: bundle-build +bundle-build: ## Build the bundle image. + docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . + +.PHONY: bundle-push +bundle-push: ## Push the bundle image. + $(MAKE) docker-push IMG=$(BUNDLE_IMG) + +.PHONY: opm +OPM = $(shell pwd)/bin/opm +opm: ## Download opm locally if necessary. +ifeq (,$(wildcard $(OPM))) +ifeq (,$(shell which opm 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPM)) ;\ + curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/$(OPM_VERSION)/$(OS)-$(ARCH)-opm ;\ + chmod +x $(OPM) ;\ + } +else +OPM = $(shell which opm) +endif +endif + +# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). +# These images MUST exist in a registry and be pull-able. +BUNDLE_IMGS ?= $(BUNDLE_IMG) + +# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) + +# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. +ifneq ($(origin CATALOG_BASE_IMG), undefined) +FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) +endif + +# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. +# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: +# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator +.PHONY: catalog-build +catalog-build: opm ## Build a catalog image. + $(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) + +# Push the catalog image. +.PHONY: catalog-push +catalog-push: ## Push a catalog image. + $(MAKE) docker-push IMG=$(CATALOG_IMG) diff --git a/infra/feast-operator/PROJECT b/infra/feast-operator/PROJECT new file mode 100644 index 00000000000..56b2532d859 --- /dev/null +++ b/infra/feast-operator/PROJECT @@ -0,0 +1,20 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html +domain: feast.dev +layout: +- helm.sdk.operatorframework.io/v1 +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} +projectName: feast-operator +resources: +- api: + crdVersion: v1 + namespaced: true + domain: feast.dev + group: charts + kind: FeastFeatureServer + version: v1alpha1 +version: "3" diff --git a/infra/feast-operator/README.md b/infra/feast-operator/README.md new file mode 100644 index 00000000000..9ebfd5bd66a --- /dev/null +++ b/infra/feast-operator/README.md @@ -0,0 +1,39 @@ +# Feast Feature Server Helm-based Operator + +This Operator was built with the [operator-sdk](https://github.com/operator-framework/operator-sdk) and leverages the [feast-feature-server helm chart](/infra/charts/feast-feature-server). + +## Installation + +1. __Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)__ +2. __Install the Operator on a Kubernetes cluster__ + +```bash +make deploy +``` + +3. __Install a Feast Feature Server on Kubernetes__ + +A base64 encoded version of the `feature_store.yaml` file is required. FeastFeatureServer CR install example: +```bash +cat < To install the aforementioned sample FeastFeatureServer, run this command - `kubectl create -f config/samples/charts_v1alpha1_feastfeatureserver.yaml` diff --git a/infra/feast-operator/config/crd/bases/charts.feast.dev_feastfeatureservers.yaml b/infra/feast-operator/config/crd/bases/charts.feast.dev_feastfeatureservers.yaml new file mode 100644 index 00000000000..8c4c6a1eceb --- /dev/null +++ b/infra/feast-operator/config/crd/bases/charts.feast.dev_feastfeatureservers.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: feastfeatureservers.charts.feast.dev +spec: + group: charts.feast.dev + names: + kind: FeastFeatureServer + listKind: FeastFeatureServerList + plural: feastfeatureservers + singular: feastfeatureserver + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: FeastFeatureServer is the Schema for the feastfeatureservers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of FeastFeatureServer + type: object + x-kubernetes-preserve-unknown-fields: true + status: + description: Status defines the observed state of FeastFeatureServer + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/infra/feast-operator/config/crd/kustomization.yaml b/infra/feast-operator/config/crd/kustomization.yaml new file mode 100644 index 00000000000..bba243307b9 --- /dev/null +++ b/infra/feast-operator/config/crd/kustomization.yaml @@ -0,0 +1,6 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/charts.feast.dev_feastfeatureservers.yaml +#+kubebuilder:scaffold:crdkustomizeresource diff --git a/infra/feast-operator/config/default/kustomization.yaml b/infra/feast-operator/config/default/kustomization.yaml new file mode 100644 index 00000000000..6cd524d5199 --- /dev/null +++ b/infra/feast-operator/config/default/kustomization.yaml @@ -0,0 +1,20 @@ +# Adds namespace to all resources. +namespace: feast-operator-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: feast-operator- + +# Labels to add to all resources and selectors. +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue + +resources: +- ../crd +- ../rbac +- ../manager diff --git a/infra/feast-operator/config/manager/kustomization.yaml b/infra/feast-operator/config/manager/kustomization.yaml new file mode 100644 index 00000000000..be181e33472 --- /dev/null +++ b/infra/feast-operator/config/manager/kustomization.yaml @@ -0,0 +1,8 @@ +resources: +- manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: feastdev/feast-operator + newTag: 0.37.1 diff --git a/infra/feast-operator/config/manager/manager.yaml b/infra/feast-operator/config/manager/manager.yaml new file mode 100644 index 00000000000..d65e8a78902 --- /dev/null +++ b/infra/feast-operator/config/manager/manager.yaml @@ -0,0 +1,101 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: namespace + app.kubernetes.io/instance: system + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: feast-operator + app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/managed-by: kustomize + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager + app.kubernetes.io/name: deployment + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: feast-operator + app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/managed-by: kustomize +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: + # TODO(user): Uncomment the following code to configure the nodeAffinity expression + # according to the platforms which are supported by your solution. + # It is considered best practice to support multiple architectures. You can + # build your manager image using the makefile target docker-buildx. + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/arch + # operator: In + # values: + # - amd64 + # - arm64 + # - ppc64le + # - s390x + # - key: kubernetes.io/os + # operator: In + # values: + # - linux + securityContext: + runAsNonRoot: true + # TODO(user): For common cases that do not require escalating privileges + # it is recommended to ensure that all your Pods/Containers are restrictive. + # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + # Please uncomment the following code if your project does NOT have to work on old Kubernetes + # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). + # seccompProfile: + # type: RuntimeDefault + containers: + - args: + - --leader-elect + - --leader-election-id=feast-operator + image: controller:latest + name: manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/infra/feast-operator/config/manifests/kustomization.yaml b/infra/feast-operator/config/manifests/kustomization.yaml new file mode 100644 index 00000000000..392c30f6b6f --- /dev/null +++ b/infra/feast-operator/config/manifests/kustomization.yaml @@ -0,0 +1,7 @@ +# These resources constitute the fully configured set of manifests +# used to generate the 'manifests/' directory in a bundle. +resources: +- bases/feast-operator.clusterserviceversion.yaml +- ../default +- ../samples +- ../scorecard diff --git a/infra/feast-operator/config/rbac/feastfeatureserver_editor_role.yaml b/infra/feast-operator/config/rbac/feastfeatureserver_editor_role.yaml new file mode 100644 index 00000000000..f03ac20fddc --- /dev/null +++ b/infra/feast-operator/config/rbac/feastfeatureserver_editor_role.yaml @@ -0,0 +1,39 @@ +# permissions for end users to edit feastfeatureservers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: feastfeatureserver-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: feast-operator + app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/managed-by: kustomize + name: feastfeatureserver-editor-role +rules: +- apiGroups: + - charts.feast.dev + resources: + - feastfeatureservers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - charts.feast.dev + resources: + - feastfeatureservers/finalizers + verbs: + - update +- apiGroups: + - charts.feast.dev + resources: + - feastfeatureservers/status + verbs: + - get + - patch + - update diff --git a/infra/feast-operator/config/rbac/feastfeatureserver_editor_rolebinding.yaml b/infra/feast-operator/config/rbac/feastfeatureserver_editor_rolebinding.yaml new file mode 100644 index 00000000000..054eb5a1a20 --- /dev/null +++ b/infra/feast-operator/config/rbac/feastfeatureserver_editor_rolebinding.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: feastfeatureserver-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: feast-operator + app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/managed-by: kustomize + name: feastfeatureserver-editor-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: feastfeatureserver-editor-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/infra/feast-operator/config/rbac/kustomization.yaml b/infra/feast-operator/config/rbac/kustomization.yaml new file mode 100644 index 00000000000..05916243907 --- /dev/null +++ b/infra/feast-operator/config/rbac/kustomization.yaml @@ -0,0 +1,13 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +- feastfeatureserver_editor_role.yaml +- feastfeatureserver_editor_rolebinding.yaml diff --git a/infra/feast-operator/config/rbac/leader_election_role.yaml b/infra/feast-operator/config/rbac/leader_election_role.yaml new file mode 100644 index 00000000000..0adc316dd39 --- /dev/null +++ b/infra/feast-operator/config/rbac/leader_election_role.yaml @@ -0,0 +1,44 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: role + app.kubernetes.io/instance: leader-election-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: feast-operator + app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/managed-by: kustomize + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/infra/feast-operator/config/rbac/leader_election_role_binding.yaml b/infra/feast-operator/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 00000000000..f745675c0e7 --- /dev/null +++ b/infra/feast-operator/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: rolebinding + app.kubernetes.io/instance: leader-election-rolebinding + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: feast-operator + app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/managed-by: kustomize + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/infra/feast-operator/config/rbac/role.yaml b/infra/feast-operator/config/rbac/role.yaml new file mode 100644 index 00000000000..2469689484e --- /dev/null +++ b/infra/feast-operator/config/rbac/role.yaml @@ -0,0 +1,30 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: manager-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: feast-operator + app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/managed-by: kustomize + name: manager-role +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +- apiGroups: + - "" + - apps + resources: + - deployments + - secrets + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/infra/feast-operator/config/rbac/role_binding.yaml b/infra/feast-operator/config/rbac/role_binding.yaml new file mode 100644 index 00000000000..3359e911695 --- /dev/null +++ b/infra/feast-operator/config/rbac/role_binding.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/instance: manager-rolebinding + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: feast-operator + app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/managed-by: kustomize + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/infra/feast-operator/config/rbac/service_account.yaml b/infra/feast-operator/config/rbac/service_account.yaml new file mode 100644 index 00000000000..7ba6f27c603 --- /dev/null +++ b/infra/feast-operator/config/rbac/service_account.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: serviceaccount + app.kubernetes.io/instance: controller-manager-sa + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: feast-operator + app.kubernetes.io/part-of: feast-operator + app.kubernetes.io/managed-by: kustomize + name: controller-manager + namespace: system diff --git a/infra/feast-operator/config/samples/charts_v1alpha1_feastfeatureserver.yaml b/infra/feast-operator/config/samples/charts_v1alpha1_feastfeatureserver.yaml new file mode 100644 index 00000000000..44b8b61af56 --- /dev/null +++ b/infra/feast-operator/config/samples/charts_v1alpha1_feastfeatureserver.yaml @@ -0,0 +1,29 @@ +apiVersion: charts.feast.dev/v1alpha1 +kind: FeastFeatureServer +metadata: + name: feastfeatureserver-sample +spec: + # Default values copied from helm-charts/feast-feature-server/values.yaml + affinity: {} + # base64 encoding of `sdk/python/feast/templates/local/feature_repo/feature_store.yaml` + feature_store_yaml_base64: "cHJvamVjdDogbXlfcHJvamVjdAojIEJ5IGRlZmF1bHQsIHRoZSByZWdpc3RyeSBpcyBhIGZpbGUgKGJ1dCBjYW4gYmUgdHVybmVkIGludG8gYSBtb3JlIHNjYWxhYmxlIFNRTC1iYWNrZWQgcmVnaXN0cnkpCnJlZ2lzdHJ5OiBkYXRhL3JlZ2lzdHJ5LmRiCiMgVGhlIHByb3ZpZGVyIHByaW1hcmlseSBzcGVjaWZpZXMgZGVmYXVsdCBvZmZsaW5lIC8gb25saW5lIHN0b3JlcyAmIHN0b3JpbmcgdGhlIHJlZ2lzdHJ5IGluIGEgZ2l2ZW4gY2xvdWQKcHJvdmlkZXI6IGxvY2FsCm9ubGluZV9zdG9yZToKICAgIHR5cGU6IHNxbGl0ZQogICAgcGF0aDogZGF0YS9vbmxpbmVfc3RvcmUuZGIKZW50aXR5X2tleV9zZXJpYWxpemF0aW9uX3ZlcnNpb246IDIK" + fullnameOverride: "" + image: {} + imagePullSecrets: [] + livenessProbe: + initialDelaySeconds: 30 + periodSeconds: 30 + nameOverride: "" + nodeSelector: {} + podAnnotations: {} + podSecurityContext: {} + readinessProbe: + initialDelaySeconds: 20 + periodSeconds: 10 + replicaCount: 1 + resources: {} + securityContext: {} + service: + port: 80 + type: ClusterIP + tolerations: [] diff --git a/infra/feast-operator/config/samples/kustomization.yaml b/infra/feast-operator/config/samples/kustomization.yaml new file mode 100644 index 00000000000..8a8cf497ead --- /dev/null +++ b/infra/feast-operator/config/samples/kustomization.yaml @@ -0,0 +1,4 @@ +## Append samples of your project ## +resources: +- charts_v1alpha1_feastfeatureserver.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/infra/feast-operator/config/scorecard/bases/config.yaml b/infra/feast-operator/config/scorecard/bases/config.yaml new file mode 100644 index 00000000000..c77047841ed --- /dev/null +++ b/infra/feast-operator/config/scorecard/bases/config.yaml @@ -0,0 +1,7 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: [] diff --git a/infra/feast-operator/config/scorecard/kustomization.yaml b/infra/feast-operator/config/scorecard/kustomization.yaml new file mode 100644 index 00000000000..50cd2d084eb --- /dev/null +++ b/infra/feast-operator/config/scorecard/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- bases/config.yaml +patchesJson6902: +- path: patches/basic.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +- path: patches/olm.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +#+kubebuilder:scaffold:patchesJson6902 diff --git a/infra/feast-operator/config/scorecard/patches/basic.config.yaml b/infra/feast-operator/config/scorecard/patches/basic.config.yaml new file mode 100644 index 00000000000..78ad61a41bd --- /dev/null +++ b/infra/feast-operator/config/scorecard/patches/basic.config.yaml @@ -0,0 +1,10 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.34.1 + labels: + suite: basic + test: basic-check-spec-test diff --git a/infra/feast-operator/config/scorecard/patches/olm.config.yaml b/infra/feast-operator/config/scorecard/patches/olm.config.yaml new file mode 100644 index 00000000000..69dda63f2eb --- /dev/null +++ b/infra/feast-operator/config/scorecard/patches/olm.config.yaml @@ -0,0 +1,50 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.34.1 + labels: + suite: olm + test: olm-bundle-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.34.1 + labels: + suite: olm + test: olm-crds-have-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.34.1 + labels: + suite: olm + test: olm-crds-have-resources-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.34.1 + labels: + suite: olm + test: olm-spec-descriptors-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.34.1 + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/infra/feast-operator/helm-charts/feast-feature-server b/infra/feast-operator/helm-charts/feast-feature-server new file mode 120000 index 00000000000..e432d2cba69 --- /dev/null +++ b/infra/feast-operator/helm-charts/feast-feature-server @@ -0,0 +1 @@ +../../charts/feast-feature-server \ No newline at end of file diff --git a/infra/feast-operator/watches.yaml b/infra/feast-operator/watches.yaml new file mode 100644 index 00000000000..bb400cb90d6 --- /dev/null +++ b/infra/feast-operator/watches.yaml @@ -0,0 +1,6 @@ +# Use the 'create api' subcommand to add watches to this file. +- group: charts.feast.dev + version: v1alpha1 + kind: FeastFeatureServer + chart: helm-charts/feast-feature-server +#+kubebuilder:scaffold:watch diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt index 61a70ac6b3c..505ef87b243 100644 --- a/infra/scripts/release/files_to_bump.txt +++ b/infra/scripts/release/files_to_bump.txt @@ -10,5 +10,7 @@ infra/charts/feast/README.md 11 68 69 infra/charts/feast-feature-server/Chart.yaml 5 infra/charts/feast-feature-server/README.md 3 infra/charts/feast-feature-server/values.yaml 12 +infra/feast-operator/Makefile 6 +infra/feast-operator/config/manager/kustomization.yaml 8 java/pom.xml 38 ui/package.json 3 From dedc1645ef1f38aa9b50a0cf55e4bc23ec60d5ad Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Wed, 1 May 2024 10:35:09 -0500 Subject: [PATCH 32/73] fix: Helm chart `feast-feature-server`, improve Service template name (#4161) fix: feast-feature-server helm chart's Service template Signed-off-by: Tommy Hughes --- examples/python-helm-demo/README.md | 4 ++-- infra/charts/feast-feature-server/templates/service.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/python-helm-demo/README.md b/examples/python-helm-demo/README.md index 44cd4799d56..90469e746d4 100644 --- a/examples/python-helm-demo/README.md +++ b/examples/python-helm-demo/README.md @@ -72,11 +72,11 @@ We use the Feast CLI to register and materialize features, and then retrieving v 3. `helm install feast-release ../../../infra/charts/feast-feature-server --set image.tag=dev --set feature_store_yaml_base64=$(base64 feature_store.yaml)` 5. (Optional): check logs of the server to make sure it’s working ```bash - kubectl logs svc/feast-feature-server + kubectl logs svc/feast-release-feast-feature-server ``` 6. Port forward to expose the grpc endpoint: ```bash - kubectl port-forward svc/feast-feature-server 6566:80 + kubectl port-forward svc/feast-release-feast-feature-server 6566:80 ``` 7. Run test fetches for online features:8. - First: change back the Redis connection string to allow localhost connections to Redis diff --git a/infra/charts/feast-feature-server/templates/service.yaml b/infra/charts/feast-feature-server/templates/service.yaml index d6914828e49..db0ac8b10b8 100644 --- a/infra/charts/feast-feature-server/templates/service.yaml +++ b/infra/charts/feast-feature-server/templates/service.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "feast-feature-server.name" . }} + name: {{ include "feast-feature-server.fullname" . }} labels: {{- include "feast-feature-server.labels" . | nindent 4 }} spec: From c86d594613b0fb1425451def4fc1d7a7496eea92 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Wed, 1 May 2024 21:32:14 +0400 Subject: [PATCH 33/73] fix: Correct nightly install command, move all installs to uv (#4164) * fix uv install, upgrade setup-python Signed-off-by: tokoko * try installing uv w/o pip Signed-off-by: tokoko * try installing uv w/o pip Signed-off-by: tokoko * try installing uv w/o pip Signed-off-by: tokoko --------- Signed-off-by: tokoko --- .../fork_pr_integration_tests_aws.yml | 31 ++++++---------- .../fork_pr_integration_tests_gcp.yml | 31 ++++++---------- .../fork_pr_integration_tests_snowflake.yml | 31 ++++++---------- .github/workflows/build_wheels.yml | 6 +-- .github/workflows/java_master_only.yml | 35 +++++++----------- .github/workflows/java_pr.yml | 37 ++++++++----------- .github/workflows/linter.yml | 5 +-- .github/workflows/master_only.yml | 7 +--- .github/workflows/nightly-ci.yml | 7 +--- .github/workflows/pr_integration_tests.yml | 5 +-- .../workflows/pr_local_integration_tests.yml | 5 +-- .github/workflows/publish.yml | 2 +- .github/workflows/unit_tests.yml | 5 +-- 13 files changed, 77 insertions(+), 130 deletions(-) diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index 4d6583abcf9..aa89ece1776 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -91,7 +91,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} @@ -109,25 +109,18 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Get pip cache dir - id: pip-cache + - name: Install uv run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - - name: Upgrade pip version + curl -LsSf https://astral.sh/uv/install.sh | sh + - name: Get uv cache dir + id: uv-cache run: | - pip install --upgrade "pip>=21.3.1,<22.3" - - name: Install pip-tools - run: pip install pip-tools + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install apache-arrow on ubuntu if: matrix.os == 'ubuntu-latest' run: | @@ -138,7 +131,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index 29a053a119f..be9844a7e93 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -33,7 +33,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} @@ -53,25 +53,18 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI run: gcloud info - - name: Get pip cache dir - id: pip-cache + - name: Install uv run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - - name: Upgrade pip version + curl -LsSf https://astral.sh/uv/install.sh | sh + - name: Get uv cache dir + id: uv-cache run: | - pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install apache-arrow on ubuntu if: matrix.os == 'ubuntu-latest' run: | @@ -82,7 +75,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest diff --git a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml index 736b066abee..a136b47b9e7 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml @@ -33,7 +33,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} @@ -43,25 +43,18 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.18.0 - - name: Get pip cache dir - id: pip-cache + - name: Install uv run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - - name: Upgrade pip version + curl -LsSf https://astral.sh/uv/install.sh | sh + - name: Get uv cache dir + id: uv-cache run: | - pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install apache-arrow on ubuntu if: matrix.os == 'ubuntu-latest' run: | @@ -72,7 +65,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 4c7caa6929c..596eef2b52c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -57,7 +57,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: "3.11" architecture: x64 @@ -84,7 +84,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup Python id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: "3.11" architecture: x64 @@ -156,7 +156,7 @@ jobs: steps: - name: Setup Python id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: x64 diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index d7f8cddfb61..95ebfe958e2 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -22,7 +22,7 @@ jobs: with: submodules: 'true' - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: "3.11" @@ -107,33 +107,26 @@ jobs: java-package: jdk architecture: x64 - name: Setup Python (to call feast apply) - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: 3.11 architecture: x64 - - name: Get pip cache dir - id: pip-cache + - name: Install uv run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - - name: Upgrade pip version + curl -LsSf https://astral.sh/uv/install.sh | sh + - name: Get uv cache dir + id: uv-cache run: | - pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install Python dependencies - run: make install-python-ci-dependencies - - uses: actions/cache@v2 + run: make install-python-ci-dependencies-uv + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 5e94e0ace9f..fa373fea23c 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -85,7 +85,7 @@ jobs: with: submodules: 'true' - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: "3.11" @@ -126,7 +126,7 @@ jobs: java-version: '11' java-package: jdk architecture: x64 - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v5 with: python-version: '3.11' architecture: 'x64' @@ -155,32 +155,25 @@ jobs: - name: Use AWS CLI run: aws sts get-caller-identity - name: Setup Python (to call feast apply) - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: 3.11 architecture: x64 - - name: Get pip cache dir - id: pip-cache + - name: Install uv run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - - name: Upgrade pip version + curl -LsSf https://astral.sh/uv/install.sh | sh + - name: Get uv cache dir + id: uv-cache run: | - pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools - - name: Install Python dependencies - run: make install-python-ci-dependencies + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} + - name: Install dependencies + run: make install-python-ci-dependencies-uv - name: Run integration tests run: make test-java-integration - name: Save report diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index d2ee734d685..ded9931737a 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -15,11 +15,8 @@ jobs: with: python-version: "3.11" architecture: x64 - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Install uv - run: pip install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Install dependencies run: | make install-python-ci-dependencies-uv diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 02bd46ab482..295e7b17e23 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -84,7 +84,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup Python id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: x64 @@ -106,11 +106,8 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Install uv - run: pip install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Get uv cache dir id: uv-cache run: | diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 89e4f1f0b90..8a6ed2d7a73 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -173,11 +173,8 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Install uv - run: pip install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Get uv cache dir id: uv-cache run: | @@ -200,7 +197,7 @@ jobs: if: matrix.os == 'macos-13' run: brew install apache-arrow - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 32eebd5cafa..aede0da23da 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -133,11 +133,8 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Install uv - run: pip install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Get uv cache dir id: uv-cache run: | diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index cedec5915e7..3de72621931 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -38,11 +38,8 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Install uv - run: pip install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Get uv cache dir id: uv-cache run: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f2b6f4a8a9f..914e5a233c7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -158,7 +158,7 @@ jobs: java-version: '11' java-package: jdk architecture: x64 - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v5 with: python-version: '3.11' architecture: 'x64' diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index b76a6490d4a..dea82da44c9 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -23,12 +23,9 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Install uv run: | - pip install uv + curl -LsSf https://astral.sh/uv/install.sh | sh - name: Get uv cache dir id: uv-cache run: | From 9015120f46644fa199857e8e3ab552cc2f8b005a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 23:13:52 -0500 Subject: [PATCH 34/73] chore: Bump ejs from 3.1.7 to 3.1.10 in /sdk/python/feast/ui (#4167) Bumps [ejs](https://github.com/mde/ejs) from 3.1.7 to 3.1.10. - [Release notes](https://github.com/mde/ejs/releases) - [Commits](https://github.com/mde/ejs/compare/v3.1.7...v3.1.10) --- updated-dependencies: - dependency-name: ejs dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/feast/ui/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 5d0101fbe52..91197e5219d 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -4812,9 +4812,9 @@ ee-first@1.1.1: integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= ejs@^3.1.6: - version "3.1.7" - resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.7.tgz#c544d9c7f715783dd92f0bddcf73a59e6962d006" - integrity sha512-BIar7R6abbUxDA3bfXrO4DSgwo8I+fB5/1zgujl3HLLjwd6+9iOnrT+t3grn2qbk9vOgBubXOFwX2m9axoFaGw== + version "3.1.10" + resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.10.tgz#69ab8358b14e896f80cc39e62087b88500c3ac3b" + integrity sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA== dependencies: jake "^10.8.5" From 0e0fad1ce17a9704679c4bf96b94e3f5f578425f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 08:28:10 -0500 Subject: [PATCH 35/73] chore: Bump ejs from 3.1.7 to 3.1.10 in /ui (#4168) Bumps [ejs](https://github.com/mde/ejs) from 3.1.7 to 3.1.10. - [Release notes](https://github.com/mde/ejs/releases) - [Commits](https://github.com/mde/ejs/compare/v3.1.7...v3.1.10) --- updated-dependencies: - dependency-name: ejs dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/yarn.lock b/ui/yarn.lock index 0bfdb63184e..9a4338a319b 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -4986,9 +4986,9 @@ ee-first@1.1.1: integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== ejs@^3.1.6: - version "3.1.7" - resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.7.tgz#c544d9c7f715783dd92f0bddcf73a59e6962d006" - integrity sha512-BIar7R6abbUxDA3bfXrO4DSgwo8I+fB5/1zgujl3HLLjwd6+9iOnrT+t3grn2qbk9vOgBubXOFwX2m9axoFaGw== + version "3.1.10" + resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.10.tgz#69ab8358b14e896f80cc39e62087b88500c3ac3b" + integrity sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA== dependencies: jake "^10.8.5" From b0c5ffc0e3c997385e7ea4c04985f1e02a02052b Mon Sep 17 00:00:00 2001 From: Jeremy Ary Date: Thu, 2 May 2024 08:31:49 -0500 Subject: [PATCH 36/73] chore: Limit integration runs back to a single python version due to rate limiting errors (#4169) Signed-off-by: Jeremy Ary --- .github/workflows/pr_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index aede0da23da..3081d418fcf 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -86,7 +86,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9", "3.10", "3.11" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} From b8087f7a181977e0e4d3bd29c857d8e137af1de2 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Thu, 2 May 2024 18:37:24 +0400 Subject: [PATCH 37/73] fix: Pass region to S3 client only if set (Java) (#4151) * pass region to s3 client only if set Signed-off-by: tokoko * java ci changes Signed-off-by: tokoko --------- Signed-off-by: tokoko --- .github/workflows/java_master_only.yml | 1 + .../serving/service/config/RegistryConfigModule.java | 11 ++++++++--- .../test/resources/docker-compose/feast10/Dockerfile | 7 +++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 95ebfe958e2..2775f500f32 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -124,6 +124,7 @@ jobs: with: path: ${{ steps.uv-cache.outputs.dir }} key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} + - name: Install Python dependencies run: make install-python-ci-dependencies-uv - uses: actions/cache@v4 diff --git a/java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java b/java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java index 5ab951c71cb..6a9c03956c4 100644 --- a/java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java +++ b/java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java @@ -41,9 +41,14 @@ Storage googleStorage(ApplicationProperties applicationProperties) { @Provides public AmazonS3 awsStorage(ApplicationProperties applicationProperties) { - return AmazonS3ClientBuilder.standard() - .withRegion(applicationProperties.getFeast().getAwsRegion()) - .build(); + AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); + String region = applicationProperties.getFeast().getAwsRegion(); + + if (region != null) { + builder = builder.withRegion(region); + } + + return builder.build(); } @Provides diff --git a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile index 94b3e708ddf..09a8d23faef 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile +++ b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile @@ -1,13 +1,16 @@ FROM python:3.11 WORKDIR /app -COPY java/serving/src/test/resources/docker-compose/feast10/ . COPY sdk/python /mnt/feast/sdk/python COPY protos /mnt/feast/protos COPY setup.py /mnt/feast/setup.py COPY pyproject.toml /mnt/feast/pyproject.toml COPY README.md /mnt/feast/README.md -RUN cd /mnt/feast && SETUPTOOLS_SCM_PRETEND_VERSION="0.1.0" pip install .[grpcio,redis] +COPY Makefile /mnt/feast/Makefile +ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.1.0 +RUN pip install uv +RUN cd /mnt/feast && uv pip install --system .[grpcio,redis] +COPY java/serving/src/test/resources/docker-compose/feast10/ . EXPOSE 8080 CMD ["./entrypoint.sh"] From 60756cb4637a7961b6caffef3242e2886e77f78a Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Thu, 2 May 2024 18:41:26 +0400 Subject: [PATCH 38/73] fix: Pass native input values to `get_online_features` from feature server (#4117) * fix: Pass native input values to get_online_features from feature server Signed-off-by: tokoko * remove unnecessary type ignore hint Signed-off-by: tokoko --------- Signed-off-by: tokoko --- sdk/python/feast/feature_server.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index 4b0e50a06da..fda8745c2d2 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -10,7 +10,7 @@ from fastapi import FastAPI, HTTPException, Request, Response, status from fastapi.logger import logger from fastapi.params import Depends -from google.protobuf.json_format import MessageToDict, Parse +from google.protobuf.json_format import MessageToDict from pydantic import BaseModel import feast @@ -18,7 +18,6 @@ from feast.constants import DEFAULT_FEATURE_SERVER_REGISTRY_TTL from feast.data_source import PushMode from feast.errors import PushSourceNotFoundException -from feast.protos.feast.serving.ServingService_pb2 import GetOnlineFeaturesRequest # TODO: deprecate this in favor of push features @@ -83,34 +82,25 @@ def shutdown_event(): @app.post("/get-online-features") def get_online_features(body=Depends(get_body)): try: - # Validate and parse the request data into GetOnlineFeaturesRequest Protobuf object - request_proto = GetOnlineFeaturesRequest() - Parse(body, request_proto) - + body = json.loads(body) # Initialize parameters for FeatureStore.get_online_features(...) call - if request_proto.HasField("feature_service"): + if "feature_service" in body: features = store.get_feature_service( - request_proto.feature_service, allow_cache=True + body["feature_service"], allow_cache=True ) else: - features = list(request_proto.features.val) - - full_feature_names = request_proto.full_feature_names + features = body["features"] - batch_sizes = [len(v.val) for v in request_proto.entities.values()] - num_entities = batch_sizes[0] - if any(batch_size != num_entities for batch_size in batch_sizes): - raise HTTPException(status_code=500, detail="Uneven number of columns") + full_feature_names = body.get("full_feature_names", False) response_proto = store._get_online_features( features=features, - entity_values=request_proto.entities, + entity_values=body["entities"], full_feature_names=full_feature_names, - native_entity_values=False, ).proto # Convert the Protobuf object to JSON and return it - return MessageToDict( # type: ignore + return MessageToDict( response_proto, preserving_proto_field_name=True, float_precision=18 ) except Exception as e: From ee51fbfe3f40e637084c5ecf51f7463b840d58b4 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Thu, 2 May 2024 18:53:43 +0400 Subject: [PATCH 39/73] chore: Remove repo-upgrade cli command (#4124) remove repo-upgrade cli command Signed-off-by: tokoko --- docs/SUMMARY.md | 1 - docs/how-to-guides/automated-feast-upgrade.md | 78 -------- sdk/python/feast/cli.py | 22 --- sdk/python/feast/repo_upgrade.py | 175 ------------------ .../requirements/py3.10-ci-requirements.txt | 13 -- .../requirements/py3.10-requirements.txt | 13 -- .../requirements/py3.9-ci-requirements.txt | 13 -- .../requirements/py3.9-requirements.txt | 13 -- setup.py | 1 - 9 files changed, 329 deletions(-) delete mode 100644 docs/how-to-guides/automated-feast-upgrade.md delete mode 100644 sdk/python/feast/repo_upgrade.py diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index ec9ce90b2e0..3673edf6cf7 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -50,7 +50,6 @@ * [Scaling Feast](how-to-guides/scaling-feast.md) * [Structuring Feature Repos](how-to-guides/structuring-repos.md) * [Running Feast in production (e.g. on Kubernetes)](how-to-guides/running-feast-in-production.md) -* [Upgrading for Feast 0.20+](how-to-guides/automated-feast-upgrade.md) * [Customizing Feast](how-to-guides/customizing-feast/README.md) * [Adding a custom batch materialization engine](how-to-guides/customizing-feast/creating-a-custom-materialization-engine.md) * [Adding a new offline store](how-to-guides/customizing-feast/adding-a-new-offline-store.md) diff --git a/docs/how-to-guides/automated-feast-upgrade.md b/docs/how-to-guides/automated-feast-upgrade.md deleted file mode 100644 index 89277fb615f..00000000000 --- a/docs/how-to-guides/automated-feast-upgrade.md +++ /dev/null @@ -1,78 +0,0 @@ -# Automated upgrades for Feast 0.20+ - -## Overview - -Starting with Feast 0.20, the APIs of many core objects (e.g. feature views and entities) have been changed. -For example, many parameters have been renamed. -These changes were made in a backwards-compatible fashion; existing Feast repositories will continue to work until Feast 0.23, without any changes required. -However, Feast 0.24 will fully deprecate all of the old parameters, so in order to use Feast 0.24+ users must modify their Feast repositories. - -There are currently deprecation warnings that indicate to users exactly how to modify their repos. -In order to make the process somewhat easier, Feast 0.23 also introduces a new CLI command, `repo-upgrade`, that will partially automate the process of upgrading Feast repositories. - -The upgrade command aims to automatically modify the object definitions in a feature repo to match the API required by Feast 0.24+. When running the command, the Feast CLI analyzes the source code in the feature repo files using [bowler](https://pybowler.io/), and attempted to rewrite the files in a best-effort way. It's possible for there to be parts of the API that are not upgraded automatically. - -The `repo-upgrade` command is specifically meant for upgrading Feast repositories that were initially created in versions 0.23 and below to be compatible with versions 0.24 and above. -It is not intended to work for any future upgrades. - -## Usage - -At the root of a feature repo, you can run `feast repo-upgrade`. By default, the CLI only echos the changes it's planning on making, and does not modify any files in place. If the changes look reasonably, you can specify the `--write` flag to have the changes be written out to disk. - -An example: -```bash -$ feast repo-upgrade --write ---- /Users/achal/feast/prompt_dory/example.py -+++ /Users/achal/feast/prompt_dory/example.py -@@ -13,7 +13,6 @@ - path="/Users/achal/feast/prompt_dory/data/driver_stats.parquet", - event_timestamp_column="event_timestamp", - created_timestamp_column="created", -- date_partition_column="created" - ) - - # Define an entity for the driver. You can think of entity as a primary key used to ---- /Users/achal/feast/prompt_dory/example.py -+++ /Users/achal/feast/prompt_dory/example.py -@@ -3,7 +3,7 @@ - from google.protobuf.duration_pb2 import Duration - import pandas as pd - --from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService, OnDemandFeatureView -+from feast import Entity, FeatureView, FileSource, ValueType, FeatureService, OnDemandFeatureView - - # Read data from parquet files. Parquet is convenient for local development mode. For - # production, you can use your favorite DWH, such as BigQuery. See Feast documentation ---- /Users/achal/feast/prompt_dory/example.py -+++ /Users/achal/feast/prompt_dory/example.py -@@ -4,6 +4,7 @@ - import pandas as pd - - from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService, OnDemandFeatureView -+from feast import Field - - # Read data from parquet files. Parquet is convenient for local development mode. For - # production, you can use your favorite DWH, such as BigQuery. See Feast documentation ---- /Users/achal/feast/prompt_dory/example.py -+++ /Users/achal/feast/prompt_dory/example.py -@@ -28,9 +29,9 @@ - entities=[driver_id], - ttl=Duration(seconds=86400 * 365), - features=[ -- Feature(name="conv_rate", dtype=ValueType.FLOAT), -- Feature(name="acc_rate", dtype=ValueType.FLOAT), -- Feature(name="avg_daily_trips", dtype=ValueType.INT64), -+ Field(name="conv_rate", dtype=ValueType.FLOAT), -+ Field(name="acc_rate", dtype=ValueType.FLOAT), -+ Field(name="avg_daily_trips", dtype=ValueType.INT64), - ], - online=True, - batch_source=driver_hourly_stats, -``` ---- -To write these changes out, you can run the same command with the `--write` flag: -```bash -$ feast repo-upgrade --write -``` - -You should see the same output, but also see the changes reflected in your feature repo on disk. \ No newline at end of file diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 88bbd4f4327..b3d7b587b27 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -43,7 +43,6 @@ registry_dump, teardown, ) -from feast.repo_upgrade import RepoUpgrader from feast.utils import maybe_local_tz _logger = logging.getLogger(__name__) @@ -834,26 +833,5 @@ def validate( exit(1) -@cli.command("repo-upgrade", cls=NoOptionDefaultFormat) -@click.option( - "--write", - is_flag=True, - default=False, - help="Upgrade a feature repo to use the API expected by feast 0.23.", -) -@click.pass_context -def repo_upgrade(ctx: click.Context, write: bool): - """ - Upgrade a feature repo in place. - """ - repo = ctx.obj["CHDIR"] - fs_yaml_file = ctx.obj["FS_YAML_FILE"] - cli_check_repo(repo, fs_yaml_file) - try: - RepoUpgrader(repo, write).upgrade() - except FeastProviderLoginError as e: - print(str(e)) - - if __name__ == "__main__": cli() diff --git a/sdk/python/feast/repo_upgrade.py b/sdk/python/feast/repo_upgrade.py deleted file mode 100644 index 6aa7a2cc1d4..00000000000 --- a/sdk/python/feast/repo_upgrade.py +++ /dev/null @@ -1,175 +0,0 @@ -import logging -from pathlib import Path -from typing import Dict, List - -from bowler import Query -from fissix.fixer_util import touch_import -from fissix.pgen2 import token -from fissix.pygram import python_symbols -from fissix.pytree import Node - -from feast.repo_operations import get_repo_files - -SOURCES = { - "FileSource", - "BigQuerySource", - "RedshiftSource", - "SnowflakeSource", - "KafkaSource", - "KinesisSource", -} - - -class RepoUpgrader: - def __init__(self, repo_path: str, write: bool): - self.repo_path = repo_path - self.write = write - self.repo_files: List[str] = [ - str(p) for p in get_repo_files(Path(self.repo_path)) - ] - logging.getLogger("RefactoringTool").setLevel(logging.WARNING) - - def upgrade(self): - self.remove_date_partition_column() - self.rename_features_to_schema() - - def rename_inputs_to_sources(self): - def _change_argument_transform(node, capture, filename) -> None: - children = node.children - self.rename_arguments_in_children(children, {"inputs": "sources"}) - - PATTERN = """ - decorator< - any * - "on_demand_feature_view" - any * - > - """ - - Query(self.repo_files).select(PATTERN).modify( - _change_argument_transform - ).execute(write=self.write, interactive=False) - - def rename_features_to_schema(self): - Query(str(self.repo_path)).select_class("Feature").modify( - self.import_remover("Feature") - ).execute(interactive=False, write=self.write) - - def _rename_class_name( - node: Node, capture: Dict[str, Node], filename: str - ) -> None: - self.rename_class_call(node, "Field") - touch_import("feast", "Field", node) - - Query(self.repo_files).select_class("Feature").is_call().modify( - _rename_class_name - ).execute(write=self.write, interactive=False) - - def remove_date_partition_column(self): - def _remove_date_partition_column( - node: Node, capture: Dict[str, Node], filename: str - ) -> None: - self.remove_argument_transform(node, "date_partition_column") - - for s in SOURCES: - Query(self.repo_files).select_class(s).is_call().modify( - _remove_date_partition_column - ).execute(write=self.write, interactive=False) - - @staticmethod - def rename_arguments_in_children( - children: List[Node], renames: Dict[str, str] - ) -> None: - """ - Renames the arguments in the children list of a node by searching for the - argument list or trailing list and renaming all keys in `renames` dict to - corresponding value. - """ - for child in children: - if not isinstance(child, Node): - continue - if ( - child.type == python_symbols.arglist - or child.type == python_symbols.trailer - ): - if not child.children: - continue - for _, child in enumerate(child.children): - if not isinstance(child, Node): - continue - else: - if child.type == python_symbols.argument: - if child.children[0].value in renames: - child.children[0].value = renames[ - child.children[0].value - ] - - @staticmethod - def rename_class_call(node: Node, new_class_name: str): - """ - Rename the class being instantiated. - f = Feature( - name="driver_id", - join_key="driver_id", - ) - into - f = Field( - name="driver_id", - ) - This method assumes that node represents a class call that already has an arglist. - """ - if len(node.children) < 2 or len(node.children[1].children) < 2: - raise ValueError(f"Expected a class call with an arglist but got {node}.") - node.children[0].value = new_class_name - - @staticmethod - def remove_argument_transform(node: Node, argument: str): - """ - Removes the specified argument. - For example, if the argument is "join_key", this method transforms - driver = Entity( - name="driver_id", - join_key="driver_id", - ) - into - driver = Entity( - name="driver_id", - ) - This method assumes that node represents a class call that already has an arglist. - """ - if len(node.children) < 2 or len(node.children[1].children) < 2: - raise ValueError(f"Expected a class call with an arglist but got {node}.") - class_args = node.children[1].children[1].children - for i, class_arg in enumerate(class_args): - if ( - class_arg.type == python_symbols.argument - and class_arg.children[0].value == argument - ): - class_args.pop(i) - if i < len(class_args) and class_args[i].type == token.COMMA: - class_args.pop(i) - if i < len(class_args) and class_args[i].type == token.NEWLINE: - class_args.pop(i) - - @staticmethod - def import_remover(class_name): - def remove_import_transformer(node, capture, filename): - if "class_import" in capture and capture["class_name"].value == class_name: - if capture["class_import"].type == python_symbols.import_from: - import_from_stmt = node.children - imported_classes = import_from_stmt[3] - - if len(imported_classes.children) > 1: - # something of the form `from feast import A, ValueType` - for i, class_leaf in enumerate(imported_classes.children): - if class_leaf.value == class_name: - imported_classes.children.pop(i) - if i == len(imported_classes.children): - imported_classes.children.pop(i - 1) - else: - imported_classes.children.pop(i) - else: - # something of the form `from feast import ValueType` - node.parent.children.remove(node) - - return remove_import_transformer diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 15dadfd8740..452b52c73ad 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -16,8 +16,6 @@ anyio==4.3.0 # jupyter-server # starlette # watchfiles -appdirs==1.4.4 - # via fissix argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -38,7 +36,6 @@ atpublic==4.1.0 # via ibis-framework attrs==23.2.0 # via - # bowler # jsonschema # referencing azure-core==1.30.1 @@ -68,8 +65,6 @@ botocore==1.34.88 # boto3 # moto # s3transfer -bowler==0.9.0 - # via feast (setup.py) build==1.2.1 # via # feast (setup.py) @@ -101,12 +96,10 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via - # bowler # dask # feast (setup.py) # geomet # great-expectations - # moreorless # pip-tools # uvicorn cloudpickle==3.0.0 @@ -187,8 +180,6 @@ filelock==3.13.4 # virtualenv firebase-admin==5.4.0 # via feast (setup.py) -fissix==21.11.13 - # via bowler fqdn==1.5.1 # via jsonschema fsspec==2023.12.2 @@ -448,8 +439,6 @@ mmh3==4.1.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moreorless==0.4.0 - # via bowler moto==4.2.14 # via feast (setup.py) msal==1.28.0 @@ -997,8 +986,6 @@ virtualenv==20.23.0 # via # feast (setup.py) # pre-commit -volatile==2.1.0 - # via bowler watchfiles==0.21.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 2cfe62c55bd..9f90db249a0 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -10,25 +10,18 @@ anyio==4.3.0 # via # starlette # watchfiles -appdirs==1.4.4 - # via fissix attrs==23.2.0 # via - # bowler # jsonschema # referencing -bowler==0.9.0 - # via feast (setup.py) certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests click==8.1.7 # via - # bowler # dask # feast (setup.py) - # moreorless # uvicorn cloudpickle==3.0.0 # via dask @@ -46,8 +39,6 @@ exceptiongroup==1.2.1 # via anyio fastapi==0.110.2 # via feast (setup.py) -fissix==21.11.13 - # via bowler fsspec==2024.3.1 # via dask greenlet==3.0.3 @@ -76,8 +67,6 @@ markupsafe==2.1.5 # via jinja2 mmh3==4.1.0 # via feast (setup.py) -moreorless==0.4.0 - # via bowler mypy==1.9.0 # via sqlalchemy mypy-extensions==1.0.0 @@ -184,8 +173,6 @@ uvicorn[standard]==0.29.0 # via feast (setup.py) uvloop==0.19.0 # via uvicorn -volatile==2.1.0 - # via bowler watchfiles==0.21.0 # via uvicorn websockets==12.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 8de9151e4e3..9486743f776 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -16,8 +16,6 @@ anyio==4.3.0 # jupyter-server # starlette # watchfiles -appdirs==1.4.4 - # via fissix argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -38,7 +36,6 @@ atpublic==4.1.0 # via ibis-framework attrs==23.2.0 # via - # bowler # jsonschema # referencing azure-core==1.30.1 @@ -68,8 +65,6 @@ botocore==1.34.88 # boto3 # moto # s3transfer -bowler==0.9.0 - # via feast (setup.py) build==1.2.1 # via # feast (setup.py) @@ -101,12 +96,10 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via - # bowler # dask # feast (setup.py) # geomet # great-expectations - # moreorless # pip-tools # uvicorn cloudpickle==3.0.0 @@ -187,8 +180,6 @@ filelock==3.13.4 # virtualenv firebase-admin==5.4.0 # via feast (setup.py) -fissix==21.11.13 - # via bowler fqdn==1.5.1 # via jsonschema fsspec==2023.12.2 @@ -457,8 +448,6 @@ mmh3==4.1.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moreorless==0.4.0 - # via bowler moto==4.2.14 # via feast (setup.py) msal==1.28.0 @@ -1010,8 +999,6 @@ virtualenv==20.23.0 # via # feast (setup.py) # pre-commit -volatile==2.1.0 - # via bowler watchfiles==0.21.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 472f3e90b99..368b2421266 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -10,25 +10,18 @@ anyio==4.3.0 # via # starlette # watchfiles -appdirs==1.4.4 - # via fissix attrs==23.2.0 # via - # bowler # jsonschema # referencing -bowler==0.9.0 - # via feast (setup.py) certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests click==8.1.7 # via - # bowler # dask # feast (setup.py) - # moreorless # uvicorn cloudpickle==3.0.0 # via dask @@ -46,8 +39,6 @@ exceptiongroup==1.2.1 # via anyio fastapi==0.110.2 # via feast (setup.py) -fissix==21.11.13 - # via bowler fsspec==2024.3.1 # via dask greenlet==3.0.3 @@ -78,8 +69,6 @@ markupsafe==2.1.5 # via jinja2 mmh3==4.1.0 # via feast (setup.py) -moreorless==0.4.0 - # via bowler mypy==1.9.0 # via sqlalchemy mypy-extensions==1.0.0 @@ -187,8 +176,6 @@ uvicorn[standard]==0.29.0 # via feast (setup.py) uvloop==0.19.0 # via uvicorn -volatile==2.1.0 - # via bowler watchfiles==0.21.0 # via uvicorn websockets==12.0 diff --git a/setup.py b/setup.py index 65d8ee27b5c..ef5986f1579 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,6 @@ "uvicorn[standard]>=0.14.0,<1", "gunicorn; platform_system != 'Windows'", "dask[dataframe]>=2024.4.2", - "bowler", # Needed for automatic repo upgrades ] GCP_REQUIRED = [ From adc2939fd419d7f8f6ba02cf24703dc42dd7d873 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Sat, 4 May 2024 10:01:21 +0400 Subject: [PATCH 40/73] docs: Add DuckDB offline store (#4174) --- docs/SUMMARY.md | 1 + docs/reference/data-sources/file.md | 6 +-- docs/reference/data-sources/overview.md | 4 +- docs/reference/offline-stores/README.md | 4 ++ docs/reference/offline-stores/duckdb.md | 56 +++++++++++++++++++++++ docs/reference/offline-stores/overview.md | 28 ++++++------ 6 files changed, 78 insertions(+), 21 deletions(-) create mode 100644 docs/reference/offline-stores/duckdb.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 3673edf6cf7..2e205dee0a1 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -80,6 +80,7 @@ * [Snowflake](reference/offline-stores/snowflake.md) * [BigQuery](reference/offline-stores/bigquery.md) * [Redshift](reference/offline-stores/redshift.md) + * [DuckDB](reference/offline-stores/duckdb.md) * [Spark (contrib)](reference/offline-stores/spark.md) * [PostgreSQL (contrib)](reference/offline-stores/postgres.md) * [Trino (contrib)](reference/offline-stores/trino.md) diff --git a/docs/reference/data-sources/file.md b/docs/reference/data-sources/file.md index 5895b1a8cee..d3fd09deca6 100644 --- a/docs/reference/data-sources/file.md +++ b/docs/reference/data-sources/file.md @@ -3,11 +3,7 @@ ## Description File data sources are files on disk or on S3. -Currently only Parquet files are supported. - -{% hint style="warning" %} -FileSource is meant for development purposes only and is not optimized for production use. -{% endhint %} +Currently only Parquet and Delta formats are supported. ## Example diff --git a/docs/reference/data-sources/overview.md b/docs/reference/data-sources/overview.md index 302c19b049c..5c2fdce9fd1 100644 --- a/docs/reference/data-sources/overview.md +++ b/docs/reference/data-sources/overview.md @@ -2,8 +2,8 @@ ## Functionality -In Feast, each batch data source is associated with a corresponding offline store. -For example, a `SnowflakeSource` can only be processed by the Snowflake offline store. +In Feast, each batch data source is associated with corresponding offline stores. +For example, a `SnowflakeSource` can only be processed by the Snowflake offline store, while a `FileSource` can be processed by both File and DuckDB offline stores. Otherwise, the primary difference between batch data sources is the set of supported types. Feast has an internal type system, and aims to support eight primitive types (`bytes`, `string`, `int32`, `int64`, `float32`, `float64`, `bool`, and `timestamp`) along with the corresponding array types. However, not every batch data source supports all of these types. diff --git a/docs/reference/offline-stores/README.md b/docs/reference/offline-stores/README.md index f4e3af2f345..33eca6d4260 100644 --- a/docs/reference/offline-stores/README.md +++ b/docs/reference/offline-stores/README.md @@ -22,6 +22,10 @@ Please see [Offline Store](../../getting-started/architecture-and-components/off [redshift.md](redshift.md) {% endcontent-ref %} +{% content-ref url="duckdb.md" %} +[duckdb.md](duckdb.md) +{% endcontent-ref %} + {% content-ref url="spark.md" %} [spark.md](spark.md) {% endcontent-ref %} diff --git a/docs/reference/offline-stores/duckdb.md b/docs/reference/offline-stores/duckdb.md new file mode 100644 index 00000000000..da3c3cd0c77 --- /dev/null +++ b/docs/reference/offline-stores/duckdb.md @@ -0,0 +1,56 @@ +# DuckDB offline store + +## Description + +The duckdb offline store provides support for reading [FileSources](../data-sources/file.md). It can read both Parquet and Delta formats. DuckDB offline store uses [ibis](https://ibis-project.org/) under the hood to convert offline store operations to DuckDB queries. + +* Entity dataframes can be provided as a Pandas dataframe. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[duckdb]'`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_project +registry: data/registry.db +provider: local +offline_store: + type: duckdb +online_store: + path: data/online_store.db +``` +{% endcode %} + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the DuckDB offline store. + +| | DuckdDB | +| :----------------------------------------------------------------- | :---- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | yes | +| `write_logged_features` (persist logged features to offline store) | yes | + +Below is a matrix indicating which functionality is supported by `IbisRetrievalJob`. + +| | DuckDB| +| ----------------------------------------------------- | ----- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | no | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | no | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/overview.md b/docs/reference/offline-stores/overview.md index 8ce90454963..4d7681e38c8 100644 --- a/docs/reference/offline-stores/overview.md +++ b/docs/reference/offline-stores/overview.md @@ -42,17 +42,17 @@ Below is a matrix indicating which offline stores support which methods. Below is a matrix indicating which `RetrievalJob`s support what functionality. -| | File | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | -| --------------------------------- | --- | --- | --- | --- | --- | --- | --- | -| export to dataframe | yes | yes | yes | yes | yes | yes | yes | -| export to arrow table | yes | yes | yes | yes | yes | yes | yes | -| export to arrow batches | no | no | no | yes | no | no | no | -| export to SQL | no | yes | yes | yes | yes | no | yes | -| export to data lake (S3, GCS, etc.) | no | no | yes | no | yes | no | no | -| export to data warehouse | no | yes | yes | yes | yes | no | no | -| export as Spark dataframe | no | no | yes | no | no | yes | no | -| local execution of Python-based on-demand transforms | yes | yes | yes | yes | yes | no | yes | -| remote execution of Python-based on-demand transforms | no | no | no | no | no | no | no | -| persist results in the offline store | yes | yes | yes | yes | yes | yes | no | -| preview the query plan before execution | yes | yes | yes | yes | yes | yes | yes | -| read partitioned data | yes | yes | yes | yes | yes | yes | yes | +| | File | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | DuckDB | +| --------------------------------- | --- | --- | --- | --- | --- | --- | --- | --- | +| export to dataframe | yes | yes | yes | yes | yes | yes | yes | yes | +| export to arrow table | yes | yes | yes | yes | yes | yes | yes | yes | +| export to arrow batches | no | no | no | yes | no | no | no | no | +| export to SQL | no | yes | yes | yes | yes | no | yes | no | +| export to data lake (S3, GCS, etc.) | no | no | yes | no | yes | no | no | no | +| export to data warehouse | no | yes | yes | yes | yes | no | no | no | +| export as Spark dataframe | no | no | yes | no | no | yes | no | no | +| local execution of Python-based on-demand transforms | yes | yes | yes | yes | yes | no | yes | yes | +| remote execution of Python-based on-demand transforms | no | no | no | no | no | no | no | no | +| persist results in the offline store | yes | yes | yes | yes | yes | yes | no | yes | +| preview the query plan before execution | yes | yes | yes | yes | yes | yes | yes | no | +| read partitioned data | yes | yes | yes | yes | yes | yes | yes | yes | From 67bea4cb73a5cb71197184b1bd8692650647a458 Mon Sep 17 00:00:00 2001 From: Jeremy Ary Date: Tue, 7 May 2024 06:44:45 -0500 Subject: [PATCH 41/73] chore: Update CI-related AWS settings due to cred swap & global namespace issues (#4171) * chore: Bumping CI to test new cloud creds, do not merge Signed-off-by: Jeremy Ary * chore: Update S3 CI bucket to avoid global namespace issues Signed-off-by: Jeremy Ary --------- Signed-off-by: Jeremy Ary --- Makefile | 2 +- README.md | 2 +- docs/reference/offline-stores/redshift.md | 4 ++-- .../contrib/athena_offline_store/tests/data_source.py | 2 +- .../tests/integration/feature_repos/repo_configuration.py | 2 +- .../feature_repos/universal/data_sources/redshift.py | 7 ++++--- .../tests/integration/registration/test_feature_store.py | 2 +- sdk/python/tests/integration/registration/test_registry.py | 2 +- .../tests/unit/infra/offline_stores/test_offline_store.py | 6 +++--- 9 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index f0abc930069..3a1d0ebc808 100644 --- a/Makefile +++ b/Makefile @@ -172,7 +172,7 @@ test-python-universal-athena: ATHENA_DATA_SOURCE=AwsDataCatalog \ ATHENA_DATABASE=default \ ATHENA_WORKGROUP=primary \ - ATHENA_S3_BUCKET_NAME=feast-integration-tests \ + ATHENA_S3_BUCKET_NAME=feast-int-bucket \ python -m pytest -n 8 --integration \ -k "not test_go_feature_server and \ not test_logged_features_validation and \ diff --git a/README.md b/README.md index a1e06774dac..e9b7ff47436 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

-
+
[![unit-tests](https://github.com/feast-dev/feast/actions/workflows/unit_tests.yml/badge.svg?branch=master&event=push)](https://github.com/feast-dev/feast/actions/workflows/unit_tests.yml) [![integration-tests-and-build](https://github.com/feast-dev/feast/actions/workflows/master_only.yml/badge.svg?branch=master&event=push)](https://github.com/feast-dev/feast/actions/workflows/master_only.yml) diff --git a/docs/reference/offline-stores/redshift.md b/docs/reference/offline-stores/redshift.md index e9bcbfeff1a..e33a1856cb2 100644 --- a/docs/reference/offline-stores/redshift.md +++ b/docs/reference/offline-stores/redshift.md @@ -130,8 +130,8 @@ The following inline policy can be used to grant Redshift necessary permissions "Action": "s3:*", "Effect": "Allow", "Resource": [ - "arn:aws:s3:::feast-integration-tests", - "arn:aws:s3:::feast-integration-tests/*" + "arn:aws:s3:::feast-int-bucket", + "arn:aws:s3:::feast-int-bucket/*" ] } ], diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py index f01144afcc4..f95a750fd14 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py @@ -31,7 +31,7 @@ def __init__(self, project_name: str, *args, **kwargs): data_source = os.getenv("ATHENA_DATA_SOURCE", "AwsDataCatalog") database = os.getenv("ATHENA_DATABASE", "default") workgroup = os.getenv("ATHENA_WORKGROUP", "primary") - bucket_name = os.getenv("ATHENA_S3_BUCKET_NAME", "feast-integration-tests") + bucket_name = os.getenv("ATHENA_S3_BUCKET_NAME", "feast-int-bucket") self.client = aws_utils.get_athena_data_client(region) self.s3 = aws_utils.get_s3_resource(region) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 096744f5472..4007106a064 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -461,7 +461,7 @@ def construct_test_environment( test_repo_config.python_feature_server and test_repo_config.provider == "aws" ) or test_repo_config.registry_location == RegistryLocation.S3: aws_registry_path = os.getenv( - "AWS_REGISTRY_PATH", "s3://feast-integration-tests/registries" + "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries" ) registry: Union[str, RegistryConfig] = ( f"{aws_registry_path}/{project}/registry.db" diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py index 60fb8950a96..8fe933fbba7 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py @@ -30,16 +30,17 @@ def __init__(self, project_name: str, *args, **kwargs): self.s3 = aws_utils.get_s3_resource(os.getenv("AWS_REGION", "us-west-2")) self.offline_store_config = RedshiftOfflineStoreConfig( - cluster_id=os.getenv("AWS_CLUSTER_ID", "feast-integration-tests"), + cluster_id=os.getenv("AWS_CLUSTER_ID", "feast-int-bucket"), region=os.getenv("AWS_REGION", "us-west-2"), user=os.getenv("AWS_USER", "admin"), database=os.getenv("AWS_DB", "feast"), s3_staging_location=os.getenv( "AWS_STAGING_LOCATION", - "s3://feast-integration-tests/redshift/tests/ingestion", + "s3://feast-int-bucket/redshift/tests/ingestion", ), iam_role=os.getenv( - "AWS_IAM_ROLE", "arn:aws:iam::402087665549:role/redshift_s3_access_role" + "AWS_IAM_ROLE", + "arn:aws:iam::585132637328:role/service-role/AmazonRedshift-CommandsAccessRole-20240403T092631", ), workgroup="", ) diff --git a/sdk/python/tests/integration/registration/test_feature_store.py b/sdk/python/tests/integration/registration/test_feature_store.py index deb1b0635f3..bf0c2fb61fd 100644 --- a/sdk/python/tests/integration/registration/test_feature_store.py +++ b/sdk/python/tests/integration/registration/test_feature_store.py @@ -226,7 +226,7 @@ def feature_store_with_gcs_registry(): @pytest.fixture def feature_store_with_s3_registry(): aws_registry_path = os.getenv( - "AWS_REGISTRY_PATH", "s3://feast-integration-tests/registries" + "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries" ) return FeatureStore( config=RepoConfig( diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 232f0356093..70d118ecf95 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -53,7 +53,7 @@ def gcs_registry() -> Registry: @pytest.fixture def s3_registry() -> Registry: aws_registry_path = os.getenv( - "AWS_REGISTRY_PATH", "s3://feast-integration-tests/registries" + "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries" ) registry_config = RegistryConfig( path=f"{aws_registry_path}/{int(time.time() * 1000)}/registry.db", diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py index 0232a8d379d..e5768a81b21 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py @@ -112,12 +112,12 @@ def retrieval_job(request, environment): return FileRetrievalJob(lambda: 1, full_feature_names=False) elif request.param is RedshiftRetrievalJob: offline_store_config = RedshiftOfflineStoreConfig( - cluster_id="feast-integration-tests", + cluster_id="feast-int-bucket", region="us-west-2", user="admin", database="feast", - s3_staging_location="s3://feast-integration-tests/redshift/tests/ingestion", - iam_role="arn:aws:iam::402087665549:role/redshift_s3_access_role", + s3_staging_location="s3://feast-int-bucket/redshift/tests/ingestion", + iam_role="arn:aws:iam::585132637328:role/service-role/AmazonRedshift-CommandsAccessRole-20240403T092631", workgroup="", ) environment.test_repo_config.offline_store = offline_store_config From ba9f4efd5eccd0548a39521a145c6573ac90c221 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Tue, 7 May 2024 08:06:42 -0400 Subject: [PATCH 42/73] feat: Enable other distance metrics for Vector DB and Update docs (#4170) * updated PGVector docs Signed-off-by: Francisco Javier Arceo * adding distance metric to arguments and defaulting to L2 Signed-off-by: Francisco Javier Arceo * linter Signed-off-by: Francisco Javier Arceo * testing other distance metric Signed-off-by: Francisco Javier Arceo * updated default Signed-off-by: Francisco Javier Arceo * linter Signed-off-by: Francisco Javier Arceo * fixed some copy Signed-off-by: Francisco Javier Arceo * updated Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo --- docs/reference/online-stores/postgres.md | 10 +++++++-- sdk/python/feast/feature_store.py | 7 +++++++ .../infra/online_stores/contrib/postgres.py | 18 +++++++++++++++- .../feast/infra/online_stores/online_store.py | 2 +- .../feast/infra/passthrough_provider.py | 8 ++++++- sdk/python/feast/infra/provider.py | 5 +++-- sdk/python/tests/foo_provider.py | 1 + .../online_store/test_universal_online.py | 21 ++++++++++++++++++- 8 files changed, 64 insertions(+), 8 deletions(-) diff --git a/docs/reference/online-stores/postgres.md b/docs/reference/online-stores/postgres.md index 34d4de34883..77a9408d2bd 100644 --- a/docs/reference/online-stores/postgres.md +++ b/docs/reference/online-stores/postgres.md @@ -65,10 +65,16 @@ To compare this set of functionality against other online stores, please see the ## PGVector The Postgres online store supports the use of [PGVector](https://github.com/pgvector/pgvector) for storing feature values. -To enable PGVector, set `pgvector_enabled: true` in the online store configuration. +To enable PGVector, set `pgvector_enabled: true` in the online store configuration. + The `vector_len` parameter can be used to specify the length of the vector. The default value is 512. -Then you can use `retrieve_online_documents` to retrieve the top k closest vectors to a query vector. +Please make sure to follow the instructions in the repository, which, as the time of this writing, requires you to +run `CREATE EXTENSION vector;` in the database. + + +Then you can use `retrieve_online_documents` to retrieve the top k closest vectors to a query vector. +For the Retrieval Augmented Generation (RAG) use-case, you have to embed the query prior to passing the query vector. {% code title="python" %} ```python diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index bc492e42086..f45dbb1bc8f 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1740,6 +1740,7 @@ def retrieve_online_documents( feature: str, query: Union[str, List[float]], top_k: int, + distance_metric: str, ) -> OnlineResponse: """ Retrieves the top k closest document features. Note, embeddings are a subset of features. @@ -1750,11 +1751,13 @@ def retrieve_online_documents( references must have format "feature_view:feature", e.g, "document_fv:document_embeddings". query: The query to retrieve the closest document features for. top_k: The number of closest document features to retrieve. + distance_metric: The distance metric to use for retrieval. """ return self._retrieve_online_documents( feature=feature, query=query, top_k=top_k, + distance_metric=distance_metric, ) def _retrieve_online_documents( @@ -1762,6 +1765,7 @@ def _retrieve_online_documents( feature: str, query: Union[str, List[float]], top_k: int, + distance_metric: str = "L2", ): if isinstance(query, str): raise ValueError( @@ -1783,6 +1787,7 @@ def _retrieve_online_documents( requested_feature, query, top_k, + distance_metric, ) # TODO Refactor to better way of populating result @@ -2025,6 +2030,7 @@ def _retrieve_from_online_store( requested_feature: str, query: List[float], top_k: int, + distance_metric: str, ) -> List[Tuple[Timestamp, "FieldStatus.ValueType", Value, Value, Value]]: """ Search and return document features from the online document store. @@ -2035,6 +2041,7 @@ def _retrieve_from_online_store( requested_feature=requested_feature, query=query, top_k=top_k, + distance_metric=distance_metric, ) read_row_protos = [] diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 6ed0885d138..f2c32fdafd1 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -21,6 +21,13 @@ from feast.repo_config import RepoConfig from feast.usage import log_exceptions_and_usage +SUPPORTED_DISTANCE_METRICS_DICT = { + "cosine": "<=>", + "L1": "<+>", + "L2": "<->", + "inner_product": "<#>", +} + class PostgreSQLOnlineStoreConfig(PostgreSQLConfig): type: Literal["postgres"] = "postgres" @@ -276,6 +283,7 @@ def retrieve_online_documents( requested_feature: str, embedding: List[float], top_k: int, + distance_metric: str = "L2", ) -> List[ Tuple[ Optional[datetime], @@ -292,6 +300,7 @@ def retrieve_online_documents( requested_feature: The requested feature as the column to search embedding: The query embedding to search for top_k: The number of items to return + distance_metric: The distance metric to use for the search.G Returns: List of tuples containing the event timestamp and the document feature @@ -303,6 +312,12 @@ def retrieve_online_documents( "pgvector is not enabled in the online store configuration" ) + if distance_metric not in SUPPORTED_DISTANCE_METRICS_DICT: + raise ValueError( + f"Distance metric {distance_metric} is not supported. Supported distance metrics are {SUPPORTED_DISTANCE_METRICS_DICT.keys()}" + ) + + distance_metric_sql = SUPPORTED_DISTANCE_METRICS_DICT[distance_metric] # Convert the embedding to a string to be used in postgres vector search query_embedding_str = f"[{','.join(str(el) for el in embedding)}]" @@ -327,13 +342,14 @@ def retrieve_online_documents( feature_name, value, vector_value, - vector_value <-> %s as distance, + vector_value {distance_metric_sql} %s as distance, event_ts FROM {table_name} WHERE feature_name = {feature_name} ORDER BY distance LIMIT {top_k}; """ ).format( + distance_metric_sql=distance_metric_sql, table_name=sql.Identifier(table_name), feature_name=sql.Literal(requested_feature), top_k=sql.Literal(top_k), diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index 67c5a931dda..2a81e370427 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -158,7 +158,7 @@ def retrieve_online_documents( table: The feature view whose feature values should be read. requested_feature: The name of the feature whose embeddings should be used for retrieval. embedding: The embeddings to use for retrieval. - top_k: The number of nearest neighbors to retrieve. + top_k: The number of documents to retrieve. Returns: object: A list of top k closest documents to the specified embedding. Each item in the list is a tuple diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 6476acbcb93..2f3e30018af 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -196,12 +196,18 @@ def retrieve_online_documents( requested_feature: str, query: List[float], top_k: int, + distance_metric: str, ) -> List: set_usage_attribute("provider", self.__class__.__name__) result = [] if self.online_store: result = self.online_store.retrieve_online_documents( - config, table, requested_feature, query, top_k + config, + table, + requested_feature, + query, + top_k, + distance_metric, ) return result diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index a45051a1b6b..02fba0c1f6b 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -303,6 +303,7 @@ def retrieve_online_documents( requested_feature: str, query: List[float], top_k: int, + distance_metric: str = "L2", ) -> List[ Tuple[ Optional[datetime], @@ -312,14 +313,14 @@ def retrieve_online_documents( ] ]: """ - Searches for the top-k nearest neighbors of the given document in the online document store. + Searches for the top-k most similar documents in the online document store. Args: config: The config for the current feature store. table: The feature view whose embeddings should be searched. requested_feature: the requested document feature name. query: The query embedding to search for. - top_k: The number of nearest neighbors to return. + top_k: The number of documents to return. Returns: A list of dictionaries, where each dictionary contains the document feature. diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index 2a830d424cc..f869d82e114 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -111,6 +111,7 @@ def retrieve_online_documents( requested_feature: str, query: List[float], top_k: int, + distance_metric: str, ) -> List[ Tuple[ Optional[datetime], diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 3ae7be9e1e4..5d6462e5e3d 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -798,6 +798,25 @@ def test_retrieve_online_documents(environment, fake_document_data): fs.write_to_online_store("item_embeddings", df) documents = fs.retrieve_online_documents( - feature="item_embeddings:embedding_float", query=[1.0, 2.0], top_k=2 + feature="item_embeddings:embedding_float", + query=[1.0, 2.0], + top_k=2, + distance_metric="L2", ).to_dict() assert len(documents["embedding_float"]) == 2 + + documents = fs.retrieve_online_documents( + feature="item_embeddings:embedding_float", + query=[1.0, 2.0], + top_k=2, + distance_metric="L1", + ).to_dict() + assert len(documents["embedding_float"]) == 2 + + with pytest.raises(ValueError): + fs.retrieve_online_documents( + feature="item_embeddings:embedding_float", + query=[1.0, 2.0], + top_k=2, + distance_metric="wrong", + ).to_dict() From 5051da75de81deed19b25fbc2826d504a8ebdc8b Mon Sep 17 00:00:00 2001 From: Tom Steenbergen <41334387+TomSteenbergen@users.noreply.github.com> Date: Tue, 7 May 2024 15:15:38 +0200 Subject: [PATCH 43/73] fix: Make sure schema is used when generating `from_expression` for Snowflake (#4177) Fix from_expression when using Snowflake offline store Signed-off-by: TomSteenbergen --- sdk/python/feast/infra/offline_stores/snowflake.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py index 907e4d44833..cc59804467a 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake.py +++ b/sdk/python/feast/infra/offline_stores/snowflake.py @@ -139,8 +139,10 @@ def pull_latest_from_table_or_query( assert isinstance(data_source, SnowflakeSource) from_expression = data_source.get_table_query_string() - if not data_source.database and data_source.table: + if not data_source.database and not data_source.schema and data_source.table: from_expression = f'"{config.offline_store.database}"."{config.offline_store.schema_}".{from_expression}' + if not data_source.database and data_source.schema and data_source.table: + from_expression = f'"{config.offline_store.database}".{from_expression}' if join_key_columns: partition_by_join_key_string = '"' + '", "'.join(join_key_columns) + '"' @@ -226,8 +228,10 @@ def pull_all_from_table_or_query( assert isinstance(data_source, SnowflakeSource) from_expression = data_source.get_table_query_string() - if not data_source.database and data_source.table: + if not data_source.database and not data_source.schema and data_source.table: from_expression = f'"{config.offline_store.database}"."{config.offline_store.schema_}".{from_expression}' + if not data_source.database and data_source.schema and data_source.table: + from_expression = f'"{config.offline_store.database}".{from_expression}' field_string = ( '"' From 946ee12873c46c6c5d44db6ac3a43ac43fce57cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 11:39:45 -0500 Subject: [PATCH 44/73] chore: Bump tqdm from 4.66.2 to 4.66.3 in /sdk/python/requirements (#4175) Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.66.2 to 4.66.3. - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.66.2...v4.66.3) --- updated-dependencies: - dependency-name: tqdm dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.10-requirements.txt | 2 +- sdk/python/requirements/py3.11-ci-requirements.txt | 2 +- sdk/python/requirements/py3.11-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 452b52c73ad..c6c2e4bf837 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -885,7 +885,7 @@ tornado==6.4 # jupyterlab # notebook # terminado -tqdm==4.66.2 +tqdm==4.66.3 # via # feast (setup.py) # great-expectations diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 9f90db249a0..93f0b50ab6f 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -149,7 +149,7 @@ toolz==0.12.1 # via # dask # partd -tqdm==4.66.2 +tqdm==4.66.3 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 71f61964be6..13f33531aee 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -908,7 +908,7 @@ tornado==6.4 # jupyterlab # notebook # terminado -tqdm==4.66.2 +tqdm==4.66.3 # via # feast (setup.py) # great-expectations diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 161e435b54e..b3ab26d2345 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -162,7 +162,7 @@ toolz==0.12.1 # via # dask # partd -tqdm==4.66.2 +tqdm==4.66.3 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 9486743f776..3a042c40fcf 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -896,7 +896,7 @@ tornado==6.4 # jupyterlab # notebook # terminado -tqdm==4.66.2 +tqdm==4.66.3 # via # feast (setup.py) # great-expectations diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 368b2421266..3c7566ce9a7 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -151,7 +151,7 @@ toolz==0.12.1 # via # dask # partd -tqdm==4.66.2 +tqdm==4.66.3 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) From 50e41ae8f9dc2744bd4ab8f92a7e17010c463737 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 11:40:04 -0500 Subject: [PATCH 45/73] chore: Bump werkzeug from 3.0.2 to 3.0.3 in /sdk/python/requirements (#4178) Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.2 to 3.0.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/3.0.2...3.0.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.11-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index c6c2e4bf837..afb18a51c83 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1002,7 +1002,7 @@ websocket-client==1.7.0 # kubernetes websockets==12.0 # via uvicorn -werkzeug==3.0.2 +werkzeug==3.0.3 # via moto wheel==0.43.0 # via pip-tools diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 13f33531aee..09da08615f4 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1029,7 +1029,7 @@ websocket-client==1.8.0 # kubernetes websockets==12.0 # via uvicorn -werkzeug==3.0.2 +werkzeug==3.0.3 # via moto wheel==0.43.0 # via pip-tools diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 3a042c40fcf..27b35ca52b1 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1015,7 +1015,7 @@ websocket-client==1.7.0 # kubernetes websockets==12.0 # via uvicorn -werkzeug==3.0.2 +werkzeug==3.0.3 # via moto wheel==0.43.0 # via pip-tools From 77f719ddd021851a6c08beb6b12592348b34888e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 11:40:27 -0500 Subject: [PATCH 46/73] chore: Bump jinja2 from 3.1.3 to 3.1.4 in /sdk/python/requirements (#4179) Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.3 to 3.1.4. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.3...3.1.4) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.10-requirements.txt | 2 +- sdk/python/requirements/py3.11-ci-requirements.txt | 2 +- sdk/python/requirements/py3.11-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index afb18a51c83..2f17714964b 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -338,7 +338,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.1 # via ipython -jinja2==3.1.3 +jinja2==3.1.4 # via # altair # feast (setup.py) diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 93f0b50ab6f..0866a4e2168 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -55,7 +55,7 @@ idna==3.7 # requests importlib-metadata==7.1.0 # via dask -jinja2==3.1.3 +jinja2==3.1.4 # via feast (setup.py) jsonschema==4.21.1 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 09da08615f4..8c9a1325483 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -358,7 +358,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.1 # via ipython -jinja2==3.1.3 +jinja2==3.1.4 # via # altair # feast (setup.py) diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index b3ab26d2345..8431afc663f 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -66,7 +66,7 @@ importlib-metadata==7.1.0 # via # dask # typeguard -jinja2==3.1.3 +jinja2==3.1.4 # via feast (setup.py) jsonschema==4.21.1 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 27b35ca52b1..874ef1d58df 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -347,7 +347,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.1 # via ipython -jinja2==3.1.3 +jinja2==3.1.4 # via # altair # feast (setup.py) diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 3c7566ce9a7..041bd0c2880 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -57,7 +57,7 @@ importlib-metadata==7.1.0 # via # dask # typeguard -jinja2==3.1.3 +jinja2==3.1.4 # via feast (setup.py) jsonschema==4.21.1 # via feast (setup.py) From e88f1e39778300fb443f1db230fe9589b74d9ed6 Mon Sep 17 00:00:00 2001 From: Jeremy Ary Date: Tue, 7 May 2024 14:00:21 -0500 Subject: [PATCH 47/73] fix: Update master-only benchmark bucket name due to credential update (#4183) Signed-off-by: Jeremy Ary --- .github/workflows/master_only.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 295e7b17e23..1d6850e4d8e 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -142,7 +142,7 @@ jobs: SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} run: pytest --verbose --color=yes sdk/python/tests --integration --benchmark --benchmark-autosave --benchmark-save-data --durations=5 - name: Upload Benchmark Artifact to S3 - run: aws s3 cp --recursive .benchmarks s3://feast-ci-pytest-benchmarks + run: aws s3 cp --recursive .benchmarks s3://feast-ci-pytest-benchmark build-all-docker-images: if: github.repository == 'feast-dev/feast' From 34d36354d7a3f892648294db798188408392fabb Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Wed, 8 May 2024 20:08:37 +0400 Subject: [PATCH 48/73] chore: Change dev workflow to use uv (and pixi) both for lock and install (#4166) * change dev workflow to use uv (and pixi) Signed-off-by: tokoko * pin ibis versions Signed-off-by: tokoko * remove redundant make command Signed-off-by: tokoko --------- Signed-off-by: tokoko --- Makefile | 17 +- docs/project/development-guide.md | 46 +-- infra/scripts/pixi/pixi.lock | 366 ++++++------------ infra/scripts/pixi/pixi.toml | 8 +- .../requirements/py3.10-ci-requirements.txt | 297 +++++--------- .../requirements/py3.10-requirements.txt | 129 +++--- .../requirements/py3.11-ci-requirements.txt | 303 ++++----------- .../requirements/py3.11-requirements.txt | 143 ++++--- .../requirements/py3.9-ci-requirements.txt | 295 +++++--------- .../requirements/py3.9-requirements.txt | 129 +++--- setup.py | 6 +- 11 files changed, 657 insertions(+), 1082 deletions(-) diff --git a/Makefile b/Makefile index 3a1d0ebc808..18006fe7d1c 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ install-python-ci-dependencies-uv: python setup.py build_python_protos --inplace lock-python-ci-dependencies: - python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt + uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt package-protos: cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos @@ -60,13 +60,15 @@ install-python: python setup.py develop lock-python-dependencies: - python -m piptools compile -U --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt + uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt lock-python-dependencies-all: - pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.9-requirements.txt" - pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt" - pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.10-requirements.txt" - pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt" + pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt" + pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt" + pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt" + pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt" + pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt" + pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt" benchmark-python: FEAST_USAGE=False IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests @@ -359,9 +361,6 @@ kill-trino-locally: install-protoc-dependencies: pip install --ignore-installed protobuf==4.24.0 "grpcio-tools>=1.56.2,<2" mypy-protobuf==3.1.0 -install-feast-ci-locally: - pip install -e ".[ci]" - # Docker build-docker: build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker diff --git a/docs/project/development-guide.md b/docs/project/development-guide.md index 2d4ab0c7c6a..146b6d7516e 100644 --- a/docs/project/development-guide.md +++ b/docs/project/development-guide.md @@ -123,43 +123,43 @@ Note that this means if you are midway through working through a PR and rebase, Setting up your development environment for Feast Python SDK / CLI: 1. Ensure that you have Docker installed in your environment. Docker is used to provision service dependencies during testing, and build images for feature servers and other components. - Please note that we use [Docker with BuiltKit](https://docs.docker.com/develop/develop-images/build_enhancements/). -2. Ensure that you have `make`, Python (3.8 and above) with `pip`, installed. +2. Ensure that you have `make` and Python (3.9 or above) installed. 3. _Recommended:_ Create a virtual environment to isolate development dependencies to be installed ```sh # create & activate a virtual environment python -m venv venv/ source venv/bin/activate ``` -4. Upgrade `pip` if outdated - ```sh - pip install --upgrade pip - ``` -5. (M1 Mac only): Follow the [dev guide](https://github.com/feast-dev/feast/issues/2105) -6. Install pip-tools - ```sh - pip install pip-tools - ``` -7. (Optional): Install Node & Yarn. Then run the following to build Feast UI artifacts for use in `feast ui` +4. (M1 Mac only): Follow the [dev guide](https://github.com/feast-dev/feast/issues/2105) +5. Install uv +It is recommended to use uv for managing python dependencies. +```sh +curl -LsSf https://astral.sh/uv/install.sh | sh +``` +or +```ssh +pip install uv +``` +6. (Optional): Install Node & Yarn. Then run the following to build Feast UI artifacts for use in `feast ui` ``` make build-ui ``` -8. Install mysql (needed for ci dependencies) +7. (Optional) install pixi +pixi is necessary to run step 8 for all python versions at once. ```sh -brew install mysql +curl -fsSL https://pixi.sh/install.sh | bash ``` -9. Install development dependencies for Feast Python SDK / CLI +8. (Optional): Recompile python lock files +If you make changes to requirements or simply want to update python lock files to reflect latest versioons. ```sh -pip install -e ".[dev]" -``` - -This will allow the installed feast version to automatically reflect changes to your local development version of Feast without needing to reinstall everytime you make code changes. - -10. Compile the protubufs +make lock-python-dependencies-all +``` +9. Install development dependencies for Feast Python SDK / CLI +This will install package versions from the lock file, install editable version of feast and compile protobufs. ```sh -make compile-protos-python +make install-python-ci-dependencies-uv ``` - -11. Spin up Docker Image +10. Spin up Docker Image ```sh docker build -t docker-whale -f ./sdk/python/feast/infra/feature_servers/multicloud/Dockerfile . ``` diff --git a/infra/scripts/pixi/pixi.lock b/infra/scripts/pixi/pixi.lock index 65b761156e1..19a32f32ae8 100644 --- a/infra/scripts/pixi/pixi.lock +++ b/infra/scripts/pixi/pixi.lock @@ -1,6 +1,17 @@ version: 4 environments: default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda + py310: channels: - url: https://conda.anaconda.org/conda-forge/ packages: @@ -9,36 +20,25 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.3-hab00c5b_0_cpython.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.14-hd12c33a_0_cpython.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda - py310: + py311: channels: - url: https://conda.anaconda.org/conda-forge/ packages: @@ -47,34 +47,25 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h55db66e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-hc881cc4_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-hc881cc4_6.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.14-hd12c33a_0_cpython.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.3.0-hd590300_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.9-hb806964_0_cpython.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda py39: channels: - url: https://conda.anaconda.org/conda-forge/ @@ -84,34 +75,24 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.9.19-h0755675_0_cpython.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda packages: - kind: conda name: _libgcc_mutex @@ -168,53 +149,6 @@ packages: license: ISC size: 155432 timestamp: 1706843687645 -- kind: conda - name: click - version: 8.1.7 - build: unix_pyh707e725_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda - sha256: f0016cbab6ac4138a429e28dbcb904a90305b34b3fe41a9b89d697c90401caec - md5: f3ad426304898027fc619827ff428eca - depends: - - __unix - - python >=3.8 - license: BSD-3-Clause - license_family: BSD - size: 84437 - timestamp: 1692311973840 -- kind: conda - name: colorama - version: 0.4.6 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 - sha256: 2c1b2e9755ce3102bca8d69e8f26e4f087ece73f50418186aee7c74bef8e1698 - md5: 3faab06a954c2a04039983f2c4a50d99 - depends: - - python >=3.7 - license: BSD-3-Clause - license_family: BSD - size: 25170 - timestamp: 1666700778190 -- kind: conda - name: importlib-metadata - version: 7.1.0 - build: pyha770c72_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda - sha256: cc2e7d1f7f01cede30feafc1118b7aefa244d0a12224513734e24165ae12ba49 - md5: 0896606848b2dc5cebdf111b6543aa04 - depends: - - python >=3.8 - - zipp >=0.5 - license: Apache-2.0 - license_family: APACHE - size: 27043 - timestamp: 1710971498183 - kind: conda name: ld_impl_linux-64 version: '2.40' @@ -229,6 +163,20 @@ packages: license_family: GPL size: 704696 timestamp: 1674833944779 +- kind: conda + name: ld_impl_linux-64 + version: '2.40' + build: h55db66e_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h55db66e_0.conda + sha256: ef969eee228cfb71e55146eaecc6af065f468cb0bc0a5239bc053b39db0b5f09 + md5: 10569984e7db886e4f1abc2b47ad79a1 + constrains: + - binutils_impl_linux-64 2.40 + license: GPL-3.0-only + license_family: GPL + size: 713322 + timestamp: 1713651222435 - kind: conda name: libexpat version: 2.6.2 @@ -278,6 +226,24 @@ packages: license_family: GPL size: 770506 timestamp: 1706819192021 +- kind: conda + name: libgcc-ng + version: 13.2.0 + build: hc881cc4_6 + build_number: 6 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-hc881cc4_6.conda + sha256: 836a0057525f1414de43642d357d0ab21ac7f85e24800b010dbc17d132e6efec + md5: df88796bd09a0d2ed292e59101478ad8 + depends: + - _libgcc_mutex 0.1 conda_forge + - _openmp_mutex >=4.5 + constrains: + - libgomp 13.2.0 hc881cc4_6 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 777315 + timestamp: 1713755001744 - kind: conda name: libgomp version: 13.2.0 @@ -293,6 +259,21 @@ packages: license_family: GPL size: 419751 timestamp: 1706819107383 +- kind: conda + name: libgomp + version: 13.2.0 + build: hc881cc4_6 + build_number: 6 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-hc881cc4_6.conda + sha256: e722b19b23b31a14b1592d5eceabb38dc52452ff5e4d346e330526971c22e52a + md5: aae89d3736661c36a5591788aebd0817 + depends: + - _libgcc_mutex 0.1 conda_forge + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 422363 + timestamp: 1713754915251 - kind: conda name: libnsl version: 2.0.1 @@ -321,6 +302,19 @@ packages: license: Unlicense size: 859858 timestamp: 1713367435849 +- kind: conda + name: libstdcxx-ng + version: 13.2.0 + build: h95c4c6d_6 + build_number: 6 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda + sha256: 2616dbf9d28431eea20b6e307145c6a92ea0328a047c725ff34b0316de2617da + md5: 3cfab3e709f77e9f1b3d380eb622494a + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 3842900 + timestamp: 1713755068572 - kind: conda name: libuuid version: 2.38.1 @@ -398,73 +392,22 @@ packages: size: 2865379 timestamp: 1710793235846 - kind: conda - name: packaging - version: '24.0' - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda - sha256: a390182d74c31dfd713c16db888c92c277feeb6d1fe96ff9d9c105f9564be48a - md5: 248f521b64ce055e7feae3105e7abeb8 + name: openssl + version: 3.3.0 + build: hd590300_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.3.0-hd590300_0.conda + sha256: fdbf05e4db88c592366c90bb82e446edbe33c6e49e5130d51c580b2629c0b5d5 + md5: c0f3abb4a16477208bbd43a39bd56f18 depends: - - python >=3.8 + - ca-certificates + - libgcc-ng >=12 + constrains: + - pyopenssl >=22.1 license: Apache-2.0 - license_family: APACHE - size: 49832 - timestamp: 1710076089469 -- kind: conda - name: pip - version: '24.0' - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda - sha256: b7c1c5d8f13e8cb491c4bd1d0d1896a4cf80fc47de01059ad77509112b664a4a - md5: f586ac1e56c8638b64f9c8122a7b8a67 - depends: - - python >=3.7 - - setuptools - - wheel - license: MIT - license_family: MIT - size: 1398245 - timestamp: 1706960660581 -- kind: conda - name: pip-tools - version: 7.4.1 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda - sha256: 5534c19a6233faed1c9109782322c9d31e536ce20448f8c90db3d864fb8f226d - md5: 73203bd783da9c37c2cdabb1f3b9d44d - depends: - - click >=7 - - pip >=21.2 - - python >=3.7 - - python-build - - setuptools - - wheel - license: BSD-3-Clause - license_family: BSD - size: 54113 - timestamp: 1709736180083 -- kind: conda - name: pyproject_hooks - version: 1.0.0 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda - sha256: 016340837fcfef57b351febcbe855eedf0c1f0ecfc910ed48c7fbd20535f9847 - md5: 21de50391d584eb7f4441b9de1ad773f - depends: - - python >=3.7 - - tomli >=1.1.0 - license: MIT - license_family: MIT - size: 13867 - timestamp: 1670268791173 + license_family: Apache + size: 2895187 + timestamp: 1714466138265 - kind: conda name: python version: 3.9.19 @@ -525,12 +468,12 @@ packages: timestamp: 1710939725109 - kind: conda name: python - version: 3.12.3 - build: hab00c5b_0_cpython + version: 3.11.9 + build: hb806964_0_cpython subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.3-hab00c5b_0_cpython.conda - sha256: f9865bcbff69f15fd89a33a2da12ad616e98d65ce7c83c644b92e66e5016b227 - md5: 2540b74d304f71d3e89c81209db4db84 + url: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.9-hb806964_0_cpython.conda + sha256: 177f33a1fb8d3476b38f73c37b42f01c0b014fa0e039a701fd9f83d83aae6d40 + md5: ac68acfa8b558ed406c75e98d3428d7b depends: - bzip2 >=1.0.8,<2.0a0 - ld_impl_linux-64 >=2.36.1 @@ -538,7 +481,7 @@ packages: - libffi >=3.4,<4.0a0 - libgcc-ng >=12 - libnsl >=2.0.1,<2.1.0a0 - - libsqlite >=3.45.2,<4.0a0 + - libsqlite >=3.45.3,<4.0a0 - libuuid >=2.38.1,<3.0a0 - libxcrypt >=4.4.36 - libzlib >=1.2.13,<1.3.0a0 @@ -549,32 +492,10 @@ packages: - tzdata - xz >=5.2.6,<6.0a0 constrains: - - python_abi 3.12.* *_cp312 + - python_abi 3.11.* *_cp311 license: Python-2.0 - size: 31991381 - timestamp: 1713208036041 -- kind: conda - name: python-build - version: 1.2.1 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda - sha256: 3104051be7279d1b15f0a4be79f4bfeaf3a42b2900d24a7ad8e980df903fe8db - md5: d657cde3b3943fcedf6038138eea84de - depends: - - colorama - - importlib-metadata >=4.6 - - packaging >=19.0 - - pyproject_hooks - - python >=3.8 - - tomli >=1.1.0 - constrains: - - build <0 - license: MIT - license_family: MIT - size: 24434 - timestamp: 1711647439510 + size: 30884494 + timestamp: 1713553104915 - kind: conda name: readline version: '8.2' @@ -591,21 +512,6 @@ packages: license_family: GPL size: 281456 timestamp: 1679532220005 -- kind: conda - name: setuptools - version: 69.5.1 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda - sha256: 72d143408507043628b32bed089730b6d5f5445eccc44b59911ec9f262e365e7 - md5: 7462280d81f639363e6e63c81276bd9e - depends: - - python >=3.8 - license: MIT - license_family: MIT - size: 501790 - timestamp: 1713094963112 - kind: conda name: tk version: 8.6.13 @@ -622,21 +528,6 @@ packages: license_family: BSD size: 3318875 timestamp: 1699202167581 -- kind: conda - name: tomli - version: 2.0.1 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 - sha256: 4cd48aba7cd026d17e86886af48d0d2ebc67ed36f87f6534f4b67138f5a5a58f - md5: 5844808ffab9ebdb694585b50ba02a96 - depends: - - python >=3.7 - license: MIT - license_family: MIT - size: 15940 - timestamp: 1644342331069 - kind: conda name: tzdata version: 2024a @@ -650,21 +541,19 @@ packages: size: 119815 timestamp: 1706886945727 - kind: conda - name: wheel - version: 0.43.0 - build: pyhd8ed1ab_1 - build_number: 1 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda - sha256: cb318f066afd6fd64619f14c030569faf3f53e6f50abf743b4c865e7d95b96bc - md5: 0b5293a157c2b5cd513dd1b03d8d3aae + name: uv + version: 0.1.39 + build: h0ea3d13_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda + sha256: 763d149b6f4f5c70c91e4106d3a48409c48283ed2e27392578998fb2441f23d8 + md5: c3206e7ca254e50b3556917886f9b12b depends: - - python >=3.8 - license: MIT - license_family: MIT - size: 57963 - timestamp: 1711546009410 + - libgcc-ng >=12 + - libstdcxx-ng >=12 + license: Apache-2.0 OR MIT + size: 11891252 + timestamp: 1714233659570 - kind: conda name: xz version: 5.2.6 @@ -678,18 +567,3 @@ packages: license: LGPL-2.1 and GPL-2.0 size: 418368 timestamp: 1660346797927 -- kind: conda - name: zipp - version: 3.17.0 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda - sha256: bced1423fdbf77bca0a735187d05d9b9812d2163f60ab426fc10f11f92ecbe26 - md5: 2e4d6bc0b14e10f895fc6791a7d9b26a - depends: - - python >=3.8 - license: MIT - license_family: MIT - size: 18954 - timestamp: 1695255262261 diff --git a/infra/scripts/pixi/pixi.toml b/infra/scripts/pixi/pixi.toml index 80a29d3a59a..f0d360fff3d 100644 --- a/infra/scripts/pixi/pixi.toml +++ b/infra/scripts/pixi/pixi.toml @@ -6,7 +6,7 @@ platforms = ["linux-64"] [tasks] [dependencies] -pip-tools = ">=7.4.1,<7.5" +uv = ">=0.1.39,<0.2" [feature.py39.dependencies] python = "~=3.9.0" @@ -14,6 +14,10 @@ python = "~=3.9.0" [feature.py310.dependencies] python = "~=3.10.0" +[feature.py311.dependencies] +python = "~=3.11.0" + [environments] py39 = ["py39"] -py310 = ["py310"] \ No newline at end of file +py310 = ["py310"] +py311 = ["py311"] diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 2f17714964b..e7ca9ca35b6 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt -# +# This file was autogenerated by uv via the following command: +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt alabaster==0.7.16 # via sphinx altair==4.2.2 @@ -25,7 +21,6 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 - # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -43,10 +38,8 @@ azure-core==1.30.1 # azure-identity # azure-storage-blob azure-identity==1.16.0 - # via feast (setup.py) azure-storage-blob==12.19.1 - # via feast (setup.py) -babel==2.14.0 +babel==2.15.0 # via # jupyterlab-server # sphinx @@ -56,25 +49,20 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.88 - # via - # feast (setup.py) - # moto -botocore==1.34.88 +boto3==1.34.99 + # via moto +botocore==1.34.99 # via # boto3 # moto # s3transfer build==1.2.1 - # via - # feast (setup.py) - # pip-tools + # via pip-tools cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 - # via feast (setup.py) certifi==2024.2.2 # via # httpcore @@ -97,28 +85,25 @@ charset-normalizer==3.3.2 click==8.1.7 # via # dask - # feast (setup.py) # geomet # great-expectations # pip-tools + # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via - # feast (setup.py) - # great-expectations + # via great-expectations comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.4.4 +coverage[toml]==7.5.1 # via pytest-cov -cryptography==42.0.5 +cryptography==42.0.7 # via # azure-identity # azure-storage-blob - # feast (setup.py) # great-expectations # moto # msal @@ -127,11 +112,9 @@ cryptography==42.0.5 # snowflake-connector-python # types-pyopenssl # types-redis -dask[array,dataframe]==2024.4.2 - # via - # dask-expr - # feast (setup.py) -dask-expr==1.0.12 +dask[dataframe]==2024.5.0 + # via dask-expr +dask-expr==1.1.0 # via dask db-dtypes==1.2.0 # via google-cloud-bigquery @@ -141,24 +124,24 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.16.4 - # via feast (setup.py) +deltalake==0.17.3 dill==0.3.8 - # via feast (setup.py) distlib==0.3.8 # via virtualenv +dnspython==2.6.1 + # via email-validator docker==7.0.0 - # via - # feast (setup.py) - # testcontainers + # via testcontainers docutils==0.19 # via sphinx duckdb==0.10.2 # via # duckdb-engine # ibis-framework -duckdb-engine==0.11.5 +duckdb-engine==0.12.0 # via ibis-framework +email-validator==2.1.1 + # via fastapi entrypoints==0.4 # via altair exceptiongroup==1.2.1 @@ -170,29 +153,27 @@ execnet==2.1.1 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.110.2 - # via feast (setup.py) +fastapi==0.111.0 + # via fastapi-cli +fastapi-cli==0.0.2 + # via fastapi fastjsonschema==2.19.1 # via nbformat -filelock==3.13.4 +filelock==3.14.0 # via # snowflake-connector-python # virtualenv firebase-admin==5.4.0 - # via feast (setup.py) fqdn==1.5.1 # via jsonschema fsspec==2023.12.2 - # via - # dask - # feast (setup.py) + # via dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.18.0 +google-api-core[grpc]==2.19.0 # via - # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -202,13 +183,14 @@ google-api-core[grpc]==2.18.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.126.0 +google-api-python-client==2.128.0 # via firebase-admin google-auth==2.29.0 # via # google-api-core # google-api-python-client # google-auth-httplib2 + # google-cloud-bigquery-storage # google-cloud-core # google-cloud-firestore # google-cloud-storage @@ -216,11 +198,8 @@ google-auth==2.29.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via feast (setup.py) -google-cloud-bigquery-storage==2.24.0 - # via feast (setup.py) +google-cloud-bigquery-storage==2.25.0 google-cloud-bigtable==2.23.1 - # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -229,13 +208,10 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 - # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.16.0 - # via - # feast (setup.py) - # firebase-admin + # via firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -246,19 +222,16 @@ google-resumable-media==2.7.0 # google-cloud-storage googleapis-common-protos[grpc]==1.63.0 # via - # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.12 - # via feast (setup.py) +great-expectations==0.18.13 greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.62.2 +grpcio==1.63.0 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -269,27 +242,19 @@ grpcio==1.62.2 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 - # via feast (setup.py) grpcio-reflection==1.62.2 - # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 - # via feast (setup.py) grpcio-tools==1.62.2 - # via feast (setup.py) -gunicorn==22.0.0 ; platform_system != "Windows" - # via feast (setup.py) +gunicorn==22.0.0 h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 - # via feast (setup.py) hazelcast-python-client==5.3.0 - # via feast (setup.py) hiredis==2.3.2 - # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -300,19 +265,17 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via - # feast (setup.py) + # fastapi # jupyterlab ibis-framework[duckdb]==8.0.0 - # via - # feast (setup.py) - # ibis-substrait + # via ibis-substrait ibis-substrait==3.2.0 - # via feast (setup.py) -identify==2.5.35 +identify==2.5.36 # via pre-commit idna==3.7 # via # anyio + # email-validator # httpx # jsonschema # requests @@ -325,7 +288,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.4 # via jupyterlab -ipython==8.23.0 +ipython==8.24.0 # via # great-expectations # ipykernel @@ -341,7 +304,7 @@ jedi==0.19.1 jinja2==3.1.4 # via # altair - # feast (setup.py) + # fastapi # great-expectations # jupyter-server # jupyterlab @@ -361,10 +324,9 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.21.1 +jsonschema[format-nongpl]==4.22.0 # via # altair - # feast (setup.py) # great-expectations # jupyter-events # jupyterlab-server @@ -398,18 +360,17 @@ jupyter-server==2.14.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.6 +jupyterlab==4.1.8 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.26.0 +jupyterlab-server==2.27.1 # via # jupyterlab # notebook jupyterlab-widgets==3.0.10 # via ipywidgets kubernetes==20.13.0 - # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -421,7 +382,7 @@ markupsafe==2.1.5 # jinja2 # nbconvert # werkzeug -marshmallow==3.21.1 +marshmallow==3.21.2 # via great-expectations matplotlib-inline==0.1.7 # via @@ -430,17 +391,13 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 - # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 - # via feast (setup.py) mock==2.0.0 - # via feast (setup.py) moto==4.2.14 - # via feast (setup.py) msal==1.28.0 # via # azure-identity @@ -451,17 +408,14 @@ msgpack==1.0.8 # via cachecontrol multipledispatch==1.0.0 # via ibis-framework -mypy==1.9.0 - # via - # feast (setup.py) - # sqlalchemy +mypy==1.10.0 + # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 - # via feast (setup.py) nbclient==0.10.0 # via nbconvert -nbconvert==7.16.3 +nbconvert==7.16.4 # via jupyter-server nbformat==5.10.4 # via @@ -484,7 +438,6 @@ numpy==1.26.4 # altair # dask # db-dtypes - # feast (setup.py) # great-expectations # ibis-framework # pandas @@ -492,6 +445,8 @@ numpy==1.26.4 # scipy oauthlib==3.2.2 # via requests-oauthlib +orjson==3.10.3 + # via fastapi overrides==7.7.0 # via jupyter-server packaging==24.0 @@ -521,7 +476,6 @@ pandas==2.2.2 # dask # dask-expr # db-dtypes - # feast (setup.py) # google-cloud-bigquery # great-expectations # ibis-framework @@ -532,27 +486,27 @@ parso==0.8.4 # via jedi parsy==2.1 # via ibis-framework -partd==1.4.1 +partd==1.4.2 # via dask pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython +pip==24.0 + # via pip-tools pip-tools==7.4.1 - # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core # snowflake-connector-python # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via pytest ply==3.11 # via thriftpy2 portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 - # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.43 @@ -567,7 +521,6 @@ proto-plus==1.23.0 # google-cloud-firestore protobuf==4.25.3 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -585,11 +538,8 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via - # feast (setup.py) - # ipykernel + # via ipykernel psycopg2-binary==2.9.9 - # via feast (setup.py) ptyprocess==0.7.0 # via # pexpect @@ -597,7 +547,6 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 - # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 @@ -607,7 +556,6 @@ pyarrow==15.0.2 # dask-expr # db-dtypes # deltalake - # feast (setup.py) # google-cloud-bigquery # ibis-framework # snowflake-connector-python @@ -622,19 +570,16 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 - # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.7.0 +pydantic==2.7.1 # via # fastapi - # feast (setup.py) # great-expectations -pydantic-core==2.18.1 +pydantic-core==2.18.2 # via pydantic -pygments==2.17.2 +pygments==2.18.0 # via - # feast (setup.py) # ipython # nbconvert # rich @@ -644,26 +589,21 @@ pyjwt[crypto]==2.8.0 # msal # snowflake-connector-python pymssql==2.3.0 - # via feast (setup.py) pymysql==1.1.0 - # via feast (setup.py) pyodbc==5.1.0 - # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 # via # great-expectations # httplib2 -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via # build # pip-tools pyspark==3.5.1 - # via feast (setup.py) pytest==7.4.4 # via - # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -673,21 +613,13 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 - # via feast (setup.py) pytest-cov==5.0.0 - # via feast (setup.py) pytest-env==1.1.3 - # via feast (setup.py) pytest-lazy-fixture==0.6.3 - # via feast (setup.py) pytest-mock==1.10.4 - # via feast (setup.py) pytest-ordering==0.6 - # via feast (setup.py) pytest-timeout==1.4.2 - # via feast (setup.py) -pytest-xdist==3.6.0 - # via feast (setup.py) +pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 # via # arrow @@ -705,6 +637,8 @@ python-dotenv==1.0.1 # via uvicorn python-json-logger==2.0.7 # via jupyter-events +python-multipart==0.0.9 + # via fastapi pytz==2024.1 # via # great-expectations @@ -715,33 +649,29 @@ pytz==2024.1 pyyaml==6.0.1 # via # dask - # feast (setup.py) # ibis-substrait # jupyter-events # kubernetes # pre-commit # responses # uvicorn -pyzmq==26.0.2 +pyzmq==26.0.3 # via # ipykernel # jupyter-client # jupyter-server redis==4.6.0 - # via feast (setup.py) -referencing==0.34.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2024.4.16 - # via feast (setup.py) +regex==2024.4.28 requests==2.31.0 # via # azure-core # cachecontrol # docker - # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-storage @@ -768,10 +698,11 @@ rfc3986-validator==0.1.1 # jsonschema # jupyter-events rich==13.7.1 - # via ibis-framework -rockset==2.1.1 - # via feast (setup.py) -rpds-py==0.18.0 + # via + # ibis-framework + # typer +rockset==2.1.2 +rpds-py==0.18.1 # via # jsonschema # referencing @@ -779,14 +710,21 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruff==0.4.1 - # via feast (setup.py) +ruff==0.4.3 s3transfer==0.10.1 # via boto3 scipy==1.13.0 # via great-expectations send2trash==1.8.3 # via jupyter-server +setuptools==69.5.1 + # via + # grpcio-tools + # kubernetes + # nodeenv + # pip-tools +shellingham==1.5.4 + # via typer six==1.16.0 # via # asttokens @@ -806,14 +744,12 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.9.0 - # via feast (setup.py) +snowflake-connector-python[pandas]==3.10.0 sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -826,12 +762,10 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==2.0.29 +sqlalchemy[mypy]==2.0.30 # via # duckdb-engine - # feast (setup.py) # ibis-framework - # sqlalchemy # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework @@ -841,24 +775,20 @@ stack-data==0.6.3 # via ipython starlette==0.37.2 # via fastapi -substrait==0.16.0 +substrait==0.17.0 # via ibis-substrait tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) +tenacity==8.3.0 terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -testcontainers==4.3.3 - # via feast (setup.py) -thriftpy2==0.4.20 +testcontainers==4.4.0 +thriftpy2==0.5.0 # via happybase -tinycss2==1.2.1 +tinycss2==1.3.0 # via nbconvert toml==0.10.2 - # via feast (setup.py) tomli==2.0.1 # via # build @@ -866,7 +796,6 @@ tomli==2.0.1 # jupyterlab # mypy # pip-tools - # pyproject-hooks # pytest # pytest-env tomlkit==0.12.4 @@ -885,10 +814,8 @@ tornado==6.4 # jupyterlab # notebook # terminado -tqdm==4.66.3 - # via - # feast (setup.py) - # great-expectations +tqdm==4.66.4 + # via great-expectations traitlets==5.14.3 # via # comm @@ -905,37 +832,25 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 - # via feast (setup.py) typeguard==4.2.1 - # via feast (setup.py) +typer==0.12.3 + # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via - # feast (setup.py) - # mypy-protobuf -types-pymysql==1.1.0.1 - # via feast (setup.py) -types-pyopenssl==24.0.0.20240417 + # via mypy-protobuf +types-pymysql==1.1.0.20240425 +types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via - # arrow - # feast (setup.py) + # via arrow types-pytz==2024.1.0.20240417 - # via feast (setup.py) types-pyyaml==6.0.12.20240311 - # via feast (setup.py) -types-redis==4.6.0.20240417 - # via feast (setup.py) +types-redis==4.6.0.20240425 types-requests==2.30.0.0 - # via feast (setup.py) -types-setuptools==69.5.0.20240415 - # via - # feast (setup.py) - # types-cffi +types-setuptools==69.5.0.20240423 + # via types-cffi types-tabulate==0.9.0.20240106 - # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 @@ -955,6 +870,7 @@ typing-extensions==4.11.0 # sqlalchemy # testcontainers # typeguard + # typer # uvicorn tzdata==2024.1 # via pandas @@ -962,6 +878,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.9.0 + # via fastapi uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 @@ -970,7 +888,6 @@ urllib3==1.26.18 # via # botocore # docker - # feast (setup.py) # great-expectations # kubernetes # minio @@ -979,13 +896,13 @@ urllib3==1.26.18 # rockset # testcontainers uvicorn[standard]==0.29.0 - # via feast (setup.py) + # via + # fastapi + # fastapi-cli uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via - # feast (setup.py) - # pre-commit + # via pre-commit watchfiles==0.21.0 # via uvicorn wcwidth==0.2.13 @@ -996,7 +913,7 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.7.0 +websocket-client==1.8.0 # via # jupyter-server # kubernetes @@ -1014,7 +931,3 @@ xmltodict==0.13.0 # via moto zipp==3.18.1 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 0866a4e2168..99c9bfc3fee 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -1,13 +1,10 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt -# +# This file was autogenerated by uv via the following command: +# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via + # httpx # starlette # watchfiles attrs==23.2.0 @@ -15,70 +12,84 @@ attrs==23.2.0 # jsonschema # referencing certifi==2024.2.2 - # via requests + # via + # httpcore + # httpx + # requests charset-normalizer==3.3.2 # via requests click==8.1.7 # via # dask - # feast (setup.py) + # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via feast (setup.py) -dask[array,dataframe]==2024.4.2 - # via - # dask-expr - # feast (setup.py) -dask-expr==1.0.12 +dask[dataframe]==2024.5.0 + # via dask-expr +dask-expr==1.1.0 # via dask dill==0.3.8 - # via feast (setup.py) +dnspython==2.6.1 + # via email-validator +email-validator==2.1.1 + # via fastapi exceptiongroup==1.2.1 # via anyio -fastapi==0.110.2 - # via feast (setup.py) +fastapi==0.111.0 + # via fastapi-cli +fastapi-cli==0.0.2 + # via fastapi fsspec==2024.3.1 # via dask greenlet==3.0.3 # via sqlalchemy -gunicorn==22.0.0 ; platform_system != "Windows" - # via feast (setup.py) +gunicorn==22.0.0 h11==0.14.0 - # via uvicorn + # via + # httpcore + # uvicorn +httpcore==1.0.5 + # via httpx httptools==0.6.1 # via uvicorn +httpx==0.27.0 + # via fastapi idna==3.7 # via # anyio + # email-validator + # httpx # requests importlib-metadata==7.1.0 # via dask jinja2==3.1.4 - # via feast (setup.py) -jsonschema==4.21.1 - # via feast (setup.py) + # via fastapi +jsonschema==4.22.0 jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 # via partd +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via jinja2 +mdurl==0.1.2 + # via markdown-it-py mmh3==4.1.0 - # via feast (setup.py) -mypy==1.9.0 +mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 - # via feast (setup.py) numpy==1.26.4 # via # dask - # feast (setup.py) # pandas # pyarrow +orjson==3.10.3 + # via fastapi packaging==24.0 # via # dask @@ -87,73 +98,66 @@ pandas==2.2.2 # via # dask # dask-expr - # feast (setup.py) -partd==1.4.1 +partd==1.4.2 # via dask protobuf==4.25.3 - # via - # feast (setup.py) - # mypy-protobuf -pyarrow==15.0.2 - # via - # dask-expr - # feast (setup.py) -pydantic==2.7.0 - # via - # fastapi - # feast (setup.py) -pydantic-core==2.18.1 + # via mypy-protobuf +pyarrow==16.0.0 + # via dask-expr +pydantic==2.7.1 + # via fastapi +pydantic-core==2.18.2 # via pydantic -pygments==2.17.2 - # via feast (setup.py) +pygments==2.18.0 + # via rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 # via uvicorn +python-multipart==0.0.9 + # via fastapi pytz==2024.1 # via pandas pyyaml==6.0.1 # via # dask - # feast (setup.py) # uvicorn -referencing==0.34.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications requests==2.31.0 - # via feast (setup.py) -rpds-py==0.18.0 +rich==13.7.1 + # via typer +rpds-py==0.18.1 # via # jsonschema # referencing +shellingham==1.5.4 + # via typer six==1.16.0 # via python-dateutil sniffio==1.3.1 - # via anyio -sqlalchemy[mypy]==2.0.29 # via - # feast (setup.py) - # sqlalchemy + # anyio + # httpx +sqlalchemy[mypy]==2.0.30 starlette==0.37.2 # via fastapi tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) +tenacity==8.3.0 toml==0.10.2 - # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 # via # dask # partd -tqdm==4.66.3 - # via feast (setup.py) +tqdm==4.66.4 typeguard==4.2.1 - # via feast (setup.py) -types-protobuf==5.26.0.20240420 +typer==0.12.3 + # via fastapi-cli +types-protobuf==5.26.0.20240422 # via mypy-protobuf typing-extensions==4.11.0 # via @@ -164,13 +168,18 @@ typing-extensions==4.11.0 # pydantic-core # sqlalchemy # typeguard + # typer # uvicorn tzdata==2024.1 # via pandas +ujson==5.9.0 + # via fastapi urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 - # via feast (setup.py) + # via + # fastapi + # fastapi-cli uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 8c9a1325483..3b76237f599 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --extra=ci --output-file=sdk/python/requirements/py3.11-ci-requirements.txt -# +# This file was autogenerated by uv via the following command: +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt alabaster==0.7.16 # via sphinx altair==4.2.2 @@ -16,10 +12,6 @@ anyio==4.3.0 # jupyter-server # starlette # watchfiles -appdirs==1.4.4 - # via fissix -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -29,18 +21,14 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 - # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab -async-timeout==4.0.3 - # via redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 # via - # bowler # jsonschema # referencing azure-core==1.30.1 @@ -48,10 +36,8 @@ azure-core==1.30.1 # azure-identity # azure-storage-blob azure-identity==1.16.0 - # via feast (setup.py) azure-storage-blob==12.19.1 - # via feast (setup.py) -babel==2.14.0 +babel==2.15.0 # via # jupyterlab-server # sphinx @@ -61,27 +47,20 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.93 - # via - # feast (setup.py) - # moto -botocore==1.34.93 +boto3==1.34.99 + # via moto +botocore==1.34.99 # via # boto3 # moto # s3transfer -bowler==0.9.0 - # via feast (setup.py) build==1.2.1 - # via - # feast (setup.py) - # pip-tools + # via pip-tools cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 - # via feast (setup.py) certifi==2024.2.2 # via # httpcore @@ -103,31 +82,26 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via - # bowler # dask - # feast (setup.py) # geomet # great-expectations - # moreorless # pip-tools + # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via - # feast (setup.py) - # great-expectations + # via great-expectations comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.5.0 +coverage[toml]==7.5.1 # via pytest-cov -cryptography==42.0.5 +cryptography==42.0.7 # via # azure-identity # azure-storage-blob - # feast (setup.py) # great-expectations # moto # msal @@ -136,11 +110,9 @@ cryptography==42.0.5 # snowflake-connector-python # types-pyopenssl # types-redis -dask[array,dataframe]==2024.4.2 - # via - # dask-expr - # feast (setup.py) -dask-expr==1.0.13 +dask[dataframe]==2024.5.0 + # via dask-expr +dask-expr==1.1.0 # via dask db-dtypes==1.2.0 # via google-cloud-bigquery @@ -150,16 +122,14 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.17.2 - # via feast (setup.py) +deltalake==0.17.3 dill==0.3.8 - # via feast (setup.py) distlib==0.3.8 # via virtualenv +dnspython==2.6.1 + # via email-validator docker==7.0.0 - # via - # feast (setup.py) - # testcontainers + # via testcontainers docutils==0.19 # via sphinx duckdb==0.10.2 @@ -168,19 +138,18 @@ duckdb==0.10.2 # ibis-framework duckdb-engine==0.12.0 # via ibis-framework +email-validator==2.1.1 + # via fastapi entrypoints==0.4 # via altair -exceptiongroup==1.2.1 - # via - # anyio - # ipython - # pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.110.2 - # via feast (setup.py) +fastapi==0.111.0 + # via fastapi-cli +fastapi-cli==0.0.2 + # via fastapi fastjsonschema==2.19.1 # via nbformat filelock==3.14.0 @@ -188,22 +157,16 @@ filelock==3.14.0 # snowflake-connector-python # virtualenv firebase-admin==5.4.0 - # via feast (setup.py) -fissix==24.4.24 - # via bowler fqdn==1.5.1 # via jsonschema fsspec==2023.12.2 - # via - # dask - # feast (setup.py) + # via dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.18.0 +google-api-core[grpc]==2.19.0 # via - # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -213,13 +176,14 @@ google-api-core[grpc]==2.18.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.127.0 +google-api-python-client==2.128.0 # via firebase-admin google-auth==2.29.0 # via # google-api-core # google-api-python-client # google-auth-httplib2 + # google-cloud-bigquery-storage # google-cloud-core # google-cloud-firestore # google-cloud-storage @@ -227,11 +191,8 @@ google-auth==2.29.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via feast (setup.py) -google-cloud-bigquery-storage==2.24.0 - # via feast (setup.py) +google-cloud-bigquery-storage==2.25.0 google-cloud-bigtable==2.23.1 - # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -240,13 +201,10 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 - # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.16.0 - # via - # feast (setup.py) - # firebase-admin + # via firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -257,19 +215,16 @@ google-resumable-media==2.7.0 # google-cloud-storage googleapis-common-protos[grpc]==1.63.0 # via - # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.12 - # via feast (setup.py) +great-expectations==0.18.13 greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.62.2 +grpcio==1.63.0 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -280,27 +235,19 @@ grpcio==1.62.2 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 - # via feast (setup.py) grpcio-reflection==1.62.2 - # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 - # via feast (setup.py) grpcio-tools==1.62.2 - # via feast (setup.py) -gunicorn==22.0.0 ; platform_system != "Windows" - # via feast (setup.py) +gunicorn==22.0.0 h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 - # via feast (setup.py) hazelcast-python-client==5.3.0 - # via feast (setup.py) hiredis==2.3.2 - # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -311,19 +258,17 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via - # feast (setup.py) + # fastapi # jupyterlab ibis-framework[duckdb]==8.0.0 - # via - # feast (setup.py) - # ibis-substrait + # via ibis-substrait ibis-substrait==3.2.0 - # via feast (setup.py) identify==2.5.36 # via pre-commit idna==3.7 # via # anyio + # email-validator # httpx # jsonschema # requests @@ -331,21 +276,12 @@ idna==3.7 imagesize==1.4.1 # via sphinx importlib-metadata==7.1.0 - # via - # build - # dask - # jupyter-client - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # nbconvert - # sphinx - # typeguard + # via dask iniconfig==2.0.0 # via pytest ipykernel==6.29.4 # via jupyterlab -ipython==8.18.1 +ipython==8.24.0 # via # great-expectations # ipykernel @@ -361,7 +297,7 @@ jedi==0.19.1 jinja2==3.1.4 # via # altair - # feast (setup.py) + # fastapi # great-expectations # jupyter-server # jupyterlab @@ -381,10 +317,9 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.21.1 +jsonschema[format-nongpl]==4.22.0 # via # altair - # feast (setup.py) # great-expectations # jupyter-events # jupyterlab-server @@ -429,7 +364,6 @@ jupyterlab-server==2.27.1 jupyterlab-widgets==3.0.10 # via ipywidgets kubernetes==20.13.0 - # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -441,7 +375,7 @@ markupsafe==2.1.5 # jinja2 # nbconvert # werkzeug -marshmallow==3.21.1 +marshmallow==3.21.2 # via great-expectations matplotlib-inline==0.1.7 # via @@ -450,19 +384,13 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 - # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 - # via feast (setup.py) mock==2.0.0 - # via feast (setup.py) -moreorless==0.4.0 - # via bowler moto==4.2.14 - # via feast (setup.py) msal==1.28.0 # via # azure-identity @@ -474,13 +402,10 @@ msgpack==1.0.8 multipledispatch==1.0.0 # via ibis-framework mypy==1.10.0 - # via - # feast (setup.py) - # sqlalchemy + # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 - # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -506,7 +431,6 @@ numpy==1.26.4 # altair # dask # db-dtypes - # feast (setup.py) # great-expectations # ibis-framework # pandas @@ -514,6 +438,8 @@ numpy==1.26.4 # scipy oauthlib==3.2.2 # via requests-oauthlib +orjson==3.10.3 + # via fastapi overrides==7.7.0 # via jupyter-server packaging==24.0 @@ -543,7 +469,6 @@ pandas==2.2.2 # dask # dask-expr # db-dtypes - # feast (setup.py) # google-cloud-bigquery # great-expectations # ibis-framework @@ -554,14 +479,15 @@ parso==0.8.4 # via jedi parsy==2.1 # via ibis-framework -partd==1.4.1 +partd==1.4.2 # via dask pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython +pip==24.0 + # via pip-tools pip-tools==7.4.1 - # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -574,7 +500,6 @@ ply==3.11 portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 - # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.43 @@ -589,7 +514,6 @@ proto-plus==1.23.0 # google-cloud-firestore protobuf==4.25.3 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -607,11 +531,8 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via - # feast (setup.py) - # ipykernel + # via ipykernel psycopg2-binary==2.9.9 - # via feast (setup.py) ptyprocess==0.7.0 # via # pexpect @@ -619,7 +540,6 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 - # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 @@ -629,7 +549,6 @@ pyarrow==15.0.2 # dask-expr # db-dtypes # deltalake - # feast (setup.py) # google-cloud-bigquery # ibis-framework # snowflake-connector-python @@ -644,19 +563,16 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 - # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.7.1 # via # fastapi - # feast (setup.py) # great-expectations pydantic-core==2.18.2 # via pydantic -pygments==2.17.2 +pygments==2.18.0 # via - # feast (setup.py) # ipython # nbconvert # rich @@ -666,11 +582,8 @@ pyjwt[crypto]==2.8.0 # msal # snowflake-connector-python pymssql==2.3.0 - # via feast (setup.py) pymysql==1.1.0 - # via feast (setup.py) pyodbc==5.1.0 - # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -682,10 +595,8 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 - # via feast (setup.py) pytest==7.4.4 # via - # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -695,21 +606,13 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 - # via feast (setup.py) pytest-cov==5.0.0 - # via feast (setup.py) pytest-env==1.1.3 - # via feast (setup.py) pytest-lazy-fixture==0.6.3 - # via feast (setup.py) pytest-mock==1.10.4 - # via feast (setup.py) pytest-ordering==0.6 - # via feast (setup.py) pytest-timeout==1.4.2 - # via feast (setup.py) pytest-xdist==3.6.1 - # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -727,6 +630,8 @@ python-dotenv==1.0.1 # via uvicorn python-json-logger==2.0.7 # via jupyter-events +python-multipart==0.0.9 + # via fastapi pytz==2024.1 # via # great-expectations @@ -737,33 +642,29 @@ pytz==2024.1 pyyaml==6.0.1 # via # dask - # feast (setup.py) # ibis-substrait # jupyter-events # kubernetes # pre-commit # responses # uvicorn -pyzmq==26.0.2 +pyzmq==26.0.3 # via # ipykernel # jupyter-client # jupyter-server redis==4.6.0 - # via feast (setup.py) -referencing==0.35.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.4.28 - # via feast (setup.py) requests==2.31.0 # via # azure-core # cachecontrol # docker - # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-storage @@ -790,10 +691,11 @@ rfc3986-validator==0.1.1 # jsonschema # jupyter-events rich==13.7.1 - # via ibis-framework -rockset==2.1.1 - # via feast (setup.py) -rpds-py==0.18.0 + # via + # ibis-framework + # typer +rockset==2.1.2 +rpds-py==0.18.1 # via # jsonschema # referencing @@ -801,16 +703,21 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruamel-yaml-clib==0.2.8 - # via ruamel-yaml -ruff==0.4.2 - # via feast (setup.py) +ruff==0.4.3 s3transfer==0.10.1 # via boto3 scipy==1.13.0 # via great-expectations send2trash==1.8.3 # via jupyter-server +setuptools==69.5.1 + # via + # grpcio-tools + # kubernetes + # nodeenv + # pip-tools +shellingham==1.5.4 + # via typer six==1.16.0 # via # asttokens @@ -830,14 +737,12 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.9.1 - # via feast (setup.py) +snowflake-connector-python[pandas]==3.10.0 sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -850,12 +755,10 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==2.0.29 +sqlalchemy[mypy]==2.0.30 # via # duckdb-engine - # feast (setup.py) # ibis-framework - # sqlalchemy # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework @@ -868,30 +771,17 @@ starlette==0.37.2 substrait==0.17.0 # via ibis-substrait tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) +tenacity==8.3.0 terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 - # via feast (setup.py) -thriftpy2==0.4.20 +thriftpy2==0.5.0 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 - # via feast (setup.py) -tomli==2.0.1 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pytest - # pytest-env tomlkit==0.12.4 # via snowflake-connector-python toolz==0.12.1 @@ -908,10 +798,8 @@ tornado==6.4 # jupyterlab # notebook # terminado -tqdm==4.66.3 - # via - # feast (setup.py) - # great-expectations +tqdm==4.66.4 + # via great-expectations traitlets==5.14.3 # via # comm @@ -928,43 +816,29 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 - # via feast (setup.py) typeguard==4.2.1 - # via feast (setup.py) +typer==0.12.3 + # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via - # feast (setup.py) - # mypy-protobuf + # via mypy-protobuf types-pymysql==1.1.0.20240425 - # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via - # arrow - # feast (setup.py) + # via arrow types-pytz==2024.1.0.20240417 - # via feast (setup.py) types-pyyaml==6.0.12.20240311 - # via feast (setup.py) types-redis==4.6.0.20240425 - # via feast (setup.py) types-requests==2.30.0.0 - # via feast (setup.py) types-setuptools==69.5.0.20240423 - # via - # feast (setup.py) - # types-cffi + # via types-cffi types-tabulate==0.9.0.20240106 - # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via - # anyio - # async-lru # azure-core # azure-storage-blob # fastapi @@ -976,16 +850,17 @@ typing-extensions==4.11.0 # pydantic-core # snowflake-connector-python # sqlalchemy - # starlette # testcontainers # typeguard - # uvicorn + # typer tzdata==2024.1 # via pandas tzlocal==5.2 # via # great-expectations # trino +ujson==5.9.0 + # via fastapi uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 @@ -994,25 +869,21 @@ urllib3==1.26.18 # via # botocore # docker - # feast (setup.py) # great-expectations # kubernetes # minio # requests # responses # rockset - # snowflake-connector-python # testcontainers uvicorn[standard]==0.29.0 - # via feast (setup.py) + # via + # fastapi + # fastapi-cli uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via - # feast (setup.py) - # pre-commit -volatile==2.1.0 - # via bowler + # via pre-commit watchfiles==0.21.0 # via uvicorn wcwidth==0.2.13 @@ -1041,7 +912,3 @@ xmltodict==0.13.0 # via moto zipp==3.18.1 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 8431afc663f..c34b610d14c 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -1,97 +1,93 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --output-file=sdk/python/requirements/py3.11-requirements.txt -# +# This file was autogenerated by uv via the following command: +# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via + # httpx # starlette # watchfiles -appdirs==1.4.4 - # via fissix attrs==23.2.0 # via - # bowler # jsonschema # referencing -bowler==0.9.0 - # via feast (setup.py) certifi==2024.2.2 - # via requests + # via + # httpcore + # httpx + # requests charset-normalizer==3.3.2 # via requests click==8.1.7 # via - # bowler # dask - # feast (setup.py) - # moreorless + # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via feast (setup.py) -dask[array,dataframe]==2024.4.2 - # via - # dask-expr - # feast (setup.py) -dask-expr==1.0.13 +dask[dataframe]==2024.5.0 + # via dask-expr +dask-expr==1.1.0 # via dask dill==0.3.8 - # via feast (setup.py) -exceptiongroup==1.2.1 - # via anyio -fastapi==0.110.2 - # via feast (setup.py) -fissix==24.4.24 - # via bowler +dnspython==2.6.1 + # via email-validator +email-validator==2.1.1 + # via fastapi +fastapi==0.111.0 + # via fastapi-cli +fastapi-cli==0.0.2 + # via fastapi fsspec==2024.3.1 # via dask greenlet==3.0.3 # via sqlalchemy -gunicorn==22.0.0 ; platform_system != "Windows" - # via feast (setup.py) +gunicorn==22.0.0 h11==0.14.0 - # via uvicorn + # via + # httpcore + # uvicorn +httpcore==1.0.5 + # via httpx httptools==0.6.1 # via uvicorn +httpx==0.27.0 + # via fastapi idna==3.7 # via # anyio + # email-validator + # httpx # requests importlib-metadata==7.1.0 - # via - # dask - # typeguard + # via dask jinja2==3.1.4 - # via feast (setup.py) -jsonschema==4.21.1 - # via feast (setup.py) + # via fastapi +jsonschema==4.22.0 jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 # via partd +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via jinja2 +mdurl==0.1.2 + # via markdown-it-py mmh3==4.1.0 - # via feast (setup.py) -moreorless==0.4.0 - # via bowler mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 - # via feast (setup.py) numpy==1.26.4 # via # dask - # feast (setup.py) # pandas # pyarrow +orjson==3.10.3 + # via fastapi packaging==24.0 # via # dask @@ -100,95 +96,86 @@ pandas==2.2.2 # via # dask # dask-expr - # feast (setup.py) -partd==1.4.1 +partd==1.4.2 # via dask protobuf==4.25.3 - # via - # feast (setup.py) - # mypy-protobuf + # via mypy-protobuf pyarrow==16.0.0 - # via - # dask-expr - # feast (setup.py) + # via dask-expr pydantic==2.7.1 - # via - # fastapi - # feast (setup.py) + # via fastapi pydantic-core==2.18.2 # via pydantic -pygments==2.17.2 - # via feast (setup.py) +pygments==2.18.0 + # via rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 # via uvicorn +python-multipart==0.0.9 + # via fastapi pytz==2024.1 # via pandas pyyaml==6.0.1 # via # dask - # feast (setup.py) # uvicorn -referencing==0.35.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications requests==2.31.0 - # via feast (setup.py) -rpds-py==0.18.0 +rich==13.7.1 + # via typer +rpds-py==0.18.1 # via # jsonschema # referencing +shellingham==1.5.4 + # via typer six==1.16.0 # via python-dateutil sniffio==1.3.1 - # via anyio -sqlalchemy[mypy]==2.0.29 # via - # feast (setup.py) - # sqlalchemy + # anyio + # httpx +sqlalchemy[mypy]==2.0.30 starlette==0.37.2 # via fastapi tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) +tenacity==8.3.0 toml==0.10.2 - # via feast (setup.py) -tomli==2.0.1 - # via mypy toolz==0.12.1 # via # dask # partd -tqdm==4.66.3 - # via feast (setup.py) +tqdm==4.66.4 typeguard==4.2.1 - # via feast (setup.py) +typer==0.12.3 + # via fastapi-cli types-protobuf==5.26.0.20240422 # via mypy-protobuf typing-extensions==4.11.0 # via - # anyio # fastapi # mypy # pydantic # pydantic-core # sqlalchemy - # starlette # typeguard - # uvicorn + # typer tzdata==2024.1 # via pandas +ujson==5.9.0 + # via fastapi urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 - # via feast (setup.py) + # via + # fastapi + # fastapi-cli uvloop==0.19.0 # via uvicorn -volatile==2.1.0 - # via bowler watchfiles==0.21.0 # via uvicorn websockets==12.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 874ef1d58df..a628f0823db 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt -# +# This file was autogenerated by uv via the following command: +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt alabaster==0.7.16 # via sphinx altair==4.2.2 @@ -25,7 +21,6 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 - # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -43,10 +38,8 @@ azure-core==1.30.1 # azure-identity # azure-storage-blob azure-identity==1.16.0 - # via feast (setup.py) azure-storage-blob==12.19.1 - # via feast (setup.py) -babel==2.14.0 +babel==2.15.0 # via # jupyterlab-server # sphinx @@ -56,25 +49,20 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.88 - # via - # feast (setup.py) - # moto -botocore==1.34.88 +boto3==1.34.99 + # via moto +botocore==1.34.99 # via # boto3 # moto # s3transfer build==1.2.1 - # via - # feast (setup.py) - # pip-tools + # via pip-tools cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 - # via feast (setup.py) certifi==2024.2.2 # via # httpcore @@ -97,28 +85,25 @@ charset-normalizer==3.3.2 click==8.1.7 # via # dask - # feast (setup.py) # geomet # great-expectations # pip-tools + # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via - # feast (setup.py) - # great-expectations + # via great-expectations comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.4.4 +coverage[toml]==7.5.1 # via pytest-cov -cryptography==42.0.5 +cryptography==42.0.7 # via # azure-identity # azure-storage-blob - # feast (setup.py) # great-expectations # moto # msal @@ -127,11 +112,9 @@ cryptography==42.0.5 # snowflake-connector-python # types-pyopenssl # types-redis -dask[array,dataframe]==2024.4.2 - # via - # dask-expr - # feast (setup.py) -dask-expr==1.0.12 +dask[dataframe]==2024.5.0 + # via dask-expr +dask-expr==1.1.0 # via dask db-dtypes==1.2.0 # via google-cloud-bigquery @@ -141,24 +124,24 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.16.4 - # via feast (setup.py) +deltalake==0.17.3 dill==0.3.8 - # via feast (setup.py) distlib==0.3.8 # via virtualenv +dnspython==2.6.1 + # via email-validator docker==7.0.0 - # via - # feast (setup.py) - # testcontainers + # via testcontainers docutils==0.19 # via sphinx duckdb==0.10.2 # via # duckdb-engine # ibis-framework -duckdb-engine==0.11.5 +duckdb-engine==0.12.0 # via ibis-framework +email-validator==2.1.1 + # via fastapi entrypoints==0.4 # via altair exceptiongroup==1.2.1 @@ -170,29 +153,27 @@ execnet==2.1.1 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.110.2 - # via feast (setup.py) +fastapi==0.111.0 + # via fastapi-cli +fastapi-cli==0.0.2 + # via fastapi fastjsonschema==2.19.1 # via nbformat -filelock==3.13.4 +filelock==3.14.0 # via # snowflake-connector-python # virtualenv firebase-admin==5.4.0 - # via feast (setup.py) fqdn==1.5.1 # via jsonschema fsspec==2023.12.2 - # via - # dask - # feast (setup.py) + # via dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.18.0 +google-api-core[grpc]==2.19.0 # via - # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -202,13 +183,14 @@ google-api-core[grpc]==2.18.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.126.0 +google-api-python-client==2.128.0 # via firebase-admin google-auth==2.29.0 # via # google-api-core # google-api-python-client # google-auth-httplib2 + # google-cloud-bigquery-storage # google-cloud-core # google-cloud-firestore # google-cloud-storage @@ -216,11 +198,8 @@ google-auth==2.29.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via feast (setup.py) -google-cloud-bigquery-storage==2.24.0 - # via feast (setup.py) +google-cloud-bigquery-storage==2.25.0 google-cloud-bigtable==2.23.1 - # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -229,13 +208,10 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 - # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.16.0 - # via - # feast (setup.py) - # firebase-admin + # via firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -246,19 +222,16 @@ google-resumable-media==2.7.0 # google-cloud-storage googleapis-common-protos[grpc]==1.63.0 # via - # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.12 - # via feast (setup.py) +great-expectations==0.18.13 greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.62.2 +grpcio==1.63.0 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -269,27 +242,19 @@ grpcio==1.62.2 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 - # via feast (setup.py) grpcio-reflection==1.62.2 - # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 - # via feast (setup.py) grpcio-tools==1.62.2 - # via feast (setup.py) -gunicorn==22.0.0 ; platform_system != "Windows" - # via feast (setup.py) +gunicorn==22.0.0 h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 - # via feast (setup.py) hazelcast-python-client==5.3.0 - # via feast (setup.py) hiredis==2.3.2 - # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -300,19 +265,17 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via - # feast (setup.py) + # fastapi # jupyterlab ibis-framework[duckdb]==8.0.0 - # via - # feast (setup.py) - # ibis-substrait + # via ibis-substrait ibis-substrait==3.2.0 - # via feast (setup.py) -identify==2.5.35 +identify==2.5.36 # via pre-commit idna==3.7 # via # anyio + # email-validator # httpx # jsonschema # requests @@ -350,7 +313,7 @@ jedi==0.19.1 jinja2==3.1.4 # via # altair - # feast (setup.py) + # fastapi # great-expectations # jupyter-server # jupyterlab @@ -370,10 +333,9 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.21.1 +jsonschema[format-nongpl]==4.22.0 # via # altair - # feast (setup.py) # great-expectations # jupyter-events # jupyterlab-server @@ -407,18 +369,17 @@ jupyter-server==2.14.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.6 +jupyterlab==4.1.8 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.26.0 +jupyterlab-server==2.27.1 # via # jupyterlab # notebook jupyterlab-widgets==3.0.10 # via ipywidgets kubernetes==20.13.0 - # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -430,7 +391,7 @@ markupsafe==2.1.5 # jinja2 # nbconvert # werkzeug -marshmallow==3.21.1 +marshmallow==3.21.2 # via great-expectations matplotlib-inline==0.1.7 # via @@ -439,17 +400,13 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 - # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 - # via feast (setup.py) mock==2.0.0 - # via feast (setup.py) moto==4.2.14 - # via feast (setup.py) msal==1.28.0 # via # azure-identity @@ -460,17 +417,14 @@ msgpack==1.0.8 # via cachecontrol multipledispatch==1.0.0 # via ibis-framework -mypy==1.9.0 - # via - # feast (setup.py) - # sqlalchemy +mypy==1.10.0 + # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 - # via feast (setup.py) nbclient==0.10.0 # via nbconvert -nbconvert==7.16.3 +nbconvert==7.16.4 # via jupyter-server nbformat==5.10.4 # via @@ -493,7 +447,6 @@ numpy==1.26.4 # altair # dask # db-dtypes - # feast (setup.py) # great-expectations # ibis-framework # pandas @@ -501,6 +454,8 @@ numpy==1.26.4 # scipy oauthlib==3.2.2 # via requests-oauthlib +orjson==3.10.3 + # via fastapi overrides==7.7.0 # via jupyter-server packaging==24.0 @@ -530,7 +485,6 @@ pandas==2.2.2 # dask # dask-expr # db-dtypes - # feast (setup.py) # google-cloud-bigquery # great-expectations # ibis-framework @@ -541,27 +495,27 @@ parso==0.8.4 # via jedi parsy==2.1 # via ibis-framework -partd==1.4.1 +partd==1.4.2 # via dask pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython +pip==24.0 + # via pip-tools pip-tools==7.4.1 - # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core # snowflake-connector-python # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via pytest ply==3.11 # via thriftpy2 portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 - # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.43 @@ -576,7 +530,6 @@ proto-plus==1.23.0 # google-cloud-firestore protobuf==4.25.3 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -594,11 +547,8 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via - # feast (setup.py) - # ipykernel + # via ipykernel psycopg2-binary==2.9.9 - # via feast (setup.py) ptyprocess==0.7.0 # via # pexpect @@ -606,7 +556,6 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 - # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 @@ -616,7 +565,6 @@ pyarrow==15.0.2 # dask-expr # db-dtypes # deltalake - # feast (setup.py) # google-cloud-bigquery # ibis-framework # snowflake-connector-python @@ -631,19 +579,16 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 - # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.7.0 +pydantic==2.7.1 # via # fastapi - # feast (setup.py) # great-expectations -pydantic-core==2.18.1 +pydantic-core==2.18.2 # via pydantic -pygments==2.17.2 +pygments==2.18.0 # via - # feast (setup.py) # ipython # nbconvert # rich @@ -653,26 +598,21 @@ pyjwt[crypto]==2.8.0 # msal # snowflake-connector-python pymssql==2.3.0 - # via feast (setup.py) pymysql==1.1.0 - # via feast (setup.py) pyodbc==5.1.0 - # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 # via # great-expectations # httplib2 -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via # build # pip-tools pyspark==3.5.1 - # via feast (setup.py) pytest==7.4.4 # via - # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -682,21 +622,13 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 - # via feast (setup.py) pytest-cov==5.0.0 - # via feast (setup.py) pytest-env==1.1.3 - # via feast (setup.py) pytest-lazy-fixture==0.6.3 - # via feast (setup.py) pytest-mock==1.10.4 - # via feast (setup.py) pytest-ordering==0.6 - # via feast (setup.py) pytest-timeout==1.4.2 - # via feast (setup.py) -pytest-xdist==3.6.0 - # via feast (setup.py) +pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 # via # arrow @@ -714,6 +646,8 @@ python-dotenv==1.0.1 # via uvicorn python-json-logger==2.0.7 # via jupyter-events +python-multipart==0.0.9 + # via fastapi pytz==2024.1 # via # great-expectations @@ -724,33 +658,29 @@ pytz==2024.1 pyyaml==6.0.1 # via # dask - # feast (setup.py) # ibis-substrait # jupyter-events # kubernetes # pre-commit # responses # uvicorn -pyzmq==26.0.2 +pyzmq==26.0.3 # via # ipykernel # jupyter-client # jupyter-server redis==4.6.0 - # via feast (setup.py) -referencing==0.34.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2024.4.16 - # via feast (setup.py) +regex==2024.4.28 requests==2.31.0 # via # azure-core # cachecontrol # docker - # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-storage @@ -777,10 +707,11 @@ rfc3986-validator==0.1.1 # jsonschema # jupyter-events rich==13.7.1 - # via ibis-framework -rockset==2.1.1 - # via feast (setup.py) -rpds-py==0.18.0 + # via + # ibis-framework + # typer +rockset==2.1.2 +rpds-py==0.18.1 # via # jsonschema # referencing @@ -790,14 +721,21 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml -ruff==0.4.1 - # via feast (setup.py) +ruff==0.4.3 s3transfer==0.10.1 # via boto3 scipy==1.13.0 # via great-expectations send2trash==1.8.3 # via jupyter-server +setuptools==69.5.1 + # via + # grpcio-tools + # kubernetes + # nodeenv + # pip-tools +shellingham==1.5.4 + # via typer six==1.16.0 # via # asttokens @@ -817,14 +755,12 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.9.0 - # via feast (setup.py) +snowflake-connector-python[pandas]==3.10.0 sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -837,12 +773,10 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==2.0.29 +sqlalchemy[mypy]==2.0.30 # via # duckdb-engine - # feast (setup.py) # ibis-framework - # sqlalchemy # sqlalchemy-views sqlalchemy-views==0.3.2 # via ibis-framework @@ -852,24 +786,20 @@ stack-data==0.6.3 # via ipython starlette==0.37.2 # via fastapi -substrait==0.16.0 +substrait==0.17.0 # via ibis-substrait tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) +tenacity==8.3.0 terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -testcontainers==4.3.3 - # via feast (setup.py) -thriftpy2==0.4.20 +testcontainers==4.4.0 +thriftpy2==0.5.0 # via happybase -tinycss2==1.2.1 +tinycss2==1.3.0 # via nbconvert toml==0.10.2 - # via feast (setup.py) tomli==2.0.1 # via # build @@ -877,7 +807,6 @@ tomli==2.0.1 # jupyterlab # mypy # pip-tools - # pyproject-hooks # pytest # pytest-env tomlkit==0.12.4 @@ -896,10 +825,8 @@ tornado==6.4 # jupyterlab # notebook # terminado -tqdm==4.66.3 - # via - # feast (setup.py) - # great-expectations +tqdm==4.66.4 + # via great-expectations traitlets==5.14.3 # via # comm @@ -916,37 +843,25 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 - # via feast (setup.py) typeguard==4.2.1 - # via feast (setup.py) +typer==0.12.3 + # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via - # feast (setup.py) - # mypy-protobuf -types-pymysql==1.1.0.1 - # via feast (setup.py) -types-pyopenssl==24.0.0.20240417 + # via mypy-protobuf +types-pymysql==1.1.0.20240425 +types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via - # arrow - # feast (setup.py) + # via arrow types-pytz==2024.1.0.20240417 - # via feast (setup.py) types-pyyaml==6.0.12.20240311 - # via feast (setup.py) -types-redis==4.6.0.20240417 - # via feast (setup.py) +types-redis==4.6.0.20240425 types-requests==2.30.0.0 - # via feast (setup.py) -types-setuptools==69.5.0.20240415 - # via - # feast (setup.py) - # types-cffi +types-setuptools==69.5.0.20240423 + # via types-cffi types-tabulate==0.9.0.20240106 - # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 @@ -967,6 +882,7 @@ typing-extensions==4.11.0 # starlette # testcontainers # typeguard + # typer # uvicorn tzdata==2024.1 # via pandas @@ -974,6 +890,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.9.0 + # via fastapi uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 @@ -982,7 +900,6 @@ urllib3==1.26.18 # via # botocore # docker - # feast (setup.py) # great-expectations # kubernetes # minio @@ -992,13 +909,13 @@ urllib3==1.26.18 # snowflake-connector-python # testcontainers uvicorn[standard]==0.29.0 - # via feast (setup.py) + # via + # fastapi + # fastapi-cli uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via - # feast (setup.py) - # pre-commit + # via pre-commit watchfiles==0.21.0 # via uvicorn wcwidth==0.2.13 @@ -1009,7 +926,7 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.7.0 +websocket-client==1.8.0 # via # jupyter-server # kubernetes @@ -1027,7 +944,3 @@ xmltodict==0.13.0 # via moto zipp==3.18.1 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 041bd0c2880..149a96626ef 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -1,13 +1,10 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --output-file=sdk/python/requirements/py3.9-requirements.txt -# +# This file was autogenerated by uv via the following command: +# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via + # httpx # starlette # watchfiles attrs==23.2.0 @@ -15,72 +12,86 @@ attrs==23.2.0 # jsonschema # referencing certifi==2024.2.2 - # via requests + # via + # httpcore + # httpx + # requests charset-normalizer==3.3.2 # via requests click==8.1.7 # via # dask - # feast (setup.py) + # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via feast (setup.py) -dask[array,dataframe]==2024.4.2 - # via - # dask-expr - # feast (setup.py) -dask-expr==1.0.12 +dask[dataframe]==2024.5.0 + # via dask-expr +dask-expr==1.1.0 # via dask dill==0.3.8 - # via feast (setup.py) +dnspython==2.6.1 + # via email-validator +email-validator==2.1.1 + # via fastapi exceptiongroup==1.2.1 # via anyio -fastapi==0.110.2 - # via feast (setup.py) +fastapi==0.111.0 + # via fastapi-cli +fastapi-cli==0.0.2 + # via fastapi fsspec==2024.3.1 # via dask greenlet==3.0.3 # via sqlalchemy -gunicorn==22.0.0 ; platform_system != "Windows" - # via feast (setup.py) +gunicorn==22.0.0 h11==0.14.0 - # via uvicorn + # via + # httpcore + # uvicorn +httpcore==1.0.5 + # via httpx httptools==0.6.1 # via uvicorn +httpx==0.27.0 + # via fastapi idna==3.7 # via # anyio + # email-validator + # httpx # requests importlib-metadata==7.1.0 # via # dask # typeguard jinja2==3.1.4 - # via feast (setup.py) -jsonschema==4.21.1 - # via feast (setup.py) + # via fastapi +jsonschema==4.22.0 jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 # via partd +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via jinja2 +mdurl==0.1.2 + # via markdown-it-py mmh3==4.1.0 - # via feast (setup.py) -mypy==1.9.0 +mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 - # via feast (setup.py) numpy==1.26.4 # via # dask - # feast (setup.py) # pandas # pyarrow +orjson==3.10.3 + # via fastapi packaging==24.0 # via # dask @@ -89,73 +100,66 @@ pandas==2.2.2 # via # dask # dask-expr - # feast (setup.py) -partd==1.4.1 +partd==1.4.2 # via dask protobuf==4.25.3 - # via - # feast (setup.py) - # mypy-protobuf -pyarrow==15.0.2 - # via - # dask-expr - # feast (setup.py) -pydantic==2.7.0 - # via - # fastapi - # feast (setup.py) -pydantic-core==2.18.1 + # via mypy-protobuf +pyarrow==16.0.0 + # via dask-expr +pydantic==2.7.1 + # via fastapi +pydantic-core==2.18.2 # via pydantic -pygments==2.17.2 - # via feast (setup.py) +pygments==2.18.0 + # via rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 # via uvicorn +python-multipart==0.0.9 + # via fastapi pytz==2024.1 # via pandas pyyaml==6.0.1 # via # dask - # feast (setup.py) # uvicorn -referencing==0.34.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications requests==2.31.0 - # via feast (setup.py) -rpds-py==0.18.0 +rich==13.7.1 + # via typer +rpds-py==0.18.1 # via # jsonschema # referencing +shellingham==1.5.4 + # via typer six==1.16.0 # via python-dateutil sniffio==1.3.1 - # via anyio -sqlalchemy[mypy]==2.0.29 # via - # feast (setup.py) - # sqlalchemy + # anyio + # httpx +sqlalchemy[mypy]==2.0.30 starlette==0.37.2 # via fastapi tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) +tenacity==8.3.0 toml==0.10.2 - # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 # via # dask # partd -tqdm==4.66.3 - # via feast (setup.py) +tqdm==4.66.4 typeguard==4.2.1 - # via feast (setup.py) -types-protobuf==5.26.0.20240420 +typer==0.12.3 + # via fastapi-cli +types-protobuf==5.26.0.20240422 # via mypy-protobuf typing-extensions==4.11.0 # via @@ -167,13 +171,18 @@ typing-extensions==4.11.0 # sqlalchemy # starlette # typeguard + # typer # uvicorn tzdata==2024.1 # via pandas +ujson==5.9.0 + # via fastapi urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 - # via feast (setup.py) + # via + # fastapi + # fastapi-cli uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/setup.py b/setup.py index ef5986f1579..6cc728ee98d 100644 --- a/setup.py +++ b/setup.py @@ -135,8 +135,8 @@ ] IBIS_REQUIRED = [ - "ibis-framework", - "ibis-substrait", + "ibis-framework>=8.0.0,<9", + "ibis-substrait<=3.2.0", ] GRPCIO_REQUIRED = [ @@ -146,7 +146,7 @@ "grpcio-health-checking>=1.56.2,<2", ] -DUCKDB_REQUIRED = ["ibis-framework[duckdb]"] +DUCKDB_REQUIRED = ["ibis-framework[duckdb]>=8.0.0,<9"] DELTA_REQUIRED = ["deltalake"] From e739745482fed1b9c2d7b788ebb088041118c642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82?= <68351136+Pawel-Drabczyk@users.noreply.github.com> Date: Wed, 8 May 2024 18:20:42 +0200 Subject: [PATCH 49/73] feat: Adding DatastoreOnlineStore 'database' argument. (#4180) * feat: adding database argument to DatastoreOnlineStore Signed-off-by: pawel * feat: adding database argument to DatastoreOnlineStore Signed-off-by: pawel * feat: adding database argument to DatastoreOnlineStore Signed-off-by: pawel * formatting and linting sdk/python/tests/unit/diff/test_infra_diff.py Signed-off-by: pawel --------- Signed-off-by: pawel --- protos/feast/core/DatastoreTable.proto | 3 ++ .../feast/infra/online_stores/datastore.py | 30 +++++++++++++++---- sdk/python/tests/unit/diff/test_infra_diff.py | 17 +++++++++-- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/protos/feast/core/DatastoreTable.proto b/protos/feast/core/DatastoreTable.proto index 4246a6ae6e7..acd3ba57b52 100644 --- a/protos/feast/core/DatastoreTable.proto +++ b/protos/feast/core/DatastoreTable.proto @@ -36,4 +36,7 @@ message DatastoreTable { // Datastore namespace google.protobuf.StringValue namespace = 4; + + // Firestore database + google.protobuf.StringValue database = 5; } \ No newline at end of file diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index 149354b4725..bf44a749661 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -80,6 +80,9 @@ class DatastoreOnlineStoreConfig(FeastConfigBaseModel): namespace: Optional[StrictStr] = None """ (optional) Datastore namespace """ + database: Optional[StrictStr] = None + """ (optional) Firestore database """ + write_concurrency: Optional[PositiveInt] = 40 """ (optional) Amount of threads to use when writing batches of feature rows into Datastore""" @@ -155,7 +158,9 @@ def teardown( def _get_client(self, online_config: DatastoreOnlineStoreConfig): if not self._client: self._client = _initialize_client( - online_config.project_id, online_config.namespace + online_config.project_id, + online_config.namespace, + online_config.database, ) return self._client @@ -344,11 +349,14 @@ def worker(shared_counter): def _initialize_client( - project_id: Optional[str], namespace: Optional[str] + project_id: Optional[str], namespace: Optional[str], database: Optional[str] ) -> datastore.Client: try: client = datastore.Client( - project=project_id, namespace=namespace, client_info=get_http_client_info() + project=project_id, + namespace=namespace, + database=database, + client_info=get_http_client_info(), ) return client except DefaultCredentialsError as e: @@ -368,11 +376,13 @@ class DatastoreTable(InfraObject): name: The name of the table. project_id (optional): The GCP project id. namespace (optional): Datastore namespace. + database (optional): Firestore database. """ project: str project_id: Optional[str] namespace: Optional[str] + database: Optional[str] def __init__( self, @@ -380,11 +390,13 @@ def __init__( name: str, project_id: Optional[str] = None, namespace: Optional[str] = None, + database: Optional[str] = None, ): super().__init__(name) self.project = project self.project_id = project_id self.namespace = namespace + self.database = database def to_infra_object_proto(self) -> InfraObjectProto: datastore_table_proto = self.to_proto() @@ -401,6 +413,8 @@ def to_proto(self) -> Any: datastore_table_proto.project_id.value = self.project_id if self.namespace: datastore_table_proto.namespace.value = self.namespace + if self.database: + datastore_table_proto.database.value = self.database return datastore_table_proto @staticmethod @@ -410,7 +424,7 @@ def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: name=infra_object_proto.datastore_table.name, ) - # Distinguish between null and empty string, since project_id and namespace are StringValues. + # Distinguish between null and empty string, since project_id, namespace and database are StringValues. if infra_object_proto.datastore_table.HasField("project_id"): datastore_table.project_id = ( infra_object_proto.datastore_table.project_id.value @@ -419,6 +433,8 @@ def from_infra_object_proto(infra_object_proto: InfraObjectProto) -> Any: datastore_table.namespace = ( infra_object_proto.datastore_table.namespace.value ) + if infra_object_proto.datastore_table.HasField("database"): + datastore_table.database = infra_object_proto.datastore_table.database.value return datastore_table @@ -434,11 +450,13 @@ def from_proto(datastore_table_proto: DatastoreTableProto) -> Any: datastore_table.project_id = datastore_table_proto.project_id.value if datastore_table_proto.HasField("namespace"): datastore_table.namespace = datastore_table_proto.namespace.value + if datastore_table_proto.HasField("database"): + datastore_table.database = datastore_table_proto.database.value return datastore_table def update(self): - client = _initialize_client(self.project_id, self.namespace) + client = _initialize_client(self.project_id, self.namespace, self.database) key = client.key("Project", self.project, "Table", self.name) entity = datastore.Entity( key=key, exclude_from_indexes=("created_ts", "event_ts", "values") @@ -447,7 +465,7 @@ def update(self): client.put(entity) def teardown(self): - client = _initialize_client(self.project_id, self.namespace) + client = _initialize_client(self.project_id, self.namespace, self.database) key = client.key("Project", self.project, "Table", self.name) _delete_all_values(client, key) diff --git a/sdk/python/tests/unit/diff/test_infra_diff.py b/sdk/python/tests/unit/diff/test_infra_diff.py index 8e3d5b765f0..3a0443e634e 100644 --- a/sdk/python/tests/unit/diff/test_infra_diff.py +++ b/sdk/python/tests/unit/diff/test_infra_diff.py @@ -39,10 +39,14 @@ def test_tag_infra_proto_objects_for_keep_delete_add(): def test_diff_between_datastore_tables(): pre_changed = DatastoreTable( - project="test", name="table", project_id="pre", namespace="pre" + project="test", name="table", project_id="pre", namespace="pre", database="pre" ).to_proto() post_changed = DatastoreTable( - project="test", name="table", project_id="post", namespace="post" + project="test", + name="table", + project_id="post", + namespace="post", + database="post", ).to_proto() infra_object_diff = diff_between(pre_changed, pre_changed, "datastore table") @@ -51,7 +55,7 @@ def test_diff_between_datastore_tables(): infra_object_diff = diff_between(pre_changed, post_changed, "datastore table") infra_object_property_diffs = infra_object_diff.infra_object_property_diffs - assert len(infra_object_property_diffs) == 2 + assert len(infra_object_property_diffs) == 3 assert infra_object_property_diffs[0].property_name == "project_id" assert infra_object_property_diffs[0].val_existing == wrappers.StringValue( @@ -67,6 +71,13 @@ def test_diff_between_datastore_tables(): assert infra_object_property_diffs[1].val_declared == wrappers.StringValue( value="post" ) + assert infra_object_property_diffs[2].property_name == "database" + assert infra_object_property_diffs[2].val_existing == wrappers.StringValue( + value="pre" + ) + assert infra_object_property_diffs[2].val_declared == wrappers.StringValue( + value="post" + ) def test_diff_infra_protos(): From 369ca98d88a5cb3c67b2363232b7c2eddfc4f333 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Thu, 9 May 2024 01:06:28 +0400 Subject: [PATCH 50/73] feat: Add materialization support to ibis/duckdb (#4173) * add materialization support to ibis/duckdb Signed-off-by: tokoko * remove unnecessary comments Signed-off-by: tokoko * pin ibis versions Signed-off-by: tokoko * refactor ibis into bunch of functions Signed-off-by: tokoko * fix requirements conflicts Signed-off-by: tokoko --------- Signed-off-by: tokoko --- .../feast/infra/offline_stores/duckdb.py | 152 +++- sdk/python/feast/infra/offline_stores/ibis.py | 656 +++++++++--------- .../requirements/py3.10-requirements.txt | 2 +- .../requirements/py3.9-requirements.txt | 2 +- .../test_universal_materialization.py | 45 ++ 5 files changed, 522 insertions(+), 335 deletions(-) create mode 100644 sdk/python/tests/integration/materialization/test_universal_materialization.py diff --git a/sdk/python/feast/infra/offline_stores/duckdb.py b/sdk/python/feast/infra/offline_stores/duckdb.py index d43286f3719..8a9390f97b1 100644 --- a/sdk/python/feast/infra/offline_stores/duckdb.py +++ b/sdk/python/feast/infra/offline_stores/duckdb.py @@ -1,8 +1,57 @@ +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, List, Optional, Union + import ibis +import pandas as pd +import pyarrow +from ibis.expr.types import Table from pydantic import StrictStr -from feast.infra.offline_stores.ibis import IbisOfflineStore -from feast.repo_config import FeastConfigBaseModel +from feast.data_format import DeltaFormat, ParquetFormat +from feast.data_source import DataSource +from feast.feature_logging import LoggingConfig, LoggingSource +from feast.feature_view import FeatureView +from feast.infra.offline_stores.file_source import FileSource +from feast.infra.offline_stores.ibis import ( + get_historical_features_ibis, + offline_write_batch_ibis, + pull_all_from_table_or_query_ibis, + pull_latest_from_table_or_query_ibis, + write_logged_features_ibis, +) +from feast.infra.offline_stores.offline_store import OfflineStore, RetrievalJob +from feast.infra.registry.base_registry import BaseRegistry +from feast.repo_config import FeastConfigBaseModel, RepoConfig + + +def _read_data_source(data_source: DataSource) -> Table: + assert isinstance(data_source, FileSource) + + if isinstance(data_source.file_format, ParquetFormat): + return ibis.read_parquet(data_source.path) + elif isinstance(data_source.file_format, DeltaFormat): + return ibis.read_delta(data_source.path) + + +def _write_data_source(table: pyarrow.Table, data_source: DataSource): + assert isinstance(data_source, FileSource) + + file_options = data_source.file_options + + if isinstance(data_source.file_format, ParquetFormat): + prev_table = ibis.read_parquet(file_options.uri).to_pyarrow() + if table.schema != prev_table.schema: + table = table.cast(prev_table.schema) + new_table = pyarrow.concat_tables([table, prev_table]) + ibis.memtable(new_table).to_parquet(file_options.uri) + elif isinstance(data_source.file_format, DeltaFormat): + from deltalake import DeltaTable + + prev_schema = DeltaTable(file_options.uri).schema().to_pyarrow() + if table.schema != prev_schema: + table = table.cast(prev_schema) + ibis.memtable(table).to_delta(file_options.uri, mode="append") class DuckDBOfflineStoreConfig(FeastConfigBaseModel): @@ -10,8 +59,99 @@ class DuckDBOfflineStoreConfig(FeastConfigBaseModel): # """ Offline store type selector""" -class DuckDBOfflineStore(IbisOfflineStore): +class DuckDBOfflineStore(OfflineStore): + @staticmethod + def pull_latest_from_table_or_query( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + created_timestamp_column: Optional[str], + start_date: datetime, + end_date: datetime, + ) -> RetrievalJob: + return pull_latest_from_table_or_query_ibis( + config=config, + data_source=data_source, + join_key_columns=join_key_columns, + feature_name_columns=feature_name_columns, + timestamp_field=timestamp_field, + created_timestamp_column=created_timestamp_column, + start_date=start_date, + end_date=end_date, + data_source_reader=_read_data_source, + ) + + @staticmethod + def get_historical_features( + config: RepoConfig, + feature_views: List[FeatureView], + feature_refs: List[str], + entity_df: Union[pd.DataFrame, str], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> RetrievalJob: + return get_historical_features_ibis( + config=config, + feature_views=feature_views, + feature_refs=feature_refs, + entity_df=entity_df, + registry=registry, + project=project, + full_feature_names=full_feature_names, + data_source_reader=_read_data_source, + ) + + @staticmethod + def pull_all_from_table_or_query( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + start_date: datetime, + end_date: datetime, + ) -> RetrievalJob: + return pull_all_from_table_or_query_ibis( + config=config, + data_source=data_source, + join_key_columns=join_key_columns, + feature_name_columns=feature_name_columns, + timestamp_field=timestamp_field, + start_date=start_date, + end_date=end_date, + data_source_reader=_read_data_source, + ) + + @staticmethod + def offline_write_batch( + config: RepoConfig, + feature_view: FeatureView, + table: pyarrow.Table, + progress: Optional[Callable[[int], Any]], + ): + offline_write_batch_ibis( + config=config, + feature_view=feature_view, + table=table, + progress=progress, + data_source_writer=_write_data_source, + ) + @staticmethod - def setup_ibis_backend(): - # there's no need to call setup as duckdb is default ibis backend - ibis.set_backend("duckdb") + def write_logged_features( + config: RepoConfig, + data: Union[pyarrow.Table, Path], + source: LoggingSource, + logging_config: LoggingConfig, + registry: BaseRegistry, + ): + write_logged_features_ibis( + config=config, + data=data, + source=source, + logging_config=logging_config, + registry=registry, + ) diff --git a/sdk/python/feast/infra/offline_stores/ibis.py b/sdk/python/feast/infra/offline_stores/ibis.py index de025ca0069..da3eefc9af9 100644 --- a/sdk/python/feast/infra/offline_stores/ibis.py +++ b/sdk/python/feast/infra/offline_stores/ibis.py @@ -25,7 +25,6 @@ SavedDatasetFileStorage, ) from feast.infra.offline_stores.offline_store import ( - OfflineStore, RetrievalJob, RetrievalMetadata, ) @@ -42,348 +41,294 @@ def _get_entity_schema(entity_df: pd.DataFrame) -> Dict[str, np.dtype]: return dict(zip(entity_df.columns, entity_df.dtypes)) -class IbisOfflineStore(OfflineStore): - @staticmethod - def pull_latest_from_table_or_query( - config: RepoConfig, - data_source: DataSource, - join_key_columns: List[str], - feature_name_columns: List[str], - timestamp_field: str, - created_timestamp_column: Optional[str], - start_date: datetime, - end_date: datetime, - ) -> RetrievalJob: - raise NotImplementedError() - - def _get_entity_df_event_timestamp_range( - entity_df: pd.DataFrame, entity_df_event_timestamp_col: str - ) -> Tuple[datetime, datetime]: - entity_df_event_timestamp = entity_df.loc[ - :, entity_df_event_timestamp_col - ].infer_objects() - if pd.api.types.is_string_dtype(entity_df_event_timestamp): - entity_df_event_timestamp = pd.to_datetime( - entity_df_event_timestamp, utc=True - ) - entity_df_event_timestamp_range = ( - entity_df_event_timestamp.min().to_pydatetime(), - entity_df_event_timestamp.max().to_pydatetime(), +def pull_latest_from_table_or_query_ibis( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + created_timestamp_column: Optional[str], + start_date: datetime, + end_date: datetime, + data_source_reader: Callable[[DataSource], Table], +) -> RetrievalJob: + fields = join_key_columns + feature_name_columns + [timestamp_field] + if created_timestamp_column: + fields.append(created_timestamp_column) + start_date = start_date.astimezone(tz=utc) + end_date = end_date.astimezone(tz=utc) + + table = data_source_reader(data_source) + + table = table.select(*fields) + + # TODO get rid of this fix + if "__log_date" in table.columns: + table = table.drop("__log_date") + + table = table.filter( + ibis.and_( + table[timestamp_field] >= ibis.literal(start_date), + table[timestamp_field] <= ibis.literal(end_date), ) + ) + + table = deduplicate( + table=table, + group_by_cols=join_key_columns, + event_timestamp_col=timestamp_field, + created_timestamp_col=created_timestamp_column, + ) + + return IbisRetrievalJob( + table=table, + on_demand_feature_views=[], + full_feature_names=False, + metadata=None, + ) + + +def _get_entity_df_event_timestamp_range( + entity_df: pd.DataFrame, entity_df_event_timestamp_col: str +) -> Tuple[datetime, datetime]: + entity_df_event_timestamp = entity_df.loc[ + :, entity_df_event_timestamp_col + ].infer_objects() + if pd.api.types.is_string_dtype(entity_df_event_timestamp): + entity_df_event_timestamp = pd.to_datetime(entity_df_event_timestamp, utc=True) + entity_df_event_timestamp_range = ( + entity_df_event_timestamp.min().to_pydatetime(), + entity_df_event_timestamp.max().to_pydatetime(), + ) + + return entity_df_event_timestamp_range + + +def _to_utc(entity_df: pd.DataFrame, event_timestamp_col): + entity_df_event_timestamp = entity_df.loc[:, event_timestamp_col].infer_objects() + if pd.api.types.is_string_dtype(entity_df_event_timestamp): + entity_df_event_timestamp = pd.to_datetime(entity_df_event_timestamp, utc=True) + + entity_df[event_timestamp_col] = entity_df_event_timestamp + return entity_df + + +def _generate_row_id( + entity_table: Table, feature_views: List[FeatureView], event_timestamp_col +) -> Table: + all_entities = [event_timestamp_col] + for fv in feature_views: + if fv.projection.join_key_map: + all_entities.extend(fv.projection.join_key_map.values()) + else: + all_entities.extend([e.name for e in fv.entity_columns]) - return entity_df_event_timestamp_range - - @staticmethod - def _to_utc(entity_df: pd.DataFrame, event_timestamp_col): - entity_df_event_timestamp = entity_df.loc[ - :, event_timestamp_col - ].infer_objects() - if pd.api.types.is_string_dtype(entity_df_event_timestamp): - entity_df_event_timestamp = pd.to_datetime( - entity_df_event_timestamp, utc=True - ) - - entity_df[event_timestamp_col] = entity_df_event_timestamp - return entity_df - - @staticmethod - def _generate_row_id( - entity_table: Table, feature_views: List[FeatureView], event_timestamp_col - ) -> Table: - all_entities = [event_timestamp_col] - for fv in feature_views: - if fv.projection.join_key_map: - all_entities.extend(fv.projection.join_key_map.values()) - else: - all_entities.extend([e.name for e in fv.entity_columns]) - - r = ibis.literal("") - - for e in set(all_entities): - r = r.concat(entity_table[e].cast("string")) # type: ignore - - entity_table = entity_table.mutate(entity_row_id=r) - - return entity_table - - @staticmethod - def _read_data_source(data_source: DataSource) -> Table: - assert isinstance(data_source, FileSource) - - if isinstance(data_source.file_format, ParquetFormat): - return ibis.read_parquet(data_source.path) - elif isinstance(data_source.file_format, DeltaFormat): - return ibis.read_delta(data_source.path) - - @staticmethod - def get_historical_features( - config: RepoConfig, - feature_views: List[FeatureView], - feature_refs: List[str], - entity_df: Union[pd.DataFrame, str], - registry: BaseRegistry, - project: str, - full_feature_names: bool = False, - ) -> RetrievalJob: - entity_schema = _get_entity_schema( - entity_df=entity_df, - ) - event_timestamp_col = offline_utils.infer_event_timestamp_from_entity_df( - entity_schema=entity_schema, - ) + r = ibis.literal("") - # TODO get range with ibis - timestamp_range = IbisOfflineStore._get_entity_df_event_timestamp_range( - entity_df, event_timestamp_col - ) + for e in set(all_entities): + r = r.concat(entity_table[e].cast("string")) # type: ignore - entity_df = IbisOfflineStore._to_utc(entity_df, event_timestamp_col) + entity_table = entity_table.mutate(entity_row_id=r) - entity_table = ibis.memtable(entity_df) - entity_table = IbisOfflineStore._generate_row_id( - entity_table, feature_views, event_timestamp_col + return entity_table + + +def get_historical_features_ibis( + config: RepoConfig, + feature_views: List[FeatureView], + feature_refs: List[str], + entity_df: Union[pd.DataFrame, str], + registry: BaseRegistry, + project: str, + data_source_reader: Callable[[DataSource], Table], + full_feature_names: bool = False, +) -> RetrievalJob: + entity_schema = _get_entity_schema( + entity_df=entity_df, + ) + event_timestamp_col = offline_utils.infer_event_timestamp_from_entity_df( + entity_schema=entity_schema, + ) + + # TODO get range with ibis + timestamp_range = _get_entity_df_event_timestamp_range( + entity_df, event_timestamp_col + ) + + entity_df = _to_utc(entity_df, event_timestamp_col) + + entity_table = ibis.memtable(entity_df) + entity_table = _generate_row_id(entity_table, feature_views, event_timestamp_col) + + def read_fv( + feature_view: FeatureView, feature_refs: List[str], full_feature_names: bool + ) -> Tuple: + fv_table: Table = data_source_reader(feature_view.batch_source) + + for old_name, new_name in feature_view.batch_source.field_mapping.items(): + if old_name in fv_table.columns: + fv_table = fv_table.rename({new_name: old_name}) + + timestamp_field = feature_view.batch_source.timestamp_field + + # TODO mutate only if tz-naive + fv_table = fv_table.mutate( + **{ + timestamp_field: fv_table[timestamp_field].cast( + dt.Timestamp(timezone="UTC") + ) + } ) - def read_fv( - feature_view: FeatureView, feature_refs: List[str], full_feature_names: bool - ) -> Tuple: - fv_table: Table = IbisOfflineStore._read_data_source( - feature_view.batch_source - ) - - for old_name, new_name in feature_view.batch_source.field_mapping.items(): - if old_name in fv_table.columns: - fv_table = fv_table.rename({new_name: old_name}) + full_name_prefix = feature_view.projection.name_alias or feature_view.name - timestamp_field = feature_view.batch_source.timestamp_field + feature_refs = [ + fr.split(":")[1] + for fr in feature_refs + if fr.startswith(f"{full_name_prefix}:") + ] - # TODO mutate only if tz-naive - fv_table = fv_table.mutate( - **{ - timestamp_field: fv_table[timestamp_field].cast( - dt.Timestamp(timezone="UTC") - ) - } + if full_feature_names: + fv_table = fv_table.rename( + {f"{full_name_prefix}__{feature}": feature for feature in feature_refs} ) - full_name_prefix = feature_view.projection.name_alias or feature_view.name - feature_refs = [ - fr.split(":")[1] - for fr in feature_refs - if fr.startswith(f"{full_name_prefix}:") + f"{full_name_prefix}__{feature}" for feature in feature_refs ] - if full_feature_names: - fv_table = fv_table.rename( - { - f"{full_name_prefix}__{feature}": feature - for feature in feature_refs - } - ) - - feature_refs = [ - f"{full_name_prefix}__{feature}" for feature in feature_refs - ] - - return ( - fv_table, - feature_view.batch_source.timestamp_field, - feature_view.batch_source.created_timestamp_column, - feature_view.projection.join_key_map - or {e.name: e.name for e in feature_view.entity_columns}, - feature_refs, - feature_view.ttl, - ) - - res = point_in_time_join( - entity_table=entity_table, - feature_tables=[ - read_fv(feature_view, feature_refs, full_feature_names) - for feature_view in feature_views - ], - event_timestamp_col=event_timestamp_col, - ) - - odfvs = OnDemandFeatureView.get_requested_odfvs(feature_refs, project, registry) - - substrait_odfvs = [fv for fv in odfvs if fv.mode == "substrait"] - for odfv in substrait_odfvs: - res = odfv.transform_ibis(res, full_feature_names) - - return IbisRetrievalJob( - res, - [fv for fv in odfvs if fv.mode != "substrait"], - full_feature_names, - metadata=RetrievalMetadata( - features=feature_refs, - keys=list(set(entity_df.columns) - {event_timestamp_col}), - min_event_timestamp=timestamp_range[0], - max_event_timestamp=timestamp_range[1], - ), + return ( + fv_table, + feature_view.batch_source.timestamp_field, + feature_view.batch_source.created_timestamp_column, + feature_view.projection.join_key_map + or {e.name: e.name for e in feature_view.entity_columns}, + feature_refs, + feature_view.ttl, ) - @staticmethod - def pull_all_from_table_or_query( - config: RepoConfig, - data_source: DataSource, - join_key_columns: List[str], - feature_name_columns: List[str], - timestamp_field: str, - start_date: datetime, - end_date: datetime, - ) -> RetrievalJob: - assert isinstance(data_source, FileSource) - - fields = join_key_columns + feature_name_columns + [timestamp_field] - start_date = start_date.astimezone(tz=utc) - end_date = end_date.astimezone(tz=utc) - - table = IbisOfflineStore._read_data_source(data_source) - - table = table.select(*fields) - - # TODO get rid of this fix - if "__log_date" in table.columns: - table = table.drop("__log_date") - - table = table.filter( - ibis.and_( - table[timestamp_field] >= ibis.literal(start_date), - table[timestamp_field] <= ibis.literal(end_date), - ) - ) - - return IbisRetrievalJob( - table=table, - on_demand_feature_views=[], - full_feature_names=False, - metadata=None, - ) - - @staticmethod - def write_logged_features( - config: RepoConfig, - data: Union[pyarrow.Table, Path], - source: LoggingSource, - logging_config: LoggingConfig, - registry: BaseRegistry, - ): - destination = logging_config.destination - assert isinstance(destination, FileLoggingDestination) - - table = ( - ibis.read_parquet(data) if isinstance(data, Path) else ibis.memtable(data) + res = point_in_time_join( + entity_table=entity_table, + feature_tables=[ + read_fv(feature_view, feature_refs, full_feature_names) + for feature_view in feature_views + ], + event_timestamp_col=event_timestamp_col, + ) + + odfvs = OnDemandFeatureView.get_requested_odfvs(feature_refs, project, registry) + + substrait_odfvs = [fv for fv in odfvs if fv.mode == "substrait"] + for odfv in substrait_odfvs: + res = odfv.transform_ibis(res, full_feature_names) + + return IbisRetrievalJob( + res, + [fv for fv in odfvs if fv.mode != "substrait"], + full_feature_names, + metadata=RetrievalMetadata( + features=feature_refs, + keys=list(set(entity_df.columns) - {event_timestamp_col}), + min_event_timestamp=timestamp_range[0], + max_event_timestamp=timestamp_range[1], + ), + ) + + +def pull_all_from_table_or_query_ibis( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + start_date: datetime, + end_date: datetime, + data_source_reader: Callable[[DataSource], Table], +) -> RetrievalJob: + fields = join_key_columns + feature_name_columns + [timestamp_field] + start_date = start_date.astimezone(tz=utc) + end_date = end_date.astimezone(tz=utc) + + table = data_source_reader(data_source) + + table = table.select(*fields) + + # TODO get rid of this fix + if "__log_date" in table.columns: + table = table.drop("__log_date") + + table = table.filter( + ibis.and_( + table[timestamp_field] >= ibis.literal(start_date), + table[timestamp_field] <= ibis.literal(end_date), ) + ) + + return IbisRetrievalJob( + table=table, + on_demand_feature_views=[], + full_feature_names=False, + metadata=None, + ) + + +def write_logged_features_ibis( + config: RepoConfig, + data: Union[pyarrow.Table, Path], + source: LoggingSource, + logging_config: LoggingConfig, + registry: BaseRegistry, +): + destination = logging_config.destination + assert isinstance(destination, FileLoggingDestination) - if destination.partition_by: - kwargs = {"partition_by": destination.partition_by} - else: - kwargs = {} - - # TODO always write to directory - table.to_parquet( - f"{destination.path}/{uuid.uuid4().hex}-{{i}}.parquet", **kwargs - ) - - @staticmethod - def offline_write_batch( - config: RepoConfig, - feature_view: FeatureView, - table: pyarrow.Table, - progress: Optional[Callable[[int], Any]], - ): - assert isinstance(feature_view.batch_source, FileSource) - - pa_schema, column_names = get_pyarrow_schema_from_batch_source( - config, feature_view.batch_source - ) - if column_names != table.column_names: - raise ValueError( - f"The input pyarrow table has schema {table.schema} with the incorrect columns {table.column_names}. " - f"The schema is expected to be {pa_schema} with the columns (in this exact order) to be {column_names}." - ) - - file_options = feature_view.batch_source.file_options - - if isinstance(feature_view.batch_source.file_format, ParquetFormat): - prev_table = ibis.read_parquet(file_options.uri).to_pyarrow() - if table.schema != prev_table.schema: - table = table.cast(prev_table.schema) - new_table = pyarrow.concat_tables([table, prev_table]) + table = ibis.read_parquet(data) if isinstance(data, Path) else ibis.memtable(data) - ibis.memtable(new_table).to_parquet(file_options.uri) - elif isinstance(feature_view.batch_source.file_format, DeltaFormat): - from deltalake import DeltaTable + if destination.partition_by: + kwargs = {"partition_by": destination.partition_by} + else: + kwargs = {} - prev_schema = DeltaTable(file_options.uri).schema().to_pyarrow() - if table.schema != prev_schema: - table = table.cast(prev_schema) - ibis.memtable(table).to_delta(file_options.uri, mode="append") + # TODO always write to directory + table.to_parquet(f"{destination.path}/{uuid.uuid4().hex}-{{i}}.parquet", **kwargs) -class IbisRetrievalJob(RetrievalJob): - def __init__( - self, table, on_demand_feature_views, full_feature_names, metadata - ) -> None: - super().__init__() - self.table = table - self._on_demand_feature_views: List[OnDemandFeatureView] = ( - on_demand_feature_views +def offline_write_batch_ibis( + config: RepoConfig, + feature_view: FeatureView, + table: pyarrow.Table, + progress: Optional[Callable[[int], Any]], + data_source_writer: Callable[[pyarrow.Table, DataSource], None], +): + pa_schema, column_names = get_pyarrow_schema_from_batch_source( + config, feature_view.batch_source + ) + if column_names != table.column_names: + raise ValueError( + f"The input pyarrow table has schema {table.schema} with the incorrect columns {table.column_names}. " + f"The schema is expected to be {pa_schema} with the columns (in this exact order) to be {column_names}." ) - self._full_feature_names = full_feature_names - self._metadata = metadata - def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: - return self.table.execute() + data_source_writer(table, feature_view.batch_source) - def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: - return self.table.to_pyarrow() - @property - def full_feature_names(self) -> bool: - return self._full_feature_names +def deduplicate( + table: Table, + group_by_cols: List[str], + event_timestamp_col: str, + created_timestamp_col: Optional[str], +): + order_by_fields = [ibis.desc(table[event_timestamp_col])] + if created_timestamp_col: + order_by_fields.append(ibis.desc(table[created_timestamp_col])) - @property - def on_demand_feature_views(self) -> List[OnDemandFeatureView]: - return self._on_demand_feature_views + table = ( + table.group_by(by=group_by_cols) + .order_by(order_by_fields) + .mutate(rn=ibis.row_number()) + ) - def persist( - self, - storage: SavedDatasetStorage, - allow_overwrite: bool = False, - timeout: Optional[int] = None, - ): - assert isinstance(storage, SavedDatasetFileStorage) - if not allow_overwrite and os.path.exists(storage.file_options.uri): - raise SavedDatasetLocationAlreadyExists(location=storage.file_options.uri) - - if isinstance(storage.file_options.file_format, ParquetFormat): - filesystem, path = FileSource.create_filesystem_and_path( - storage.file_options.uri, - storage.file_options.s3_endpoint_override, - ) - - if path.endswith(".parquet"): - pyarrow.parquet.write_table( - self.to_arrow(), where=path, filesystem=filesystem - ) - else: - # otherwise assume destination is directory - pyarrow.parquet.write_to_dataset( - self.to_arrow(), root_path=path, filesystem=filesystem - ) - elif isinstance(storage.file_options.file_format, DeltaFormat): - mode = ( - "overwrite" - if allow_overwrite and os.path.exists(storage.file_options.uri) - else "error" - ) - self.table.to_delta(storage.file_options.uri, mode=mode) - - @property - def metadata(self) -> Optional[RetrievalMetadata]: - return self._metadata + return table.filter(table["rn"] == ibis.literal(0)).drop("rn") def point_in_time_join( @@ -440,20 +385,13 @@ def point_in_time_join( feature_table = feature_table.drop(s.endswith("_y")) - order_by_fields = [ibis.desc(feature_table[timestamp_field])] - if created_timestamp_field: - order_by_fields.append(ibis.desc(feature_table[created_timestamp_field])) - - feature_table = ( - feature_table.group_by(by="entity_row_id") - .order_by(order_by_fields) - .mutate(rn=ibis.row_number()) + feature_table = deduplicate( + table=feature_table, + group_by_cols=["entity_row_id"], + event_timestamp_col=timestamp_field, + created_timestamp_col=created_timestamp_field, ) - feature_table = feature_table.filter( - feature_table["rn"] == ibis.literal(0) - ).drop("rn") - select_cols = ["entity_row_id"] select_cols.extend(feature_refs) feature_table = feature_table.select(select_cols) @@ -470,3 +408,67 @@ def point_in_time_join( acc_table = acc_table.drop("entity_row_id") return acc_table + + +class IbisRetrievalJob(RetrievalJob): + def __init__( + self, table, on_demand_feature_views, full_feature_names, metadata + ) -> None: + super().__init__() + self.table = table + self._on_demand_feature_views: List[OnDemandFeatureView] = ( + on_demand_feature_views + ) + self._full_feature_names = full_feature_names + self._metadata = metadata + + def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: + return self.table.execute() + + def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: + return self.table.to_pyarrow() + + @property + def full_feature_names(self) -> bool: + return self._full_feature_names + + @property + def on_demand_feature_views(self) -> List[OnDemandFeatureView]: + return self._on_demand_feature_views + + def persist( + self, + storage: SavedDatasetStorage, + allow_overwrite: bool = False, + timeout: Optional[int] = None, + ): + assert isinstance(storage, SavedDatasetFileStorage) + if not allow_overwrite and os.path.exists(storage.file_options.uri): + raise SavedDatasetLocationAlreadyExists(location=storage.file_options.uri) + + if isinstance(storage.file_options.file_format, ParquetFormat): + filesystem, path = FileSource.create_filesystem_and_path( + storage.file_options.uri, + storage.file_options.s3_endpoint_override, + ) + + if path.endswith(".parquet"): + pyarrow.parquet.write_table( + self.to_arrow(), where=path, filesystem=filesystem + ) + else: + # otherwise assume destination is directory + pyarrow.parquet.write_to_dataset( + self.to_arrow(), root_path=path, filesystem=filesystem + ) + elif isinstance(storage.file_options.file_format, DeltaFormat): + mode = ( + "overwrite" + if allow_overwrite and os.path.exists(storage.file_options.uri) + else "error" + ) + self.table.to_delta(storage.file_options.uri, mode=mode) + + @property + def metadata(self) -> Optional[RetrievalMetadata]: + return self._metadata diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 99c9bfc3fee..56a8259ab43 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -187,4 +187,4 @@ watchfiles==0.21.0 websockets==12.0 # via uvicorn zipp==3.18.1 - # via importlib-metadata + # via importlib-metadata \ No newline at end of file diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 149a96626ef..1092aac9d09 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -190,4 +190,4 @@ watchfiles==0.21.0 websockets==12.0 # via uvicorn zipp==3.18.1 - # via importlib-metadata + # via importlib-metadata \ No newline at end of file diff --git a/sdk/python/tests/integration/materialization/test_universal_materialization.py b/sdk/python/tests/integration/materialization/test_universal_materialization.py new file mode 100644 index 00000000000..37030b1bb30 --- /dev/null +++ b/sdk/python/tests/integration/materialization/test_universal_materialization.py @@ -0,0 +1,45 @@ +from datetime import timedelta + +import pytest + +from feast.entity import Entity +from feast.feature_view import FeatureView +from feast.field import Field +from feast.types import Float32 +from tests.data.data_creator import create_basic_driver_dataset +from tests.utils.e2e_test_validation import validate_offline_online_store_consistency + + +@pytest.mark.integration +@pytest.mark.universal_offline_stores +def test_universal_materialization_consistency(environment): + fs = environment.feature_store + + df = create_basic_driver_dataset() + + ds = environment.data_source_creator.create_data_source( + df, + fs.project, + field_mapping={"ts_1": "ts"}, + ) + + driver = Entity( + name="driver_id", + join_keys=["driver_id"], + ) + + driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(weeks=52), + schema=[Field(name="value", dtype=Float32)], + source=ds, + ) + + fs.apply([driver, driver_stats_fv]) + + # materialization is run in two steps and + # we use timestamp from generated dataframe as a split point + split_dt = df["ts_1"][4].to_pydatetime() - timedelta(seconds=1) + + validate_offline_online_store_consistency(fs, driver_stats_fv, split_dt) From 311efc5005b24d1fc9bc389ee7579e102e2cd4ea Mon Sep 17 00:00:00 2001 From: Breno Costa <35263725+breno-costa@users.noreply.github.com> Date: Thu, 9 May 2024 14:57:20 +0200 Subject: [PATCH 51/73] feat: Adding get_online_features_async to feature store sdk (#4172) * feat: Adding get_online_features_async to feature store sdk Signed-off-by: Breno Costa * add more unit tests Signed-off-by: Breno Costa * fix redis key generation Signed-off-by: Breno Costa * fix unit tests Signed-off-by: Breno Costa --------- Signed-off-by: Breno Costa --- README.md | 2 +- sdk/python/feast/feature_store.py | 233 ++++++++++++++++-- .../feast/infra/online_stores/online_store.py | 25 ++ sdk/python/feast/infra/online_stores/redis.py | 133 ++++++++-- .../feast/infra/passthrough_provider.py | 16 ++ sdk/python/feast/infra/provider.py | 24 ++ sdk/python/feast/online_response.py | 6 +- sdk/python/tests/foo_provider.py | 9 + .../unit/infra/online_store/test_redis.py | 130 ++++++++++ 9 files changed, 522 insertions(+), 56 deletions(-) create mode 100644 sdk/python/tests/unit/infra/online_store/test_redis.py diff --git a/README.md b/README.md index e9b7ff47436..a1e06774dac 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

-
+
[![unit-tests](https://github.com/feast-dev/feast/actions/workflows/unit_tests.yml/badge.svg?branch=master&event=push)](https://github.com/feast-dev/feast/actions/workflows/unit_tests.yml) [![integration-tests-and-build](https://github.com/feast-dev/feast/actions/workflows/master_only.yml/badge.svg?branch=master&event=push)](https://github.com/feast-dev/feast/actions/workflows/master_only.yml) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index f45dbb1bc8f..270ea45a26c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1550,6 +1550,54 @@ def get_online_features( native_entity_values=True, ) + @log_exceptions_and_usage + async def get_online_features_async( + self, + features: Union[List[str], FeatureService], + entity_rows: List[Dict[str, Any]], + full_feature_names: bool = False, + ) -> OnlineResponse: + """ + [Alpha] Retrieves the latest online feature data asynchronously. + + Note: This method will download the full feature registry the first time it is run. If you are using a + remote registry like GCS or S3 then that may take a few seconds. The registry remains cached up to a TTL + duration (which can be set to infinity). If the cached registry is stale (more time than the TTL has + passed), then a new registry will be downloaded synchronously by this method. This download may + introduce latency to online feature retrieval. In order to avoid synchronous downloads, please call + refresh_registry() prior to the TTL being reached. Remember it is possible to set the cache TTL to + infinity (cache forever). + + Args: + features: The list of features that should be retrieved from the online store. These features can be + specified either as a list of string feature references or as a feature service. String feature + references must have format "feature_view:feature", e.g. "customer_fv:daily_transactions". + entity_rows: A list of dictionaries where each key-value is an entity-name, entity-value pair. + full_feature_names: If True, feature names will be prefixed with the corresponding feature view name, + changing them from the format "feature" to "feature_view__feature" (e.g. "daily_transactions" + changes to "customer_fv__daily_transactions"). + + Returns: + OnlineResponse containing the feature data in records. + + Raises: + Exception: No entity with the specified name exists. + """ + columnar: Dict[str, List[Any]] = {k: [] for k in entity_rows[0].keys()} + for entity_row in entity_rows: + for key, value in entity_row.items(): + try: + columnar[key].append(value) + except KeyError as e: + raise ValueError("All entity_rows must have the same keys.") from e + + return await self._get_online_features_async( + features=features, + entity_values=columnar, + full_feature_names=full_feature_names, + native_entity_values=True, + ) + def _get_online_request_context( self, features: Union[List[str], FeatureService], full_feature_names: bool ): @@ -1609,7 +1657,7 @@ def _get_online_request_context( entityless_case, ) - def _get_online_features( + def _prepare_entities_to_read_from_online_store( self, features: Union[List[str], FeatureService], entity_values: Mapping[ @@ -1619,7 +1667,7 @@ def _get_online_features( native_entity_values: bool = True, ): ( - _feature_refs, + feature_refs, requested_on_demand_feature_views, entity_name_to_join_key_map, entity_type_map, @@ -1694,6 +1742,40 @@ def _get_online_features( [DUMMY_ENTITY_VAL] * num_rows, DUMMY_ENTITY.value_type ) + return ( + join_key_values, + grouped_refs, + entity_name_to_join_key_map, + requested_on_demand_feature_views, + feature_refs, + requested_result_row_names, + online_features_response, + ) + + def _get_online_features( + self, + features: Union[List[str], FeatureService], + entity_values: Mapping[ + str, Union[Sequence[Any], Sequence[Value], RepeatedValue] + ], + full_feature_names: bool = False, + native_entity_values: bool = True, + ): + ( + join_key_values, + grouped_refs, + entity_name_to_join_key_map, + requested_on_demand_feature_views, + feature_refs, + requested_result_row_names, + online_features_response, + ) = self._prepare_entities_to_read_from_online_store( + features=features, + entity_values=entity_values, + full_feature_names=full_feature_names, + native_entity_values=native_entity_values, + ) + provider = self._get_provider() for table, requested_features in grouped_refs: # Get the correct set of entity values with the correct join keys. @@ -1724,7 +1806,71 @@ def _get_online_features( if requested_on_demand_feature_views: self._augment_response_with_on_demand_transforms( online_features_response, - _feature_refs, + feature_refs, + requested_on_demand_feature_views, + full_feature_names, + ) + + self._drop_unneeded_columns( + online_features_response, requested_result_row_names + ) + return OnlineResponse(online_features_response) + + async def _get_online_features_async( + self, + features: Union[List[str], FeatureService], + entity_values: Mapping[ + str, Union[Sequence[Any], Sequence[Value], RepeatedValue] + ], + full_feature_names: bool = False, + native_entity_values: bool = True, + ): + ( + join_key_values, + grouped_refs, + entity_name_to_join_key_map, + requested_on_demand_feature_views, + feature_refs, + requested_result_row_names, + online_features_response, + ) = self._prepare_entities_to_read_from_online_store( + features=features, + entity_values=entity_values, + full_feature_names=full_feature_names, + native_entity_values=native_entity_values, + ) + + provider = self._get_provider() + for table, requested_features in grouped_refs: + # Get the correct set of entity values with the correct join keys. + table_entity_values, idxs = self._get_unique_entities( + table, + join_key_values, + entity_name_to_join_key_map, + ) + + # Fetch feature data for the minimum set of Entities. + feature_data = await self._read_from_online_store_async( + table_entity_values, + provider, + requested_features, + table, + ) + + # Populate the result_rows with the Features from the OnlineStore inplace. + self._populate_response_from_feature_data( + feature_data, + idxs, + online_features_response, + full_feature_names, + requested_features, + table, + ) + + if requested_on_demand_feature_views: + self._augment_response_with_on_demand_transforms( + online_features_response, + feature_refs, requested_on_demand_feature_views, full_feature_names, ) @@ -1965,38 +2111,24 @@ def _get_unique_entities( ) return unique_entities, indexes - def _read_from_online_store( + def _get_entity_key_protos( self, entity_rows: Iterable[Mapping[str, Value]], - provider: Provider, - requested_features: List[str], - table: FeatureView, - ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[Value]]]: - """Read and process data from the OnlineStore for a given FeatureView. - - This method guarantees that the order of the data in each element of the - List returned is the same as the order of `requested_features`. - - This method assumes that `provider.online_read` returns data for each - combination of Entities in `entity_rows` in the same order as they - are provided. - """ + ) -> List[EntityKeyProto]: # Instantiate one EntityKeyProto per Entity. entity_key_protos = [ EntityKeyProto(join_keys=row.keys(), entity_values=row.values()) for row in entity_rows ] + return entity_key_protos - # Fetch data for Entities. - read_rows = provider.online_read( - config=self.config, - table=table, - entity_keys=entity_key_protos, - requested_features=requested_features, - ) - - # Each row is a set of features for a given entity key. We only need to convert - # the data to Protobuf once. + def _convert_rows_to_protobuf( + self, + requested_features: List[str], + read_rows: List[Tuple[Optional[datetime], Optional[Dict[str, Value]]]], + ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[Value]]]: + # Each row is a set of features for a given entity key. + # We only need to convert the data to Protobuf once. null_value = Value() read_row_protos = [] for read_row in read_rows: @@ -2023,6 +2155,53 @@ def _read_from_online_store( read_row_protos.append((event_timestamps, statuses, values)) return read_row_protos + def _read_from_online_store( + self, + entity_rows: Iterable[Mapping[str, Value]], + provider: Provider, + requested_features: List[str], + table: FeatureView, + ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[Value]]]: + """Read and process data from the OnlineStore for a given FeatureView. + + This method guarantees that the order of the data in each element of the + List returned is the same as the order of `requested_features`. + + This method assumes that `provider.online_read` returns data for each + combination of Entities in `entity_rows` in the same order as they + are provided. + """ + entity_key_protos = self._get_entity_key_protos(entity_rows) + + # Fetch data for Entities. + read_rows = provider.online_read( + config=self.config, + table=table, + entity_keys=entity_key_protos, + requested_features=requested_features, + ) + + return self._convert_rows_to_protobuf(requested_features, read_rows) + + async def _read_from_online_store_async( + self, + entity_rows: Iterable[Mapping[str, Value]], + provider: Provider, + requested_features: List[str], + table: FeatureView, + ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[Value]]]: + entity_key_protos = self._get_entity_key_protos(entity_rows) + + # Fetch data for Entities. + read_rows = await provider.online_read_async( + config=self.config, + table=table, + entity_keys=entity_key_protos, + requested_features=requested_features, + ) + + return self._convert_rows_to_protobuf(requested_features, read_rows) + def _retrieve_from_online_store( self, provider: Provider, diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index 2a81e370427..7dd03a84178 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -80,6 +80,31 @@ def online_read( """ pass + async def online_read_async( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + """ + Reads features values for the given entity keys asynchronously. + + Args: + config: The config for the current feature store. + table: The feature view whose feature values should be read. + entity_keys: The list of entity keys for which feature values should be read. + requested_features: The list of features that should be read. + + Returns: + A list of the same length as entity_keys. Each item in the list is a tuple where the first + item is the event timestamp for the row, and the second item is a dict mapping feature names + to values, which are returned in proto format. + """ + raise NotImplementedError( + f"Online store {self.__class__.__name__} does not support online read async" + ) + @abstractmethod def update( self, diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index 6f6c2fb45c6..f681d8473e4 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -42,6 +42,7 @@ try: from redis import Redis + from redis import asyncio as redis_asyncio from redis.cluster import ClusterNode, RedisCluster from redis.sentinel import Sentinel except ImportError as e: @@ -90,6 +91,9 @@ class RedisOnlineStore(OnlineStore): """ _client: Optional[Union[Redis, RedisCluster]] = None + _client_async: Optional[Union[redis_asyncio.Redis, redis_asyncio.RedisCluster]] = ( + None + ) def delete_entity_values(self, config: RepoConfig, join_keys: List[str]): client = self._get_client(config.online_store) @@ -234,6 +238,30 @@ def _get_client(self, online_store_config: RedisOnlineStoreConfig): self._client = Redis(**kwargs) return self._client + async def _get_client_async(self, online_store_config: RedisOnlineStoreConfig): + if not self._client_async: + startup_nodes, kwargs = self._parse_connection_string( + online_store_config.connection_string + ) + if online_store_config.redis_type == RedisType.redis_cluster: + kwargs["startup_nodes"] = [ + redis_asyncio.cluster.ClusterNode(**node) for node in startup_nodes + ] + self._client_async = redis_asyncio.RedisCluster(**kwargs) + elif online_store_config.redis_type == RedisType.redis_sentinel: + sentinel_hosts = [] + for item in startup_nodes: + sentinel_hosts.append((item["host"], int(item["port"]))) + + sentinel = redis_asyncio.Sentinel(sentinel_hosts, **kwargs) + master = sentinel.master_for(online_store_config.sentinel_master) + self._client_async = master + else: + kwargs["host"] = startup_nodes[0]["host"] + kwargs["port"] = startup_nodes[0]["port"] + self._client_async = redis_asyncio.Redis(**kwargs) + return self._client_async + @log_exceptions_and_usage(online_store="redis") def online_write_batch( self, @@ -304,6 +332,49 @@ def online_write_batch( if progress: progress(len(results)) + def _generate_redis_keys_for_entities( + self, config: RepoConfig, entity_keys: List[EntityKeyProto] + ) -> List[bytes]: + keys = [] + for entity_key in entity_keys: + redis_key_bin = _redis_key( + config.project, + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + keys.append(redis_key_bin) + return keys + + def _generate_hset_keys_for_features( + self, + feature_view: FeatureView, + requested_features: Optional[List[str]] = None, + ) -> Tuple[List[str], List[str]]: + if not requested_features: + requested_features = [f.name for f in feature_view.features] + + hset_keys = [_mmh3(f"{feature_view.name}:{k}") for k in requested_features] + + ts_key = f"_ts:{feature_view.name}" + hset_keys.append(ts_key) + requested_features.append(ts_key) + + return requested_features, hset_keys + + def _convert_redis_values_to_protobuf( + self, + redis_values: List[List[ByteString]], + feature_view: str, + requested_features: List[str], + ): + result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + for values in redis_values: + features = self._get_features_for_entity( + values, feature_view, requested_features + ) + result.append(features) + return result + @log_exceptions_and_usage(online_store="redis") def online_read( self, @@ -316,39 +387,51 @@ def online_read( assert isinstance(online_store_config, RedisOnlineStoreConfig) client = self._get_client(online_store_config) - feature_view = table.name - project = config.project + feature_view = table - result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + requested_features, hset_keys = self._generate_hset_keys_for_features( + feature_view, requested_features + ) + keys = self._generate_redis_keys_for_entities(config, entity_keys) - if not requested_features: - requested_features = [f.name for f in table.features] + with client.pipeline(transaction=False) as pipe: + for redis_key_bin in keys: + pipe.hmget(redis_key_bin, hset_keys) + with tracing_span(name="remote_call"): + redis_values = pipe.execute() - hset_keys = [_mmh3(f"{feature_view}:{k}") for k in requested_features] + return self._convert_redis_values_to_protobuf( + redis_values, feature_view.name, requested_features + ) - ts_key = f"_ts:{feature_view}" - hset_keys.append(ts_key) - requested_features.append(ts_key) + @log_exceptions_and_usage(online_store="redis") + async def online_read_async( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + online_store_config = config.online_store + assert isinstance(online_store_config, RedisOnlineStoreConfig) - keys = [] - for entity_key in entity_keys: - redis_key_bin = _redis_key( - project, - entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, - ) - keys.append(redis_key_bin) - with client.pipeline(transaction=False) as pipe: + client = await self._get_client_async(online_store_config) + feature_view = table + + requested_features, hset_keys = self._generate_hset_keys_for_features( + feature_view, requested_features + ) + keys = self._generate_redis_keys_for_entities(config, entity_keys) + + async with client.pipeline(transaction=False) as pipe: for redis_key_bin in keys: pipe.hmget(redis_key_bin, hset_keys) with tracing_span(name="remote_call"): - redis_values = pipe.execute() - for values in redis_values: - features = self._get_features_for_entity( - values, feature_view, requested_features - ) - result.append(features) - return result + redis_values = await pipe.execute() + + return self._convert_redis_values_to_protobuf( + redis_values, feature_view.name, requested_features + ) def _get_features_for_entity( self, diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 2f3e30018af..97c2820d415 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -188,6 +188,22 @@ def online_read( ) return result + @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.001)) + async def online_read_async( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List: + set_usage_attribute("provider", self.__class__.__name__) + result = [] + if self.online_store: + result = await self.online_store.online_read_async( + config, table, entity_keys, requested_features + ) + return result + @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.001)) def retrieve_online_documents( self, diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 02fba0c1f6b..68d36da17f7 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -229,6 +229,30 @@ def online_read( """ pass + @abstractmethod + async def online_read_async( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + """ + Reads features values for the given entity keys asynchronously. + + Args: + config: The config for the current feature store. + table: The feature view whose feature values should be read. + entity_keys: The list of entity keys for which feature values should be read. + requested_features: The list of features that should be read. + + Returns: + A list of the same length as entity_keys. Each item in the list is a tuple where the first + item is the event timestamp for the row, and the second item is a dict mapping feature names + to values, which are returned in proto format. + """ + pass + @abstractmethod def retrieve_saved_dataset( self, config: RepoConfig, dataset: SavedDataset diff --git a/sdk/python/feast/online_response.py b/sdk/python/feast/online_response.py index 050b374340e..a4e5694127f 100644 --- a/sdk/python/feast/online_response.py +++ b/sdk/python/feast/online_response.py @@ -50,7 +50,7 @@ def to_dict(self, include_event_timestamps: bool = False) -> Dict[str, Any]: Converts GetOnlineFeaturesResponse features into a dictionary form. Args: - is_with_event_timestamps: bool Optionally include feature timestamps in the dictionary + include_event_timestamps: bool Optionally include feature timestamps in the dictionary """ response: Dict[str, List[Any]] = {} @@ -74,7 +74,7 @@ def to_df(self, include_event_timestamps: bool = False) -> pd.DataFrame: Converts GetOnlineFeaturesResponse features into Panda dataframe form. Args: - is_with_event_timestamps: bool Optionally include feature timestamps in the dataframe + include_event_timestamps: bool Optionally include feature timestamps in the dataframe """ return pd.DataFrame(self.to_dict(include_event_timestamps)) @@ -84,7 +84,7 @@ def to_arrow(self, include_event_timestamps: bool = False) -> pa.Table: Converts GetOnlineFeaturesResponse features into pyarrow Table. Args: - is_with_event_timestamps: bool Optionally include feature timestamps in the table + include_event_timestamps: bool Optionally include feature timestamps in the table """ return pa.Table.from_pydict(self.to_dict(include_event_timestamps)) diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index f869d82e114..eb7fe5d6ac8 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -82,6 +82,15 @@ def online_read( ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: return [] + async def online_read_async( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + return [] + def retrieve_saved_dataset(self, config: RepoConfig, dataset: SavedDataset): pass diff --git a/sdk/python/tests/unit/infra/online_store/test_redis.py b/sdk/python/tests/unit/infra/online_store/test_redis.py new file mode 100644 index 00000000000..c26c2f25c5f --- /dev/null +++ b/sdk/python/tests/unit/infra/online_store/test_redis.py @@ -0,0 +1,130 @@ +import pytest +from google.protobuf.timestamp_pb2 import Timestamp + +from feast import Entity, FeatureView, Field, FileSource, RepoConfig +from feast.infra.online_stores.redis import RedisOnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.types import Int32 + + +@pytest.fixture +def redis_online_store() -> RedisOnlineStore: + return RedisOnlineStore() + + +@pytest.fixture +def repo_config(): + return RepoConfig( + provider="local", + project="test", + entity_key_serialization_version=2, + registry="dummy_registry.db", + ) + + +@pytest.fixture +def feature_view(): + file_source = FileSource(name="my_file_source", path="test.parquet") + entity = Entity(name="entity", join_keys=["entity"]) + feature_view = FeatureView( + name="feature_view_1", + entities=[entity], + schema=[ + Field(name="feature_10", dtype=Int32), + Field(name="feature_11", dtype=Int32), + Field(name="feature_12", dtype=Int32), + ], + source=file_source, + ) + return feature_view + + +def test_generate_entity_redis_keys(redis_online_store: RedisOnlineStore, repo_config): + entity_keys = [ + EntityKeyProto(join_keys=["entity"], entity_values=[ValueProto(int32_val=1)]), + ] + + actual = redis_online_store._generate_redis_keys_for_entities( + repo_config, entity_keys + ) + expected = [ + b"\x02\x00\x00\x00entity\x03\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00test" + ] + assert actual == expected + + +def test_generate_hset_keys_for_features( + redis_online_store: RedisOnlineStore, feature_view +): + actual = redis_online_store._generate_hset_keys_for_features(feature_view) + expected = ( + ["feature_10", "feature_11", "feature_12", "_ts:feature_view_1"], + [b"&m_9", b"\xc37\x9a\xbf", b"wr\xb5d", "_ts:feature_view_1"], + ) + assert actual == expected + + +def test_generate_hset_keys_for_features_with_requested_features( + redis_online_store: RedisOnlineStore, feature_view +): + actual = redis_online_store._generate_hset_keys_for_features( + feature_view=feature_view, requested_features=["my-feature-view:feature1"] + ) + expected = ( + ["my-feature-view:feature1", "_ts:feature_view_1"], + [b"Si\x86J", "_ts:feature_view_1"], + ) + assert actual == expected + + +def test_convert_redis_values_to_protobuf( + redis_online_store: RedisOnlineStore, feature_view +): + requested_features = [ + "feature_view_1:feature_10", + "feature_view_1:feature_11", + "_ts:feature_view_1", + ] + values = [ + [ + ValueProto(int32_val=1).SerializeToString(), + ValueProto(int32_val=2).SerializeToString(), + Timestamp().SerializeToString(), + ] + ] + + features = redis_online_store._convert_redis_values_to_protobuf( + redis_values=values, + feature_view=feature_view.name, + requested_features=requested_features, + ) + assert isinstance(features, list) + assert len(features) == 1 + + timestamp, features = features[0] + assert features["feature_view_1:feature_10"].int32_val == 1 + assert features["feature_view_1:feature_11"].int32_val == 2 + + +def test_get_features_for_entity(redis_online_store: RedisOnlineStore, feature_view): + requested_features = [ + "feature_view_1:feature_10", + "feature_view_1:feature_11", + "_ts:feature_view_1", + ] + values = [ + ValueProto(int32_val=1).SerializeToString(), + ValueProto(int32_val=2).SerializeToString(), + Timestamp().SerializeToString(), + ] + + timestamp, features = redis_online_store._get_features_for_entity( + values=values, + feature_view=feature_view.name, + requested_features=requested_features, + ) + assert "feature_view_1:feature_10" in features + assert "feature_view_1:feature_11" in features + assert features["feature_view_1:feature_10"].int32_val == 1 + assert features["feature_view_1:feature_11"].int32_val == 2 From 48712a516fbbc68904826cd81e49144b84d345bb Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Fri, 10 May 2024 22:01:30 +0400 Subject: [PATCH 52/73] chore: Move FileSource handling from ibis to duckdb (#4191) move FileSource handling from ibis to duckdb Signed-off-by: tokoko --- .../feast/infra/offline_stores/duckdb.py | 59 +++++++++++++++---- sdk/python/feast/infra/offline_stores/ibis.py | 50 ++++++---------- 2 files changed, 65 insertions(+), 44 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/duckdb.py b/sdk/python/feast/infra/offline_stores/duckdb.py index 8a9390f97b1..8e392425ea5 100644 --- a/sdk/python/feast/infra/offline_stores/duckdb.py +++ b/sdk/python/feast/infra/offline_stores/duckdb.py @@ -1,3 +1,4 @@ +import os from datetime import datetime from pathlib import Path from typing import Any, Callable, List, Optional, Union @@ -10,6 +11,7 @@ from feast.data_format import DeltaFormat, ParquetFormat from feast.data_source import DataSource +from feast.errors import SavedDatasetLocationAlreadyExists from feast.feature_logging import LoggingConfig, LoggingSource from feast.feature_view import FeatureView from feast.infra.offline_stores.file_source import FileSource @@ -34,24 +36,56 @@ def _read_data_source(data_source: DataSource) -> Table: return ibis.read_delta(data_source.path) -def _write_data_source(table: pyarrow.Table, data_source: DataSource): +def _write_data_source( + table: Table, + data_source: DataSource, + mode: str = "append", + allow_overwrite: bool = False, +): assert isinstance(data_source, FileSource) file_options = data_source.file_options + if mode == "overwrite" and not allow_overwrite and os.path.exists(file_options.uri): + raise SavedDatasetLocationAlreadyExists(location=file_options.uri) + if isinstance(data_source.file_format, ParquetFormat): - prev_table = ibis.read_parquet(file_options.uri).to_pyarrow() - if table.schema != prev_table.schema: - table = table.cast(prev_table.schema) - new_table = pyarrow.concat_tables([table, prev_table]) - ibis.memtable(new_table).to_parquet(file_options.uri) + if mode == "overwrite": + table = table.to_pyarrow() + filesystem, path = FileSource.create_filesystem_and_path( + file_options.uri, + file_options.s3_endpoint_override, + ) + + if path.endswith(".parquet"): + pyarrow.parquet.write_table(table, where=path, filesystem=filesystem) + else: + # otherwise assume destination is directory + pyarrow.parquet.write_to_dataset( + table, root_path=path, filesystem=filesystem + ) + elif mode == "append": + table = table.to_pyarrow() + prev_table = ibis.read_parquet(file_options.uri).to_pyarrow() + if table.schema != prev_table.schema: + table = table.cast(prev_table.schema) + new_table = pyarrow.concat_tables([table, prev_table]) + ibis.memtable(new_table).to_parquet(file_options.uri) elif isinstance(data_source.file_format, DeltaFormat): - from deltalake import DeltaTable + if mode == "append": + from deltalake import DeltaTable + + prev_schema = DeltaTable(file_options.uri).schema().to_pyarrow() + table = table.cast(ibis.Schema.from_pyarrow(prev_schema)) + write_mode = "append" + elif mode == "overwrite": + write_mode = ( + "overwrite" + if allow_overwrite and os.path.exists(file_options.uri) + else "error" + ) - prev_schema = DeltaTable(file_options.uri).schema().to_pyarrow() - if table.schema != prev_schema: - table = table.cast(prev_schema) - ibis.memtable(table).to_delta(file_options.uri, mode="append") + table.to_delta(file_options.uri, mode=write_mode) class DuckDBOfflineStoreConfig(FeastConfigBaseModel): @@ -81,6 +115,7 @@ def pull_latest_from_table_or_query( start_date=start_date, end_date=end_date, data_source_reader=_read_data_source, + data_source_writer=_write_data_source, ) @staticmethod @@ -102,6 +137,7 @@ def get_historical_features( project=project, full_feature_names=full_feature_names, data_source_reader=_read_data_source, + data_source_writer=_write_data_source, ) @staticmethod @@ -123,6 +159,7 @@ def pull_all_from_table_or_query( start_date=start_date, end_date=end_date, data_source_reader=_read_data_source, + data_source_writer=_write_data_source, ) @staticmethod diff --git a/sdk/python/feast/infra/offline_stores/ibis.py b/sdk/python/feast/infra/offline_stores/ibis.py index da3eefc9af9..b9efb87a36a 100644 --- a/sdk/python/feast/infra/offline_stores/ibis.py +++ b/sdk/python/feast/infra/offline_stores/ibis.py @@ -1,4 +1,3 @@ -import os import uuid from datetime import datetime, timedelta from pathlib import Path @@ -13,16 +12,12 @@ from ibis.expr.types import Table from pytz import utc -from feast.data_format import DeltaFormat, ParquetFormat from feast.data_source import DataSource -from feast.errors import SavedDatasetLocationAlreadyExists from feast.feature_logging import LoggingConfig, LoggingSource from feast.feature_view import FeatureView from feast.infra.offline_stores import offline_utils from feast.infra.offline_stores.file_source import ( FileLoggingDestination, - FileSource, - SavedDatasetFileStorage, ) from feast.infra.offline_stores.offline_store import ( RetrievalJob, @@ -51,6 +46,7 @@ def pull_latest_from_table_or_query_ibis( start_date: datetime, end_date: datetime, data_source_reader: Callable[[DataSource], Table], + data_source_writer: Callable[[pyarrow.Table, DataSource], None], ) -> RetrievalJob: fields = join_key_columns + feature_name_columns + [timestamp_field] if created_timestamp_column: @@ -85,6 +81,7 @@ def pull_latest_from_table_or_query_ibis( on_demand_feature_views=[], full_feature_names=False, metadata=None, + data_source_writer=data_source_writer, ) @@ -141,6 +138,7 @@ def get_historical_features_ibis( registry: BaseRegistry, project: str, data_source_reader: Callable[[DataSource], Table], + data_source_writer: Callable[[pyarrow.Table, DataSource], None], full_feature_names: bool = False, ) -> RetrievalJob: entity_schema = _get_entity_schema( @@ -232,6 +230,7 @@ def read_fv( min_event_timestamp=timestamp_range[0], max_event_timestamp=timestamp_range[1], ), + data_source_writer=data_source_writer, ) @@ -244,6 +243,7 @@ def pull_all_from_table_or_query_ibis( start_date: datetime, end_date: datetime, data_source_reader: Callable[[DataSource], Table], + data_source_writer: Callable[[pyarrow.Table, DataSource], None], ) -> RetrievalJob: fields = join_key_columns + feature_name_columns + [timestamp_field] start_date = start_date.astimezone(tz=utc) @@ -269,6 +269,7 @@ def pull_all_from_table_or_query_ibis( on_demand_feature_views=[], full_feature_names=False, metadata=None, + data_source_writer=data_source_writer, ) @@ -309,7 +310,7 @@ def offline_write_batch_ibis( f"The schema is expected to be {pa_schema} with the columns (in this exact order) to be {column_names}." ) - data_source_writer(table, feature_view.batch_source) + data_source_writer(ibis.memtable(table), feature_view.batch_source) def deduplicate( @@ -412,7 +413,12 @@ def point_in_time_join( class IbisRetrievalJob(RetrievalJob): def __init__( - self, table, on_demand_feature_views, full_feature_names, metadata + self, + table, + on_demand_feature_views, + full_feature_names, + metadata, + data_source_writer, ) -> None: super().__init__() self.table = table @@ -421,6 +427,7 @@ def __init__( ) self._full_feature_names = full_feature_names self._metadata = metadata + self.data_source_writer = data_source_writer def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: return self.table.execute() @@ -442,32 +449,9 @@ def persist( allow_overwrite: bool = False, timeout: Optional[int] = None, ): - assert isinstance(storage, SavedDatasetFileStorage) - if not allow_overwrite and os.path.exists(storage.file_options.uri): - raise SavedDatasetLocationAlreadyExists(location=storage.file_options.uri) - - if isinstance(storage.file_options.file_format, ParquetFormat): - filesystem, path = FileSource.create_filesystem_and_path( - storage.file_options.uri, - storage.file_options.s3_endpoint_override, - ) - - if path.endswith(".parquet"): - pyarrow.parquet.write_table( - self.to_arrow(), where=path, filesystem=filesystem - ) - else: - # otherwise assume destination is directory - pyarrow.parquet.write_to_dataset( - self.to_arrow(), root_path=path, filesystem=filesystem - ) - elif isinstance(storage.file_options.file_format, DeltaFormat): - mode = ( - "overwrite" - if allow_overwrite and os.path.exists(storage.file_options.uri) - else "error" - ) - self.table.to_delta(storage.file_options.uri, mode=mode) + self.data_source_writer( + self.table, storage.to_data_source(), "overwrite", allow_overwrite + ) @property def metadata(self) -> Optional[RetrievalMetadata]: From 48a081e3b54b9b0bd397f80e5d0e8011ebc1584f Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Fri, 10 May 2024 13:02:12 -0500 Subject: [PATCH 53/73] docs: Add docs for using `podman` on RHEL or Fedora machines (#4192) add docs for using podman on rhel or fedora Signed-off-by: Tommy Hughes --- docs/project/development-guide.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/project/development-guide.md b/docs/project/development-guide.md index 146b6d7516e..e3b09294bc3 100644 --- a/docs/project/development-guide.md +++ b/docs/project/development-guide.md @@ -123,6 +123,7 @@ Note that this means if you are midway through working through a PR and rebase, Setting up your development environment for Feast Python SDK / CLI: 1. Ensure that you have Docker installed in your environment. Docker is used to provision service dependencies during testing, and build images for feature servers and other components. - Please note that we use [Docker with BuiltKit](https://docs.docker.com/develop/develop-images/build_enhancements/). + - _Alternatively_ - To use [podman](https://podman.io/) on a Fedora or RHEL machine, follow this [guide](https://github.com/feast-dev/feast/issues/4190) 2. Ensure that you have `make` and Python (3.9 or above) installed. 3. _Recommended:_ Create a virtual environment to isolate development dependencies to be installed ```sh From 8e4412589d2450c49b113e3b921ae9dad2ee56a0 Mon Sep 17 00:00:00 2001 From: Jiwon Park Date: Sun, 12 May 2024 01:09:23 +0900 Subject: [PATCH 54/73] refactor: Replace deprecated lifespan event (#4187) --- sdk/python/feast/feature_server.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index fda8745c2d2..98a8c0caf49 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -3,6 +3,7 @@ import threading import traceback import warnings +from contextlib import asynccontextmanager from typing import List, Optional import pandas as pd @@ -50,15 +51,16 @@ def get_app( registry_ttl_sec: int = DEFAULT_FEATURE_SERVER_REGISTRY_TTL, ): proto_json.patch() - - app = FastAPI() # Asynchronously refresh registry, notifying shutdown and canceling the active timer if the app is shutting down registry_proto = None shutting_down = False active_timer: Optional[threading.Timer] = None - async def get_body(request: Request): - return await request.body() + def stop_refresh(): + nonlocal shutting_down + shutting_down = True + if active_timer: + active_timer.cancel() def async_refresh(): store.refresh_registry() @@ -70,14 +72,16 @@ def async_refresh(): active_timer = threading.Timer(registry_ttl_sec, async_refresh) active_timer.start() - @app.on_event("shutdown") - def shutdown_event(): - nonlocal shutting_down - shutting_down = True - if active_timer: - active_timer.cancel() + @asynccontextmanager + async def lifespan(app: FastAPI): + async_refresh() + yield + stop_refresh() - async_refresh() + app = FastAPI(lifespan=lifespan) + + async def get_body(request: Request): + return await request.body() @app.post("/get-online-features") def get_online_features(body=Depends(get_body)): From 37f36b681bde0c1ae83303803c89d3ed0b2ac8a9 Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Sat, 11 May 2024 10:10:41 -0700 Subject: [PATCH 55/73] fix: Add vector database doc (#4165) --- docs/reference/alpha-vector-database.md | 111 ++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 docs/reference/alpha-vector-database.md diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md new file mode 100644 index 00000000000..3b0c924d84b --- /dev/null +++ b/docs/reference/alpha-vector-database.md @@ -0,0 +1,111 @@ +# [Alpha] Vector Database +**Warning**: This is an _experimental_ feature. To our knowledge, this is stable, but there are still rough edges in the experience. Contributions are welcome! + +## Overview +Vector database allows user to store and retrieve embeddings. Feast provides general APIs to store and retrieve embeddings. + +## Integration +Below are supported vector databases and implemented features: + +| Vector Database | Retrieval | Indexing | +|-----------------|-----------|----------| +| Pgvector | [x] | [ ] | +| Elasticsearch | [ ] | [ ] | +| Milvus | [ ] | [ ] | +| Faiss | [ ] | [ ] | + + +## Example + +See [https://github.com/feast-dev/feast-workshop/blob/rag/module_4_rag](https://github.com/feast-dev/feast-workshop/blob/rag/module_4_rag) for an example on how to use vector database. + +### **Prepare offline embedding dataset** +Run the following commands to prepare the embedding dataset: +```shell +python pull_states.py +python batch_score_documents.py +``` +The output will be stored in `data/city_wikipedia_summaries.csv.` + +### **Initialize Feast feature store and materialize the data to the online store** +Use the feature_tore.yaml file to initialize the feature store. This will use the data as offline store, and Pgvector as online store. + +```yaml +project: feast_demo_local +provider: local +registry: + registry_type: sql + path: postgresql://@localhost:5432/feast +online_store: + type: postgres + pgvector_enabled: true + vector_len: 384 + host: 127.0.0.1 + port: 5432 + database: feast + user: "" + password: "" + + +offline_store: + type: file +entity_key_serialization_version: 2 +``` +Run the following command in terminal to apply the feature store configuration: + +```shell +feast apply +``` + +Note that when you run `feast apply` you are going to apply the following Feature View that we will use for retrieval later: + +```python +city_embeddings_feature_view = FeatureView( + name="city_embeddings", + entities=[item], + schema=[ + Field(name="Embeddings", dtype=Array(Float32)), + ], + source=source, + ttl=timedelta(hours=2), +) +``` + +Then run the following command in the terminal to materialize the data to the online store: + +```shell +CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%S") +feast materialize-incremental $CURRENT_TIME +``` + +### **Prepare a query embedding** +```python +from batch_score_documents import run_model, TOKENIZER, MODEL +from transformers import AutoTokenizer, AutoModel + +question = "the most populous city in the U.S. state of Texas?" + +tokenizer = AutoTokenizer.from_pretrained(TOKENIZER) +model = AutoModel.from_pretrained(MODEL) +query_embedding = run_model(question, tokenizer, model) +query = query_embedding.detach().cpu().numpy().tolist()[0] +``` + +### **Retrieve the top 5 similar documents** +First create a feature store instance, and use the `retrieve_online_documents` API to retrieve the top 5 similar documents to the specified query. + +```python +from feast import FeatureStore +store = FeatureStore(repo_path=".") +features = store.retrieve_online_documents( + feature="city_embeddings:Embeddings", + query=query, + top_k=5 +).to_dict() + +def print_online_features(features): + for key, value in sorted(features.items()): + print(key, " : ", value) + +print_online_features(features) +``` \ No newline at end of file From bf99640c0bcfd9ee7c1e66d24cb791bfa0e5ac4a Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Sun, 12 May 2024 20:35:37 -0700 Subject: [PATCH 56/73] feat: Elasticsearch vector database (#4188) --- Makefile | 19 ++ docs/reference/alpha-vector-database.md | 2 +- docs/reference/online-stores/elasticsearch.md | 125 ++++++++ sdk/python/feast/feature_store.py | 6 +- .../online_stores/contrib/elasticsearch.py | 276 ++++++++++++++++++ .../elasticsearch_repo_configuration.py | 13 + .../infra/online_stores/contrib/postgres.py | 2 +- .../feast/infra/online_stores/online_store.py | 2 + .../feast/infra/passthrough_provider.py | 2 +- sdk/python/feast/infra/provider.py | 3 +- sdk/python/feast/repo_config.py | 1 + sdk/python/tests/foo_provider.py | 2 +- .../universal/online_store/elasticsearch.py | 28 ++ .../online_store/test_universal_online.py | 2 +- setup.py | 4 + 15 files changed, 478 insertions(+), 9 deletions(-) create mode 100644 docs/reference/online-stores/elasticsearch.md create mode 100644 sdk/python/feast/infra/online_stores/contrib/elasticsearch.py create mode 100644 sdk/python/feast/infra/online_stores/contrib/elasticsearch_repo_configuration.py create mode 100644 sdk/python/tests/integration/feature_repos/universal/online_store/elasticsearch.py diff --git a/Makefile b/Makefile index 18006fe7d1c..9b537522181 100644 --- a/Makefile +++ b/Makefile @@ -310,6 +310,25 @@ test-python-universal-cassandra-no-cloud-providers: not test_snowflake" \ sdk/python/tests + test-python-universal-elasticsearch-online: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.elasticsearch_repo_configuration \ + PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.elasticsearch \ + python -m pytest -n 8 --integration \ + -k "not test_universal_cli and \ + not test_go_feature_server and \ + not test_feature_logging and \ + not test_reorder_columns and \ + not test_logged_features_validation and \ + not test_lambda_materialization_consistency and \ + not test_offline_write and \ + not test_push_features_to_offline_store and \ + not gcs_registry and \ + not s3_registry and \ + not test_universal_types and \ + not test_snowflake" \ + sdk/python/tests + test-python-universal: python -m pytest -n 8 --integration sdk/python/tests diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index 3b0c924d84b..37d9b9cdf87 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -10,7 +10,7 @@ Below are supported vector databases and implemented features: | Vector Database | Retrieval | Indexing | |-----------------|-----------|----------| | Pgvector | [x] | [ ] | -| Elasticsearch | [ ] | [ ] | +| Elasticsearch | [x] | [x] | | Milvus | [ ] | [ ] | | Faiss | [ ] | [ ] | diff --git a/docs/reference/online-stores/elasticsearch.md b/docs/reference/online-stores/elasticsearch.md new file mode 100644 index 00000000000..bf6f9a58db1 --- /dev/null +++ b/docs/reference/online-stores/elasticsearch.md @@ -0,0 +1,125 @@ +# ElasticSearch online store (contrib) + +## Description + +The ElasticSearch online store provides support for materializing tabular feature values, as well as embedding feature vectors, into an ElasticSearch index for serving online features. \ +The embedding feature vectors are stored as dense vectors, and can be used for similarity search. More information on dense vectors can be found [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/dense-vector.html). + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[elasticsearch]'`. You can get started by then running `feast init -t elasticsearch`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: elasticsearch + host: ES_HOST + port: ES_PORT + user: ES_USERNAME + password: ES_PASSWORD + vector_len: 512 + write_batch_size: 1000 +``` +{% endcode %} + +The full set of configuration options is available in [ElasticsearchOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.contrib.elasticsearch.ElasticsearchOnlineStoreConfig). + +## Functionality Matrix + + +| | Postgres | +| :-------------------------------------------------------- | :------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + +## Retrieving online document vectors + +The ElasticSearch online store supports retrieving document vectors for a given list of entity keys. The document vectors are returned as a dictionary where the key is the entity key and the value is the document vector. The document vector is a dense vector of floats. + +{% code title="python" %} +```python +from feast import FeatureStore + +feature_store = FeatureStore(repo_path="feature_store.yaml") + +query_vector = [1.0, 2.0, 3.0, 4.0, 5.0] +top_k = 5 + +# Retrieve the top k closest features to the query vector + +feature_values = feature_store.retrieve_online_documents( + feature="my_feature", + query=query_vector, + top_k=top_k +) +``` +{% endcode %} + +## Indexing +Currently, the indexing mapping in the ElasticSearch online store is configured as: + +{% code title="indexing_mapping" %} +```json +"properties": { + "entity_key": {"type": "binary"}, + "feature_name": {"type": "keyword"}, + "feature_value": {"type": "binary"}, + "timestamp": {"type": "date"}, + "created_ts": {"type": "date"}, + "vector_value": { + "type": "dense_vector", + "dims": config.online_store.vector_len, + "index": "true", + "similarity": config.online_store.similarity, + }, +} +``` +{% endcode %} +And the online_read API mapping is configured as: + +{% code title="online_read_mapping" %} +```json +"query": { + "bool": { + "must": [ + {"terms": {"entity_key": entity_keys}}, + {"terms": {"feature_name": requested_features}}, + ] + } +}, +``` +{% endcode %} + +And the similarity search API mapping is configured as: + +{% code title="similarity_search_mapping" %} +```json +{ + "field": "vector_value", + "query_vector": embedding_vector, + "k": top_k, +} +``` +{% endcode %} + +These APIs are subject to change in future versions of Feast to improve performance and usability. \ No newline at end of file diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 270ea45a26c..2fe885865d9 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1886,7 +1886,7 @@ def retrieve_online_documents( feature: str, query: Union[str, List[float]], top_k: int, - distance_metric: str, + distance_metric: Optional[str] = None, ) -> OnlineResponse: """ Retrieves the top k closest document features. Note, embeddings are a subset of features. @@ -1911,7 +1911,7 @@ def _retrieve_online_documents( feature: str, query: Union[str, List[float]], top_k: int, - distance_metric: str = "L2", + distance_metric: Optional[str] = None, ): if isinstance(query, str): raise ValueError( @@ -2209,7 +2209,7 @@ def _retrieve_from_online_store( requested_feature: str, query: List[float], top_k: int, - distance_metric: str, + distance_metric: Optional[str], ) -> List[Tuple[Timestamp, "FieldStatus.ValueType", Value, Value, Value]]: """ Search and return document features from the online document store. diff --git a/sdk/python/feast/infra/online_stores/contrib/elasticsearch.py b/sdk/python/feast/infra/online_stores/contrib/elasticsearch.py new file mode 100644 index 00000000000..429327e6518 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/elasticsearch.py @@ -0,0 +1,276 @@ +from __future__ import absolute_import + +import base64 +import json +import logging +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +import pytz +from elasticsearch import Elasticsearch, helpers + +from feast import Entity, FeatureView, RepoConfig +from feast.infra.key_encoding_utils import get_list_val_str, serialize_entity_key +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel + + +class ElasticSearchOnlineStoreConfig(FeastConfigBaseModel): + """ + Configuration for the ElasticSearch online store. + NOTE: The class *must* end with the `OnlineStoreConfig` suffix. + """ + + type: str = "elasticsearch" + + host: Optional[str] = None + user: Optional[str] = None + password: Optional[str] = None + port: Optional[int] = None + index: Optional[str] = None + scheme: Optional[str] = "http" + + # The number of rows to write in a single batch + write_batch_size: Optional[int] = 40 + + # The length of the vector value + vector_len: Optional[int] = 512 + + # The vector similarity metric to use in KNN search + # more details: https://www.elastic.co/guide/en/elasticsearch/reference/current/dense-vector.html + similarity: Optional[str] = "cosine" + + +class ElasticSearchOnlineStore(OnlineStore): + _client: Optional[Elasticsearch] = None + + def _get_client(self, config: RepoConfig) -> Elasticsearch: + online_store_config = config.online_store + assert isinstance(online_store_config, ElasticSearchOnlineStoreConfig) + + user = online_store_config.user if online_store_config.user is not None else "" + password = ( + online_store_config.password + if online_store_config.password is not None + else "" + ) + + if self._client: + return self._client + else: + self._client = Elasticsearch( + hosts=[ + { + "host": online_store_config.host or "localhost", + "port": online_store_config.port or 9200, + "scheme": online_store_config.scheme or "http", + } + ], + basic_auth=(user, password), + ) + return self._client + + def _bulk_batch_actions(self, table: FeatureView, batch: List[Dict[str, Any]]): + for row in batch: + yield { + "_index": table.name, + "_id": f"{row['entity_key']}_{row['feature_name']}_{row['timestamp']}", + "_source": row, + } + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + insert_values = [] + for entity_key, values, timestamp, created_ts in data: + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + encoded_entity_key = base64.b64encode(entity_key_bin).decode("utf-8") + timestamp = _to_naive_utc(timestamp) + if created_ts is not None: + created_ts = _to_naive_utc(created_ts) + for feature_name, value in values.items(): + encoded_value = base64.b64encode(value.SerializeToString()).decode( + "utf-8" + ) + vector_val = json.loads(get_list_val_str(value)) + insert_values.append( + { + "entity_key": encoded_entity_key, + "feature_name": feature_name, + "feature_value": encoded_value, + "timestamp": timestamp, + "created_ts": created_ts, + "vector_value": vector_val, + } + ) + + batch_size = config.online_store.write_batch_size + for i in range(0, len(insert_values), batch_size): + batch = insert_values[i : i + batch_size] + actions = self._bulk_batch_actions(table, batch) + helpers.bulk(self._get_client(config), actions) + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + if not requested_features: + body = { + "_source": {"excludes": ["vector_value"]}, + "query": {"match": {"entity_key": entity_keys}}, + } + else: + body = { + "_source": {"excludes": ["vector_value"]}, + "query": { + "bool": { + "must": [ + {"terms": {"entity_key": entity_keys}}, + {"terms": {"feature_name": requested_features}}, + ] + } + }, + } + response = self._get_client(config).search(index=table.name, body=body) + results: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + for hit in response["hits"]["hits"]: + results.append( + ( + hit["_source"]["timestamp"], + {hit["_source"]["feature_name"]: hit["_source"]["feature_value"]}, + ) + ) + return results + + def create_index(self, config: RepoConfig, table: FeatureView): + """ + Create an index in ElasticSearch for the given table. + TODO: This method can be exposed to users to customize the indexing functionality. + Args: + config: Feast repo configuration object. + table: FeatureView table for which the index needs to be created. + """ + index_mapping = { + "properties": { + "entity_key": {"type": "binary"}, + "feature_name": {"type": "keyword"}, + "feature_value": {"type": "binary"}, + "timestamp": {"type": "date"}, + "created_ts": {"type": "date"}, + "vector_value": { + "type": "dense_vector", + "dims": config.online_store.vector_len, + "index": "true", + "similarity": config.online_store.similarity, + }, + } + } + self._get_client(config).indices.create( + index=table.name, mappings=index_mapping + ) + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + # implement the update method + for table in tables_to_delete: + self._get_client(config).delete_by_query(index=table.name) + for table in tables_to_keep: + self.create_index(config, table) + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ): + project = config.project + try: + for table in tables: + self._get_client(config).indices.delete(index=table.name) + except Exception as e: + logging.exception(f"Error deleting index in project {project}: {e}") + raise + + def retrieve_online_documents( + self, + config: RepoConfig, + table: FeatureView, + requested_feature: str, + embedding: List[float], + top_k: int, + *args, + **kwargs, + ) -> List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: + result: List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ] = [] + response = self._get_client(config).search( + index=table.name, + knn={ + "field": "vector_value", + "query_vector": embedding, + "k": top_k, + }, + ) + rows = response["hits"]["hits"][0:top_k] + for row in rows: + feature_value = row["_source"]["feature_value"] + vector_value = row["_source"]["vector_value"] + timestamp = row["_source"]["timestamp"] + distance = row["_score"] + timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f") + + feature_value_proto = ValueProto() + feature_value_proto.ParseFromString(base64.b64decode(feature_value)) + + vector_value_proto = ValueProto(string_val=str(vector_value)) + distance_value_proto = ValueProto(float_val=distance) + result.append( + ( + timestamp, + feature_value_proto, + vector_value_proto, + distance_value_proto, + ) + ) + return result + + +def _to_naive_utc(ts: datetime): + if ts.tzinfo is None: + return ts + else: + return ts.astimezone(pytz.utc).replace(tzinfo=None) diff --git a/sdk/python/feast/infra/online_stores/contrib/elasticsearch_repo_configuration.py b/sdk/python/feast/infra/online_stores/contrib/elasticsearch_repo_configuration.py new file mode 100644 index 00000000000..4d1f2c3ca18 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/elasticsearch_repo_configuration.py @@ -0,0 +1,13 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.elasticsearch import ( + ElasticSearchOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig( + online_store="elasticsearch", + online_store_creator=ElasticSearchOnlineStoreCreator, + ), +] diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index f2c32fdafd1..1043208ab33 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -283,7 +283,7 @@ def retrieve_online_documents( requested_feature: str, embedding: List[float], top_k: int, - distance_metric: str = "L2", + distance_metric: Optional[str] = "L2", ) -> List[ Tuple[ Optional[datetime], diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index 7dd03a84178..05983a494c0 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -167,6 +167,7 @@ def retrieve_online_documents( requested_feature: str, embedding: List[float], top_k: int, + distance_metric: Optional[str] = None, ) -> List[ Tuple[ Optional[datetime], @@ -179,6 +180,7 @@ def retrieve_online_documents( Retrieves online feature values for the specified embeddings. Args: + distance_metric: distance metric to use for retrieval. config: The config for the current feature store. table: The feature view whose feature values should be read. requested_feature: The name of the feature whose embeddings should be used for retrieval. diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 97c2820d415..48d2f8ef185 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -212,7 +212,7 @@ def retrieve_online_documents( requested_feature: str, query: List[float], top_k: int, - distance_metric: str, + distance_metric: Optional[str] = None, ) -> List: set_usage_attribute("provider", self.__class__.__name__) result = [] diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 68d36da17f7..22f6088474a 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -327,7 +327,7 @@ def retrieve_online_documents( requested_feature: str, query: List[float], top_k: int, - distance_metric: str = "L2", + distance_metric: Optional[str] = None, ) -> List[ Tuple[ Optional[datetime], @@ -340,6 +340,7 @@ def retrieve_online_documents( Searches for the top-k most similar documents in the online document store. Args: + distance_metric: distance metric to use for the search. config: The config for the current feature store. table: The feature view whose embeddings should be searched. requested_feature: the requested document feature name. diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 5e38fd17758..00cbac19081 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -64,6 +64,7 @@ "rockset": "feast.infra.online_stores.contrib.rockset_online_store.rockset.RocksetOnlineStore", "hazelcast": "feast.infra.online_stores.contrib.hazelcast_online_store.hazelcast_online_store.HazelcastOnlineStore", "ikv": "feast.infra.online_stores.contrib.ikv_online_store.ikv.IKVOnlineStore", + "elasticsearch": "feast.infra.online_stores.contrib.elasticsearch.ElasticSearchOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index eb7fe5d6ac8..3b9146a7b96 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -120,7 +120,7 @@ def retrieve_online_documents( requested_feature: str, query: List[float], top_k: int, - distance_metric: str, + distance_metric: Optional[str] = None, ) -> List[ Tuple[ Optional[datetime], diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/elasticsearch.py b/sdk/python/tests/integration/feature_repos/universal/online_store/elasticsearch.py new file mode 100644 index 00000000000..c62a9009caf --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/elasticsearch.py @@ -0,0 +1,28 @@ +from typing import Dict + +from testcontainers.elasticsearch import ElasticSearchContainer + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class ElasticSearchOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.container = ElasticSearchContainer( + "elasticsearch:8.3.3", + ).with_exposed_ports(9200) + + def create_online_store(self) -> Dict[str, str]: + self.container.start() + return { + "host": "localhost", + "type": "elasticsearch", + "port": self.container.get_exposed_port(9200), + "vector_len": 2, + "similarity": "cosine", + } + + def teardown(self): + self.container.stop() diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 5d6462e5e3d..9beba4d72b5 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -789,7 +789,7 @@ def assert_feature_service_entity_mapping_correctness( @pytest.mark.integration -@pytest.mark.universal_online_stores(only=["pgvector"]) +@pytest.mark.universal_online_stores(only=["pgvector", "elasticsearch"]) def test_retrieve_online_documents(environment, fake_document_data): fs = environment.feature_store df, data_source = fake_document_data diff --git a/setup.py b/setup.py index 6cc728ee98d..9181e64c2f6 100644 --- a/setup.py +++ b/setup.py @@ -150,6 +150,8 @@ DELTA_REQUIRED = ["deltalake"] +ELASTICSEARCH_REQUIRED = ["elasticsearch>=8.13.0"] + CI_REQUIRED = ( [ "build", @@ -211,6 +213,7 @@ + GRPCIO_REQUIRED + DUCKDB_REQUIRED + DELTA_REQUIRED + + ELASTICSEARCH_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -377,6 +380,7 @@ def run(self): "duckdb": DUCKDB_REQUIRED, "ikv": IKV_REQUIRED, "delta": DELTA_REQUIRED, + "elasticsearch": ELASTICSEARCH_REQUIRED, }, include_package_data=True, license="Apache", From 0e4215060f97b7629015ab65ac526dfef0a1f7d4 Mon Sep 17 00:00:00 2001 From: Pushkar Gupta Date: Tue, 14 May 2024 09:25:54 -0700 Subject: [PATCH 57/73] feat: Feast/IKV upgrade client version (#4200) --- .../contrib/ikv_online_store/ikv.py | 34 ++++++++++++------- setup.py | 2 +- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py b/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py index 9d888aad3d8..90df7f46860 100644 --- a/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py +++ b/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py @@ -82,9 +82,8 @@ def online_write_batch( progress: Function to be called once a batch of rows is written to the online store, used to show progress. """ - # update should have been called before - if self._writer is None: - return + self._init_writer(config=config) + assert self._writer is not None for entity_key, features, event_timestamp, _ in data: entity_id: str = compute_entity_id( @@ -120,6 +119,8 @@ def online_read( item is the event timestamp for the row, and the second item is a dict mapping feature names to values, which are returned in proto format. """ + self._init_reader(config=config) + if not len(entity_keys): return [] @@ -174,7 +175,6 @@ def _decode_fields_for_primary_key( return dt, features - # called before any read/write requests are issued @log_exceptions_and_usage(online_store="ikv") def update( self, @@ -199,7 +199,7 @@ def update( partial: If true, tables_to_delete and tables_to_keep are not exhaustive lists, so infrastructure corresponding to other feature views should be not be touched. """ - self._init_clients(config=config) + self._init_writer(config=config) assert self._writer is not None # note: we assume tables_to_keep does not overlap with tables_to_delete @@ -223,7 +223,7 @@ def teardown( tables: Feature views whose corresponding infrastructure should be deleted. entities: Entities whose corresponding infrastructure should be deleted. """ - self._init_clients(config=config) + self._init_writer(config=config) assert self._writer is not None # drop fields corresponding to this feature-view @@ -269,20 +269,28 @@ def _create_document( return builder.build() - def _init_clients(self, config: RepoConfig): - """Initializes (if required) reader/writer ikv clients.""" - online_config = config.online_store - assert isinstance(online_config, IKVOnlineStoreConfig) - client_options = IKVOnlineStore._config_to_client_options(online_config) - + def _init_writer(self, config: RepoConfig): + """Initializes ikv writer client.""" # initialize writer if self._writer is None: + online_config = config.online_store + assert isinstance(online_config, IKVOnlineStoreConfig) + client_options = IKVOnlineStore._config_to_client_options(online_config) + self._writer = create_new_writer(client_options) + self._writer.startup() # blocking operation - # initialize reader, iff mount_dir is specified + def _init_reader(self, config: RepoConfig): + """Initializes ikv reader client.""" + # initialize reader if self._reader is None: + online_config = config.online_store + assert isinstance(online_config, IKVOnlineStoreConfig) + client_options = IKVOnlineStore._config_to_client_options(online_config) + if online_config.mount_directory and len(online_config.mount_directory) > 0: self._reader = create_new_reader(client_options) + self._reader.startup() # blocking operation @staticmethod def _config_to_client_options(config: IKVOnlineStoreConfig) -> ClientOptions: diff --git a/setup.py b/setup.py index 9181e64c2f6..cdab69b6848 100644 --- a/setup.py +++ b/setup.py @@ -127,7 +127,7 @@ ] IKV_REQUIRED = [ - "ikvpy>=0.0.23", + "ikvpy>=0.0.36", ] HAZELCAST_REQUIRED = [ From d91d7e0da69d15c7aa14e736b608ed9f5ece3504 Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Wed, 15 May 2024 08:31:05 -0400 Subject: [PATCH 58/73] fix: Improve the code related to on-demand-featureview. (#4203) * fix: Improve the code related to on-demand-featureview. Signed-off-by: Shuchu Han * fix: add the pyarrow.substrait import. Signed-off-by: Shuchu Han --------- Signed-off-by: Shuchu Han --- sdk/python/feast/on_demand_feature_view.py | 90 +++++++++---------- .../transformation/pandas_transformation.py | 31 ++----- .../transformation/python_transformation.py | 20 ++--- .../substrait_transformation.py | 6 +- .../unit/infra/test_inference_unit_tests.py | 4 +- .../unit/online_store/test_online_writes.py | 6 +- .../tests/unit/test_on_demand_feature_view.py | 9 -- .../test_on_demand_python_transformation.py | 16 ++-- 8 files changed, 73 insertions(+), 109 deletions(-) diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index e5de1f20a65..b501b87a0e4 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -4,7 +4,7 @@ import warnings from datetime import datetime from types import FunctionType -from typing import Any, Dict, List, Optional, Type, Union +from typing import Any, Optional, Union import dill import pandas as pd @@ -62,15 +62,15 @@ class OnDemandFeatureView(BaseFeatureView): """ name: str - features: List[Field] - source_feature_view_projections: Dict[str, FeatureViewProjection] - source_request_sources: Dict[str, RequestSource] + features: list[Field] + source_feature_view_projections: dict[str, FeatureViewProjection] + source_request_sources: dict[str, RequestSource] feature_transformation: Union[ PandasTransformation, PythonTransformation, SubstraitTransformation ] mode: str description: str - tags: Dict[str, str] + tags: dict[str, str] owner: str @log_exceptions # noqa: C901 @@ -78,8 +78,8 @@ def __init__( # noqa: C901 self, *, name: str, - schema: List[Field], - sources: List[ + schema: list[Field], + sources: list[ Union[ FeatureView, RequestSource, @@ -93,7 +93,7 @@ def __init__( # noqa: C901 ], mode: str = "pandas", description: str = "", - tags: Optional[Dict[str, str]] = None, + tags: Optional[dict[str, str]] = None, owner: str = "", ): """ @@ -124,12 +124,13 @@ def __init__( # noqa: C901 owner=owner, ) - if mode not in {"python", "pandas", "substrait"}: - raise Exception( - f"Unknown mode {mode}. OnDemandFeatureView only supports python or pandas UDFs and substrait." + self.mode = mode.lower() + + if self.mode not in {"python", "pandas", "substrait"}: + raise ValueError( + f"Unknown mode {self.mode}. OnDemandFeatureView only supports python or pandas UDFs and substrait." ) - else: - self.mode = mode + if not feature_transformation: if udf: warnings.warn( @@ -137,19 +138,17 @@ def __init__( # noqa: C901 DeprecationWarning, ) # Note inspecting the return signature won't work with isinstance so this is the best alternative - if mode == "pandas": + if self.mode == "pandas": feature_transformation = PandasTransformation(udf, udf_string) - elif mode == "python": + elif self.mode == "python": feature_transformation = PythonTransformation(udf, udf_string) - else: - pass else: - raise Exception( + raise ValueError( "OnDemandFeatureView needs to be initialized with either feature_transformation or udf arguments" ) - self.source_feature_view_projections: Dict[str, FeatureViewProjection] = {} - self.source_request_sources: Dict[str, RequestSource] = {} + self.source_feature_view_projections: dict[str, FeatureViewProjection] = {} + self.source_request_sources: dict[str, RequestSource] = {} for odfv_source in sources: if isinstance(odfv_source, RequestSource): self.source_request_sources[odfv_source.name] = odfv_source @@ -163,7 +162,7 @@ def __init__( # noqa: C901 self.feature_transformation = feature_transformation @property - def proto_class(self) -> Type[OnDemandFeatureViewProto]: + def proto_class(self) -> type[OnDemandFeatureViewProto]: return OnDemandFeatureViewProto def __copy__(self): @@ -336,7 +335,7 @@ def from_proto( user_defined_function_proto=backwards_compatible_udf, ) else: - raise Exception("At least one transformation type needs to be provided") + raise ValueError("At least one transformation type needs to be provided") on_demand_feature_view_obj = cls( name=on_demand_feature_view_proto.spec.name, @@ -372,18 +371,18 @@ def from_proto( return on_demand_feature_view_obj - def get_request_data_schema(self) -> Dict[str, ValueType]: - schema: Dict[str, ValueType] = {} + def get_request_data_schema(self) -> dict[str, ValueType]: + schema: dict[str, ValueType] = {} for request_source in self.source_request_sources.values(): - if isinstance(request_source.schema, List): + if isinstance(request_source.schema, list): new_schema = {} for field in request_source.schema: new_schema[field.name] = field.dtype.to_value_type() schema.update(new_schema) - elif isinstance(request_source.schema, Dict): + elif isinstance(request_source.schema, dict): schema.update(request_source.schema) else: - raise Exception( + raise TypeError( f"Request source schema is not correct type: ${str(type(request_source.schema))}" ) return schema @@ -401,7 +400,10 @@ def transform_ibis( if not isinstance(ibis_table, Table): raise TypeError("transform_ibis only accepts ibis.expr.types.Table") - assert type(self.feature_transformation) == SubstraitTransformation + if not isinstance(self.feature_transformation, SubstraitTransformation): + raise TypeError( + "The feature_transformation is not SubstraitTransformation type while calling transform_ibis()." + ) columns_to_cleanup = [] for source_fv_projection in self.source_feature_view_projections.values(): @@ -423,7 +425,7 @@ def transform_ibis( transformed_table = transformed_table.drop(*columns_to_cleanup) - rename_columns: Dict[str, str] = {} + rename_columns: dict[str, str] = {} for feature in self.features: short_name = feature.name long_name = self._get_projected_feature_name(feature.name) @@ -454,11 +456,9 @@ def transform_arrow( pa_table = pa_table.append_column( feature.name, pa_table[full_feature_ref] ) - # pa_table[feature.name] = pa_table[full_feature_ref] columns_to_cleanup.append(feature.name) elif feature.name in pa_table.column_names: # Make sure the full feature name is always present - # pa_table[full_feature_ref] = pa_table[feature.name] pa_table = pa_table.append_column( full_feature_ref, pa_table[feature.name] ) @@ -469,7 +469,7 @@ def transform_arrow( ) # Work out whether the correct columns names are used. - rename_columns: Dict[str, str] = {} + rename_columns: dict[str, str] = {} for feature in self.features: short_name = feature.name long_name = self._get_projected_feature_name(feature.name) @@ -494,12 +494,12 @@ def transform_arrow( def transform_dict( self, - feature_dict: Dict[str, Any], # type: ignore - ) -> Dict[str, Any]: + feature_dict: dict[str, Any], # type: ignore + ) -> dict[str, Any]: # we need a mapping from full feature name to short and back to do a renaming # The simplest thing to do is to make the full reference, copy the columns with the short reference # and rerun - columns_to_cleanup: List[str] = [] + columns_to_cleanup: list[str] = [] for source_fv_projection in self.source_feature_view_projections.values(): for feature in source_fv_projection.features: full_feature_ref = f"{source_fv_projection.name}__{feature.name}" @@ -512,7 +512,7 @@ def transform_dict( feature_dict[full_feature_ref] = feature_dict[feature.name] columns_to_cleanup.append(str(full_feature_ref)) - output_dict: Dict[str, Any] = self.feature_transformation.transform( + output_dict: dict[str, Any] = self.feature_transformation.transform( feature_dict ) for feature_name in columns_to_cleanup: @@ -542,8 +542,8 @@ def infer_features(self) -> None: f"Could not infer Features for the feature view '{self.name}'.", ) - def _construct_random_input(self) -> Dict[str, List[Any]]: - rand_dict_value: Dict[ValueType, List[Any]] = { + def _construct_random_input(self) -> dict[str, list[Any]]: + rand_dict_value: dict[ValueType, list[Any]] = { ValueType.BYTES: [str.encode("hello world")], ValueType.STRING: ["hello world"], ValueType.INT32: [1], @@ -582,11 +582,11 @@ def _construct_random_input(self) -> Dict[str, List[Any]]: @staticmethod def get_requested_odfvs( feature_refs, project, registry - ) -> List["OnDemandFeatureView"]: + ) -> list["OnDemandFeatureView"]: all_on_demand_feature_views = registry.list_on_demand_feature_views( project, allow_cache=True ) - requested_on_demand_feature_views: List[OnDemandFeatureView] = [] + requested_on_demand_feature_views: list[OnDemandFeatureView] = [] for odfv in all_on_demand_feature_views: for feature in odfv.features: if f"{odfv.name}:{feature.name}" in feature_refs: @@ -597,8 +597,8 @@ def get_requested_odfvs( def on_demand_feature_view( *, - schema: List[Field], - sources: List[ + schema: list[Field], + sources: list[ Union[ FeatureView, RequestSource, @@ -607,7 +607,7 @@ def on_demand_feature_view( ], mode: str = "pandas", description: str = "", - tags: Optional[Dict[str, str]] = None, + tags: Optional[dict[str, str]] = None, owner: str = "", ): """ @@ -643,9 +643,9 @@ def decorator(user_function): ) transformation = PandasTransformation(user_function, udf_string) elif mode == "python": - if return_annotation not in (inspect._empty, Dict[str, Any]): + if return_annotation not in (inspect._empty, dict[str, Any]): raise TypeError( - f"return signature for {user_function} is {return_annotation} but should be Dict[str, Any]" + f"return signature for {user_function} is {return_annotation} but should be dict[str, Any]" ) transformation = PythonTransformation(user_function, udf_string) elif mode == "substrait": diff --git a/sdk/python/feast/transformation/pandas_transformation.py b/sdk/python/feast/transformation/pandas_transformation.py index 7e706810cb4..e9dab721608 100644 --- a/sdk/python/feast/transformation/pandas_transformation.py +++ b/sdk/python/feast/transformation/pandas_transformation.py @@ -1,5 +1,5 @@ from types import FunctionType -from typing import Any, Dict, List +from typing import Any import dill import pandas as pd @@ -28,35 +28,16 @@ def __init__(self, udf: FunctionType, udf_string: str = ""): self.udf_string = udf_string def transform_arrow( - self, pa_table: pyarrow.Table, features: List[Field] + self, pa_table: pyarrow.Table, features: list[Field] ) -> pyarrow.Table: - if not isinstance(pa_table, pyarrow.Table): - raise TypeError( - f"pa_table should be type pyarrow.Table but got {type(pa_table).__name__}" - ) - output_df = self.udf.__call__(pa_table.to_pandas()) - output_df = pyarrow.Table.from_pandas(output_df) - if not isinstance(output_df, pyarrow.Table): - raise TypeError( - f"output_df should be type pyarrow.Table but got {type(output_df).__name__}" - ) - return output_df + output_df_pandas = self.udf.__call__(pa_table.to_pandas()) + return pyarrow.Table.from_pandas(output_df_pandas) def transform(self, input_df: pd.DataFrame) -> pd.DataFrame: - if not isinstance(input_df, pd.DataFrame): - raise TypeError( - f"input_df should be type pd.DataFrame but got {type(input_df).__name__}" - ) - output_df = self.udf.__call__(input_df) - if not isinstance(output_df, pd.DataFrame): - raise TypeError( - f"output_df should be type pd.DataFrame but got {type(output_df).__name__}" - ) - return output_df + return self.udf.__call__(input_df) - def infer_features(self, random_input: Dict[str, List[Any]]) -> List[Field]: + def infer_features(self, random_input: dict[str, list[Any]]) -> list[Field]: df = pd.DataFrame.from_dict(random_input) - output_df: pd.DataFrame = self.transform(df) return [ diff --git a/sdk/python/feast/transformation/python_transformation.py b/sdk/python/feast/transformation/python_transformation.py index 88cde7cc726..2a9c7db8763 100644 --- a/sdk/python/feast/transformation/python_transformation.py +++ b/sdk/python/feast/transformation/python_transformation.py @@ -1,5 +1,5 @@ from types import FunctionType -from typing import Any, Dict, List +from typing import Any import dill import pyarrow @@ -26,27 +26,19 @@ def __init__(self, udf: FunctionType, udf_string: str = ""): self.udf_string = udf_string def transform_arrow( - self, pa_table: pyarrow.Table, features: List[Field] + self, pa_table: pyarrow.Table, features: list[Field] ) -> pyarrow.Table: raise Exception( - 'OnDemandFeatureView mode "python" not supported for offline processing.' + 'OnDemandFeatureView with mode "python" does not support offline processing.' ) - def transform(self, input_dict: Dict) -> Dict: - if not isinstance(input_dict, Dict): - raise TypeError( - f"input_dict should be type Dict[str, Any] but got {type(input_dict).__name__}" - ) + def transform(self, input_dict: dict) -> dict: # Ensuring that the inputs are included as well output_dict = self.udf.__call__(input_dict) - if not isinstance(output_dict, Dict): - raise TypeError( - f"output_dict should be type Dict[str, Any] but got {type(output_dict).__name__}" - ) return {**input_dict, **output_dict} - def infer_features(self, random_input: Dict[str, List[Any]]) -> List[Field]: - output_dict: Dict[str, List[Any]] = self.transform(random_input) + def infer_features(self, random_input: dict[str, list[Any]]) -> list[Field]: + output_dict: dict[str, list[Any]] = self.transform(random_input) return [ Field( diff --git a/sdk/python/feast/transformation/substrait_transformation.py b/sdk/python/feast/transformation/substrait_transformation.py index 02b94d85726..17c40cf0a16 100644 --- a/sdk/python/feast/transformation/substrait_transformation.py +++ b/sdk/python/feast/transformation/substrait_transformation.py @@ -1,5 +1,5 @@ from types import FunctionType -from typing import Any, Dict, List +from typing import Any import dill import pandas as pd @@ -42,7 +42,7 @@ def transform_ibis(self, table): return self.ibis_function(table) def transform_arrow( - self, pa_table: pyarrow.Table, features: List[Field] = [] + self, pa_table: pyarrow.Table, features: list[Field] = [] ) -> pyarrow.Table: def table_provider(names, schema: pyarrow.Schema): return pa_table.select(schema.names) @@ -56,7 +56,7 @@ def table_provider(names, schema: pyarrow.Schema): return table - def infer_features(self, random_input: Dict[str, List[Any]]) -> List[Field]: + def infer_features(self, random_input: dict[str, list[Any]]) -> list[Field]: df = pd.DataFrame.from_dict(random_input) output_df: pd.DataFrame = self.transform(df) diff --git a/sdk/python/tests/unit/infra/test_inference_unit_tests.py b/sdk/python/tests/unit/infra/test_inference_unit_tests.py index e4acef97136..3d8fe8c9677 100644 --- a/sdk/python/tests/unit/infra/test_inference_unit_tests.py +++ b/sdk/python/tests/unit/infra/test_inference_unit_tests.py @@ -83,8 +83,8 @@ def test_view(features_df: pd.DataFrame) -> pd.DataFrame: ], mode="python", ) - def python_native_test_view(input_dict: Dict[str, Any]) -> Dict[str, Any]: - output_dict: Dict[str, Any] = { + def python_native_test_view(input_dict: dict[str, Any]) -> dict[str, Any]: + output_dict: dict[str, Any] = { "output": input_dict["some_date"], "object_output": str(input_dict["some_date"]), } diff --git a/sdk/python/tests/unit/online_store/test_online_writes.py b/sdk/python/tests/unit/online_store/test_online_writes.py index 5fb13519692..0f7547a93b5 100644 --- a/sdk/python/tests/unit/online_store/test_online_writes.py +++ b/sdk/python/tests/unit/online_store/test_online_writes.py @@ -16,7 +16,7 @@ import tempfile import unittest from datetime import datetime, timedelta -from typing import Any, Dict +from typing import Any from feast import Entity, FeatureStore, FeatureView, FileSource, RepoConfig from feast.driver_test_data import create_driver_hourly_stats_df @@ -81,8 +81,8 @@ def setUp(self): schema=[Field(name="conv_rate_plus_acc", dtype=Float64)], mode="python", ) - def test_view(inputs: Dict[str, Any]) -> Dict[str, Any]: - output: Dict[str, Any] = { + def test_view(inputs: dict[str, Any]) -> dict[str, Any]: + output: dict[str, Any] = { "conv_rate_plus_acc": [ conv_rate + acc_rate for conv_rate, acc_rate in zip( diff --git a/sdk/python/tests/unit/test_on_demand_feature_view.py b/sdk/python/tests/unit/test_on_demand_feature_view.py index 402aa4e0e33..d9cc5dee50d 100644 --- a/sdk/python/tests/unit/test_on_demand_feature_view.py +++ b/sdk/python/tests/unit/test_on_demand_feature_view.py @@ -195,15 +195,6 @@ def test_python_native_transformation_mode(): == PythonTransformation(python_native_udf, "python native udf source code") ) - with pytest.raises(TypeError): - # This should fail - on_demand_feature_view_python_native_err.feature_transformation.transform( - { - "feature1": 0, - "feature2": 1, - } - ) - assert on_demand_feature_view_python_native.transform_dict( { "feature1": 0, diff --git a/sdk/python/tests/unit/test_on_demand_python_transformation.py b/sdk/python/tests/unit/test_on_demand_python_transformation.py index e2db96c5f87..ebe797ffdbf 100644 --- a/sdk/python/tests/unit/test_on_demand_python_transformation.py +++ b/sdk/python/tests/unit/test_on_demand_python_transformation.py @@ -2,7 +2,7 @@ import tempfile import unittest from datetime import datetime, timedelta -from typing import Any, Dict +from typing import Any import pandas as pd import pytest @@ -82,8 +82,8 @@ def pandas_view(inputs: pd.DataFrame) -> pd.DataFrame: schema=[Field(name="conv_rate_plus_acc_python", dtype=Float64)], mode="python", ) - def python_view(inputs: Dict[str, Any]) -> Dict[str, Any]: - output: Dict[str, Any] = { + def python_view(inputs: dict[str, Any]) -> dict[str, Any]: + output: dict[str, Any] = { "conv_rate_plus_acc_python": [ conv_rate + acc_rate for conv_rate, acc_rate in zip( @@ -101,8 +101,8 @@ def python_view(inputs: Dict[str, Any]) -> Dict[str, Any]: ], mode="python", ) - def python_demo_view(inputs: Dict[str, Any]) -> Dict[str, Any]: - output: Dict[str, Any] = { + def python_demo_view(inputs: dict[str, Any]) -> dict[str, Any]: + output: dict[str, Any] = { "conv_rate_plus_val1_python": [ conv_rate + acc_rate for conv_rate, acc_rate in zip( @@ -125,8 +125,8 @@ def python_demo_view(inputs: Dict[str, Any]) -> Dict[str, Any]: ], mode="python", ) - def python_singleton_view(inputs: Dict[str, Any]) -> Dict[str, Any]: - output: Dict[str, Any] = dict(conv_rate_plus_acc_python=float("-inf")) + def python_singleton_view(inputs: dict[str, Any]) -> dict[str, Any]: + output: dict[str, Any] = dict(conv_rate_plus_acc_python=float("-inf")) output["conv_rate_plus_acc_python_singleton"] = ( inputs["conv_rate"] + inputs["acc_rate"] ) @@ -134,7 +134,7 @@ def python_singleton_view(inputs: Dict[str, Any]) -> Dict[str, Any]: with pytest.raises(TypeError): # Note the singleton view will fail as the type is - # expected to be a List which can be confirmed in _infer_features_dict + # expected to be a list which can be confirmed in _infer_features_dict self.store.apply( [ driver, From a17725daec9e7355591e7ff2bc57202d5fa3f0c1 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Thu, 16 May 2024 23:13:08 +0400 Subject: [PATCH 59/73] feat: Move data source validation entrypoint to offline store (#4197) * move validate_data_source entrypoint to offline store Signed-off-by: tokoko * add validate_data_source to foo provider Signed-off-by: tokoko --------- Signed-off-by: tokoko --- .../feast/infra/offline_stores/offline_store.py | 14 ++++++++++++++ sdk/python/feast/infra/passthrough_provider.py | 8 ++++++++ sdk/python/feast/infra/provider.py | 16 ++++++++++++++++ sdk/python/feast/repo_operations.py | 6 ++++-- sdk/python/tests/foo_provider.py | 8 ++++++++ 5 files changed, 50 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index 4851aecae28..d9738445313 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -351,3 +351,17 @@ def offline_write_batch( to show progress. """ raise NotImplementedError + + @staticmethod + def validate_data_source( + config: RepoConfig, + data_source: DataSource, + ): + """ + Validates the underlying data source. + + Args: + config: Configuration object used to configure a feature store. + data_source: DataSource object that needs to be validated + """ + data_source.validate(config=config) diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 48d2f8ef185..b96c3433ea7 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -7,6 +7,7 @@ from feast import importer from feast.batch_feature_view import BatchFeatureView +from feast.data_source import DataSource from feast.entity import Entity from feast.feature_logging import FeatureServiceLoggingSource from feast.feature_service import FeatureService @@ -383,3 +384,10 @@ def retrieve_feature_service_logs( start_date=make_tzaware(start_date), end_date=make_tzaware(end_date), ) + + def validate_data_source( + self, + config: RepoConfig, + data_source: DataSource, + ): + self.offline_store.validate_data_source(config=config, data_source=data_source) diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 22f6088474a..93077f40b97 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -8,6 +8,7 @@ from tqdm import tqdm from feast import FeatureService, errors +from feast.data_source import DataSource from feast.entity import Entity from feast.feature_view import FeatureView from feast.importer import import_class @@ -352,6 +353,21 @@ def retrieve_online_documents( """ pass + @abstractmethod + def validate_data_source( + self, + config: RepoConfig, + data_source: DataSource, + ): + """ + Validates the underlying data source. + + Args: + config: Configuration object used to configure a feature store. + data_source: DataSource object that needs to be validated + """ + pass + def get_provider(config: RepoConfig) -> Provider: if "." not in config.provider: diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 0b659b960c2..296155ce464 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -205,10 +205,11 @@ def plan(repo_config: RepoConfig, repo_path: Path, skip_source_validation: bool) project, registry, repo, store = _prepare_registry_and_repo(repo_config, repo_path) if not skip_source_validation: + provider = store._get_provider() data_sources = [t.batch_source for t in repo.feature_views] # Make sure the data source used by this feature view is supported by Feast for data_source in data_sources: - data_source.validate(store.config) + provider.validate_data_source(store.config, data_source) registry_diff, infra_diff, _ = store.plan(repo) click.echo(registry_diff.to_string()) @@ -282,10 +283,11 @@ def apply_total_with_repo_instance( skip_source_validation: bool, ): if not skip_source_validation: + provider = store._get_provider() data_sources = [t.batch_source for t in repo.feature_views] # Make sure the data source used by this feature view is supported by Feast for data_source in data_sources: - data_source.validate(store.config) + provider.validate_data_source(store.config, data_source) # For each object in the registry, determine whether it should be kept or deleted. ( diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index 3b9146a7b96..bd1e247a7b9 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -7,6 +7,7 @@ from tqdm import tqdm from feast import Entity, FeatureService, FeatureView, RepoConfig +from feast.data_source import DataSource from feast.infra.offline_stores.offline_store import RetrievalJob from feast.infra.provider import Provider from feast.infra.registry.base_registry import BaseRegistry @@ -130,3 +131,10 @@ def retrieve_online_documents( ] ]: return [] + + def validate_data_source( + self, + config: RepoConfig, + data_source: DataSource, + ): + pass From 6a04c48b4b84fb9905df638e5c4041c12532b053 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Thu, 16 May 2024 23:17:18 +0400 Subject: [PATCH 60/73] feat: Add s3 remote storage export for duckdb (#4195) add s3 remote export, tests for duckdb Signed-off-by: tokoko --- .../feast/infra/offline_stores/duckdb.py | 30 ++++++- .../feast/infra/offline_stores/file_source.py | 10 ++- sdk/python/feast/infra/offline_stores/ibis.py | 45 +++++++++++ sdk/python/pytest.ini | 8 +- sdk/python/tests/conftest.py | 8 +- .../feature_repos/repo_configuration.py | 9 +++ .../universal/data_sources/file.py | 78 +++++++++++++++++++ .../contrib/spark/test_spark.py | 2 +- .../test_universal_historical_retrieval.py | 3 +- .../integration/registration/test_registry.py | 24 ++---- 10 files changed, 191 insertions(+), 26 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/duckdb.py b/sdk/python/feast/infra/offline_stores/duckdb.py index 8e392425ea5..a639d54add5 100644 --- a/sdk/python/feast/infra/offline_stores/duckdb.py +++ b/sdk/python/feast/infra/offline_stores/duckdb.py @@ -33,7 +33,11 @@ def _read_data_source(data_source: DataSource) -> Table: if isinstance(data_source.file_format, ParquetFormat): return ibis.read_parquet(data_source.path) elif isinstance(data_source.file_format, DeltaFormat): - return ibis.read_delta(data_source.path) + storage_options = { + "AWS_ENDPOINT_URL": data_source.s3_endpoint_override, + } + + return ibis.read_delta(data_source.path, storage_options=storage_options) def _write_data_source( @@ -72,10 +76,18 @@ def _write_data_source( new_table = pyarrow.concat_tables([table, prev_table]) ibis.memtable(new_table).to_parquet(file_options.uri) elif isinstance(data_source.file_format, DeltaFormat): + storage_options = { + "AWS_ENDPOINT_URL": str(data_source.s3_endpoint_override), + } + if mode == "append": from deltalake import DeltaTable - prev_schema = DeltaTable(file_options.uri).schema().to_pyarrow() + prev_schema = ( + DeltaTable(file_options.uri, storage_options=storage_options) + .schema() + .to_pyarrow() + ) table = table.cast(ibis.Schema.from_pyarrow(prev_schema)) write_mode = "append" elif mode == "overwrite": @@ -85,13 +97,19 @@ def _write_data_source( else "error" ) - table.to_delta(file_options.uri, mode=write_mode) + table.to_delta( + file_options.uri, mode=write_mode, storage_options=storage_options + ) class DuckDBOfflineStoreConfig(FeastConfigBaseModel): type: StrictStr = "duckdb" # """ Offline store type selector""" + staging_location: Optional[str] = None + + staging_location_endpoint_override: Optional[str] = None + class DuckDBOfflineStore(OfflineStore): @staticmethod @@ -116,6 +134,8 @@ def pull_latest_from_table_or_query( end_date=end_date, data_source_reader=_read_data_source, data_source_writer=_write_data_source, + staging_location=config.offline_store.staging_location, + staging_location_endpoint_override=config.offline_store.staging_location_endpoint_override, ) @staticmethod @@ -138,6 +158,8 @@ def get_historical_features( full_feature_names=full_feature_names, data_source_reader=_read_data_source, data_source_writer=_write_data_source, + staging_location=config.offline_store.staging_location, + staging_location_endpoint_override=config.offline_store.staging_location_endpoint_override, ) @staticmethod @@ -160,6 +182,8 @@ def pull_all_from_table_or_query( end_date=end_date, data_source_reader=_read_data_source, data_source_writer=_write_data_source, + staging_location=config.offline_store.staging_location, + staging_location_endpoint_override=config.offline_store.staging_location_endpoint_override, ) @staticmethod diff --git a/sdk/python/feast/infra/offline_stores/file_source.py b/sdk/python/feast/infra/offline_stores/file_source.py index 596f3464a99..3fdc6cba31a 100644 --- a/sdk/python/feast/infra/offline_stores/file_source.py +++ b/sdk/python/feast/infra/offline_stores/file_source.py @@ -179,7 +179,15 @@ def get_table_column_names_and_types( elif isinstance(self.file_format, DeltaFormat): from deltalake import DeltaTable - schema = DeltaTable(self.path).schema().to_pyarrow() + storage_options = { + "AWS_ENDPOINT_URL": str(self.s3_endpoint_override), + } + + schema = ( + DeltaTable(self.path, storage_options=storage_options) + .schema() + .to_pyarrow() + ) else: raise Exception(f"Unknown FileFormat -> {self.file_format}") diff --git a/sdk/python/feast/infra/offline_stores/ibis.py b/sdk/python/feast/infra/offline_stores/ibis.py index b9efb87a36a..6cc1606a458 100644 --- a/sdk/python/feast/infra/offline_stores/ibis.py +++ b/sdk/python/feast/infra/offline_stores/ibis.py @@ -47,6 +47,8 @@ def pull_latest_from_table_or_query_ibis( end_date: datetime, data_source_reader: Callable[[DataSource], Table], data_source_writer: Callable[[pyarrow.Table, DataSource], None], + staging_location: Optional[str] = None, + staging_location_endpoint_override: Optional[str] = None, ) -> RetrievalJob: fields = join_key_columns + feature_name_columns + [timestamp_field] if created_timestamp_column: @@ -82,6 +84,8 @@ def pull_latest_from_table_or_query_ibis( full_feature_names=False, metadata=None, data_source_writer=data_source_writer, + staging_location=staging_location, + staging_location_endpoint_override=staging_location_endpoint_override, ) @@ -140,6 +144,8 @@ def get_historical_features_ibis( data_source_reader: Callable[[DataSource], Table], data_source_writer: Callable[[pyarrow.Table, DataSource], None], full_feature_names: bool = False, + staging_location: Optional[str] = None, + staging_location_endpoint_override: Optional[str] = None, ) -> RetrievalJob: entity_schema = _get_entity_schema( entity_df=entity_df, @@ -231,6 +237,8 @@ def read_fv( max_event_timestamp=timestamp_range[1], ), data_source_writer=data_source_writer, + staging_location=staging_location, + staging_location_endpoint_override=staging_location_endpoint_override, ) @@ -244,6 +252,8 @@ def pull_all_from_table_or_query_ibis( end_date: datetime, data_source_reader: Callable[[DataSource], Table], data_source_writer: Callable[[pyarrow.Table, DataSource], None], + staging_location: Optional[str] = None, + staging_location_endpoint_override: Optional[str] = None, ) -> RetrievalJob: fields = join_key_columns + feature_name_columns + [timestamp_field] start_date = start_date.astimezone(tz=utc) @@ -270,6 +280,8 @@ def pull_all_from_table_or_query_ibis( full_feature_names=False, metadata=None, data_source_writer=data_source_writer, + staging_location=staging_location, + staging_location_endpoint_override=staging_location_endpoint_override, ) @@ -411,6 +423,23 @@ def point_in_time_join( return acc_table +def list_s3_files(path: str, endpoint_url: str) -> List[str]: + import boto3 + + s3 = boto3.client("s3", endpoint_url=endpoint_url) + if path.startswith("s3://"): + path = path[len("s3://") :] + bucket, prefix = path.split("/", 1) + objects = s3.list_objects_v2(Bucket=bucket, Prefix=prefix) + contents = objects["Contents"] + files = [ + f"s3://{bucket}/{content['Key']}" + for content in contents + if content["Key"].endswith("parquet") + ] + return files + + class IbisRetrievalJob(RetrievalJob): def __init__( self, @@ -419,6 +448,8 @@ def __init__( full_feature_names, metadata, data_source_writer, + staging_location, + staging_location_endpoint_override, ) -> None: super().__init__() self.table = table @@ -428,6 +459,8 @@ def __init__( self._full_feature_names = full_feature_names self._metadata = metadata self.data_source_writer = data_source_writer + self.staging_location = staging_location + self.staging_location_endpoint_override = staging_location_endpoint_override def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: return self.table.execute() @@ -456,3 +489,15 @@ def persist( @property def metadata(self) -> Optional[RetrievalMetadata]: return self._metadata + + def supports_remote_storage_export(self) -> bool: + return self.staging_location is not None + + def to_remote_storage(self) -> List[str]: + path = self.staging_location + f"/{str(uuid.uuid4())}" + + storage_options = {"AWS_ENDPOINT_URL": self.staging_location_endpoint_override} + + self.table.to_delta(path, storage_options=storage_options) + + return list_s3_files(path, self.staging_location_endpoint_override) diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini index 83317d36c98..8a162943221 100644 --- a/sdk/python/pytest.ini +++ b/sdk/python/pytest.ini @@ -5,4 +5,10 @@ markers = env = FEAST_USAGE=False - IS_TEST=True \ No newline at end of file + IS_TEST=True + +filterwarnings = + ignore::DeprecationWarning:pyspark.sql.pandas.*: + ignore::DeprecationWarning:pyspark.sql.connect.*: + ignore::DeprecationWarning:httpx.*: + ignore::FutureWarning:ibis_substrait.compiler.*: diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 6abe30822f2..c4a62be0c0a 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -13,11 +13,13 @@ # limitations under the License. import logging import multiprocessing +import os import random from datetime import datetime, timedelta from multiprocessing import Process from sys import platform from typing import Any, Dict, List, Tuple, no_type_check +from unittest import mock import pandas as pd import pytest @@ -180,7 +182,11 @@ def environment(request, worker_id): request.param, worker_id=worker_id, fixture_request=request ) - yield e + if hasattr(e.data_source_creator, "mock_environ"): + with mock.patch.dict(os.environ, e.data_source_creator.mock_environ): + yield e + else: + yield e e.feature_store.teardown() e.data_source_creator.teardown() diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 4007106a064..311325536ed 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -33,6 +33,7 @@ from tests.integration.feature_repos.universal.data_sources.file import ( DuckDBDataSourceCreator, DuckDBDeltaDataSourceCreator, + DuckDBDeltaS3DataSourceCreator, FileDataSourceCreator, ) from tests.integration.feature_repos.universal.data_sources.redshift import ( @@ -122,6 +123,14 @@ ("local", DuckDBDeltaDataSourceCreator), ] +if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "True": + AVAILABLE_OFFLINE_STORES.extend( + [ + ("local", DuckDBDeltaS3DataSourceCreator), + ] + ) + + AVAILABLE_ONLINE_STORES: Dict[ str, Tuple[Union[str, Dict[Any, Any]], Optional[Type[OnlineStoreCreator]]] ] = { diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index 9cdc91a6c87..6f0ac02a003 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -10,6 +10,7 @@ from minio import Minio from testcontainers.core.generic import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs +from testcontainers.minio import MinioContainer from feast import FileSource from feast.data_format import DeltaFormat, ParquetFormat @@ -134,6 +135,74 @@ def create_logged_features_destination(self) -> LoggingDestination: return FileLoggingDestination(path=d) +class DeltaS3FileSourceCreator(FileDataSourceCreator): + def __init__(self, project_name: str, *args, **kwargs): + super().__init__(project_name) + self.minio = MinioContainer() + self.minio.start() + client = self.minio.get_client() + client.make_bucket("test") + host_ip = self.minio.get_container_host_ip() + exposed_port = self.minio.get_exposed_port(self.minio.port) + self.endpoint_url = f"http://{host_ip}:{exposed_port}" + + self.mock_environ = { + "AWS_ACCESS_KEY_ID": self.minio.access_key, + "AWS_SECRET_ACCESS_KEY": self.minio.secret_key, + "AWS_EC2_METADATA_DISABLED": "true", + "AWS_REGION": "us-east-1", + "AWS_ALLOW_HTTP": "true", + "AWS_S3_ALLOW_UNSAFE_RENAME": "true", + } + + def create_data_source( + self, + df: pd.DataFrame, + destination_name: str, + created_timestamp_column="created_ts", + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", + ) -> DataSource: + from deltalake.writer import write_deltalake + + destination_name = self.get_prefixed_table_name(destination_name) + + storage_options = { + "AWS_ACCESS_KEY_ID": self.minio.access_key, + "AWS_SECRET_ACCESS_KEY": self.minio.secret_key, + "AWS_ENDPOINT_URL": self.endpoint_url, + } + + path = f"s3://test/{str(uuid.uuid4())}/{destination_name}" + + write_deltalake(path, df, storage_options=storage_options) + + return FileSource( + file_format=DeltaFormat(), + path=path, + timestamp_field=timestamp_field, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping or {"ts_1": "ts"}, + s3_endpoint_override=self.endpoint_url, + ) + + def create_saved_dataset_destination(self) -> SavedDatasetFileStorage: + return SavedDatasetFileStorage( + path=f"s3://test/{str(uuid.uuid4())}", + file_format=DeltaFormat(), + s3_endpoint_override=self.endpoint_url, + ) + + # LoggingDestination is parquet-only + def create_logged_features_destination(self) -> LoggingDestination: + d = tempfile.mkdtemp(prefix=self.project_name) + self.keep.append(d) + return FileLoggingDestination(path=d) + + def teardown(self): + self.minio.stop() + + class FileParquetDatasetSourceCreator(FileDataSourceCreator): def create_data_source( self, @@ -273,3 +342,12 @@ class DuckDBDeltaDataSourceCreator(DeltaFileSourceCreator): def create_offline_store_config(self): self.duckdb_offline_store_config = DuckDBOfflineStoreConfig() return self.duckdb_offline_store_config + + +class DuckDBDeltaS3DataSourceCreator(DeltaS3FileSourceCreator): + def create_offline_store_config(self): + self.duckdb_offline_store_config = DuckDBOfflineStoreConfig( + staging_location="s3://test/staging", + staging_location_endpoint_override=self.endpoint_url, + ) + return self.duckdb_offline_store_config diff --git a/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py b/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py index e85c1d73119..bb4c4e63fc2 100644 --- a/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py +++ b/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py @@ -31,7 +31,7 @@ def test_spark_materialization_consistency(): batch_engine={"type": "spark.engine", "partitions": 10}, ) spark_environment = construct_test_environment( - spark_config, None, entity_key_serialization_version=1 + spark_config, None, entity_key_serialization_version=2 ) df = create_basic_driver_dataset() diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py index 958b829a60b..6d355e093ca 100644 --- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py +++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py @@ -139,8 +139,7 @@ def test_historical_features( if job_from_df.supports_remote_storage_export(): files = job_from_df.to_remote_storage() - print(files) - assert len(files) > 0 # This test should be way more detailed + assert len(files) # 0 # This test should be way more detailed start_time = datetime.utcnow() actual_df_from_df_entities = job_from_df.to_df() diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 70d118ecf95..9ad1a98a050 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -18,7 +18,7 @@ import pytest from pytest_lazyfixture import lazy_fixture -from testcontainers.core.container import DockerContainer +from testcontainers.minio import MinioContainer from feast import FileSource from feast.data_format import ParquetFormat @@ -64,25 +64,15 @@ def s3_registry() -> Registry: @pytest.fixture def minio_registry() -> Registry: - minio_user = "minio99" - minio_password = "minio123" bucket_name = "test-bucket" - container: DockerContainer = ( - DockerContainer("quay.io/minio/minio") - .with_exposed_ports(9000, 9001) - .with_env("MINIO_ROOT_USER", minio_user) - .with_env("MINIO_ROOT_PASSWORD", minio_password) - .with_command('server /data --console-address ":9001"') - .with_exposed_ports() - ) - + container = MinioContainer() container.start() + client = container.get_client() + client.make_bucket(bucket_name) - exposed_port = container.get_exposed_port("9000") container_host = container.get_container_host_ip() - - container.exec(f"mkdir /data/{bucket_name}") + exposed_port = container.get_exposed_port(container.port) registry_config = RegistryConfig( path=f"s3://{bucket_name}/registry.db", cache_ttl_seconds=600 @@ -90,8 +80,8 @@ def minio_registry() -> Registry: mock_environ = { "FEAST_S3_ENDPOINT_URL": f"http://{container_host}:{exposed_port}", - "AWS_ACCESS_KEY_ID": minio_user, - "AWS_SECRET_ACCESS_KEY": minio_password, + "AWS_ACCESS_KEY_ID": container.access_key, + "AWS_SECRET_ACCESS_KEY": container.secret_key, "AWS_SESSION_TOKEN": "", } From a417ea8c4e2151896fc098f929163f641e5dcced Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Fri, 17 May 2024 00:07:21 +0400 Subject: [PATCH 61/73] chore: Skip test_historical_features_main for Snowflake (#4206) skip test_historical_features_main for snowflake Signed-off-by: tokoko --- Makefile | 2 +- .../offline_store/test_universal_historical_retrieval.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9b537522181..f1d8c107bad 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ test-python-unit: python -m pytest -n 8 --color=yes sdk/python/tests test-python-integration: - python -m pytest -n 8 --integration -k "not minio_registry" --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests + python -m pytest -n 8 --integration -k "(not snowflake or not test_historical_features_main) and not minio_registry" --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests test-python-integration-local: @(docker info > /dev/null 2>&1 && \ diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py index 6d355e093ca..2a2820c10a7 100644 --- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py +++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py @@ -44,7 +44,7 @@ @pytest.mark.parametrize( "use_substrait_odfv", [True, False], ids=lambda v: f"substrait:{v}" ) -def test_historical_features( +def test_historical_features_main( environment, universal_data_sources, full_feature_names, use_substrait_odfv ): store = environment.feature_store From 20f5419d30c32b533e91043a9690007a84000512 Mon Sep 17 00:00:00 2001 From: ArijeetC <10762479+ArijeetC@users.noreply.github.com> Date: Fri, 17 May 2024 17:22:07 +0200 Subject: [PATCH 62/73] feat: Add optional private key params to Snowflake config (#4205) feat: Add private key params to Snowflake config Signed-off-by: Arijeet Chakrabarty --- sdk/python/feast/infra/materialization/snowflake_engine.py | 6 ++++++ sdk/python/feast/infra/offline_stores/snowflake.py | 6 ++++++ sdk/python/feast/infra/online_stores/snowflake.py | 6 ++++++ sdk/python/feast/infra/registry/snowflake.py | 6 ++++++ 4 files changed, 24 insertions(+) diff --git a/sdk/python/feast/infra/materialization/snowflake_engine.py b/sdk/python/feast/infra/materialization/snowflake_engine.py index 4a81982dcde..f77239398e6 100644 --- a/sdk/python/feast/infra/materialization/snowflake_engine.py +++ b/sdk/python/feast/infra/materialization/snowflake_engine.py @@ -67,6 +67,12 @@ class SnowflakeMaterializationEngineConfig(FeastConfigBaseModel): authenticator: Optional[str] = None """ Snowflake authenticator name """ + private_key: Optional[str] = None + """ Snowflake private key file path""" + + private_key_passphrase: Optional[str] = None + """ Snowflake private key file passphrase""" + database: StrictStr """ Snowflake database name """ diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py index cc59804467a..64c84318c9d 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake.py +++ b/sdk/python/feast/infra/offline_stores/snowflake.py @@ -105,6 +105,12 @@ class SnowflakeOfflineStoreConfig(FeastConfigBaseModel): authenticator: Optional[str] = None """ Snowflake authenticator name """ + private_key: Optional[str] = None + """ Snowflake private key file path""" + + private_key_passphrase: Optional[str] = None + """ Snowflake private key file passphrase""" + database: StrictStr """ Snowflake database name """ diff --git a/sdk/python/feast/infra/online_stores/snowflake.py b/sdk/python/feast/infra/online_stores/snowflake.py index f5600249c91..57e3bbbb8dc 100644 --- a/sdk/python/feast/infra/online_stores/snowflake.py +++ b/sdk/python/feast/infra/online_stores/snowflake.py @@ -51,6 +51,12 @@ class SnowflakeOnlineStoreConfig(FeastConfigBaseModel): authenticator: Optional[str] = None """ Snowflake authenticator name """ + private_key: Optional[str] = None + """ Snowflake private key file path""" + + private_key_passphrase: Optional[str] = None + """ Snowflake private key file passphrase""" + database: StrictStr """ Snowflake database name """ diff --git a/sdk/python/feast/infra/registry/snowflake.py b/sdk/python/feast/infra/registry/snowflake.py index 326d2e02266..169c8ae43ec 100644 --- a/sdk/python/feast/infra/registry/snowflake.py +++ b/sdk/python/feast/infra/registry/snowflake.py @@ -93,6 +93,12 @@ class SnowflakeRegistryConfig(RegistryConfig): authenticator: Optional[str] = None """ Snowflake authenticator name """ + private_key: Optional[str] = None + """ Snowflake private key file path""" + + private_key_passphrase: Optional[str] = None + """ Snowflake private key file passphrase""" + database: StrictStr """ Snowflake database name """ From 08c44ae35a4a91228f9f78c7323b4b7a73ef33aa Mon Sep 17 00:00:00 2001 From: Breno Costa <35263725+breno-costa@users.noreply.github.com> Date: Fri, 17 May 2024 23:56:07 +0200 Subject: [PATCH 63/73] fix: Integration tests for async sdk method (#4201) --- .../online_store/test_universal_online.py | 71 ++++++++++++++----- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 9beba4d72b5..4822a8d4f71 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -1,3 +1,4 @@ +import asyncio import datetime import os import time @@ -12,6 +13,7 @@ import requests from botocore.exceptions import BotoCoreError +from feast import FeatureStore from feast.entity import Entity from feast.errors import FeatureNameCollisionError from feast.feature_service import FeatureService @@ -400,19 +402,15 @@ def test_online_retrieval_with_shared_batch_source(environment, universal_data_s ) -@pytest.mark.integration -@pytest.mark.universal_online_stores -@pytest.mark.parametrize("full_feature_names", [True, False], ids=lambda v: str(v)) -def test_online_retrieval_with_event_timestamps( - environment, universal_data_sources, full_feature_names -): - fs = environment.feature_store +def setup_feature_store_universal_feature_views( + environment, universal_data_sources +) -> FeatureStore: + fs: FeatureStore = environment.feature_store entities, datasets, data_sources = universal_data_sources feature_views = construct_universal_feature_views(data_sources) fs.apply([driver(), feature_views.driver, feature_views.global_fv]) - # fake data to ingest into Online Store data = { "driver_id": [1, 2], "conv_rate": [0.5, 0.3], @@ -429,18 +427,11 @@ def test_online_retrieval_with_event_timestamps( } df_ingest = pd.DataFrame(data) - # directly ingest data into the Online Store fs.write_to_online_store("driver_stats", df_ingest) + return fs - response = fs.get_online_features( - features=[ - "driver_stats:avg_daily_trips", - "driver_stats:acc_rate", - "driver_stats:conv_rate", - ], - entity_rows=[{"driver_id": 1}, {"driver_id": 2}], - ) - df = response.to_df(True) + +def assert_feature_store_universal_feature_views_response(df: pd.DataFrame): assertpy.assert_that(len(df)).is_equal_to(2) assertpy.assert_that(df["driver_id"].iloc[0]).is_equal_to(1) assertpy.assert_that(df["driver_id"].iloc[1]).is_equal_to(2) @@ -464,6 +455,50 @@ def test_online_retrieval_with_event_timestamps( ) +@pytest.mark.integration +@pytest.mark.universal_online_stores +def test_online_retrieval_with_event_timestamps(environment, universal_data_sources): + fs = setup_feature_store_universal_feature_views( + environment, universal_data_sources + ) + + response = fs.get_online_features( + features=[ + "driver_stats:avg_daily_trips", + "driver_stats:acc_rate", + "driver_stats:conv_rate", + ], + entity_rows=[{"driver_id": 1}, {"driver_id": 2}], + ) + df = response.to_df(True) + + assert_feature_store_universal_feature_views_response(df) + + +@pytest.mark.integration +@pytest.mark.universal_online_stores(only=["redis"]) +def test_async_online_retrieval_with_event_timestamps( + environment, universal_data_sources +): + fs = setup_feature_store_universal_feature_views( + environment, universal_data_sources + ) + + response = asyncio.run( + fs.get_online_features_async( + features=[ + "driver_stats:avg_daily_trips", + "driver_stats:acc_rate", + "driver_stats:conv_rate", + ], + entity_rows=[{"driver_id": 1}, {"driver_id": 2}], + ) + ) + df = response.to_df(True) + + assert_feature_store_universal_feature_views_response(df) + + @pytest.mark.integration @pytest.mark.universal_online_stores(only=["redis"]) def test_online_store_cleanup(environment, universal_data_sources): From ff970531d207050c6959b6e37b8fb6b70f08e7e7 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Sat, 18 May 2024 12:32:30 -0400 Subject: [PATCH 64/73] chore: Update OWNERS (#4208) misc: Update OWNERS --- OWNERS | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/OWNERS b/OWNERS index d726837e570..52c5e436d30 100644 --- a/OWNERS +++ b/OWNERS @@ -17,6 +17,11 @@ approvers: - toping4445 - DvirDukhan - hemidactylus + - franciscojavierarceo + - haoxuai + - jeremyary + - shuchu + reviewers: - woop - achals @@ -34,4 +39,8 @@ reviewers: - toping4445 - DvirDukhan - hemidactylus + - franciscojavierarceo + - haoxuai + - jeremyary + - shuchu \ No newline at end of file From bdae562ea4582d8e47763736b639c70e56d79b2d Mon Sep 17 00:00:00 2001 From: Pushkar Gupta Date: Tue, 21 May 2024 05:32:00 -0700 Subject: [PATCH 65/73] feat: Feast/IKV datetime edgecase errors (#4211) * feat: Feast/IKV datetime edgecase errors Signed-off-by: Pushkar Gupta * linter Signed-off-by: Pushkar Gupta * linter Signed-off-by: Pushkar Gupta --------- Signed-off-by: Pushkar Gupta --- .../online_stores/contrib/ikv_online_store/ikv.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py b/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py index 90df7f46860..5d62fd701f4 100644 --- a/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py +++ b/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py @@ -11,6 +11,8 @@ Tuple, ) +import pytz +from google.protobuf.timestamp_pb2 import Timestamp from ikvpy.client import IKVReader, IKVWriter from ikvpy.clientoptions import ClientOptions, ClientOptionsBuilder from ikvpy.document import IKVDocument, IKVDocumentBuilder @@ -162,7 +164,9 @@ def _decode_fields_for_primary_key( dt: Optional[datetime] = None dt_bytes = next(value_iter) if dt_bytes: - dt = datetime.fromisoformat(str(dt_bytes, "utf-8")) + proto_timestamp = Timestamp() + proto_timestamp.ParseFromString(dt_bytes) + dt = datetime.fromtimestamp(proto_timestamp.seconds, tz=pytz.utc) # decode other features features = {} @@ -252,12 +256,17 @@ def _create_document( """Converts feast key-value pairs into an IKV document.""" # initialie builder by inserting primary key and row creation timestamp - event_timestamp_str: str = utils.make_tzaware(event_timestamp).isoformat() + event_timestamp_seconds = int(utils.make_tzaware(event_timestamp).timestamp()) + event_timestamp_seconds_proto = Timestamp() + event_timestamp_seconds_proto.seconds = event_timestamp_seconds + + # event_timestamp_str: str = utils.make_tzaware(event_timestamp).isoformat() builder = ( IKVDocumentBuilder() .put_string_field(PRIMARY_KEY_FIELD_NAME, entity_id) .put_bytes_field( - EVENT_CREATION_TIMESTAMP_FIELD_NAME, event_timestamp_str.encode("utf-8") + EVENT_CREATION_TIMESTAMP_FIELD_NAME, + event_timestamp_seconds_proto.SerializeToString(), ) ) From 9284a5053d5a0dfc37fcbc139900b2a3df4a432e Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 21 May 2024 21:25:06 +0400 Subject: [PATCH 66/73] chore: Remove usage tracking (#4214) remove usage Signed-off-by: tokoko --- Makefile | 6 +- docs/how-to-guides/adding-or-reusing-tests.md | 4 - .../adding-a-new-offline-store.md | 2 +- .../adding-support-for-a-new-online-store.md | 1 - docs/reference/usage.md | 12 - infra/scripts/test-end-to-end.sh | 1 - sdk/python/docs/source/feast.rst | 8 - sdk/python/feast/batch_feature_view.py | 4 +- sdk/python/feast/cli.py | 10 +- sdk/python/feast/constants.py | 6 - .../embedded_go/online_features_service.py | 2 +- sdk/python/feast/entity.py | 2 - sdk/python/feast/feature_service.py | 2 - sdk/python/feast/feature_store.py | 49 +-- sdk/python/feast/feature_view.py | 2 - sdk/python/feast/infra/aws.py | 12 +- sdk/python/feast/infra/contrib/grpc_server.py | 9 +- .../feast/infra/offline_stores/bigquery.py | 14 +- .../infra/offline_stores/bigquery_source.py | 2 +- .../contrib/athena_offline_store/athena.py | 10 +- .../contrib/mssql_offline_store/mssql.py | 4 - .../postgres_offline_store/postgres.py | 8 +- .../contrib/spark_offline_store/spark.py | 8 +- .../contrib/trino_offline_store/trino.py | 8 +- sdk/python/feast/infra/offline_stores/file.py | 6 - .../feast/infra/offline_stores/redshift.py | 8 - .../feast/infra/offline_stores/snowflake.py | 4 - .../feast/infra/online_stores/bigtable.py | 3 - .../cassandra_online_store.py | 50 +-- .../hazelcast_online_store.py | 2 - .../contrib/hbase_online_store/hbase.py | 4 - .../contrib/ikv_online_store/ikv.py | 5 - .../infra/online_stores/contrib/postgres.py | 4 - .../contrib/rockset_online_store/rockset.py | 5 - .../feast/infra/online_stores/datastore.py | 8 +- .../feast/infra/online_stores/dynamodb.py | 13 +- sdk/python/feast/infra/online_stores/redis.py | 12 +- .../feast/infra/online_stores/snowflake.py | 8 +- .../feast/infra/online_stores/sqlite.py | 36 +- .../feast/infra/passthrough_provider.py | 21 - .../feast/infra/registry/caching_registry.py | 5 +- sdk/python/feast/infra/registry/file.py | 3 - sdk/python/feast/infra/registry/gcs.py | 3 - .../infra/registry/proto_registry_utils.py | 2 - sdk/python/feast/infra/registry/registry.py | 5 +- sdk/python/feast/infra/registry/s3.py | 3 - sdk/python/feast/infra/registry/snowflake.py | 11 +- sdk/python/feast/infra/registry/sql.py | 6 +- .../feast/infra/transformation_servers/app.py | 8 +- sdk/python/feast/infra/utils/aws_utils.py | 2 +- sdk/python/feast/on_demand_feature_view.py | 2 - sdk/python/feast/project_metadata.py | 2 - sdk/python/feast/repo_config.py | 2 - sdk/python/feast/repo_operations.py | 13 +- sdk/python/feast/stream_feature_view.py | 8 +- sdk/python/feast/usage.py | 405 ------------------ sdk/python/feast/utils.py | 9 + sdk/python/pytest.ini | 3 +- sdk/python/tests/README.md | 4 - .../tests/integration/e2e/test_usage_e2e.py | 149 ------- .../tests/unit/infra/test_local_registry.py | 6 +- sdk/python/tests/unit/test_usage.py | 237 ---------- sdk/python/tests/utils/e2e_test_validation.py | 1 + 63 files changed, 116 insertions(+), 1148 deletions(-) delete mode 100644 docs/reference/usage.md delete mode 100644 sdk/python/feast/usage.py delete mode 100644 sdk/python/tests/integration/e2e/test_usage_e2e.py delete mode 100644 sdk/python/tests/unit/test_usage.py diff --git a/Makefile b/Makefile index f1d8c107bad..aed58ed465b 100644 --- a/Makefile +++ b/Makefile @@ -71,10 +71,10 @@ lock-python-dependencies-all: pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt" benchmark-python: - FEAST_USAGE=False IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests + IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests benchmark-python-local: - FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests + IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests test-python-unit: python -m pytest -n 8 --color=yes sdk/python/tests @@ -372,7 +372,7 @@ start-trino-locally: sleep 15 test-trino-plugin-locally: - cd ${ROOT_DIR}/sdk/python; FULL_REPO_CONFIGS_MODULE=feast.infra.offline_stores.contrib.trino_offline_store.test_config.manual_tests FEAST_USAGE=False IS_TEST=True python -m pytest --integration tests/ + cd ${ROOT_DIR}/sdk/python; FULL_REPO_CONFIGS_MODULE=feast.infra.offline_stores.contrib.trino_offline_store.test_config.manual_tests IS_TEST=True python -m pytest --integration tests/ kill-trino-locally: cd ${ROOT_DIR}; docker stop trino diff --git a/docs/how-to-guides/adding-or-reusing-tests.md b/docs/how-to-guides/adding-or-reusing-tests.md index d68e47df5c6..b7c01a04b02 100644 --- a/docs/how-to-guides/adding-or-reusing-tests.md +++ b/docs/how-to-guides/adding-or-reusing-tests.md @@ -21,7 +21,6 @@ $ tree │ ├── test_go_feature_server.py │ ├── test_python_feature_server.py │ ├── test_universal_e2e.py -│ ├── test_usage_e2e.py │ └── test_validation.py ├── feature_repos │ ├── integration_test_repo_config.py @@ -99,8 +98,6 @@ If a test can be run purely locally (where locally includes Docker resources), i * `test_go_feature_server.py` * python http server * `test_python_feature_server.py` - * usage tracking - * `test_usage_e2e.py` * data quality monitoring feature validation * `test_validation.py` 2. Offline and Online Store Tests @@ -149,7 +146,6 @@ If a test can be run purely locally (where locally includes Docker resources), i * Type mapping * Feast types * Serialization tests due to this [issue](https://github.com/feast-dev/feast/issues/2345) - * Feast usage tracking unit tests #### Docstring tests diff --git a/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md b/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md index 9c4ed1c45a3..28592f0cd1a 100644 --- a/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md +++ b/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md @@ -417,7 +417,7 @@ test-python-universal-spark: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.spark_repo_configuration \ PYTEST_PLUGINS=feast.infra.offline_stores.contrib.spark_offline_store.tests \ - FEAST_USAGE=False IS_TEST=True \ + IS_TEST=True \ python -m pytest -n 8 --integration \ -k "not test_historical_retrieval_fails_on_validation and \ not test_historical_retrieval_with_validation and \ diff --git a/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md b/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md index e3f1fcf4286..440205f8f11 100644 --- a/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md +++ b/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md @@ -374,7 +374,6 @@ test-python-universal-cassandra: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \ - FEAST_USAGE=False \ IS_TEST=True \ python -m pytest -x --integration \ sdk/python/tests diff --git a/docs/reference/usage.md b/docs/reference/usage.md deleted file mode 100644 index 8c9c9046612..00000000000 --- a/docs/reference/usage.md +++ /dev/null @@ -1,12 +0,0 @@ -# Usage - -## How Feast SDK usage is measured - -The Feast project has a feature to log usage statistics and errors. Several client methods are tracked, beginning in Feast 0.9. Users are assigned a UUID which is sent along with the name of the method, the Feast version, the OS \(using `sys.platform`\), and the current time. - -The [source code](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/usage.py) is available here. - -## How to enable the usage logging - -Set the environment variable `FEAST_USAGE` to `True` (in String type) and config your endpoint by the variable `FEAST_USAGE_ENDPOINT`. - diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index 9f086d0d8c0..4cfc2307f94 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -10,6 +10,5 @@ make build-java-no-tests REVISION=develop python -m pip install --upgrade pip setuptools wheel pip-tools make install-python python -m pip install -qr tests/requirements.txt -export FEAST_USAGE="False" su -p postgres -c "PATH=$PATH HOME=/tmp pytest -v tests/e2e/ --feast-version develop" diff --git a/sdk/python/docs/source/feast.rst b/sdk/python/docs/source/feast.rst index abb8783bf09..4730fdf725d 100644 --- a/sdk/python/docs/source/feast.rst +++ b/sdk/python/docs/source/feast.rst @@ -321,14 +321,6 @@ feast.ui\_server module :undoc-members: :show-inheritance: -feast.usage module ------------------- - -.. automodule:: feast.usage - :members: - :undoc-members: - :show-inheritance: - feast.utils module ------------------ diff --git a/sdk/python/feast/batch_feature_view.py b/sdk/python/feast/batch_feature_view.py index 707529a1a85..af7a5e68fd6 100644 --- a/sdk/python/feast/batch_feature_view.py +++ b/sdk/python/feast/batch_feature_view.py @@ -1,6 +1,6 @@ import warnings from datetime import datetime, timedelta -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple from feast import flags_helper from feast.data_source import DataSource @@ -60,7 +60,7 @@ def __init__( *, name: str, source: DataSource, - entities: Optional[Union[List[Entity], List[str]]] = None, + entities: Optional[List[Entity]] = None, ttl: Optional[timedelta] = None, tags: Optional[Dict[str, str]] = None, online: bool = True, diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index b3d7b587b27..f239c2dfad5 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -162,7 +162,7 @@ def ui( host: str, port: int, registry_ttl_sec: int, - root_path: Optional[str] = "", + root_path: str = "", ): """ Shows the Feast UI over the current directory @@ -807,12 +807,12 @@ def validate( """ store = create_feature_store(ctx) - feature_service = store.get_feature_service(name=feature_service) - reference = store.get_validation_reference(reference) + _feature_service = store.get_feature_service(name=feature_service) + _reference = store.get_validation_reference(reference) result = store.validate_logged_features( - source=feature_service, - reference=reference, + source=_feature_service, + reference=_reference, start=maybe_local_tz(datetime.fromisoformat(start_ts)), end=maybe_local_tz(datetime.fromisoformat(end_ts)), throw_exception=False, diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index f9865771172..6aad3e60bbf 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -29,12 +29,6 @@ # Environment variable for registry REGISTRY_ENV_NAME: str = "REGISTRY_BASE64" -# Environment variable for toggling the Usage feature -FEAST_USAGE = "FEAST_USAGE" - -# Environment variable for FEAST_USAGE_ENDPOINT -FEAST_USAGE_ENDPOINT = "FEAST_USAGE_ENDPOINT" - # Environment variable for the path for overwriting universal test configs FULL_REPO_CONFIGS_MODULE_ENV_NAME: str = "FULL_REPO_CONFIGS_MODULE" diff --git a/sdk/python/feast/embedded_go/online_features_service.py b/sdk/python/feast/embedded_go/online_features_service.py index 56427f61e6f..867431fcf85 100644 --- a/sdk/python/feast/embedded_go/online_features_service.py +++ b/sdk/python/feast/embedded_go/online_features_service.py @@ -256,7 +256,7 @@ def transformation_callback( f"OnDemandFeatureView mode '{odfv.mode} not supported by EmbeddedOnlineFeatureServer." ) - output = odfv.get_transformed_features_df( + output = odfv.get_transformed_features_df( # type: ignore input_record.to_pandas(), full_feature_names=full_feature_names ) output_record = pa.RecordBatch.from_pandas(output) diff --git a/sdk/python/feast/entity.py b/sdk/python/feast/entity.py index 30f04e9c068..a988c200d7c 100644 --- a/sdk/python/feast/entity.py +++ b/sdk/python/feast/entity.py @@ -20,7 +20,6 @@ from feast.protos.feast.core.Entity_pb2 import Entity as EntityProto from feast.protos.feast.core.Entity_pb2 import EntityMeta as EntityMetaProto from feast.protos.feast.core.Entity_pb2 import EntitySpecV2 as EntitySpecProto -from feast.usage import log_exceptions from feast.value_type import ValueType @@ -52,7 +51,6 @@ class Entity: created_timestamp: Optional[datetime] last_updated_timestamp: Optional[datetime] - @log_exceptions def __init__( self, *, diff --git a/sdk/python/feast/feature_service.py b/sdk/python/feast/feature_service.py index 7ec923205a3..8b8cbac8ea2 100644 --- a/sdk/python/feast/feature_service.py +++ b/sdk/python/feast/feature_service.py @@ -19,7 +19,6 @@ from feast.protos.feast.core.FeatureService_pb2 import ( FeatureServiceSpec as FeatureServiceSpecProto, ) -from feast.usage import log_exceptions @typechecked @@ -50,7 +49,6 @@ class FeatureService: last_updated_timestamp: Optional[datetime] = None logging_config: Optional[LoggingConfig] = None - @log_exceptions def __init__( self, *, diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 2fe885865d9..343aa04d604 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -95,7 +95,6 @@ from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference from feast.stream_feature_view import StreamFeatureView from feast.type_map import python_values_to_proto_values -from feast.usage import log_exceptions, log_exceptions_and_usage, set_usage_attribute from feast.value_type import ValueType from feast.version import get_version @@ -118,7 +117,6 @@ class FeatureStore: _registry: BaseRegistry _provider: Provider - @log_exceptions def __init__( self, repo_path: Optional[str] = None, @@ -176,7 +174,6 @@ def __init__( self._provider = get_provider(self.config) - @log_exceptions def version(self) -> str: """Returns the version of the current Feast SDK/CLI.""" return get_version() @@ -195,7 +192,6 @@ def _get_provider(self) -> Provider: # TODO: Bake self.repo_path into self.config so that we dont only have one interface to paths return self._provider - @log_exceptions_and_usage def refresh_registry(self): """Fetches and caches a copy of the feature registry in memory. @@ -218,7 +214,6 @@ def refresh_registry(self): self._registry = registry - @log_exceptions_and_usage def list_entities(self, allow_cache: bool = False) -> List[Entity]: """ Retrieves the list of entities from the registry. @@ -243,7 +238,6 @@ def _list_entities( if entity.name != DUMMY_ENTITY_NAME or not hide_dummy_entity ] - @log_exceptions_and_usage def list_feature_services(self) -> List[FeatureService]: """ Retrieves the list of feature services from the registry. @@ -253,7 +247,6 @@ def list_feature_services(self) -> List[FeatureService]: """ return self._registry.list_feature_services(self.project) - @log_exceptions_and_usage def list_feature_views(self, allow_cache: bool = False) -> List[FeatureView]: """ Retrieves the list of feature views from the registry. @@ -300,7 +293,6 @@ def _list_stream_feature_views( stream_feature_views.append(sfv) return stream_feature_views - @log_exceptions_and_usage def list_on_demand_feature_views( self, allow_cache: bool = False ) -> List[OnDemandFeatureView]: @@ -314,7 +306,6 @@ def list_on_demand_feature_views( self.project, allow_cache=allow_cache ) - @log_exceptions_and_usage def list_stream_feature_views( self, allow_cache: bool = False ) -> List[StreamFeatureView]: @@ -326,7 +317,6 @@ def list_stream_feature_views( """ return self._list_stream_feature_views(allow_cache) - @log_exceptions_and_usage def list_data_sources(self, allow_cache: bool = False) -> List[DataSource]: """ Retrieves the list of data sources from the registry. @@ -339,7 +329,6 @@ def list_data_sources(self, allow_cache: bool = False) -> List[DataSource]: """ return self._registry.list_data_sources(self.project, allow_cache=allow_cache) - @log_exceptions_and_usage def get_entity(self, name: str, allow_registry_cache: bool = False) -> Entity: """ Retrieves an entity. @@ -358,7 +347,6 @@ def get_entity(self, name: str, allow_registry_cache: bool = False) -> Entity: name, self.project, allow_cache=allow_registry_cache ) - @log_exceptions_and_usage def get_feature_service( self, name: str, allow_cache: bool = False ) -> FeatureService: @@ -377,7 +365,6 @@ def get_feature_service( """ return self._registry.get_feature_service(name, self.project, allow_cache) - @log_exceptions_and_usage def get_feature_view( self, name: str, allow_registry_cache: bool = False ) -> FeatureView: @@ -409,7 +396,6 @@ def _get_feature_view( feature_view.entities = [] return feature_view - @log_exceptions_and_usage def get_stream_feature_view( self, name: str, allow_registry_cache: bool = False ) -> StreamFeatureView: @@ -443,7 +429,6 @@ def _get_stream_feature_view( stream_feature_view.entities = [] return stream_feature_view - @log_exceptions_and_usage def get_on_demand_feature_view(self, name: str) -> OnDemandFeatureView: """ Retrieves a feature view. @@ -459,7 +444,6 @@ def get_on_demand_feature_view(self, name: str) -> OnDemandFeatureView: """ return self._registry.get_on_demand_feature_view(name, self.project) - @log_exceptions_and_usage def get_data_source(self, name: str) -> DataSource: """ Retrieves the list of data sources from the registry. @@ -475,7 +459,6 @@ def get_data_source(self, name: str) -> DataSource: """ return self._registry.get_data_source(name, self.project) - @log_exceptions_and_usage def delete_feature_view(self, name: str): """ Deletes a feature view. @@ -488,7 +471,6 @@ def delete_feature_view(self, name: str): """ return self._registry.delete_feature_view(name, self.project) - @log_exceptions_and_usage def delete_feature_service(self, name: str): """ Deletes a feature service. @@ -554,7 +536,6 @@ def _validate_all_feature_views( "This API is stable, but the functionality does not scale well for offline retrieval", RuntimeWarning, ) - set_usage_attribute("odfv", bool(odfvs_to_update)) _validate_feature_views( [ *views_to_update, @@ -657,7 +638,6 @@ def _get_feature_views_to_materialize( return feature_views_to_materialize - @log_exceptions_and_usage def plan( self, desired_repo_contents: RepoContents ) -> Tuple[RegistryDiff, InfraDiff, Infra]: @@ -734,7 +714,6 @@ def plan( return registry_diff, infra_diff, new_infra - @log_exceptions_and_usage def _apply_diffs( self, registry_diff: RegistryDiff, infra_diff: InfraDiff, new_infra: Infra ): @@ -752,7 +731,6 @@ def _apply_diffs( self._registry.update_infra(new_infra, self.project, commit=True) - @log_exceptions_and_usage def apply( self, objects: Union[ @@ -968,7 +946,6 @@ def apply( self._registry.commit() - @log_exceptions_and_usage def teardown(self): """Tears down all local and cloud resources for the feature store.""" tables: List[FeatureView] = [] @@ -981,7 +958,6 @@ def teardown(self): self._get_provider().teardown_infra(self.project, tables, entities) self._registry.teardown() - @log_exceptions_and_usage def get_historical_features( self, entity_df: Union[pd.DataFrame, str], @@ -1061,8 +1037,6 @@ def get_historical_features( feature_views = list(view for view, _ in fvs) on_demand_feature_views = list(view for view, _ in odfvs) - set_usage_attribute("odfv", bool(on_demand_feature_views)) - # Check that the right request data is present in the entity_df if type(entity_df) == pd.DataFrame: if self.config.coerce_tz_aware: @@ -1091,7 +1065,6 @@ def get_historical_features( return job - @log_exceptions_and_usage def create_saved_dataset( self, from_: RetrievalJob, @@ -1159,7 +1132,6 @@ def create_saved_dataset( self._registry.apply_saved_dataset(dataset, self.project, commit=True) return dataset - @log_exceptions_and_usage def get_saved_dataset(self, name: str) -> SavedDataset: """ Find a saved dataset in the registry by provided name and @@ -1191,7 +1163,6 @@ def get_saved_dataset(self, name: str) -> SavedDataset: ) return dataset.with_retrieval_job(retrieval_job) - @log_exceptions_and_usage def materialize_incremental( self, end_date: datetime, @@ -1283,7 +1254,6 @@ def tqdm_builder(length): end_date, ) - @log_exceptions_and_usage def materialize( self, start_date: datetime, @@ -1358,7 +1328,6 @@ def tqdm_builder(length): end_date, ) - @log_exceptions_and_usage def push( self, push_source_name: str, @@ -1403,7 +1372,6 @@ def push( fv.name, df, allow_registry_cache=allow_registry_cache ) - @log_exceptions_and_usage def write_to_online_store( self, feature_view_name: str, @@ -1422,7 +1390,7 @@ def write_to_online_store( """ # TODO: restrict this to work with online StreamFeatureViews and validate the FeatureView type try: - feature_view = self.get_stream_feature_view( + feature_view: FeatureView = self.get_stream_feature_view( feature_view_name, allow_registry_cache=allow_registry_cache ) except FeatureViewNotFoundException: @@ -1444,7 +1412,6 @@ def write_to_online_store( provider = self._get_provider() provider.ingest_df(feature_view, df) - @log_exceptions_and_usage def write_to_offline_store( self, feature_view_name: str, @@ -1460,7 +1427,7 @@ def write_to_offline_store( """ # TODO: restrict this to work with online StreamFeatureViews and validate the FeatureView type try: - feature_view = self.get_stream_feature_view( + feature_view: FeatureView = self.get_stream_feature_view( feature_view_name, allow_registry_cache=allow_registry_cache ) except FeatureViewNotFoundException: @@ -1487,7 +1454,6 @@ def write_to_offline_store( provider = self._get_provider() provider.ingest_df_to_offline_store(feature_view, table) - @log_exceptions_and_usage def get_online_features( self, features: Union[List[str], FeatureService], @@ -1550,7 +1516,6 @@ def get_online_features( native_entity_values=True, ) - @log_exceptions_and_usage async def get_online_features_async( self, features: Union[List[str], FeatureService], @@ -1625,7 +1590,6 @@ def _get_online_request_context( requested_feature_views, requested_on_demand_feature_views, ) - set_usage_attribute("odfv", bool(grouped_odfv_refs)) requested_result_row_names = { feat_ref.replace(":", "__") for feat_ref in _feature_refs @@ -1880,7 +1844,6 @@ async def _get_online_features_async( ) return OnlineResponse(online_features_response) - @log_exceptions_and_usage def retrieve_online_documents( self, feature: str, @@ -2481,7 +2444,6 @@ def _get_feature_views_to_use( return views_to_use - @log_exceptions_and_usage def serve( self, host: str, @@ -2510,12 +2472,10 @@ def serve( registry_ttl_sec=registry_ttl_sec, ) - @log_exceptions_and_usage def get_feature_server_endpoint(self) -> Optional[str]: """Returns endpoint for the feature server, if it exists.""" return self._provider.get_feature_server_endpoint() - @log_exceptions_and_usage def serve_ui( self, host: str, @@ -2541,14 +2501,12 @@ def serve_ui( root_path=root_path, ) - @log_exceptions_and_usage def serve_registry(self, port: int) -> None: """Start registry server locally on a given port.""" from feast import registry_server registry_server.start_server(self, port) - @log_exceptions_and_usage def serve_transformations(self, port: int) -> None: """Start the feature transformation server locally on a given port.""" warnings.warn( @@ -2561,7 +2519,6 @@ def serve_transformations(self, port: int) -> None: transformation_server.start_server(self, port) - @log_exceptions_and_usage def write_logged_features( self, logs: Union[pa.Table, Path], source: FeatureService ): @@ -2589,7 +2546,6 @@ def write_logged_features( registry=self._registry, ) - @log_exceptions_and_usage def validate_logged_features( self, source: FeatureService, @@ -2650,7 +2606,6 @@ def validate_logged_features( return None - @log_exceptions_and_usage def get_validation_reference( self, name: str, allow_cache: bool = False ) -> ValidationReference: diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index f87ae7ab132..ff41400eace 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -37,7 +37,6 @@ MaterializationInterval as MaterializationIntervalProto, ) from feast.types import from_value_type -from feast.usage import log_exceptions from feast.value_type import ValueType warnings.simplefilter("once", DeprecationWarning) @@ -94,7 +93,6 @@ class FeatureView(BaseFeatureView): owner: str materialization_intervals: List[Tuple[datetime, datetime]] - @log_exceptions def __init__( self, *, diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py index 5a045de4016..bb896fa961f 100644 --- a/sdk/python/feast/infra/aws.py +++ b/sdk/python/feast/infra/aws.py @@ -13,7 +13,6 @@ AWS_LAMBDA_FEATURE_SERVER_IMAGE, AWS_LAMBDA_FEATURE_SERVER_REPOSITORY, DOCKER_IMAGE_TAG_ENV_NAME, - FEAST_USAGE, FEATURE_STORE_YAML_ENV_NAME, ) from feast.entity import Entity @@ -29,7 +28,6 @@ from feast.infra.registry.registry import get_registry_store_class_from_scheme from feast.infra.registry.s3 import S3RegistryStore from feast.infra.utils import aws_utils -from feast.usage import log_exceptions_and_usage from feast.version import get_version try: @@ -43,7 +41,6 @@ class AwsProvider(PassthroughProvider): - @log_exceptions_and_usage(provider="AwsProvider") def update_infra( self, project: str, @@ -140,12 +137,7 @@ def _deploy_feature_server(self, project: str, image_uri: str): Code={"ImageUri": image_uri}, PackageType="Image", MemorySize=1769, - Environment={ - "Variables": { - FEATURE_STORE_YAML_ENV_NAME: config_base64, - FEAST_USAGE: "False", - } - }, + Environment={"Variables": {FEATURE_STORE_YAML_ENV_NAME: config_base64}}, Tags={ "feast-owned": "True", "project": project, @@ -200,7 +192,6 @@ def _deploy_feature_server(self, project: str, image_uri: str): SourceArn=f"arn:aws:execute-api:{region}:{account_id}:{api_id}/*/*/get-online-features", ) - @log_exceptions_and_usage(provider="AwsProvider") def teardown_infra( self, project: str, @@ -229,7 +220,6 @@ def teardown_infra( _logger.info(" Tearing down AWS API Gateway...") aws_utils.delete_api_gateway(api_gateway_client, api["ApiId"]) - @log_exceptions_and_usage(provider="AwsProvider") def get_feature_server_endpoint(self) -> Optional[str]: project = self.repo_config.project resource_name = _get_lambda_name(project) diff --git a/sdk/python/feast/infra/contrib/grpc_server.py b/sdk/python/feast/infra/contrib/grpc_server.py index 27ac45e77cc..2bd1b27755b 100644 --- a/sdk/python/feast/infra/contrib/grpc_server.py +++ b/sdk/python/feast/infra/contrib/grpc_server.py @@ -1,7 +1,7 @@ import logging import threading from concurrent import futures -from typing import Optional +from typing import Optional, Union import grpc import pandas as pd @@ -9,6 +9,7 @@ from feast.data_source import PushMode from feast.errors import FeatureServiceNotFoundException, PushSourceNotFoundException +from feast.feature_service import FeatureService from feast.feature_store import FeatureStore from feast.protos.feast.serving.GrpcServer_pb2 import ( PushResponse, @@ -100,8 +101,10 @@ def GetOnlineFeatures(self, request: GetOnlineFeaturesRequest, context): if request.HasField("feature_service"): logger.info(f"Requesting feature service: {request.feature_service}") try: - features = self.fs.get_feature_service( - request.feature_service, allow_cache=True + features: Union[list[str], FeatureService] = ( + self.fs.get_feature_service( + request.feature_service, allow_cache=True + ) ) except FeatureServiceNotFoundException as e: logger.error(f"Feature service {request.feature_service} not found") diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 897647bfc2b..36334b606d4 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -45,7 +45,7 @@ from feast.on_demand_feature_view import OnDemandFeatureView from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage -from feast.usage import get_user_agent, log_exceptions_and_usage +from feast.utils import get_user_agent from .bigquery_source import ( BigQueryLoggingDestination, @@ -114,7 +114,6 @@ def project_id_exists(cls, v, values, **kwargs): class BigQueryOfflineStore(OfflineStore): @staticmethod - @log_exceptions_and_usage(offline_store="bigquery") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -168,7 +167,6 @@ def pull_latest_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="bigquery") def pull_all_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -204,7 +202,6 @@ def pull_all_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="bigquery") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], @@ -467,10 +464,9 @@ def on_demand_feature_views(self) -> List[OnDemandFeatureView]: def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: with self._query_generator() as query: - df = self._execute_query(query=query, timeout=timeout).to_dataframe( - create_bqstorage_client=True - ) - return df + query_job = self._execute_query(query=query, timeout=timeout) + assert query_job + return query_job.to_dataframe(create_bqstorage_client=True) def to_sql(self) -> str: """Returns the underlying SQL query.""" @@ -521,6 +517,7 @@ def to_bigquery( bq_job = self._execute_query(query, job_config, timeout) if not job_config.dry_run: + assert bq_job config = bq_job.to_api_repr()["configuration"] # get temp table created by BQ tmp_dest = config["query"]["destinationTable"] @@ -539,7 +536,6 @@ def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: assert q return q.to_arrow() - @log_exceptions_and_usage def _execute_query( self, query, job_config=None, timeout: Optional[int] = None ) -> Optional[bigquery.job.query.QueryJob]: diff --git a/sdk/python/feast/infra/offline_stores/bigquery_source.py b/sdk/python/feast/infra/offline_stores/bigquery_source.py index 4888707c09c..1f667d66003 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery_source.py +++ b/sdk/python/feast/infra/offline_stores/bigquery_source.py @@ -15,7 +15,7 @@ ) from feast.repo_config import RepoConfig from feast.saved_dataset import SavedDatasetStorage -from feast.usage import get_user_agent +from feast.utils import get_user_agent from feast.value_type import ValueType diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py index 43960d87d54..ce731f01988 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py @@ -38,11 +38,9 @@ RetrievalMetadata, ) from feast.infra.registry.base_registry import BaseRegistry -from feast.infra.registry.registry import Registry from feast.infra.utils import aws_utils from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage -from feast.usage import log_exceptions_and_usage class AthenaOfflineStoreConfig(FeastConfigBaseModel): @@ -69,7 +67,6 @@ class AthenaOfflineStoreConfig(FeastConfigBaseModel): class AthenaOfflineStore(OfflineStore): @staticmethod - @log_exceptions_and_usage(offline_store="athena") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -129,7 +126,6 @@ def pull_latest_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="athena") def pull_all_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -168,13 +164,12 @@ def pull_all_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="athena") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], feature_refs: List[str], entity_df: Union[pd.DataFrame, str], - registry: Registry, + registry: BaseRegistry, project: str, full_feature_names: bool = False, ) -> RetrievalJob: @@ -372,7 +367,6 @@ def get_temp_table_dml_header( """ return temp_table_dml_header - @log_exceptions_and_usage def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: with self._query_generator() as query: temp_table_name = "_" + str(uuid.uuid4()).replace("-", "") @@ -389,7 +383,6 @@ def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: temp_table_name, ) - @log_exceptions_and_usage def _to_arrow_internal(self, timeout: Optional[int] = None) -> pa.Table: with self._query_generator() as query: temp_table_name = "_" + str(uuid.uuid4()).replace("-", "") @@ -419,7 +412,6 @@ def persist( assert isinstance(storage, SavedDatasetAthenaStorage) self.to_athena(table_name=storage.athena_options.table) - @log_exceptions_and_usage def to_athena(self, table_name: str) -> None: if self.on_demand_feature_views: transformed_df = self.to_df() diff --git a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py index f7f42e196b2..5fe58571466 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py +++ b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py @@ -34,7 +34,6 @@ from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage from feast.type_map import pa_to_mssql_type -from feast.usage import log_exceptions_and_usage # Make sure warning doesn't raise more than once. warnings.simplefilter("once", RuntimeWarning) @@ -66,7 +65,6 @@ class MsSqlServerOfflineStore(OfflineStore): """ @staticmethod - @log_exceptions_and_usage(offline_store="mssql") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -117,7 +115,6 @@ def pull_latest_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="mssql") def pull_all_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -156,7 +153,6 @@ def pull_all_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="mssql") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index 1bf10202e19..cb08b5f0168 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -34,7 +34,7 @@ RetrievalJob, RetrievalMetadata, ) -from feast.infra.registry.registry import Registry +from feast.infra.registry.base_registry import BaseRegistry from feast.infra.utils.postgres.connection_utils import ( _get_conn, df_to_postgres_table, @@ -45,7 +45,6 @@ from feast.repo_config import RepoConfig from feast.saved_dataset import SavedDatasetStorage from feast.type_map import pg_type_code_to_arrow -from feast.usage import log_exceptions_and_usage from .postgres_source import PostgreSQLSource @@ -56,7 +55,6 @@ class PostgreSQLOfflineStoreConfig(PostgreSQLConfig): class PostgreSQLOfflineStore(OfflineStore): @staticmethod - @log_exceptions_and_usage(offline_store="postgres") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -108,13 +106,12 @@ def pull_latest_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="postgres") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], feature_refs: List[str], entity_df: Union[pd.DataFrame, str], - registry: Registry, + registry: BaseRegistry, project: str, full_feature_names: bool = False, ) -> RetrievalJob: @@ -200,7 +197,6 @@ def query_generator() -> Iterator[str]: ) @staticmethod - @log_exceptions_and_usage(offline_store="postgres") def pull_all_from_table_or_query( config: RepoConfig, data_source: DataSource, diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py index 43902f33cf3..2d5a00c2965 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py @@ -30,12 +30,11 @@ RetrievalJob, RetrievalMetadata, ) -from feast.infra.registry.registry import Registry +from feast.infra.registry.base_registry import BaseRegistry from feast.infra.utils import aws_utils from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage from feast.type_map import spark_schema_to_np_dtypes -from feast.usage import log_exceptions_and_usage # Make sure spark warning doesn't raise more than once. warnings.simplefilter("once", RuntimeWarning) @@ -58,7 +57,6 @@ class SparkOfflineStoreConfig(FeastConfigBaseModel): class SparkOfflineStore(OfflineStore): @staticmethod - @log_exceptions_and_usage(offline_store="spark") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -120,13 +118,12 @@ def pull_latest_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="spark") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], feature_refs: List[str], entity_df: Union[pandas.DataFrame, str, pyspark.sql.DataFrame], - registry: Registry, + registry: BaseRegistry, project: str, full_feature_names: bool = False, ) -> RetrievalJob: @@ -259,7 +256,6 @@ def offline_write_batch( ) @staticmethod - @log_exceptions_and_usage(offline_store="spark") def pull_all_from_table_or_query( config: RepoConfig, data_source: DataSource, diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py index cdc94350244..b034d4f9923 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py @@ -31,11 +31,10 @@ RetrievalJob, RetrievalMetadata, ) -from feast.infra.registry.registry import Registry +from feast.infra.registry.base_registry import BaseRegistry from feast.on_demand_feature_view import OnDemandFeatureView from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage -from feast.usage import log_exceptions_and_usage class BasicAuthModel(FeastConfigBaseModel): @@ -266,7 +265,6 @@ def metadata(self) -> Optional[RetrievalMetadata]: class TrinoOfflineStore(OfflineStore): @staticmethod - @log_exceptions_and_usage(offline_store="trino") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -316,13 +314,12 @@ def pull_latest_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="trino") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], feature_refs: List[str], entity_df: Union[pd.DataFrame, str], - registry: Registry, + registry: BaseRegistry, project: str, full_feature_names: bool = False, ) -> TrinoRetrievalJob: @@ -402,7 +399,6 @@ def get_historical_features( ) @staticmethod - @log_exceptions_and_usage(offline_store="trino") def pull_all_from_table_or_query( config: RepoConfig, data_source: DataSource, diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index 1ae9a1558d1..af2570ebc08 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -37,7 +37,6 @@ from feast.on_demand_feature_view import OnDemandFeatureView from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage -from feast.usage import log_exceptions_and_usage from feast.utils import _get_requested_feature_views_to_features_dict # FileRetrievalJob will cast string objects to string[pyarrow] from dask version 2023.7.1 @@ -77,14 +76,12 @@ def full_feature_names(self) -> bool: def on_demand_feature_views(self) -> List[OnDemandFeatureView]: return self._on_demand_feature_views - @log_exceptions_and_usage def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: # Only execute the evaluation function to build the final historical retrieval dataframe at the last moment. df = self.evaluation_function().compute() df = df.reset_index(drop=True) return df - @log_exceptions_and_usage def _to_arrow_internal(self, timeout: Optional[int] = None): # Only execute the evaluation function to build the final historical retrieval dataframe at the last moment. df = self.evaluation_function().compute() @@ -127,7 +124,6 @@ def supports_remote_storage_export(self) -> bool: class FileOfflineStore(OfflineStore): @staticmethod - @log_exceptions_and_usage(offline_store="file") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], @@ -303,7 +299,6 @@ def evaluate_historical_retrieval(): return job @staticmethod - @log_exceptions_and_usage(offline_store="file") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -383,7 +378,6 @@ def evaluate_offline_job(): ) @staticmethod - @log_exceptions_and_usage(offline_store="file") def pull_all_from_table_or_query( config: RepoConfig, data_source: DataSource, diff --git a/sdk/python/feast/infra/offline_stores/redshift.py b/sdk/python/feast/infra/offline_stores/redshift.py index 2565a569ad1..cec21c35c1f 100644 --- a/sdk/python/feast/infra/offline_stores/redshift.py +++ b/sdk/python/feast/infra/offline_stores/redshift.py @@ -42,7 +42,6 @@ from feast.infra.utils import aws_utils from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage -from feast.usage import log_exceptions_and_usage class RedshiftOfflineStoreConfig(FeastConfigBaseModel): @@ -95,7 +94,6 @@ def require_cluster_and_user_or_workgroup(self): class RedshiftOfflineStore(OfflineStore): @staticmethod - @log_exceptions_and_usage(offline_store="redshift") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -154,7 +152,6 @@ def pull_latest_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="redshift") def pull_all_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -195,7 +192,6 @@ def pull_all_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="redshift") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], @@ -426,7 +422,6 @@ def full_feature_names(self) -> bool: def on_demand_feature_views(self) -> List[OnDemandFeatureView]: return self._on_demand_feature_views - @log_exceptions_and_usage def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: with self._query_generator() as query: return aws_utils.unload_redshift_query_to_df( @@ -441,7 +436,6 @@ def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: query, ) - @log_exceptions_and_usage def _to_arrow_internal(self, timeout: Optional[int] = None) -> pa.Table: with self._query_generator() as query: return aws_utils.unload_redshift_query_to_pa( @@ -456,7 +450,6 @@ def _to_arrow_internal(self, timeout: Optional[int] = None) -> pa.Table: query, ) - @log_exceptions_and_usage def to_s3(self) -> str: """Export dataset to S3 in Parquet format and return path""" if self.on_demand_feature_views: @@ -477,7 +470,6 @@ def to_s3(self) -> str: ) return self._s3_path - @log_exceptions_and_usage def to_redshift(self, table_name: str) -> None: """Save dataset as a new Redshift table""" if self.on_demand_feature_views: diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py index 64c84318c9d..96552ff87ec 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake.py +++ b/sdk/python/feast/infra/offline_stores/snowflake.py @@ -63,7 +63,6 @@ String, UnixTimestamp, ) -from feast.usage import log_exceptions_and_usage try: from snowflake.connector import SnowflakeConnection @@ -130,7 +129,6 @@ class SnowflakeOfflineStoreConfig(FeastConfigBaseModel): class SnowflakeOfflineStore(OfflineStore): @staticmethod - @log_exceptions_and_usage(offline_store="snowflake") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -220,7 +218,6 @@ def pull_latest_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="snowflake") def pull_all_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -265,7 +262,6 @@ def pull_all_from_table_or_query( ) @staticmethod - @log_exceptions_and_usage(offline_store="snowflake") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], diff --git a/sdk/python/feast/infra/online_stores/bigtable.py b/sdk/python/feast/infra/online_stores/bigtable.py index 3a83d23cedb..3479f7f289a 100644 --- a/sdk/python/feast/infra/online_stores/bigtable.py +++ b/sdk/python/feast/infra/online_stores/bigtable.py @@ -16,7 +16,6 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig -from feast.usage import log_exceptions_and_usage logger = logging.getLogger(__name__) @@ -49,7 +48,6 @@ class BigtableOnlineStore(OnlineStore): feature_column_family: str = "features" - @log_exceptions_and_usage(online_store="bigtable") def online_read( self, config: RepoConfig, @@ -116,7 +114,6 @@ def _process_bt_row( return (event_ts, res) - @log_exceptions_and_usage(online_store="bigtable") def online_write_batch( self, config: RepoConfig, diff --git a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py index c672e18db03..0870bc709db 100644 --- a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py +++ b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py @@ -51,7 +51,6 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel -from feast.usage import log_exceptions_and_usage, tracing_span # Error messages E_CASSANDRA_UNEXPECTED_CONFIGURATION_CLASS = ( @@ -319,7 +318,6 @@ def __del__(self): """ pass - @log_exceptions_and_usage(online_store="cassandra") def online_write_batch( self, config: RepoConfig, @@ -367,18 +365,16 @@ def unroll_insertion_tuples() -> Iterable[Tuple[str, bytes, str, datetime]]: if progress: progress(1) - with tracing_span(name="remote_call"): - self._write_rows_concurrently( - config, - project, - table, - unroll_insertion_tuples(), - ) - # correction for the last missing call to `progress`: - if progress: - progress(1) - - @log_exceptions_and_usage(online_store="cassandra") + self._write_rows_concurrently( + config, + project, + table, + unroll_insertion_tuples(), + ) + # correction for the last missing call to `progress`: + if progress: + progress(1) + def online_read( self, config: RepoConfig, @@ -408,14 +404,13 @@ def online_read( for entity_key in entity_keys ] - with tracing_span(name="remote_call"): - feature_rows_sequence = self._read_rows_by_entity_keys( - config, - project, - table, - entity_key_bins, - columns=["feature_name", "value", "event_ts"], - ) + feature_rows_sequence = self._read_rows_by_entity_keys( + config, + project, + table, + entity_key_bins, + columns=["feature_name", "value", "event_ts"], + ) for entity_key_bin, feature_rows in zip(entity_key_bins, feature_rows_sequence): res = {} @@ -436,7 +431,6 @@ def online_read( result.append((res_ts, res)) return result - @log_exceptions_and_usage(online_store="cassandra") def update( self, config: RepoConfig, @@ -457,13 +451,10 @@ def update( project = config.project for table in tables_to_keep: - with tracing_span(name="remote_call"): - self._create_table(config, project, table) + self._create_table(config, project, table) for table in tables_to_delete: - with tracing_span(name="remote_call"): - self._drop_table(config, project, table) + self._drop_table(config, project, table) - @log_exceptions_and_usage(online_store="cassandra") def teardown( self, config: RepoConfig, @@ -480,8 +471,7 @@ def teardown( project = config.project for table in tables: - with tracing_span(name="remote_call"): - self._drop_table(config, project, table) + self._drop_table(config, project, table) @staticmethod def _fq_table_name(keyspace: str, project: str, table: FeatureView) -> str: diff --git a/sdk/python/feast/infra/online_stores/contrib/hazelcast_online_store/hazelcast_online_store.py b/sdk/python/feast/infra/online_stores/contrib/hazelcast_online_store/hazelcast_online_store.py index 2537ecbf458..497d8909af4 100644 --- a/sdk/python/feast/infra/online_stores/contrib/hazelcast_online_store/hazelcast_online_store.py +++ b/sdk/python/feast/infra/online_stores/contrib/hazelcast_online_store/hazelcast_online_store.py @@ -35,7 +35,6 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel -from feast.usage import log_exceptions_and_usage # Exception messages EXCEPTION_HAZELCAST_UNEXPECTED_CONFIGURATION_CLASS = ( @@ -143,7 +142,6 @@ def _get_client(self, config: HazelcastOnlineStoreConfig): ) return self._client - @log_exceptions_and_usage(online_store="hazelcast") def online_write_batch( self, config: RepoConfig, diff --git a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py index d46b848c120..dc48d2c4efc 100644 --- a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py +++ b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py @@ -15,7 +15,6 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig -from feast.usage import log_exceptions_and_usage class HbaseOnlineStoreConfig(FeastConfigBaseModel): @@ -71,7 +70,6 @@ def _get_conn(self, config: RepoConfig): ) return self._conn - @log_exceptions_and_usage(online_store="hbase") def online_write_batch( self, config: RepoConfig, @@ -129,7 +127,6 @@ def online_write_batch( if progress: progress(len(data)) - @log_exceptions_and_usage(online_store="hbase") def online_read( self, config: RepoConfig, @@ -180,7 +177,6 @@ def online_read( result.append((res_ts, res)) return result - @log_exceptions_and_usage(online_store="hbase") def update( self, config: RepoConfig, diff --git a/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py b/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py index 5d62fd701f4..6b721bddf89 100644 --- a/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py +++ b/sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py @@ -25,7 +25,6 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig -from feast.usage import log_exceptions_and_usage PRIMARY_KEY_FIELD_NAME: str = "_entity_key" EVENT_CREATION_TIMESTAMP_FIELD_NAME: str = "_event_timestamp" @@ -60,7 +59,6 @@ class IKVOnlineStore(OnlineStore): _reader: Optional[IKVReader] = None _writer: Optional[IKVWriter] = None - @log_exceptions_and_usage(online_store="ikv") def online_write_batch( self, config: RepoConfig, @@ -99,7 +97,6 @@ def online_write_batch( if progress: progress(1) - @log_exceptions_and_usage(online_store="ikv") def online_read( self, config: RepoConfig, @@ -179,7 +176,6 @@ def _decode_fields_for_primary_key( return dt, features - @log_exceptions_and_usage(online_store="ikv") def update( self, config: RepoConfig, @@ -212,7 +208,6 @@ def update( # each field in an IKV document is prefixed by the feature-view's name self._writer.drop_fields_by_name_prefix([feature_view.name]) - @log_exceptions_and_usage(online_store="ikv") def teardown( self, config: RepoConfig, diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 1043208ab33..3eddd8ba203 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -19,7 +19,6 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RepoConfig -from feast.usage import log_exceptions_and_usage SUPPORTED_DISTANCE_METRICS_DICT = { "cosine": "<=>", @@ -57,7 +56,6 @@ def _get_conn(self, config: RepoConfig): self._conn = _get_conn(config.online_store) yield self._conn - @log_exceptions_and_usage(online_store="postgres") def online_write_batch( self, config: RepoConfig, @@ -120,7 +118,6 @@ def online_write_batch( if progress: progress(len(cur_batch)) - @log_exceptions_and_usage(online_store="postgres") def online_read( self, config: RepoConfig, @@ -191,7 +188,6 @@ def online_read( return result - @log_exceptions_and_usage(online_store="postgres") def update( self, config: RepoConfig, diff --git a/sdk/python/feast/infra/online_stores/contrib/rockset_online_store/rockset.py b/sdk/python/feast/infra/online_stores/contrib/rockset_online_store/rockset.py index 37cfbd86afd..31de7f9e9b1 100644 --- a/sdk/python/feast/infra/online_stores/contrib/rockset_online_store/rockset.py +++ b/sdk/python/feast/infra/online_stores/contrib/rockset_online_store/rockset.py @@ -33,7 +33,6 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig -from feast.usage import log_exceptions_and_usage logger = logging.getLogger(__name__) @@ -85,7 +84,6 @@ class RocksetOnlineStore(OnlineStore): _rockset_client = None - @log_exceptions_and_usage(online_store="rockset") def online_write_batch( self, config: RepoConfig, @@ -164,7 +162,6 @@ def online_write_batch( return None - @log_exceptions_and_usage(online_store="rockset") def online_read( self, config: RepoConfig, @@ -258,7 +255,6 @@ def online_read( return results_list - @log_exceptions_and_usage(online_store="rockset") def update( self, config: RepoConfig, @@ -303,7 +299,6 @@ def update( rs, created_collections, online_config=online_config ) - @log_exceptions_and_usage(online_store="rockset") def teardown( self, config: RepoConfig, diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index bf44a749661..b33767cea56 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -44,7 +44,7 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig -from feast.usage import get_user_agent, log_exceptions_and_usage, tracing_span +from feast.utils import get_user_agent LOGGER = logging.getLogger(__name__) @@ -103,7 +103,6 @@ class DatastoreOnlineStore(OnlineStore): _client: Optional[datastore.Client] = None - @log_exceptions_and_usage(online_store="datastore") def update( self, config: RepoConfig, @@ -164,7 +163,6 @@ def _get_client(self, online_config: DatastoreOnlineStoreConfig): ) return self._client - @log_exceptions_and_usage(online_store="datastore") def online_write_batch( self, config: RepoConfig, @@ -255,7 +253,6 @@ def _write_minibatch( if progress: progress(len(entities)) - @log_exceptions_and_usage(online_store="datastore") def online_read( self, config: RepoConfig, @@ -283,8 +280,7 @@ def online_read( # NOTE: get_multi doesn't return values in the same order as the keys in the request. # Also, len(values) can be less than len(keys) in the case of missing values. - with tracing_span(name="remote_call"): - values = client.get_multi(keys) + values = client.get_multi(keys) values_dict = {v.key: v for v in values} if values is not None else {} for key in keys: if key in values_dict: diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index 4a74f9e27f3..0ee9af185d3 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -29,7 +29,7 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig -from feast.usage import get_user_agent, log_exceptions_and_usage, tracing_span +from feast.utils import get_user_agent try: import boto3 @@ -81,7 +81,6 @@ class DynamoDBOnlineStore(OnlineStore): _dynamodb_client = None _dynamodb_resource = None - @log_exceptions_and_usage(online_store="dynamodb") def update( self, config: RepoConfig, @@ -172,7 +171,6 @@ def teardown( dynamodb_resource, _get_table_name(online_config, config, table) ) - @log_exceptions_and_usage(online_store="dynamodb") def online_write_batch( self, config: RepoConfig, @@ -208,7 +206,6 @@ def online_write_batch( ) self._write_batch_non_duplicates(table_instance, data, progress, config) - @log_exceptions_and_usage(online_store="dynamodb") def online_read( self, config: RepoConfig, @@ -257,10 +254,9 @@ def online_read( "ConsistentRead": online_config.consistent_reads, } } - with tracing_span(name="remote_call"): - response = dynamodb_resource.batch_get_item( - RequestItems=batch_entity_ids, - ) + response = dynamodb_resource.batch_get_item( + RequestItems=batch_entity_ids, + ) response = response.get("Responses") table_responses = response.get(table_instance.name) if table_responses: @@ -316,7 +312,6 @@ def _sort_dynamodb_response(self, responses: list, order: list) -> Any: _, table_responses_ordered = zip(*table_responses_ordered) return table_responses_ordered - @log_exceptions_and_usage(online_store="dynamodb") def _write_batch_non_duplicates( self, table_instance, diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index f681d8473e4..7428eb8bea4 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -38,7 +38,6 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel -from feast.usage import log_exceptions_and_usage, tracing_span try: from redis import Redis @@ -143,7 +142,6 @@ def delete_table(self, config: RepoConfig, table: FeatureView): logger.debug(f"Deleted {deleted_count} rows for feature view {table.name}") - @log_exceptions_and_usage(online_store="redis") def update( self, config: RepoConfig, @@ -262,7 +260,6 @@ async def _get_client_async(self, online_store_config: RedisOnlineStoreConfig): self._client_async = redis_asyncio.Redis(**kwargs) return self._client_async - @log_exceptions_and_usage(online_store="redis") def online_write_batch( self, config: RepoConfig, @@ -375,7 +372,6 @@ def _convert_redis_values_to_protobuf( result.append(features) return result - @log_exceptions_and_usage(online_store="redis") def online_read( self, config: RepoConfig, @@ -397,14 +393,13 @@ def online_read( with client.pipeline(transaction=False) as pipe: for redis_key_bin in keys: pipe.hmget(redis_key_bin, hset_keys) - with tracing_span(name="remote_call"): - redis_values = pipe.execute() + + redis_values = pipe.execute() return self._convert_redis_values_to_protobuf( redis_values, feature_view.name, requested_features ) - @log_exceptions_and_usage(online_store="redis") async def online_read_async( self, config: RepoConfig, @@ -426,8 +421,7 @@ async def online_read_async( async with client.pipeline(transaction=False) as pipe: for redis_key_bin in keys: pipe.hmget(redis_key_bin, hset_keys) - with tracing_span(name="remote_call"): - redis_values = await pipe.execute() + redis_values = await pipe.execute() return self._convert_redis_values_to_protobuf( redis_values, feature_view.name, requested_features diff --git a/sdk/python/feast/infra/online_stores/snowflake.py b/sdk/python/feast/infra/online_stores/snowflake.py index 57e3bbbb8dc..fef804a3773 100644 --- a/sdk/python/feast/infra/online_stores/snowflake.py +++ b/sdk/python/feast/infra/online_stores/snowflake.py @@ -20,7 +20,6 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig -from feast.usage import log_exceptions_and_usage from feast.utils import to_naive_utc @@ -66,7 +65,6 @@ class SnowflakeOnlineStoreConfig(FeastConfigBaseModel): class SnowflakeOnlineStore(OnlineStore): - @log_exceptions_and_usage(online_store="snowflake") def online_write_batch( self, config: RepoConfig, @@ -151,18 +149,19 @@ def online_write_batch( return None - @log_exceptions_and_usage(online_store="snowflake") def online_read( self, config: RepoConfig, table: FeatureView, entity_keys: List[EntityKeyProto], - requested_features: List[str], + requested_features: Optional[List[str]] = None, ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: assert isinstance(config.online_store, SnowflakeOnlineStoreConfig) result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + requested_features = requested_features if requested_features else [] + entity_fetch_str = ",".join( [ ( @@ -211,7 +210,6 @@ def online_read( result.append((res_ts, res)) return result - @log_exceptions_and_usage(online_store="snowflake") def update( self, config: RepoConfig, diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 745a9ed5a5e..63d3ef03f51 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -31,7 +31,6 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig -from feast.usage import log_exceptions_and_usage, tracing_span from feast.utils import to_naive_utc @@ -76,7 +75,6 @@ def _get_conn(self, config: RepoConfig): self._conn = _initialize_conn(db_path) return self._conn - @log_exceptions_and_usage(online_store="sqlite") def online_write_batch( self, config: RepoConfig, @@ -133,7 +131,6 @@ def online_write_batch( if progress: progress(1) - @log_exceptions_and_usage(online_store="sqlite") def online_read( self, config: RepoConfig, @@ -146,22 +143,21 @@ def online_read( result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] - with tracing_span(name="remote_call"): - # Fetch all entities in one go - cur.execute( - f"SELECT entity_key, feature_name, value, event_ts " - f"FROM {_table_id(config.project, table)} " - f"WHERE entity_key IN ({','.join('?' * len(entity_keys))}) " - f"ORDER BY entity_key", - [ - serialize_entity_key( - entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, - ) - for entity_key in entity_keys - ], - ) - rows = cur.fetchall() + # Fetch all entities in one go + cur.execute( + f"SELECT entity_key, feature_name, value, event_ts " + f"FROM {_table_id(config.project, table)} " + f"WHERE entity_key IN ({','.join('?' * len(entity_keys))}) " + f"ORDER BY entity_key", + [ + serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + for entity_key in entity_keys + ], + ) + rows = cur.fetchall() rows = { k: list(group) for k, group in itertools.groupby(rows, key=lambda r: r[0]) @@ -185,7 +181,6 @@ def online_read( result.append((res_ts, res)) return result - @log_exceptions_and_usage(online_store="sqlite") def update( self, config: RepoConfig, @@ -209,7 +204,6 @@ def update( for table in tables_to_delete: conn.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") - @log_exceptions_and_usage(online_store="sqlite") def plan( self, config: RepoConfig, desired_registry_proto: RegistryProto ) -> List[InfraObject]: diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index b96c3433ea7..e707f9495db 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -27,7 +27,6 @@ from feast.repo_config import BATCH_ENGINE_CLASS_FOR_TYPE, RepoConfig from feast.saved_dataset import SavedDataset from feast.stream_feature_view import StreamFeatureView -from feast.usage import RatioSampler, log_exceptions_and_usage, set_usage_attribute from feast.utils import ( _convert_arrow_to_proto, _run_pyarrow_field_mapping, @@ -113,8 +112,6 @@ def update_infra( entities_to_keep: Sequence[Entity], partial: bool, ): - set_usage_attribute("provider", self.__class__.__name__) - # Call update only if there is an online store if self.online_store: self.online_store.update( @@ -140,7 +137,6 @@ def teardown_infra( tables: Sequence[FeatureView], entities: Sequence[Entity], ) -> None: - set_usage_attribute("provider", self.__class__.__name__) if self.online_store: self.online_store.teardown(self.repo_config, tables, entities) if self.batch_engine: @@ -155,7 +151,6 @@ def online_write_batch( ], progress: Optional[Callable[[int], Any]], ) -> None: - set_usage_attribute("provider", self.__class__.__name__) if self.online_store: self.online_store.online_write_batch(config, table, data, progress) @@ -166,14 +161,11 @@ def offline_write_batch( data: pa.Table, progress: Optional[Callable[[int], Any]], ) -> None: - set_usage_attribute("provider", self.__class__.__name__) - if self.offline_store: self.offline_store.__class__.offline_write_batch( config, feature_view, data, progress ) - @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.001)) def online_read( self, config: RepoConfig, @@ -181,7 +173,6 @@ def online_read( entity_keys: List[EntityKeyProto], requested_features: Optional[List[str]] = None, ) -> List: - set_usage_attribute("provider", self.__class__.__name__) result = [] if self.online_store: result = self.online_store.online_read( @@ -189,7 +180,6 @@ def online_read( ) return result - @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.001)) async def online_read_async( self, config: RepoConfig, @@ -197,7 +187,6 @@ async def online_read_async( entity_keys: List[EntityKeyProto], requested_features: Optional[List[str]] = None, ) -> List: - set_usage_attribute("provider", self.__class__.__name__) result = [] if self.online_store: result = await self.online_store.online_read_async( @@ -205,7 +194,6 @@ async def online_read_async( ) return result - @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.001)) def retrieve_online_documents( self, config: RepoConfig, @@ -215,7 +203,6 @@ def retrieve_online_documents( top_k: int, distance_metric: Optional[str] = None, ) -> List: - set_usage_attribute("provider", self.__class__.__name__) result = [] if self.online_store: result = self.online_store.retrieve_online_documents( @@ -233,7 +220,6 @@ def ingest_df( feature_view: FeatureView, df: pd.DataFrame, ): - set_usage_attribute("provider", self.__class__.__name__) table = pa.Table.from_pandas(df) if feature_view.batch_source.field_mapping is not None: @@ -252,8 +238,6 @@ def ingest_df( ) def ingest_df_to_offline_store(self, feature_view: FeatureView, table: pa.Table): - set_usage_attribute("provider", self.__class__.__name__) - if feature_view.batch_source.field_mapping is not None: table = _run_pyarrow_field_mapping( table, feature_view.batch_source.field_mapping @@ -271,7 +255,6 @@ def materialize_single_feature_view( project: str, tqdm_builder: Callable[[int], tqdm], ) -> None: - set_usage_attribute("provider", self.__class__.__name__) assert ( isinstance(feature_view, BatchFeatureView) or isinstance(feature_view, StreamFeatureView) @@ -301,8 +284,6 @@ def get_historical_features( project: str, full_feature_names: bool, ) -> RetrievalJob: - set_usage_attribute("provider", self.__class__.__name__) - job = self.offline_store.get_historical_features( config=config, feature_views=feature_views, @@ -318,8 +299,6 @@ def get_historical_features( def retrieve_saved_dataset( self, config: RepoConfig, dataset: SavedDataset ) -> RetrievalJob: - set_usage_attribute("provider", self.__class__.__name__) - feature_name_columns = [ ref.replace(":", "__") if dataset.full_feature_names else ref.split(":")[1] for ref in dataset.features diff --git a/sdk/python/feast/infra/registry/caching_registry.py b/sdk/python/feast/infra/registry/caching_registry.py index 3101b073d51..0f660128086 100644 --- a/sdk/python/feast/infra/registry/caching_registry.py +++ b/sdk/python/feast/infra/registry/caching_registry.py @@ -4,7 +4,6 @@ from threading import Lock from typing import List, Optional -from feast import usage from feast.data_source import DataSource from feast.entity import Entity from feast.feature_service import FeatureService @@ -282,9 +281,7 @@ def refresh(self, project: Optional[str] = None): project_metadata = proto_registry_utils.get_project_metadata( registry_proto=self.cached_registry_proto, project=project ) - if project_metadata: - usage.set_current_project_uuid(project_metadata.project_uuid) - else: + if not project_metadata: proto_registry_utils.init_project_metadata( self.cached_registry_proto, project ) diff --git a/sdk/python/feast/infra/registry/file.py b/sdk/python/feast/infra/registry/file.py index 3ee75a78805..7117a0d2c6b 100644 --- a/sdk/python/feast/infra/registry/file.py +++ b/sdk/python/feast/infra/registry/file.py @@ -5,7 +5,6 @@ from feast.infra.registry.registry_store import RegistryStore from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.repo_config import RegistryConfig -from feast.usage import log_exceptions_and_usage class FileRegistryStore(RegistryStore): @@ -16,7 +15,6 @@ def __init__(self, registry_config: RegistryConfig, repo_path: Path): else: self._filepath = repo_path.joinpath(registry_path) - @log_exceptions_and_usage(registry="local") def get_registry_proto(self): registry_proto = RegistryProto() if self._filepath.exists(): @@ -26,7 +24,6 @@ def get_registry_proto(self): f'Registry not found at path "{self._filepath}". Have you run "feast apply"?' ) - @log_exceptions_and_usage(registry="local") def update_registry_proto(self, registry_proto: RegistryProto): self._write_registry(registry_proto) diff --git a/sdk/python/feast/infra/registry/gcs.py b/sdk/python/feast/infra/registry/gcs.py index 6f922d4ea20..7e4b7104cf1 100644 --- a/sdk/python/feast/infra/registry/gcs.py +++ b/sdk/python/feast/infra/registry/gcs.py @@ -7,7 +7,6 @@ from feast.infra.registry.registry_store import RegistryStore from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.repo_config import RegistryConfig -from feast.usage import log_exceptions_and_usage class GCSRegistryStore(RegistryStore): @@ -25,7 +24,6 @@ def __init__(self, registry_config: RegistryConfig, repo_path: Path): self._bucket = self._uri.hostname self._blob = self._uri.path.lstrip("/") - @log_exceptions_and_usage(registry="gs") def get_registry_proto(self): import google.cloud.storage as storage from google.cloud.exceptions import NotFound @@ -49,7 +47,6 @@ def get_registry_proto(self): f'Registry not found at path "{self._uri.geturl()}". Have you run "feast apply"?' ) - @log_exceptions_and_usage(registry="gs") def update_registry_proto(self, registry_proto: RegistryProto): self._write_registry(registry_proto) diff --git a/sdk/python/feast/infra/registry/proto_registry_utils.py b/sdk/python/feast/infra/registry/proto_registry_utils.py index 4d2e16cb022..60e9cfa3abc 100644 --- a/sdk/python/feast/infra/registry/proto_registry_utils.py +++ b/sdk/python/feast/infra/registry/proto_registry_utils.py @@ -2,7 +2,6 @@ from functools import wraps from typing import List, Optional -from feast import usage from feast.data_source import DataSource from feast.entity import Entity from feast.errors import ( @@ -45,7 +44,6 @@ def wrapper(registry_proto: RegistryProto, project: str): def init_project_metadata(cached_registry_proto: RegistryProto, project: str): new_project_uuid = f"{uuid.uuid4()}" - usage.set_current_project_uuid(new_project_uuid) cached_registry_proto.project_metadata.append( ProjectMetadata(project_name=project, project_uuid=new_project_uuid).to_proto() ) diff --git a/sdk/python/feast/infra/registry/registry.py b/sdk/python/feast/infra/registry/registry.py index d949b6079da..b1efbb2c7c3 100644 --- a/sdk/python/feast/infra/registry/registry.py +++ b/sdk/python/feast/infra/registry/registry.py @@ -22,7 +22,6 @@ from google.protobuf.internal.containers import RepeatedCompositeFieldContainer from google.protobuf.message import Message -from feast import usage from feast.base_feature_view import BaseFeatureView from feast.data_source import DataSource from feast.entity import Entity @@ -827,9 +826,7 @@ def _get_registry_proto( project_metadata = proto_registry_utils.get_project_metadata( registry_proto=registry_proto, project=project ) - if project_metadata: - usage.set_current_project_uuid(project_metadata.project_uuid) - else: + if not project_metadata: proto_registry_utils.init_project_metadata(registry_proto, project) self.commit() diff --git a/sdk/python/feast/infra/registry/s3.py b/sdk/python/feast/infra/registry/s3.py index 0a94c942e18..cbae3af11cc 100644 --- a/sdk/python/feast/infra/registry/s3.py +++ b/sdk/python/feast/infra/registry/s3.py @@ -9,7 +9,6 @@ from feast.infra.registry.registry_store import RegistryStore from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.repo_config import RegistryConfig -from feast.usage import log_exceptions_and_usage try: import boto3 @@ -31,7 +30,6 @@ def __init__(self, registry_config: RegistryConfig, repo_path: Path): "s3", endpoint_url=os.environ.get("FEAST_S3_ENDPOINT_URL") ) - @log_exceptions_and_usage(registry="s3") def get_registry_proto(self): file_obj = TemporaryFile() registry_proto = RegistryProto() @@ -64,7 +62,6 @@ def get_registry_proto(self): f"Error while trying to locate Registry at path {self._uri.geturl()}" ) from e - @log_exceptions_and_usage(registry="s3") def update_registry_proto(self, registry_proto: RegistryProto): self._write_registry(registry_proto) diff --git a/sdk/python/feast/infra/registry/snowflake.py b/sdk/python/feast/infra/registry/snowflake.py index 169c8ae43ec..87d89af9c87 100644 --- a/sdk/python/feast/infra/registry/snowflake.py +++ b/sdk/python/feast/infra/registry/snowflake.py @@ -10,7 +10,6 @@ from pydantic import ConfigDict, Field, StrictStr import feast -from feast import usage from feast.base_feature_view import BaseFeatureView from feast.data_source import DataSource from feast.entity import Entity @@ -149,9 +148,7 @@ def refresh(self, project: Optional[str] = None): project_metadata = proto_registry_utils.get_project_metadata( registry_proto=self.cached_registry_proto, project=project ) - if project_metadata: - usage.set_current_project_uuid(project_metadata.project_uuid) - else: + if not project_metadata: proto_registry_utils.init_project_metadata( self.cached_registry_proto, project ) @@ -1003,9 +1000,7 @@ def _maybe_init_project_metadata(self, project): """ df = execute_snowflake_statement(conn, query).fetch_pandas_all() - if not df.empty: - usage.set_current_project_uuid(df.squeeze()) - else: + if df.empty: new_project_uuid = f"{uuid.uuid4()}" query = f""" INSERT INTO {self.registry_path}."FEAST_METADATA" @@ -1014,8 +1009,6 @@ def _maybe_init_project_metadata(self, project): """ execute_snowflake_statement(conn, query) - usage.set_current_project_uuid(new_project_uuid) - def _set_last_updated_metadata(self, last_updated: datetime, project: str): with GetSnowflakeConnection(self.registry_config) as conn: query = f""" diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index 98b23e1943e..26f9da19e18 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -21,7 +21,6 @@ ) from sqlalchemy.engine import Engine -from feast import usage from feast.base_feature_view import BaseFeatureView from feast.data_source import DataSource from feast.entity import Entity @@ -734,9 +733,7 @@ def _maybe_init_project_metadata(self, project): feast_metadata.c.project_id == project, ) row = conn.execute(stmt).first() - if row: - usage.set_current_project_uuid(row._mapping["metadata_value"]) - else: + if not row: new_project_uuid = f"{uuid.uuid4()}" values = { "metadata_key": FeastMetadataKeys.PROJECT_UUID.value, @@ -746,7 +743,6 @@ def _maybe_init_project_metadata(self, project): } insert_stmt = insert(feast_metadata).values(values) conn.execute(insert_stmt) - usage.set_current_project_uuid(new_project_uuid) def _delete_object( self, diff --git a/sdk/python/feast/infra/transformation_servers/app.py b/sdk/python/feast/infra/transformation_servers/app.py index 7afba69beb7..167e7b9245d 100644 --- a/sdk/python/feast/infra/transformation_servers/app.py +++ b/sdk/python/feast/infra/transformation_servers/app.py @@ -56,8 +56,10 @@ def async_refresh(): async_refresh() # Start the feature transformation server -port = ( - os.environ.get(FEATURE_TRANSFORMATION_SERVER_PORT_ENV_NAME) - or DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT +port = int( + os.environ.get( + FEATURE_TRANSFORMATION_SERVER_PORT_ENV_NAME, + DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT, + ) ) store.serve_transformations(port) diff --git a/sdk/python/feast/infra/utils/aws_utils.py b/sdk/python/feast/infra/utils/aws_utils.py index c3604ee41f0..8e1b182249a 100644 --- a/sdk/python/feast/infra/utils/aws_utils.py +++ b/sdk/python/feast/infra/utils/aws_utils.py @@ -22,7 +22,7 @@ RedshiftTableNameTooLong, ) from feast.type_map import pa_to_athena_value_type, pa_to_redshift_value_type -from feast.usage import get_user_agent +from feast.utils import get_user_agent try: import boto3 diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index b501b87a0e4..839ce4d64ca 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -34,7 +34,6 @@ from feast.transformation.pandas_transformation import PandasTransformation from feast.transformation.python_transformation import PythonTransformation from feast.transformation.substrait_transformation import SubstraitTransformation -from feast.usage import log_exceptions from feast.value_type import ValueType warnings.simplefilter("once", DeprecationWarning) @@ -73,7 +72,6 @@ class OnDemandFeatureView(BaseFeatureView): tags: dict[str, str] owner: str - @log_exceptions # noqa: C901 def __init__( # noqa: C901 self, *, diff --git a/sdk/python/feast/project_metadata.py b/sdk/python/feast/project_metadata.py index 829e9ff0d54..64488a03629 100644 --- a/sdk/python/feast/project_metadata.py +++ b/sdk/python/feast/project_metadata.py @@ -18,7 +18,6 @@ from typeguard import typechecked from feast.protos.feast.core.Registry_pb2 import ProjectMetadata as ProjectMetadataProto -from feast.usage import log_exceptions @typechecked @@ -34,7 +33,6 @@ class ProjectMetadata: project_name: str project_uuid: str - @log_exceptions def __init__( self, *args, diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 00cbac19081..6ef81794bf8 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -26,7 +26,6 @@ FeastRegistryTypeInvalidError, ) from feast.importer import import_class -from feast.usage import log_exceptions warnings.simplefilter("once", RuntimeWarning) @@ -307,7 +306,6 @@ def batch_engine(self): return self._batch_engine @model_validator(mode="before") - @log_exceptions def _validate_online_store_config(cls, values: Any) -> Any: # This method will validate whether the online store configurations are set correctly. This explicit validation # is necessary because Pydantic Unions throw very verbose and cryptic exceptions. We also use this method to diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 296155ce464..274a0af02b0 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -9,7 +9,7 @@ from importlib.abc import Loader from importlib.machinery import ModuleSpec from pathlib import Path -from typing import List, Set, Union +from typing import List, Optional, Set, Union import click from click.exceptions import BadParameter @@ -30,7 +30,6 @@ from feast.repo_config import RepoConfig from feast.repo_contents import RepoContents from feast.stream_feature_view import StreamFeatureView -from feast.usage import log_exceptions_and_usage def py_path_to_module(path: Path) -> str: @@ -169,8 +168,8 @@ def parse_repo(repo_root: Path) -> RepoContents: res.data_sources.append(batch_source) # Handle stream sources defined with feature views. + assert obj.stream_source stream_source = obj.stream_source - assert stream_source if not any((stream_source is ds) for ds in res.data_sources): res.data_sources.append(stream_source) elif isinstance(obj, BatchFeatureView) and not any( @@ -199,7 +198,6 @@ def parse_repo(repo_root: Path) -> RepoContents: return res -@log_exceptions_and_usage def plan(repo_config: RepoConfig, repo_path: Path, skip_source_validation: bool): os.chdir(repo_path) project, registry, repo, store = _prepare_registry_and_repo(repo_config, repo_path) @@ -323,7 +321,6 @@ def log_infra_changes( ) -@log_exceptions_and_usage def create_feature_store( ctx: click.Context, ) -> FeatureStore: @@ -344,7 +341,6 @@ def create_feature_store( return FeatureStore(repo_path=str(repo), fs_yaml_file=fs_yaml_file) -@log_exceptions_and_usage def apply_total(repo_config: RepoConfig, repo_path: Path, skip_source_validation: bool): os.chdir(repo_path) project, registry, repo, store = _prepare_registry_and_repo(repo_config, repo_path) @@ -353,14 +349,12 @@ def apply_total(repo_config: RepoConfig, repo_path: Path, skip_source_validation ) -@log_exceptions_and_usage -def teardown(repo_config: RepoConfig, repo_path: Path): +def teardown(repo_config: RepoConfig, repo_path: Optional[str]): # Cannot pass in both repo_path and repo_config to FeatureStore. feature_store = FeatureStore(repo_path=repo_path, config=None) feature_store.teardown() -@log_exceptions_and_usage def registry_dump(repo_config: RepoConfig, repo_path: Path) -> str: """For debugging only: output contents of the metadata registry""" registry_config = repo_config.registry @@ -380,7 +374,6 @@ def cli_check_repo(repo_path: Path, fs_yaml_file: Path): sys.exit(1) -@log_exceptions_and_usage def init_repo(repo_name: str, template: str): import os from distutils.dir_util import copy_tree diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 301cf6cba57..50e1a221456 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -87,12 +87,12 @@ def __init__( *, name: str, source: DataSource, - entities: Optional[Union[List[Entity], List[str]]] = None, + entities: Optional[List[Entity]] = None, ttl: timedelta = timedelta(days=0), tags: Optional[Dict[str, str]] = None, - online: Optional[bool] = True, - description: Optional[str] = "", - owner: Optional[str] = "", + online: bool = True, + description: str = "", + owner: str = "", schema: Optional[List[Field]] = None, aggregations: Optional[List[Aggregation]] = None, mode: Optional[str] = "spark", diff --git a/sdk/python/feast/usage.py b/sdk/python/feast/usage.py deleted file mode 100644 index faf734cf01d..00000000000 --- a/sdk/python/feast/usage.py +++ /dev/null @@ -1,405 +0,0 @@ -# Copyright 2019 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import concurrent.futures -import contextlib -import contextvars -import dataclasses -import hashlib -import logging -import os -import platform -import sys -import typing -import uuid -from datetime import datetime -from functools import wraps -from os.path import expanduser, join -from pathlib import Path - -import requests - -from feast import flags_helper -from feast.constants import FEAST_USAGE, FEAST_USAGE_ENDPOINT -from feast.version import get_version - -_logger = logging.getLogger(__name__) -_executor = concurrent.futures.ThreadPoolExecutor(max_workers=3) - -_is_enabled = os.getenv(FEAST_USAGE, default="False") == "True" - -# Default usage endpoint value. -# Will raise an exception if the configured value is not working. -_usage_endpoint = os.getenv(FEAST_USAGE_ENDPOINT, default="") - -_constant_attributes = { - "project_id": "", - "session_id": str(uuid.uuid4()), - "installation_id": None, - "version": get_version(), - "python_version": platform.python_version(), - "platform": platform.platform(), - "env_signature": hashlib.md5( - ",".join( - sorted([k for k in os.environ.keys() if not k.startswith("FEAST")]) - ).encode(), - usedforsecurity=False, - ).hexdigest(), -} - -APPLICATION_NAME = "feast-dev/feast" -USER_AGENT = "{}/{}".format(APPLICATION_NAME, get_version()) - - -def get_user_agent(): - return USER_AGENT - - -def set_current_project_uuid(project_uuid: str): - _constant_attributes["project_id"] = project_uuid - - -@dataclasses.dataclass -class FnCall: - fn_name: str - id: str - - start: datetime - end: typing.Optional[datetime] = None - - parent_id: typing.Optional[str] = None - - -class Sampler: - def should_record(self) -> bool: - raise NotImplementedError - - @property - def priority(self): - return 0 - - -class AlwaysSampler(Sampler): - def should_record(self) -> bool: - return True - - -class RatioSampler(Sampler): - MAX_COUNTER = (1 << 32) - 1 - - def __init__(self, ratio): - assert 0 < ratio <= 1, "Ratio must be within (0, 1]" - self.ratio = ratio - self.total_counter = 0 - self.sampled_counter = 0 - - def should_record(self) -> bool: - self.total_counter += 1 - if self.total_counter == self.MAX_COUNTER: - self.total_counter = 1 - self.sampled_counter = 1 - - decision = self.sampled_counter < self.ratio * self.total_counter - self.sampled_counter += int(decision) - return decision - - @property - def priority(self): - return int(1 / self.ratio) - - -class UsageContext: - attributes: typing.Dict[str, typing.Any] - - call_stack: typing.List[FnCall] - completed_calls: typing.List[FnCall] - - exception: typing.Optional[Exception] = None - traceback: typing.Optional[typing.Tuple[str, int, str]] = None - - sampler: Sampler = AlwaysSampler() - - def __init__(self): - self.attributes = {} - self.call_stack = [] - self.completed_calls = [] - - -_context = contextvars.ContextVar("usage_context", default=UsageContext()) - - -def _set_installation_id(): - if os.getenv("FEAST_FORCE_USAGE_UUID"): - _constant_attributes["installation_id"] = os.getenv("FEAST_FORCE_USAGE_UUID") - _constant_attributes["installation_ts"] = datetime.utcnow().isoformat() - return - - feast_home_dir = join(expanduser("~"), ".feast") - installation_timestamp = datetime.utcnow() - - try: - Path(feast_home_dir).mkdir(exist_ok=True) - usage_filepath = join(feast_home_dir, "usage") - - if os.path.exists(usage_filepath): - installation_timestamp = datetime.utcfromtimestamp( - os.path.getmtime(usage_filepath) - ) - with open(usage_filepath, "r") as f: - installation_id = f.read() - else: - installation_id = str(uuid.uuid4()) - - with open(usage_filepath, "w") as f: - f.write(installation_id) - print( - "Feast is an open source project that collects " - "anonymized error reporting and usage statistics. To opt out or learn" - " more see https://docs.feast.dev/reference/usage" - ) - except OSError as e: - _logger.debug(f"Unable to configure usage {e}") - installation_id = "undefined" - - _constant_attributes["installation_id"] = installation_id - _constant_attributes["installation_ts"] = installation_timestamp.isoformat() - - -_set_installation_id() - - -def _export(event: typing.Dict[str, typing.Any]): - _executor.submit(requests.post, _usage_endpoint, json=event, timeout=2) - - -def _produce_event(ctx: UsageContext): - if ctx.sampler and not ctx.sampler.should_record(): - return - # Cannot check for unittest because typeguard pulls in unittest - is_test = flags_helper.is_test() or bool({"pytest"} & sys.modules.keys()) - event = { - "timestamp": datetime.utcnow().isoformat(), - "is_test": is_test, - "is_webserver": ( - not is_test and bool({"uwsgi", "gunicorn", "fastapi"} & sys.modules.keys()) - ), - "calls": [ - dict( - fn_name=c.fn_name, - id=c.id, - parent_id=c.parent_id, - start=c.start and c.start.isoformat(), - end=c.end and c.end.isoformat(), - ) - for c in reversed(ctx.completed_calls) - ], - "entrypoint": ctx.completed_calls[-1].fn_name, - "exception": repr(ctx.exception) if ctx.exception else None, - "traceback": ctx.traceback if ctx.exception else None, - **_constant_attributes, - } - event.update(ctx.attributes) - _export(event) - - -@contextlib.contextmanager -def tracing_span(name): - """ - Context manager for wrapping heavy parts of code in tracing span - """ - if _is_enabled: - ctx = _context.get() - if not ctx.call_stack: - raise RuntimeError("tracing_span must be called in usage context") - - last_call = ctx.call_stack[-1] - fn_call = FnCall( - id=uuid.uuid4().hex, - parent_id=last_call.id, - fn_name=f"{last_call.fn_name}.{name}", - start=datetime.utcnow(), - ) - try: - yield - finally: - if _is_enabled: - fn_call.end = datetime.utcnow() - ctx.completed_calls.append(fn_call) - - -def log_exceptions_and_usage(*args, **attrs): - """ - This function decorator enables three components: - 1. Error tracking - 2. Usage statistic collection - 3. Time profiling - - This data is being collected, anonymized and sent to Feast Developers. - All events from nested decorated functions are being grouped into single event - to build comprehensive context useful for profiling and error tracking. - - Usage example (will result in one output event): - @log_exceptions_and_usage - def fn(...): - nested() - - @log_exceptions_and_usage(attr='value') - def nested(...): - deeply_nested() - - @log_exceptions_and_usage(attr2='value2', sample=RateSampler(rate=0.1)) - def deeply_nested(...): - ... - """ - sampler = attrs.pop("sampler", AlwaysSampler()) - - def clear_context(ctx): - _context.set(UsageContext()) # reset context to default values - # TODO: Figure out why without this, new contexts.get aren't reset - ctx.call_stack = [] - ctx.completed_calls = [] - ctx.attributes = {} - - def decorator(func): - if not _is_enabled: - return func - - @wraps(func) - def wrapper(*args, **kwargs): - ctx = _context.get() - ctx.call_stack.append( - FnCall( - id=uuid.uuid4().hex, - parent_id=ctx.call_stack[-1].id if ctx.call_stack else None, - fn_name=_fn_fullname(func), - start=datetime.utcnow(), - ) - ) - ctx.attributes.update(attrs) - - try: - return func(*args, **kwargs) - except Exception: - if ctx.exception: - # exception was already recorded - raise - - _, exc, traceback = sys.exc_info() - ctx.exception = exc - ctx.traceback = _trace_to_log(traceback) - - if traceback: - raise exc.with_traceback(traceback) - - raise exc - finally: - ctx.sampler = ( - sampler if sampler.priority > ctx.sampler.priority else ctx.sampler - ) - last_call = ctx.call_stack.pop(-1) - last_call.end = datetime.utcnow() - ctx.completed_calls.append(last_call) - - if not ctx.call_stack or ( - len(ctx.call_stack) == 1 - and "feast.feature_store.FeatureStore.serve" - in str(ctx.call_stack[0].fn_name) - ): - # When running `feast serve`, the serve method never exits so it gets - # stuck otherwise - _produce_event(ctx) - clear_context(ctx) - - return wrapper - - if args: - return decorator(args[0]) - - return decorator - - -def log_exceptions(*args, **attrs): - """ - Function decorator that track errors and send them to Feast Developers - """ - - def decorator(func): - if not _is_enabled: - return func - - @wraps(func) - def wrapper(*args, **kwargs): - if _context.get().call_stack: - # we're already inside usage context - # let it handle exception - return func(*args, **kwargs) - - fn_call = FnCall( - id=uuid.uuid4().hex, fn_name=_fn_fullname(func), start=datetime.utcnow() - ) - try: - return func(*args, **kwargs) - except Exception: - _, exc, traceback = sys.exc_info() - - fn_call.end = datetime.utcnow() - - ctx = UsageContext() - ctx.exception = exc - ctx.traceback = _trace_to_log(traceback) - ctx.attributes = attrs - ctx.completed_calls.append(fn_call) - _produce_event(ctx) - - if traceback: - raise exc.with_traceback(traceback) - - raise exc - - return wrapper - - if args: - return decorator(args[0]) - - return decorator - - -def set_usage_attribute(name, value): - """ - Extend current context with custom attribute - """ - ctx = _context.get() - ctx.attributes[name] = value - - -def _trim_filename(filename: str) -> str: - return filename.split("/")[-1] - - -def _fn_fullname(fn: typing.Callable): - return fn.__module__ + "." + fn.__qualname__ - - -def _trace_to_log(traceback): - log = [] - while traceback is not None: - log.append( - ( - _trim_filename(traceback.tb_frame.f_code.co_filename), - traceback.tb_lineno, - traceback.tb_frame.f_code.co_name, - ) - ) - traceback = traceback.tb_next - - return log diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 89a1a9ab41c..47faa7d8c48 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -16,12 +16,21 @@ from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.type_map import python_values_to_proto_values from feast.value_type import ValueType +from feast.version import get_version if typing.TYPE_CHECKING: from feast.feature_view import FeatureView from feast.on_demand_feature_view import OnDemandFeatureView +APPLICATION_NAME = "feast-dev/feast" +USER_AGENT = "{}/{}".format(APPLICATION_NAME, get_version()) + + +def get_user_agent(): + return USER_AGENT + + def make_tzaware(t: datetime) -> datetime: """We assume tz-naive datetimes are UTC""" if t.tzinfo is None: diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini index 8a162943221..d87e4c07cb3 100644 --- a/sdk/python/pytest.ini +++ b/sdk/python/pytest.ini @@ -4,11 +4,12 @@ markers = universal_online_stores: mark a test as using all online stores. env = - FEAST_USAGE=False IS_TEST=True filterwarnings = ignore::DeprecationWarning:pyspark.sql.pandas.*: ignore::DeprecationWarning:pyspark.sql.connect.*: ignore::DeprecationWarning:httpx.*: + ignore::DeprecationWarning:happybase.*: + ignore::DeprecationWarning:pkg_resources.*: ignore::FutureWarning:ibis_substrait.compiler.*: diff --git a/sdk/python/tests/README.md b/sdk/python/tests/README.md index 3212f02482c..5b930129026 100644 --- a/sdk/python/tests/README.md +++ b/sdk/python/tests/README.md @@ -19,7 +19,6 @@ $ tree │ ├── test_go_feature_server.py │ ├── test_python_feature_server.py │ ├── test_universal_e2e.py -│ ├── test_usage_e2e.py │ └── test_validation.py ├── feature_repos │ ├── integration_test_repo_config.py @@ -97,8 +96,6 @@ Tests in Feast are split into integration and unit tests. * `test_go_feature_server.py` * python http server * `test_python_feature_server.py` - * usage tracking - * `test_usage_e2e.py` * data quality monitoring feature validation * `test_validation.py` 2. Offline and Online Store Tests @@ -147,7 +144,6 @@ Tests in Feast are split into integration and unit tests. * Type mapping * Feast types * Serialization tests due to this [issue](https://github.com/feast-dev/feast/issues/2345) - * Feast usage tracking unit tests #### Docstring tests diff --git a/sdk/python/tests/integration/e2e/test_usage_e2e.py b/sdk/python/tests/integration/e2e/test_usage_e2e.py deleted file mode 100644 index 4c8be468901..00000000000 --- a/sdk/python/tests/integration/e2e/test_usage_e2e.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright 2020 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file tests our usage tracking system in `usage.py`. -import os -import sys -import tempfile -from importlib import reload -from unittest.mock import patch - -import pytest - -from feast import Entity, RepoConfig -from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig - - -@pytest.fixture(scope="function") -def dummy_exporter(): - event_log = [] - - with patch("feast.usage._export", new=event_log.append): - yield event_log - - -@pytest.fixture(scope="function") -def enabling_toggle(): - with patch("feast.usage._is_enabled") as p: - p.__bool__.return_value = True - yield p - - # return to initial state - _reload_feast() - - -@pytest.mark.integration -def test_usage_on(dummy_exporter, enabling_toggle): - _reload_feast() - from feast.feature_store import FeatureStore - - with tempfile.TemporaryDirectory() as temp_dir: - test_feature_store = FeatureStore( - config=RepoConfig( - registry=os.path.join(temp_dir, "registry.db"), - project="fake_project", - provider="local", - online_store=SqliteOnlineStoreConfig( - path=os.path.join(temp_dir, "online.db") - ), - entity_key_serialization_version=2, - ) - ) - entity = Entity( - name="driver_car_id", - description="Car driver id", - tags={"team": "matchmaking"}, - ) - - test_feature_store.apply([entity]) - - assert len(dummy_exporter) == 3 - assert { - "entrypoint": "feast.infra.registry.file.FileRegistryStore.get_registry_proto" - }.items() <= dummy_exporter[0].items() - assert { - "entrypoint": "feast.infra.registry.file.FileRegistryStore.update_registry_proto" - }.items() <= dummy_exporter[1].items() - assert { - "entrypoint": "feast.feature_store.FeatureStore.apply" - }.items() <= dummy_exporter[2].items() - - -@pytest.mark.integration -def test_usage_off(dummy_exporter, enabling_toggle): - enabling_toggle.__bool__.return_value = False - - _reload_feast() - from feast.feature_store import FeatureStore - - with tempfile.TemporaryDirectory() as temp_dir: - test_feature_store = FeatureStore( - config=RepoConfig( - registry=os.path.join(temp_dir, "registry.db"), - project="fake_project", - provider="local", - online_store=SqliteOnlineStoreConfig( - path=os.path.join(temp_dir, "online.db") - ), - entity_key_serialization_version=2, - ) - ) - entity = Entity( - name="driver_car_id", - description="Car driver id", - tags={"team": "matchmaking"}, - ) - test_feature_store.apply([entity]) - - assert not dummy_exporter - - -@pytest.mark.integration -def test_exception_usage_on(dummy_exporter, enabling_toggle): - _reload_feast() - from feast.feature_store import FeatureStore - - with pytest.raises(OSError): - FeatureStore("/tmp/non_existent_directory") - - assert len(dummy_exporter) == 1 - assert { - "entrypoint": "feast.feature_store.FeatureStore.__init__", - "exception": repr(FileNotFoundError(2, "No such file or directory")), - }.items() <= dummy_exporter[0].items() - - -@pytest.mark.integration -def test_exception_usage_off(dummy_exporter, enabling_toggle): - enabling_toggle.__bool__.return_value = False - - _reload_feast() - from feast.feature_store import FeatureStore - - with pytest.raises(OSError): - FeatureStore("/tmp/non_existent_directory") - - assert not dummy_exporter - - -def _reload_feast(): - """After changing environment need to reload modules and rerun usage decorators""" - modules = ( - "feast.infra.registry.file", - "feast.infra.online_stores.sqlite", - "feast.feature_store", - ) - for mod in modules: - if mod in sys.modules: - reload(sys.modules[mod]) diff --git a/sdk/python/tests/unit/infra/test_local_registry.py b/sdk/python/tests/unit/infra/test_local_registry.py index b5e7d23a979..73f5cd91a5d 100644 --- a/sdk/python/tests/unit/infra/test_local_registry.py +++ b/sdk/python/tests/unit/infra/test_local_registry.py @@ -106,6 +106,7 @@ def test_apply_feature_view_success(test_registry): fv1 = FeatureView( name="my_feature_view_1", schema=[ + Field(name="test", dtype=Int64), Field(name="fs1_my_feature_1", dtype=Int64), Field(name="fs1_my_feature_2", dtype=String), Field(name="fs1_my_feature_3", dtype=Array(String)), @@ -331,7 +332,10 @@ def test_modify_feature_views_success(test_registry): fv1 = FeatureView( name="my_feature_view_1", - schema=[Field(name="fs1_my_feature_1", dtype=Int64)], + schema=[ + Field(name="test", dtype=Int64), + Field(name="fs1_my_feature_1", dtype=Int64), + ], entities=[entity], tags={"team": "matchmaking"}, source=batch_source, diff --git a/sdk/python/tests/unit/test_usage.py b/sdk/python/tests/unit/test_usage.py deleted file mode 100644 index ca842474307..00000000000 --- a/sdk/python/tests/unit/test_usage.py +++ /dev/null @@ -1,237 +0,0 @@ -import datetime -import json -import time -from unittest.mock import patch - -import pytest - -from feast.usage import ( - RatioSampler, - log_exceptions, - log_exceptions_and_usage, - set_usage_attribute, - tracing_span, -) - - -@pytest.fixture(scope="function") -def dummy_exporter(): - event_log = [] - - with patch( - "feast.usage._export", - new=lambda e: event_log.append(json.loads(json.dumps(e))), - ): - yield event_log - - -@pytest.fixture(scope="function", autouse=True) -def enabling_patch(): - with patch("feast.usage._is_enabled") as p: - p.__bool__.return_value = True - yield p - - -def test_logging_disabled(dummy_exporter, enabling_patch): - enabling_patch.__bool__.return_value = False - - @log_exceptions_and_usage(event="test-event") - def entrypoint(): - pass - - @log_exceptions(event="test-event") - def entrypoint2(): - raise ValueError(1) - - entrypoint() - with pytest.raises(ValueError): - entrypoint2() - - assert not dummy_exporter - - -def test_global_context_building(dummy_exporter): - @log_exceptions_and_usage(event="test-event") - def entrypoint(provider): - if provider == "one": - provider_one() - if provider == "two": - provider_two() - - @log_exceptions_and_usage(provider="provider-one") - def provider_one(): - dummy_layer() - - @log_exceptions_and_usage(provider="provider-two") - def provider_two(): - set_usage_attribute("new-attr", "new-val") - - @log_exceptions_and_usage - def dummy_layer(): - redis_store() - - @log_exceptions_and_usage(store="redis") - def redis_store(): - set_usage_attribute("attr", "val") - - entrypoint(provider="one") - entrypoint(provider="two") - - scope_name = "test_usage.test_global_context_building." - - assert dummy_exporter - assert { - "event": "test-event", - "provider": "provider-one", - "store": "redis", - "attr": "val", - "entrypoint": f"{scope_name}.entrypoint", - }.items() <= dummy_exporter[0].items() - assert dummy_exporter[0]["calls"][0]["fn_name"] == f"{scope_name}.entrypoint" - assert dummy_exporter[0]["calls"][1]["fn_name"] == f"{scope_name}.provider_one" - assert dummy_exporter[0]["calls"][2]["fn_name"] == f"{scope_name}.dummy_layer" - assert dummy_exporter[0]["calls"][3]["fn_name"] == f"{scope_name}.redis_store" - - assert ( - not {"store", "attr"} & dummy_exporter[1].keys() - ) # check that context was reset - assert { - "event": "test-event", - "provider": "provider-two", - "new-attr": "new-val", - }.items() <= dummy_exporter[1].items() - - -def test_exception_recording(dummy_exporter): - @log_exceptions_and_usage(event="test-event") - def entrypoint(): - provider() - - @log_exceptions_and_usage(provider="provider-one") - def provider(): - raise ValueError(1) - - with pytest.raises(ValueError): - entrypoint() - - assert dummy_exporter - assert { - "event": "test-event", - "provider": "provider-one", - "exception": repr(ValueError(1)), - "entrypoint": "test_usage.test_exception_recording..entrypoint", - }.items() <= dummy_exporter[0].items() - - -def test_only_exception_logging(dummy_exporter): - @log_exceptions(scope="exception-only") - def failing_fn(): - raise ValueError(1) - - @log_exceptions_and_usage(scope="usage-and-exception") - def entrypoint(): - failing_fn() - - with pytest.raises(ValueError): - failing_fn() - - assert { - "exception": repr(ValueError(1)), - "scope": "exception-only", - "entrypoint": "test_usage.test_only_exception_logging..failing_fn", - }.items() <= dummy_exporter[0].items() - - with pytest.raises(ValueError): - entrypoint() - - assert { - "exception": repr(ValueError(1)), - "scope": "usage-and-exception", - "entrypoint": "test_usage.test_only_exception_logging..entrypoint", - }.items() <= dummy_exporter[1].items() - - -def test_ratio_based_sampling(dummy_exporter): - @log_exceptions_and_usage() - def entrypoint(): - expensive_fn() - - @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.1)) - def expensive_fn(): - pass - - for _ in range(100): - entrypoint() - - assert len(dummy_exporter) == 10 - - -def test_sampling_priority(dummy_exporter): - @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.3)) - def entrypoint(): - expensive_fn() - - @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.01)) - def expensive_fn(): - other_fn() - - @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.1)) - def other_fn(): - pass - - for _ in range(300): - entrypoint() - - assert len(dummy_exporter) == 3 - - -def test_time_recording(dummy_exporter): - @log_exceptions_and_usage() - def entrypoint(): - time.sleep(0.1) - expensive_fn() - - @log_exceptions_and_usage() - def expensive_fn(): - time.sleep(0.5) - other_fn() - - @log_exceptions_and_usage() - def other_fn(): - time.sleep(0.2) - - entrypoint() - - assert dummy_exporter - calls = dummy_exporter[0]["calls"] - assert call_length_ms(calls[0]) >= 800 - assert call_length_ms(calls[0]) > call_length_ms(calls[1]) >= 700 - assert call_length_ms(calls[1]) > call_length_ms(calls[2]) >= 200 - - -def test_profiling_decorator(dummy_exporter): - @log_exceptions_and_usage() - def entrypoint(): - with tracing_span("custom_span"): - time.sleep(0.1) - - entrypoint() - - assert dummy_exporter - - calls = dummy_exporter[0]["calls"] - assert len(calls) - assert call_length_ms(calls[0]) >= 100 - assert call_length_ms(calls[1]) >= 100 - - assert ( - calls[1]["fn_name"] - == "test_usage.test_profiling_decorator..entrypoint.custom_span" - ) - - -def call_length_ms(call): - return ( - datetime.datetime.fromisoformat(call["end"]) - - datetime.datetime.fromisoformat(call["start"]) - ).total_seconds() * 10**3 diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py index 798e82de9b9..47ef503a7f5 100644 --- a/sdk/python/tests/utils/e2e_test_validation.py +++ b/sdk/python/tests/utils/e2e_test_validation.py @@ -251,6 +251,7 @@ def validate_registry_data_source_apply(test_registry: Registry): fv1 = FeatureView( name="my_feature_view_1", schema=[ + Field(name="test", dtype=Int64), Field(name="fs1_my_feature_1", dtype=Int64), Field(name="fs1_my_feature_2", dtype=String), Field(name="fs1_my_feature_3", dtype=Array(String)), From 2b398dc9a203f3abe848f49b5b4d85adf10cb893 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 21 May 2024 21:28:43 +0400 Subject: [PATCH 67/73] chore: Refactor test environment setup (#4210) * refactor it test environment setup Signed-off-by: tokoko * fix test_offline_store env setup Signed-off-by: tokoko * fix it tests Signed-off-by: tokoko * fix it tests Signed-off-by: tokoko --------- Signed-off-by: tokoko --- sdk/python/tests/conftest.py | 7 +- .../feature_repos/repo_configuration.py | 76 +++++++++++-------- .../universal/data_source_creator.py | 5 +- .../contrib/spark/test_spark.py | 2 + .../materialization/test_snowflake.py | 3 + .../test_universal_historical_retrieval.py | 4 +- .../registration/test_universal_cli.py | 9 ++- .../registration/test_universal_types.py | 9 +-- .../offline_stores/test_offline_store.py | 31 ++++---- sdk/python/tests/utils/e2e_test_validation.py | 9 ++- 10 files changed, 87 insertions(+), 68 deletions(-) diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index c4a62be0c0a..7c875fc9bde 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -182,16 +182,15 @@ def environment(request, worker_id): request.param, worker_id=worker_id, fixture_request=request ) + e.setup() + if hasattr(e.data_source_creator, "mock_environ"): with mock.patch.dict(os.environ, e.data_source_creator.mock_environ): yield e else: yield e - e.feature_store.teardown() - e.data_source_creator.teardown() - if e.online_store_creator: - e.online_store_creator.teardown() + e.teardown() _config_cache: Any = {} diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 311325536ed..2f260e87a60 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -1,6 +1,5 @@ import dataclasses import importlib -import json import os import tempfile import uuid @@ -11,13 +10,15 @@ import pandas as pd import pytest -import yaml from feast import FeatureStore, FeatureView, OnDemandFeatureView, driver_test_data from feast.constants import FULL_REPO_CONFIGS_MODULE_ENV_NAME from feast.data_source import DataSource from feast.errors import FeastModuleImportError -from feast.infra.feature_servers.base_config import FeatureLoggingConfig +from feast.infra.feature_servers.base_config import ( + BaseFeatureServerConfig, + FeatureLoggingConfig, +) from feast.infra.feature_servers.local_process.config import LocalFeatureServerConfig from feast.repo_config import RegistryConfig, RepoConfig from tests.integration.feature_repos.integration_test_repo_config import ( @@ -397,18 +398,48 @@ def construct_universal_feature_views( @dataclass class Environment: name: str - test_repo_config: IntegrationTestRepoConfig - feature_store: FeatureStore + project: str + provider: str + registry: RegistryConfig data_source_creator: DataSourceCreator + online_store_creator: Optional[OnlineStoreCreator] + online_store: Optional[Union[str, Dict]] + batch_engine: Optional[Union[str, Dict]] python_feature_server: bool worker_id: str - online_store_creator: Optional[OnlineStoreCreator] = None + feature_server: BaseFeatureServerConfig + entity_key_serialization_version: int + repo_dir_name: str fixture_request: Optional[pytest.FixtureRequest] = None def __post_init__(self): self.end_date = datetime.utcnow().replace(microsecond=0, second=0, minute=0) self.start_date: datetime = self.end_date - timedelta(days=3) + def setup(self): + self.data_source_creator.setup(self.registry) + + self.config = RepoConfig( + registry=self.registry, + project=self.project, + provider=self.provider, + offline_store=self.data_source_creator.create_offline_store_config(), + online_store=self.online_store_creator.create_online_store() + if self.online_store_creator + else self.online_store, + batch_engine=self.batch_engine, + repo_path=self.repo_dir_name, + feature_server=self.feature_server, + entity_key_serialization_version=self.entity_key_serialization_version, + ) + self.feature_store = FeatureStore(config=self.config) + + def teardown(self): + self.feature_store.teardown() + self.data_source_creator.teardown() + if self.online_store_creator: + self.online_store_creator.teardown() + def table_name_from_data_source(ds: DataSource) -> Optional[str]: if hasattr(ds, "table_ref"): @@ -436,16 +467,13 @@ def construct_test_environment( offline_creator: DataSourceCreator = test_repo_config.offline_store_creator( project, fixture_request=fixture_request ) - offline_store_config = offline_creator.create_offline_store_config() if test_repo_config.online_store_creator: online_creator = test_repo_config.online_store_creator( project, fixture_request=fixture_request ) - online_store = online_creator.create_online_store() else: online_creator = None - online_store = test_repo_config.online_store if test_repo_config.python_feature_server and test_repo_config.provider == "aws": from feast.infra.feature_servers.aws_lambda.config import ( @@ -481,35 +509,21 @@ def construct_test_environment( cache_ttl_seconds=1, ) - config = RepoConfig( - registry=registry, - project=project, - provider=test_repo_config.provider, - offline_store=offline_store_config, - online_store=online_store, - batch_engine=test_repo_config.batch_engine, - repo_path=repo_dir_name, - feature_server=feature_server, - entity_key_serialization_version=entity_key_serialization_version, - ) - - # Create feature_store.yaml out of the config - with open(Path(repo_dir_name) / "feature_store.yaml", "w") as f: - yaml.safe_dump(json.loads(config.model_dump_json(by_alias=True)), f) - - fs = FeatureStore(repo_dir_name) - # We need to initialize the registry, because if nothing is applied in the test before tearing down - # the feature store, that will cause the teardown method to blow up. - fs.registry._initialize_registry(project) environment = Environment( name=project, - test_repo_config=test_repo_config, - feature_store=fs, + provider=test_repo_config.provider, data_source_creator=offline_creator, python_feature_server=test_repo_config.python_feature_server, worker_id=worker_id, online_store_creator=online_creator, fixture_request=fixture_request, + project=project, + registry=registry, + feature_server=feature_server, + entity_key_serialization_version=entity_key_serialization_version, + repo_dir_name=repo_dir_name, + batch_engine=test_repo_config.batch_engine, + online_store=test_repo_config.online_store, ) return environment diff --git a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py index 5e5062291d5..62d458d6f4a 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py @@ -5,7 +5,7 @@ from feast.data_source import DataSource from feast.feature_logging import LoggingDestination -from feast.repo_config import FeastConfigBaseModel +from feast.repo_config import FeastConfigBaseModel, RegistryConfig from feast.saved_dataset import SavedDatasetStorage @@ -44,6 +44,9 @@ def create_data_source( """ raise NotImplementedError + def setup(self, registry: RegistryConfig): + pass + @abstractmethod def create_offline_store_config(self) -> FeastConfigBaseModel: raise NotImplementedError diff --git a/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py b/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py index bb4c4e63fc2..ae0e03c9441 100644 --- a/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py +++ b/sdk/python/tests/integration/materialization/contrib/spark/test_spark.py @@ -34,6 +34,8 @@ def test_spark_materialization_consistency(): spark_config, None, entity_key_serialization_version=2 ) + spark_environment.setup() + df = create_basic_driver_dataset() ds = spark_environment.data_source_creator.create_data_source( diff --git a/sdk/python/tests/integration/materialization/test_snowflake.py b/sdk/python/tests/integration/materialization/test_snowflake.py index 60fa9b30aab..adb2bd7e7df 100644 --- a/sdk/python/tests/integration/materialization/test_snowflake.py +++ b/sdk/python/tests/integration/materialization/test_snowflake.py @@ -52,6 +52,7 @@ def test_snowflake_materialization_consistency(online_store): batch_engine=SNOWFLAKE_ENGINE_CONFIG, ) snowflake_environment = construct_test_environment(snowflake_config, None) + snowflake_environment.setup() df = create_basic_driver_dataset() ds = snowflake_environment.data_source_creator.create_data_source( @@ -112,6 +113,7 @@ def test_snowflake_materialization_consistency_internal_with_lists( batch_engine=SNOWFLAKE_ENGINE_CONFIG, ) snowflake_environment = construct_test_environment(snowflake_config, None) + snowflake_environment.setup() df = create_basic_driver_dataset(Int32, feature_dtype, True, feature_is_empty_list) ds = snowflake_environment.data_source_creator.create_data_source( @@ -195,6 +197,7 @@ def test_snowflake_materialization_entityless_fv(): batch_engine=SNOWFLAKE_ENGINE_CONFIG, ) snowflake_environment = construct_test_environment(snowflake_config, None) + snowflake_environment.setup() df = create_basic_driver_dataset() entityless_df = df.drop("driver_id", axis=1) diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py index 2a2820c10a7..a6db7f2535c 100644 --- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py +++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py @@ -269,8 +269,8 @@ def test_historical_features_with_entities_from_query( if not orders_table: raise pytest.skip("Offline source is not sql-based") - data_source_creator = environment.test_repo_config.offline_store_creator - if data_source_creator.__name__ == SnowflakeDataSourceCreator.__name__: + data_source_creator = environment.data_source_creator + if isinstance(data_source_creator, SnowflakeDataSourceCreator): entity_df_query = f""" SELECT "customer_id", "driver_id", "order_id", "origin_id", "destination_id", "event_timestamp" FROM "{orders_table}" diff --git a/sdk/python/tests/integration/registration/test_universal_cli.py b/sdk/python/tests/integration/registration/test_universal_cli.py index e7f7a7cb633..e7331a07894 100644 --- a/sdk/python/tests/integration/registration/test_universal_cli.py +++ b/sdk/python/tests/integration/registration/test_universal_cli.py @@ -27,9 +27,10 @@ def test_universal_cli(environment: Environment): repo_path = Path(repo_dir_name) feature_store_yaml = make_feature_store_yaml( project, - environment.test_repo_config, repo_path, environment.data_source_creator, + environment.provider, + environment.online_store, ) repo_config = repo_path / "feature_store.yaml" @@ -124,9 +125,10 @@ def test_odfv_apply(environment) -> None: repo_path = Path(repo_dir_name) feature_store_yaml = make_feature_store_yaml( project, - environment.test_repo_config, repo_path, environment.data_source_creator, + environment.provider, + environment.online_store, ) repo_config = repo_path / "feature_store.yaml" @@ -158,9 +160,10 @@ def test_nullable_online_store(test_nullable_online_store) -> None: repo_path = Path(repo_dir_name) feature_store_yaml = make_feature_store_yaml( project, - test_nullable_online_store, repo_path, test_nullable_online_store.offline_store_creator(project), + test_nullable_online_store.provider, + test_nullable_online_store.online_store, ) repo_config = repo_path / "feature_store.yaml" diff --git a/sdk/python/tests/integration/registration/test_universal_types.py b/sdk/python/tests/integration/registration/test_universal_types.py index 3ce5876bd60..ca15681c9b2 100644 --- a/sdk/python/tests/integration/registration/test_universal_types.py +++ b/sdk/python/tests/integration/registration/test_universal_types.py @@ -110,7 +110,7 @@ def test_feature_get_historical_features_types_match( if config.feature_is_list: assert_feature_list_types( - environment.test_repo_config.provider, + environment.provider, config.feature_dtype, historical_features_df, ) @@ -119,7 +119,7 @@ def test_feature_get_historical_features_types_match( config.feature_dtype, historical_features_df ) assert_expected_arrow_types( - environment.test_repo_config.provider, + environment.provider, config.feature_dtype, config.feature_is_list, historical_features, @@ -335,10 +335,7 @@ class TypeTestConfig: ) def offline_types_test_fixtures(request, environment): config: TypeTestConfig = request.param - if ( - environment.test_repo_config.provider == "aws" - and config.feature_is_list is True - ): + if environment.provider == "aws" and config.feature_is_list is True: pytest.skip("Redshift doesn't support list features") return get_fixtures(request, environment) diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py index e5768a81b21..79a3a27b67a 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py @@ -10,7 +10,6 @@ AthenaRetrievalJob, ) from feast.infra.offline_stores.contrib.mssql_offline_store.mssql import ( - MsSqlServerOfflineStoreConfig, MsSqlServerRetrievalJob, ) from feast.infra.offline_stores.contrib.postgres_offline_store.postgres import ( @@ -120,12 +119,14 @@ def retrieval_job(request, environment): iam_role="arn:aws:iam::585132637328:role/service-role/AmazonRedshift-CommandsAccessRole-20240403T092631", workgroup="", ) - environment.test_repo_config.offline_store = offline_store_config + config = environment.config.copy( + update={"offline_config": offline_store_config} + ) return RedshiftRetrievalJob( query="query", redshift_client="", s3_resource="", - config=environment.test_repo_config, + config=config, full_feature_names=False, ) elif request.param is SnowflakeRetrievalJob: @@ -141,12 +142,14 @@ def retrieval_job(request, environment): storage_integration_name="FEAST_S3", blob_export_location="s3://feast-snowflake-offload/export", ) - environment.test_repo_config.offline_store = offline_store_config - environment.test_repo_config.project = "project" + config = environment.config.copy( + update={"offline_config": offline_store_config} + ) + environment.project = "project" return SnowflakeRetrievalJob( query="query", snowflake_conn=MagicMock(), - config=environment.test_repo_config, + config=config, full_feature_names=False, ) elif request.param is AthenaRetrievalJob: @@ -158,21 +161,18 @@ def retrieval_job(request, environment): s3_staging_location="athena", ) - environment.test_repo_config.offline_store = offline_store_config return AthenaRetrievalJob( query="query", athena_client="client", s3_resource="", - config=environment.test_repo_config.offline_store, + config=environment.config, full_feature_names=False, ) elif request.param is MsSqlServerRetrievalJob: return MsSqlServerRetrievalJob( query="query", engine=MagicMock(), - config=MsSqlServerOfflineStoreConfig( - connection_string="str" - ), # TODO: this does not match the RetrievalJob pattern. Suppose to be RepoConfig + config=environment.config, full_feature_names=False, ) elif request.param is PostgreSQLRetrievalJob: @@ -182,28 +182,25 @@ def retrieval_job(request, environment): user="str", password="str", ) - environment.test_repo_config.offline_store = offline_store_config return PostgreSQLRetrievalJob( query="query", - config=environment.test_repo_config.offline_store, + config=environment.config, full_feature_names=False, ) elif request.param is SparkRetrievalJob: offline_store_config = SparkOfflineStoreConfig() - environment.test_repo_config.offline_store = offline_store_config return SparkRetrievalJob( spark_session=MagicMock(), query="str", full_feature_names=False, - config=environment.test_repo_config, + config=environment.config, ) elif request.param is TrinoRetrievalJob: offline_store_config = SparkOfflineStoreConfig() - environment.test_repo_config.offline_store = offline_store_config return TrinoRetrievalJob( query="str", client=MagicMock(), - config=environment.test_repo_config, + config=environment.config, full_feature_names=False, ) else: diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py index 47ef503a7f5..37e57558678 100644 --- a/sdk/python/tests/utils/e2e_test_validation.py +++ b/sdk/python/tests/utils/e2e_test_validation.py @@ -3,7 +3,7 @@ import time from datetime import datetime, timedelta from pathlib import Path -from typing import List, Optional +from typing import Dict, List, Optional, Union import pandas as pd import pytest @@ -176,17 +176,18 @@ def _check_offline_and_online_features( def make_feature_store_yaml( project, - test_repo_config, repo_dir_name: Path, offline_creator: DataSourceCreator, + provider: str, + online_store: Optional[Union[str, Dict]], ): offline_store_config = offline_creator.create_offline_store_config() - online_store = test_repo_config.online_store + online_store = online_store config = RepoConfig( registry=str(Path(repo_dir_name) / "registry.db"), project=project, - provider=test_repo_config.provider, + provider=provider, offline_store=offline_store_config, online_store=online_store, repo_path=str(Path(repo_dir_name)), From 1f17b9bcd635d1344e1f9be4dcb745d8e206e071 Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Thu, 23 May 2024 06:28:30 -0500 Subject: [PATCH 68/73] docs: Operator readme fix (#4219) operator readme fix Signed-off-by: Tommy Hughes --- infra/feast-operator/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/feast-operator/README.md b/infra/feast-operator/README.md index 9ebfd5bd66a..ba9fe17fa3c 100644 --- a/infra/feast-operator/README.md +++ b/infra/feast-operator/README.md @@ -21,7 +21,7 @@ kind: FeastFeatureServer metadata: name: example spec: - feature_store_yaml_base64: $(cat feature_store.yaml | base64) + feature_store_yaml_base64: $(cat feature_store.yaml | base64 | tr -d '\n\r') EOF ``` Ensure it was successfully created on the cluster and that the `feature_store_yaml_base64` field was properly set. The following command should return output which is identical to your `feature_store.yaml`: From c1e3faa25564bc43697c43aac272b55c16c8069b Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Fri, 24 May 2024 14:46:19 +0400 Subject: [PATCH 69/73] chore: Refactor registry tests (#4223) chore: refactor registry tests Signed-off-by: tokoko --- sdk/python/tests/integration/e2e/__init__.py | 0 .../test_universal_e2e.py | 0 .../{e2e => offline_store}/test_validation.py | 0 .../test_python_feature_server.py | 0 .../integration/registration/test_registry.py | 232 --------- .../registration/test_universal_registry.py} | 488 ++++++++++++------ .../tests/integration/scaffolding/__init__.py | 0 .../tests/unit/infra/test_local_registry.py | 424 --------------- sdk/python/tests/utils/e2e_test_validation.py | 70 +-- 9 files changed, 323 insertions(+), 891 deletions(-) delete mode 100644 sdk/python/tests/integration/e2e/__init__.py rename sdk/python/tests/integration/{e2e => materialization}/test_universal_e2e.py (100%) rename sdk/python/tests/integration/{e2e => offline_store}/test_validation.py (100%) rename sdk/python/tests/integration/{e2e => online_store}/test_python_feature_server.py (100%) delete mode 100644 sdk/python/tests/integration/registration/test_registry.py rename sdk/python/tests/{unit/test_sql_registry.py => integration/registration/test_universal_registry.py} (63%) delete mode 100644 sdk/python/tests/integration/scaffolding/__init__.py diff --git a/sdk/python/tests/integration/e2e/__init__.py b/sdk/python/tests/integration/e2e/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/tests/integration/e2e/test_universal_e2e.py b/sdk/python/tests/integration/materialization/test_universal_e2e.py similarity index 100% rename from sdk/python/tests/integration/e2e/test_universal_e2e.py rename to sdk/python/tests/integration/materialization/test_universal_e2e.py diff --git a/sdk/python/tests/integration/e2e/test_validation.py b/sdk/python/tests/integration/offline_store/test_validation.py similarity index 100% rename from sdk/python/tests/integration/e2e/test_validation.py rename to sdk/python/tests/integration/offline_store/test_validation.py diff --git a/sdk/python/tests/integration/e2e/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py similarity index 100% rename from sdk/python/tests/integration/e2e/test_python_feature_server.py rename to sdk/python/tests/integration/online_store/test_python_feature_server.py diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py deleted file mode 100644 index 9ad1a98a050..00000000000 --- a/sdk/python/tests/integration/registration/test_registry.py +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright 2021 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import time -from datetime import timedelta -from unittest import mock - -import pytest -from pytest_lazyfixture import lazy_fixture -from testcontainers.minio import MinioContainer - -from feast import FileSource -from feast.data_format import ParquetFormat -from feast.entity import Entity -from feast.feature_view import FeatureView -from feast.field import Field -from feast.infra.registry.registry import Registry -from feast.repo_config import RegistryConfig -from feast.types import Array, Bytes, Int64, String -from tests.utils.e2e_test_validation import validate_registry_data_source_apply - - -@pytest.fixture -def gcs_registry() -> Registry: - from google.cloud import storage - - storage_client = storage.Client() - bucket_name = f"feast-registry-test-{int(time.time() * 1000)}" - bucket = storage_client.bucket(bucket_name) - bucket = storage_client.create_bucket(bucket) - bucket.add_lifecycle_delete_rule( - age=14 - ) # delete buckets automatically after 14 days - bucket.patch() - bucket.blob("registry.db") - registry_config = RegistryConfig( - path=f"gs://{bucket_name}/registry.db", cache_ttl_seconds=600 - ) - return Registry("project", registry_config, None) - - -@pytest.fixture -def s3_registry() -> Registry: - aws_registry_path = os.getenv( - "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries" - ) - registry_config = RegistryConfig( - path=f"{aws_registry_path}/{int(time.time() * 1000)}/registry.db", - cache_ttl_seconds=600, - ) - return Registry("project", registry_config, None) - - -@pytest.fixture -def minio_registry() -> Registry: - bucket_name = "test-bucket" - - container = MinioContainer() - container.start() - client = container.get_client() - client.make_bucket(bucket_name) - - container_host = container.get_container_host_ip() - exposed_port = container.get_exposed_port(container.port) - - registry_config = RegistryConfig( - path=f"s3://{bucket_name}/registry.db", cache_ttl_seconds=600 - ) - - mock_environ = { - "FEAST_S3_ENDPOINT_URL": f"http://{container_host}:{exposed_port}", - "AWS_ACCESS_KEY_ID": container.access_key, - "AWS_SECRET_ACCESS_KEY": container.secret_key, - "AWS_SESSION_TOKEN": "", - } - - with mock.patch.dict(os.environ, mock_environ): - yield Registry("project", registry_config, None) - - container.stop() - - -@pytest.mark.integration -@pytest.mark.parametrize( - "test_registry", - [ - lazy_fixture("gcs_registry"), - lazy_fixture("s3_registry"), - lazy_fixture("minio_registry"), - ], -) -def test_apply_entity_integration(test_registry): - entity = Entity( - name="driver_car_id", - description="Car driver id", - tags={"team": "matchmaking"}, - ) - - project = "project" - - # Register Entity - test_registry.apply_entity(entity, project) - - entities = test_registry.list_entities(project) - - entity = entities[0] - assert ( - len(entities) == 1 - and entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" - ) - - entity = test_registry.get_entity("driver_car_id", project) - assert ( - entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" - ) - - test_registry.teardown() - - # Will try to reload registry, which will fail because the file has been deleted - with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto(project=project) - - -@pytest.mark.integration -@pytest.mark.parametrize( - "test_registry", - [ - lazy_fixture("gcs_registry"), - lazy_fixture("s3_registry"), - lazy_fixture("minio_registry"), - ], -) -def test_apply_feature_view_integration(test_registry): - # Create Feature Views - batch_source = FileSource( - file_format=ParquetFormat(), - path="file://feast/*", - timestamp_field="ts_col", - created_timestamp_column="timestamp", - ) - - entity = Entity(name="fs1_my_entity_1", join_keys=["test"]) - - fv1 = FeatureView( - name="my_feature_view_1", - schema=[ - Field(name="fs1_my_feature_1", dtype=Int64), - Field(name="fs1_my_feature_2", dtype=String), - Field(name="fs1_my_feature_3", dtype=Array(String)), - Field(name="fs1_my_feature_4", dtype=Array(Bytes)), - ], - entities=[entity], - tags={"team": "matchmaking"}, - source=batch_source, - ttl=timedelta(minutes=5), - ) - - project = "project" - - # Register Feature View - test_registry.apply_feature_view(fv1, project) - - feature_views = test_registry.list_feature_views(project) - - # List Feature Views - assert ( - len(feature_views) == 1 - and feature_views[0].name == "my_feature_view_1" - and feature_views[0].features[0].name == "fs1_my_feature_1" - and feature_views[0].features[0].dtype == Int64 - and feature_views[0].features[1].name == "fs1_my_feature_2" - and feature_views[0].features[1].dtype == String - and feature_views[0].features[2].name == "fs1_my_feature_3" - and feature_views[0].features[2].dtype == Array(String) - and feature_views[0].features[3].name == "fs1_my_feature_4" - and feature_views[0].features[3].dtype == Array(Bytes) - and feature_views[0].entities[0] == "fs1_my_entity_1" - ) - - feature_view = test_registry.get_feature_view("my_feature_view_1", project) - assert ( - feature_view.name == "my_feature_view_1" - and feature_view.features[0].name == "fs1_my_feature_1" - and feature_view.features[0].dtype == Int64 - and feature_view.features[1].name == "fs1_my_feature_2" - and feature_view.features[1].dtype == String - and feature_view.features[2].name == "fs1_my_feature_3" - and feature_view.features[2].dtype == Array(String) - and feature_view.features[3].name == "fs1_my_feature_4" - and feature_view.features[3].dtype == Array(Bytes) - and feature_view.entities[0] == "fs1_my_entity_1" - ) - - test_registry.delete_feature_view("my_feature_view_1", project) - feature_views = test_registry.list_feature_views(project) - assert len(feature_views) == 0 - - test_registry.teardown() - - # Will try to reload registry, which will fail because the file has been deleted - with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto(project=project) - - -@pytest.mark.integration -@pytest.mark.parametrize( - "test_registry", - [ - lazy_fixture("gcs_registry"), - lazy_fixture("s3_registry"), - lazy_fixture("minio_registry"), - ], -) -def test_apply_data_source_integration(test_registry: Registry): - validate_registry_data_source_apply(test_registry) diff --git a/sdk/python/tests/unit/test_sql_registry.py b/sdk/python/tests/integration/registration/test_universal_registry.py similarity index 63% rename from sdk/python/tests/unit/test_sql_registry.py rename to sdk/python/tests/integration/registration/test_universal_registry.py index a1460663aeb..1f0ccb4f6b5 100644 --- a/sdk/python/tests/unit/test_sql_registry.py +++ b/sdk/python/tests/integration/registration/test_universal_registry.py @@ -13,31 +13,105 @@ # limitations under the License. import logging import os -import sys +import time from datetime import timedelta +from tempfile import mkstemp +from unittest import mock import pandas as pd import pytest from pytest_lazyfixture import lazy_fixture from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs +from testcontainers.minio import MinioContainer from testcontainers.mysql import MySqlContainer from feast import FileSource, RequestSource -from feast.data_format import ParquetFormat +from feast.data_format import AvroFormat, ParquetFormat +from feast.data_source import KafkaSource from feast.entity import Entity from feast.errors import FeatureViewNotFoundException from feast.feature_view import FeatureView from feast.field import Field from feast.infra.infra_object import Infra from feast.infra.online_stores.sqlite import SqliteTable +from feast.infra.registry.registry import Registry from feast.infra.registry.sql import SqlRegistry from feast.on_demand_feature_view import on_demand_feature_view from feast.repo_config import RegistryConfig +from feast.stream_feature_view import Aggregation, StreamFeatureView from feast.types import Array, Bytes, Float32, Int32, Int64, String from feast.value_type import ValueType from tests.integration.feature_repos.universal.entities import driver + +@pytest.fixture +def local_registry() -> Registry: + fd, registry_path = mkstemp() + registry_config = RegistryConfig(path=registry_path, cache_ttl_seconds=600) + return Registry("project", registry_config, None) + + +@pytest.fixture +def gcs_registry() -> Registry: + from google.cloud import storage + + storage_client = storage.Client() + bucket_name = f"feast-registry-test-{int(time.time() * 1000)}" + bucket = storage_client.bucket(bucket_name) + bucket = storage_client.create_bucket(bucket) + bucket.add_lifecycle_delete_rule( + age=14 + ) # delete buckets automatically after 14 days + bucket.patch() + bucket.blob("registry.db") + registry_config = RegistryConfig( + path=f"gs://{bucket_name}/registry.db", cache_ttl_seconds=600 + ) + return Registry("project", registry_config, None) + + +@pytest.fixture +def s3_registry() -> Registry: + aws_registry_path = os.getenv( + "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries" + ) + registry_config = RegistryConfig( + path=f"{aws_registry_path}/{int(time.time() * 1000)}/registry.db", + cache_ttl_seconds=600, + ) + return Registry("project", registry_config, None) + + +@pytest.fixture(scope="session") +def minio_registry() -> Registry: + bucket_name = "test-bucket" + + container = MinioContainer() + container.start() + client = container.get_client() + client.make_bucket(bucket_name) + + container_host = container.get_container_host_ip() + exposed_port = container.get_exposed_port(container.port) + + registry_config = RegistryConfig( + path=f"s3://{bucket_name}/registry.db", cache_ttl_seconds=600 + ) + + mock_environ = { + "FEAST_S3_ENDPOINT_URL": f"http://{container_host}:{exposed_port}", + "AWS_ACCESS_KEY_ID": container.access_key, + "AWS_SECRET_ACCESS_KEY": container.secret_key, + "AWS_SESSION_TOKEN": "", + } + + with mock.patch.dict(os.environ, mock_environ): + yield Registry("project", registry_config, None) + + container.stop() + + POSTGRES_USER = "test" POSTGRES_PASSWORD = "test" POSTGRES_DB = "test" @@ -113,19 +187,20 @@ def sqlite_registry(): yield SqlRegistry(registry_config, "project", None) -@pytest.mark.skipif( - sys.platform == "darwin" and "GITHUB_REF" in os.environ, - reason="does not run on mac github actions", -) +@pytest.mark.integration @pytest.mark.parametrize( - "sql_registry", + "test_registry", [ - lazy_fixture("mysql_registry"), + lazy_fixture("local_registry"), + lazy_fixture("gcs_registry"), + lazy_fixture("s3_registry"), + lazy_fixture("minio_registry"), lazy_fixture("pg_registry"), + lazy_fixture("mysql_registry"), lazy_fixture("sqlite_registry"), ], ) -def test_apply_entity_success(sql_registry): +def test_apply_entity_success(test_registry): entity = Entity( name="driver_car_id", description="Car driver id", @@ -135,15 +210,15 @@ def test_apply_entity_success(sql_registry): project = "project" # Register Entity - sql_registry.apply_entity(entity, project) - project_metadata = sql_registry.list_project_metadata(project=project) + test_registry.apply_entity(entity, project) + project_metadata = test_registry.list_project_metadata(project=project) assert len(project_metadata) == 1 project_uuid = project_metadata[0].project_uuid assert len(project_metadata[0].project_uuid) == 36 - assert_project_uuid(project, project_uuid, sql_registry) + assert_project_uuid(project, project_uuid, test_registry) - entities = sql_registry.list_entities(project) - assert_project_uuid(project, project_uuid, sql_registry) + entities = test_registry.list_entities(project) + assert_project_uuid(project, project_uuid, test_registry) entity = entities[0] assert ( @@ -154,7 +229,7 @@ def test_apply_entity_success(sql_registry): and entity.tags["team"] == "matchmaking" ) - entity = sql_registry.get_entity("driver_car_id", project) + entity = test_registry.get_entity("driver_car_id", project) assert ( entity.name == "driver_car_id" and entity.description == "Car driver id" @@ -165,34 +240,35 @@ def test_apply_entity_success(sql_registry): # After the first apply, the created_timestamp should be the same as the last_update_timestamp. assert entity.created_timestamp == entity.last_updated_timestamp - sql_registry.delete_entity("driver_car_id", project) - assert_project_uuid(project, project_uuid, sql_registry) - entities = sql_registry.list_entities(project) - assert_project_uuid(project, project_uuid, sql_registry) + test_registry.delete_entity("driver_car_id", project) + assert_project_uuid(project, project_uuid, test_registry) + entities = test_registry.list_entities(project) + assert_project_uuid(project, project_uuid, test_registry) assert len(entities) == 0 - sql_registry.teardown() + test_registry.teardown() -def assert_project_uuid(project, project_uuid, sql_registry): - project_metadata = sql_registry.list_project_metadata(project=project) +def assert_project_uuid(project, project_uuid, test_registry): + project_metadata = test_registry.list_project_metadata(project=project) assert len(project_metadata) == 1 assert project_metadata[0].project_uuid == project_uuid -@pytest.mark.skipif( - sys.platform == "darwin" and "GITHUB_REF" in os.environ, - reason="does not run on mac github actions", -) +@pytest.mark.integration @pytest.mark.parametrize( - "sql_registry", + "test_registry", [ - lazy_fixture("mysql_registry"), + lazy_fixture("local_registry"), + lazy_fixture("gcs_registry"), + lazy_fixture("s3_registry"), + lazy_fixture("minio_registry"), lazy_fixture("pg_registry"), + lazy_fixture("mysql_registry"), lazy_fixture("sqlite_registry"), ], ) -def test_apply_feature_view_success(sql_registry): +def test_apply_feature_view_success(test_registry): # Create Feature Views batch_source = FileSource( file_format=ParquetFormat(), @@ -221,9 +297,9 @@ def test_apply_feature_view_success(sql_registry): project = "project" # Register Feature View - sql_registry.apply_feature_view(fv1, project) + test_registry.apply_feature_view(fv1, project) - feature_views = sql_registry.list_feature_views(project) + feature_views = test_registry.list_feature_views(project) # List Feature Views assert ( @@ -240,7 +316,7 @@ def test_apply_feature_view_success(sql_registry): and feature_views[0].entities[0] == "fs1_my_entity_1" ) - feature_view = sql_registry.get_feature_view("my_feature_view_1", project) + feature_view = test_registry.get_feature_view("my_feature_view_1", project) assert ( feature_view.name == "my_feature_view_1" and feature_view.features[0].name == "fs1_my_feature_1" @@ -260,33 +336,34 @@ def test_apply_feature_view_success(sql_registry): # Modify the feature view and apply again to test if diffing the online store table works fv1.ttl = timedelta(minutes=6) - sql_registry.apply_feature_view(fv1, project) - feature_views = sql_registry.list_feature_views(project) + test_registry.apply_feature_view(fv1, project) + feature_views = test_registry.list_feature_views(project) assert len(feature_views) == 1 - feature_view = sql_registry.get_feature_view("my_feature_view_1", project) + feature_view = test_registry.get_feature_view("my_feature_view_1", project) assert feature_view.ttl == timedelta(minutes=6) # Delete feature view - sql_registry.delete_feature_view("my_feature_view_1", project) - feature_views = sql_registry.list_feature_views(project) + test_registry.delete_feature_view("my_feature_view_1", project) + feature_views = test_registry.list_feature_views(project) assert len(feature_views) == 0 - sql_registry.teardown() + test_registry.teardown() -@pytest.mark.skipif( - sys.platform == "darwin" and "GITHUB_REF" in os.environ, - reason="does not run on mac github actions", -) +@pytest.mark.integration @pytest.mark.parametrize( - "sql_registry", + "test_registry", [ - lazy_fixture("mysql_registry"), + # lazy_fixture("local_registry"), + # lazy_fixture("gcs_registry"), + # lazy_fixture("s3_registry"), + # lazy_fixture("minio_registry"), lazy_fixture("pg_registry"), + lazy_fixture("mysql_registry"), lazy_fixture("sqlite_registry"), ], ) -def test_apply_on_demand_feature_view_success(sql_registry): +def test_apply_on_demand_feature_view_success(test_registry): # Create Feature Views driver_stats = FileSource( name="driver_stats_source", @@ -326,18 +403,18 @@ def location_features_from_push(inputs: pd.DataFrame) -> pd.DataFrame: project = "project" with pytest.raises(FeatureViewNotFoundException): - sql_registry.get_user_metadata(project, location_features_from_push) + test_registry.get_user_metadata(project, location_features_from_push) # Register Feature View - sql_registry.apply_feature_view(location_features_from_push, project) + test_registry.apply_feature_view(location_features_from_push, project) - assert not sql_registry.get_user_metadata(project, location_features_from_push) + assert not test_registry.get_user_metadata(project, location_features_from_push) b = "metadata".encode("utf-8") - sql_registry.apply_user_metadata(project, location_features_from_push, b) - assert sql_registry.get_user_metadata(project, location_features_from_push) == b + test_registry.apply_user_metadata(project, location_features_from_push, b) + assert test_registry.get_user_metadata(project, location_features_from_push) == b - feature_views = sql_registry.list_on_demand_feature_views(project) + feature_views = test_registry.list_on_demand_feature_views(project) # List Feature Views assert ( @@ -347,7 +424,7 @@ def location_features_from_push(inputs: pd.DataFrame) -> pd.DataFrame: and feature_views[0].features[0].dtype == String ) - feature_view = sql_registry.get_on_demand_feature_view( + feature_view = test_registry.get_on_demand_feature_view( "location_features_from_push", project ) assert ( @@ -356,26 +433,98 @@ def location_features_from_push(inputs: pd.DataFrame) -> pd.DataFrame: and feature_view.features[0].dtype == String ) - sql_registry.delete_feature_view("location_features_from_push", project) - feature_views = sql_registry.list_on_demand_feature_views(project) + test_registry.delete_feature_view("location_features_from_push", project) + feature_views = test_registry.list_on_demand_feature_views(project) assert len(feature_views) == 0 - sql_registry.teardown() + test_registry.teardown() -@pytest.mark.skipif( - sys.platform == "darwin" and "GITHUB_REF" in os.environ, - reason="does not run on mac github actions", -) +@pytest.mark.integration @pytest.mark.parametrize( - "sql_registry", + "test_registry", [ + lazy_fixture("local_registry"), + lazy_fixture("gcs_registry"), + lazy_fixture("s3_registry"), + lazy_fixture("minio_registry"), + lazy_fixture("pg_registry"), lazy_fixture("mysql_registry"), + lazy_fixture("sqlite_registry"), + ], +) +def test_apply_data_source(test_registry): + # Create Feature Views + batch_source = FileSource( + name="test_source", + file_format=ParquetFormat(), + path="file://feast/*", + timestamp_field="ts_col", + created_timestamp_column="timestamp", + ) + + entity = Entity(name="fs1_my_entity_1", join_keys=["test"]) + + fv1 = FeatureView( + name="my_feature_view_1", + schema=[ + Field(name="test", dtype=Int64), + Field(name="fs1_my_feature_1", dtype=Int64), + Field(name="fs1_my_feature_2", dtype=String), + Field(name="fs1_my_feature_3", dtype=Array(String)), + Field(name="fs1_my_feature_4", dtype=Array(Bytes)), + ], + entities=[entity], + tags={"team": "matchmaking"}, + source=batch_source, + ttl=timedelta(minutes=5), + ) + + project = "project" + + # Register data source and feature view + test_registry.apply_data_source(batch_source, project, commit=False) + test_registry.apply_feature_view(fv1, project, commit=True) + + registry_feature_views = test_registry.list_feature_views(project) + registry_data_sources = test_registry.list_data_sources(project) + assert len(registry_feature_views) == 1 + assert len(registry_data_sources) == 1 + registry_feature_view = registry_feature_views[0] + assert registry_feature_view.batch_source == batch_source + registry_data_source = registry_data_sources[0] + assert registry_data_source == batch_source + + # Check that change to batch source propagates + batch_source.timestamp_field = "new_ts_col" + test_registry.apply_data_source(batch_source, project, commit=False) + test_registry.apply_feature_view(fv1, project, commit=True) + registry_feature_views = test_registry.list_feature_views(project) + registry_data_sources = test_registry.list_data_sources(project) + assert len(registry_feature_views) == 1 + assert len(registry_data_sources) == 1 + registry_feature_view = registry_feature_views[0] + assert registry_feature_view.batch_source == batch_source + registry_batch_source = test_registry.list_data_sources(project)[0] + assert registry_batch_source == batch_source + + test_registry.teardown() + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_registry", + [ + lazy_fixture("local_registry"), + lazy_fixture("gcs_registry"), + lazy_fixture("s3_registry"), + lazy_fixture("minio_registry"), lazy_fixture("pg_registry"), + lazy_fixture("mysql_registry"), lazy_fixture("sqlite_registry"), ], ) -def test_modify_feature_views_success(sql_registry): +def test_modify_feature_views_success(test_registry): # Create Feature Views batch_source = FileSource( file_format=ParquetFormat(), @@ -419,8 +568,8 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: project = "project" # Register Feature Views - sql_registry.apply_feature_view(odfv1, project) - sql_registry.apply_feature_view(fv1, project) + test_registry.apply_feature_view(odfv1, project) + test_registry.apply_feature_view(fv1, project) # Modify odfv by changing a single feature dtype @on_demand_feature_view( @@ -437,10 +586,10 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: return data # Apply the modified odfv - sql_registry.apply_feature_view(odfv1, project) + test_registry.apply_feature_view(odfv1, project) # Check odfv - on_demand_feature_views = sql_registry.list_on_demand_feature_views(project) + on_demand_feature_views = test_registry.list_on_demand_feature_views(project) assert ( len(on_demand_feature_views) == 1 @@ -456,7 +605,7 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: and list(request_schema.values())[0] == ValueType.INT32 ) - feature_view = sql_registry.get_on_demand_feature_view("odfv1", project) + feature_view = test_registry.get_on_demand_feature_view("odfv1", project) assert ( feature_view.name == "odfv1" and feature_view.features[0].name == "odfv1_my_feature_1" @@ -471,7 +620,7 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: ) # Make sure fv1 is untouched - feature_views = sql_registry.list_feature_views(project) + feature_views = test_registry.list_feature_views(project) # List Feature Views assert ( @@ -482,7 +631,7 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: and feature_views[0].entities[0] == "fs1_my_entity_1" ) - feature_view = sql_registry.get_feature_view("my_feature_view_1", project) + feature_view = test_registry.get_feature_view("my_feature_view_1", project) assert ( feature_view.name == "my_feature_view_1" and feature_view.features[0].name == "fs1_my_feature_1" @@ -490,92 +639,62 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: and feature_view.entities[0] == "fs1_my_entity_1" ) - sql_registry.teardown() + test_registry.teardown() -@pytest.mark.skipif( - sys.platform == "darwin" and "GITHUB_REF" in os.environ, - reason="does not run on mac github actions", -) +@pytest.mark.integration @pytest.mark.parametrize( - "sql_registry", + "test_registry", [ - lazy_fixture("mysql_registry"), + # lazy_fixture("local_registry"), + # lazy_fixture("gcs_registry"), + # lazy_fixture("s3_registry"), + # lazy_fixture("minio_registry"), lazy_fixture("pg_registry"), + lazy_fixture("mysql_registry"), lazy_fixture("sqlite_registry"), ], ) -def test_apply_data_source(sql_registry): - # Create Feature Views - batch_source = FileSource( - name="test_source", - file_format=ParquetFormat(), - path="file://feast/*", - timestamp_field="ts_col", - created_timestamp_column="timestamp", - ) - - entity = Entity(name="fs1_my_entity_1", join_keys=["test"]) - - fv1 = FeatureView( - name="my_feature_view_1", - schema=[ - Field(name="test", dtype=Int64), - Field(name="fs1_my_feature_1", dtype=Int64), - Field(name="fs1_my_feature_2", dtype=String), - Field(name="fs1_my_feature_3", dtype=Array(String)), - Field(name="fs1_my_feature_4", dtype=Array(Bytes)), - ], - entities=[entity], - tags={"team": "matchmaking"}, - source=batch_source, - ttl=timedelta(minutes=5), - ) - +def test_update_infra(test_registry): + # Create infra object project = "project" + infra = test_registry.get_infra(project=project) - # Register data source and feature view - sql_registry.apply_data_source(batch_source, project, commit=False) - sql_registry.apply_feature_view(fv1, project, commit=True) + assert len(infra.infra_objects) == 0 - registry_feature_views = sql_registry.list_feature_views(project) - registry_data_sources = sql_registry.list_data_sources(project) - assert len(registry_feature_views) == 1 - assert len(registry_data_sources) == 1 - registry_feature_view = registry_feature_views[0] - assert registry_feature_view.batch_source == batch_source - registry_data_source = registry_data_sources[0] - assert registry_data_source == batch_source + # Should run update infra successfully + test_registry.update_infra(infra, project) - # Check that change to batch source propagates - batch_source.timestamp_field = "new_ts_col" - sql_registry.apply_data_source(batch_source, project, commit=False) - sql_registry.apply_feature_view(fv1, project, commit=True) - registry_feature_views = sql_registry.list_feature_views(project) - registry_data_sources = sql_registry.list_data_sources(project) - assert len(registry_feature_views) == 1 - assert len(registry_data_sources) == 1 - registry_feature_view = registry_feature_views[0] - assert registry_feature_view.batch_source == batch_source - registry_batch_source = sql_registry.list_data_sources(project)[0] - assert registry_batch_source == batch_source + # Should run update infra successfully when adding + new_infra = Infra() + new_infra.infra_objects.append( + SqliteTable( + path="/tmp/my_path.db", + name="my_table", + ) + ) + test_registry.update_infra(new_infra, project) + infra = test_registry.get_infra(project=project) + assert len(infra.infra_objects) == 1 - sql_registry.teardown() + # Try again since second time, infra should be not-empty + test_registry.teardown() -@pytest.mark.skipif( - sys.platform == "darwin" and "GITHUB_REF" in os.environ, - reason="does not run on mac github actions", -) +@pytest.mark.integration @pytest.mark.parametrize( - "sql_registry", + "test_registry", [ - lazy_fixture("mysql_registry"), + # lazy_fixture("local_registry"), + # lazy_fixture("gcs_registry"), + # lazy_fixture("s3_registry"), + # lazy_fixture("minio_registry"), lazy_fixture("pg_registry"), + lazy_fixture("mysql_registry"), lazy_fixture("sqlite_registry"), ], ) -def test_registry_cache(sql_registry): +def test_registry_cache(test_registry): # Create Feature Views batch_source = FileSource( name="test_source", @@ -605,23 +724,23 @@ def test_registry_cache(sql_registry): project = "project" # Register data source and feature view - sql_registry.apply_data_source(batch_source, project) - sql_registry.apply_feature_view(fv1, project) - registry_feature_views_cached = sql_registry.list_feature_views( + test_registry.apply_data_source(batch_source, project) + test_registry.apply_feature_view(fv1, project) + registry_feature_views_cached = test_registry.list_feature_views( project, allow_cache=True ) - registry_data_sources_cached = sql_registry.list_data_sources( + registry_data_sources_cached = test_registry.list_data_sources( project, allow_cache=True ) # Not refreshed cache, so cache miss assert len(registry_feature_views_cached) == 0 assert len(registry_data_sources_cached) == 0 - sql_registry.refresh(project) + test_registry.refresh(project) # Now objects exist - registry_feature_views_cached = sql_registry.list_feature_views( + registry_feature_views_cached = test_registry.list_feature_views( project, allow_cache=True ) - registry_data_sources_cached = sql_registry.list_data_sources( + registry_data_sources_cached = test_registry.list_data_sources( project, allow_cache=True ) assert len(registry_feature_views_cached) == 1 @@ -631,42 +750,79 @@ def test_registry_cache(sql_registry): registry_data_source = registry_data_sources_cached[0] assert registry_data_source == batch_source - sql_registry.teardown() + test_registry.teardown() -@pytest.mark.skipif( - sys.platform == "darwin" and "GITHUB_REF" in os.environ, - reason="does not run on mac github actions", -) +@pytest.mark.integration @pytest.mark.parametrize( - "sql_registry", + "test_registry", [ - lazy_fixture("mysql_registry"), + lazy_fixture("local_registry"), + lazy_fixture("gcs_registry"), + lazy_fixture("s3_registry"), + lazy_fixture("minio_registry"), lazy_fixture("pg_registry"), + lazy_fixture("mysql_registry"), lazy_fixture("sqlite_registry"), ], ) -def test_update_infra(sql_registry): - # Create infra object - project = "project" - infra = sql_registry.get_infra(project=project) +def test_apply_stream_feature_view_success(test_registry): + # Create Feature Views + def simple_udf(x: int): + return x + 3 - assert len(infra.infra_objects) == 0 + entity = Entity(name="driver_entity", join_keys=["test_key"]) - # Should run update infra successfully - sql_registry.update_infra(infra, project) + stream_source = KafkaSource( + name="kafka", + timestamp_field="event_timestamp", + kafka_bootstrap_servers="", + message_format=AvroFormat(""), + topic="topic", + batch_source=FileSource(path="some path"), + watermark_delay_threshold=timedelta(days=1), + ) - # Should run update infra successfully when adding - new_infra = Infra() - new_infra.infra_objects.append( - SqliteTable( - path="/tmp/my_path.db", - name="my_table", - ) + sfv = StreamFeatureView( + name="test kafka stream feature view", + entities=[entity], + ttl=timedelta(days=30), + owner="test@example.com", + online=True, + schema=[Field(name="dummy_field", dtype=Float32)], + description="desc", + aggregations=[ + Aggregation( + column="dummy_field", + function="max", + time_window=timedelta(days=1), + ), + Aggregation( + column="dummy_field2", + function="count", + time_window=timedelta(days=24), + ), + ], + timestamp_field="event_timestamp", + mode="spark", + source=stream_source, + udf=simple_udf, + tags={}, ) - sql_registry.update_infra(new_infra, project) - infra = sql_registry.get_infra(project=project) - assert len(infra.infra_objects) == 1 - # Try again since second time, infra should be not-empty - sql_registry.teardown() + project = "project" + + # Register Feature View + test_registry.apply_feature_view(sfv, project) + + stream_feature_views = test_registry.list_stream_feature_views(project) + + # List Feature Views + assert len(stream_feature_views) == 1 + assert stream_feature_views[0] == sfv + + test_registry.delete_feature_view("test kafka stream feature view", project) + stream_feature_views = test_registry.list_stream_feature_views(project) + assert len(stream_feature_views) == 0 + + test_registry.teardown() diff --git a/sdk/python/tests/integration/scaffolding/__init__.py b/sdk/python/tests/integration/scaffolding/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/tests/unit/infra/test_local_registry.py b/sdk/python/tests/unit/infra/test_local_registry.py index 73f5cd91a5d..c86a616c406 100644 --- a/sdk/python/tests/unit/infra/test_local_registry.py +++ b/sdk/python/tests/unit/infra/test_local_registry.py @@ -11,437 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from datetime import timedelta from tempfile import mkstemp -import pandas as pd import pytest -from pytest_lazyfixture import lazy_fixture -from feast import FileSource -from feast.aggregation import Aggregation -from feast.data_format import AvroFormat, ParquetFormat -from feast.data_source import KafkaSource from feast.entity import Entity -from feast.feature_view import FeatureView -from feast.field import Field from feast.infra.registry.registry import Registry -from feast.on_demand_feature_view import RequestSource, on_demand_feature_view from feast.repo_config import RegistryConfig -from feast.stream_feature_view import StreamFeatureView -from feast.types import Array, Bytes, Float32, Int32, Int64, String -from feast.value_type import ValueType -from tests.integration.feature_repos.universal.entities import driver -from tests.utils.e2e_test_validation import validate_registry_data_source_apply - - -@pytest.fixture -def local_registry() -> Registry: - fd, registry_path = mkstemp() - registry_config = RegistryConfig(path=registry_path, cache_ttl_seconds=600) - return Registry("project", registry_config, None) - - -@pytest.mark.parametrize( - "test_registry", - [lazy_fixture("local_registry")], -) -def test_apply_entity_success(test_registry): - entity = Entity( - name="driver_car_id", - description="Car driver id", - tags={"team": "matchmaking"}, - ) - - project = "project" - - # Register Entity - test_registry.apply_entity(entity, project) - - entities = test_registry.list_entities(project) - - entity = entities[0] - assert ( - len(entities) == 1 - and entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" - ) - - entity = test_registry.get_entity("driver_car_id", project) - assert ( - entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" - ) - - test_registry.delete_entity("driver_car_id", project) - entities = test_registry.list_entities(project) - assert len(entities) == 0 - - test_registry.teardown() - - # Will try to reload registry, which will fail because the file has been deleted - with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto(project=project) - - -@pytest.mark.parametrize( - "test_registry", - [lazy_fixture("local_registry")], -) -def test_apply_feature_view_success(test_registry): - # Create Feature Views - batch_source = FileSource( - file_format=ParquetFormat(), - path="file://feast/*", - timestamp_field="ts_col", - created_timestamp_column="timestamp", - ) - - entity = Entity(name="fs1_my_entity_1", join_keys=["test"]) - - fv1 = FeatureView( - name="my_feature_view_1", - schema=[ - Field(name="test", dtype=Int64), - Field(name="fs1_my_feature_1", dtype=Int64), - Field(name="fs1_my_feature_2", dtype=String), - Field(name="fs1_my_feature_3", dtype=Array(String)), - Field(name="fs1_my_feature_4", dtype=Array(Bytes)), - ], - entities=[entity], - tags={"team": "matchmaking"}, - source=batch_source, - ttl=timedelta(minutes=5), - ) - - project = "project" - - # Register Feature View - test_registry.apply_feature_view(fv1, project) - - feature_views = test_registry.list_feature_views(project) - - # List Feature Views - assert ( - len(feature_views) == 1 - and feature_views[0].name == "my_feature_view_1" - and feature_views[0].features[0].name == "fs1_my_feature_1" - and feature_views[0].features[0].dtype == Int64 - and feature_views[0].features[1].name == "fs1_my_feature_2" - and feature_views[0].features[1].dtype == String - and feature_views[0].features[2].name == "fs1_my_feature_3" - and feature_views[0].features[2].dtype == Array(String) - and feature_views[0].features[3].name == "fs1_my_feature_4" - and feature_views[0].features[3].dtype == Array(Bytes) - and feature_views[0].entities[0] == "fs1_my_entity_1" - ) - - feature_view = test_registry.get_feature_view("my_feature_view_1", project) - assert ( - feature_view.name == "my_feature_view_1" - and feature_view.features[0].name == "fs1_my_feature_1" - and feature_view.features[0].dtype == Int64 - and feature_view.features[1].name == "fs1_my_feature_2" - and feature_view.features[1].dtype == String - and feature_view.features[2].name == "fs1_my_feature_3" - and feature_view.features[2].dtype == Array(String) - and feature_view.features[3].name == "fs1_my_feature_4" - and feature_view.features[3].dtype == Array(Bytes) - and feature_view.entities[0] == "fs1_my_entity_1" - ) - - test_registry.delete_feature_view("my_feature_view_1", project) - feature_views = test_registry.list_feature_views(project) - assert len(feature_views) == 0 - - test_registry.teardown() - - # Will try to reload registry, which will fail because the file has been deleted - with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto(project=project) - - -@pytest.mark.parametrize( - "test_registry", - [lazy_fixture("local_registry")], -) -def test_apply_on_demand_feature_view_success(test_registry): - # Create Feature Views - driver_stats = FileSource( - name="driver_stats_source", - path="data/driver_stats_lat_lon.parquet", - timestamp_field="event_timestamp", - created_timestamp_column="created", - description="A table describing the stats of a driver based on hourly logs", - owner="test2@gmail.com", - ) - - driver_daily_features_view = FeatureView( - name="driver_daily_features", - entities=[driver()], - ttl=timedelta(seconds=8640000000), - schema=[ - Field(name="daily_miles_driven", dtype=Float32), - Field(name="lat", dtype=Float32), - Field(name="lon", dtype=Float32), - Field(name="string_feature", dtype=String), - ], - online=True, - source=driver_stats, - tags={"production": "True"}, - owner="test2@gmail.com", - ) - - @on_demand_feature_view( - sources=[driver_daily_features_view], - schema=[Field(name="first_char", dtype=String)], - ) - def location_features_from_push(inputs: pd.DataFrame) -> pd.DataFrame: - df = pd.DataFrame() - df["first_char"] = inputs["string_feature"].str[:1].astype("string") - return df - - project = "project" - - # Register Feature View - test_registry.apply_feature_view(location_features_from_push, project) - - feature_views = test_registry.list_on_demand_feature_views(project) - - # List Feature Views - assert ( - len(feature_views) == 1 - and feature_views[0].name == "location_features_from_push" - and feature_views[0].features[0].name == "first_char" - and feature_views[0].features[0].dtype == String - ) - - feature_view = test_registry.get_on_demand_feature_view( - "location_features_from_push", project - ) - assert ( - feature_view.name == "location_features_from_push" - and feature_view.features[0].name == "first_char" - and feature_view.features[0].dtype == String - ) - - test_registry.delete_feature_view("location_features_from_push", project) - feature_views = test_registry.list_on_demand_feature_views(project) - assert len(feature_views) == 0 - - test_registry.teardown() - - # Will try to reload registry, which will fail because the file has been deleted - with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto(project=project) - - -@pytest.mark.parametrize( - "test_registry", - [lazy_fixture("local_registry")], -) -def test_apply_stream_feature_view_success(test_registry): - # Create Feature Views - def simple_udf(x: int): - return x + 3 - - entity = Entity(name="driver_entity", join_keys=["test_key"]) - - stream_source = KafkaSource( - name="kafka", - timestamp_field="event_timestamp", - kafka_bootstrap_servers="", - message_format=AvroFormat(""), - topic="topic", - batch_source=FileSource(path="some path"), - watermark_delay_threshold=timedelta(days=1), - ) - - sfv = StreamFeatureView( - name="test kafka stream feature view", - entities=[entity], - ttl=timedelta(days=30), - owner="test@example.com", - online=True, - schema=[Field(name="dummy_field", dtype=Float32)], - description="desc", - aggregations=[ - Aggregation( - column="dummy_field", - function="max", - time_window=timedelta(days=1), - ), - Aggregation( - column="dummy_field2", - function="count", - time_window=timedelta(days=24), - ), - ], - timestamp_field="event_timestamp", - mode="spark", - source=stream_source, - udf=simple_udf, - tags={}, - ) - - project = "project" - - # Register Feature View - test_registry.apply_feature_view(sfv, project) - - stream_feature_views = test_registry.list_stream_feature_views(project) - - # List Feature Views - assert len(stream_feature_views) == 1 - assert stream_feature_views[0] == sfv - - test_registry.delete_feature_view("test kafka stream feature view", project) - stream_feature_views = test_registry.list_stream_feature_views(project) - assert len(stream_feature_views) == 0 - - test_registry.teardown() - - # Will try to reload registry, which will fail because the file has been deleted - with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto(project=project) - - -@pytest.mark.parametrize( - "test_registry", - [lazy_fixture("local_registry")], -) -def test_modify_feature_views_success(test_registry): - # Create Feature Views - batch_source = FileSource( - file_format=ParquetFormat(), - path="file://feast/*", - timestamp_field="ts_col", - created_timestamp_column="timestamp", - ) - - request_source = RequestSource( - name="request_source", - schema=[Field(name="my_input_1", dtype=Int32)], - ) - - entity = Entity(name="fs1_my_entity_1", join_keys=["test"]) - - fv1 = FeatureView( - name="my_feature_view_1", - schema=[ - Field(name="test", dtype=Int64), - Field(name="fs1_my_feature_1", dtype=Int64), - ], - entities=[entity], - tags={"team": "matchmaking"}, - source=batch_source, - ttl=timedelta(minutes=5), - ) - - @on_demand_feature_view( - schema=[ - Field(name="odfv1_my_feature_1", dtype=String), - Field(name="odfv1_my_feature_2", dtype=Int32), - ], - sources=[request_source], - ) - def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: - data = pd.DataFrame() - data["odfv1_my_feature_1"] = feature_df["my_input_1"].astype("category") - data["odfv1_my_feature_2"] = feature_df["my_input_1"].astype("int32") - return data - - project = "project" - - # Register Feature Views - test_registry.apply_feature_view(odfv1, project) - test_registry.apply_feature_view(fv1, project) - - # Modify odfv by changing a single feature dtype - @on_demand_feature_view( - schema=[ - Field(name="odfv1_my_feature_1", dtype=Float32), - Field(name="odfv1_my_feature_2", dtype=Int32), - ], - sources=[request_source], - ) - def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: - data = pd.DataFrame() - data["odfv1_my_feature_1"] = feature_df["my_input_1"].astype("float") - data["odfv1_my_feature_2"] = feature_df["my_input_1"].astype("int32") - return data - - # Apply the modified odfv - test_registry.apply_feature_view(odfv1, project) - - # Check odfv - on_demand_feature_views = test_registry.list_on_demand_feature_views(project) - - assert ( - len(on_demand_feature_views) == 1 - and on_demand_feature_views[0].name == "odfv1" - and on_demand_feature_views[0].features[0].name == "odfv1_my_feature_1" - and on_demand_feature_views[0].features[0].dtype == Float32 - and on_demand_feature_views[0].features[1].name == "odfv1_my_feature_2" - and on_demand_feature_views[0].features[1].dtype == Int32 - ) - request_schema = on_demand_feature_views[0].get_request_data_schema() - assert ( - list(request_schema.keys())[0] == "my_input_1" - and list(request_schema.values())[0] == ValueType.INT32 - ) - - feature_view = test_registry.get_on_demand_feature_view("odfv1", project) - assert ( - feature_view.name == "odfv1" - and feature_view.features[0].name == "odfv1_my_feature_1" - and feature_view.features[0].dtype == Float32 - and feature_view.features[1].name == "odfv1_my_feature_2" - and feature_view.features[1].dtype == Int32 - ) - request_schema = feature_view.get_request_data_schema() - assert ( - list(request_schema.keys())[0] == "my_input_1" - and list(request_schema.values())[0] == ValueType.INT32 - ) - - # Make sure fv1 is untouched - feature_views = test_registry.list_feature_views(project) - - # List Feature Views - assert ( - len(feature_views) == 1 - and feature_views[0].name == "my_feature_view_1" - and feature_views[0].features[0].name == "fs1_my_feature_1" - and feature_views[0].features[0].dtype == Int64 - and feature_views[0].entities[0] == "fs1_my_entity_1" - ) - - feature_view = test_registry.get_feature_view("my_feature_view_1", project) - assert ( - feature_view.name == "my_feature_view_1" - and feature_view.features[0].name == "fs1_my_feature_1" - and feature_view.features[0].dtype == Int64 - and feature_view.entities[0] == "fs1_my_entity_1" - ) - - test_registry.teardown() - - # Will try to reload registry, which will fail because the file has been deleted - with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto(project=project) - - -@pytest.mark.parametrize( - "test_registry", - [lazy_fixture("local_registry")], -) -def test_apply_data_source(test_registry: Registry): - validate_registry_data_source_apply(test_registry) def test_commit(): diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py index 37e57558678..985c1661d5a 100644 --- a/sdk/python/tests/utils/e2e_test_validation.py +++ b/sdk/python/tests/utils/e2e_test_validation.py @@ -6,16 +6,10 @@ from typing import Dict, List, Optional, Union import pandas as pd -import pytest import yaml from pytz import utc -from feast import FeatureStore, FeatureView, FileSource, RepoConfig -from feast.data_format import ParquetFormat -from feast.entity import Entity -from feast.field import Field -from feast.infra.registry.registry import Registry -from feast.types import Array, Bytes, Int64, String +from feast import FeatureStore, FeatureView, RepoConfig from tests.integration.feature_repos.integration_test_repo_config import ( IntegrationTestRepoConfig, ) @@ -235,65 +229,3 @@ def make_feature_store_yaml( ), ] ) - - -def validate_registry_data_source_apply(test_registry: Registry): - # Create Feature Views - batch_source = FileSource( - name="test_source", - file_format=ParquetFormat(), - path="file://feast/*", - timestamp_field="ts_col", - created_timestamp_column="timestamp", - ) - - entity = Entity(name="fs1_my_entity_1", join_keys=["test"]) - - fv1 = FeatureView( - name="my_feature_view_1", - schema=[ - Field(name="test", dtype=Int64), - Field(name="fs1_my_feature_1", dtype=Int64), - Field(name="fs1_my_feature_2", dtype=String), - Field(name="fs1_my_feature_3", dtype=Array(String)), - Field(name="fs1_my_feature_4", dtype=Array(Bytes)), - ], - entities=[entity], - tags={"team": "matchmaking"}, - source=batch_source, - ttl=timedelta(minutes=5), - ) - - project = "project" - - # Register data source and feature view - test_registry.apply_data_source(batch_source, project, commit=False) - test_registry.apply_feature_view(fv1, project, commit=True) - - registry_feature_views = test_registry.list_feature_views(project) - registry_data_sources = test_registry.list_data_sources(project) - assert len(registry_feature_views) == 1 - assert len(registry_data_sources) == 1 - registry_feature_view = registry_feature_views[0] - assert registry_feature_view.batch_source == batch_source - registry_data_source = registry_data_sources[0] - assert registry_data_source == batch_source - - # Check that change to batch source propagates - batch_source.timestamp_field = "new_ts_col" - test_registry.apply_data_source(batch_source, project, commit=False) - test_registry.apply_feature_view(fv1, project, commit=True) - registry_feature_views = test_registry.list_feature_views(project) - registry_data_sources = test_registry.list_data_sources(project) - assert len(registry_feature_views) == 1 - assert len(registry_data_sources) == 1 - registry_feature_view = registry_feature_views[0] - assert registry_feature_view.batch_source == batch_source - registry_batch_source = test_registry.list_data_sources(project)[0] - assert registry_batch_source == batch_source - - test_registry.teardown() - - # Will try to reload registry, which will fail because the file has been deleted - with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto(project=project) From fffc36609b9c9165164113f325ce9eee41859734 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Fri, 24 May 2024 06:54:37 -0400 Subject: [PATCH 70/73] chore: Adjusting the file version validation for minor releases (#4221) * adjusting the validation for branch updates Signed-off-by: Francisco Javier Arceo * updated to get parsed version Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo --- infra/scripts/release/bump_file_versions.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/infra/scripts/release/bump_file_versions.py b/infra/scripts/release/bump_file_versions.py index e17463c2c7b..c913e9f43f7 100644 --- a/infra/scripts/release/bump_file_versions.py +++ b/infra/scripts/release/bump_file_versions.py @@ -1,5 +1,6 @@ # This script will bump the versions found in files (charts, pom.xml) during the Feast release process. +import re import pathlib import sys @@ -45,7 +46,9 @@ def main() -> None: with open(repo_root.joinpath(file_path), "r") as f: file_contents = f.readlines() for line in lines: - file_contents[int(line) - 1] = file_contents[int(line) - 1].replace(current_version, new_version) + # note we validate the version above already + current_parsed_version = _get_semantic_version(file_contents[int(line) - 1]) + file_contents[int(line) - 1] = file_contents[int(line) - 1].replace(current_parsed_version, new_version) with open(repo_root.joinpath(file_path), "w") as f: f.write(''.join(file_contents)) @@ -73,11 +76,19 @@ def validate_files_to_bump(current_version, files_to_bump, repo_root): with open(repo_root.joinpath(file_path), "r") as f: file_contents = f.readlines() for line in lines: - assert current_version in file_contents[int(line) - 1], ( + new_version = _get_semantic_version(file_contents[int(line) - 1]) + current_major_minor_version = '.'.join(current_version.split(".")[0:1]) + assert current_version in new_version or current_major_minor_version in new_version, ( f"File `{file_path}` line `{line}` didn't contain version {current_version}. " f"Contents: {file_contents[int(line) - 1]}" ) +def _get_semantic_version(input_string: str) -> str: + semver_pattern = r'\bv?(\d+\.\d+\.\d+)\b' + match = re.search(semver_pattern, input_string) + return match.group(1) + + if __name__ == "__main__": main() From 7032bd22795dcecf683e7d96c7dd95c5d3c90435 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Fri, 24 May 2024 17:24:40 -0400 Subject: [PATCH 71/73] chore: Revert 371 bump (#4227) --- infra/charts/feast-feature-server/Chart.yaml | 2 +- infra/charts/feast-feature-server/README.md | 2 +- infra/charts/feast-feature-server/values.yaml | 2 +- infra/charts/feast/Chart.yaml | 2 +- infra/charts/feast/README.md | 6 +++--- infra/charts/feast/charts/feature-server/Chart.yaml | 4 ++-- infra/charts/feast/charts/feature-server/README.md | 4 ++-- infra/charts/feast/charts/feature-server/values.yaml | 2 +- infra/charts/feast/charts/transformation-service/Chart.yaml | 4 ++-- infra/charts/feast/charts/transformation-service/README.md | 4 ++-- .../charts/feast/charts/transformation-service/values.yaml | 2 +- infra/charts/feast/requirements.yaml | 4 ++-- infra/feast-operator/Makefile | 2 +- infra/feast-operator/config/manager/kustomization.yaml | 2 +- java/pom.xml | 2 +- ui/package.json | 2 +- 16 files changed, 23 insertions(+), 23 deletions(-) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index 8d564f3b420..bd4bc606a70 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.37.1 +version: 0.37.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index a9c609c3d62..ceb8637b45b 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.37.1` +Current chart version is `0.37.0` ## Installation diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index df5241ebb2d..0de071ef3db 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.37.1 + tag: 0.37.0 imagePullSecrets: [] nameOverride: "" diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 21c00e4483b..26a00d80c63 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.37.1 +version: 0.37.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 8ab816dc707..47959047dc2 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.37.1` +Feature store for machine learning Current chart version is `0.37.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.37.1 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.37.1 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.37.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.37.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index 08563c6e069..f2f8c748dc4 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.37.1 -appVersion: v0.37.1 +version: 0.37.0 +appVersion: v0.37.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 3018b31c96c..531adce92da 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.37.1](https://img.shields.io/badge/Version-0.37.1-informational?style=flat-square) ![AppVersion: v0.37.1](https://img.shields.io/badge/AppVersion-v0.37.1-informational?style=flat-square) +![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![AppVersion: v0.37.0](https://img.shields.io/badge/AppVersion-v0.37.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.37.1"` | Image tag | +| image.tag | string | `"0.37.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 1d86059c1fd..1bf1a03f4a7 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.37.1 + tag: 0.37.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index bad9befa0bf..056e00473fb 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.37.1 -appVersion: v0.37.1 +version: 0.37.0 +appVersion: v0.37.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index f912b4c02f7..4b11861d539 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.37.1](https://img.shields.io/badge/Version-0.37.1-informational?style=flat-square) ![AppVersion: v0.37.1](https://img.shields.io/badge/AppVersion-v0.37.1-informational?style=flat-square) +![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![AppVersion: v0.37.0](https://img.shields.io/badge/AppVersion-v0.37.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.37.1"` | Image tag | +| image.tag | string | `"0.37.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index df5ea64c347..a04dfeb3e04 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.37.1 + tag: 0.37.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 80b8c861326..9a2d3e0e807 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.37.1 + version: 0.37.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.37.1 + version: 0.37.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 84e69d6eaca..1388778f9fe 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.37.1 +VERSION ?= 0.37.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") diff --git a/infra/feast-operator/config/manager/kustomization.yaml b/infra/feast-operator/config/manager/kustomization.yaml index be181e33472..226b87118d2 100644 --- a/infra/feast-operator/config/manager/kustomization.yaml +++ b/infra/feast-operator/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: feastdev/feast-operator - newTag: 0.37.1 + newTag: 0.37.0 diff --git a/java/pom.xml b/java/pom.xml index 8ba8ed4ac53..2d7e2c3e7d2 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.37.1 + 0.37.0 https://github.com/feast-dev/feast UTF-8 diff --git a/ui/package.json b/ui/package.json index ea69e571fb5..9209d7b03c7 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.37.1", + "version": "0.37.0", "private": false, "files": [ "dist" From d52426a9a3c1643d1f3eb1c71599d2c2ed795190 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Fri, 24 May 2024 17:35:14 -0400 Subject: [PATCH 72/73] chore: Add tokoko to OWNERS (#4228) --- OWNERS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/OWNERS b/OWNERS index 52c5e436d30..1072fc2187b 100644 --- a/OWNERS +++ b/OWNERS @@ -21,6 +21,7 @@ approvers: - haoxuai - jeremyary - shuchu + - tokoko reviewers: - woop @@ -43,4 +44,4 @@ reviewers: - haoxuai - jeremyary - shuchu - \ No newline at end of file + - tokoko From 65de0d291ca98466535c1d9434b9ea35e97ff230 Mon Sep 17 00:00:00 2001 From: feast-ci-bot Date: Fri, 24 May 2024 23:31:49 +0000 Subject: [PATCH 73/73] chore(release): release 0.38.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # [0.38.0](https://github.com/feast-dev/feast/compare/v0.37.0...v0.38.0) (2024-05-24) ### Bug Fixes * Add vector database doc ([#4165](https://github.com/feast-dev/feast/issues/4165)) ([37f36b6](https://github.com/feast-dev/feast/commit/37f36b681bde0c1ae83303803c89d3ed0b2ac8a9)) * Change checkout action back to v3 from v5 which isn't released yet ([#4147](https://github.com/feast-dev/feast/issues/4147)) ([9523fff](https://github.com/feast-dev/feast/commit/9523fff2dda2e0d53bffa7f5c0d6f2f69f6b8c02)) * Change numpy version <1.25 dependency to <2 in setup.py ([#4085](https://github.com/feast-dev/feast/issues/4085)) ([2ba71ff](https://github.com/feast-dev/feast/commit/2ba71fff5f76ed05066e94f3b11d08bc30b54b39)), closes [#4084](https://github.com/feast-dev/feast/issues/4084) * Changed the code the way mysql container is initialized. ([#4140](https://github.com/feast-dev/feast/issues/4140)) ([8b5698f](https://github.com/feast-dev/feast/commit/8b5698fefa965fc08fdb5e07d739d0ca276a3522)), closes [#4126](https://github.com/feast-dev/feast/issues/4126) * Correct nightly install command, move all installs to uv ([#4164](https://github.com/feast-dev/feast/issues/4164)) ([c86d594](https://github.com/feast-dev/feast/commit/c86d594613b0fb1425451def4fc1d7a7496eea92)) * Default value is not set in Redis connection string using environment variable ([#4136](https://github.com/feast-dev/feast/issues/4136)) ([95acfb4](https://github.com/feast-dev/feast/commit/95acfb4cefc10f96f8ed61f148e24b238d400a68)), closes [#3669](https://github.com/feast-dev/feast/issues/3669) * Get container host addresses from testcontainers (java) ([#4125](https://github.com/feast-dev/feast/issues/4125)) ([9184dde](https://github.com/feast-dev/feast/commit/9184dde1fcd57de5765c850615eb5e70cbafe70f)) * Get rid of empty string `name_alias` during feature view projection deserialization ([#4116](https://github.com/feast-dev/feast/issues/4116)) ([65056ce](https://github.com/feast-dev/feast/commit/65056cea6c4537834a1c40be2ad37e1659310a47)) * Helm chart `feast-feature-server`, improve Service template name ([#4161](https://github.com/feast-dev/feast/issues/4161)) ([dedc164](https://github.com/feast-dev/feast/commit/dedc1645ef1f38aa9b50a0cf55e4bc23ec60d5ad)) * Improve the code related to on-demand-featureview. ([#4203](https://github.com/feast-dev/feast/issues/4203)) ([d91d7e0](https://github.com/feast-dev/feast/commit/d91d7e0da69d15c7aa14e736b608ed9f5ece3504)) * Integration tests for async sdk method ([#4201](https://github.com/feast-dev/feast/issues/4201)) ([08c44ae](https://github.com/feast-dev/feast/commit/08c44ae35a4a91228f9f78c7323b4b7a73ef33aa)) * Make sure schema is used when calling `get_table_query_string` method for Snowflake datasource ([#4131](https://github.com/feast-dev/feast/issues/4131)) ([c1579c7](https://github.com/feast-dev/feast/commit/c1579c77324cebb0514422235956812403316c80)) * Make sure schema is used when generating `from_expression` for Snowflake ([#4177](https://github.com/feast-dev/feast/issues/4177)) ([5051da7](https://github.com/feast-dev/feast/commit/5051da75de81deed19b25fbc2826d504a8ebdc8b)) * Pass native input values to `get_online_features` from feature server ([#4117](https://github.com/feast-dev/feast/issues/4117)) ([60756cb](https://github.com/feast-dev/feast/commit/60756cb4637a7961b6caffef3242e2886e77f78a)) * Pass region to S3 client only if set (Java) ([#4151](https://github.com/feast-dev/feast/issues/4151)) ([b8087f7](https://github.com/feast-dev/feast/commit/b8087f7a181977e0e4d3bd29c857d8e137af1de2)) * Pgvector patch ([#4108](https://github.com/feast-dev/feast/issues/4108)) ([ad45bb4](https://github.com/feast-dev/feast/commit/ad45bb4ac2dd83b530adda6196f85d46decaf98e)) * Update doc ([#4153](https://github.com/feast-dev/feast/issues/4153)) ([e873636](https://github.com/feast-dev/feast/commit/e873636b4a5f3a05666f9284c31e488f27257ed0)) * Update master-only benchmark bucket name due to credential update ([#4183](https://github.com/feast-dev/feast/issues/4183)) ([e88f1e3](https://github.com/feast-dev/feast/commit/e88f1e39778300fb443f1db230fe9589b74d9ed6)) * Updating the instructions for quickstart guide. ([#4120](https://github.com/feast-dev/feast/issues/4120)) ([0c30e96](https://github.com/feast-dev/feast/commit/0c30e96da144babe725a3f168c05d2fbeca65507)) * Upgrading the test container so that local tests works with updated d… ([#4155](https://github.com/feast-dev/feast/issues/4155)) ([93ddb11](https://github.com/feast-dev/feast/commit/93ddb11bf5a182cea44435147e39f40b30a69db7)) ### Features * Add a Kubernetes Operator for the Feast Feature Server ([#4145](https://github.com/feast-dev/feast/issues/4145)) ([4a696dc](https://github.com/feast-dev/feast/commit/4a696dc4b0fd96d51872a5e629ab5f3ca785d708)) * Add delta format to `FileSource`, add support for it in ibis/duckdb ([#4123](https://github.com/feast-dev/feast/issues/4123)) ([2b6f1d0](https://github.com/feast-dev/feast/commit/2b6f1d0945e8dbf13d01e045f87c5e58546b4af6)) * Add materialization support to ibis/duckdb ([#4173](https://github.com/feast-dev/feast/issues/4173)) ([369ca98](https://github.com/feast-dev/feast/commit/369ca98d88a5cb3c67b2363232b7c2eddfc4f333)) * Add optional private key params to Snowflake config ([#4205](https://github.com/feast-dev/feast/issues/4205)) ([20f5419](https://github.com/feast-dev/feast/commit/20f5419d30c32b533e91043a9690007a84000512)) * Add s3 remote storage export for duckdb ([#4195](https://github.com/feast-dev/feast/issues/4195)) ([6a04c48](https://github.com/feast-dev/feast/commit/6a04c48b4b84fb9905df638e5c4041c12532b053)) * Adding DatastoreOnlineStore 'database' argument. ([#4180](https://github.com/feast-dev/feast/issues/4180)) ([e739745](https://github.com/feast-dev/feast/commit/e739745482fed1b9c2d7b788ebb088041118c642)) * Adding get_online_features_async to feature store sdk ([#4172](https://github.com/feast-dev/feast/issues/4172)) ([311efc5](https://github.com/feast-dev/feast/commit/311efc5005b24d1fc9bc389ee7579e102e2cd4ea)) * Adding support for dictionary writes to online store ([#4156](https://github.com/feast-dev/feast/issues/4156)) ([abfac01](https://github.com/feast-dev/feast/commit/abfac011ad1f94caef001539591d03b1552f65e5)) * Elasticsearch vector database ([#4188](https://github.com/feast-dev/feast/issues/4188)) ([bf99640](https://github.com/feast-dev/feast/commit/bf99640c0bcfd9ee7c1e66d24cb791bfa0e5ac4a)) * Enable other distance metrics for Vector DB and Update docs ([#4170](https://github.com/feast-dev/feast/issues/4170)) ([ba9f4ef](https://github.com/feast-dev/feast/commit/ba9f4efd5eccd0548a39521a145c6573ac90c221)) * Feast/IKV datetime edgecase errors ([#4211](https://github.com/feast-dev/feast/issues/4211)) ([bdae562](https://github.com/feast-dev/feast/commit/bdae562ea4582d8e47763736b639c70e56d79b2d)) * Feast/IKV documenation language changes ([#4149](https://github.com/feast-dev/feast/issues/4149)) ([690a621](https://github.com/feast-dev/feast/commit/690a6212e9f2b14fc4bf65513e5d30e70e229d0a)) * Feast/IKV online store contrib plugin integration ([#4068](https://github.com/feast-dev/feast/issues/4068)) ([f2b4eb9](https://github.com/feast-dev/feast/commit/f2b4eb94add8f86afa4e168236e8fcd11968510e)) * Feast/IKV online store documentation ([#4146](https://github.com/feast-dev/feast/issues/4146)) ([73601e4](https://github.com/feast-dev/feast/commit/73601e45e2fc57dc889644b1d28115b3c94bd8ea)) * Feast/IKV upgrade client version ([#4200](https://github.com/feast-dev/feast/issues/4200)) ([0e42150](https://github.com/feast-dev/feast/commit/0e4215060f97b7629015ab65ac526dfef0a1f7d4)) * Incorporate substrait ODFVs into ibis-based offline store queries ([#4102](https://github.com/feast-dev/feast/issues/4102)) ([c3a102f](https://github.com/feast-dev/feast/commit/c3a102f1b1941c8681ec876b54d7d16a32862925)) * Isolate input-dependent calculations in `get_online_features` ([#4041](https://github.com/feast-dev/feast/issues/4041)) ([2a6edea](https://github.com/feast-dev/feast/commit/2a6edeae42a2ebba7d9fc69af917bdc41ae6ecb0)) * Make arrow primary interchange for online ODFV execution ([#4143](https://github.com/feast-dev/feast/issues/4143)) ([3fdb716](https://github.com/feast-dev/feast/commit/3fdb71631fbb1b9cfb8d1cad69dbc2d2d50cea0d)) * Move data source validation entrypoint to offline store ([#4197](https://github.com/feast-dev/feast/issues/4197)) ([a17725d](https://github.com/feast-dev/feast/commit/a17725daec9e7355591e7ff2bc57202d5fa3f0c1)) * Upgrading python version to 3.11, adding support for 3.11 as well. ([#4159](https://github.com/feast-dev/feast/issues/4159)) ([4b1634f](https://github.com/feast-dev/feast/commit/4b1634f4da7ba47a29dfd4a0d573dfe515a8863d)), closes [#4152](https://github.com/feast-dev/feast/issues/4152) [#4114](https://github.com/feast-dev/feast/issues/4114) ### Reverts * Reverts "fix: Using version args to install the correct feast version" ([#4112](https://github.com/feast-dev/feast/issues/4112)) ([b66baa4](https://github.com/feast-dev/feast/commit/b66baa46f48c72f4704bfe3980a8df49e1a06507)), closes [#3953](https://github.com/feast-dev/feast/issues/3953) --- CHANGELOG.md | 55 +++++++++++++++++++ infra/charts/feast-feature-server/Chart.yaml | 2 +- infra/charts/feast-feature-server/README.md | 5 +- infra/charts/feast-feature-server/values.yaml | 2 +- infra/charts/feast/Chart.yaml | 2 +- infra/charts/feast/README.md | 6 +- .../feast/charts/feature-server/Chart.yaml | 4 +- .../feast/charts/feature-server/README.md | 4 +- .../feast/charts/feature-server/values.yaml | 2 +- .../charts/transformation-service/Chart.yaml | 4 +- .../charts/transformation-service/README.md | 4 +- .../charts/transformation-service/values.yaml | 2 +- infra/charts/feast/requirements.yaml | 4 +- java/pom.xml | 2 +- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 8 +-- ui/package.json | 2 +- 17 files changed, 82 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19dc5d86d7c..fc569e5fbba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,60 @@ # Changelog +# [0.38.0](https://github.com/feast-dev/feast/compare/v0.37.0...v0.38.0) (2024-05-24) + + +### Bug Fixes + +* Add vector database doc ([#4165](https://github.com/feast-dev/feast/issues/4165)) ([37f36b6](https://github.com/feast-dev/feast/commit/37f36b681bde0c1ae83303803c89d3ed0b2ac8a9)) +* Change checkout action back to v3 from v5 which isn't released yet ([#4147](https://github.com/feast-dev/feast/issues/4147)) ([9523fff](https://github.com/feast-dev/feast/commit/9523fff2dda2e0d53bffa7f5c0d6f2f69f6b8c02)) +* Change numpy version <1.25 dependency to <2 in setup.py ([#4085](https://github.com/feast-dev/feast/issues/4085)) ([2ba71ff](https://github.com/feast-dev/feast/commit/2ba71fff5f76ed05066e94f3b11d08bc30b54b39)), closes [#4084](https://github.com/feast-dev/feast/issues/4084) +* Changed the code the way mysql container is initialized. ([#4140](https://github.com/feast-dev/feast/issues/4140)) ([8b5698f](https://github.com/feast-dev/feast/commit/8b5698fefa965fc08fdb5e07d739d0ca276a3522)), closes [#4126](https://github.com/feast-dev/feast/issues/4126) +* Correct nightly install command, move all installs to uv ([#4164](https://github.com/feast-dev/feast/issues/4164)) ([c86d594](https://github.com/feast-dev/feast/commit/c86d594613b0fb1425451def4fc1d7a7496eea92)) +* Default value is not set in Redis connection string using environment variable ([#4136](https://github.com/feast-dev/feast/issues/4136)) ([95acfb4](https://github.com/feast-dev/feast/commit/95acfb4cefc10f96f8ed61f148e24b238d400a68)), closes [#3669](https://github.com/feast-dev/feast/issues/3669) +* Get container host addresses from testcontainers (java) ([#4125](https://github.com/feast-dev/feast/issues/4125)) ([9184dde](https://github.com/feast-dev/feast/commit/9184dde1fcd57de5765c850615eb5e70cbafe70f)) +* Get rid of empty string `name_alias` during feature view projection deserialization ([#4116](https://github.com/feast-dev/feast/issues/4116)) ([65056ce](https://github.com/feast-dev/feast/commit/65056cea6c4537834a1c40be2ad37e1659310a47)) +* Helm chart `feast-feature-server`, improve Service template name ([#4161](https://github.com/feast-dev/feast/issues/4161)) ([dedc164](https://github.com/feast-dev/feast/commit/dedc1645ef1f38aa9b50a0cf55e4bc23ec60d5ad)) +* Improve the code related to on-demand-featureview. ([#4203](https://github.com/feast-dev/feast/issues/4203)) ([d91d7e0](https://github.com/feast-dev/feast/commit/d91d7e0da69d15c7aa14e736b608ed9f5ece3504)) +* Integration tests for async sdk method ([#4201](https://github.com/feast-dev/feast/issues/4201)) ([08c44ae](https://github.com/feast-dev/feast/commit/08c44ae35a4a91228f9f78c7323b4b7a73ef33aa)) +* Make sure schema is used when calling `get_table_query_string` method for Snowflake datasource ([#4131](https://github.com/feast-dev/feast/issues/4131)) ([c1579c7](https://github.com/feast-dev/feast/commit/c1579c77324cebb0514422235956812403316c80)) +* Make sure schema is used when generating `from_expression` for Snowflake ([#4177](https://github.com/feast-dev/feast/issues/4177)) ([5051da7](https://github.com/feast-dev/feast/commit/5051da75de81deed19b25fbc2826d504a8ebdc8b)) +* Pass native input values to `get_online_features` from feature server ([#4117](https://github.com/feast-dev/feast/issues/4117)) ([60756cb](https://github.com/feast-dev/feast/commit/60756cb4637a7961b6caffef3242e2886e77f78a)) +* Pass region to S3 client only if set (Java) ([#4151](https://github.com/feast-dev/feast/issues/4151)) ([b8087f7](https://github.com/feast-dev/feast/commit/b8087f7a181977e0e4d3bd29c857d8e137af1de2)) +* Pgvector patch ([#4108](https://github.com/feast-dev/feast/issues/4108)) ([ad45bb4](https://github.com/feast-dev/feast/commit/ad45bb4ac2dd83b530adda6196f85d46decaf98e)) +* Update doc ([#4153](https://github.com/feast-dev/feast/issues/4153)) ([e873636](https://github.com/feast-dev/feast/commit/e873636b4a5f3a05666f9284c31e488f27257ed0)) +* Update master-only benchmark bucket name due to credential update ([#4183](https://github.com/feast-dev/feast/issues/4183)) ([e88f1e3](https://github.com/feast-dev/feast/commit/e88f1e39778300fb443f1db230fe9589b74d9ed6)) +* Updating the instructions for quickstart guide. ([#4120](https://github.com/feast-dev/feast/issues/4120)) ([0c30e96](https://github.com/feast-dev/feast/commit/0c30e96da144babe725a3f168c05d2fbeca65507)) +* Upgrading the test container so that local tests works with updated d… ([#4155](https://github.com/feast-dev/feast/issues/4155)) ([93ddb11](https://github.com/feast-dev/feast/commit/93ddb11bf5a182cea44435147e39f40b30a69db7)) + + +### Features + +* Add a Kubernetes Operator for the Feast Feature Server ([#4145](https://github.com/feast-dev/feast/issues/4145)) ([4a696dc](https://github.com/feast-dev/feast/commit/4a696dc4b0fd96d51872a5e629ab5f3ca785d708)) +* Add delta format to `FileSource`, add support for it in ibis/duckdb ([#4123](https://github.com/feast-dev/feast/issues/4123)) ([2b6f1d0](https://github.com/feast-dev/feast/commit/2b6f1d0945e8dbf13d01e045f87c5e58546b4af6)) +* Add materialization support to ibis/duckdb ([#4173](https://github.com/feast-dev/feast/issues/4173)) ([369ca98](https://github.com/feast-dev/feast/commit/369ca98d88a5cb3c67b2363232b7c2eddfc4f333)) +* Add optional private key params to Snowflake config ([#4205](https://github.com/feast-dev/feast/issues/4205)) ([20f5419](https://github.com/feast-dev/feast/commit/20f5419d30c32b533e91043a9690007a84000512)) +* Add s3 remote storage export for duckdb ([#4195](https://github.com/feast-dev/feast/issues/4195)) ([6a04c48](https://github.com/feast-dev/feast/commit/6a04c48b4b84fb9905df638e5c4041c12532b053)) +* Adding DatastoreOnlineStore 'database' argument. ([#4180](https://github.com/feast-dev/feast/issues/4180)) ([e739745](https://github.com/feast-dev/feast/commit/e739745482fed1b9c2d7b788ebb088041118c642)) +* Adding get_online_features_async to feature store sdk ([#4172](https://github.com/feast-dev/feast/issues/4172)) ([311efc5](https://github.com/feast-dev/feast/commit/311efc5005b24d1fc9bc389ee7579e102e2cd4ea)) +* Adding support for dictionary writes to online store ([#4156](https://github.com/feast-dev/feast/issues/4156)) ([abfac01](https://github.com/feast-dev/feast/commit/abfac011ad1f94caef001539591d03b1552f65e5)) +* Elasticsearch vector database ([#4188](https://github.com/feast-dev/feast/issues/4188)) ([bf99640](https://github.com/feast-dev/feast/commit/bf99640c0bcfd9ee7c1e66d24cb791bfa0e5ac4a)) +* Enable other distance metrics for Vector DB and Update docs ([#4170](https://github.com/feast-dev/feast/issues/4170)) ([ba9f4ef](https://github.com/feast-dev/feast/commit/ba9f4efd5eccd0548a39521a145c6573ac90c221)) +* Feast/IKV datetime edgecase errors ([#4211](https://github.com/feast-dev/feast/issues/4211)) ([bdae562](https://github.com/feast-dev/feast/commit/bdae562ea4582d8e47763736b639c70e56d79b2d)) +* Feast/IKV documenation language changes ([#4149](https://github.com/feast-dev/feast/issues/4149)) ([690a621](https://github.com/feast-dev/feast/commit/690a6212e9f2b14fc4bf65513e5d30e70e229d0a)) +* Feast/IKV online store contrib plugin integration ([#4068](https://github.com/feast-dev/feast/issues/4068)) ([f2b4eb9](https://github.com/feast-dev/feast/commit/f2b4eb94add8f86afa4e168236e8fcd11968510e)) +* Feast/IKV online store documentation ([#4146](https://github.com/feast-dev/feast/issues/4146)) ([73601e4](https://github.com/feast-dev/feast/commit/73601e45e2fc57dc889644b1d28115b3c94bd8ea)) +* Feast/IKV upgrade client version ([#4200](https://github.com/feast-dev/feast/issues/4200)) ([0e42150](https://github.com/feast-dev/feast/commit/0e4215060f97b7629015ab65ac526dfef0a1f7d4)) +* Incorporate substrait ODFVs into ibis-based offline store queries ([#4102](https://github.com/feast-dev/feast/issues/4102)) ([c3a102f](https://github.com/feast-dev/feast/commit/c3a102f1b1941c8681ec876b54d7d16a32862925)) +* Isolate input-dependent calculations in `get_online_features` ([#4041](https://github.com/feast-dev/feast/issues/4041)) ([2a6edea](https://github.com/feast-dev/feast/commit/2a6edeae42a2ebba7d9fc69af917bdc41ae6ecb0)) +* Make arrow primary interchange for online ODFV execution ([#4143](https://github.com/feast-dev/feast/issues/4143)) ([3fdb716](https://github.com/feast-dev/feast/commit/3fdb71631fbb1b9cfb8d1cad69dbc2d2d50cea0d)) +* Move data source validation entrypoint to offline store ([#4197](https://github.com/feast-dev/feast/issues/4197)) ([a17725d](https://github.com/feast-dev/feast/commit/a17725daec9e7355591e7ff2bc57202d5fa3f0c1)) +* Upgrading python version to 3.11, adding support for 3.11 as well. ([#4159](https://github.com/feast-dev/feast/issues/4159)) ([4b1634f](https://github.com/feast-dev/feast/commit/4b1634f4da7ba47a29dfd4a0d573dfe515a8863d)), closes [#4152](https://github.com/feast-dev/feast/issues/4152) [#4114](https://github.com/feast-dev/feast/issues/4114) + + +### Reverts + +* Reverts "fix: Using version args to install the correct feast version" ([#4112](https://github.com/feast-dev/feast/issues/4112)) ([b66baa4](https://github.com/feast-dev/feast/commit/b66baa46f48c72f4704bfe3980a8df49e1a06507)), closes [#3953](https://github.com/feast-dev/feast/issues/3953) + ## [0.37.1](https://github.com/feast-dev/feast/compare/v0.37.0...v0.37.1) (2024-04-17) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index bd4bc606a70..fca8f0c98c5 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.37.0 +version: 0.38.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index ceb8637b45b..457aeff2452 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.37.0` +Current chart version is `0.38.0` ## Installation @@ -17,7 +17,6 @@ A base64 encoded version of the `feature_store.yaml` file is needed. Helm instal ``` helm install feast-feature-server feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml) ``` -> Alternatively, deploy this helm chart with a [Kubernetes Operator](/infra/feast-operator). ## Tutorial See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-demo) for a sample tutorial on testing this helm chart with a demo feature repository and a local Redis instance. @@ -31,7 +30,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.37.1"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.38.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 0de071ef3db..168164ffe9d 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.37.0 + tag: 0.38.0 imagePullSecrets: [] nameOverride: "" diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 26a00d80c63..109b6713933 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.37.0 +version: 0.38.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 47959047dc2..70296aa130c 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.37.0` +Feature store for machine learning Current chart version is `0.38.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.37.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.37.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.38.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.38.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index f2f8c748dc4..3df922d7994 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.37.0 -appVersion: v0.37.0 +version: 0.38.0 +appVersion: v0.38.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 531adce92da..8266efeda3d 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![AppVersion: v0.37.0](https://img.shields.io/badge/AppVersion-v0.37.0-informational?style=flat-square) +![Version: 0.38.0](https://img.shields.io/badge/Version-0.38.0-informational?style=flat-square) ![AppVersion: v0.38.0](https://img.shields.io/badge/AppVersion-v0.38.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.37.0"` | Image tag | +| image.tag | string | `"0.38.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 1bf1a03f4a7..fac64c18c7b 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.37.0 + tag: 0.38.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 056e00473fb..91f0781f523 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.37.0 -appVersion: v0.37.0 +version: 0.38.0 +appVersion: v0.38.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 4b11861d539..7b33e4b4a13 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![AppVersion: v0.37.0](https://img.shields.io/badge/AppVersion-v0.37.0-informational?style=flat-square) +![Version: 0.38.0](https://img.shields.io/badge/Version-0.38.0-informational?style=flat-square) ![AppVersion: v0.38.0](https://img.shields.io/badge/AppVersion-v0.38.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.37.0"` | Image tag | +| image.tag | string | `"0.38.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index a04dfeb3e04..8c116cf7783 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.37.0 + tag: 0.38.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 9a2d3e0e807..d9c5f747b8a 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.37.0 + version: 0.38.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.37.0 + version: 0.38.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/java/pom.xml b/java/pom.xml index 2d7e2c3e7d2..6aabb87d0cc 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.37.0 + 0.38.0 https://github.com/feast-dev/feast UTF-8 diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 61c89f648f2..d4b5decaac1 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.37.1", + "@feast-dev/feast-ui": "0.38.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 91197e5219d..cb1e3154049 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1451,10 +1451,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.37.1": - version "0.37.1" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.37.1.tgz#b84618d1fd2e1dbc463ab2889964006b555d9ec4" - integrity sha512-xhHK3hWvW58ukB+kx04ut+7OIT+zuITw6eYKjuJmjzAZ2S8uVcqDso4T9Ma88qX+qhn4NWzNBUyM2Gz1xOhzKQ== +"@feast-dev/feast-ui@0.38.0": + version "0.38.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.38.0.tgz#3a2b8325b15a1e789741523bd5113b54a80b4325" + integrity sha512-i2F4yMwbaWOOPE+FOyDxrqAsb1GETDUsZ/AYJQJiQYyWgXtVFBZpShrJcOQkOwBvV5eX/2jtj9o7SaFQpUcM8A== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" diff --git a/ui/package.json b/ui/package.json index 9209d7b03c7..ec00624a823 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.37.0", + "version": "0.38.0", "private": false, "files": [ "dist"