From 21e47d2ce62378241a5eef86798c73e9c469c656 Mon Sep 17 00:00:00 2001 From: David Y Liu Date: Fri, 11 Jun 2021 17:38:25 -0700 Subject: [PATCH 01/43] test Signed-off-by: David Y Liu Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- README.md | 1 + docs/quickstart.md | 3 +- sdk/python/feast/errors.py | 7 + sdk/python/feast/feature_store.py | 56 ++++- sdk/python/feast/infra/gcp.py | 2 + sdk/python/feast/infra/local.py | 2 + .../feast/infra/offline_stores/bigquery.py | 13 +- sdk/python/feast/infra/offline_stores/file.py | 14 +- .../infra/offline_stores/offline_store.py | 1 + sdk/python/feast/infra/provider.py | 21 +- sdk/python/tests/foo_provider.py | 1 + sdk/python/tests/test_e2e_local.py | 1 + sdk/python/tests/test_historical_retrieval.py | 191 +++++++++++++++++- .../test_offline_online_store_consistency.py | 3 +- sdk/python/tests/test_online_retrieval.py | 34 +++- 15 files changed, 316 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index f64f91c27dd..00604e2f788 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ training_df = store.get_historical_features( 'driver_hourly_stats:acc_rate', 'driver_hourly_stats:avg_daily_trips' ], + full_feature_names=True ).to_df() print(training_df.head()) diff --git a/docs/quickstart.md b/docs/quickstart.md index 6b94b04a812..b98bfe8acac 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -234,7 +234,8 @@ feature_vector = store.get_online_features( 'driver_hourly_stats:acc_rate', 'driver_hourly_stats:avg_daily_trips' ], - entity_rows=[{"driver_id": 1001}] + entity_rows=[{"driver_id": 1001}], + full_feature_names=True ).to_dict() pprint(feature_vector) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index a0b3e2bf49a..0c026af31a0 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -71,6 +71,13 @@ def __init__(self, offline_store_name: str, data_source_name: str): ) +class FeatureNameCollisionError(Exception): + def __init__(self, feature_name_collisions: str): + super().__init__( + f"The following feature name(s) have collisions: {feature_name_collisions}. Set 'feature_names_only' argument in the data retrieval function to False to use the full feature name which is prefixed by the feature view name." + ) + + class FeastOnlineStoreUnsupportedDataSource(Exception): def __init__(self, online_store_name: str, data_source_name: str): super().__init__( diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 497ba3a368d..df22aab944b 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -24,7 +24,11 @@ from feast import utils from feast.entity import Entity -from feast.errors import FeastProviderLoginError, FeatureViewNotFoundException +from feast.errors import ( + FeastProviderLoginError, + FeatureNameCollisionError, + FeatureViewNotFoundException, +) from feast.feature_view import FeatureView from feast.inference import infer_entity_value_type_from_feature_views from feast.infra.provider import Provider, RetrievalJob, get_provider @@ -244,7 +248,10 @@ def apply( @log_exceptions_and_usage def get_historical_features( - self, entity_df: Union[pd.DataFrame, str], feature_refs: List[str], + self, + entity_df: Union[pd.DataFrame, str], + feature_refs: List[str], + full_feature_names: bool = False, ) -> RetrievalJob: """Enrich an entity dataframe with historical feature values for either training or batch scoring. @@ -266,6 +273,10 @@ def get_historical_features( SQL query. The query must be of a format supported by the configured offline store (e.g., BigQuery) feature_refs: A list of features that should be retrieved from the offline store. Feature references are of the format "feature_view:feature", e.g., "customer_fv:daily_transactions". + full_feature_names: By default, this value is set to False. This strips the feature view prefixes from the data + and returns only the feature name, changing them from the format "feature_view__feature" to "feature" + (e.g., "customer_fv__daily_transactions" changes to "daily_transactions"). Set the value to True for + the feature names to be prefixed by the feature view name in the format "feature_view__feature". Returns: RetrievalJob which can be used to materialize the results. @@ -278,12 +289,12 @@ def get_historical_features( >>> fs = FeatureStore(config=RepoConfig(provider="gcp")) >>> retrieval_job = fs.get_historical_features( >>> entity_df="SELECT event_timestamp, order_id, customer_id from gcp_project.my_ds.customer_orders", - >>> feature_refs=["customer:age", "customer:avg_orders_1d", "customer:avg_orders_7d"] - >>> ) + >>> feature_refs=["customer:age", "customer:avg_orders_1d", "customer:avg_orders_7d"], + >>> full_feature_names=False + >>> ) >>> feature_data = retrieval_job.to_df() >>> model.fit(feature_data) # insert your modeling framework here. """ - all_feature_views = self._registry.list_feature_views(project=self.project) try: feature_views = _get_requested_feature_views( @@ -301,6 +312,7 @@ def get_historical_features( entity_df, self._registry, self.project, + full_feature_names, ) except FeastProviderLoginError as e: sys.exit(e) @@ -467,7 +479,10 @@ def tqdm_builder(length): @log_exceptions_and_usage def get_online_features( - self, feature_refs: List[str], entity_rows: List[Dict[str, Any]], + self, + feature_refs: List[str], + entity_rows: List[Dict[str, Any]], + full_feature_names: bool = False, ) -> OnlineResponse: """ Retrieves the latest online feature data. @@ -535,7 +550,7 @@ def get_online_features( project=self.project, allow_cache=True ) - grouped_refs = _group_refs(feature_refs, all_feature_views) + grouped_refs = _group_refs(feature_refs, all_feature_views, full_feature_names) for table, requested_features in grouped_refs: entity_keys = _get_table_entity_keys( table, union_of_entity_keys, entity_name_to_join_key_map @@ -552,13 +567,21 @@ def get_online_features( if feature_data is None: for feature_name in requested_features: - feature_ref = f"{table.name}__{feature_name}" + feature_ref = ( + f"{table.name}__{feature_name}" + if full_feature_names + else feature_name + ) result_row.statuses[ feature_ref ] = GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND else: for feature_name in feature_data: - feature_ref = f"{table.name}__{feature_name}" + feature_ref = ( + f"{table.name}__{feature_name}" + if full_feature_names + else feature_name + ) if feature_name in requested_features: result_row.fields[feature_ref].CopyFrom( feature_data[feature_name] @@ -587,7 +610,9 @@ def _entity_row_to_field_values( def _group_refs( - feature_refs: List[str], all_feature_views: List[FeatureView] + feature_refs: List[str], + all_feature_views: List[FeatureView], + full_feature_names: bool = False, ) -> List[Tuple[FeatureView, List[str]]]: """ Get list of feature views and corresponding feature names based on feature references""" @@ -597,12 +622,23 @@ def _group_refs( # view name to feature names views_features = defaultdict(list) + feature_set = set() + feature_collision_set = set() + for ref in feature_refs: view_name, feat_name = ref.split(":") + if feat_name in feature_set: + feature_collision_set.add(feat_name) + else: + feature_set.add(feat_name) if view_name not in view_index: raise FeatureViewNotFoundException(view_name) views_features[view_name].append(feat_name) + if not full_feature_names and len(feature_collision_set) > 0: + err = ", ".join(x for x in feature_collision_set) + raise FeatureNameCollisionError(err) + result = [] for view_name, feature_names in views_features.items(): result.append((view_index[view_name], feature_names)) diff --git a/sdk/python/feast/infra/gcp.py b/sdk/python/feast/infra/gcp.py index f33b501d62e..9e307d761b4 100644 --- a/sdk/python/feast/infra/gcp.py +++ b/sdk/python/feast/infra/gcp.py @@ -128,6 +128,7 @@ def get_historical_features( entity_df: Union[pandas.DataFrame, str], registry: Registry, project: str, + full_feature_names: bool = False, ) -> RetrievalJob: job = self.offline_store.get_historical_features( config=config, @@ -136,5 +137,6 @@ def get_historical_features( entity_df=entity_df, registry=registry, project=project, + full_feature_names=full_feature_names, ) return job diff --git a/sdk/python/feast/infra/local.py b/sdk/python/feast/infra/local.py index a76f49b2c4d..23c813e6083 100644 --- a/sdk/python/feast/infra/local.py +++ b/sdk/python/feast/infra/local.py @@ -127,6 +127,7 @@ def get_historical_features( entity_df: Union[pd.DataFrame, str], registry: Registry, project: str, + full_feature_names: bool = False, ) -> RetrievalJob: return self.offline_store.get_historical_features( config=config, @@ -135,6 +136,7 @@ def get_historical_features( entity_df=entity_df, registry=registry, project=project, + full_feature_names=full_feature_names, ) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 44b83caff1f..9be327f5609 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -86,6 +86,7 @@ def get_historical_features( entity_df: Union[pandas.DataFrame, str], registry: Registry, project: str, + full_feature_names: bool = False, ) -> RetrievalJob: # TODO: Add entity_df validation in order to fail before interacting with BigQuery @@ -107,7 +108,7 @@ def get_historical_features( # Build a query context containing all information required to template the BigQuery SQL query query_context = get_feature_view_query_context( - feature_refs, feature_views, registry, project + feature_refs, feature_views, registry, project, full_feature_names ) # TODO: Infer min_timestamp and max_timestamp from entity_df @@ -118,6 +119,7 @@ def get_historical_features( max_timestamp=datetime.now() + timedelta(days=1), left_table_query_string=str(table.reference), entity_df_event_timestamp_col=entity_df_event_timestamp_col, + full_feature_names=full_feature_names, ) job = BigQueryRetrievalJob(query=query, client=client, config=config) @@ -320,11 +322,12 @@ def get_feature_view_query_context( feature_views: List[FeatureView], registry: Registry, project: str, + full_feature_names: bool = False, ) -> List[FeatureViewQueryContext]: """Build a query context containing all information required to template a BigQuery point-in-time SQL query""" feature_views_to_feature_map = _get_requested_feature_views_to_features_dict( - feature_refs, feature_views + feature_refs, feature_views, full_feature_names ) query_context = [] @@ -379,6 +382,7 @@ def build_point_in_time_query( max_timestamp: datetime, left_table_query_string: str, entity_df_event_timestamp_col: str, + full_feature_names: bool = False, ): """Build point-in-time query between each feature view table and the entity dataframe""" template = Environment(loader=BaseLoader()).from_string( @@ -395,6 +399,7 @@ def build_point_in_time_query( [entity for fv in feature_view_query_contexts for entity in fv.entities] ), "featureviews": [asdict(context) for context in feature_view_query_contexts], + "full_feature_names": full_feature_names, } query = template.render(template_context) @@ -468,7 +473,7 @@ def _get_bigquery_client(): {{ featureview.created_timestamp_column ~ ' as created_timestamp,' if featureview.created_timestamp_column else '' }} {{ featureview.entity_selections | join(', ')}}, {% for feature in featureview.features %} - {{ feature }} as {{ featureview.name }}__{{ feature }}{% if loop.last %}{% else %}, {% endif %} + {{ feature }} as {% if full_feature_names %}{{ featureview.name }}__{{feature}}{% else %}{{ feature }}{% endif %}{% if loop.last %}{% else %}, {% endif %} {% endfor %} FROM {{ featureview.table_subquery }} ), @@ -561,7 +566,7 @@ def _get_bigquery_client(): SELECT entity_row_unique_id, {% for feature in featureview.features %} - {{ featureview.name }}__{{ feature }}, + {% if full_feature_names %}{{ featureview.name }}__{{feature}}{% else %}{{ feature }}{% endif %}, {% endfor %} FROM {{ featureview.name }}__cleaned ) USING (entity_row_unique_id) diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index acd12ff9003..0513dc884a4 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -40,6 +40,7 @@ def get_historical_features( entity_df: Union[pd.DataFrame, str], registry: Registry, project: str, + full_feature_names: bool = False, ) -> FileRetrievalJob: if not isinstance(entity_df, pd.DataFrame): raise ValueError( @@ -59,9 +60,8 @@ def get_historical_features( raise ValueError( f"Please provide an entity_df with a column named {DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL} representing the time of events." ) - feature_views_to_features = _get_requested_feature_views_to_features_dict( - feature_refs, feature_views + feature_refs, feature_views, full_feature_names ) # Create lazy function that is only called from the RetrievalJob object @@ -125,14 +125,16 @@ def evaluate_historical_retrieval(): # Modify the separator for feature refs in column names to double underscore. We are using # double underscore as separator for consistency with other databases like BigQuery, # where there are very few characters available for use as separators - prefixed_feature_name = f"{feature_view.name}__{feature}" - + if full_feature_names: + formatted_feature_name = f"{feature_view.name}__{feature}" + else: + formatted_feature_name = feature # Add the feature name to the list of columns - feature_names.append(prefixed_feature_name) + feature_names.append(formatted_feature_name) # Ensure that the source dataframe feature column includes the feature view name as a prefix df_to_join.rename( - columns={feature: prefixed_feature_name}, inplace=True, + columns={feature: formatted_feature_name}, inplace=True, ) # Build a list of entity columns to join on (from the right table) diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index d31d11aae2a..c1c2279dc61 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -66,5 +66,6 @@ def get_historical_features( entity_df: Union[pd.DataFrame, str], registry: Registry, project: str, + full_feature_names: bool = False, ) -> RetrievalJob: pass diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 5d4f8d6cf0c..353be43b766 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -2,7 +2,7 @@ import importlib from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple, Union import pandas import pyarrow @@ -116,6 +116,7 @@ def get_historical_features( entity_df: Union[pandas.DataFrame, str], registry: Registry, project: str, + full_feature_names: bool = False, ) -> RetrievalJob: pass @@ -179,15 +180,24 @@ def get_provider(config: RepoConfig, repo_path: Path) -> Provider: def _get_requested_feature_views_to_features_dict( - feature_refs: List[str], feature_views: List[FeatureView] + feature_refs: List[str], feature_views: List[FeatureView], full_feature_names: bool ) -> Dict[FeatureView, List[str]]: - """Create a dict of FeatureView -> List[Feature] for all requested features""" + """Create a dict of FeatureView -> List[Feature] for all requested features. + Features are prefixed by the feature view name, set value to True to obtain only the feature names.""" feature_views_to_feature_map = {} # type: Dict[FeatureView, List[str]] + feature_set = set() # type: Set[str] + feature_collision_set = set() # type: Set[str] + for ref in feature_refs: ref_parts = ref.split(":") feature_view_from_ref = ref_parts[0] feature_from_ref = ref_parts[1] + if feature_from_ref in feature_set: + feature_collision_set.add(feature_from_ref) + else: + feature_set.add(feature_from_ref) + found = False for feature_view_from_registry in feature_views: if feature_view_from_registry.name == feature_view_from_ref: @@ -203,6 +213,11 @@ def _get_requested_feature_views_to_features_dict( if not found: raise ValueError(f"Could not find feature view from reference {ref}") + + if not full_feature_names and len(feature_collision_set) > 0: + err = ", ".join(x for x in feature_collision_set) + raise errors.FeatureNameCollisionError(err) + return feature_views_to_feature_map diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index 8b7e5f4d368..ac902376f52 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -62,6 +62,7 @@ def get_historical_features( entity_df: Union[pandas.DataFrame, str], registry: Registry, project: str, + full_feature_names: bool = False, ) -> RetrievalJob: pass diff --git a/sdk/python/tests/test_e2e_local.py b/sdk/python/tests/test_e2e_local.py index d61d8caa7b1..6057226b738 100644 --- a/sdk/python/tests/test_e2e_local.py +++ b/sdk/python/tests/test_e2e_local.py @@ -32,6 +32,7 @@ def _assert_online_features( "driver_hourly_stats:avg_daily_trips", ], entity_rows=[{"driver_id": 1001}], + full_feature_names=True, ) assert "driver_hourly_stats__avg_daily_trips" in result.to_dict() diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index 5cced84bb66..81b3a2609d8 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -17,6 +17,7 @@ from feast import errors, utils from feast.data_source import BigQuerySource, FileSource from feast.entity import Entity +from feast.errors import FeatureNameCollisionError from feast.feature import Feature from feast.feature_store import FeatureStore from feast.feature_view import FeatureView @@ -111,6 +112,7 @@ def create_customer_daily_profile_feature_view(source): Feature(name="current_balance", dtype=ValueType.FLOAT), Feature(name="avg_passenger_count", dtype=ValueType.FLOAT), Feature(name="lifetime_trip_count", dtype=ValueType.INT32), + Feature(name="avg_daily_trips", dtype=ValueType.INT32), ], input=source, ttl=timedelta(days=2), @@ -142,6 +144,7 @@ def get_expected_training_df( driver_fv: FeatureView, orders_df: pd.DataFrame, event_timestamp: str, + full_feature_names: bool = False, ): # Convert all pandas dataframes into records with UTC timestamps order_records = convert_timestamp_records_to_utc( @@ -177,6 +180,10 @@ def get_expected_training_df( f"driver_stats__{k}": driver_record.get(k, None) for k in ("conv_rate", "avg_daily_trips") } + if full_feature_names + else { + k: driver_record.get(k, None) for k in ("conv_rate", "avg_daily_trips") + } ) order_record.update( { @@ -187,6 +194,15 @@ def get_expected_training_df( "lifetime_trip_count", ) } + if full_feature_names + else { + k: customer_record.get(k, None) + for k in ( + "current_balance", + "avg_passenger_count", + "lifetime_trip_count", + ) + } ) # Convert records back to pandas dataframe @@ -199,12 +215,21 @@ def get_expected_training_df( # Cast some columns to expected types, since we lose information when converting pandas DFs into Python objects. expected_df["order_is_success"] = expected_df["order_is_success"].astype("int32") - expected_df["customer_profile__current_balance"] = expected_df[ - "customer_profile__current_balance" - ].astype("float32") - expected_df["customer_profile__avg_passenger_count"] = expected_df[ - "customer_profile__avg_passenger_count" - ].astype("float32") + + if full_feature_names: + expected_df["customer_profile__current_balance"] = expected_df[ + "customer_profile__current_balance" + ].astype("float32") + expected_df["customer_profile__avg_passenger_count"] = expected_df[ + "customer_profile__avg_passenger_count" + ].astype("float32") + else: + expected_df["current_balance"] = expected_df["current_balance"].astype( + "float32" + ) + expected_df["avg_passenger_count"] = expected_df["avg_passenger_count"].astype( + "float32" + ) return expected_df @@ -294,6 +319,7 @@ def test_historical_features_from_parquet_sources(infer_event_timestamp_col): "customer_profile:avg_passenger_count", "customer_profile:lifetime_trip_count", ], + full_feature_names=True, ) actual_df = job.to_df() @@ -303,7 +329,13 @@ def test_historical_features_from_parquet_sources(infer_event_timestamp_col): else "e_ts" ) expected_df = get_expected_training_df( - customer_df, customer_fv, driver_df, driver_fv, orders_df, event_timestamp, + customer_df, + customer_fv, + driver_df, + driver_fv, + orders_df, + event_timestamp, + full_feature_names=True, ) assert_frame_equal( expected_df.sort_values( @@ -314,6 +346,59 @@ def test_historical_features_from_parquet_sources(infer_event_timestamp_col): ).reset_index(drop=True), ) + event_timestamp = ( + DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL + if DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL in orders_df.columns + else "e_ts" + ) + expected_df_fno = get_expected_training_df( + customer_df, + customer_fv, + driver_df, + driver_fv, + orders_df, + event_timestamp, + full_feature_names=False, + ) + + # Test parquet sources when using feature names only (strip prefixed feature views) + job = store.get_historical_features( + entity_df=orders_df, + feature_refs=[ + "driver_stats:conv_rate", + "driver_stats:avg_daily_trips", + "customer_profile:current_balance", + "customer_profile:avg_passenger_count", + "customer_profile:lifetime_trip_count", + ], + full_feature_names=False, + ) + + actual_df_fno = job.to_df() + assert_frame_equal( + expected_df_fno.sort_values( + by=[event_timestamp, "order_id", "driver_id", "customer_id"] + ).reset_index(drop=True), + actual_df_fno.sort_values( + by=[event_timestamp, "order_id", "driver_id", "customer_id"] + ).reset_index(drop=True), + ) + + # Test for colliding feature names when featureview prefixes are stripped + with pytest.raises(FeatureNameCollisionError): + store.get_historical_features( + entity_df=orders_df, + feature_refs=[ + "driver_stats:conv_rate", + "driver_stats:avg_daily_trips", + "customer_profile:current_balance", + "customer_profile:avg_passenger_count", + "customer_profile:lifetime_trip_count", + "customer_profile:avg_daily_trips", + ], + full_feature_names=False, + ) + @pytest.mark.integration @pytest.mark.parametrize( @@ -427,7 +512,13 @@ def test_historical_features_from_bigquery_sources( else "e_ts" ) expected_df = get_expected_training_df( - customer_df, customer_fv, driver_df, driver_fv, orders_df, event_timestamp, + customer_df, + customer_fv, + driver_df, + driver_fv, + orders_df, + event_timestamp, + full_feature_names=True, ) job_from_sql = store.get_historical_features( @@ -439,6 +530,7 @@ def test_historical_features_from_bigquery_sources( "customer_profile:avg_passenger_count", "customer_profile:lifetime_trip_count", ], + full_feature_names=True, ) # Just a dry run, should not create table @@ -513,6 +605,7 @@ def test_historical_features_from_bigquery_sources( "customer_profile:avg_passenger_count", "customer_profile:lifetime_trip_count", ], + full_feature_names=True, ) # Rename the join key; this should now raise an error. @@ -560,3 +653,85 @@ def test_historical_features_from_bigquery_sources( .reset_index(drop=True), check_dtype=False, ) + + # Test BigQuery sources when using feature names only (strip prefixed feature views) + + expected_df_fno = get_expected_training_df( + customer_df, + customer_fv, + driver_df, + driver_fv, + orders_df, + event_timestamp, + full_feature_names=True, + ) + + job_from_sql = store.get_historical_features( + entity_df=entity_df_query, + feature_refs=[ + "driver_stats:conv_rate", + "driver_stats:avg_daily_trips", + "customer_profile:current_balance", + "customer_profile:avg_passenger_count", + "customer_profile:lifetime_trip_count", + ], + full_feature_names=False, + ) + + actual_df_from_sql_entities_fno = job_from_sql.to_df() + + assert_frame_equal( + expected_df_fno.sort_values( + by=[event_timestamp, "order_id", "driver_id", "customer_id"] + ).reset_index(drop=True), + actual_df_from_sql_entities_fno.sort_values( + by=[event_timestamp, "order_id", "driver_id", "customer_id"] + ).reset_index(drop=True), + check_dtype=False, + ) + + job_from_df = store.get_historical_features( + entity_df=orders_df, + feature_refs=[ + "driver_stats:conv_rate", + "driver_stats:avg_daily_trips", + "customer_profile:current_balance", + "customer_profile:avg_passenger_count", + "customer_profile:lifetime_trip_count", + ], + full_feature_names=False, + ) + + if provider_type == "gcp_custom_offline_config": + # Make sure that custom dataset name is being used from the offline_store config + assertpy.assert_that(job_from_df.query).contains("foo.entity_df") + else: + # If the custom dataset name isn't provided in the config, use default `feast` name + assertpy.assert_that(job_from_df.query).contains("feast.entity_df") + + actual_df_from_df_entities_fno = job_from_df.to_df() + + assert_frame_equal( + expected_df_fno.sort_values( + by=[event_timestamp, "order_id", "driver_id", "customer_id"] + ).reset_index(drop=True), + actual_df_from_df_entities_fno.sort_values( + by=[event_timestamp, "order_id", "driver_id", "customer_id"] + ).reset_index(drop=True), + check_dtype=False, + ) + + # Test for colliding feature names when featureview prefixes are stripped + with pytest.raises(FeatureNameCollisionError): + store.get_historical_features( + entity_df=orders_df, + feature_refs=[ + "driver_stats:conv_rate", + "driver_stats:avg_daily_trips", + "customer_profile:current_balance", + "customer_profile:avg_passenger_count", + "customer_profile:lifetime_trip_count", + "customer_profile:avg_daily_trips", + ], + full_feature_names=False, + ) diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py index 02943fd2eb8..d7a58692817 100644 --- a/sdk/python/tests/test_offline_online_store_consistency.py +++ b/sdk/python/tests/test_offline_online_store_consistency.py @@ -196,7 +196,7 @@ def check_offline_and_online_features( ) -> None: # Check online store response_dict = fs.get_online_features( - [f"{fv.name}:value"], [{"driver": driver_id}] + [f"{fv.name}:value"], [{"driver": driver_id}], full_feature_names=True ).to_dict() if expected_value: @@ -210,6 +210,7 @@ def check_offline_and_online_features( {"driver_id": [driver_id], "event_timestamp": [event_timestamp]} ), feature_refs=[f"{fv.name}:value"], + full_feature_names=True, ).to_df() if expected_value: diff --git a/sdk/python/tests/test_online_retrieval.py b/sdk/python/tests/test_online_retrieval.py index 3f5df6b3e0b..e98ae552a54 100644 --- a/sdk/python/tests/test_online_retrieval.py +++ b/sdk/python/tests/test_online_retrieval.py @@ -100,6 +100,7 @@ def test_online() -> None: "customer_driver_combined:trips", ], entity_rows=[{"driver": 1, "customer": 5}, {"driver": 1, "customer": 5}], + full_feature_names=True, ).to_dict() assert "driver_locations__lon" in result @@ -112,10 +113,34 @@ def test_online() -> None: assert result["customer_profile__name"] == ["John", "John"] assert result["customer_driver_combined__trips"] == [7, 7] + # Ensure setting full_feature_names to False strips featureview prefixes + # from feature names + result = store.get_online_features( + feature_refs=[ + "driver_locations:lon", + "customer_profile:avg_orders_day", + "customer_profile:name", + "customer_driver_combined:trips", + ], + entity_rows=[{"driver": 1, "customer": 5}, {"driver": 1, "customer": 5}], + full_feature_names=False, + ).to_dict() + + assert "lon" in result + assert "avg_orders_day" in result + assert "name" in result + assert result["driver"] == [1, 1] + assert result["customer"] == [5, 5] + assert result["lon"] == ["1.0", "1.0"] + assert result["avg_orders_day"] == [1.0, 1.0] + assert result["name"] == ["John", "John"] + assert result["trips"] == [7, 7] + # Ensure features are still in result when keys not found result = store.get_online_features( feature_refs=["customer_driver_combined:trips"], entity_rows=[{"driver": 0, "customer": 0}], + full_feature_names=True, ).to_dict() assert "customer_driver_combined__trips" in result @@ -123,7 +148,9 @@ def test_online() -> None: # invalid table reference with pytest.raises(FeatureViewNotFoundException): store.get_online_features( - feature_refs=["driver_locations_bad:lon"], entity_rows=[{"driver": 1}], + feature_refs=["driver_locations_bad:lon"], + entity_rows=[{"driver": 1}], + full_feature_names=True, ) # Create new FeatureStore object with fast cache invalidation @@ -148,6 +175,7 @@ def test_online() -> None: "customer_driver_combined:trips", ], entity_rows=[{"driver": 1, "customer": 5}], + full_feature_names=True, ).to_dict() assert result["driver_locations__lon"] == ["1.0"] assert result["customer_driver_combined__trips"] == [7] @@ -168,6 +196,7 @@ def test_online() -> None: "customer_driver_combined:trips", ], entity_rows=[{"driver": 1, "customer": 5}], + full_feature_names=True, ).to_dict() # Restore registry.db so that we can see if it actually reloads registry @@ -182,6 +211,7 @@ def test_online() -> None: "customer_driver_combined:trips", ], entity_rows=[{"driver": 1, "customer": 5}], + full_feature_names=True, ).to_dict() assert result["driver_locations__lon"] == ["1.0"] assert result["customer_driver_combined__trips"] == [7] @@ -207,6 +237,7 @@ def test_online() -> None: "customer_driver_combined:trips", ], entity_rows=[{"driver": 1, "customer": 5}], + full_feature_names=True, ).to_dict() assert result["driver_locations__lon"] == ["1.0"] assert result["customer_driver_combined__trips"] == [7] @@ -226,6 +257,7 @@ def test_online() -> None: "customer_driver_combined:trips", ], entity_rows=[{"driver": 1, "customer": 5}], + full_feature_names=True, ).to_dict() assert result["driver_locations__lon"] == ["1.0"] assert result["customer_driver_combined__trips"] == [7] From 06e0c77d923757676e8f63bfe28a5b49e8dc1f5e Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Wed, 16 Jun 2021 09:29:00 -0400 Subject: [PATCH 02/43] refactored existing tests to test full_feature_names feature on data retreival, added new tests also. Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/errors.py | 2 +- sdk/python/feast/feature_store.py | 23 +- sdk/python/feast/infra/provider.py | 14 +- sdk/python/tests/test_historical_retrieval.py | 200 +++++++----------- .../test_offline_online_store_consistency.py | 81 +++++-- sdk/python/tests/test_online_retrieval.py | 55 ++--- 6 files changed, 169 insertions(+), 206 deletions(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 0c026af31a0..53a4f22cf52 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -74,7 +74,7 @@ def __init__(self, offline_store_name: str, data_source_name: str): class FeatureNameCollisionError(Exception): def __init__(self, feature_name_collisions: str): super().__init__( - f"The following feature name(s) have collisions: {feature_name_collisions}. Set 'feature_names_only' argument in the data retrieval function to False to use the full feature name which is prefixed by the feature view name." + f"The following feature name(s) have collisions: {feature_name_collisions}. Set 'full_feature_names' argument in the data retrieval function to True to use the full feature name which is prefixed by the feature view name." ) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index df22aab944b..f68e61279c6 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -273,10 +273,9 @@ def get_historical_features( SQL query. The query must be of a format supported by the configured offline store (e.g., BigQuery) feature_refs: A list of features that should be retrieved from the offline store. Feature references are of the format "feature_view:feature", e.g., "customer_fv:daily_transactions". - full_feature_names: By default, this value is set to False. This strips the feature view prefixes from the data - and returns only the feature name, changing them from the format "feature_view__feature" to "feature" - (e.g., "customer_fv__daily_transactions" changes to "daily_transactions"). Set the value to True for - the feature names to be prefixed by the feature view name in the format "feature_view__feature". + full_feature_names: By default, this value is set to False. By setting the value to True, this adds the + feature view prefixes to the feature names, changing them from the format "feature" to + "feature_view__feature" (e.g., "daily_transactions" changes to "customer_fv__daily_transactions"). Returns: RetrievalJob which can be used to materialize the results. @@ -290,7 +289,7 @@ def get_historical_features( >>> retrieval_job = fs.get_historical_features( >>> entity_df="SELECT event_timestamp, order_id, customer_id from gcp_project.my_ds.customer_orders", >>> feature_refs=["customer:age", "customer:avg_orders_1d", "customer:avg_orders_7d"], - >>> full_feature_names=False + >>> full_feature_names=True >>> ) >>> feature_data = retrieval_job.to_df() >>> model.fit(feature_data) # insert your modeling framework here. @@ -298,9 +297,9 @@ def get_historical_features( all_feature_views = self._registry.list_feature_views(project=self.project) try: feature_views = _get_requested_feature_views( - feature_refs, all_feature_views + feature_refs, all_feature_views, full_feature_names ) - except FeatureViewNotFoundException as e: + except (FeatureNameCollisionError, FeatureViewNotFoundException) as e: sys.exit(e) provider = self._get_provider() @@ -635,6 +634,7 @@ def _group_refs( raise FeatureViewNotFoundException(view_name) views_features[view_name].append(feat_name) + print(full_feature_names) if not full_feature_names and len(feature_collision_set) > 0: err = ", ".join(x for x in feature_collision_set) raise FeatureNameCollisionError(err) @@ -646,11 +646,16 @@ def _group_refs( def _get_requested_feature_views( - feature_refs: List[str], all_feature_views: List[FeatureView] + feature_refs: List[str], + all_feature_views: List[FeatureView], + full_feature_names: bool, ) -> List[FeatureView]: """Get list of feature views based on feature references""" # TODO: Get rid of this function. We only need _group_refs - return list(view for view, _ in _group_refs(feature_refs, all_feature_views)) + return list( + view + for view, _ in _group_refs(feature_refs, all_feature_views, full_feature_names) + ) def _get_table_entity_keys( diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 353be43b766..50d3d8873d4 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -2,7 +2,7 @@ import importlib from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union import pandas import pyarrow @@ -183,20 +183,14 @@ def _get_requested_feature_views_to_features_dict( feature_refs: List[str], feature_views: List[FeatureView], full_feature_names: bool ) -> Dict[FeatureView, List[str]]: """Create a dict of FeatureView -> List[Feature] for all requested features. - Features are prefixed by the feature view name, set value to True to obtain only the feature names.""" + Set full_feature_names to True to get feature names prefixed by its featureview.""" feature_views_to_feature_map = {} # type: Dict[FeatureView, List[str]] - feature_set = set() # type: Set[str] - feature_collision_set = set() # type: Set[str] for ref in feature_refs: ref_parts = ref.split(":") feature_view_from_ref = ref_parts[0] feature_from_ref = ref_parts[1] - if feature_from_ref in feature_set: - feature_collision_set.add(feature_from_ref) - else: - feature_set.add(feature_from_ref) found = False for feature_view_from_registry in feature_views: @@ -214,10 +208,6 @@ def _get_requested_feature_views_to_features_dict( if not found: raise ValueError(f"Could not find feature view from reference {ref}") - if not full_feature_names and len(feature_collision_set) > 0: - err = ", ".join(x for x in feature_collision_set) - raise errors.FeatureNameCollisionError(err) - return feature_views_to_feature_map diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index 81b3a2609d8..48c64505280 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -19,7 +19,7 @@ from feast.entity import Entity from feast.errors import FeatureNameCollisionError from feast.feature import Feature -from feast.feature_store import FeatureStore +from feast.feature_store import FeatureStore, _group_refs from feast.feature_view import FeatureView from feast.infra.provider import DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL from feast.repo_config import ( @@ -271,7 +271,12 @@ def __exit__(self, exc_type, exc_value, exc_traceback): @pytest.mark.parametrize( "infer_event_timestamp_col", [False, True], ) -def test_historical_features_from_parquet_sources(infer_event_timestamp_col): +@pytest.mark.parametrize( + "full_feature_names", [False, True], +) +def test_historical_features_from_parquet_sources( + infer_event_timestamp_col, full_feature_names +): start_date = datetime.now().replace(microsecond=0, second=0, minute=0) ( customer_entities, @@ -319,7 +324,7 @@ def test_historical_features_from_parquet_sources(infer_event_timestamp_col): "customer_profile:avg_passenger_count", "customer_profile:lifetime_trip_count", ], - full_feature_names=True, + full_feature_names=full_feature_names, ) actual_df = job.to_df() @@ -335,7 +340,7 @@ def test_historical_features_from_parquet_sources(infer_event_timestamp_col): driver_fv, orders_df, event_timestamp, - full_feature_names=True, + full_feature_names=full_feature_names, ) assert_frame_equal( expected_df.sort_values( @@ -346,59 +351,6 @@ def test_historical_features_from_parquet_sources(infer_event_timestamp_col): ).reset_index(drop=True), ) - event_timestamp = ( - DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL - if DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL in orders_df.columns - else "e_ts" - ) - expected_df_fno = get_expected_training_df( - customer_df, - customer_fv, - driver_df, - driver_fv, - orders_df, - event_timestamp, - full_feature_names=False, - ) - - # Test parquet sources when using feature names only (strip prefixed feature views) - job = store.get_historical_features( - entity_df=orders_df, - feature_refs=[ - "driver_stats:conv_rate", - "driver_stats:avg_daily_trips", - "customer_profile:current_balance", - "customer_profile:avg_passenger_count", - "customer_profile:lifetime_trip_count", - ], - full_feature_names=False, - ) - - actual_df_fno = job.to_df() - assert_frame_equal( - expected_df_fno.sort_values( - by=[event_timestamp, "order_id", "driver_id", "customer_id"] - ).reset_index(drop=True), - actual_df_fno.sort_values( - by=[event_timestamp, "order_id", "driver_id", "customer_id"] - ).reset_index(drop=True), - ) - - # Test for colliding feature names when featureview prefixes are stripped - with pytest.raises(FeatureNameCollisionError): - store.get_historical_features( - entity_df=orders_df, - feature_refs=[ - "driver_stats:conv_rate", - "driver_stats:avg_daily_trips", - "customer_profile:current_balance", - "customer_profile:avg_passenger_count", - "customer_profile:lifetime_trip_count", - "customer_profile:avg_daily_trips", - ], - full_feature_names=False, - ) - @pytest.mark.integration @pytest.mark.parametrize( @@ -407,8 +359,11 @@ def test_historical_features_from_parquet_sources(infer_event_timestamp_col): @pytest.mark.parametrize( "infer_event_timestamp_col", [False, True], ) +@pytest.mark.parametrize( + "full_feature_names", [False, True], +) def test_historical_features_from_bigquery_sources( - provider_type, infer_event_timestamp_col, capsys + provider_type, infer_event_timestamp_col, capsys, full_feature_names ): start_date = datetime.now().replace(microsecond=0, second=0, minute=0) ( @@ -518,7 +473,7 @@ def test_historical_features_from_bigquery_sources( driver_fv, orders_df, event_timestamp, - full_feature_names=True, + full_feature_names, ) job_from_sql = store.get_historical_features( @@ -530,7 +485,7 @@ def test_historical_features_from_bigquery_sources( "customer_profile:avg_passenger_count", "customer_profile:lifetime_trip_count", ], - full_feature_names=True, + full_feature_names=full_feature_names, ) # Just a dry run, should not create table @@ -605,7 +560,7 @@ def test_historical_features_from_bigquery_sources( "customer_profile:avg_passenger_count", "customer_profile:lifetime_trip_count", ], - full_feature_names=True, + full_feature_names=full_feature_names, ) # Rename the join key; this should now raise an error. @@ -654,77 +609,31 @@ def test_historical_features_from_bigquery_sources( check_dtype=False, ) - # Test BigQuery sources when using feature names only (strip prefixed feature views) - - expected_df_fno = get_expected_training_df( - customer_df, - customer_fv, - driver_df, - driver_fv, - orders_df, - event_timestamp, - full_feature_names=True, - ) - - job_from_sql = store.get_historical_features( - entity_df=entity_df_query, - feature_refs=[ - "driver_stats:conv_rate", - "driver_stats:avg_daily_trips", - "customer_profile:current_balance", - "customer_profile:avg_passenger_count", - "customer_profile:lifetime_trip_count", - ], - full_feature_names=False, - ) - actual_df_from_sql_entities_fno = job_from_sql.to_df() +@pytest.mark.integration +def test_feature_name_collision_on_historical_retrieval_from_parquet_sources(): + start_date = datetime.now().replace(microsecond=0, second=0, minute=0) + (customer_entities, driver_entities, end_date, _, start_date,) = generate_entities( + start_date, True + ) - assert_frame_equal( - expected_df_fno.sort_values( - by=[event_timestamp, "order_id", "driver_id", "customer_id"] - ).reset_index(drop=True), - actual_df_from_sql_entities_fno.sort_values( - by=[event_timestamp, "order_id", "driver_id", "customer_id"] - ).reset_index(drop=True), - check_dtype=False, + with TemporaryDirectory() as temp_dir: + driver_df = driver_data.create_driver_hourly_stats_df( + driver_entities, start_date, end_date ) - - job_from_df = store.get_historical_features( - entity_df=orders_df, - feature_refs=[ - "driver_stats:conv_rate", - "driver_stats:avg_daily_trips", - "customer_profile:current_balance", - "customer_profile:avg_passenger_count", - "customer_profile:lifetime_trip_count", - ], - full_feature_names=False, + driver_source = stage_driver_hourly_stats_parquet_source(temp_dir, driver_df) + driver_fv = create_driver_hourly_stats_feature_view(driver_source) + customer_df = driver_data.create_customer_daily_profile_df( + customer_entities, start_date, end_date ) - - if provider_type == "gcp_custom_offline_config": - # Make sure that custom dataset name is being used from the offline_store config - assertpy.assert_that(job_from_df.query).contains("foo.entity_df") - else: - # If the custom dataset name isn't provided in the config, use default `feast` name - assertpy.assert_that(job_from_df.query).contains("feast.entity_df") - - actual_df_from_df_entities_fno = job_from_df.to_df() - - assert_frame_equal( - expected_df_fno.sort_values( - by=[event_timestamp, "order_id", "driver_id", "customer_id"] - ).reset_index(drop=True), - actual_df_from_df_entities_fno.sort_values( - by=[event_timestamp, "order_id", "driver_id", "customer_id"] - ).reset_index(drop=True), - check_dtype=False, + customer_source = stage_customer_daily_profile_parquet_source( + temp_dir, customer_df ) + customer_fv = create_customer_daily_profile_feature_view(customer_source) - # Test for colliding feature names when featureview prefixes are stripped + # _group_refs is the function that checks for colliding feature names with pytest.raises(FeatureNameCollisionError): - store.get_historical_features( - entity_df=orders_df, + _group_refs( feature_refs=[ "driver_stats:conv_rate", "driver_stats:avg_daily_trips", @@ -733,5 +642,46 @@ def test_historical_features_from_bigquery_sources( "customer_profile:lifetime_trip_count", "customer_profile:avg_daily_trips", ], + all_feature_views=[driver_fv, customer_fv], full_feature_names=False, ) + + +def test_feature_name_collision_on_historical_retrieval_from_bigquery_sources(): + bigquery_dataset = ( + f"test_hist_retrieval_{int(time.time_ns())}_{random.randint(1000, 9999)}" + ) + + gcp_project = bigquery.Client().project + + # Driver Feature View + driver_table_id = f"{gcp_project}.{bigquery_dataset}.driver_hourly" + driver_source = BigQuerySource( + table_ref=driver_table_id, + event_timestamp_column="datetime", + created_timestamp_column="created", + ) + driver_fv = create_driver_hourly_stats_feature_view(driver_source) + + customer_table_id = f"{gcp_project}.{bigquery_dataset}.customer_profile" + customer_source = BigQuerySource( + table_ref=customer_table_id, + event_timestamp_column="datetime", + created_timestamp_column="", + ) + customer_fv = create_customer_daily_profile_feature_view(customer_source) + + # _group_refs is the function that checks for colliding feature names + with pytest.raises(FeatureNameCollisionError): + _group_refs( + feature_refs=[ + "driver_stats:conv_rate", + "driver_stats:avg_daily_trips", + "customer_profile:current_balance", + "customer_profile:avg_passenger_count", + "customer_profile:lifetime_trip_count", + "customer_profile:avg_daily_trips", + ], + all_feature_views=[driver_fv, customer_fv], + full_feature_names=False, + ) diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py index d7a58692817..237ee8cda75 100644 --- a/sdk/python/tests/test_offline_online_store_consistency.py +++ b/sdk/python/tests/test_offline_online_store_consistency.py @@ -193,16 +193,25 @@ def check_offline_and_online_features( driver_id: int, event_timestamp: datetime, expected_value: Optional[float], + full_feature_names: bool, ) -> None: # Check online store response_dict = fs.get_online_features( - [f"{fv.name}:value"], [{"driver": driver_id}], full_feature_names=True + [f"{fv.name}:value"], + [{"driver": driver_id}], + full_feature_names=full_feature_names, ).to_dict() - if expected_value: - assert abs(response_dict[f"{fv.name}__value"][0] - expected_value) < 1e-6 + if full_feature_names: + if expected_value: + assert abs(response_dict[f"{fv.name}__value"][0] - expected_value) < 1e-6 + else: + assert response_dict[f"{fv.name}__value"][0] is None else: - assert response_dict[f"{fv.name}__value"][0] is None + if expected_value: + assert abs(response_dict["value"][0] - expected_value) < 1e-6 + else: + assert response_dict["value"][0] is None # Check offline store df = fs.get_historical_features( @@ -210,18 +219,25 @@ def check_offline_and_online_features( {"driver_id": [driver_id], "event_timestamp": [event_timestamp]} ), feature_refs=[f"{fv.name}:value"], - full_feature_names=True, + full_feature_names=full_feature_names, ).to_df() - if expected_value: - assert abs(df.to_dict()[f"{fv.name}__value"][0] - expected_value) < 1e-6 + if full_feature_names: + if expected_value: + assert abs(df.to_dict()[f"{fv.name}__value"][0] - expected_value) < 1e-6 + else: + df = df.where(pd.notnull(df), None) + assert df.to_dict()[f"{fv.name}__value"][0] is None else: - df = df.where(pd.notnull(df), None) - assert df.to_dict()[f"{fv.name}__value"][0] is None + if expected_value: + assert abs(df.to_dict()["value"][0] - expected_value) < 1e-6 + else: + df = df.where(pd.notnull(df), None) + assert df.to_dict()["value"][0] is None def run_offline_online_store_consistency_test( - fs: FeatureStore, fv: FeatureView + fs: FeatureStore, fv: FeatureView, ffn: bool ) -> None: now = datetime.utcnow() # Run materialize() @@ -232,16 +248,31 @@ def run_offline_online_store_consistency_test( # check result of materialize() check_offline_and_online_features( - fs=fs, fv=fv, driver_id=1, event_timestamp=end_date, expected_value=0.3 + fs=fs, + fv=fv, + driver_id=1, + event_timestamp=end_date, + expected_value=0.3, + full_feature_names=ffn, ) check_offline_and_online_features( - fs=fs, fv=fv, driver_id=2, event_timestamp=end_date, expected_value=None + fs=fs, + fv=fv, + driver_id=2, + event_timestamp=end_date, + expected_value=None, + full_feature_names=ffn, ) # check prior value for materialize_incremental() check_offline_and_online_features( - fs=fs, fv=fv, driver_id=3, event_timestamp=end_date, expected_value=4 + fs=fs, + fv=fv, + driver_id=3, + event_timestamp=end_date, + expected_value=4, + full_feature_names=ffn, ) # run materialize_incremental() @@ -249,7 +280,12 @@ def run_offline_online_store_consistency_test( # check result of materialize_incremental() check_offline_and_online_features( - fs=fs, fv=fv, driver_id=3, event_timestamp=now, expected_value=5 + fs=fs, + fv=fv, + driver_id=3, + event_timestamp=now, + expected_value=5, + full_feature_names=ffn, ) @@ -257,17 +293,22 @@ def run_offline_online_store_consistency_test( @pytest.mark.parametrize( "bq_source_type", ["query", "table"], ) -def test_bq_offline_online_store_consistency(bq_source_type: str): +@pytest.mark.parametrize("full_feature_names", [True, False]) +def test_bq_offline_online_store_consistency( + bq_source_type: str, full_feature_names: bool +): with prep_bq_fs_and_fv(bq_source_type) as (fs, fv): - run_offline_online_store_consistency_test(fs, fv) + run_offline_online_store_consistency_test(fs, fv, full_feature_names) +@pytest.mark.parametrize("full_feature_names", [True, False]) @pytest.mark.integration -def test_redis_offline_online_store_consistency(): +def test_redis_offline_online_store_consistency(full_feature_names: bool): with prep_redis_fs_and_fv() as (fs, fv): - run_offline_online_store_consistency_test(fs, fv) + run_offline_online_store_consistency_test(fs, fv, full_feature_names) -def test_local_offline_online_store_consistency(): +@pytest.mark.parametrize("full_feature_names", [True, False]) +def test_local_offline_online_store_consistency(full_feature_names: bool): with prep_local_fs_and_fv() as (fs, fv): - run_offline_online_store_consistency_test(fs, fv) + run_offline_online_store_consistency_test(fs, fv, full_feature_names) diff --git a/sdk/python/tests/test_online_retrieval.py b/sdk/python/tests/test_online_retrieval.py index e98ae552a54..6172d5e93b1 100644 --- a/sdk/python/tests/test_online_retrieval.py +++ b/sdk/python/tests/test_online_retrieval.py @@ -92,29 +92,6 @@ def test_online() -> None: ) # Retrieve two features using two keys, one valid one non-existing - result = store.get_online_features( - feature_refs=[ - "driver_locations:lon", - "customer_profile:avg_orders_day", - "customer_profile:name", - "customer_driver_combined:trips", - ], - entity_rows=[{"driver": 1, "customer": 5}, {"driver": 1, "customer": 5}], - full_feature_names=True, - ).to_dict() - - assert "driver_locations__lon" in result - assert "customer_profile__avg_orders_day" in result - assert "customer_profile__name" in result - assert result["driver"] == [1, 1] - assert result["customer"] == [5, 5] - assert result["driver_locations__lon"] == ["1.0", "1.0"] - assert result["customer_profile__avg_orders_day"] == [1.0, 1.0] - assert result["customer_profile__name"] == ["John", "John"] - assert result["customer_driver_combined__trips"] == [7, 7] - - # Ensure setting full_feature_names to False strips featureview prefixes - # from feature names result = store.get_online_features( feature_refs=[ "driver_locations:lon", @@ -140,17 +117,17 @@ def test_online() -> None: result = store.get_online_features( feature_refs=["customer_driver_combined:trips"], entity_rows=[{"driver": 0, "customer": 0}], - full_feature_names=True, + full_feature_names=False, ).to_dict() - assert "customer_driver_combined__trips" in result + assert "trips" in result # invalid table reference with pytest.raises(FeatureViewNotFoundException): store.get_online_features( feature_refs=["driver_locations_bad:lon"], entity_rows=[{"driver": 1}], - full_feature_names=True, + full_feature_names=False, ) # Create new FeatureStore object with fast cache invalidation @@ -175,10 +152,10 @@ def test_online() -> None: "customer_driver_combined:trips", ], entity_rows=[{"driver": 1, "customer": 5}], - full_feature_names=True, + full_feature_names=False, ).to_dict() - assert result["driver_locations__lon"] == ["1.0"] - assert result["customer_driver_combined__trips"] == [7] + assert result["lon"] == ["1.0"] + assert result["trips"] == [7] # Rename the registry.db so that it cant be used for refreshes os.rename(store.config.registry, store.config.registry + "_fake") @@ -196,7 +173,7 @@ def test_online() -> None: "customer_driver_combined:trips", ], entity_rows=[{"driver": 1, "customer": 5}], - full_feature_names=True, + full_feature_names=False, ).to_dict() # Restore registry.db so that we can see if it actually reloads registry @@ -211,10 +188,10 @@ def test_online() -> None: "customer_driver_combined:trips", ], entity_rows=[{"driver": 1, "customer": 5}], - full_feature_names=True, + full_feature_names=False, ).to_dict() - assert result["driver_locations__lon"] == ["1.0"] - assert result["customer_driver_combined__trips"] == [7] + assert result["lon"] == ["1.0"] + assert result["trips"] == [7] # Create a registry with infinite cache (for users that want to manually refresh the registry) fs_infinite_ttl = FeatureStore( @@ -237,10 +214,10 @@ def test_online() -> None: "customer_driver_combined:trips", ], entity_rows=[{"driver": 1, "customer": 5}], - full_feature_names=True, + full_feature_names=False, ).to_dict() - assert result["driver_locations__lon"] == ["1.0"] - assert result["customer_driver_combined__trips"] == [7] + assert result["lon"] == ["1.0"] + assert result["trips"] == [7] # Wait a bit so that an arbitrary TTL would take effect time.sleep(2) @@ -257,10 +234,10 @@ def test_online() -> None: "customer_driver_combined:trips", ], entity_rows=[{"driver": 1, "customer": 5}], - full_feature_names=True, + full_feature_names=False, ).to_dict() - assert result["driver_locations__lon"] == ["1.0"] - assert result["customer_driver_combined__trips"] == [7] + assert result["lon"] == ["1.0"] + assert result["trips"] == [7] # Force registry reload (should fail because file is missing) with pytest.raises(FileNotFoundError): From 4b7dd1897d5e8b7891233959cba40d01dfa523f4 Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Wed, 16 Jun 2021 19:52:59 -0400 Subject: [PATCH 03/43] removed full_feature_names usage from quickstart and README to have more simple examples. Resolved failing tests. Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- README.md | 11 ++-- docs/quickstart.md | 9 ++-- sdk/python/feast/feature_store.py | 1 - sdk/python/tests/test_historical_retrieval.py | 3 +- sdk/python/tests/test_online_retrieval.py | 54 +++++++++---------- 5 files changed, 35 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 00604e2f788..4fe0c11083f 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,6 @@ training_df = store.get_historical_features( 'driver_hourly_stats:acc_rate', 'driver_hourly_stats:avg_daily_trips' ], - full_feature_names=True ).to_df() print(training_df.head()) @@ -76,11 +75,11 @@ print(training_df.head()) # model = ml.fit(training_df) ``` ```commandline - event_timestamp driver_id driver_hourly_stats__conv_rate driver_hourly_stats__acc_rate - 2021-04-12 08:12:10 1002 0.497279 0.357702 - 2021-04-12 10:59:42 1001 0.979747 0.008166 - 2021-04-12 15:01:12 1004 0.151432 0.551748 - 2021-04-12 16:40:26 1003 0.951506 0.753572 + event_timestamp driver_id conv_rate acc_rate avg_daily_trips +0 2021-04-12 08:12:10+00:00 1002 0.713465 0.597095 531 +1 2021-04-12 10:59:42+00:00 1001 0.072752 0.044344 11 +2 2021-04-12 15:01:12+00:00 1004 0.658182 0.079150 220 +3 2021-04-12 16:40:26+00:00 1003 0.162092 0.309035 959 ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index b98bfe8acac..66d4f0f00d6 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -234,8 +234,7 @@ feature_vector = store.get_online_features( 'driver_hourly_stats:acc_rate', 'driver_hourly_stats:avg_daily_trips' ], - entity_rows=[{"driver_id": 1001}], - full_feature_names=True + entity_rows=[{"driver_id": 1001}] ).to_dict() pprint(feature_vector) @@ -245,9 +244,9 @@ pprint(feature_vector) ```text { 'driver_id': [1001], - 'driver_hourly_stats__conv_rate': [0.49274], - 'driver_hourly_stats__acc_rate': [0.92743], - 'driver_hourly_stats__avg_daily_trips': [72], + 'conv_rate': [0.49274], + 'acc_rate': [0.92743], + 'avg_daily_trips': [72], } ``` diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index f68e61279c6..dd08586be35 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -634,7 +634,6 @@ def _group_refs( raise FeatureViewNotFoundException(view_name) views_features[view_name].append(feat_name) - print(full_feature_names) if not full_feature_names and len(feature_collision_set) > 0: err = ", ".join(x for x in feature_collision_set) raise FeatureNameCollisionError(err) diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index 48c64505280..79e9b14d906 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -651,8 +651,7 @@ def test_feature_name_collision_on_historical_retrieval_from_bigquery_sources(): bigquery_dataset = ( f"test_hist_retrieval_{int(time.time_ns())}_{random.randint(1000, 9999)}" ) - - gcp_project = bigquery.Client().project + gcp_project = "project_name" # Driver Feature View driver_table_id = f"{gcp_project}.{bigquery_dataset}.driver_hourly" diff --git a/sdk/python/tests/test_online_retrieval.py b/sdk/python/tests/test_online_retrieval.py index 6172d5e93b1..b76f901bd4d 100644 --- a/sdk/python/tests/test_online_retrieval.py +++ b/sdk/python/tests/test_online_retrieval.py @@ -276,7 +276,7 @@ def test_online_to_df(): for (d, c) in zip(driver_ids, customer_ids): """ driver table: - driver driver_locations__lon driver_locations__lat + lon lat 1 1.0 0.1 2 2.0 0.2 3 3.0 0.3 @@ -303,10 +303,10 @@ def test_online_to_df(): """ customer table - customer customer_profile__avg_orders_day customer_profile__name customer_profile__age - 4 4.0 foo4 40 - 5 5.0 foo5 50 - 6 6.0 foo6 60 + customer avg_orders_day name age + 4 4.0 foo4 40 + 5 5.0 foo5 50 + 6 6.0 foo6 60 """ customer_key = EntityKeyProto( join_keys=["customer"], entity_values=[ValueProto(int64_val=c)] @@ -332,10 +332,10 @@ def test_online_to_df(): ) """ customer_driver_combined table - customer driver customer_driver_combined__trips - 4 1 4 - 5 2 10 - 6 3 18 + customer driver trips + 4 1 4 + 5 2 10 + 6 3 18 """ combo_keys = EntityKeyProto( join_keys=["customer", "driver"], @@ -373,35 +373,31 @@ def test_online_to_df(): ).to_df() """ Construct the expected dataframe with reversed row order like so: - driver customer driver_locations__lon driver_locations__lat customer_profile__avg_orders_day customer_profile__name customer_profile__age customer_driver_combined__trips - 3 6 3.0 0.3 6.0 foo6 60 18 - 2 5 2.0 0.2 5.0 foo5 50 10 - 1 4 1.0 0.1 4.0 foo4 40 4 + driver customer lon lat avg_orders_day name age trips + 3 6 3.0 0.3 6.0 foo6 60 18 + 2 5 2.0 0.2 5.0 foo5 50 10 + 1 4 1.0 0.1 4.0 foo4 40 4 """ df_dict = { "driver": driver_ids, "customer": customer_ids, - "driver_locations__lon": [str(d * lon_multiply) for d in driver_ids], - "driver_locations__lat": [d * lat_multiply for d in driver_ids], - "customer_profile__avg_orders_day": [ - c * avg_order_day_multiply for c in customer_ids - ], - "customer_profile__name": [name + str(c) for c in customer_ids], - "customer_profile__age": [c * age_multiply for c in customer_ids], - "customer_driver_combined__trips": [ - d * c for (d, c) in zip(driver_ids, customer_ids) - ], + "lon": [str(d * lon_multiply) for d in driver_ids], + "lat": [d * lat_multiply for d in driver_ids], + "avg_orders_day": [c * avg_order_day_multiply for c in customer_ids], + "name": [name + str(c) for c in customer_ids], + "age": [c * age_multiply for c in customer_ids], + "trips": [d * c for (d, c) in zip(driver_ids, customer_ids)], } # Requested column order ordered_column = [ "driver", "customer", - "driver_locations__lon", - "driver_locations__lat", - "customer_profile__avg_orders_day", - "customer_profile__name", - "customer_profile__age", - "customer_driver_combined__trips", + "lon", + "lat", + "avg_orders_day", + "name", + "age", + "trips", ] expected_df = pd.DataFrame({k: reversed(v) for (k, v) in df_dict.items()}) assert_frame_equal(result_df[ordered_column], expected_df) From 579e08f508933530d3a70e7226439d970158a4d1 Mon Sep 17 00:00:00 2001 From: Tsotne Tabidze Date: Wed, 16 Jun 2021 18:55:40 -0700 Subject: [PATCH 04/43] Update CHANGELOG for Feast v0.10.8 Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 544b5a41e28..54d598be5b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [v0.10.8](https://github.com/feast-dev/feast/tree/v0.10.8) (2021-06-17) + +[Full Changelog](https://github.com/feast-dev/feast/compare/v0.10.7...v0.10.8) + +**Implemented enhancements:** + +- Add `to_bigquery()` function to BigQueryRetrievalJob [\#1634](https://github.com/feast-dev/feast/pull/1634) ([vtao2](https://github.com/vtao2)) + +**Fixed bugs:** + +- Don't use .result\(\) in BigQueryOfflineStore, since it still leads to OOM [\#1642](https://github.com/feast-dev/feast/pull/1642) ([tsotnet](https://github.com/tsotnet)) +- Don't load entire bigquery query results in memory [\#1638](https://github.com/feast-dev/feast/pull/1638) ([tsotnet](https://github.com/tsotnet)) +- Add entity column validations when getting historical features from bigquery [\#1614](https://github.com/feast-dev/feast/pull/1614) ([achals](https://github.com/achals)) + +**Merged pull requests:** + +- Make test historical retrieval longer [\#1630](https://github.com/feast-dev/feast/pull/1630) ([MattDelac](https://github.com/MattDelac)) +- Fix failing historical retrieval assertion [\#1622](https://github.com/feast-dev/feast/pull/1622) ([woop](https://github.com/woop)) +- Optimize historical retrieval with BigQuery offline store [\#1602](https://github.com/feast-dev/feast/pull/1602) ([MattDelac](https://github.com/MattDelac)) + ## [v0.10.7](https://github.com/feast-dev/feast/tree/v0.10.7) (2021-06-07) [Full Changelog](https://github.com/feast-dev/feast/compare/v0.10.6...v0.10.7) From 462da438deacec503651f373d9e7c86e95fbd15c Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Thu, 17 Jun 2021 06:25:16 +0000 Subject: [PATCH 05/43] GitBook: [master] 2 pages modified Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- docs/SUMMARY.md | 1 + .../user-guide/extending-feast.md | 106 ++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 docs/feast-on-kubernetes/user-guide/extending-feast.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 299066a7ef3..9288a077182 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -56,6 +56,7 @@ * [Getting online features](feast-on-kubernetes/user-guide/getting-online-features.md) * [Getting training features](feast-on-kubernetes/user-guide/getting-training-features.md) * [Define and ingest features](feast-on-kubernetes/user-guide/define-and-ingest-features.md) + * [Extending Feast](feast-on-kubernetes/user-guide/extending-feast.md) * [Reference](feast-on-kubernetes/reference-1/README.md) * [Configuration Reference](feast-on-kubernetes/reference-1/configuration-reference.md) * [Feast and Spark](feast-on-kubernetes/reference-1/feast-and-spark.md) diff --git a/docs/feast-on-kubernetes/user-guide/extending-feast.md b/docs/feast-on-kubernetes/user-guide/extending-feast.md new file mode 100644 index 00000000000..b124e2f948b --- /dev/null +++ b/docs/feast-on-kubernetes/user-guide/extending-feast.md @@ -0,0 +1,106 @@ +# Extending Feast + +## Custom OnlineStore + +Feast allow users to create their own OnlineStore implementations, allowing Feast to read and write feature values to stores other than first-party implementations already in Feast directly. The interface for the is found at [here](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/online_stores/online_store.py), and consists of four methods that need to be implemented. + +### Update/Teardown methods + +The `update` method is should be set up any state in the OnlineStore that is required before any data can be ingested into it. This can be things like tables in sqlite, or keyspaces in Cassandra, etc. The update method should be idempotent. Similarly, the `teardown` method should remove any state in the online store. + +```python +def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[Union[FeatureTable, FeatureView]], + tables_to_keep: Sequence[Union[FeatureTable, FeatureView]], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, +): + ... + +def teardown( + self, + config: RepoConfig, + tables: Sequence[Union[FeatureTable, FeatureView]], + entities: Sequence[Entity], +): + ... + +``` + +### Write/Read methods + +The `online_write_batch` method is responsible for writing the data into the online store - and `online_read` method is responsible for reading data from the online store. + +```python +def online_write_batch( + self, + config: RepoConfig, + table: Union[FeatureTable, FeatureView], + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], +) -> None: + + ... + +def online_read( + self, + config: RepoConfig, + table: Union[FeatureTable, FeatureView], + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, +) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + ... +``` + +## Custom OfflineStore + +Feast allow users to create their own OfflineStore implementations, allowing Feast to read and write feature values to stores other than first-party implementations already in Feast directly. The interface for the is found at [here](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/offline_stores/offline_store.py), and consists of two methods that need to be implemented. + +### Write method + +The `pull_latest_from_table_or_query` method is used to read data from a source for materialization into the OfflineStore. + +```python +def pull_latest_from_table_or_query( + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + event_timestamp_column: str, + created_timestamp_column: Optional[str], + start_date: datetime, + end_date: datetime, +) -> pyarrow.Table: + ... + +``` + +### Read method + +The read method is responsible for reading historical features from the OfflineStore. The feature retrieval may be asynchronous, so the read method is expected to return an object that should produce a DataFrame representing the historical features once the feature retrieval job is complete. + +```python +class RetrievalJob: + + @abstractmethod + def to_df(self): + pass + +def get_historical_features( + config: RepoConfig, + feature_views: List[FeatureView], + feature_refs: List[str], + entity_df: Union[pd.DataFrame, str], + registry: Registry, + project: str, +) -> RetrievalJob: + pass + +``` + + + From df95ee84bc4ff24ebbcec8cf3c3749015de9e2e9 Mon Sep 17 00:00:00 2001 From: David Y Liu <7172604+mavysavydav@users.noreply.github.com> Date: Fri, 18 Jun 2021 14:18:16 -0700 Subject: [PATCH 06/43] Schema Inferencing should happen at apply time (#1646) * wip1 Signed-off-by: David Y Liu * just need to do clean up Signed-off-by: David Y Liu * linted Signed-off-by: David Y Liu * improve test coverage Signed-off-by: David Y Liu * changed placement of inference methods in repo_operation apply_total Signed-off-by: David Y Liu * updated inference method name + changed to void return since it updates in place Signed-off-by: David Y Liu * fixed integration test and added comments Signed-off-by: David Y Liu * Made DataSource event_timestamp_column optional Signed-off-by: David Y Liu Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/data_source.py | 64 +++---------------- sdk/python/feast/errors.py | 8 +++ sdk/python/feast/feature_store.py | 10 ++- sdk/python/feast/feature_view.py | 56 ++++++++++------- sdk/python/feast/inference.py | 69 +++++++++++++++++++-- sdk/python/feast/repo_operations.py | 10 ++- sdk/python/tests/test_inference.py | 47 +++++++++----- sdk/python/tests/utils/data_source_utils.py | 6 +- 8 files changed, 167 insertions(+), 103 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 96ef9e46a0a..44badcb83b6 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -14,7 +14,6 @@ import enum -import re from typing import Callable, Dict, Iterable, Optional, Tuple from pyarrow.parquet import ParquetFile @@ -371,7 +370,7 @@ class DataSource: def __init__( self, - event_timestamp_column: str, + event_timestamp_column: Optional[str] = "", created_timestamp_column: Optional[str] = "", field_mapping: Optional[Dict[str, str]] = None, date_partition_column: Optional[str] = "", @@ -520,45 +519,11 @@ def to_proto(self) -> DataSourceProto: """ raise NotImplementedError - def _infer_event_timestamp_column(self, ts_column_type_regex_pattern): - ERROR_MSG_PREFIX = "Unable to infer DataSource event_timestamp_column" - USER_GUIDANCE = "Please specify event_timestamp_column explicitly." - - if isinstance(self, FileSource) or isinstance(self, BigQuerySource): - event_timestamp_column, matched_flag = None, False - for col_name, col_datatype in self.get_table_column_names_and_types(): - if re.match(ts_column_type_regex_pattern, col_datatype): - if matched_flag: - raise TypeError( - f""" - {ERROR_MSG_PREFIX} due to multiple possible columns satisfying - the criteria. {USER_GUIDANCE} - """ - ) - matched_flag = True - event_timestamp_column = col_name - if matched_flag: - return event_timestamp_column - else: - raise TypeError( - f""" - {ERROR_MSG_PREFIX} due to an absence of columns that satisfy the criteria. - {USER_GUIDANCE} - """ - ) - else: - raise TypeError( - f""" - {ERROR_MSG_PREFIX} because this DataSource currently does not support this inference. - {USER_GUIDANCE} - """ - ) - class FileSource(DataSource): def __init__( self, - event_timestamp_column: Optional[str] = None, + event_timestamp_column: Optional[str] = "", file_url: Optional[str] = None, path: Optional[str] = None, file_format: FileFormat = None, @@ -598,7 +563,7 @@ def __init__( self._file_options = FileOptions(file_format=file_format, file_url=file_url) super().__init__( - event_timestamp_column or self._infer_event_timestamp_column(r"^timestamp"), + event_timestamp_column, created_timestamp_column, field_mapping, date_partition_column, @@ -662,7 +627,7 @@ def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: class BigQuerySource(DataSource): def __init__( self, - event_timestamp_column: Optional[str] = None, + event_timestamp_column: Optional[str] = "", table_ref: Optional[str] = None, created_timestamp_column: Optional[str] = "", field_mapping: Optional[Dict[str, str]] = None, @@ -672,8 +637,7 @@ def __init__( self._bigquery_options = BigQueryOptions(table_ref=table_ref, query=query) super().__init__( - event_timestamp_column - or self._infer_event_timestamp_column("TIMESTAMP|DATETIME"), + event_timestamp_column, created_timestamp_column, field_mapping, date_partition_column, @@ -743,20 +707,12 @@ def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: from google.cloud import bigquery client = bigquery.Client() - name_type_pairs = [] if self.table_ref is not None: - project_id, dataset_id, table_id = self.table_ref.split(".") - bq_columns_query = f""" - SELECT COLUMN_NAME, DATA_TYPE FROM {project_id}.{dataset_id}.INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_NAME = '{table_id}' - """ - table_schema = ( - client.query(bq_columns_query).result().to_dataframe_iterable() - ) - for df in table_schema: - name_type_pairs.extend( - list(zip(df["COLUMN_NAME"].to_list(), df["DATA_TYPE"].to_list())) - ) + table_schema = client.get_table(self.table_ref).schema + if not isinstance(table_schema[0], bigquery.schema.SchemaField): + raise TypeError("Could not parse BigQuery table schema.") + + name_type_pairs = [(field.name, field.field_type) for field in table_schema] else: bq_columns_query = f"SELECT * FROM ({self.query}) LIMIT 1" queryRes = client.query(bq_columns_query).result() diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 53a4f22cf52..b55fe61df37 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -101,3 +101,11 @@ def __init__( f"The DataFrame from {source} being materialized must have at least {join_key_columns} columns present, " f"but these were missing: {join_key_columns - source_columns} " ) + + +class RegistryInferenceFailure(Exception): + def __init__(self, repo_obj_type: str, specific_issue: str): + super().__init__( + f"Inference to fill in missing information for {repo_obj_type} failed. {specific_issue}. " + "Try filling the information explicitly." + ) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index dd08586be35..e9b9263bc8b 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -30,7 +30,10 @@ FeatureViewNotFoundException, ) from feast.feature_view import FeatureView -from feast.inference import infer_entity_value_type_from_feature_views +from feast.inference import ( + infer_entity_value_type_from_feature_views, + update_data_sources_with_inferred_event_timestamp_col, +) from feast.infra.provider import Provider, RetrievalJob, get_provider from feast.online_response import OnlineResponse, _infer_online_entity_rows from feast.protos.feast.serving.ServingService_pb2 import ( @@ -228,6 +231,11 @@ def apply( entities_to_update = infer_entity_value_type_from_feature_views( [ob for ob in objects if isinstance(ob, Entity)], views_to_update ) + update_data_sources_with_inferred_event_timestamp_col( + [view.input for view in views_to_update] + ) + for view in views_to_update: + view.infer_features_from_input_source() if len(views_to_update) + len(entities_to_update) != len(objects): raise ValueError("Unknown object type provided as part of apply() call") diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index db756dda795..114bb37e613 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -21,6 +21,7 @@ from feast import utils from feast.data_source import BigQuerySource, DataSource, FileSource +from feast.errors import RegistryInferenceFailure from feast.feature import Feature from feast.protos.feast.core.FeatureView_pb2 import FeatureView as FeatureViewProto from feast.protos.feast.core.FeatureView_pb2 import ( @@ -64,29 +65,6 @@ def __init__( tags: Optional[Dict[str, str]] = None, online: bool = True, ): - if not features: - features = [] # to handle python's mutable default arguments - columns_to_exclude = { - input.event_timestamp_column, - input.created_timestamp_column, - } | set(entities) - - for col_name, col_datatype in input.get_table_column_names_and_types(): - if col_name not in columns_to_exclude and not re.match( - "^__|__$", col_name - ): - features.append( - Feature( - col_name, - input.source_datatype_to_feast_value_type()(col_datatype), - ) - ) - - if not features: - raise ValueError( - f"Could not infer Features for the FeatureView named {name}. Please specify Features explicitly for this FeatureView." - ) - cols = [entity for entity in entities] + [feat.name for feat in features] for col in cols: if input.field_mapping is not None and col in input.field_mapping.keys(): @@ -241,3 +219,35 @@ def most_recent_end_time(self) -> Optional[datetime]: if len(self.materialization_intervals) == 0: return None return max([interval[1] for interval in self.materialization_intervals]) + + def infer_features_from_input_source(self): + if not self.features: + columns_to_exclude = { + self.input.event_timestamp_column, + self.input.created_timestamp_column, + } | set(self.entities) + + for col_name, col_datatype in self.input.get_table_column_names_and_types(): + if col_name not in columns_to_exclude and not re.match( + "^__|__$", + col_name, # double underscores often signal an internal-use column + ): + feature_name = ( + self.input.field_mapping[col_name] + if col_name in self.input.field_mapping.keys() + else col_name + ) + self.features.append( + Feature( + feature_name, + self.input.source_datatype_to_feast_value_type()( + col_datatype + ), + ) + ) + + if not self.features: + raise RegistryInferenceFailure( + "FeatureView", + f"Could not infer Features for the FeatureView named {self.name}.", + ) diff --git a/sdk/python/feast/inference.py b/sdk/python/feast/inference.py index 54105a9bc2c..fac2155ee21 100644 --- a/sdk/python/feast/inference.py +++ b/sdk/python/feast/inference.py @@ -1,6 +1,9 @@ -from typing import List +import re +from typing import List, Union from feast import Entity +from feast.data_source import BigQuerySource, FileSource +from feast.errors import RegistryInferenceFailure from feast.feature_view import FeatureView from feast.value_type import ValueType @@ -45,12 +48,70 @@ def infer_entity_value_type_from_feature_views( entity.value_type != ValueType.UNKNOWN and entity.value_type != inferred_value_type ) or (len(extracted_entity_name_type_pairs) > 1): - raise ValueError( + raise RegistryInferenceFailure( + "Entity", f"""Entity value_type inference failed for {entity_name} entity. - Multiple viable matches. Please explicitly specify the entity value_type - for this entity.""" + Multiple viable matches. + """, ) entity.value_type = inferred_value_type return entities + + +def update_data_sources_with_inferred_event_timestamp_col( + data_sources: List[Union[BigQuerySource, FileSource]], +) -> None: + ERROR_MSG_PREFIX = "Unable to infer DataSource event_timestamp_column" + + for data_source in data_sources: + if ( + data_source.event_timestamp_column is None + or data_source.event_timestamp_column == "" + ): + # prepare right match pattern for data source + ts_column_type_regex_pattern = "" + if isinstance(data_source, FileSource): + ts_column_type_regex_pattern = r"^timestamp" + elif isinstance(data_source, BigQuerySource): + ts_column_type_regex_pattern = "TIMESTAMP|DATETIME" + else: + raise RegistryInferenceFailure( + "DataSource", + """ + DataSource inferencing of event_timestamp_column is currently only supported + for FileSource and BigQuerySource. + """, + ) + # for informing the type checker + assert isinstance(data_source, FileSource) or isinstance( + data_source, BigQuerySource + ) + + # loop through table columns to find singular match + event_timestamp_column, matched_flag = None, False + for ( + col_name, + col_datatype, + ) in data_source.get_table_column_names_and_types(): + if re.match(ts_column_type_regex_pattern, col_datatype): + if matched_flag: + raise RegistryInferenceFailure( + "DataSource", + f""" + {ERROR_MSG_PREFIX} due to multiple possible columns satisfying + the criteria. {ts_column_type_regex_pattern} {col_name} + """, + ) + matched_flag = True + event_timestamp_column = col_name + if matched_flag: + data_source.event_timestamp_column = event_timestamp_column + else: + raise RegistryInferenceFailure( + "DataSource", + f""" + {ERROR_MSG_PREFIX} due to an absence of columns that satisfy the criteria. + """, + ) diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 63ed5c74d72..b3cc7fa0c39 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -13,7 +13,10 @@ from feast import Entity, FeatureTable from feast.feature_view import FeatureView -from feast.inference import infer_entity_value_type_from_feature_views +from feast.inference import ( + infer_entity_value_type_from_feature_views, + update_data_sources_with_inferred_event_timestamp_col, +) from feast.infra.offline_stores.helpers import assert_offline_store_supports_data_source from feast.infra.provider import get_provider from feast.names import adjectives, animals @@ -136,6 +139,7 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): ), feature_views=repo.feature_views, ) + sys.dont_write_bytecode = False for entity in repo.entities: registry.apply_entity(entity, project=project) @@ -156,6 +160,10 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): repo_config.offline_store, data_source ) + update_data_sources_with_inferred_event_timestamp_col(data_sources) + for view in repo.feature_views: + view.infer_features_from_input_source() + tables_to_delete = [] for registry_table in registry.list_feature_tables(project=project): if registry_table.name not in repo_table_names: diff --git a/sdk/python/tests/test_inference.py b/sdk/python/tests/test_inference.py index 886aca8ab2a..1f626ac3cd3 100644 --- a/sdk/python/tests/test_inference.py +++ b/sdk/python/tests/test_inference.py @@ -6,23 +6,12 @@ ) from feast import Entity, ValueType +from feast.errors import RegistryInferenceFailure from feast.feature_view import FeatureView -from feast.inference import infer_entity_value_type_from_feature_views - - -@pytest.mark.integration -def test_data_source_ts_col_inference_success(simple_dataset_1): - with prep_file_source(df=simple_dataset_1) as file_source: - actual_file_source = file_source.event_timestamp_column - actual_bq_1 = simple_bq_source_using_table_ref_arg( - simple_dataset_1 - ).event_timestamp_column - actual_bq_2 = simple_bq_source_using_query_arg( - simple_dataset_1 - ).event_timestamp_column - expected = "ts_1" - - assert expected == actual_file_source == actual_bq_1 == actual_bq_2 +from feast.inference import ( + infer_entity_value_type_from_feature_views, + update_data_sources_with_inferred_event_timestamp_col, +) def test_infer_entity_value_type_from_feature_views(simple_dataset_1, simple_dataset_2): @@ -44,6 +33,30 @@ def test_infer_entity_value_type_from_feature_views(simple_dataset_1, simple_dat assert actual_1 == [Entity(name="id", value_type=ValueType.INT64)] assert actual_2 == [Entity(name="id", value_type=ValueType.STRING)] - with pytest.raises(ValueError): + with pytest.raises(RegistryInferenceFailure): # two viable data types infer_entity_value_type_from_feature_views([Entity(name="id")], [fv1, fv2]) + + +@pytest.mark.integration +def test_infer_event_timestamp_column_for_data_source(simple_dataset_1): + df_with_two_viable_timestamp_cols = simple_dataset_1.copy(deep=True) + df_with_two_viable_timestamp_cols["ts_2"] = simple_dataset_1["ts_1"] + + with prep_file_source(df=simple_dataset_1) as file_source: + data_sources = [ + file_source, + simple_bq_source_using_table_ref_arg(simple_dataset_1), + simple_bq_source_using_query_arg(simple_dataset_1), + ] + update_data_sources_with_inferred_event_timestamp_col(data_sources) + actual_event_timestamp_cols = [ + source.event_timestamp_column for source in data_sources + ] + + assert actual_event_timestamp_cols == ["ts_1", "ts_1", "ts_1"] + + with prep_file_source(df=df_with_two_viable_timestamp_cols) as file_source: + with pytest.raises(RegistryInferenceFailure): + # two viable event_timestamp_columns + update_data_sources_with_inferred_event_timestamp_col([file_source]) diff --git a/sdk/python/tests/utils/data_source_utils.py b/sdk/python/tests/utils/data_source_utils.py index 0aec0c6f1a8..c848b8ea647 100644 --- a/sdk/python/tests/utils/data_source_utils.py +++ b/sdk/python/tests/utils/data_source_utils.py @@ -8,7 +8,7 @@ @contextlib.contextmanager -def prep_file_source(df, event_timestamp_column="") -> FileSource: +def prep_file_source(df, event_timestamp_column=None) -> FileSource: with tempfile.NamedTemporaryFile(suffix=".parquet") as f: f.close() df.to_parquet(f.name) @@ -21,7 +21,7 @@ def prep_file_source(df, event_timestamp_column="") -> FileSource: def simple_bq_source_using_table_ref_arg( - df, event_timestamp_column="" + df, event_timestamp_column=None ) -> BigQuerySource: client = bigquery.Client() gcp_project = client.project @@ -46,7 +46,7 @@ def simple_bq_source_using_table_ref_arg( ) -def simple_bq_source_using_query_arg(df, event_timestamp_column="") -> BigQuerySource: +def simple_bq_source_using_query_arg(df, event_timestamp_column=None) -> BigQuerySource: bq_source_using_table_ref = simple_bq_source_using_table_ref_arg( df, event_timestamp_column ) From e38357545553083e83cc4fab5a74c36239a9e7b7 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sat, 19 Jun 2021 23:31:00 +0000 Subject: [PATCH 07/43] GitBook: [master] 80 pages modified Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- docs/SUMMARY.md | 28 +++- docs/concepts/architecture-and-components.md | 8 +- docs/concepts/data-model-and-concepts.md | 124 +---------------- docs/concepts/feature-view.md | 71 ++++++++++ docs/concepts/offline-store.md | 15 +++ docs/concepts/online-store.md | 16 +++ docs/concepts/overview.md | 21 ++- docs/concepts/provider.md | 10 ++ .../create-a-feature-repository.md | 2 +- .../getting-started/deploy-a-feature-store.md | 2 +- docs/quickstart.md | 2 +- docs/reference/data-sources/README.md | 10 ++ docs/reference/data-sources/bigquery.md | 34 +++++ docs/reference/data-sources/file.md | 20 +++ docs/reference/feature-repository/README.md | 125 ++++++++++++++++++ .../feature-repository/feast-ignore.md | 33 +++++ .../feature-repository/feature-store-yaml.md | 29 ++++ docs/reference/offline-stores/README.md | 8 ++ docs/reference/offline-stores/file.md | 23 ++++ docs/reference/offline-stores/untitled.md | 26 ++++ docs/reference/online-stores/README.md | 10 ++ docs/reference/online-stores/datastore.md | 22 +++ docs/reference/online-stores/redis.md | 24 ++++ docs/reference/online-stores/sqlite.md | 24 ++++ docs/reference/providers/README.md | 8 ++ .../providers/google-cloud-platform.md | 102 ++++++++++++++ docs/reference/providers/local.md | 17 +++ docs/reference/telemetry.md | 2 +- docs/roadmap.md | 2 +- 29 files changed, 668 insertions(+), 150 deletions(-) create mode 100644 docs/concepts/feature-view.md create mode 100644 docs/concepts/offline-store.md create mode 100644 docs/concepts/online-store.md create mode 100644 docs/concepts/provider.md create mode 100644 docs/reference/data-sources/README.md create mode 100644 docs/reference/data-sources/bigquery.md create mode 100644 docs/reference/data-sources/file.md create mode 100644 docs/reference/feature-repository/README.md create mode 100644 docs/reference/feature-repository/feast-ignore.md create mode 100644 docs/reference/feature-repository/feature-store-yaml.md create mode 100644 docs/reference/offline-stores/README.md create mode 100644 docs/reference/offline-stores/file.md create mode 100644 docs/reference/offline-stores/untitled.md create mode 100644 docs/reference/online-stores/README.md create mode 100644 docs/reference/online-stores/datastore.md create mode 100644 docs/reference/online-stores/redis.md create mode 100644 docs/reference/online-stores/sqlite.md create mode 100644 docs/reference/providers/README.md create mode 100644 docs/reference/providers/google-cloud-platform.md create mode 100644 docs/reference/providers/local.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 9288a077182..84da9a4ad30 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -15,15 +15,33 @@ ## Concepts -* [Data model and concepts](concepts/data-model-and-concepts.md) -* [Architecture and components](concepts/architecture-and-components.md) +* [Overview](concepts/overview.md) +* [Feature view](concepts/feature-view.md) +* [Data model](concepts/data-model-and-concepts.md) +* [Online Store](concepts/online-store.md) +* [Offline Store](concepts/offline-store.md) +* [Provider](concepts/provider.md) +* [Architecture](concepts/architecture-and-components.md) ## Reference -* [Feature repository](reference/feature-repository.md) -* [feature\_store.yaml](reference/feature-store-yaml.md) +* [Data Sources](reference/data-sources/README.md) + * [BigQuery](reference/data-sources/bigquery.md) + * [File](reference/data-sources/file.md) +* [Online stores](reference/online-stores/README.md) + * [SQLite](reference/online-stores/sqlite.md) + * [Redis](reference/online-stores/redis.md) + * [Datastore](reference/online-stores/datastore.md) +* [Offline stores](reference/offline-stores/README.md) + * [File](reference/offline-stores/file.md) + * [BigQuery](reference/offline-stores/untitled.md) +* [Providers](reference/providers/README.md) + * [Local](reference/providers/local.md) + * [Google Cloud Platform](reference/providers/google-cloud-platform.md) * [Feast CLI reference](reference/feast-cli-commands.md) -* [.feastignore](reference/feast-ignore.md) +* [Feature repository](reference/feature-repository/README.md) + * [feature\_store.yaml](reference/feature-repository/feature-store-yaml.md) + * [.feastignore](reference/feature-repository/feast-ignore.md) * [Python API reference](http://rtd.feast.dev/) * [Telemetry](reference/telemetry.md) diff --git a/docs/concepts/architecture-and-components.md b/docs/concepts/architecture-and-components.md index 9d6ef3924e8..8c72bdf3b63 100644 --- a/docs/concepts/architecture-and-components.md +++ b/docs/concepts/architecture-and-components.md @@ -1,8 +1,8 @@ -# Architecture and components +# Architecture -![Feast 0.10 Architecture Diagram](../.gitbook/assets/image%20%284%29.png) +![Feast Architecture Diagram](../.gitbook/assets/image%20%284%29.png) -### Functionality +#### Functionality * **Create Batch Features:** ELT/ETL systems like Spark and SQL are used to transform data in the batch store. * **Feast Apply:** The user \(or CI\) publishes versioned controlled feature definitions using `feast apply`. This CLI command updates infrastructure and persists definitions in the object store registry. @@ -13,7 +13,7 @@ * **Prediction:** A backend system makes a request for a prediction from the model serving service. * **Get Online Features:** The model serving service makes a request to the Feast Online Serving service for online features using a Feast SDK. -### Components +#### Components A complete Feast deployment contains the following components: diff --git a/docs/concepts/data-model-and-concepts.md b/docs/concepts/data-model-and-concepts.md index 47c8c33810e..270d98e3d3b 100644 --- a/docs/concepts/data-model-and-concepts.md +++ b/docs/concepts/data-model-and-concepts.md @@ -1,87 +1,4 @@ -# Data model and concepts - -### Concepts - -The top-level namespace within Feast is a [project](data-model-and-concepts.md#project). Users define one or more [feature views](data-model-and-concepts.md#feature-view) within a project. Each feature view contains one or more [features](data-model-and-concepts.md#feature) that relate to a specific [entity](data-model-and-concepts.md#entity). A feature view must always have a [data source](data-model-and-concepts.md#data-source). This source is used during the generation of training [datasets](data-model-and-concepts.md#dataset) and when materializing feature values into the online store. - -![](../.gitbook/assets/image%20%287%29.png) - -### Project - -Projects provide complete isolation of feature stores at the infrastructure level. This is accomplished through resource namespacing, e.g., prefixing table names with the associated project. Each project should be considered a completely separate universe of entities and features. It is not possible to retrieve features from multiple projects in a single request. We recommend having a single feature store and a single project per environment \(`dev`, `staging`, `prod`\). - -{% hint style="info" %} -Projects are currently being supported for backward compatibility reasons. The concept and functionality provided by Projects may change in the future as we simplify the Feast API. -{% endhint %} - -### Data Source - -Feast uses a time-series data model to represent data. This data model is used to interpret feature data in data sources in order to build training datasets or when materializing features into an online store. - -Below is an example data source with a single entity \(`driver`\) and two features \(`trips_today`, and `rating`\). - -![Ride-hailing data source](../.gitbook/assets/image%20%2816%29.png) - -### Entity - -An entity is a collection of semantically related features. Users define entities to map to the domain of their use case. For example, a ride-hailing service could have customers and drivers as their entities, which group related features that correspond to these customers and drivers. - -```python -driver = Entity(name='driver', value_type=ValueType.STRING, join_key='driver_id') -``` - -Entities are defined as part of feature views. Entities are used to identify the primary key on which feature values should be stored and retrieved. These keys are used during the lookup of feature values from the online store and the join process in point-in-time joins. It is possible to define composite entities \(more than one entity object\) in a feature view. - -Entities should be reused across feature views. - -### Feature - -A feature is an individual measurable property observed on an entity. For example, a feature of a `customer` entity could be the number of transactions they have made on an average month. - -Features are defined as part of feature views. Since Feast does not transform data, a feature is essentially a schema that only contains a name and a type: - -```python -trips_today = Feature( - name="trips_today", - dtype=ValueType.FLOAT -) -``` - -Together with [data sources](data-model-and-concepts.md#data-source), they indicate to Feast where to find your feature values, e.g., in a specific parquet file or BigQuery table. Feature definitions are also used when reading features from the feature store, using [feature references](data-model-and-concepts.md#feature-references). - -Feature names must be unique within a [feature view](data-model-and-concepts.md#feature-view). - -### Feature View - -A feature view is an object that represents a logical group of time-series feature data as it is found in a data source. Feature views consist of one or more entities, features, and a data source. Feature views allow Feast to model your existing feature data in a consistent way in both an offline \(training\) and online \(serving\) environment. - -{% tabs %} -{% tab title="driver\_trips\_feature\_view.py" %} -```python -driver_stats_fv = FeatureView( - name="driver_activity", - entities=["driver"], - features=[ - Feature(name="trips_today", dtype=ValueType.INT64), - Feature(name="rating", dtype=ValueType.FLOAT), - ], - input=BigQuerySource( - table_ref="feast-oss.demo_data.driver_activity" - ) -) -``` -{% endtab %} -{% endtabs %} - -Feature views are used during - -* The generation of training datasets by querying the data source of feature views in order to find historical feature values. A single training dataset may consist of features from multiple feature views. -* Loading of feature values into an online store. Feature views determine the storage schema in the online store. -* Retrieval of features from the online store. Feature views provide the schema definition to Feast in order to look up features from the online store. - -{% hint style="info" %} -Feast does not generate feature values. It acts as the ingestion and serving system. The data sources described within feature views should reference feature values in their already computed form. -{% endhint %} +# Data model ### Dataset @@ -147,42 +64,3 @@ Example of an entity dataframe with feature values joined to it: ![](../.gitbook/assets/image%20%2817%29.png) -### **Online Store** - -The Feast online store is used for low-latency online feature value lookups. Feature values are loaded into the online store from data sources in feature views using the `materialize` command. - -The storage schema of features within the online store mirrors that of the data source used to populate the online store. One key difference between the online store and data sources is that only the latest feature values are stored per entity key. No historical values are stored. - -Example batch data source - -![](../.gitbook/assets/image%20%286%29.png) - -Once the above data source is materialized into Feast \(using `feast materialize`\), the feature values will be stored as follows: - -![](../.gitbook/assets/image%20%285%29.png) - -### Offline Store - -An offline store is a storage and compute system where historic feature data can be stored or accessed for building training datasets or for sourcing data for materialization into the online store. - -Offline stores are used primarily for two reasons - -1. Building training datasets -2. Querying data sources for feature data in order to load these features into your online store - -Feast does not actively manage your offline store. Instead, you are asked to select an offline store \(like `BigQuery` or the `File` offline store\) and then to introduce batch sources from these stores using [data sources](data-model-and-concepts.md#data-source) inside feature views. - -Feast will use your offline store to query these sources. It is not possible to query all data sources from all offline stores, and only a single offline store can be used at a time. For example, it is not possible to query a BigQuery table from a `File` offline store, nor is it possible for a `BigQuery` offline store to query files in your local file system. - -Please see [feature\_store.yaml](../reference/feature-store-yaml.md#overview) for configuring your offline store. - -### **Provider** - -A provider is an implementation of a feature store using specific feature store components targeting a specific environment**.** More specifically, a provider is the target environment to which you have configured your feature store to deploy and run. - -Providers are built to orchestrate various components \(offline store, online store, infrastructure, compute\) inside an environment. For example, the `gcp` provider may only support `BigQuery` as an offline store and `datastore` as the online store, but it ensures that these components can work together seamlessly. - -Providers also come with default configurations which makes it easier for users to start a feature store in a specific environment. - -Please see [feature\_store.yaml](../reference/feature-store-yaml.md#overview) for configuring a provider. - diff --git a/docs/concepts/feature-view.md b/docs/concepts/feature-view.md new file mode 100644 index 00000000000..8326723fd69 --- /dev/null +++ b/docs/concepts/feature-view.md @@ -0,0 +1,71 @@ +# Feature view + +### Feature View + +A feature view is an object that represents a logical group of time-series feature data as it is found in a [data source](feature-view.md#data-source). Feature views consist of one or more [entities](feature-view.md#entity), [features](feature-view.md#feature), and a [data source](feature-view.md#data-source). Feature views allow Feast to model your existing feature data in a consistent way in both an offline \(training\) and online \(serving\) environment. + +{% tabs %} +{% tab title="driver\_trips\_feature\_view.py" %} +```python +driver_stats_fv = FeatureView( + name="driver_activity", + entities=["driver"], + features=[ + Feature(name="trips_today", dtype=ValueType.INT64), + Feature(name="rating", dtype=ValueType.FLOAT), + ], + input=BigQuerySource( + table_ref="feast-oss.demo_data.driver_activity" + ) +) +``` +{% endtab %} +{% endtabs %} + +Feature views are used during + +* The generation of training datasets by querying the data source of feature views in order to find historical feature values. A single training dataset may consist of features from multiple feature views. +* Loading of feature values into an online store. Feature views determine the storage schema in the online store. +* Retrieval of features from the online store. Feature views provide the schema definition to Feast in order to look up features from the online store. + +{% hint style="info" %} +Feast does not generate feature values. It acts as the ingestion and serving system. The data sources described within feature views should reference feature values in their already computed form. +{% endhint %} + +### Data Source + +Feast uses a time-series data model to represent data. This data model is used to interpret feature data in data sources in order to build training datasets or when materializing features into an online store. + +Below is an example data source with a single entity \(`driver`\) and two features \(`trips_today`, and `rating`\). + +![Ride-hailing data source](../.gitbook/assets/image%20%2816%29.png) + +### Entity + +An entity is a collection of semantically related features. Users define entities to map to the domain of their use case. For example, a ride-hailing service could have customers and drivers as their entities, which group related features that correspond to these customers and drivers. + +```python +driver = Entity(name='driver', value_type=ValueType.STRING, join_key='driver_id') +``` + +Entities are defined as part of feature views. Entities are used to identify the primary key on which feature values should be stored and retrieved. These keys are used during the lookup of feature values from the online store and the join process in point-in-time joins. It is possible to define composite entities \(more than one entity object\) in a feature view. + +Entities should be reused across feature views. + +### Feature + +A feature is an individual measurable property observed on an entity. For example, a feature of a `customer` entity could be the number of transactions they have made on an average month. + +Features are defined as part of feature views. Since Feast does not transform data, a feature is essentially a schema that only contains a name and a type: + +```python +trips_today = Feature( + name="trips_today", + dtype=ValueType.FLOAT +) +``` + +Together with [data sources](data-model-and-concepts.md#data-source), they indicate to Feast where to find your feature values, e.g., in a specific parquet file or BigQuery table. Feature definitions are also used when reading features from the feature store, using [feature references](data-model-and-concepts.md#feature-references). + +Feature names must be unique within a [feature view](data-model-and-concepts.md#feature-view). + diff --git a/docs/concepts/offline-store.md b/docs/concepts/offline-store.md new file mode 100644 index 00000000000..d926dd9da6e --- /dev/null +++ b/docs/concepts/offline-store.md @@ -0,0 +1,15 @@ +# Offline Store + +An offline store is a storage and compute system where historic feature data can be stored or accessed for building training datasets or for sourcing data for materialization into the online store. + +Offline stores are used primarily for two reasons + +1. Building training datasets +2. Querying data sources for feature data in order to load these features into your online store + +Feast does not actively manage your offline store. Instead, you are asked to select an offline store \(like `BigQuery` or the `File` offline store\) and then to introduce batch sources from these stores using [data sources](data-model-and-concepts.md#data-source) inside feature views. + +Feast will use your offline store to query these sources. It is not possible to query all data sources from all offline stores, and only a single offline store can be used at a time. For example, it is not possible to query a BigQuery table from a `File` offline store, nor is it possible for a `BigQuery` offline store to query files in your local file system. + +Please see [feature\_store.yaml](../reference/feature-repository/feature-store-yaml.md#overview) for configuring your offline store. + diff --git a/docs/concepts/online-store.md b/docs/concepts/online-store.md new file mode 100644 index 00000000000..ef3b74a0426 --- /dev/null +++ b/docs/concepts/online-store.md @@ -0,0 +1,16 @@ +# Online Store + +The Feast online store is used for low-latency online feature value lookups. Feature values are loaded into the online store from data sources in feature views using the `materialize` command. + +The storage schema of features within the online store mirrors that of the data source used to populate the online store. One key difference between the online store and data sources is that only the latest feature values are stored per entity key. No historical values are stored. + +Example batch data source + +![](../.gitbook/assets/image%20%286%29.png) + +Once the above data source is materialized into Feast \(using `feast materialize`\), the feature values will be stored as follows: + +![](../.gitbook/assets/image%20%285%29.png) + +### + diff --git a/docs/concepts/overview.md b/docs/concepts/overview.md index 00dccbd4ea2..a721be99c5a 100644 --- a/docs/concepts/overview.md +++ b/docs/concepts/overview.md @@ -1,21 +1,16 @@ # Overview -### Concepts +The top-level namespace within Feast is a [project](data-model-and-concepts.md#project). Users define one or more [feature views](data-model-and-concepts.md#feature-view) within a project. Each feature view contains one or more [features](data-model-and-concepts.md#feature) that relate to a specific [entity](data-model-and-concepts.md#entity). A feature view must always have a [data source](data-model-and-concepts.md#data-source), which in turn is used during the generation of training [datasets](data-model-and-concepts.md#dataset) and when materializing feature values into the online store. -[Entities](entities.md) are objects in an organization like customers, transactions, and drivers, products, etc. +![](../.gitbook/assets/image%20%287%29.png) -[Sources](sources.md) are external sources of data where feature data can be found. +### Project -[Feature Tables](feature-tables.md) are objects that define logical groupings of features, data sources, and other related metadata. +Projects provide complete isolation of feature stores at the infrastructure level. This is accomplished through resource namespacing, e.g., prefixing table names with the associated project. Each project should be considered a completely separate universe of entities and features. It is not possible to retrieve features from multiple projects in a single request. We recommend having a single feature store and a single project per environment \(`dev`, `staging`, `prod`\). -### Concept Hierarchy +{% hint style="info" %} +Projects are currently being supported for backward compatibility reasons. Projects may change in the future as we simplify the Feast API. +{% endhint %} -![](../.gitbook/assets/image%20%284%29%20%282%29%20%282%29%20%282%29%20%282%29%20%282%29%20%282%29%20%282%29%20%281%29.png) - -Feast contains the following core concepts: - -* **Projects:** Serve as a top level namespace for all Feast resources. Each project is a completely independent environment in Feast. Users can only work in a single project at a time. -* **Entities:** Entities are the objects in an organization on which features occur. They map to your business domain \(users, products, transactions, locations\). -* **Feature Tables:** Defines a group of features that occur on a specific entity. -* **Features:** Individual feature within a feature table. +### diff --git a/docs/concepts/provider.md b/docs/concepts/provider.md new file mode 100644 index 00000000000..8941f61b058 --- /dev/null +++ b/docs/concepts/provider.md @@ -0,0 +1,10 @@ +# Provider + +A provider is an implementation of a feature store using specific feature store components targeting a specific environment**.** More specifically, a provider is the target environment to which you have configured your feature store to deploy and run. + +Providers are built to orchestrate various components \(offline store, online store, infrastructure, compute\) inside an environment. For example, the `gcp` provider supports [BigQuery](https://cloud.google.com/bigquery) as an offline store and [Datastore](https://cloud.google.com/datastore) as an online store, ensuring that these components can work together seamlessly. + +Providers also come with default configurations which makes it easier for users to start a feature store in a specific environment. + +Please see [feature\_store.yaml](../reference/feature-repository/feature-store-yaml.md#overview) for configuring providers. + diff --git a/docs/getting-started/create-a-feature-repository.md b/docs/getting-started/create-a-feature-repository.md index 05bf0f331bf..160a65b7d91 100644 --- a/docs/getting-started/create-a-feature-repository.md +++ b/docs/getting-started/create-a-feature-repository.md @@ -1,6 +1,6 @@ # Create a feature repository -A feature repository is a directory that contains the configuration of the feature store and individual features. This configuration is written as code \(Python/YAML\) and it's highly recommended that teams track it centrally using git. See [Feature Repository](../reference/feature-repository.md) for a detailed explanation of feature repositories. +A feature repository is a directory that contains the configuration of the feature store and individual features. This configuration is written as code \(Python/YAML\) and it's highly recommended that teams track it centrally using git. See [Feature Repository](../reference/feature-repository/) for a detailed explanation of feature repositories. The easiest way to create a new feature repository to use `feast init` command: diff --git a/docs/getting-started/deploy-a-feature-store.md b/docs/getting-started/deploy-a-feature-store.md index 55f28b78a9b..bc61fe4ee27 100644 --- a/docs/getting-started/deploy-a-feature-store.md +++ b/docs/getting-started/deploy-a-feature-store.md @@ -1,6 +1,6 @@ # Deploy a feature store -The Feast CLI can be used to deploy a feature store to your infrastructure, spinning up any necessary persistent resources like buckets or tables in data stores. The deployment target and effects depend on the `provider` that has been configured in your [feature\_store.yaml](../reference/feature-store-yaml.md) file, as well as the feature definitions found in your feature repository. +The Feast CLI can be used to deploy a feature store to your infrastructure, spinning up any necessary persistent resources like buckets or tables in data stores. The deployment target and effects depend on the `provider` that has been configured in your [feature\_store.yaml](../reference/feature-repository/feature-store-yaml.md) file, as well as the feature definitions found in your feature repository. {% hint style="info" %} Here we'll be using the example repository we created in the previous guide, [Create a feature store](create-a-feature-repository.md). You can re-create it by running `feast init` in a new directory. diff --git a/docs/quickstart.md b/docs/quickstart.md index 66d4f0f00d6..7ec2dc7fe03 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -71,7 +71,7 @@ online_store: ``` {% endcode %} -An important field to be aware of is `provider`, which specifies the environment that Feast will run in. We've initialized `provider=local`, indicating that Feast will run the feature store on our local machine. See [Repository Config](reference/feature-store-yaml.md) for more details. +An important field to be aware of is `provider`, which specifies the environment that Feast will run in. We've initialized `provider=local`, indicating that Feast will run the feature store on our local machine. See [Repository Config](reference/feature-repository/feature-store-yaml.md) for more details. Next, take a look at `example.py`, which defines some example features: diff --git a/docs/reference/data-sources/README.md b/docs/reference/data-sources/README.md new file mode 100644 index 00000000000..c1347563968 --- /dev/null +++ b/docs/reference/data-sources/README.md @@ -0,0 +1,10 @@ +# Data Sources + +Please see [Data Source](../../concepts/feature-view.md#data-source) for an explanation of data sources. + +{% page-ref page="bigquery.md" %} + +{% page-ref page="file.md" %} + + + diff --git a/docs/reference/data-sources/bigquery.md b/docs/reference/data-sources/bigquery.md new file mode 100644 index 00000000000..0d6e80424cd --- /dev/null +++ b/docs/reference/data-sources/bigquery.md @@ -0,0 +1,34 @@ +# BigQuery + +### Description + +BigQuery data sources allow for the retrieval of historical feature values from BigQuery for building training datasets as well as materializing features into an online store. + +* Either a table reference or a SQL query can be provided. +* No performance guarantees can be provided over SQL query-based sources. Please use table references where possible. + +### Examples + +Using a table reference + +```python +from feast import BigQuerySource + +my_bigquery_source = BigQuerySource( + table_ref="gcp_project:bq_dataset.bq_table", +) +``` + +Using a query + +```python +from feast import BigQuerySource + +BigQuerySource( + query="SELECT timestamp as ts, created, f1, f2 " + "FROM `my_project.my_dataset.my_features`", +) +``` + +Configuration options are available [here](https://rtd.feast.dev/en/latest/index.html#feast.data_source.BigQuerySource). + diff --git a/docs/reference/data-sources/file.md b/docs/reference/data-sources/file.md new file mode 100644 index 00000000000..f009f5be352 --- /dev/null +++ b/docs/reference/data-sources/file.md @@ -0,0 +1,20 @@ +# File + +### Description + +File data sources allow for the retrieval of historical feature values from files on disk for building training datasets, as well as for materializing features into an online store. + +### Example + +```python +from feast import FileSource +from feast.data_format import ParquetFormat + +parquet_file_source = FileSource( + file_format=ParquetFormat(), + file_url="file:///feast/customer.parquet", +) +``` + +Configuration options are available [here](https://rtd.feast.dev/en/latest/index.html#feast.data_source.FileSource). + diff --git a/docs/reference/feature-repository/README.md b/docs/reference/feature-repository/README.md new file mode 100644 index 00000000000..b5ec5faa8c2 --- /dev/null +++ b/docs/reference/feature-repository/README.md @@ -0,0 +1,125 @@ +# Feature repository + +Feast manages two important sets of configuration: feature definitions, and configuration about how to run the feature store. With Feast, this configuration can be written declaratively and stored as code in a central location. This central location is called a feature repository, and it's essentially just a directory that contains some code files. + +The feature repository is the declarative source of truth for what the desired state of a feature store should be. The Feast CLI uses the feature repository to configure your infrastructure, e.g., migrate tables. + +## What is a feature repository? + +A feature repository consists of: + +* A collection of Python files containing feature declarations. +* A `feature_store.yaml` file containing infrastructural configuration. +* A `.feastignore` file containing paths in the feature repository to ignore. + +{% hint style="info" %} +Typically, users store their feature repositories in a Git repository, especially when working in teams. However, using Git is not a requirement. +{% endhint %} + +## Structure of a feature repository + +The structure of a feature repository is as follows: + +* The root of the repository should contain a `feature_store.yaml` file and may contain a `.feastignore` file. +* The repository should contain Python files that contain feature definitions. +* The repository can contain other files as well, including documentation and potentially data files. + +An example structure of a feature repository is shown below: + +```text +$ tree -a +. +├── data +│ └── driver_stats.parquet +├── driver_features.py +├── feature_store.yaml +└── .feastignore + +1 directory, 4 files +``` + +A couple of things to note about the feature repository: + +* Feast reads _all_ Python files recursively when `feast apply` is ran, including subdirectories, even if they don't contain feature definitions. +* It's recommended to add `.feastignore` and add paths to all imperative scripts if you need to store them inside the feature registry. + +## The feature\_store.yaml configuration file + +The configuration for a feature store is stored in a file named `feature_store.yaml` , which must be located at the root of a feature repository. An example `feature_store.yaml` file is shown below: + +{% code title="feature\_store.yaml" %} +```yaml +project: my_feature_repo_1 +registry: data/metadata.db +provider: local +online_store: + path: data/online_store.db +``` +{% endcode %} + +The `feature_store.yaml` file configures how the feature store should run. See [feature\_store.yaml](feature-store-yaml.md) for more details. + +## The .feastignore file + +This file contains paths that should be ignored when running `feast apply`. An example `.feastignore` is shown below: + +{% code title=".feastignore" %} +```text +# Ignore virtual environment +venv + +# Ignore a specific Python file +scripts/foo.py + +# Ignore all Python files directly under scripts directory +scripts/*.py + +# Ignore all "foo.py" anywhere under scripts directory +scripts/**/foo.py +``` +{% endcode %} + +See [.feastignore](feast-ignore.md) for more details. + +## Feature definitions + +A feature repository can also contain one or more Python files that contain feature definitions. An example feature definition file is shown below: + +{% code title="driver\_features.py" %} +```python +from datetime import timedelta + +from feast import BigQuerySource, Entity, Feature, FeatureView, ValueType + +driver_locations_source = BigQuerySource( + table_ref="rh_prod.ride_hailing_co.drivers", + event_timestamp_column="event_timestamp", + created_timestamp_column="created_timestamp", +) + +driver = Entity( + name="driver", + value_type=ValueType.INT64, + description="driver id", +) + +driver_locations = FeatureView( + name="driver_locations", + entities=["driver"], + ttl=timedelta(days=1), + features=[ + Feature(name="lat", dtype=ValueType.FLOAT), + Feature(name="lon", dtype=ValueType.STRING), + ], + input=driver_locations_source, +) +``` +{% endcode %} + +To declare new feature definitions, just add code to the feature repository, either in existing files or in a new file. For more information on how to define features, see [Feature Views](../../concepts/data-model-and-concepts.md#feature-view). + +### Next steps + +* See [Create a feature repository](../../getting-started/create-a-feature-repository.md) to get started with an example feature repository. +* See [feature\_store.yaml](feature-store-yaml.md), [.feastignore](feast-ignore.md), or [Feature Views](../../concepts/data-model-and-concepts.md#feature-view) for more information on the configuration files that live in a feature registry. + diff --git a/docs/reference/feature-repository/feast-ignore.md b/docs/reference/feature-repository/feast-ignore.md new file mode 100644 index 00000000000..072a44b23de --- /dev/null +++ b/docs/reference/feature-repository/feast-ignore.md @@ -0,0 +1,33 @@ +# .feastignore + +## Overview + +`.feastignore` is a file that is placed at the root of the [Feature Repository](./). This file contains paths that should be ignored when running `feast apply`. An example `.feastignore` is shown below: + +{% code title=".feastignore" %} +```text +# Ignore virtual environment +venv + +# Ignore a specific Python file +scripts/foo.py + +# Ignore all Python files directly under scripts directory +scripts/*.py + +# Ignore all "foo.py" anywhere under scripts directory +scripts/**/foo.py +``` +{% endcode %} + +`.feastignore` file is optional. If the file can not be found, every Python in the feature repo directory will be parsed by `feast apply`. + +## Feast Ignore Patterns + +| Pattern | Example matches | Explanation | +| :--- | :--- | :--- | +| venv | venv/foo.py venv/a/foo.py | You can specify a path to a specific directory. Everything in that directory will be ignored. | +| scripts/foo.py | scripts/foo.py | You can specify a path to a specific file. Only that file will be ignored. | +| scripts/\*.py | scripts/foo.py scripts/bar.py | You can specify an asterisk \(\*\) anywhere in the expression. An asterisk matches zero or more characters, except "/". | +| scripts/\*\*/foo.py | scripts/foo.py scripts/a/foo.py scripts/a/b/foo.py | You can specify a double asterisk \(\*\*\) anywhere in the expression. A double asterisk matches zero or more directories. | + diff --git a/docs/reference/feature-repository/feature-store-yaml.md b/docs/reference/feature-repository/feature-store-yaml.md new file mode 100644 index 00000000000..0f8be6654e6 --- /dev/null +++ b/docs/reference/feature-repository/feature-store-yaml.md @@ -0,0 +1,29 @@ +# feature\_store.yaml + +## Overview + +`feature_store.yaml` is used to configure a feature store. The file must be located at the root of a [feature repository](./). An example `feature_store.yaml` is shown below: + +{% code title="feature\_store.yaml" %} +```yaml +project: loyal_spider +registry: data/registry.db +provider: local +online_store: + type: sqlite + path: data/online_store.db +``` +{% endcode %} + +## Options + +The following top-level configuration options exist in the `feature_store.yaml` file. + +* **provider** — Configures the environment in which Feast will deploy and operate. +* **registry** — Configures the location of the feature registry. +* **online\_store** — Configures the online store. +* **offline\_store** — Configures the offline store. +* **project** — Defines a namespace for the entire feature store. Can be used to isolate multiple deployments in a single installation of Feast. + +Please see the [RepoConfig](https://rtd.feast.dev/en/latest/#feast.repo_config.RepoConfig) API reference for the full list of configuration options. + diff --git a/docs/reference/offline-stores/README.md b/docs/reference/offline-stores/README.md new file mode 100644 index 00000000000..e2f4e1b8fdf --- /dev/null +++ b/docs/reference/offline-stores/README.md @@ -0,0 +1,8 @@ +# Offline stores + +Please see [Offline Store](../../concepts/offline-store.md) for an explanation of offline stores. + +{% page-ref page="file.md" %} + +{% page-ref page="untitled.md" %} + diff --git a/docs/reference/offline-stores/file.md b/docs/reference/offline-stores/file.md new file mode 100644 index 00000000000..1f1d06fdedc --- /dev/null +++ b/docs/reference/offline-stores/file.md @@ -0,0 +1,23 @@ +# File + +### Description + +The File offline store provides support for reading [FileSources](https://github.com/feast-dev/feast/blob/c50a36ec1ad5b8d81c6f773c23204db7c7a7d218/sdk/python/feast/data_source.py#L523). + +* Only Parquet files are currently supported. +* All data is downloaded and joined using Python and may not scale to production workloads. + +### Example + +{% code title="feature\_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +offline_store: + type: file +``` +{% endcode %} + +Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.FileOfflineStoreConfig). + diff --git a/docs/reference/offline-stores/untitled.md b/docs/reference/offline-stores/untitled.md new file mode 100644 index 00000000000..db12cba8337 --- /dev/null +++ b/docs/reference/offline-stores/untitled.md @@ -0,0 +1,26 @@ +# BigQuery + +### Description + +The BigQuery offline store provides support for reading [BigQuerySources](https://github.com/feast-dev/feast/blob/c50a36ec1ad5b8d81c6f773c23204db7c7a7d218/sdk/python/feast/data_source.py#L627). + +* BigQuery tables and views are allowed as sources. +* All joins happen within BigQuery. +* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. Pandas dataframes will be uploaded to BigQuery in order to complete join operations. +* A [BigQueryRetrievalJob](https://github.com/feast-dev/feast/blob/c50a36ec1ad5b8d81c6f773c23204db7c7a7d218/sdk/python/feast/infra/offline_stores/bigquery.py#L210) is returned when calling `get_historical_features()`. + +### Example + +{% code title="feature\_store.yaml" %} +```yaml +project: my_feature_repo +registry: gs://my-bucket/data/registry.db +provider: gcp +offline_store: + type: bigquery + dataset: feast_bq_dataset +``` +{% endcode %} + +Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.BigQueryOfflineStoreConfig). + diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md new file mode 100644 index 00000000000..0a240ffb102 --- /dev/null +++ b/docs/reference/online-stores/README.md @@ -0,0 +1,10 @@ +# Online stores + +Please see [Online Store](../../concepts/online-store.md) for an explanation of online stores. + +{% page-ref page="sqlite.md" %} + +{% page-ref page="redis.md" %} + +{% page-ref page="datastore.md" %} + diff --git a/docs/reference/online-stores/datastore.md b/docs/reference/online-stores/datastore.md new file mode 100644 index 00000000000..730fab5969f --- /dev/null +++ b/docs/reference/online-stores/datastore.md @@ -0,0 +1,22 @@ +# Datastore + +### Description + +The [Datastore](https://cloud.google.com/datastore) online store provides support for materializing feature values into Cloud Datastore. The data model used to store feature values in Datastore is described in more detail [here](https://github.com/feast-dev/feast/blob/master/docs/specs/online_store_format.md#google-datastore-online-store-format). + +### Example + +{% code title="feature\_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: gcp +online_store: + type: datastore + project_id: my_gcp_project + namespace: my_datastore_namespace +``` +{% endcode %} + +Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.DatastoreOnlineStoreConfig). + diff --git a/docs/reference/online-stores/redis.md b/docs/reference/online-stores/redis.md new file mode 100644 index 00000000000..0cff4b37ead --- /dev/null +++ b/docs/reference/online-stores/redis.md @@ -0,0 +1,24 @@ +# Redis + +### Description + +The [Redis](https://redis.io/) online store provides support for materializing feature values into Redis. + +* Both Redis and Redis Cluster are supported +* The data model used to store feature values in Redis is described in more detail [here](https://github.com/feast-dev/feast/blob/master/docs/specs/online_store_format.md). + +### Example + +{% code title="feature\_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: redis + connection_string: localhost:6379 +``` +{% endcode %} + +Configuration options are available [here](https://rtd.feast.dev/en/master/#feast.repo_config.RedisOnlineStoreConfig). + diff --git a/docs/reference/online-stores/sqlite.md b/docs/reference/online-stores/sqlite.md new file mode 100644 index 00000000000..2191b0cc5da --- /dev/null +++ b/docs/reference/online-stores/sqlite.md @@ -0,0 +1,24 @@ +# SQLite + +### Description + +The [SQLite](https://www.sqlite.org/index.html) online store provides support for materializing feature values into an SQLite database for serving online features. + +* All feature values are stored in an on-disk SQLite database +* Only the latest feature values are persisted + +### Example + +{% code title="feature\_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: sqlite + path: data/online_store.db +``` +{% endcode %} + +Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.SqliteOnlineStoreConfig). + diff --git a/docs/reference/providers/README.md b/docs/reference/providers/README.md new file mode 100644 index 00000000000..ffc1c9805f3 --- /dev/null +++ b/docs/reference/providers/README.md @@ -0,0 +1,8 @@ +# Providers + +Please see [Provider](../../concepts/provider.md) for an explanation of providers. + +{% page-ref page="local.md" %} + +{% page-ref page="google-cloud-platform.md" %} + diff --git a/docs/reference/providers/google-cloud-platform.md b/docs/reference/providers/google-cloud-platform.md new file mode 100644 index 00000000000..af6444187df --- /dev/null +++ b/docs/reference/providers/google-cloud-platform.md @@ -0,0 +1,102 @@ +# Google Cloud Platform + +### Description + +* Offline Store: Uses the **BigQuery** offline store by default. Also supports File as the offline store. +* Online Store: Uses the **Datastore** online store by default. Also supports Sqlite as an online store. + +### Example + +{% code title="feature\_store.yaml" %} +```yaml +project: my_feature_repo +registry: gs://my-bucket/data/registry.db +provider: gcp +``` +{% endcode %} + +### **Permissions** + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Command + ComponentPermissionsRecommended Role
Apply + BigQuery (source) +

bigquery.jobs.create

+

bigquery.readsessions.create

+

bigquery.readsessions.getData

+ 		roles/bigquery.user
Apply + Datastore (destination) +

datastore.entities.allocateIds

+

datastore.entities.create

+

datastore.entities.delete

+

datastore.entities.get

+

datastore.entities.list

+

datastore.entities.update

+ 		roles/datastore.owner
Materialize + BigQuery (source)bigquery.jobs.createroles/bigquery.user
Materialize + Datastore (destination) +

datastore.entities.allocateIds

+

datastore.entities.create

+

datastore.entities.delete

+

datastore.entities.get

+

datastore.entities.list

+

datastore.entities.update

+

datastore.databases.get

+ 		roles/datastore.owner
Get Online Features + Datastoredatastore.entities.getroles/datastore.user
Get Historical Features + BigQuery (source) +

bigquery.datasets.get

+

bigquery.tables.get

+

bigquery.tables.create

+

bigquery.tables.updateData

+

bigquery.tables.update

+

bigquery.tables.delete

+

bigquery.tables.getData

+ 		roles/bigquery.dataEditor
+ diff --git a/docs/reference/providers/local.md b/docs/reference/providers/local.md new file mode 100644 index 00000000000..210be132ad6 --- /dev/null +++ b/docs/reference/providers/local.md @@ -0,0 +1,17 @@ +# Local + +### Description + +* Offline Store: Uses the File offline store by default. Also supports BigQuery as the offline store. +* Online Store: Uses the Sqlite online store by default. Also supports Datastore as an online store. + +### Example + +{% code title="feature\_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +``` +{% endcode %} + diff --git a/docs/reference/telemetry.md b/docs/reference/telemetry.md index bd7f7e5954f..96dcb71129a 100644 --- a/docs/reference/telemetry.md +++ b/docs/reference/telemetry.md @@ -6,5 +6,5 @@ The Feast maintainers use anonymous usage statistics and error tracking to help ## How to disable telemetry -To opt out of all telemetry, simply set the environment variable `FEAST_TELEMETRY` to `False` in the environment in which the Feast client is run. +To opt-out of all telemetry, simply set the environment variable `FEAST_TELEMETRY` to `False` in the environment in which the Feast SDK/CLI is run. diff --git a/docs/roadmap.md b/docs/roadmap.md index d1486e1c40e..a11a9b82cef 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2,7 +2,7 @@ ## Backlog -* Add On demand transformations support +* Add On-demand transformations support * Add Data quality monitoring * Add Snowflake offline store support * Add Bigtable support From dd25ad6d4f1c6921a39c5e279b534072f5459fa9 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Sun, 20 Jun 2021 00:06:46 +0000 Subject: [PATCH 08/43] GitBook: [master] 80 pages modified Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- docs/SUMMARY.md | 14 +- docs/concepts/offline-store.md | 14 +- docs/concepts/online-store.md | 4 +- docs/quickstart.md | 235 +++++----------------- docs/reference/data-sources/README.md | 2 +- docs/reference/feast-cli-commands.md | 4 - docs/reference/offline-stores/file.md | 2 +- docs/reference/offline-stores/untitled.md | 2 +- docs/reference/online-stores/redis.md | 20 +- docs/reference/telemetry.md | 10 +- 10 files changed, 95 insertions(+), 212 deletions(-) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 84da9a4ad30..30b19cfdcc8 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -18,30 +18,30 @@ * [Overview](concepts/overview.md) * [Feature view](concepts/feature-view.md) * [Data model](concepts/data-model-and-concepts.md) -* [Online Store](concepts/online-store.md) -* [Offline Store](concepts/offline-store.md) +* [Online store](concepts/online-store.md) +* [Offline store](concepts/offline-store.md) * [Provider](concepts/provider.md) * [Architecture](concepts/architecture-and-components.md) ## Reference -* [Data Sources](reference/data-sources/README.md) +* [Data sources](reference/data-sources/README.md) * [BigQuery](reference/data-sources/bigquery.md) * [File](reference/data-sources/file.md) +* [Offline stores](reference/offline-stores/README.md) + * [File](reference/offline-stores/file.md) + * [BigQuery](reference/offline-stores/untitled.md) * [Online stores](reference/online-stores/README.md) * [SQLite](reference/online-stores/sqlite.md) * [Redis](reference/online-stores/redis.md) * [Datastore](reference/online-stores/datastore.md) -* [Offline stores](reference/offline-stores/README.md) - * [File](reference/offline-stores/file.md) - * [BigQuery](reference/offline-stores/untitled.md) * [Providers](reference/providers/README.md) * [Local](reference/providers/local.md) * [Google Cloud Platform](reference/providers/google-cloud-platform.md) -* [Feast CLI reference](reference/feast-cli-commands.md) * [Feature repository](reference/feature-repository/README.md) * [feature\_store.yaml](reference/feature-repository/feature-store-yaml.md) * [.feastignore](reference/feature-repository/feast-ignore.md) +* [Feast CLI reference](reference/feast-cli-commands.md) * [Python API reference](http://rtd.feast.dev/) * [Telemetry](reference/telemetry.md) diff --git a/docs/concepts/offline-store.md b/docs/concepts/offline-store.md index d926dd9da6e..a5f059ba184 100644 --- a/docs/concepts/offline-store.md +++ b/docs/concepts/offline-store.md @@ -1,15 +1,15 @@ -# Offline Store +# Offline store -An offline store is a storage and compute system where historic feature data can be stored or accessed for building training datasets or for sourcing data for materialization into the online store. +Feast uses offline stores as storage and compute systems. Offline stores store historic time-series feature values. Feast does not generate these features, but instead uses the offline store as the interface for querying existing features in your organization. Offline stores are used primarily for two reasons -1. Building training datasets -2. Querying data sources for feature data in order to load these features into your online store +1. Building training datasets from time-series features. +2. Materializing \(loading\) features from the offline store into an online store in order to serve those features at low latency for prediction. -Feast does not actively manage your offline store. Instead, you are asked to select an offline store \(like `BigQuery` or the `File` offline store\) and then to introduce batch sources from these stores using [data sources](data-model-and-concepts.md#data-source) inside feature views. +Offline stores are configured through the [feature\_store.yaml](../reference/offline-stores/). When building training datasets or materializing features into an online store, Feast will use the configured offline store along with the data sources you have defined as part of feature views to execute the necessary data operations. -Feast will use your offline store to query these sources. It is not possible to query all data sources from all offline stores, and only a single offline store can be used at a time. For example, it is not possible to query a BigQuery table from a `File` offline store, nor is it possible for a `BigQuery` offline store to query files in your local file system. +It is not possible to query all data sources from all offline stores, and only a single offline store can be used at a time. For example, it is not possible to query a BigQuery table from a `File` offline store, nor is it possible for a `BigQuery` offline store to query files from your local file system. -Please see [feature\_store.yaml](../reference/feature-repository/feature-store-yaml.md#overview) for configuring your offline store. +Please see the [Offline Stores](../reference/offline-stores/) reference for more details on configuring offline stores. diff --git a/docs/concepts/online-store.md b/docs/concepts/online-store.md index ef3b74a0426..9830701d1e7 100644 --- a/docs/concepts/online-store.md +++ b/docs/concepts/online-store.md @@ -1,4 +1,4 @@ -# Online Store +# Online store The Feast online store is used for low-latency online feature value lookups. Feature values are loaded into the online store from data sources in feature views using the `materialize` command. @@ -12,5 +12,3 @@ Once the above data source is materialized into Feast \(using `feast materialize ![](../.gitbook/assets/image%20%285%29.png) -### - diff --git a/docs/quickstart.md b/docs/quickstart.md index 7ec2dc7fe03..1fdb6f42d17 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,136 +1,40 @@ # Quickstart -Welcome to the Feast quickstart! This quickstart is intended to get you up and running with Feast in your local environment. It covers the following workflows: +In this tutorial we will -1. Setting up Feast -2. Registering features -3. Constructing training datasets from offline data -4. Materializing feature data to the online feature store -5. Fetching feature vectors for real-time inference +1. Deploy a local feature store with a **Parquet file offline store** and **Sqlite online store**. +2. Build a training dataset using our time series features from our **Parquet files**. +3. Materialize feature values from the offline store into the online store. +4. Read the latest features from the online store for inference. -This quickstart uses some example data about a ride-hailing app to walk through Feast. Let's get into it! +### Install Feast -## 1. Setting up Feast - -A Feast installation includes a Python SDK and a CLI. Both can be installed from `pip`: +Install the Feast SDK and CLI using pip: ```bash pip install feast ``` -You can test your installation by running`feast version` from your command line: - -```bash -$ feast version - -# 0.10 -``` - -## 2. Registering features to Feast +### Create a feature repository -We can bootstrap a feature repository using the `feast init` command: - -```bash -feast init feature_repo - -# Creating a new Feast repository in /feature_repo. -``` - -This command generates an example repository containing the following files. - -{% code title="CLI" %} -```bash -tree - -# . -# └── feature_repo -# ├── data -# │ └── driver_stats.parquet -# ├── example.py -# └── feature_store.yaml -``` -{% endcode %} +Bootstrap a new feature repository using `feast init` from the command line: -Now, let's take a look at these files. First, `cd` into the feature repository: - -{% code title="CLI" %} ```text +feast init feature_repo cd feature_repo ``` -{% endcode %} - -Next, take a look at the `feature_store.yaml` file, which configures how the feature store runs: - -{% code title="feature\_store.yaml" %} -```yaml -project: feature_repo -registry: data/registry.db -provider: local -online_store: - path: data/online_store.db -``` -{% endcode %} - -An important field to be aware of is `provider`, which specifies the environment that Feast will run in. We've initialized `provider=local`, indicating that Feast will run the feature store on our local machine. See [Repository Config](reference/feature-repository/feature-store-yaml.md) for more details. - -Next, take a look at `example.py`, which defines some example features: -{% code title="example.py" %} -```python -# This is an example feature definition file - -from google.protobuf.duration_pb2 import Duration - -from feast import Entity, Feature, FeatureView, ValueType -from feast.data_source import FileSource - -# Read data from parquet files. Parquet is convenient for local development mode. For -# production, you can use your favorite DWH, such as BigQuery. See Feast documentation -# for more info. -driver_hourly_stats = FileSource( - path="//feature_repo/data/driver_stats.parquet", - event_timestamp_column="datetime", - created_timestamp_column="created", -) - -# Define an entity for the driver. You can think of entity as a primary key used to -# fetch features. -driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id",) - -# Our parquet files contain sample data that includes a driver_id column, timestamps and -# three feature column. Here we define a Feature View that will allow us to serve this -# data to our model online. -driver_hourly_stats_view = FeatureView( - name="driver_hourly_stats", - entities=["driver_id"], - ttl=Duration(seconds=86400 * 1), - features=[ - Feature(name="conv_rate", dtype=ValueType.FLOAT), - Feature(name="acc_rate", dtype=ValueType.FLOAT), - Feature(name="avg_daily_trips", dtype=ValueType.INT64), - ], - online=True, - input=driver_hourly_stats, - tags={}, -) +```text +Creating a new Feast repository in /home/Jovyan/feature_repo. ``` -{% endcode %} - -There are three objects defined in this file: - -* A `DataSource`, which is a pointer to persistent feature data. In this example, we're using a `FileSource`, which points to a set of parquet files on our local machine. -* An `Entity`, which is a metadata object that is used to organize and join features. In this example, our entity is `driver_id`, indicating that our features are modeling attributes of drivers. -* A `FeatureView`, which defines a group of features. In this example, our features are statistics about drivers, like their conversion rate and average daily trips. -Feature definitions in Feast work similarly to Terraform: local definitions don't actually affect what's running in production until we explicitly register them with Feast. At this point, we have a set of feature definitions, but we haven't registered them with Feast yet. +### Register feature definitions and deploy your feature store -We can register our features by running `feast apply` from the CLI: +The `apply` command registers all the objects in your feature repository and deploys a feature store: -{% code title="CLI" %} ```bash feast apply ``` -{% endcode %} ```text Registered entity driver_id @@ -138,90 +42,62 @@ Registered feature view driver_hourly_stats Deploying infrastructure for driver_hourly_stats ``` -This command has registered our features to Feast. They're now ready for offline retrieval and materialization. +### Generating training data -## 3. Generating training data +The `apply` command builds a training dataset based on the time-series features defined in the feature repository: -Feast generates point-in-time accurate training data. In our ride-hailing example, we are using statistics about drivers to predict the likelihood of a booking completion. When we generate training data, we want to know what the features of the drivers were _at the time of prediction_ \(in the past.\) - -![](.gitbook/assets/ride-hailing.png) - -Generating training datasets is a workflow best done from an interactive computing environment, like a Jupyter notebook. You can start a Jupyter notebook by running `jupyter notebook` from the command line. Then, run the following code to generate an _entity DataFrame_: - -{% code title="jupyter notebook" %} ```python -import pandas as pd from datetime import datetime -# entity_df generally comes from upstream systems -entity_df = pd.DataFrame.from_dict({ - "driver_id": [1001, 1002, 1003, 1004], - "event_timestamp": [ - datetime(2021, 4, 12, 10, 59, 42), - datetime(2021, 4, 12, 8, 12, 10), - datetime(2021, 4, 12, 16, 40, 26), - datetime(2021, 4, 12, 15, 1 , 12) - ] -}) - -entity_df.head() -``` -{% endcode %} - -![](.gitbook/assets/feast-landing-page-blog-post-page-5%20%281%29%20%281%29%20%281%29%20%282%29%20%282%29%20%285%29%20%287%29%20%287%29%20%283%29%20%287%29.png) - -This DataFrame represents the entity keys and timestamps that we want feature values for. We can pass this Entity DataFrame into Feast, and Feast will fetch point-in-time correct features for each row: +import pandas as pd -{% code title="jupyter notebook" %} -```python from feast import FeatureStore +entity_df = pd.DataFrame.from_dict( + { + "driver_id": [1001, 1002, 1003, 1004], + "event_timestamp": [ + datetime(2021, 4, 12, 10, 59, 42), + datetime(2021, 4, 12, 8, 12, 10), + datetime(2021, 4, 12, 16, 40, 26), + datetime(2021, 4, 12, 15, 1, 12), + ], + } +) + store = FeatureStore(repo_path=".") training_df = store.get_historical_features( - entity_df=entity_df, - feature_refs = [ - 'driver_hourly_stats:conv_rate', - 'driver_hourly_stats:acc_rate', - 'driver_hourly_stats:avg_daily_trips' + entity_df=entity_df, + feature_refs=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", ], ).to_df() -training_df.head() +print(training_df.head()) ``` -{% endcode %} - -![\(These feature values are non-deterministic, by the way.\)](.gitbook/assets/feast-landing-page-blog-post-feature-df.png) - -Feast has joined on the correct feature values for the drivers that specified, as of the timestamp we specified. - -This DataFrame contains all the necessary signals needed to train a model, excluding labels, which are typically managed outside of Feast. Before you can train a model, you'll need to join on labels from external systems. -## 4. Materializing features to the online store - -We have just seen how we can use Feast in the model training workflow. Now, we'll see how Feast fits into the model inferencing workflow. +```bash +event_timestamp driver_id driver_hourly_stats__conv_rate driver_hourly_stats__acc_rate driver_hourly_stats__avg_daily_trips +2021-04-12 1002 0.328245 0.993218 329 +2021-04-12 1001 0.448272 0.873785 767 +2021-04-12 1004 0.822571 0.571790 673 +2021-04-12 1003 0.556326 0.605357 335 +``` -When running inference on Feast features, the first step is to populate the online store to make our features available for real-time inference. When using the `local` provider, the online store is a SQLite database. +### Load features into your online store -To materialize features, run the following command from the CLI: +The `materialize` command loads the latest feature values from your feature views into your online store: -{% code title="CLI" %} ```bash CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%S") feast materialize-incremental $CURRENT_TIME - -# Materializing feature view driver_hourly_stats from 2021-04-13 23:50:05.754655-04:00 -# to 2021-04-14 23:50:04-04:00 done! ``` -{% endcode %} - -We've just populated the online store with the most recent features from the offline store. Our feature values are now ready for real-time retrieval. -## 5. Fetching feature vectors for inference +### Fetching feature vectors for inference -After we materialize our features, we can use the `store.get_online_features` to fetch the latest feature values for real-time inference: - -{% code title="jupyter notebook" %} ```python from pprint import pprint from feast import FeatureStore @@ -230,18 +106,17 @@ store = FeatureStore(repo_path=".") feature_vector = store.get_online_features( feature_refs=[ - 'driver_hourly_stats:conv_rate', - 'driver_hourly_stats:acc_rate', - 'driver_hourly_stats:avg_daily_trips' + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", ], - entity_rows=[{"driver_id": 1001}] + entity_rows=[{"driver_id": 1001}], ).to_dict() pprint(feature_vector) ``` -{% endcode %} -```text +```python { 'driver_id': [1001], 'conv_rate': [0.49274], @@ -250,12 +125,8 @@ pprint(feature_vector) } ``` -This feature vector can be used for real-time inference, for example, in a model serving microservice. - -## Next steps - -This quickstart covered the essential workflows of using Feast in your local environment. The next step is to `pip install "feast[gcp]"` and set `provider="gcp"` in your `feature_store.yaml` file and push your work to production deployment. You can also use the `feast init -t gcp` command in the CLI to initialize a feature repository with example features in the GCP environment. +### Next steps -* See [Create a feature repository](getting-started/create-a-feature-repository.md) for more information on the workflows we covered. -* Join our [Slack group](https://slack.feast.dev) to talk to other Feast users and the maintainers! +* Follow our [Getting Started](getting-started/) guide for a hands tutorial in using Feast +* Join other Feast users and contributors in [Slack](https://slack.feast.dev/) and become part of the community! diff --git a/docs/reference/data-sources/README.md b/docs/reference/data-sources/README.md index c1347563968..ef4fcaa33b0 100644 --- a/docs/reference/data-sources/README.md +++ b/docs/reference/data-sources/README.md @@ -1,4 +1,4 @@ -# Data Sources +# Data sources Please see [Data Source](../../concepts/feature-view.md#data-source) for an explanation of data sources. diff --git a/docs/reference/feast-cli-commands.md b/docs/reference/feast-cli-commands.md index 6b654a80e22..423c9bcd455 100644 --- a/docs/reference/feast-cli-commands.md +++ b/docs/reference/feast-cli-commands.md @@ -165,7 +165,3 @@ Print the current Feast version feast version ``` - - -## - diff --git a/docs/reference/offline-stores/file.md b/docs/reference/offline-stores/file.md index 1f1d06fdedc..b4ce3b1b668 100644 --- a/docs/reference/offline-stores/file.md +++ b/docs/reference/offline-stores/file.md @@ -2,7 +2,7 @@ ### Description -The File offline store provides support for reading [FileSources](https://github.com/feast-dev/feast/blob/c50a36ec1ad5b8d81c6f773c23204db7c7a7d218/sdk/python/feast/data_source.py#L523). +The File offline store provides support for reading [FileSources](../data-sources/file.md). * Only Parquet files are currently supported. * All data is downloaded and joined using Python and may not scale to production workloads. diff --git a/docs/reference/offline-stores/untitled.md b/docs/reference/offline-stores/untitled.md index db12cba8337..8ffa566a70f 100644 --- a/docs/reference/offline-stores/untitled.md +++ b/docs/reference/offline-stores/untitled.md @@ -2,7 +2,7 @@ ### Description -The BigQuery offline store provides support for reading [BigQuerySources](https://github.com/feast-dev/feast/blob/c50a36ec1ad5b8d81c6f773c23204db7c7a7d218/sdk/python/feast/data_source.py#L627). +The BigQuery offline store provides support for reading [BigQuerySources](../data-sources/bigquery.md). * BigQuery tables and views are allowed as sources. * All joins happen within BigQuery. diff --git a/docs/reference/online-stores/redis.md b/docs/reference/online-stores/redis.md index 0cff4b37ead..adcff9a8ea4 100644 --- a/docs/reference/online-stores/redis.md +++ b/docs/reference/online-stores/redis.md @@ -7,7 +7,22 @@ The [Redis](https://redis.io/) online store provides support for materializing f * Both Redis and Redis Cluster are supported * The data model used to store feature values in Redis is described in more detail [here](https://github.com/feast-dev/feast/blob/master/docs/specs/online_store_format.md). -### Example +### Examples + +Connecting to a single Redis instance + +{% code title="feature\_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: redis + connection_string: "localhost:6379" +``` +{% endcode %} + +Connecting to a Redis Cluster with SSL enabled and password authentication {% code title="feature\_store.yaml" %} ```yaml @@ -16,7 +31,8 @@ registry: data/registry.db provider: local online_store: type: redis - connection_string: localhost:6379 + redis_type: redis_cluster + connection_string: "redis1:6379,redis2:6379,ssl=true,password=my_password" ``` {% endcode %} diff --git a/docs/reference/telemetry.md b/docs/reference/telemetry.md index 96dcb71129a..f8f76787645 100644 --- a/docs/reference/telemetry.md +++ b/docs/reference/telemetry.md @@ -1,10 +1,12 @@ # Telemetry -## How telemetry is used +### How telemetry is used -The Feast maintainers use anonymous usage statistics and error tracking to help shape the Feast roadmap. Several client methods are tracked, beginning in Feast 0.9. Users are assigned a UUID which is sent along with the name of the method, the Feast version, the OS \(using `sys.platform`\), and the current time. For more detailed information see [the source code](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/telemetry.py). +The Feast project logs anonymous usage statistics and errors in order to inform our planning. Several client methods are tracked, beginning in Feast 0.9. Users are assigned a UUID which is sent along with the name of the method, the Feast version, the OS \(using `sys.platform`\), and the current time. -## How to disable telemetry +The [source code](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/telemetry.py) is available here. -To opt-out of all telemetry, simply set the environment variable `FEAST_TELEMETRY` to `False` in the environment in which the Feast SDK/CLI is run. +### How to disable telemetry + +Set the environment variable `FEAST_TELEMETRY` to `False`. From cef286952aaedb21716a91121ebc3f988f0e3326 Mon Sep 17 00:00:00 2001 From: codyjlin <31944154+codyjlin@users.noreply.github.com> Date: Mon, 21 Jun 2021 11:05:40 -0700 Subject: [PATCH 09/43] Provide descriptive error on invalid table reference (#1627) * Initial commit to catch nonexistent table Signed-off-by: Cody Lin Signed-off-by: Cody Lin * simplify nonexistent BQ table test Signed-off-by: Cody Lin * clean up table_exists exception Signed-off-by: Cody Lin * remove unneeded variable Signed-off-by: Cody Lin * function name change to _assert_table_exists Signed-off-by: Cody Lin * Initial commit to catch nonexistent table Signed-off-by: Cody Lin Signed-off-by: Cody Lin * simplify nonexistent BQ table test Signed-off-by: Cody Lin * clean up table_exists exception Signed-off-by: Cody Lin * function name change to _assert_table_exists Signed-off-by: Cody Lin * fix lint errors and rebase Signed-off-by: Cody Lin * Fix get_table(None) error Signed-off-by: Cody Lin * custom exception for both missing file and BQ source Signed-off-by: Cody Lin * revert FileSource checks Signed-off-by: Cody Lin * Use DataSourceNotFoundException instead of subclassing Signed-off-by: Cody Lin * Moved assert_table_exists out of the BQ constructor to apply_total Signed-off-by: Cody Lin * rename test and test asset Signed-off-by: Cody Lin * move validate logic back to data_source Signed-off-by: Cody Lin * fixed tests Signed-off-by: Cody Lin * Set pytest.integration for tests that access BQ Signed-off-by: Cody Lin * Import pytest in failed test files Signed-off-by: Cody Lin Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/data_source.py | 22 ++++++++++++ sdk/python/feast/errors.py | 7 ++++ sdk/python/feast/repo_operations.py | 3 +- ...ple_feature_repo_with_missing_bq_source.py | 20 +++++++++++ sdk/python/tests/test_cli_gcp.py | 35 +++++++++++++++++++ sdk/python/tests/test_cli_local.py | 3 ++ sdk/python/tests/test_online_retrieval.py | 2 ++ sdk/python/tests/test_partial_apply.py | 2 ++ 8 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 sdk/python/tests/example_feature_repo_with_missing_bq_source.py diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 44badcb83b6..c25b64c82f4 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -20,6 +20,7 @@ from feast import type_map from feast.data_format import FileFormat, StreamFormat +from feast.errors import DataSourceNotFoundException from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.value_type import ValueType @@ -519,6 +520,12 @@ def to_proto(self) -> DataSourceProto: """ raise NotImplementedError + def validate(self): + """ + Validates the underlying data source. + """ + raise NotImplementedError + class FileSource(DataSource): def __init__( @@ -615,6 +622,10 @@ def to_proto(self) -> DataSourceProto: return data_source_proto + def validate(self): + # TODO: validate a FileSource + pass + @staticmethod def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: return type_map.pa_to_feast_value_type @@ -692,6 +703,17 @@ def to_proto(self) -> DataSourceProto: return data_source_proto + def validate(self): + if not self.query: + from google.api_core.exceptions import NotFound + from google.cloud import bigquery + + client = bigquery.Client() + try: + client.get_table(self.table_ref) + except NotFound: + raise DataSourceNotFoundException(self.table_ref) + def get_table_query_string(self) -> str: """Returns a string that can directly be used to reference this table in SQL""" if self.table_ref: diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index b55fe61df37..2e7d537a876 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -3,6 +3,13 @@ from colorama import Fore, Style +class DataSourceNotFoundException(Exception): + def __init__(self, path): + super().__init__( + f"Unable to find table at '{path}'. Please check that table exists." + ) + + class FeastObjectNotFoundException(Exception): pass diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index b3cc7fa0c39..3ed219138b5 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -154,11 +154,12 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): data_sources = [t.input for t in repo.feature_views] - # Make sure the data source used by this feature view is supported by + # Make sure the data source used by this feature view is supported by Feast for data_source in data_sources: assert_offline_store_supports_data_source( repo_config.offline_store, data_source ) + data_source.validate() update_data_sources_with_inferred_event_timestamp_col(data_sources) for view in repo.feature_views: diff --git a/sdk/python/tests/example_feature_repo_with_missing_bq_source.py b/sdk/python/tests/example_feature_repo_with_missing_bq_source.py new file mode 100644 index 00000000000..3d1dc3394b9 --- /dev/null +++ b/sdk/python/tests/example_feature_repo_with_missing_bq_source.py @@ -0,0 +1,20 @@ +from datetime import timedelta + +from feast import BigQuerySource, Entity, Feature, FeatureView, ValueType + +nonexistent_source = BigQuerySource( + table_ref="project.dataset.nonexistent_table", event_timestamp_column="" +) + +driver = Entity(name="driver", value_type=ValueType.INT64, description="driver id",) + +nonexistent_features = FeatureView( + name="driver_locations", + entities=["driver"], + ttl=timedelta(days=1), + features=[ + Feature(name="lat", dtype=ValueType.FLOAT), + Feature(name="lon", dtype=ValueType.STRING), + ], + input=nonexistent_source, +) diff --git a/sdk/python/tests/test_cli_gcp.py b/sdk/python/tests/test_cli_gcp.py index 486b04b7efb..ac005e3c36a 100644 --- a/sdk/python/tests/test_cli_gcp.py +++ b/sdk/python/tests/test_cli_gcp.py @@ -53,3 +53,38 @@ def test_basic() -> None: result = runner.run(["teardown"], cwd=repo_path) assert result.returncode == 0 + + +@pytest.mark.integration +def test_missing_bq_source_fail() -> None: + project_id = "".join( + random.choice(string.ascii_lowercase + string.digits) for _ in range(10) + ) + runner = CliRunner() + with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name: + + repo_path = Path(repo_dir_name) + data_path = Path(data_dir_name) + + repo_config = repo_path / "feature_store.yaml" + + repo_config.write_text( + dedent( + f""" + project: {project_id} + registry: {data_path / "registry.db"} + provider: gcp + """ + ) + ) + + repo_example = repo_path / "example.py" + repo_example.write_text( + ( + Path(__file__).parent / "example_feature_repo_with_missing_bq_source.py" + ).read_text() + ) + + returncode, output = runner.run_with_output(["apply"], cwd=repo_path) + assert returncode == 1 + assert b"DataSourceNotFoundException" in output diff --git a/sdk/python/tests/test_cli_local.py b/sdk/python/tests/test_cli_local.py index 5b1a988a99b..288a2462452 100644 --- a/sdk/python/tests/test_cli_local.py +++ b/sdk/python/tests/test_cli_local.py @@ -4,12 +4,14 @@ from textwrap import dedent import assertpy +import pytest from feast.feature_store import FeatureStore from tests.cli_utils import CliRunner from tests.online_read_write_test import basic_rw_test +@pytest.mark.integration def test_workflow() -> None: """ Test running apply on a sample repo, and make sure the infra gets created. @@ -78,6 +80,7 @@ def test_workflow() -> None: assertpy.assert_that(result.returncode).is_equal_to(0) +@pytest.mark.integration def test_non_local_feature_repo() -> None: """ Test running apply on a sample repo, and make sure the infra gets created. diff --git a/sdk/python/tests/test_online_retrieval.py b/sdk/python/tests/test_online_retrieval.py index b76f901bd4d..3d0da04e18b 100644 --- a/sdk/python/tests/test_online_retrieval.py +++ b/sdk/python/tests/test_online_retrieval.py @@ -14,6 +14,7 @@ from tests.cli_utils import CliRunner, get_example_repo +@pytest.mark.integration def test_online() -> None: """ Test reading from the online store in local mode. @@ -247,6 +248,7 @@ def test_online() -> None: os.rename(store.config.registry + "_fake", store.config.registry) +@pytest.mark.integration def test_online_to_df(): """ Test dataframe conversion. Make sure the response columns and rows are diff --git a/sdk/python/tests/test_partial_apply.py b/sdk/python/tests/test_partial_apply.py index c8c9de76b12..062d9664186 100644 --- a/sdk/python/tests/test_partial_apply.py +++ b/sdk/python/tests/test_partial_apply.py @@ -1,3 +1,4 @@ +import pytest from google.protobuf.duration_pb2 import Duration from feast import BigQuerySource, Feature, FeatureView, ValueType @@ -5,6 +6,7 @@ from tests.online_read_write_test import basic_rw_test +@pytest.mark.integration def test_partial() -> None: """ Add another table to existing repo using partial apply API. Make sure both the table From c2e2b4dc9713941918e37c759d417b8798374a61 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 21 Jun 2021 12:03:40 -0700 Subject: [PATCH 10/43] Refactor OnlineStoreConfig classes into owning modules (#1649) * Refactor OnlineStoreConfig classes into owning modules Signed-off-by: Achal Shah * make format Signed-off-by: Achal Shah * Move redis too Signed-off-by: Achal Shah * update test_telemetery Signed-off-by: Achal Shah * add a create_repo_config method that should be called instead of RepoConfig ctor directly Signed-off-by: Achal Shah * fix the table reference in repo_operations Signed-off-by: Achal Shah * reuse create_repo_config Signed-off-by: Achal Shah Remove redis provider reference * CR comments Signed-off-by: Achal Shah * Remove create_repo_config in favor of __init__ Signed-off-by: Achal Shah * make format Signed-off-by: Achal Shah * Remove print statement Signed-off-by: Achal Shah Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/errors.py | 26 +++- .../feast/infra/online_stores/datastore.py | 23 ++- .../feast/infra/online_stores/helpers.py | 78 ++++------ sdk/python/feast/infra/online_stores/redis.py | 25 +++- .../feast/infra/online_stores/sqlite.py | 15 +- sdk/python/feast/infra/provider.py | 6 +- sdk/python/feast/repo_config.py | 141 ++++++++---------- sdk/python/feast/repo_operations.py | 2 +- sdk/python/telemetry_tests/test_telemetry.py | 4 +- sdk/python/tests/foo_provider.py | 2 +- sdk/python/tests/test_feature_store.py | 13 +- sdk/python/tests/test_historical_retrieval.py | 9 +- .../test_offline_online_store_consistency.py | 11 +- 13 files changed, 192 insertions(+), 163 deletions(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 2e7d537a876..b4848d66685 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -49,15 +49,15 @@ def __init__(self, provider_name): super().__init__(f"Provider '{provider_name}' is not implemented") -class FeastProviderModuleImportError(Exception): - def __init__(self, module_name): - super().__init__(f"Could not import provider module '{module_name}'") +class FeastModuleImportError(Exception): + def __init__(self, module_name, module_type="provider"): + super().__init__(f"Could not import {module_type} module '{module_name}'") -class FeastProviderClassImportError(Exception): - def __init__(self, module_name, class_name): +class FeastClassImportError(Exception): + def __init__(self, module_name, class_name, class_type="provider"): super().__init__( - f"Could not import provider '{class_name}' from module '{module_name}'" + f"Could not import {class_type} '{class_name}' from module '{module_name}'" ) @@ -84,6 +84,20 @@ def __init__(self, feature_name_collisions: str): f"The following feature name(s) have collisions: {feature_name_collisions}. Set 'full_feature_names' argument in the data retrieval function to True to use the full feature name which is prefixed by the feature view name." ) + +class FeastOnlineStoreInvalidName(Exception): + def __init__(self, online_store_class_name: str): + super().__init__( + f"Online Store Class '{online_store_class_name}' should end with the string `OnlineStore`.'" + ) + + +class FeastOnlineStoreConfigInvalidName(Exception): + def __init__(self, online_store_config_class_name: str): + super().__init__( + f"Online Store Config Class '{online_store_config_class_name}' should end with the string `OnlineStoreConfig`.'" + ) + class FeastOnlineStoreUnsupportedDataSource(Exception): def __init__(self, online_store_name: str, data_source_name: str): diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index 1f2c5abac49..c623af1c1f8 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -17,6 +17,8 @@ from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple, Union import mmh3 +from pydantic import PositiveInt, StrictStr +from pydantic.typing import Literal from feast import Entity, FeatureTable, utils from feast.feature_view import FeatureView @@ -24,7 +26,7 @@ from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto -from feast.repo_config import DatastoreOnlineStoreConfig, RepoConfig +from feast.repo_config import FeastConfigBaseModel, RepoConfig try: from google.auth.exceptions import DefaultCredentialsError @@ -40,6 +42,25 @@ ] +class DatastoreOnlineStoreConfig(FeastConfigBaseModel): + """ Online store config for GCP Datastore """ + + type: Literal["datastore"] = "datastore" + """ Online store type selector""" + + project_id: Optional[StrictStr] = None + """ (optional) GCP Project Id """ + + namespace: Optional[StrictStr] = None + """ (optional) Datastore namespace """ + + write_concurrency: Optional[PositiveInt] = 40 + """ (optional) Amount of threads to use when writing batches of feature rows into Datastore""" + + write_batch_size: Optional[PositiveInt] = 50 + """ (optional) Amount of feature rows per batch being written into Datastore""" + + class DatastoreOnlineStore(OnlineStore): """ OnlineStore is an object used for all interaction between Feast and the service used for offline storage of diff --git a/sdk/python/feast/infra/online_stores/helpers.py b/sdk/python/feast/infra/online_stores/helpers.py index 391794d20e0..71693ce1354 100644 --- a/sdk/python/feast/infra/online_stores/helpers.py +++ b/sdk/python/feast/infra/online_stores/helpers.py @@ -1,65 +1,41 @@ +import importlib import struct -from typing import Any, Dict, Set +from typing import Any import mmh3 -from feast.data_source import BigQuerySource, DataSource, FileSource -from feast.errors import FeastOnlineStoreUnsupportedDataSource +from feast import errors from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.storage.Redis_pb2 import RedisKeyV2 as RedisKeyProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto -from feast.repo_config import ( - DatastoreOnlineStoreConfig, - OnlineStoreConfig, - RedisOnlineStoreConfig, - SqliteOnlineStoreConfig, -) -def get_online_store_from_config( - online_store_config: OnlineStoreConfig, -) -> OnlineStore: +def get_online_store_from_config(online_store_config: Any,) -> OnlineStore: """Get the offline store from offline store config""" - if isinstance(online_store_config, SqliteOnlineStoreConfig): - from feast.infra.online_stores.sqlite import SqliteOnlineStore - - return SqliteOnlineStore() - elif isinstance(online_store_config, DatastoreOnlineStoreConfig): - from feast.infra.online_stores.datastore import DatastoreOnlineStore - - return DatastoreOnlineStore() - elif isinstance(online_store_config, RedisOnlineStoreConfig): - from feast.infra.online_stores.redis import RedisOnlineStore - - return RedisOnlineStore() - raise ValueError(f"Unsupported offline store config '{online_store_config}'") - - -SUPPORTED_SOURCES: Dict[Any, Set[Any]] = { - SqliteOnlineStoreConfig: {FileSource}, - DatastoreOnlineStoreConfig: {BigQuerySource}, - RedisOnlineStoreConfig: {FileSource, BigQuerySource}, -} - - -def assert_online_store_supports_data_source( - online_store_config: OnlineStoreConfig, data_source: DataSource -): - supported_sources: Set[Any] = SUPPORTED_SOURCES.get( - online_store_config.__class__, set() - ) - # This is needed because checking for `in` with Union types breaks mypy. - # https://github.com/python/mypy/issues/4954 - # We can replace this with `data_source.__class__ in SUPPORTED_SOURCES[online_store_config.__class__]` - # Once ^ is resolved. - if supported_sources: - for source in supported_sources: - if source == data_source.__class__: - return - raise FeastOnlineStoreUnsupportedDataSource( - online_store_config.type, data_source.__class__.__name__ - ) + module_name = online_store_config.__module__ + qualified_name = type(online_store_config).__name__ + store_class_name = qualified_name.replace("Config", "") + try: + module = importlib.import_module(module_name) + except Exception as e: + # The original exception can be anything - either module not found, + # or any other kind of error happening during the module import time. + # So we should include the original error as well in the stack trace. + raise errors.FeastModuleImportError( + module_name, module_type="OnlineStore" + ) from e + + # Try getting the provider class definition + try: + online_store_class = getattr(module, store_class_name) + except AttributeError: + # This can only be one type of error, when class_name attribute does not exist in the module + # So we don't have to include the original exception here + raise errors.FeastClassImportError( + module_name, store_class_name, class_type="OnlineStore" + ) from None + return online_store_class() def _redis_key(project: str, entity_key: EntityKeyProto): diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index abab030dced..bb85a8e853d 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -13,16 +13,19 @@ # limitations under the License. import json from datetime import datetime +from enum import Enum from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union from google.protobuf.timestamp_pb2 import Timestamp +from pydantic import StrictStr +from pydantic.typing import Literal from feast import Entity, FeatureTable, FeatureView, RepoConfig, utils from feast.infra.online_stores.helpers import _mmh3, _redis_key from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto -from feast.repo_config import RedisOnlineStoreConfig, RedisType +from feast.repo_config import FeastConfigBaseModel try: from redis import Redis @@ -35,6 +38,25 @@ EX_SECONDS = 253402300799 +class RedisType(str, Enum): + redis = "redis" + redis_cluster = "redis_cluster" + + +class RedisOnlineStoreConfig(FeastConfigBaseModel): + """Online store config for Redis store""" + + type: Literal["redis"] = "redis" + """Online store type selector""" + + redis_type: RedisType = RedisType.redis + """Redis type: redis or redis_cluster""" + + connection_string: StrictStr = "localhost:6379" + """Connection string containing the host, port, and configuration parameters for Redis + format: host:port,parameter1,parameter2 eg. redis:6379,db=0 """ + + class RedisOnlineStore(OnlineStore): _client: Optional[Union[Redis, RedisCluster]] = None @@ -99,7 +121,6 @@ def _get_client(self, online_store_config: RedisOnlineStoreConfig): startup_nodes, kwargs = self._parse_connection_string( online_store_config.connection_string ) - print(f"Startup nodes: {startup_nodes}, {kwargs}") if online_store_config.type == RedisType.redis_cluster: kwargs["startup_nodes"] = startup_nodes self._client = RedisCluster(**kwargs) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 9c2e5cd251a..7385ae25a94 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -19,6 +19,8 @@ from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union import pytz +from pydantic import StrictStr +from pydantic.schema import Literal from feast import Entity, FeatureTable from feast.feature_view import FeatureView @@ -26,7 +28,17 @@ from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto -from feast.repo_config import RepoConfig +from feast.repo_config import FeastConfigBaseModel, RepoConfig + + +class SqliteOnlineStoreConfig(FeastConfigBaseModel): + """ Online store config for local (SQLite-based) store """ + + type: Literal["sqlite"] = "sqlite" + """ Online store type selector""" + + path: StrictStr = "data/online.db" + """ (optional) Path to sqlite db """ class SqliteOnlineStore(OnlineStore): @@ -65,6 +77,7 @@ def online_write_batch( ], progress: Optional[Callable[[int], Any]], ) -> None: + conn = self._get_conn(config) project = config.project diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 50d3d8873d4..20ceaf0acf0 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -164,7 +164,7 @@ def get_provider(config: RepoConfig, repo_path: Path) -> Provider: # The original exception can be anything - either module not found, # or any other kind of error happening during the module import time. # So we should include the original error as well in the stack trace. - raise errors.FeastProviderModuleImportError(module_name) from e + raise errors.FeastModuleImportError(module_name) from e # Try getting the provider class definition try: @@ -172,9 +172,7 @@ def get_provider(config: RepoConfig, repo_path: Path) -> Provider: except AttributeError: # This can only be one type of error, when class_name attribute does not exist in the module # So we don't have to include the original exception here - raise errors.FeastProviderClassImportError( - module_name, class_name - ) from None + raise errors.FeastClassImportError(module_name, class_name) from None return ProviderCls(config, repo_path) diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 7e70efb799b..25c55dcc2bc 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -1,80 +1,39 @@ -from enum import Enum +import importlib from pathlib import Path +from typing import Any import yaml -from pydantic import ( - BaseModel, - PositiveInt, - StrictInt, - StrictStr, - ValidationError, - root_validator, -) +from pydantic import BaseModel, StrictInt, StrictStr, ValidationError, root_validator from pydantic.error_wrappers import ErrorWrapper from pydantic.typing import Dict, Literal, Optional, Union +from feast import errors from feast.telemetry import log_exceptions +# This dict exists so that: +# - existing values for the online store type in featurestore.yaml files continue to work in a backwards compatible way +# - first party and third party implementations can use the same class loading code path. +ONLINE_CONFIG_CLASS_FOR_TYPE = { + "sqlite": "feast.infra.online_stores.sqlite.SqliteOnlineStore", + "datastore": "feast.infra.online_stores.datastore.DatastoreOnlineStore", + "redis": "feast.infra.online_stores.redis.RedisOnlineStore", +} + class FeastBaseModel(BaseModel): """ Feast Pydantic Configuration Class """ class Config: arbitrary_types_allowed = True - extra = "forbid" - - -class SqliteOnlineStoreConfig(FeastBaseModel): - """ Online store config for local (SQLite-based) store """ - - type: Literal["sqlite"] = "sqlite" - """ Online store type selector""" - - path: StrictStr = "data/online.db" - """ (optional) Path to sqlite db """ - - -class DatastoreOnlineStoreConfig(FeastBaseModel): - """ Online store config for GCP Datastore """ - - type: Literal["datastore"] = "datastore" - """ Online store type selector""" - - project_id: Optional[StrictStr] = None - """ (optional) GCP Project Id """ - - namespace: Optional[StrictStr] = None - """ (optional) Datastore namespace """ - - write_concurrency: Optional[PositiveInt] = 40 - """ (optional) Amount of threads to use when writing batches of feature rows into Datastore""" - - write_batch_size: Optional[PositiveInt] = 50 - """ (optional) Amount of feature rows per batch being written into Datastore""" - - -class RedisType(str, Enum): - redis = "redis" - redis_cluster = "redis_cluster" - + extra = "allow" -class RedisOnlineStoreConfig(FeastBaseModel): - """Online store config for Redis store""" - - type: Literal["redis"] = "redis" - """Online store type selector""" - - redis_type: RedisType = RedisType.redis - """Redis type: redis or redis_cluster""" - - connection_string: StrictStr = "localhost:6379" - """Connection string containing the host, port, and configuration parameters for Redis - format: host:port,parameter1,parameter2 eg. redis:6379,db=0 """ +class FeastConfigBaseModel(BaseModel): + """ Feast Pydantic Configuration Class """ -OnlineStoreConfig = Union[ - DatastoreOnlineStoreConfig, SqliteOnlineStoreConfig, RedisOnlineStoreConfig -] + class Config: + arbitrary_types_allowed = True + extra = "forbid" class FileOfflineStoreConfig(FeastBaseModel): @@ -123,9 +82,9 @@ class RepoConfig(FeastBaseModel): """ provider: StrictStr - """ str: local or gcp or redis """ + """ str: local or gcp """ - online_store: OnlineStoreConfig = SqliteOnlineStoreConfig() + online_store: Any """ OnlineStoreConfig: Online store configuration (optional depending on provider) """ offline_store: OfflineStoreConfig = FileOfflineStoreConfig() @@ -133,6 +92,13 @@ class RepoConfig(FeastBaseModel): repo_path: Optional[Path] = None + def __init__(self, **data: Any): + super().__init__(**data) + if isinstance(self.online_store, Dict): + self.online_store = get_online_config_from_type(self.online_store["type"])( + **self.online_store + ) + def get_registry_config(self): if isinstance(self.registry, str): return RegistryConfig(path=self.registry) @@ -160,6 +126,8 @@ def _validate_online_store_config(cls, values): assert "provider" in values # Set the default type + # This is only direct reference to a provider or online store that we should have + # for backwards compatibility. if "type" not in values["online_store"]: if values["provider"] == "local": values["online_store"]["type"] = "sqlite" @@ -168,22 +136,13 @@ def _validate_online_store_config(cls, values): online_store_type = values["online_store"]["type"] - # Make sure the user hasn't provided the wrong type - assert online_store_type in ["datastore", "sqlite", "redis"] - # Validate the dict to ensure one of the union types match try: - if online_store_type == "sqlite": - SqliteOnlineStoreConfig(**values["online_store"]) - elif online_store_type == "datastore": - DatastoreOnlineStoreConfig(**values["online_store"]) - elif online_store_type == "redis": - RedisOnlineStoreConfig(**values["online_store"]) - else: - raise ValueError(f"Invalid online store type {online_store_type}") + online_config_class = get_online_config_from_type(online_store_type) + online_config_class(**values["online_store"]) except ValidationError as e: raise ValidationError( - [ErrorWrapper(e, loc="online_store")], model=SqliteOnlineStoreConfig, + [ErrorWrapper(e, loc="online_store")], model=RepoConfig, ) return values @@ -203,7 +162,7 @@ def _validate_offline_store_config(cls, values): # Set the default type if "type" not in values["offline_store"]: - if values["provider"] == "local" or values["provider"] == "redis": + if values["provider"] == "local": values["offline_store"]["type"] = "file" elif values["provider"] == "gcp": values["offline_store"]["type"] = "bigquery" @@ -246,6 +205,38 @@ def __repr__(self) -> str: ) +def get_online_config_from_type(online_store_type: str): + if online_store_type in ONLINE_CONFIG_CLASS_FOR_TYPE: + online_store_type = ONLINE_CONFIG_CLASS_FOR_TYPE[online_store_type] + module_name, class_name = online_store_type.rsplit(".", 1) + + if not class_name.endswith("OnlineStore"): + raise errors.FeastOnlineStoreConfigInvalidName(class_name) + config_class_name = f"{class_name}Config" + + # Try importing the module that contains the custom provider + try: + module = importlib.import_module(module_name) + except Exception as e: + # The original exception can be anything - either module not found, + # or any other kind of error happening during the module import time. + # So we should include the original error as well in the stack trace. + raise errors.FeastModuleImportError( + module_name, module_type="OnlineStore" + ) from e + + # Try getting the provider class definition + try: + online_store_config_class = getattr(module, config_class_name) + except AttributeError: + # This can only be one type of error, when class_name attribute does not exist in the module + # So we don't have to include the original exception here + raise errors.FeastClassImportError( + module_name, config_class_name, class_type="OnlineStoreConfig" + ) from None + return online_store_config_class + + def load_repo_config(repo_path: Path) -> RepoConfig: config_path = repo_path / "feature_store.yaml" diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 3ed219138b5..59ff0c60bf7 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -186,7 +186,7 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): for table in repo.feature_tables: registry.apply_feature_table(table, project) click.echo( - f"Registered feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL}" + f"Registered feature table {Style.BRIGHT + Fore.GREEN}{table.name}{Style.RESET_ALL}" ) # Delete views that should not exist diff --git a/sdk/python/telemetry_tests/test_telemetry.py b/sdk/python/telemetry_tests/test_telemetry.py index be6f2cb6219..9b35bf3c17a 100644 --- a/sdk/python/telemetry_tests/test_telemetry.py +++ b/sdk/python/telemetry_tests/test_telemetry.py @@ -15,15 +15,15 @@ import uuid from datetime import datetime +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig from tenacity import retry, wait_exponential, stop_after_attempt from google.cloud import bigquery import os from time import sleep -from importlib import reload from feast import Client, Entity, ValueType, FeatureStore, RepoConfig -from feast.repo_config import SqliteOnlineStoreConfig + TELEMETRY_BIGQUERY_TABLE = ( "kf-feast.feast_telemetry.cloudfunctions_googleapis_com_cloud_functions" diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index ac902376f52..0352645d983 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -54,8 +54,8 @@ def materialize_single_feature_view( ) -> None: pass - @staticmethod def get_historical_features( + self, config: RepoConfig, feature_views: List[FeatureView], feature_refs: List[str], diff --git a/sdk/python/tests/test_feature_store.py b/sdk/python/tests/test_feature_store.py index e1e82a9ec27..49a3a9a63b0 100644 --- a/sdk/python/tests/test_feature_store.py +++ b/sdk/python/tests/test_feature_store.py @@ -17,11 +17,6 @@ import pytest from pytest_lazyfixture import lazy_fixture -from utils.data_source_utils import ( - prep_file_source, - simple_bq_source_using_query_arg, - simple_bq_source_using_table_ref_arg, -) from feast.data_format import ParquetFormat from feast.data_source import FileSource @@ -29,9 +24,15 @@ from feast.feature import Feature from feast.feature_store import FeatureStore from feast.feature_view import FeatureView +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig from feast.protos.feast.types import Value_pb2 as ValueProto -from feast.repo_config import RepoConfig, SqliteOnlineStoreConfig +from feast.repo_config import RepoConfig from feast.value_type import ValueType +from tests.utils.data_source_utils import ( + prep_file_source, + simple_bq_source_using_query_arg, + simple_bq_source_using_table_ref_arg, +) @pytest.fixture diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index 79e9b14d906..03a44b0fd5d 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -14,19 +14,16 @@ from pytz import utc import feast.driver_test_data as driver_data -from feast import errors, utils +from feast import RepoConfig, errors, utils from feast.data_source import BigQuerySource, FileSource from feast.entity import Entity from feast.errors import FeatureNameCollisionError from feast.feature import Feature from feast.feature_store import FeatureStore, _group_refs from feast.feature_view import FeatureView +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig from feast.infra.provider import DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL -from feast.repo_config import ( - BigQueryOfflineStoreConfig, - RepoConfig, - SqliteOnlineStoreConfig, -) +from feast.repo_config import BigQueryOfflineStoreConfig from feast.value_type import ValueType np.random.seed(0) diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py index 237ee8cda75..e2faec2d6db 100644 --- a/sdk/python/tests/test_offline_online_store_consistency.py +++ b/sdk/python/tests/test_offline_online_store_consistency.py @@ -17,13 +17,10 @@ from feast.feature import Feature from feast.feature_store import FeatureStore from feast.feature_view import FeatureView -from feast.repo_config import ( - DatastoreOnlineStoreConfig, - RedisOnlineStoreConfig, - RedisType, - RepoConfig, - SqliteOnlineStoreConfig, -) +from feast.infra.online_stores.datastore import DatastoreOnlineStoreConfig +from feast.infra.online_stores.redis import RedisOnlineStoreConfig, RedisType +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig +from feast.repo_config import RepoConfig from feast.value_type import ValueType From d2cda245e7158206808726728cf4a52d5754d7f0 Mon Sep 17 00:00:00 2001 From: Matt Delacour Date: Mon, 21 Jun 2021 17:31:39 -0400 Subject: [PATCH 11/43] Possibility to specify a project for BigQuery queries (#1656) Signed-off-by: Matt Delacour Co-authored-by: Achal Shah Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- .../feast/infra/offline_stores/bigquery.py | 22 ++++++++++++++----- sdk/python/feast/repo_config.py | 5 ++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 9be327f5609..b1deb1c7bca 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -95,8 +95,14 @@ def get_historical_features( expected_join_keys = _get_join_keys(project, feature_views, registry) assert isinstance(config.offline_store, BigQueryOfflineStoreConfig) + dataset_project = config.offline_store.project_id or client.project + table = _upload_entity_df_into_bigquery( - client, config.project, config.offline_store.dataset, entity_df + client=client, + project=config.project, + dataset_name=config.offline_store.dataset, + dataset_project=dataset_project, + entity_df=entity_df, ) entity_df_event_timestamp_col = _infer_event_timestamp_from_bigquery_query( @@ -227,7 +233,8 @@ def _block_until_done(): today = date.today().strftime("%Y%m%d") rand_id = str(uuid.uuid4())[:7] - path = f"{self.client.project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" + dataset_project = self.config.offline_store.project_id or self.client.project + path = f"{dataset_project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" job_config = bigquery.QueryJobConfig(destination=path, dry_run=dry_run) bq_job = self.client.query(self.query, job_config=job_config) @@ -263,12 +270,12 @@ class FeatureViewQueryContext: def _get_table_id_for_new_entity( - client: Client, project: str, dataset_name: str + client: Client, project: str, dataset_name: str, dataset_project: str ) -> str: """Gets the table_id for the new entity to be uploaded.""" # First create the BigQuery dataset if it doesn't exist - dataset = bigquery.Dataset(f"{client.project}.{dataset_name}") + dataset = bigquery.Dataset(f"{dataset_project}.{dataset_name}") dataset.location = "US" try: @@ -277,18 +284,21 @@ def _get_table_id_for_new_entity( # Only create the dataset if it does not exist client.create_dataset(dataset, exists_ok=True) - return f"{client.project}.{dataset_name}.entity_df_{project}_{int(time.time())}" + return f"{dataset_project}.{dataset_name}.entity_df_{project}_{int(time.time())}" def _upload_entity_df_into_bigquery( client: Client, project: str, dataset_name: str, + dataset_project: str, entity_df: Union[pandas.DataFrame, str], ) -> Table: """Uploads a Pandas entity dataframe into a BigQuery table and returns the resulting table""" - table_id = _get_table_id_for_new_entity(client, project, dataset_name) + table_id = _get_table_id_for_new_entity( + client, project, dataset_name, dataset_project + ) if type(entity_df) is str: job = client.query(f"CREATE TABLE {table_id} AS ({entity_df})") diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 25c55dcc2bc..7c72fce9440 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -50,7 +50,10 @@ class BigQueryOfflineStoreConfig(FeastBaseModel): """ Offline store type selector""" dataset: StrictStr = "feast" - """ (optional) BigQuery Dataset name for temporary tables """ + """ (optional) BigQuery dataset name used for the BigQuery offline store """ + + project_id: Optional[StrictStr] = None + """ (optional) GCP project name used for the BigQuery offline store """ OfflineStoreConfig = Union[FileOfflineStoreConfig, BigQueryOfflineStoreConfig] From 4ab4c607a5fddfc11dfc637836f903769ad50f28 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 22 Jun 2021 11:59:25 -0700 Subject: [PATCH 12/43] Refactor OfflineStoreConfig classes into their owning modules (#1657) * Refactor OfflineStoreConfig classes into their owning modules Signed-off-by: Achal Shah * Fix error string Signed-off-by: Achal Shah * Generic error class Signed-off-by: Achal Shah * Merge conflicts Signed-off-by: Achal Shah * make the store type work, and add a test that uses the fully qualified name of the OnlineStore Signed-off-by: Achal Shah * Address comments from previous PR Signed-off-by: Achal Shah * CR updates Signed-off-by: Achal Shah Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/errors.py | 9 +- sdk/python/feast/importer.py | 28 +++++ .../feast/infra/offline_stores/bigquery.py | 17 ++- sdk/python/feast/infra/offline_stores/file.py | 10 +- .../feast/infra/offline_stores/helpers.py | 62 +++++----- .../feast/infra/online_stores/helpers.py | 4 +- .../feast/infra/online_stores/sqlite.py | 9 +- sdk/python/feast/infra/provider.py | 24 +--- sdk/python/feast/repo_config.py | 106 ++++++------------ sdk/python/feast/repo_operations.py | 4 - sdk/python/tests/test_cli_local.py | 6 +- sdk/python/tests/test_historical_retrieval.py | 2 +- sdk/python/tests/test_repo_config.py | 16 +++ 13 files changed, 152 insertions(+), 145 deletions(-) create mode 100644 sdk/python/feast/importer.py diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index b4848d66685..6b8ff45d714 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -50,7 +50,7 @@ def __init__(self, provider_name): class FeastModuleImportError(Exception): - def __init__(self, module_name, module_type="provider"): + def __init__(self, module_name: str, module_type: str): super().__init__(f"Could not import {module_type} module '{module_name}'") @@ -92,10 +92,11 @@ def __init__(self, online_store_class_name: str): ) -class FeastOnlineStoreConfigInvalidName(Exception): - def __init__(self, online_store_config_class_name: str): +class FeastClassInvalidName(Exception): + def __init__(self, class_name: str, class_type: str): super().__init__( - f"Online Store Config Class '{online_store_config_class_name}' should end with the string `OnlineStoreConfig`.'" + f"Config Class '{class_name}' " + f"should end with the string `{class_type}`.'" ) diff --git a/sdk/python/feast/importer.py b/sdk/python/feast/importer.py new file mode 100644 index 00000000000..5dcd7c71c12 --- /dev/null +++ b/sdk/python/feast/importer.py @@ -0,0 +1,28 @@ +import importlib + +from feast import errors + + +def get_class_from_type(module_name: str, class_name: str, class_type: str): + if not class_name.endswith(class_type): + raise errors.FeastClassInvalidName(class_name, class_type) + + # Try importing the module that contains the custom provider + try: + module = importlib.import_module(module_name) + except Exception as e: + # The original exception can be anything - either module not found, + # or any other kind of error happening during the module import time. + # So we should include the original error as well in the stack trace. + raise errors.FeastModuleImportError(module_name, class_type) from e + + # Try getting the provider class definition + try: + _class = getattr(module, class_name) + except AttributeError: + # This can only be one type of error, when class_name attribute does not exist in the module + # So we don't have to include the original exception here + raise errors.FeastClassImportError( + module_name, class_name, class_type=class_type + ) from None + return _class diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index b1deb1c7bca..2bd36867b3c 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -7,6 +7,8 @@ import pandas import pyarrow from jinja2 import BaseLoader, Environment +from pydantic import StrictStr +from pydantic.typing import Literal from tenacity import retry, stop_after_delay, wait_fixed from feast import errors @@ -20,7 +22,7 @@ _get_requested_feature_views_to_features_dict, ) from feast.registry import Registry -from feast.repo_config import BigQueryOfflineStoreConfig, RepoConfig +from feast.repo_config import FeastConfigBaseModel, RepoConfig try: from google.api_core.exceptions import NotFound @@ -34,6 +36,19 @@ raise FeastExtrasDependencyImportError("gcp", str(e)) +class BigQueryOfflineStoreConfig(FeastConfigBaseModel): + """ Offline store config for GCP BigQuery """ + + type: Literal["bigquery"] = "bigquery" + """ Offline store type selector""" + + dataset: StrictStr = "feast" + """ (optional) BigQuery Dataset name for temporary tables """ + + project_id: Optional[StrictStr] = None + """ (optional) GCP project name used for the BigQuery offline store """ + + class BigQueryOfflineStore(OfflineStore): @staticmethod def pull_latest_from_table_or_query( diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index 0513dc884a4..f2f700cc661 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -4,6 +4,7 @@ import pandas as pd import pyarrow import pytz +from pydantic.typing import Literal from feast.data_source import DataSource, FileSource from feast.errors import FeastJoinKeysDuringMaterialization @@ -15,7 +16,14 @@ _run_field_mapping, ) from feast.registry import Registry -from feast.repo_config import RepoConfig +from feast.repo_config import FeastConfigBaseModel, RepoConfig + + +class FileOfflineStoreConfig(FeastConfigBaseModel): + """ Offline store config for local (file-based) store """ + + type: Literal["file"] = "file" + """ Offline store type selector""" class FileRetrievalJob(RetrievalJob): diff --git a/sdk/python/feast/infra/offline_stores/helpers.py b/sdk/python/feast/infra/offline_stores/helpers.py index af1d1b92123..dff604c7ed1 100644 --- a/sdk/python/feast/infra/offline_stores/helpers.py +++ b/sdk/python/feast/infra/offline_stores/helpers.py @@ -1,41 +1,31 @@ -from feast.data_source import BigQuerySource, DataSource, FileSource -from feast.errors import FeastOfflineStoreUnsupportedDataSource +import importlib +from typing import Any + +from feast import errors from feast.infra.offline_stores.offline_store import OfflineStore -from feast.repo_config import ( - BigQueryOfflineStoreConfig, - FileOfflineStoreConfig, - OfflineStoreConfig, -) -def get_offline_store_from_config( - offline_store_config: OfflineStoreConfig, -) -> OfflineStore: +def get_offline_store_from_config(offline_store_config: Any,) -> OfflineStore: """Get the offline store from offline store config""" - if isinstance(offline_store_config, FileOfflineStoreConfig): - from feast.infra.offline_stores.file import FileOfflineStore - - return FileOfflineStore() - elif isinstance(offline_store_config, BigQueryOfflineStoreConfig): - from feast.infra.offline_stores.bigquery import BigQueryOfflineStore - - return BigQueryOfflineStore() - - raise ValueError(f"Unsupported offline store config '{offline_store_config}'") - - -def assert_offline_store_supports_data_source( - offline_store_config: OfflineStoreConfig, data_source: DataSource -): - if ( - isinstance(offline_store_config, FileOfflineStoreConfig) - and isinstance(data_source, FileSource) - ) or ( - isinstance(offline_store_config, BigQueryOfflineStoreConfig) - and isinstance(data_source, BigQuerySource) - ): - return - raise FeastOfflineStoreUnsupportedDataSource( - offline_store_config.type, data_source.__class__.__name__ - ) + module_name = offline_store_config.__module__ + qualified_name = type(offline_store_config).__name__ + store_class_name = qualified_name.replace("Config", "") + try: + module = importlib.import_module(module_name) + except Exception as e: + # The original exception can be anything - either module not found, + # or any other kind of error happening during the module import time. + # So we should include the original error as well in the stack trace. + raise errors.FeastModuleImportError(module_name, "OfflineStore") from e + + # Try getting the provider class definition + try: + offline_store_class = getattr(module, store_class_name) + except AttributeError: + # This can only be one type of error, when class_name attribute does not exist in the module + # So we don't have to include the original exception here + raise errors.FeastClassImportError( + module_name, store_class_name, class_type="OfflineStore" + ) from None + return offline_store_class() diff --git a/sdk/python/feast/infra/online_stores/helpers.py b/sdk/python/feast/infra/online_stores/helpers.py index 71693ce1354..9c42c5ea002 100644 --- a/sdk/python/feast/infra/online_stores/helpers.py +++ b/sdk/python/feast/infra/online_stores/helpers.py @@ -22,9 +22,7 @@ def get_online_store_from_config(online_store_config: Any,) -> OnlineStore: # The original exception can be anything - either module not found, # or any other kind of error happening during the module import time. # So we should include the original error as well in the stack trace. - raise errors.FeastModuleImportError( - module_name, module_type="OnlineStore" - ) from e + raise errors.FeastModuleImportError(module_name, "OnlineStore") from e # Try getting the provider class definition try: diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 7385ae25a94..dbd837c5dfc 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -34,7 +34,9 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel): """ Online store config for local (SQLite-based) store """ - type: Literal["sqlite"] = "sqlite" + type: Literal[ + "sqlite", "feast.infra.online_stores.sqlite.SqliteOnlineStore" + ] = "sqlite" """ Online store type selector""" path: StrictStr = "data/online.db" @@ -51,7 +53,10 @@ class SqliteOnlineStore(OnlineStore): @staticmethod def _get_db_path(config: RepoConfig) -> str: - assert config.online_store.type == "sqlite" + assert ( + config.online_store.type == "sqlite" + or config.online_store.type.endswith("SqliteOnlineStore") + ) if config.repo_path and not Path(config.online_store.path).is_absolute(): db_path = str(config.repo_path / config.online_store.path) diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 20ceaf0acf0..d82d9c3f2dd 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -1,5 +1,4 @@ import abc -import importlib from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union @@ -8,7 +7,7 @@ import pyarrow from tqdm import tqdm -from feast import errors +from feast import errors, importer from feast.entity import Entity from feast.feature_table import FeatureTable from feast.feature_view import FeatureView @@ -157,24 +156,9 @@ def get_provider(config: RepoConfig, repo_path: Path) -> Provider: # For example, provider 'foo.bar.MyProvider' will be parsed into 'foo.bar' and 'MyProvider' module_name, class_name = config.provider.rsplit(".", 1) - # Try importing the module that contains the custom provider - try: - module = importlib.import_module(module_name) - except Exception as e: - # The original exception can be anything - either module not found, - # or any other kind of error happening during the module import time. - # So we should include the original error as well in the stack trace. - raise errors.FeastModuleImportError(module_name) from e - - # Try getting the provider class definition - try: - ProviderCls = getattr(module, class_name) - except AttributeError: - # This can only be one type of error, when class_name attribute does not exist in the module - # So we don't have to include the original exception here - raise errors.FeastClassImportError(module_name, class_name) from None - - return ProviderCls(config, repo_path) + cls = importer.get_class_from_type(module_name, class_name, "Provider") + + return cls(config, repo_path) def _get_requested_feature_views_to_features_dict( diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 7c72fce9440..5587fb59053 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -1,24 +1,28 @@ -import importlib from pathlib import Path from typing import Any import yaml from pydantic import BaseModel, StrictInt, StrictStr, ValidationError, root_validator from pydantic.error_wrappers import ErrorWrapper -from pydantic.typing import Dict, Literal, Optional, Union +from pydantic.typing import Dict, Optional, Union -from feast import errors +from feast.importer import get_class_from_type from feast.telemetry import log_exceptions -# This dict exists so that: +# These dict exists so that: # - existing values for the online store type in featurestore.yaml files continue to work in a backwards compatible way # - first party and third party implementations can use the same class loading code path. -ONLINE_CONFIG_CLASS_FOR_TYPE = { +ONLINE_STORE_CLASS_FOR_TYPE = { "sqlite": "feast.infra.online_stores.sqlite.SqliteOnlineStore", "datastore": "feast.infra.online_stores.datastore.DatastoreOnlineStore", "redis": "feast.infra.online_stores.redis.RedisOnlineStore", } +OFFLINE_STORE_CLASS_FOR_TYPE = { + "file": "feast.infra.offline_stores.file.FileOfflineStore", + "bigquery": "feast.infra.offline_stores.bigquery.BigQueryOfflineStore", +} + class FeastBaseModel(BaseModel): """ Feast Pydantic Configuration Class """ @@ -36,29 +40,6 @@ class Config: extra = "forbid" -class FileOfflineStoreConfig(FeastBaseModel): - """ Offline store config for local (file-based) store """ - - type: Literal["file"] = "file" - """ Offline store type selector""" - - -class BigQueryOfflineStoreConfig(FeastBaseModel): - """ Offline store config for GCP BigQuery """ - - type: Literal["bigquery"] = "bigquery" - """ Offline store type selector""" - - dataset: StrictStr = "feast" - """ (optional) BigQuery dataset name used for the BigQuery offline store """ - - project_id: Optional[StrictStr] = None - """ (optional) GCP project name used for the BigQuery offline store """ - - -OfflineStoreConfig = Union[FileOfflineStoreConfig, BigQueryOfflineStoreConfig] - - class RegistryConfig(FeastBaseModel): """ Metadata Store Configuration. Configuration that relates to reading from and writing to the Feast registry.""" @@ -90,7 +71,7 @@ class RepoConfig(FeastBaseModel): online_store: Any """ OnlineStoreConfig: Online store configuration (optional depending on provider) """ - offline_store: OfflineStoreConfig = FileOfflineStoreConfig() + offline_store: Any """ OfflineStoreConfig: Offline store configuration (optional depending on provider) """ repo_path: Optional[Path] = None @@ -101,6 +82,10 @@ def __init__(self, **data: Any): self.online_store = get_online_config_from_type(self.online_store["type"])( **self.online_store ) + if isinstance(self.offline_store, Dict): + self.offline_store = get_offline_config_from_type( + self.offline_store["type"] + )(**self.offline_store) def get_registry_config(self): if isinstance(self.registry, str): @@ -172,22 +157,13 @@ def _validate_offline_store_config(cls, values): offline_store_type = values["offline_store"]["type"] - # Make sure the user hasn't provided the wrong type - assert offline_store_type in ["file", "bigquery"] - # Validate the dict to ensure one of the union types match try: - if offline_store_type == "file": - FileOfflineStoreConfig(**values["offline_store"]) - elif offline_store_type == "bigquery": - BigQueryOfflineStoreConfig(**values["offline_store"]) - else: - raise ValidationError( - f"Invalid offline store type {offline_store_type}" - ) + offline_config_class = get_offline_config_from_type(offline_store_type) + offline_config_class(**values["offline_store"]) except ValidationError as e: raise ValidationError( - [ErrorWrapper(e, loc="offline_store")], model=FileOfflineStoreConfig, + [ErrorWrapper(e, loc="offline_store")], model=RepoConfig, ) return values @@ -209,35 +185,25 @@ def __repr__(self) -> str: def get_online_config_from_type(online_store_type: str): - if online_store_type in ONLINE_CONFIG_CLASS_FOR_TYPE: - online_store_type = ONLINE_CONFIG_CLASS_FOR_TYPE[online_store_type] - module_name, class_name = online_store_type.rsplit(".", 1) - - if not class_name.endswith("OnlineStore"): - raise errors.FeastOnlineStoreConfigInvalidName(class_name) - config_class_name = f"{class_name}Config" - - # Try importing the module that contains the custom provider - try: - module = importlib.import_module(module_name) - except Exception as e: - # The original exception can be anything - either module not found, - # or any other kind of error happening during the module import time. - # So we should include the original error as well in the stack trace. - raise errors.FeastModuleImportError( - module_name, module_type="OnlineStore" - ) from e - - # Try getting the provider class definition - try: - online_store_config_class = getattr(module, config_class_name) - except AttributeError: - # This can only be one type of error, when class_name attribute does not exist in the module - # So we don't have to include the original exception here - raise errors.FeastClassImportError( - module_name, config_class_name, class_type="OnlineStoreConfig" - ) from None - return online_store_config_class + if online_store_type in ONLINE_STORE_CLASS_FOR_TYPE: + online_store_type = ONLINE_STORE_CLASS_FOR_TYPE[online_store_type] + else: + assert online_store_type.endswith("OnlineStore") + module_name, online_store_class_type = online_store_type.rsplit(".", 1) + config_class_name = f"{online_store_class_type}Config" + + return get_class_from_type(module_name, config_class_name, config_class_name) + + +def get_offline_config_from_type(offline_store_type: str): + if offline_store_type in OFFLINE_STORE_CLASS_FOR_TYPE: + offline_store_type = OFFLINE_STORE_CLASS_FOR_TYPE[offline_store_type] + else: + assert offline_store_type.endswith("OfflineStore") + module_name, offline_store_class_type = offline_store_type.rsplit(".", 1) + config_class_name = f"{offline_store_class_type}Config" + + return get_class_from_type(module_name, config_class_name, config_class_name) def load_repo_config(repo_path: Path) -> RepoConfig: diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 59ff0c60bf7..f4a44a74559 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -17,7 +17,6 @@ infer_entity_value_type_from_feature_views, update_data_sources_with_inferred_event_timestamp_col, ) -from feast.infra.offline_stores.helpers import assert_offline_store_supports_data_source from feast.infra.provider import get_provider from feast.names import adjectives, animals from feast.registry import Registry @@ -156,9 +155,6 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): # Make sure the data source used by this feature view is supported by Feast for data_source in data_sources: - assert_offline_store_supports_data_source( - repo_config.offline_store, data_source - ) data_source.validate() update_data_sources_with_inferred_event_timestamp_col(data_sources) diff --git a/sdk/python/tests/test_cli_local.py b/sdk/python/tests/test_cli_local.py index 288a2462452..43998190737 100644 --- a/sdk/python/tests/test_cli_local.py +++ b/sdk/python/tests/test_cli_local.py @@ -164,18 +164,18 @@ def test_3rd_party_providers() -> None: assertpy.assert_that(return_code).is_equal_to(1) assertpy.assert_that(output).contains(b"Provider 'feast123' is not implemented") # Check with incorrect third-party provider name (with dots) - with setup_third_party_provider_repo("feast_foo.provider") as repo_path: + with setup_third_party_provider_repo("feast_foo.Provider") as repo_path: return_code, output = runner.run_with_output(["apply"], cwd=repo_path) assertpy.assert_that(return_code).is_equal_to(1) assertpy.assert_that(output).contains( - b"Could not import provider module 'feast_foo'" + b"Could not import Provider module 'feast_foo'" ) # Check with incorrect third-party provider name (with dots) with setup_third_party_provider_repo("foo.FooProvider") as repo_path: return_code, output = runner.run_with_output(["apply"], cwd=repo_path) assertpy.assert_that(return_code).is_equal_to(1) assertpy.assert_that(output).contains( - b"Could not import provider 'FooProvider' from module 'foo'" + b"Could not import Provider 'FooProvider' from module 'foo'" ) # Check with correct third-party provider name with setup_third_party_provider_repo("foo.provider.FooProvider") as repo_path: diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index 03a44b0fd5d..61f9564f87f 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -21,9 +21,9 @@ from feast.feature import Feature from feast.feature_store import FeatureStore, _group_refs from feast.feature_view import FeatureView +from feast.infra.offline_stores.bigquery import BigQueryOfflineStoreConfig from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig from feast.infra.provider import DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL -from feast.repo_config import BigQueryOfflineStoreConfig from feast.value_type import ValueType np.random.seed(0) diff --git a/sdk/python/tests/test_repo_config.py b/sdk/python/tests/test_repo_config.py index 19c8ee4dcc0..b6a6a330119 100644 --- a/sdk/python/tests/test_repo_config.py +++ b/sdk/python/tests/test_repo_config.py @@ -26,6 +26,7 @@ def _test_config(config_text, expect_error: Optional[str]): if expect_error is not None: assert expect_error in str(error) else: + print(f"error: {error}") assert error is None @@ -42,6 +43,21 @@ def test_local_config(): ) +def test_local_config_with_full_online_class(): + _test_config( + dedent( + """ + project: foo + registry: "registry.db" + provider: local + online_store: + type: feast.infra.online_stores.sqlite.SqliteOnlineStore + """ + ), + expect_error=None, + ) + + def test_gcp_config(): _test_config( dedent( From 64a2cb589058d7ca729e61c989bc8503240204b4 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 22 Jun 2021 13:39:25 -0700 Subject: [PATCH 13/43] Run python unit tests in parallel (#1652) Signed-off-by: Achal Shah Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- Makefile | 2 +- sdk/python/setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e1911a93158..475d729e9e9 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ install-python: python -m pip install -e sdk/python -U --use-deprecated=legacy-resolver test-python: - FEAST_TELEMETRY=False pytest --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests + FEAST_TELEMETRY=False pytest -n 4 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests format-python: # Sort diff --git a/sdk/python/setup.py b/sdk/python/setup.py index c198e4ff75e..293e6804e77 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -87,6 +87,7 @@ "urllib3>=1.25.4", "pytest==6.0.0", "pytest-cov", + "pytest-xdist", "pytest-lazy-fixture==0.6.3", "pytest-timeout==1.4.2", "pytest-ordering==0.6.*", From 9e4c90735a13dc74b73cbfc490a74c65d3c7087a Mon Sep 17 00:00:00 2001 From: Tsotne Tabidze Date: Tue, 22 Jun 2021 14:33:25 -0700 Subject: [PATCH 14/43] Rename telemetry to usage (#1660) * Rename telemetry to usage Signed-off-by: Tsotne Tabidze * Update docs Signed-off-by: Tsotne Tabidze * Update .prow and infra Signed-off-by: Tsotne Tabidze * Rename file Signed-off-by: Tsotne Tabidze * Change url Signed-off-by: Tsotne Tabidze * Re-add telemetry.md for backwards-compatibility Signed-off-by: Tsotne Tabidze Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- .prow.yaml | 4 +- docs/SUMMARY.md | 2 +- docs/advanced/telemetry.md | 10 --- docs/reference/usage.md | 11 +++ infra/scripts/test-docker-compose.sh | 2 +- infra/scripts/test-end-to-end.sh | 2 +- infra/scripts/test-integration.sh | 2 +- .../{test-telemetry.sh => test-usage.sh} | 2 +- sdk/python/feast/client.py | 14 +-- sdk/python/feast/constants.py | 4 +- sdk/python/feast/entity.py | 2 +- sdk/python/feast/feature_store.py | 2 +- sdk/python/feast/feature_view.py | 2 +- sdk/python/feast/repo_config.py | 2 +- sdk/python/feast/repo_operations.py | 2 +- sdk/python/feast/{telemetry.py => usage.py} | 76 ++++++++-------- .../test_usage.py} | 88 +++++++++---------- 17 files changed, 112 insertions(+), 115 deletions(-) delete mode 100644 docs/advanced/telemetry.md create mode 100644 docs/reference/usage.md rename infra/scripts/{test-telemetry.sh => test-usage.sh} (91%) rename sdk/python/feast/{telemetry.py => usage.py} (67%) rename sdk/python/{telemetry_tests/test_telemetry.py => usage_tests/test_usage.py} (67%) diff --git a/.prow.yaml b/.prow.yaml index b973cffbbf2..e614e4a2f96 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -64,13 +64,13 @@ presubmits: branches: - ^v0\.(3|4)-branch$ -- name: test-telemetry +- name: test-usage decorate: true run_if_changed: "sdk/python/.*" spec: containers: - image: python:3.7 - command: ["infra/scripts/test-telemetry.sh"] + command: ["infra/scripts/test-usage.sh"] env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /etc/gcloud/service-account.json diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 30b19cfdcc8..b057db87a7e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -43,7 +43,7 @@ * [.feastignore](reference/feature-repository/feast-ignore.md) * [Feast CLI reference](reference/feast-cli-commands.md) * [Python API reference](http://rtd.feast.dev/) -* [Telemetry](reference/telemetry.md) +* [Usage](reference/usage.md) ## Feast on Kubernetes diff --git a/docs/advanced/telemetry.md b/docs/advanced/telemetry.md deleted file mode 100644 index c817e1ce15a..00000000000 --- a/docs/advanced/telemetry.md +++ /dev/null @@ -1,10 +0,0 @@ -# Telemetry - -## How telemetry is used - -The Feast maintainers use anonymous usage statistics to help shape the Feast roadmap. Several client methods are tracked, beginning in Feast 0.9. Users are assigned a UUID which is sent along with the name of the method, the Feast version, the OS \(using `sys.platform`\), and the current time. For more detailed information see [the source code](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/telemetry.py). - -## How to disable telemetry - -To opt out of telemetry, simply set the environment variable `FEAST_TELEMETRY` to `False` in the environment in which the Feast client is run. - diff --git a/docs/reference/usage.md b/docs/reference/usage.md new file mode 100644 index 00000000000..6d37bdfd46a --- /dev/null +++ b/docs/reference/usage.md @@ -0,0 +1,11 @@ +# Usage + +### How Feast SDK usage is measured + +The Feast project logs anonymous usage statistics and errors in order to inform our planning. Several client methods are tracked, beginning in Feast 0.9. Users are assigned a UUID which is sent along with the name of the method, the Feast version, the OS \(using `sys.platform`\), and the current time. + +The [source code](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/usage.py) is available here. + +### How to disable usage logging + +Set the environment variable `FEAST_USAGE` to `False`. diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh index 3681255e465..69cdaac82fa 100755 --- a/infra/scripts/test-docker-compose.sh +++ b/infra/scripts/test-docker-compose.sh @@ -70,4 +70,4 @@ docker exec \ -e DISABLE_FEAST_SERVICE_FIXTURES=true \ --user root \ feast_jupyter_1 bash \ - -c 'cd /feast/tests && python -m pip install -r requirements.txt && FEAST_TELEMETRY=False pytest e2e/ --ingestion-jar https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --job-service-url jobservice:6568 --staging-path file:///shared/staging/ --kafka-brokers kafka:9092 --statsd-url prometheus_statsd:9125 --prometheus-url prometheus_statsd:9102 --feast-version develop' + -c 'cd /feast/tests && python -m pip install -r requirements.txt && FEAST_USAGE=False pytest e2e/ --ingestion-jar https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --job-service-url jobservice:6568 --staging-path file:///shared/staging/ --kafka-brokers kafka:9092 --statsd-url prometheus_statsd:9125 --prometheus-url prometheus_statsd:9102 --feast-version develop' diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index 8120d83efd4..1c94f9c02d3 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -10,6 +10,6 @@ make build-java-no-tests REVISION=develop python -m pip install --upgrade pip setuptools wheel make install-python python -m pip install -qr tests/requirements.txt -export FEAST_TELEMETRY="False" +export FEAST_USAGE="False" su -p postgres -c "PATH=$PATH HOME=/tmp pytest -v tests/e2e/ --feast-version develop" diff --git a/infra/scripts/test-integration.sh b/infra/scripts/test-integration.sh index ad5dd29a4ff..5e88e0281b2 100755 --- a/infra/scripts/test-integration.sh +++ b/infra/scripts/test-integration.sh @@ -4,5 +4,5 @@ python -m pip install --upgrade pip setuptools wheel make install-python python -m pip install -qr tests/requirements.txt -export FEAST_TELEMETRY="False" +export FEAST_USAGE="False" pytest tests/integration --dataproc-cluster-name feast-e2e --dataproc-project kf-feast --dataproc-region us-central1 --dataproc-staging-location gs://feast-templocation-kf-feast diff --git a/infra/scripts/test-telemetry.sh b/infra/scripts/test-usage.sh similarity index 91% rename from infra/scripts/test-telemetry.sh rename to infra/scripts/test-usage.sh index c17f9b107c2..f70fdd62479 100755 --- a/infra/scripts/test-telemetry.sh +++ b/infra/scripts/test-usage.sh @@ -7,5 +7,5 @@ LOGS_ARTIFACT_PATH=/logs/artifacts cd sdk/python/ pip install -e ".[ci]" -cd telemetry_tests/ +cd usage_tests/ pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 9afc4ba9268..c7cd1259ae2 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -69,7 +69,7 @@ ) from feast.protos.feast.serving.ServingService_pb2_grpc import ServingServiceStub from feast.registry import Registry -from feast.telemetry import Telemetry +from feast.usage import Usage _logger = logging.getLogger(__name__) @@ -117,7 +117,7 @@ def __init__(self, options: Optional[Dict[str, str]] = None, **kwargs): if self._config.getboolean(opt.ENABLE_AUTH): self._auth_metadata = feast_auth.get_auth_metadata_plugin(self._config) - self._tele = Telemetry() + self._usage = Usage() @property def config(self) -> Config: @@ -467,7 +467,7 @@ def apply( >>> feast_client.apply(entity) """ - self._tele.log("apply") + self._usage.log("apply") if project is None: project = self.project @@ -581,7 +581,7 @@ def get_entity(self, name: str, project: str = None) -> Entity: none is found """ - self._tele.log("get_entity") + self._usage.log("get_entity") if project is None: project = self.project @@ -706,7 +706,7 @@ def get_feature_table(self, name: str, project: str = None) -> FeatureTable: none is found """ - self._tele.log("get_feature_table") + self._usage.log("get_feature_table") if project is None: project = self.project @@ -847,7 +847,7 @@ def ingest( >>> client.ingest(driver_ft, ft_df) """ - self._tele.log("ingest") + self._usage.log("ingest") if project is None: project = self.project if isinstance(feature_table, str): @@ -972,7 +972,7 @@ def get_online_features( {'sales:daily_transactions': [1.1,1.2], 'sales:customer_id': [0,1]} """ - self._tele.log("get_online_features") + self._usage.log("get_online_features") try: response = self._serving_service.GetOnlineFeaturesV2( GetOnlineFeaturesRequestV2( diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 259920edaee..8bf6ea16ec3 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -260,8 +260,8 @@ class ConfigOptions(metaclass=ConfigMeta): #: Oauth token request url OAUTH_TOKEN_REQUEST_URL: Optional[str] = None - #: Telemetry enabled - TELEMETRY = "True" + #: Usage enabled + USAGE = "True" #: Object store registry REGISTRY_PATH: Optional[str] = None diff --git a/sdk/python/feast/entity.py b/sdk/python/feast/entity.py index 832b9e4db8d..f73e8c70efe 100644 --- a/sdk/python/feast/entity.py +++ b/sdk/python/feast/entity.py @@ -23,7 +23,7 @@ from feast.protos.feast.core.Entity_pb2 import Entity as EntityV2Proto from feast.protos.feast.core.Entity_pb2 import EntityMeta as EntityMetaProto from feast.protos.feast.core.Entity_pb2 import EntitySpecV2 as EntitySpecProto -from feast.telemetry import log_exceptions +from feast.usage import log_exceptions from feast.value_type import ValueType diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index e9b9263bc8b..18526d00609 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -43,7 +43,7 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.registry import Registry from feast.repo_config import RepoConfig, load_repo_config -from feast.telemetry import log_exceptions, log_exceptions_and_usage +from feast.usage import log_exceptions, log_exceptions_and_usage from feast.version import get_version diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 114bb37e613..d89a8a7b6be 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -33,7 +33,7 @@ from feast.protos.feast.core.FeatureView_pb2 import ( MaterializationInterval as MaterializationIntervalProto, ) -from feast.telemetry import log_exceptions +from feast.usage import log_exceptions from feast.value_type import ValueType diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 5587fb59053..c680d94d07a 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -7,7 +7,7 @@ from pydantic.typing import Dict, Optional, Union from feast.importer import get_class_from_type -from feast.telemetry import log_exceptions +from feast.usage import log_exceptions # These dict exists so that: # - existing values for the online store type in featurestore.yaml files continue to work in a backwards compatible way diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index f4a44a74559..4f2cb1981d3 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -21,7 +21,7 @@ from feast.names import adjectives, animals from feast.registry import Registry from feast.repo_config import RepoConfig -from feast.telemetry import log_exceptions_and_usage +from feast.usage import log_exceptions_and_usage def py_path_to_module(path: Path, repo_root: Path) -> str: diff --git a/sdk/python/feast/telemetry.py b/sdk/python/feast/usage.py similarity index 67% rename from sdk/python/feast/telemetry.py rename to sdk/python/feast/usage.py index 7c8247f3af0..a089f660db3 100644 --- a/sdk/python/feast/telemetry.py +++ b/sdk/python/feast/usage.py @@ -25,76 +25,72 @@ from feast.version import get_version -TELEMETRY_ENDPOINT = ( - "https://us-central1-kf-feast.cloudfunctions.net/bq_telemetry_logger" -) +USAGE_ENDPOINT = "https://us-central1-kf-feast.cloudfunctions.net/bq_telemetry_logger" _logger = logging.getLogger(__name__) -class Telemetry: +class Usage: def __init__(self): - self._telemetry_enabled: bool = False + self._usage_enabled: bool = False self.check_env_and_configure() def check_env_and_configure(self): - telemetry_enabled = ( - os.getenv("FEAST_TELEMETRY", default="True") == "True" + usage_enabled = ( + os.getenv("FEAST_USAGE", default="True") == "True" ) # written this way to turn the env var string into a boolean # Check if it changed - if telemetry_enabled != self._telemetry_enabled: - self._telemetry_enabled = telemetry_enabled + if usage_enabled != self._usage_enabled: + self._usage_enabled = usage_enabled - if self._telemetry_enabled: + if self._usage_enabled: try: feast_home_dir = join(expanduser("~"), ".feast") Path(feast_home_dir).mkdir(exist_ok=True) - telemetry_filepath = join(feast_home_dir, "telemetry") + usage_filepath = join(feast_home_dir, "usage") - self._is_test = ( - os.getenv("FEAST_IS_TELEMETRY_TEST", "False") == "True" - ) - self._telemetry_counter = {"get_online_features": 0} + self._is_test = os.getenv("FEAST_IS_USAGE_TEST", "False") == "True" + self._usage_counter = {"get_online_features": 0} - if os.path.exists(telemetry_filepath): - with open(telemetry_filepath, "r") as f: - self._telemetry_id = f.read() + if os.path.exists(usage_filepath): + with open(usage_filepath, "r") as f: + self._usage_id = f.read() else: - self._telemetry_id = str(uuid.uuid4()) + self._usage_id = str(uuid.uuid4()) - with open(telemetry_filepath, "w") as f: - f.write(self._telemetry_id) + with open(usage_filepath, "w") as f: + f.write(self._usage_id) print( "Feast is an open source project that collects anonymized error reporting and usage statistics. To opt out or learn" - " more see https://docs.feast.dev/reference/telemetry" + " more see https://docs.feast.dev/reference/usage" ) except Exception as e: - _logger.debug(f"Unable to configure telemetry {e}") + _logger.debug(f"Unable to configure usage {e}") @property - def telemetry_id(self) -> Optional[str]: - if os.getenv("FEAST_FORCE_TELEMETRY_UUID"): - return os.getenv("FEAST_FORCE_TELEMETRY_UUID") - return self._telemetry_id + def usage_id(self) -> Optional[str]: + if os.getenv("FEAST_FORCE_USAGE_UUID"): + return os.getenv("FEAST_FORCE_USAGE_UUID") + return self._usage_id def log(self, function_name: str): self.check_env_and_configure() - if self._telemetry_enabled and self.telemetry_id: + if self._usage_enabled and self.usage_id: if function_name == "get_online_features": - if self._telemetry_counter["get_online_features"] % 10000 != 0: - self._telemetry_counter["get_online_features"] += 1 + if self._usage_counter["get_online_features"] % 10000 != 0: + self._usage_counter["get_online_features"] += 1 return json = { "function_name": function_name, - "telemetry_id": self.telemetry_id, + "telemetry_id": self.usage_id, "timestamp": datetime.utcnow().isoformat(), "version": get_version(), "os": sys.platform, "is_test": self._is_test, } try: - requests.post(TELEMETRY_ENDPOINT, json=json) + requests.post(USAGE_ENDPOINT, json=json) except Exception as e: if self._is_test: raise e @@ -104,17 +100,17 @@ def log(self, function_name: str): def log_exception(self, error_type: str, traceback: List[Tuple[str, int, str]]): self.check_env_and_configure() - if self._telemetry_enabled and self.telemetry_id: + if self._usage_enabled and self.usage_id: json = { "error_type": error_type, "traceback": traceback, - "telemetry_id": self.telemetry_id, + "telemetry_id": self.usage_id, "version": get_version(), "os": sys.platform, "is_test": self._is_test, } try: - requests.post(TELEMETRY_ENDPOINT, json=json) + requests.post(USAGE_ENDPOINT, json=json) except Exception as e: if self._is_test: raise e @@ -141,7 +137,7 @@ def exception_logging_wrapper(*args, **kwargs): ) ) tb = tb.tb_next - tele.log_exception(error_type, trace_to_log) + usage.log_exception(error_type, trace_to_log) raise return result @@ -153,7 +149,7 @@ def log_exceptions_and_usage(func): def exception_logging_wrapper(*args, **kwargs): try: result = func(*args, **kwargs) - tele.log(func.__name__) + usage.log(func.__name__) except Exception as e: error_type = type(e).__name__ trace_to_log = [] @@ -167,7 +163,7 @@ def exception_logging_wrapper(*args, **kwargs): ) ) tb = tb.tb_next - tele.log_exception(error_type, trace_to_log) + usage.log_exception(error_type, trace_to_log) raise return result @@ -178,5 +174,5 @@ def _trim_filename(filename: str) -> str: return filename.split("/")[-1] -# Single global telemetry object -tele = Telemetry() +# Single global usage object +usage = Usage() diff --git a/sdk/python/telemetry_tests/test_telemetry.py b/sdk/python/usage_tests/test_usage.py similarity index 67% rename from sdk/python/telemetry_tests/test_telemetry.py rename to sdk/python/usage_tests/test_usage.py index 9b35bf3c17a..e6b7760fabb 100644 --- a/sdk/python/telemetry_tests/test_telemetry.py +++ b/sdk/python/usage_tests/test_usage.py @@ -25,18 +25,18 @@ from feast import Client, Entity, ValueType, FeatureStore, RepoConfig -TELEMETRY_BIGQUERY_TABLE = ( +USAGE_BIGQUERY_TABLE = ( "kf-feast.feast_telemetry.cloudfunctions_googleapis_com_cloud_functions" ) -def test_telemetry_on_v09(mocker): +def test_usage_on_v09(mocker): # Setup environment old_environ = dict(os.environ) - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id - test_client = Client(serving_url=None, core_url=None, telemetry=True) + os.environ["FEAST_IS_USAGE_TEST"] = "True" + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id + test_client = Client(serving_url=None, core_url=None, usage=True) test_client.set_project("project1") entity = Entity( name="driver_car_id", @@ -54,17 +54,17 @@ def test_telemetry_on_v09(mocker): os.environ.clear() os.environ.update(old_environ) - ensure_bigquery_telemetry_id_with_retry(test_telemetry_id) + ensure_bigquery_usage_id_with_retry(test_usage_id) -def test_telemetry_off_v09(mocker): +def test_usage_off_v09(mocker): old_environ = dict(os.environ) - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id - os.environ["FEAST_TELEMETRY"] = "False" + os.environ["FEAST_IS_USAGE_TEST"] = "True" + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id + os.environ["FEAST_USAGE"] = "False" - test_client = Client(serving_url=None, core_url=None, telemetry=False) + test_client = Client(serving_url=None, core_url=None, usage=False) test_client.set_project("project1") entity = Entity( name="driver_car_id", @@ -82,16 +82,16 @@ def test_telemetry_off_v09(mocker): os.environ.clear() os.environ.update(old_environ) sleep(30) - rows = read_bigquery_telemetry_id(test_telemetry_id) + rows = read_bigquery_usage_id(test_usage_id) assert rows.total_rows == 0 -def test_telemetry_on(): +def test_usage_on(): old_environ = dict(os.environ) - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - os.environ["FEAST_TELEMETRY"] = "True" + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id + os.environ["FEAST_IS_USAGE_TEST"] = "True" + os.environ["FEAST_USAGE"] = "True" with tempfile.TemporaryDirectory() as temp_dir: test_feature_store = FeatureStore( @@ -115,15 +115,15 @@ def test_telemetry_on(): os.environ.clear() os.environ.update(old_environ) - ensure_bigquery_telemetry_id_with_retry(test_telemetry_id) + ensure_bigquery_usage_id_with_retry(test_usage_id) -def test_telemetry_off(): +def test_usage_off(): old_environ = dict(os.environ) - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - os.environ["FEAST_TELEMETRY"] = "False" - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_IS_USAGE_TEST"] = "True" + os.environ["FEAST_USAGE"] = "False" + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id with tempfile.TemporaryDirectory() as temp_dir: test_feature_store = FeatureStore( @@ -147,16 +147,16 @@ def test_telemetry_off(): os.environ.clear() os.environ.update(old_environ) sleep(30) - rows = read_bigquery_telemetry_id(test_telemetry_id) + rows = read_bigquery_usage_id(test_usage_id) assert rows.total_rows == 0 -def test_exception_telemetry_on(): +def test_exception_usage_on(): old_environ = dict(os.environ) - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - os.environ["FEAST_TELEMETRY"] = "True" + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id + os.environ["FEAST_IS_USAGE_TEST"] = "True" + os.environ["FEAST_USAGE"] = "True" try: test_feature_store = FeatureStore("/tmp/non_existent_directory") @@ -165,15 +165,15 @@ def test_exception_telemetry_on(): os.environ.clear() os.environ.update(old_environ) - ensure_bigquery_telemetry_id_with_retry(test_telemetry_id) + ensure_bigquery_usage_id_with_retry(test_usage_id) -def test_exception_telemetry_off(): +def test_exception_usage_off(): old_environ = dict(os.environ) - test_telemetry_id = str(uuid.uuid4()) - os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" - os.environ["FEAST_TELEMETRY"] = "False" - os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id + test_usage_id = str(uuid.uuid4()) + os.environ["FEAST_IS_USAGE_TEST"] = "True" + os.environ["FEAST_USAGE"] = "False" + os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id try: test_feature_store = FeatureStore("/tmp/non_existent_directory") @@ -183,18 +183,18 @@ def test_exception_telemetry_off(): os.environ.clear() os.environ.update(old_environ) sleep(30) - rows = read_bigquery_telemetry_id(test_telemetry_id) + rows = read_bigquery_usage_id(test_usage_id) assert rows.total_rows == 0 @retry(wait=wait_exponential(multiplier=1, min=1, max=10), stop=stop_after_attempt(5)) -def ensure_bigquery_telemetry_id_with_retry(telemetry_id): - rows = read_bigquery_telemetry_id(telemetry_id) +def ensure_bigquery_usage_id_with_retry(usage_id): + rows = read_bigquery_usage_id(usage_id) if rows.total_rows != 1: - raise Exception(f"Could not find telemetry id: {telemetry_id}") + raise Exception(f"Could not find usage id: {usage_id}") -def read_bigquery_telemetry_id(telemetry_id): +def read_bigquery_usage_id(usage_id): bq_client = bigquery.Client() query = f""" SELECT @@ -203,11 +203,11 @@ def read_bigquery_telemetry_id(telemetry_id): SELECT JSON_EXTRACT(textPayload, '$.telemetry_id') AS telemetry_id FROM - `{TELEMETRY_BIGQUERY_TABLE}` + `{USAGE_BIGQUERY_TABLE}` WHERE timestamp >= TIMESTAMP(\"{datetime.utcnow().date().isoformat()}\")) WHERE - telemetry_id = '\"{telemetry_id}\"' + telemetry_id = '\"{usage_id}\"' """ query_job = bq_client.query(query) return query_job.result() From b951282ee79265e3870843ad83969d21da8504b8 Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Wed, 23 Jun 2021 12:51:36 -0400 Subject: [PATCH 15/43] resolved final comments on PR (variable renaming, refactor tests) Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/feature_store.py | 18 +++-- sdk/python/tests/test_historical_retrieval.py | 65 +++++++++---------- .../test_offline_online_store_consistency.py | 10 +-- 3 files changed, 46 insertions(+), 47 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 18526d00609..dbd1bdb1f4b 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -281,9 +281,9 @@ def get_historical_features( SQL query. The query must be of a format supported by the configured offline store (e.g., BigQuery) feature_refs: A list of features that should be retrieved from the offline store. Feature references are of the format "feature_view:feature", e.g., "customer_fv:daily_transactions". - full_feature_names: By default, this value is set to False. By setting the value to True, this adds the - feature view prefixes to the feature names, changing them from the format "feature" to - "feature_view__feature" (e.g., "daily_transactions" changes to "customer_fv__daily_transactions"). + full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, + changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to + "customer_fv__daily_transactions"). By default, this value is set to False. Returns: RetrievalJob which can be used to materialize the results. @@ -557,7 +557,9 @@ def get_online_features( project=self.project, allow_cache=True ) - grouped_refs = _group_refs(feature_refs, all_feature_views, full_feature_names) + grouped_refs = _validate_and_group_feature_refs( + feature_refs, all_feature_views, full_feature_names + ) for table, requested_features in grouped_refs: entity_keys = _get_table_entity_keys( table, union_of_entity_keys, entity_name_to_join_key_map @@ -616,7 +618,7 @@ def _entity_row_to_field_values( return result -def _group_refs( +def _validate_and_group_feature_refs( feature_refs: List[str], all_feature_views: List[FeatureView], full_feature_names: bool = False, @@ -658,10 +660,12 @@ def _get_requested_feature_views( full_feature_names: bool, ) -> List[FeatureView]: """Get list of feature views based on feature references""" - # TODO: Get rid of this function. We only need _group_refs + # TODO: Get rid of this function. We only need _validate_and_group_feature_refs return list( view - for view, _ in _group_refs(feature_refs, all_feature_views, full_feature_names) + for view, _ in _validate_and_group_feature_refs( + feature_refs, all_feature_views, full_feature_names + ) ) diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index 61f9564f87f..c3bbdbbe469 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -19,7 +19,7 @@ from feast.entity import Entity from feast.errors import FeatureNameCollisionError from feast.feature import Feature -from feast.feature_store import FeatureStore, _group_refs +from feast.feature_store import FeatureStore, _validate_and_group_feature_refs from feast.feature_view import FeatureView from feast.infra.offline_stores.bigquery import BigQueryOfflineStoreConfig from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig @@ -31,7 +31,7 @@ PROJECT_NAME = "default" -def generate_entities(date, infer_event_timestamp_col): +def generate_entities(date, infer_event_timestamp_col, order_count: int = 1000): end_date = date before_start_date = end_date - timedelta(days=365) start_date = end_date - timedelta(days=7) @@ -43,7 +43,7 @@ def generate_entities(date, infer_event_timestamp_col): drivers=driver_entities, start_date=before_start_date, end_date=after_end_date, - order_count=1000, + order_count=order_count, infer_event_timestamp_col=infer_event_timestamp_col, ) return customer_entities, driver_entities, end_date, orders_df, start_date @@ -609,39 +609,34 @@ def test_historical_features_from_bigquery_sources( @pytest.mark.integration def test_feature_name_collision_on_historical_retrieval_from_parquet_sources(): - start_date = datetime.now().replace(microsecond=0, second=0, minute=0) - (customer_entities, driver_entities, end_date, _, start_date,) = generate_entities( - start_date, True + + driver_source = FileSource( + path="driver_stats_path", + event_timestamp_column="datetime", + created_timestamp_column="created", ) + driver_fv = create_driver_hourly_stats_feature_view(driver_source) + customer_source = FileSource( + path="customer_profile_path", + event_timestamp_column="datetime", + created_timestamp_column="created", + ) + customer_fv = create_customer_daily_profile_feature_view(customer_source) - with TemporaryDirectory() as temp_dir: - driver_df = driver_data.create_driver_hourly_stats_df( - driver_entities, start_date, end_date - ) - driver_source = stage_driver_hourly_stats_parquet_source(temp_dir, driver_df) - driver_fv = create_driver_hourly_stats_feature_view(driver_source) - customer_df = driver_data.create_customer_daily_profile_df( - customer_entities, start_date, end_date - ) - customer_source = stage_customer_daily_profile_parquet_source( - temp_dir, customer_df + # _validate_and_group_feature_refs is the function that checks for colliding feature names + with pytest.raises(FeatureNameCollisionError): + _validate_and_group_feature_refs( + feature_refs=[ + "driver_stats:conv_rate", + "driver_stats:avg_daily_trips", + "customer_profile:current_balance", + "customer_profile:avg_passenger_count", + "customer_profile:lifetime_trip_count", + "customer_profile:avg_daily_trips", + ], + all_feature_views=[driver_fv, customer_fv], + full_feature_names=False, ) - customer_fv = create_customer_daily_profile_feature_view(customer_source) - - # _group_refs is the function that checks for colliding feature names - with pytest.raises(FeatureNameCollisionError): - _group_refs( - feature_refs=[ - "driver_stats:conv_rate", - "driver_stats:avg_daily_trips", - "customer_profile:current_balance", - "customer_profile:avg_passenger_count", - "customer_profile:lifetime_trip_count", - "customer_profile:avg_daily_trips", - ], - all_feature_views=[driver_fv, customer_fv], - full_feature_names=False, - ) def test_feature_name_collision_on_historical_retrieval_from_bigquery_sources(): @@ -667,9 +662,9 @@ def test_feature_name_collision_on_historical_retrieval_from_bigquery_sources(): ) customer_fv = create_customer_daily_profile_feature_view(customer_source) - # _group_refs is the function that checks for colliding feature names + # _validate_and_group_feature_refs is the function that checks for colliding feature names with pytest.raises(FeatureNameCollisionError): - _group_refs( + _validate_and_group_feature_refs( feature_refs=[ "driver_stats:conv_rate", "driver_stats:avg_daily_trips", diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py index e2faec2d6db..b7fb1304947 100644 --- a/sdk/python/tests/test_offline_online_store_consistency.py +++ b/sdk/python/tests/test_offline_online_store_consistency.py @@ -234,7 +234,7 @@ def check_offline_and_online_features( def run_offline_online_store_consistency_test( - fs: FeatureStore, fv: FeatureView, ffn: bool + fs: FeatureStore, fv: FeatureView, full_feature_names: bool ) -> None: now = datetime.utcnow() # Run materialize() @@ -250,7 +250,7 @@ def run_offline_online_store_consistency_test( driver_id=1, event_timestamp=end_date, expected_value=0.3, - full_feature_names=ffn, + full_feature_names=full_feature_names, ) check_offline_and_online_features( @@ -259,7 +259,7 @@ def run_offline_online_store_consistency_test( driver_id=2, event_timestamp=end_date, expected_value=None, - full_feature_names=ffn, + full_feature_names=full_feature_names, ) # check prior value for materialize_incremental() @@ -269,7 +269,7 @@ def run_offline_online_store_consistency_test( driver_id=3, event_timestamp=end_date, expected_value=4, - full_feature_names=ffn, + full_feature_names=full_feature_names, ) # run materialize_incremental() @@ -282,7 +282,7 @@ def run_offline_online_store_consistency_test( driver_id=3, event_timestamp=now, expected_value=5, - full_feature_names=ffn, + full_feature_names=full_feature_names, ) From a68b12b5a62313e0f528c7a96028793205ff13f7 Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Wed, 23 Jun 2021 15:58:41 -0400 Subject: [PATCH 16/43] reformatted after merge conflict Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/errors.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 6b8ff45d714..58e34dede42 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -81,9 +81,11 @@ def __init__(self, offline_store_name: str, data_source_name: str): class FeatureNameCollisionError(Exception): def __init__(self, feature_name_collisions: str): super().__init__( - f"The following feature name(s) have collisions: {feature_name_collisions}. Set 'full_feature_names' argument in the data retrieval function to True to use the full feature name which is prefixed by the feature view name." + f"The following feature name(s) have collisions: {feature_name_collisions}. Set 'full_feature_names' " + f"argument in the data retrieval function to True to use the full feature name which is prefixed by the feature view name." ) + class FeastOnlineStoreInvalidName(Exception): def __init__(self, online_store_class_name: str): From 094dbf319fa3c8d444145a3db3153bfc0f24dbf4 Mon Sep 17 00:00:00 2001 From: Willem Pienaar Date: Wed, 23 Jun 2021 22:24:05 -0700 Subject: [PATCH 17/43] Update CHANGELOG for Feast v0.11.0 Signed-off-by: Willem Pienaar Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- CHANGELOG.md | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54d598be5b8..da7d7933dbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,83 @@ # Changelog +## [v0.11.0](https://github.com/feast-dev/feast/tree/v0.11.0) (2021-06-24) + +[Full Changelog](https://github.com/feast-dev/feast/compare/v0.10.8...v0.11.0) + +**Implemented enhancements:** + +- Allow BigQuery project to be configured [\#1656](https://github.com/feast-dev/feast/pull/1656) ([MattDelac](https://github.com/MattDelac)) +- Add to_bigquery function to BigQueryRetrievalJob [\#1634](https://github.com/feast-dev/feast/pull/1634) ([vtao2](https://github.com/vtao2)) +- Add AWS authentication using github actions [\#1629](https://github.com/feast-dev/feast/pull/1629) ([tsotnet](https://github.com/tsotnet)) +- Introduce an OnlineStore interface [\#1628](https://github.com/feast-dev/feast/pull/1628) ([achals](https://github.com/achals)) +- Add to_df to convert get_online_feature response to pandas dataframe [\#1623](https://github.com/feast-dev/feast/pull/1623) ([tedhtchang](https://github.com/tedhtchang)) +- Add datastore namespace option in configs [\#1581](https://github.com/feast-dev/feast/pull/1581) ([tsotnet](https://github.com/tsotnet)) +- Add offline_store config [\#1552](https://github.com/feast-dev/feast/pull/1552) ([tsotnet](https://github.com/tsotnet)) +- Entity value_type inference for Feature Repo registration [\#1538](https://github.com/feast-dev/feast/pull/1538) ([mavysavydav](https://github.com/mavysavydav)) +- Inferencing of Features in FeatureView and timestamp column of DataSource [\#1523](https://github.com/feast-dev/feast/pull/1523) ([mavysavydav](https://github.com/mavysavydav)) +- Add Unix Timestamp value type [\#1520](https://github.com/feast-dev/feast/pull/1520) ([MattDelac](https://github.com/MattDelac)) +- Add support for Redis and Redis Cluster [\#1511](https://github.com/feast-dev/feast/pull/1511) ([qooba](https://github.com/qooba)) +- Add path option to cli [\#1509](https://github.com/feast-dev/feast/pull/1509) ([tedhtchang](https://github.com/tedhtchang)) + +**Fixed bugs:** + +- Schema Inferencing should happen at apply time [\#1646](https://github.com/feast-dev/feast/pull/1646) ([mavysavydav](https://github.com/mavysavydav)) +- Don't use .result\(\) in BigQueryOfflineStore, since it still leads to OOM [\#1642](https://github.com/feast-dev/feast/pull/1642) ([tsotnet](https://github.com/tsotnet)) +- Don't load entire bigquery query results in memory [\#1638](https://github.com/feast-dev/feast/pull/1638) ([tsotnet](https://github.com/tsotnet)) +- Remove file loader & its test [\#1632](https://github.com/feast-dev/feast/pull/1632) ([tsotnet](https://github.com/tsotnet)) +- Provide descriptive error on invalid table reference [\#1627](https://github.com/feast-dev/feast/pull/1627) ([codyjlin](https://github.com/codyjlin)) +- Fix ttl duration when ttl is None [\#1624](https://github.com/feast-dev/feast/pull/1624) ([MattDelac](https://github.com/MattDelac)) +- Fix race condition in historical e2e tests [\#1620](https://github.com/feast-dev/feast/pull/1620) ([woop](https://github.com/woop)) +- Add validations when materializing from file sources [\#1615](https://github.com/feast-dev/feast/pull/1615) ([achals](https://github.com/achals)) +- Add entity column validations when getting historical features from bigquery [\#1614](https://github.com/feast-dev/feast/pull/1614) ([achals](https://github.com/achals)) +- Allow telemetry configuration to fail gracefully [\#1612](https://github.com/feast-dev/feast/pull/1612) ([achals](https://github.com/achals)) +- Update type conversion from pandas to timestamp to support various the timestamp types [\#1603](https://github.com/feast-dev/feast/pull/1603) ([achals](https://github.com/achals)) +- Add current directory in sys path for CLI commands that might depend on custom providers [\#1594](https://github.com/feast-dev/feast/pull/1594) ([MattDelac](https://github.com/MattDelac)) +- Fix contention issue [\#1582](https://github.com/feast-dev/feast/pull/1582) ([woop](https://github.com/woop)) +- Ensure that only None types fail predicate [\#1580](https://github.com/feast-dev/feast/pull/1580) ([woop](https://github.com/woop)) +- Don't create bigquery dataset if it already exists [\#1569](https://github.com/feast-dev/feast/pull/1569) ([tsotnet](https://github.com/tsotnet)) +- Don't lose materialization interval tracking when re-applying feature views [\#1559](https://github.com/feast-dev/feast/pull/1559) ([jklegar](https://github.com/jklegar)) +- Validate project and repo names for apply and init commands [\#1558](https://github.com/feast-dev/feast/pull/1558) ([tedhtchang](https://github.com/tedhtchang)) +- Bump supported Python version to 3.7 [\#1504](https://github.com/feast-dev/feast/pull/1504) ([tsotnet](https://github.com/tsotnet)) + +**Merged pull requests:** + +- Rename telemetry to usage [\#1660](https://github.com/feast-dev/feast/pull/1660) ([tsotnet](https://github.com/tsotnet)) +- Refactor OfflineStoreConfig classes into their owning modules [\#1657](https://github.com/feast-dev/feast/pull/1657) ([achals](https://github.com/achals)) +- Run python unit tests in parallel [\#1652](https://github.com/feast-dev/feast/pull/1652) ([achals](https://github.com/achals)) +- Refactor OnlineStoreConfig classes into owning modules [\#1649](https://github.com/feast-dev/feast/pull/1649) ([achals](https://github.com/achals)) +- Fix table\_refs in BigQuerySource definitions [\#1644](https://github.com/feast-dev/feast/pull/1644) ([tsotnet](https://github.com/tsotnet)) +- Make test historical retrieval longer [\#1630](https://github.com/feast-dev/feast/pull/1630) ([MattDelac](https://github.com/MattDelac)) +- Fix failing historical retrieval assertion [\#1622](https://github.com/feast-dev/feast/pull/1622) ([woop](https://github.com/woop)) +- Add a specific error for missing columns during materialization [\#1619](https://github.com/feast-dev/feast/pull/1619) ([achals](https://github.com/achals)) +- Use drop\_duplicates\(\) instead of groupby \(about 1.5~2x faster\) [\#1617](https://github.com/feast-dev/feast/pull/1617) ([rightx2](https://github.com/rightx2)) +- Optimize historical retrieval with BigQuery offline store [\#1602](https://github.com/feast-dev/feast/pull/1602) ([MattDelac](https://github.com/MattDelac)) +- Use CONCAT\(\) instead of ROW\_NUMBER\(\) [\#1601](https://github.com/feast-dev/feast/pull/1601) ([MattDelac](https://github.com/MattDelac)) +- Minor doc fix in the code snippet: Fix to reference the right instance for the retrieved job instance object [\#1599](https://github.com/feast-dev/feast/pull/1599) ([dmatrix](https://github.com/dmatrix)) +- Repo and project names should not start with an underscore [\#1597](https://github.com/feast-dev/feast/pull/1597) ([tedhtchang](https://github.com/tedhtchang)) +- Append nanoseconds to dataset name in test\_historical\_retrival to prevent tests stomping over each other [\#1593](https://github.com/feast-dev/feast/pull/1593) ([achals](https://github.com/achals)) +- Make start and end timestamps tz aware in the CLI [\#1590](https://github.com/feast-dev/feast/pull/1590) ([achals](https://github.com/achals)) +- Bump fastavro version [\#1585](https://github.com/feast-dev/feast/pull/1585) ([kevinhu](https://github.com/kevinhu)) +- Change OfflineStore class description [\#1571](https://github.com/feast-dev/feast/pull/1571) ([tedhtchang](https://github.com/tedhtchang)) +- Fix Sphinx documentation building [\#1563](https://github.com/feast-dev/feast/pull/1563) ([woop](https://github.com/woop)) +- Add test coverage and remove MacOS integration tests [\#1562](https://github.com/feast-dev/feast/pull/1562) ([woop](https://github.com/woop)) +- Improve GCP exception handling [\#1561](https://github.com/feast-dev/feast/pull/1561) ([woop](https://github.com/woop)) +- Update default cli no option help message [\#1550](https://github.com/feast-dev/feast/pull/1550) ([tedhtchang](https://github.com/tedhtchang)) +- Add opt-out exception logging telemetry [\#1535](https://github.com/feast-dev/feast/pull/1535) ([jklegar](https://github.com/jklegar)) +- Add instruction for install Feast on IKS and OpenShift using Kustomize [\#1534](https://github.com/feast-dev/feast/pull/1534) ([tedhtchang](https://github.com/tedhtchang)) +- BigQuery type to Feast type conversion chart update [\#1530](https://github.com/feast-dev/feast/pull/1530) ([mavysavydav](https://github.com/mavysavydav)) +- remove unnecessay path join in setup.py [\#1529](https://github.com/feast-dev/feast/pull/1529) ([shihabuddinbuet](https://github.com/shihabuddinbuet)) +- Add roadmap to documentation [\#1528](https://github.com/feast-dev/feast/pull/1528) ([woop](https://github.com/woop)) +- Add test matrix for different Python versions [\#1526](https://github.com/feast-dev/feast/pull/1526) ([woop](https://github.com/woop)) +- Update broken urls in the github pr template file [\#1521](https://github.com/feast-dev/feast/pull/1521) ([tedhtchang](https://github.com/tedhtchang)) +- Add a fixed timestamp to quickstart data [\#1513](https://github.com/feast-dev/feast/pull/1513) ([jklegar](https://github.com/jklegar)) +- Make gcp imports optional [\#1512](https://github.com/feast-dev/feast/pull/1512) ([jklegar](https://github.com/jklegar)) +- Fix documentation inconsistency [\#1510](https://github.com/feast-dev/feast/pull/1510) ([jongillham](https://github.com/jongillham)) +- Upgrade grpcio version in python SDK [\#1508](https://github.com/feast-dev/feast/pull/1508) ([szalai1](https://github.com/szalai1)) +- pre-commit command typo fix in CONTRIBUTING.md [\#1506](https://github.com/feast-dev/feast/pull/1506) ([mavysavydav](https://github.com/mavysavydav)) +- Add optional telemetry to other CLI commands [\#1505](https://github.com/feast-dev/feast/pull/1505) ([jklegar](https://github.com/jklegar)) + + ## [v0.10.8](https://github.com/feast-dev/feast/tree/v0.10.8) (2021-06-17) [Full Changelog](https://github.com/feast-dev/feast/compare/v0.10.7...v0.10.8) From 0a148f98d851a86801ecd2362933809a942b5ae7 Mon Sep 17 00:00:00 2001 From: Peter Szalai Date: Fri, 25 Jun 2021 22:47:19 +0200 Subject: [PATCH 18/43] Update charts README (#1659) Adding feast jupyter link to it. + Fix the helm 'feast-serving' name in aws/azure terraform. Signed-off-by: szalai1 Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- infra/charts/README.md | 3 +-- infra/terraform/aws/helm.tf | 2 +- infra/terraform/azure/helm.tf | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/infra/charts/README.md b/infra/charts/README.md index f9df0958091..751bd4f483a 100644 --- a/infra/charts/README.md +++ b/infra/charts/README.md @@ -5,5 +5,4 @@ Feast Helm Charts have been moved out of this repository. * The master chart can now be found in at: https://github.com/feast-dev/feast-helm-charts * Feast Serving & Core Charts are at: https://github.com/feast-dev/feast-java/tree/master/infra * Feast Spark (Job Service) is at: https://github.com/feast-dev/feast-spark/tree/master/infra/charts/feast-spark - -This repository still contains the Feast Jupyter Helm Chart, which installs a complete Feast Jupyter Server into a Kubernetes cluster. \ No newline at end of file +* Feast Jupyter server is at: https://github.com/feast-dev/feast-helm-charts/blob/main/charts/feast-jupyter/README.md diff --git a/infra/terraform/aws/helm.tf b/infra/terraform/aws/helm.tf index 8a13fe8d407..179097bd155 100644 --- a/infra/terraform/aws/helm.tf +++ b/infra/terraform/aws/helm.tf @@ -51,7 +51,7 @@ locals { } } - "feast-online-serving" = { + "feast-serving" = { "application-override.yaml" = { enabled = true feast = { diff --git a/infra/terraform/azure/helm.tf b/infra/terraform/azure/helm.tf index 8c28762a438..c1f85a3699c 100644 --- a/infra/terraform/azure/helm.tf +++ b/infra/terraform/azure/helm.tf @@ -23,7 +23,7 @@ locals { } } - feast-online-serving = { + feast-serving = { enabled = true "application-override.yaml" = { feast = { From 0ce821021300463dacda52ca4348a78b0d1fe218 Mon Sep 17 00:00:00 2001 From: Nel Swanepoel Date: Fri, 25 Jun 2021 21:53:23 +0100 Subject: [PATCH 19/43] Added Redis to list of online stores for local provider in providers reference doc. (#1668) Signed-off-by: Nel Swanepoel Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- docs/reference/providers/local.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/providers/local.md b/docs/reference/providers/local.md index 210be132ad6..a63fe6ee588 100644 --- a/docs/reference/providers/local.md +++ b/docs/reference/providers/local.md @@ -3,7 +3,7 @@ ### Description * Offline Store: Uses the File offline store by default. Also supports BigQuery as the offline store. -* Online Store: Uses the Sqlite online store by default. Also supports Datastore as an online store. +* Online Store: Uses the Sqlite online store by default. Also supports Redis and Datastore as online stores. ### Example From d71e4c50b06b04fd1371a92bcac2e8317f60469a Mon Sep 17 00:00:00 2001 From: David Y Liu <7172604+mavysavydav@users.noreply.github.com> Date: Fri, 25 Jun 2021 15:19:23 -0700 Subject: [PATCH 20/43] Grouped inferencing statements together in apply methods for easier readability (#1667) * grouped inferencing statements together Signed-off-by: David Y Liu * update in testing Signed-off-by: David Y Liu Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/feature_store.py | 9 ++++++--- sdk/python/feast/inference.py | 6 ++---- sdk/python/feast/repo_operations.py | 30 +++++++++++++---------------- sdk/python/tests/test_inference.py | 27 ++++++++++++++------------ 4 files changed, 36 insertions(+), 36 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index dbd1bdb1f4b..22d9df8d176 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -31,8 +31,8 @@ ) from feast.feature_view import FeatureView from feast.inference import ( - infer_entity_value_type_from_feature_views, update_data_sources_with_inferred_event_timestamp_col, + update_entities_with_inferred_types_from_feature_views, ) from feast.infra.provider import Provider, RetrievalJob, get_provider from feast.online_response import OnlineResponse, _infer_online_entity_rows @@ -228,8 +228,11 @@ def apply( assert isinstance(objects, list) views_to_update = [ob for ob in objects if isinstance(ob, FeatureView)] - entities_to_update = infer_entity_value_type_from_feature_views( - [ob for ob in objects if isinstance(ob, Entity)], views_to_update + entities_to_update = [ob for ob in objects if isinstance(ob, Entity)] + + # Make inferences + update_entities_with_inferred_types_from_feature_views( + entities_to_update, views_to_update ) update_data_sources_with_inferred_event_timestamp_col( [view.input for view in views_to_update] diff --git a/sdk/python/feast/inference.py b/sdk/python/feast/inference.py index fac2155ee21..af95f9d2551 100644 --- a/sdk/python/feast/inference.py +++ b/sdk/python/feast/inference.py @@ -8,9 +8,9 @@ from feast.value_type import ValueType -def infer_entity_value_type_from_feature_views( +def update_entities_with_inferred_types_from_feature_views( entities: List[Entity], feature_views: List[FeatureView] -) -> List[Entity]: +) -> None: """ Infer entity value type by examining schema of feature view input sources """ @@ -57,8 +57,6 @@ def infer_entity_value_type_from_feature_views( entity.value_type = inferred_value_type - return entities - def update_data_sources_with_inferred_event_timestamp_col( data_sources: List[Union[BigQuerySource, FileSource]], diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 4f2cb1981d3..f707f19c41b 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -14,8 +14,8 @@ from feast import Entity, FeatureTable from feast.feature_view import FeatureView from feast.inference import ( - infer_entity_value_type_from_feature_views, update_data_sources_with_inferred_event_timestamp_col, + update_entities_with_inferred_types_from_feature_views, ) from feast.infra.provider import get_provider from feast.names import adjectives, animals @@ -131,13 +131,19 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): registry._initialize_registry() sys.dont_write_bytecode = True repo = parse_repo(repo_path) - repo = ParsedRepo( - feature_tables=repo.feature_tables, - entities=infer_entity_value_type_from_feature_views( - repo.entities, repo.feature_views - ), - feature_views=repo.feature_views, + data_sources = [t.input for t in repo.feature_views] + + # Make sure the data source used by this feature view is supported by Feast + for data_source in data_sources: + data_source.validate() + + # Make inferences + update_entities_with_inferred_types_from_feature_views( + repo.entities, repo.feature_views ) + update_data_sources_with_inferred_event_timestamp_col(data_sources) + for view in repo.feature_views: + view.infer_features_from_input_source() sys.dont_write_bytecode = False for entity in repo.entities: @@ -151,16 +157,6 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): for t in repo.feature_views: repo_table_names.add(t.name) - data_sources = [t.input for t in repo.feature_views] - - # Make sure the data source used by this feature view is supported by Feast - for data_source in data_sources: - data_source.validate() - - update_data_sources_with_inferred_event_timestamp_col(data_sources) - for view in repo.feature_views: - view.infer_features_from_input_source() - tables_to_delete = [] for registry_table in registry.list_feature_tables(project=project): if registry_table.name not in repo_table_names: diff --git a/sdk/python/tests/test_inference.py b/sdk/python/tests/test_inference.py index 1f626ac3cd3..9405bce1f28 100644 --- a/sdk/python/tests/test_inference.py +++ b/sdk/python/tests/test_inference.py @@ -9,12 +9,14 @@ from feast.errors import RegistryInferenceFailure from feast.feature_view import FeatureView from feast.inference import ( - infer_entity_value_type_from_feature_views, update_data_sources_with_inferred_event_timestamp_col, + update_entities_with_inferred_types_from_feature_views, ) -def test_infer_entity_value_type_from_feature_views(simple_dataset_1, simple_dataset_2): +def test_update_entities_with_inferred_types_from_feature_views( + simple_dataset_1, simple_dataset_2 +): with prep_file_source( df=simple_dataset_1, event_timestamp_column="ts_1" ) as file_source, prep_file_source( @@ -24,22 +26,23 @@ def test_infer_entity_value_type_from_feature_views(simple_dataset_1, simple_dat fv1 = FeatureView(name="fv1", entities=["id"], input=file_source, ttl=None,) fv2 = FeatureView(name="fv2", entities=["id"], input=file_source_2, ttl=None,) - actual_1 = infer_entity_value_type_from_feature_views( - [Entity(name="id")], [fv1] - ) - actual_2 = infer_entity_value_type_from_feature_views( - [Entity(name="id")], [fv2] - ) - assert actual_1 == [Entity(name="id", value_type=ValueType.INT64)] - assert actual_2 == [Entity(name="id", value_type=ValueType.STRING)] + actual_1 = Entity(name="id") + actual_2 = Entity(name="id") + + update_entities_with_inferred_types_from_feature_views([actual_1], [fv1]) + update_entities_with_inferred_types_from_feature_views([actual_2], [fv2]) + assert actual_1 == Entity(name="id", value_type=ValueType.INT64) + assert actual_2 == Entity(name="id", value_type=ValueType.STRING) with pytest.raises(RegistryInferenceFailure): # two viable data types - infer_entity_value_type_from_feature_views([Entity(name="id")], [fv1, fv2]) + update_entities_with_inferred_types_from_feature_views( + [Entity(name="id")], [fv1, fv2] + ) @pytest.mark.integration -def test_infer_event_timestamp_column_for_data_source(simple_dataset_1): +def test_update_data_sources_with_inferred_event_timestamp_col(simple_dataset_1): df_with_two_viable_timestamp_cols = simple_dataset_1.copy(deep=True) df_with_two_viable_timestamp_cols["ts_2"] = simple_dataset_1["ts_1"] From c14023f5c15f4233f0ffd30b945f78c1106674b1 Mon Sep 17 00:00:00 2001 From: Tsotne Tabidze Date: Mon, 28 Jun 2021 11:05:59 -0700 Subject: [PATCH 21/43] Add RedshiftDataSource (#1669) * Add RedshiftDataSource Signed-off-by: Tsotne Tabidze * Call parent __init__ first Signed-off-by: Tsotne Tabidze Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- protos/feast/core/DataSource.proto | 12 + sdk/python/feast/__init__.py | 2 + sdk/python/feast/data_source.py | 266 +++++++++++++++++- sdk/python/feast/errors.py | 10 + sdk/python/feast/feature_store.py | 6 +- sdk/python/feast/feature_view.py | 13 +- sdk/python/feast/inference.py | 15 +- .../feast/infra/offline_stores/redshift.py | 60 ++++ sdk/python/feast/repo_config.py | 3 + sdk/python/feast/repo_operations.py | 8 +- sdk/python/feast/type_map.py | 24 ++ .../tensorflow_metadata/proto/v0/path_pb2.py | 2 +- .../proto/v0/schema_pb2.py | 2 +- .../proto/v0/statistics_pb2.py | 2 +- sdk/python/tests/test_inference.py | 22 +- 15 files changed, 414 insertions(+), 33 deletions(-) create mode 100644 sdk/python/feast/infra/offline_stores/redshift.py diff --git a/protos/feast/core/DataSource.proto b/protos/feast/core/DataSource.proto index a4c46e75084..1200c1b9bea 100644 --- a/protos/feast/core/DataSource.proto +++ b/protos/feast/core/DataSource.proto @@ -33,6 +33,7 @@ message DataSource { BATCH_BIGQUERY = 2; STREAM_KAFKA = 3; STREAM_KINESIS = 4; + BATCH_REDSHIFT = 5; } SourceType type = 1; @@ -100,11 +101,22 @@ message DataSource { StreamFormat record_format = 3; } + // Defines options for DataSource that sources features from a Redshift Query + message RedshiftOptions { + // Redshift table name + string table = 1; + + // SQL query that returns a table containing feature data. Must contain an event_timestamp column, and respective + // entity columns + string query = 2; + } + // DataSource options. oneof options { FileOptions file_options = 11; BigQueryOptions bigquery_options = 12; KafkaOptions kafka_options = 13; KinesisOptions kinesis_options = 14; + RedshiftOptions redshift_options = 15; } } diff --git a/sdk/python/feast/__init__.py b/sdk/python/feast/__init__.py index 7a1a70f23cf..6f1cb58451e 100644 --- a/sdk/python/feast/__init__.py +++ b/sdk/python/feast/__init__.py @@ -8,6 +8,7 @@ FileSource, KafkaSource, KinesisSource, + RedshiftSource, SourceType, ) from .entity import Entity @@ -37,6 +38,7 @@ "FileSource", "KafkaSource", "KinesisSource", + "RedshiftSource", "Feature", "FeatureStore", "FeatureTable", diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index c25b64c82f4..a5a620b55bb 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -17,11 +17,17 @@ from typing import Callable, Dict, Iterable, Optional, Tuple from pyarrow.parquet import ParquetFile +from tenacity import retry, retry_unless_exception_type, wait_exponential from feast import type_map from feast.data_format import FileFormat, StreamFormat -from feast.errors import DataSourceNotFoundException +from feast.errors import ( + DataSourceNotFoundException, + RedshiftCredentialsError, + RedshiftQueryError, +) from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto +from feast.repo_config import RepoConfig from feast.value_type import ValueType @@ -477,6 +483,15 @@ def from_proto(data_source): date_partition_column=data_source.date_partition_column, query=data_source.bigquery_options.query, ) + elif data_source.redshift_options.table or data_source.redshift_options.query: + data_source_obj = RedshiftSource( + field_mapping=data_source.field_mapping, + table=data_source.redshift_options.table, + event_timestamp_column=data_source.event_timestamp_column, + created_timestamp_column=data_source.created_timestamp_column, + date_partition_column=data_source.date_partition_column, + query=data_source.redshift_options.query, + ) elif ( data_source.kafka_options.bootstrap_servers and data_source.kafka_options.topic @@ -520,12 +535,27 @@ def to_proto(self) -> DataSourceProto: """ raise NotImplementedError - def validate(self): + def validate(self, config: RepoConfig): """ Validates the underlying data source. """ raise NotImplementedError + @staticmethod + def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: + """ + Get the callable method that returns Feast type given the raw column type + """ + raise NotImplementedError + + def get_table_column_names_and_types( + self, config: RepoConfig + ) -> Iterable[Tuple[str, str]]: + """ + Get the list of column names and raw column types + """ + raise NotImplementedError + class FileSource(DataSource): def __init__( @@ -622,7 +652,7 @@ def to_proto(self) -> DataSourceProto: return data_source_proto - def validate(self): + def validate(self, config: RepoConfig): # TODO: validate a FileSource pass @@ -630,7 +660,9 @@ def validate(self): def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: return type_map.pa_to_feast_value_type - def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: + def get_table_column_names_and_types( + self, config: RepoConfig + ) -> Iterable[Tuple[str, str]]: schema = ParquetFile(self.path).schema_arrow return zip(schema.names, map(str, schema.types)) @@ -703,7 +735,7 @@ def to_proto(self) -> DataSourceProto: return data_source_proto - def validate(self): + def validate(self, config: RepoConfig): if not self.query: from google.api_core.exceptions import NotFound from google.cloud import bigquery @@ -725,7 +757,9 @@ def get_table_query_string(self) -> str: def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: return type_map.bq_to_feast_value_type - def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: + def get_table_column_names_and_types( + self, config: RepoConfig + ) -> Iterable[Tuple[str, str]]: from google.cloud import bigquery client = bigquery.Client() @@ -875,3 +909,223 @@ def to_proto(self) -> DataSourceProto: data_source_proto.date_partition_column = self.date_partition_column return data_source_proto + + +class RedshiftOptions: + """ + DataSource Redshift options used to source features from Redshift query + """ + + def __init__(self, table: Optional[str], query: Optional[str]): + self._table = table + self._query = query + + @property + def query(self): + """ + Returns the Redshift SQL query referenced by this source + """ + return self._query + + @query.setter + def query(self, query): + """ + Sets the Redshift SQL query referenced by this source + """ + self._query = query + + @property + def table(self): + """ + Returns the table name of this Redshift table + """ + return self._table + + @table.setter + def table(self, table_name): + """ + Sets the table ref of this Redshift table + """ + self._table = table_name + + @classmethod + def from_proto(cls, redshift_options_proto: DataSourceProto.RedshiftOptions): + """ + Creates a RedshiftOptions from a protobuf representation of a Redshift option + + Args: + redshift_options_proto: A protobuf representation of a DataSource + + Returns: + Returns a RedshiftOptions object based on the redshift_options protobuf + """ + + redshift_options = cls( + table=redshift_options_proto.table, query=redshift_options_proto.query, + ) + + return redshift_options + + def to_proto(self) -> DataSourceProto.RedshiftOptions: + """ + Converts an RedshiftOptionsProto object to its protobuf representation. + + Returns: + RedshiftOptionsProto protobuf + """ + + redshift_options_proto = DataSourceProto.RedshiftOptions( + table=self.table, query=self.query, + ) + + return redshift_options_proto + + +class RedshiftSource(DataSource): + def __init__( + self, + event_timestamp_column: Optional[str] = "", + table: Optional[str] = None, + created_timestamp_column: Optional[str] = "", + field_mapping: Optional[Dict[str, str]] = None, + date_partition_column: Optional[str] = "", + query: Optional[str] = None, + ): + super().__init__( + event_timestamp_column, + created_timestamp_column, + field_mapping, + date_partition_column, + ) + + self._redshift_options = RedshiftOptions(table=table, query=query) + + def __eq__(self, other): + if not isinstance(other, RedshiftSource): + raise TypeError( + "Comparisons should only involve RedshiftSource class objects." + ) + + return ( + self.redshift_options.table == other.redshift_options.table + and self.redshift_options.query == other.redshift_options.query + and self.event_timestamp_column == other.event_timestamp_column + and self.created_timestamp_column == other.created_timestamp_column + and self.field_mapping == other.field_mapping + ) + + @property + def table(self): + return self._redshift_options.table + + @property + def query(self): + return self._redshift_options.query + + @property + def redshift_options(self): + """ + Returns the Redshift options of this data source + """ + return self._redshift_options + + @redshift_options.setter + def redshift_options(self, _redshift_options): + """ + Sets the Redshift options of this data source + """ + self._redshift_options = _redshift_options + + def to_proto(self) -> DataSourceProto: + data_source_proto = DataSourceProto( + type=DataSourceProto.BATCH_REDSHIFT, + field_mapping=self.field_mapping, + redshift_options=self.redshift_options.to_proto(), + ) + + data_source_proto.event_timestamp_column = self.event_timestamp_column + data_source_proto.created_timestamp_column = self.created_timestamp_column + data_source_proto.date_partition_column = self.date_partition_column + + return data_source_proto + + def validate(self, config: RepoConfig): + # As long as the query gets successfully executed, or the table exists, + # the data source is validated. We don't need the results though. + # TODO: uncomment this + # self.get_table_column_names_and_types(config) + print("Validate", self.get_table_column_names_and_types(config)) + + def get_table_query_string(self) -> str: + """Returns a string that can directly be used to reference this table in SQL""" + if self.table: + return f"`{self.table}`" + else: + return f"({self.query})" + + @staticmethod + def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: + return type_map.redshift_to_feast_value_type + + def get_table_column_names_and_types( + self, config: RepoConfig + ) -> Iterable[Tuple[str, str]]: + import boto3 + from botocore.config import Config + from botocore.exceptions import ClientError + + from feast.infra.offline_stores.redshift import RedshiftOfflineStoreConfig + + assert isinstance(config.offline_store, RedshiftOfflineStoreConfig) + + client = boto3.client( + "redshift-data", config=Config(region_name=config.offline_store.region) + ) + + try: + if self.table is not None: + table = client.describe_table( + ClusterIdentifier=config.offline_store.cluster_id, + Database=config.offline_store.database, + DbUser=config.offline_store.user, + Table=self.table, + ) + # The API returns valid JSON with empty column list when the table doesn't exist + if len(table["ColumnList"]) == 0: + raise DataSourceNotFoundException(self.table) + + columns = table["ColumnList"] + else: + statement = client.execute_statement( + ClusterIdentifier=config.offline_store.cluster_id, + Database=config.offline_store.database, + DbUser=config.offline_store.user, + Sql=f"SELECT * FROM ({self.query}) LIMIT 1", + ) + + # Need to retry client.describe_statement(...) until the task is finished. We don't want to bombard + # Redshift with queries, and neither do we want to wait for a long time on the initial call. + # The solution is exponential backoff. The backoff starts with 0.1 seconds and doubles exponentially + # until reaching 30 seconds, at which point the backoff is fixed. + @retry( + wait=wait_exponential(multiplier=0.1, max=30), + retry=retry_unless_exception_type(RedshiftQueryError), + ) + def wait_for_statement(): + desc = client.describe_statement(Id=statement["Id"]) + if desc["Status"] in ("SUBMITTED", "STARTED", "PICKED"): + raise Exception # Retry + if desc["Status"] != "FINISHED": + raise RedshiftQueryError(desc) # Don't retry. Raise exception. + + wait_for_statement() + + result = client.get_statement_result(Id=statement["Id"]) + + columns = result["ColumnMetadata"] + except ClientError as e: + if e.response["Error"]["Code"] == "ValidationException": + raise RedshiftCredentialsError() from e + raise + + return [(column["name"], column["typeName"].upper()) for column in columns] diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 58e34dede42..78d39852557 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -133,3 +133,13 @@ def __init__(self, repo_obj_type: str, specific_issue: str): f"Inference to fill in missing information for {repo_obj_type} failed. {specific_issue}. " "Try filling the information explicitly." ) + + +class RedshiftCredentialsError(Exception): + def __init__(self): + super().__init__("Redshift API failed due to incorrect credentials") + + +class RedshiftQueryError(Exception): + def __init__(self, details): + super().__init__(f"Redshift SQL Query failed to finish. Details: {details}") diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 22d9df8d176..c2f58cd9fe1 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -232,13 +232,13 @@ def apply( # Make inferences update_entities_with_inferred_types_from_feature_views( - entities_to_update, views_to_update + entities_to_update, views_to_update, self.config ) update_data_sources_with_inferred_event_timestamp_col( - [view.input for view in views_to_update] + [view.input for view in views_to_update], self.config ) for view in views_to_update: - view.infer_features_from_input_source() + view.infer_features_from_input_source(self.config) if len(views_to_update) + len(entities_to_update) != len(objects): raise ValueError("Unknown object type provided as part of apply() call") diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index d89a8a7b6be..db22fd7e4a3 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -20,7 +20,7 @@ from google.protobuf.timestamp_pb2 import Timestamp from feast import utils -from feast.data_source import BigQuerySource, DataSource, FileSource +from feast.data_source import DataSource from feast.errors import RegistryInferenceFailure from feast.feature import Feature from feast.protos.feast.core.FeatureView_pb2 import FeatureView as FeatureViewProto @@ -33,6 +33,7 @@ from feast.protos.feast.core.FeatureView_pb2 import ( MaterializationInterval as MaterializationIntervalProto, ) +from feast.repo_config import RepoConfig from feast.usage import log_exceptions from feast.value_type import ValueType @@ -48,7 +49,7 @@ class FeatureView: tags: Optional[Dict[str, str]] ttl: Optional[timedelta] online: bool - input: Union[BigQuerySource, FileSource] + input: DataSource created_timestamp: Optional[Timestamp] = None last_updated_timestamp: Optional[Timestamp] = None @@ -60,7 +61,7 @@ def __init__( name: str, entities: List[str], ttl: Optional[Union[Duration, timedelta]], - input: Union[BigQuerySource, FileSource], + input: DataSource, features: List[Feature] = [], tags: Optional[Dict[str, str]] = None, online: bool = True, @@ -220,14 +221,16 @@ def most_recent_end_time(self) -> Optional[datetime]: return None return max([interval[1] for interval in self.materialization_intervals]) - def infer_features_from_input_source(self): + def infer_features_from_input_source(self, config: RepoConfig): if not self.features: columns_to_exclude = { self.input.event_timestamp_column, self.input.created_timestamp_column, } | set(self.entities) - for col_name, col_datatype in self.input.get_table_column_names_and_types(): + for col_name, col_datatype in self.input.get_table_column_names_and_types( + config + ): if col_name not in columns_to_exclude and not re.match( "^__|__$", col_name, # double underscores often signal an internal-use column diff --git a/sdk/python/feast/inference.py b/sdk/python/feast/inference.py index af95f9d2551..28b764fd808 100644 --- a/sdk/python/feast/inference.py +++ b/sdk/python/feast/inference.py @@ -1,15 +1,16 @@ import re -from typing import List, Union +from typing import List from feast import Entity -from feast.data_source import BigQuerySource, FileSource +from feast.data_source import BigQuerySource, DataSource, FileSource, RedshiftSource from feast.errors import RegistryInferenceFailure from feast.feature_view import FeatureView +from feast.repo_config import RepoConfig from feast.value_type import ValueType def update_entities_with_inferred_types_from_feature_views( - entities: List[Entity], feature_views: List[FeatureView] + entities: List[Entity], feature_views: List[FeatureView], config: RepoConfig ) -> None: """ Infer entity value type by examining schema of feature view input sources @@ -25,7 +26,7 @@ def update_entities_with_inferred_types_from_feature_views( if not (incomplete_entities_keys & set(view.entities)): continue # skip if view doesn't contain any entities that need inference - col_names_and_types = view.input.get_table_column_names_and_types() + col_names_and_types = view.input.get_table_column_names_and_types(config) for entity_name in view.entities: if entity_name in incomplete_entities: # get entity information from information extracted from the view input source @@ -59,7 +60,7 @@ def update_entities_with_inferred_types_from_feature_views( def update_data_sources_with_inferred_event_timestamp_col( - data_sources: List[Union[BigQuerySource, FileSource]], + data_sources: List[DataSource], config: RepoConfig ) -> None: ERROR_MSG_PREFIX = "Unable to infer DataSource event_timestamp_column" @@ -74,6 +75,8 @@ def update_data_sources_with_inferred_event_timestamp_col( ts_column_type_regex_pattern = r"^timestamp" elif isinstance(data_source, BigQuerySource): ts_column_type_regex_pattern = "TIMESTAMP|DATETIME" + elif isinstance(data_source, RedshiftSource): + ts_column_type_regex_pattern = "TIMESTAMP[A-Z]*" else: raise RegistryInferenceFailure( "DataSource", @@ -92,7 +95,7 @@ def update_data_sources_with_inferred_event_timestamp_col( for ( col_name, col_datatype, - ) in data_source.get_table_column_names_and_types(): + ) in data_source.get_table_column_names_and_types(config): if re.match(ts_column_type_regex_pattern, col_datatype): if matched_flag: raise RegistryInferenceFailure( diff --git a/sdk/python/feast/infra/offline_stores/redshift.py b/sdk/python/feast/infra/offline_stores/redshift.py new file mode 100644 index 00000000000..06a437564a4 --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/redshift.py @@ -0,0 +1,60 @@ +from datetime import datetime +from typing import List, Optional, Union + +import pandas as pd +import pyarrow +from pydantic import StrictStr +from pydantic.typing import Literal + +from feast.data_source import DataSource +from feast.feature_view import FeatureView +from feast.infra.offline_stores.offline_store import OfflineStore, RetrievalJob +from feast.registry import Registry +from feast.repo_config import FeastConfigBaseModel, RepoConfig + + +class RedshiftOfflineStoreConfig(FeastConfigBaseModel): + """ Offline store config for AWS Redshift """ + + type: Literal["redshift"] = "redshift" + """ Offline store type selector""" + + cluster_id: StrictStr + """ Redshift cluster identifier """ + + region: StrictStr + """ Redshift cluster's AWS region """ + + user: StrictStr + """ Redshift user name """ + + database: StrictStr + """ Redshift database name """ + + s3_path: StrictStr + """ S3 path for importing & exporting data to Redshift """ + + +class RedshiftOfflineStore(OfflineStore): + @staticmethod + def pull_latest_from_table_or_query( + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + event_timestamp_column: str, + created_timestamp_column: Optional[str], + start_date: datetime, + end_date: datetime, + ) -> pyarrow.Table: + pass + + @staticmethod + def get_historical_features( + config: RepoConfig, + feature_views: List[FeatureView], + feature_refs: List[str], + entity_df: Union[pd.DataFrame, str], + registry: Registry, + project: str, + ) -> RetrievalJob: + pass diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index c680d94d07a..968af8bc9eb 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -21,6 +21,7 @@ OFFLINE_STORE_CLASS_FOR_TYPE = { "file": "feast.infra.offline_stores.file.FileOfflineStore", "bigquery": "feast.infra.offline_stores.bigquery.BigQueryOfflineStore", + "redshift": "feast.infra.offline_stores.redshift.RedshiftOfflineStore", } @@ -154,6 +155,8 @@ def _validate_offline_store_config(cls, values): values["offline_store"]["type"] = "file" elif values["provider"] == "gcp": values["offline_store"]["type"] = "bigquery" + elif values["provider"] == "aws": + values["offline_store"]["type"] = "redshift" offline_store_type = values["offline_store"]["type"] diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index f707f19c41b..eeae8f3d890 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -135,15 +135,15 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): # Make sure the data source used by this feature view is supported by Feast for data_source in data_sources: - data_source.validate() + data_source.validate(repo_config) # Make inferences update_entities_with_inferred_types_from_feature_views( - repo.entities, repo.feature_views + repo.entities, repo.feature_views, repo_config ) - update_data_sources_with_inferred_event_timestamp_col(data_sources) + update_data_sources_with_inferred_event_timestamp_col(data_sources, repo_config) for view in repo.feature_views: - view.infer_features_from_input_source() + view.infer_features_from_input_source(repo_config) sys.dont_write_bytecode = False for entity in repo.entities: diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 576a0b7f354..54d30cbd224 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -528,3 +528,27 @@ def bq_to_feast_value_type(bq_type_as_str): } return type_map[bq_type_as_str] + + +def redshift_to_feast_value_type(redshift_type_as_str): + # Type names from https://docs.aws.amazon.com/redshift/latest/dg/c_Supported_data_types.html + type_map: Dict[ValueType, Union[str, Dict[str, Any]]] = { + "INT": ValueType.INT32, + "INT4": ValueType.INT32, + "INT8": ValueType.INT64, + "FLOAT4": ValueType.FLOAT, + "FLOAT8": ValueType.DOUBLE, + "FLOAT": ValueType.DOUBLE, + "NUMERIC": ValueType.DOUBLE, + "BOOL": ValueType.BOOL, + "CHARACTER": ValueType.STRING, + "NCHAR": ValueType.STRING, + "BPCHAR": ValueType.STRING, + "CHARACTER VARYING": ValueType.STRING, + "NVARCHAR": ValueType.STRING, + "TEXT": ValueType.STRING, + "TIMESTAMP WITHOUT TIME ZONE": ValueType.UNIX_TIMESTAMP, + "TIMESTAMP WITH TIME ZONE": ValueType.UNIX_TIMESTAMP, + } + + return type_map[redshift_type_as_str.upper()] diff --git a/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py b/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py index 4b6dec828cf..d732119ead5 100644 --- a/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py +++ b/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: tensorflow_metadata/proto/v0/path.proto -"""Generated protocol buffer code.""" + from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection diff --git a/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py b/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py index d3bfc50616c..78fda8003da 100644 --- a/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py +++ b/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: tensorflow_metadata/proto/v0/schema.proto -"""Generated protocol buffer code.""" + from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message diff --git a/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py b/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py index 21473adc75c..d8e12bd1209 100644 --- a/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py +++ b/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: tensorflow_metadata/proto/v0/statistics.proto -"""Generated protocol buffer code.""" + from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection diff --git a/sdk/python/tests/test_inference.py b/sdk/python/tests/test_inference.py index 9405bce1f28..cff5f33f74e 100644 --- a/sdk/python/tests/test_inference.py +++ b/sdk/python/tests/test_inference.py @@ -5,7 +5,7 @@ simple_bq_source_using_table_ref_arg, ) -from feast import Entity, ValueType +from feast import Entity, RepoConfig, ValueType from feast.errors import RegistryInferenceFailure from feast.feature_view import FeatureView from feast.inference import ( @@ -29,15 +29,21 @@ def test_update_entities_with_inferred_types_from_feature_views( actual_1 = Entity(name="id") actual_2 = Entity(name="id") - update_entities_with_inferred_types_from_feature_views([actual_1], [fv1]) - update_entities_with_inferred_types_from_feature_views([actual_2], [fv2]) + update_entities_with_inferred_types_from_feature_views( + [actual_1], [fv1], RepoConfig(provider="local", project="test") + ) + update_entities_with_inferred_types_from_feature_views( + [actual_2], [fv2], RepoConfig(provider="local", project="test") + ) assert actual_1 == Entity(name="id", value_type=ValueType.INT64) assert actual_2 == Entity(name="id", value_type=ValueType.STRING) with pytest.raises(RegistryInferenceFailure): # two viable data types update_entities_with_inferred_types_from_feature_views( - [Entity(name="id")], [fv1, fv2] + [Entity(name="id")], + [fv1, fv2], + RepoConfig(provider="local", project="test"), ) @@ -52,7 +58,9 @@ def test_update_data_sources_with_inferred_event_timestamp_col(simple_dataset_1) simple_bq_source_using_table_ref_arg(simple_dataset_1), simple_bq_source_using_query_arg(simple_dataset_1), ] - update_data_sources_with_inferred_event_timestamp_col(data_sources) + update_data_sources_with_inferred_event_timestamp_col( + data_sources, RepoConfig(provider="local", project="test") + ) actual_event_timestamp_cols = [ source.event_timestamp_column for source in data_sources ] @@ -62,4 +70,6 @@ def test_update_data_sources_with_inferred_event_timestamp_col(simple_dataset_1) with prep_file_source(df=df_with_two_viable_timestamp_cols) as file_source: with pytest.raises(RegistryInferenceFailure): # two viable event_timestamp_columns - update_data_sources_with_inferred_event_timestamp_col([file_source]) + update_data_sources_with_inferred_event_timestamp_col( + [file_source], RepoConfig(provider="local", project="test") + ) From d1386485f890b944b5878f18d11b62a746ead6c2 Mon Sep 17 00:00:00 2001 From: codyjlin <31944154+codyjlin@users.noreply.github.com> Date: Mon, 28 Jun 2021 14:15:58 -0400 Subject: [PATCH 22/43] Provide the user with more options for setting the to_bigquery config (#1661) * Provide more options for to_bigquery config Signed-off-by: Cody Lin * Fix default job_config when none; remove excessive testing Signed-off-by: Cody Lin * Add param type and docstring Signed-off-by: Cody Lin * add docstrings and typing Signed-off-by: Cody Lin * Apply docstring suggestions from code review Co-authored-by: Willem Pienaar <6728866+woop@users.noreply.github.com> Signed-off-by: Cody Lin Co-authored-by: Willem Pienaar <6728866+woop@users.noreply.github.com> Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- .../feast/infra/offline_stores/bigquery.py | 48 +++++++++++++------ sdk/python/tests/test_historical_retrieval.py | 18 +------ 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 2bd36867b3c..bdf30f1db67 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -241,31 +241,51 @@ def to_df(self): df = self.client.query(self.query).to_dataframe(create_bqstorage_client=True) return df - def to_bigquery(self, dry_run=False) -> Optional[str]: + def to_sql(self) -> str: + """ + Returns the SQL query that will be executed in BigQuery to build the historical feature table. + """ + return self.query + + def to_bigquery(self, job_config: bigquery.QueryJobConfig = None) -> Optional[str]: + """ + Triggers the execution of a historical feature retrieval query and exports the results to a BigQuery table. + + Args: + job_config: An optional bigquery.QueryJobConfig to specify options like destination table, dry run, etc. + + Returns: + Returns the destination table name or returns None if job_config.dry_run is True. + """ + @retry(wait=wait_fixed(10), stop=stop_after_delay(1800), reraise=True) def _block_until_done(): return self.client.get_job(bq_job.job_id).state in ["PENDING", "RUNNING"] - today = date.today().strftime("%Y%m%d") - rand_id = str(uuid.uuid4())[:7] - dataset_project = self.config.offline_store.project_id or self.client.project - path = f"{dataset_project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" - job_config = bigquery.QueryJobConfig(destination=path, dry_run=dry_run) - bq_job = self.client.query(self.query, job_config=job_config) - - if dry_run: - print( - "This query will process {} bytes.".format(bq_job.total_bytes_processed) + if not job_config: + today = date.today().strftime("%Y%m%d") + rand_id = str(uuid.uuid4())[:7] + dataset_project = ( + self.config.offline_store.project_id or self.client.project ) - return None + path = f"{dataset_project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" + job_config = bigquery.QueryJobConfig(destination=path) + + bq_job = self.client.query(self.query, job_config=job_config) _block_until_done() if bq_job.exception(): raise bq_job.exception() - print(f"Done writing to '{path}'.") - return path + if job_config.dry_run: + print( + "This query will process {} bytes.".format(bq_job.total_bytes_processed) + ) + return None + + print(f"Done writing to '{job_config.destination}'.") + return str(job_config.destination) @dataclass(frozen=True) diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index c3bbdbbe469..4a2df9120f5 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -485,29 +485,13 @@ def test_historical_features_from_bigquery_sources( full_feature_names=full_feature_names, ) - # Just a dry run, should not create table - bq_dry_run = job_from_sql.to_bigquery(dry_run=True) - assert bq_dry_run is None - - bq_temp_table_path = job_from_sql.to_bigquery() - assert bq_temp_table_path.split(".")[0] == gcp_project - - if provider_type == "gcp_custom_offline_config": - assert bq_temp_table_path.split(".")[1] == "foo" - else: - assert bq_temp_table_path.split(".")[1] == bigquery_dataset - - # Check that this table actually exists - actual_bq_temp_table = bigquery.Client().get_table(bq_temp_table_path) - assert actual_bq_temp_table.table_id == bq_temp_table_path.split(".")[-1] - start_time = datetime.utcnow() actual_df_from_sql_entities = job_from_sql.to_df() end_time = datetime.utcnow() with capsys.disabled(): print( str( - f"\nTime to execute job_from_df.to_df() = '{(end_time - start_time)}'" + f"\nTime to execute job_from_sql.to_df() = '{(end_time - start_time)}'" ) ) From c02b9eb6ab41a3f2c87d436c6c3236e2a171bd12 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 28 Jun 2021 14:00:57 -0700 Subject: [PATCH 23/43] Add streaming sources to the FeatureView API (#1664) * Add a streaming source to the FeatureView API This diff only updates the API. It is currently up to the providers to actually use this information to spin up resources to consume events from the stream sources. Signed-off-by: Achal Shah * remove stuff from rebase Signed-off-by: Achal Shah * make format Signed-off-by: Achal Shah * Update protos Signed-off-by: Achal Shah * lint Signed-off-by: Achal Shah * format Signed-off-by: Achal Shah * CR Signed-off-by: Achal Shah * fix test Signed-off-by: Achal Shah * lint Signed-off-by: Achal Shah Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- protos/feast/core/FeatureView.proto | 4 +- sdk/python/feast/data_source.py | 3 ++ sdk/python/feast/feature_view.py | 39 ++++++++++++++++--- .../tensorflow_metadata/proto/v0/path_pb2.py | 2 +- .../proto/v0/schema_pb2.py | 2 +- .../proto/v0/statistics_pb2.py | 2 +- 6 files changed, 42 insertions(+), 10 deletions(-) diff --git a/protos/feast/core/FeatureView.proto b/protos/feast/core/FeatureView.proto index d98f54825a6..f39fcf5e732 100644 --- a/protos/feast/core/FeatureView.proto +++ b/protos/feast/core/FeatureView.proto @@ -59,7 +59,9 @@ message FeatureViewSpec { google.protobuf.Duration ttl = 6; // Batch/Offline DataSource where this view can retrieve offline feature data. - DataSource input = 7; + DataSource batch_source = 7; + // Streaming DataSource from where this view can consume "online" feature data. + DataSource stream_source = 9; // Whether these features should be served online or not bool online = 8; diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index a5a620b55bb..7480d7fd4f4 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -869,6 +869,9 @@ def __init__( ) def __eq__(self, other): + if other is None: + return False + if not isinstance(other, KinesisSource): raise TypeError( "Comparisons should only involve KinesisSource class objects." diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index db22fd7e4a3..3d20b9334f4 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -50,7 +50,8 @@ class FeatureView: ttl: Optional[timedelta] online: bool input: DataSource - + batch_source: Optional[DataSource] = None + stream_source: Optional[DataSource] = None created_timestamp: Optional[Timestamp] = None last_updated_timestamp: Optional[Timestamp] = None materialization_intervals: List[Tuple[datetime, datetime]] @@ -62,15 +63,22 @@ def __init__( entities: List[str], ttl: Optional[Union[Duration, timedelta]], input: DataSource, + batch_source: Optional[DataSource] = None, + stream_source: Optional[DataSource] = None, features: List[Feature] = [], tags: Optional[Dict[str, str]] = None, online: bool = True, ): + _input = input or batch_source + assert _input is not None + cols = [entity for entity in entities] + [feat.name for feat in features] for col in cols: - if input.field_mapping is not None and col in input.field_mapping.keys(): + if _input.field_mapping is not None and col in _input.field_mapping.keys(): raise ValueError( - f"The field {col} is mapped to {input.field_mapping[col]} for this data source. Please either remove this field mapping or use {input.field_mapping[col]} as the Entity or Feature name." + f"The field {col} is mapped to {_input.field_mapping[col]} for this data source. " + f"Please either remove this field mapping or use {_input.field_mapping[col]} as the " + f"Entity or Feature name." ) self.name = name @@ -84,7 +92,9 @@ def __init__( self.ttl = ttl self.online = online - self.input = input + self.input = _input + self.batch_source = _input + self.stream_source = stream_source self.materialization_intervals = [] @@ -118,6 +128,8 @@ def __eq__(self, other): return False if self.input != other.input: return False + if self.stream_source != other.stream_source: + return False return True @@ -157,6 +169,8 @@ def to_proto(self) -> FeatureViewProto: ttl_duration = Duration() ttl_duration.FromTimedelta(self.ttl) + print(f"Stream soruce: {self.stream_source}, {type(self.stream_source)}") + spec = FeatureViewSpecProto( name=self.name, entities=self.entities, @@ -164,7 +178,12 @@ def to_proto(self) -> FeatureViewProto: tags=self.tags, ttl=(ttl_duration if ttl_duration is not None else None), online=self.online, - input=self.input.to_proto(), + batch_source=self.input.to_proto(), + stream_source=( + self.stream_source.to_proto() + if self.stream_source is not None + else None + ), ) return FeatureViewProto(spec=spec, meta=meta) @@ -181,6 +200,12 @@ def from_proto(cls, feature_view_proto: FeatureViewProto): Returns a FeatureViewProto object based on the feature view protobuf """ + _input = DataSource.from_proto(feature_view_proto.spec.batch_source) + stream_source = ( + DataSource.from_proto(feature_view_proto.spec.stream_source) + if feature_view_proto.spec.HasField("stream_source") + else None + ) feature_view = cls( name=feature_view_proto.spec.name, entities=[entity for entity in feature_view_proto.spec.entities], @@ -200,7 +225,9 @@ def from_proto(cls, feature_view_proto: FeatureViewProto): and feature_view_proto.spec.ttl.nanos == 0 else feature_view_proto.spec.ttl ), - input=DataSource.from_proto(feature_view_proto.spec.input), + input=_input, + batch_source=_input, + stream_source=stream_source, ) feature_view.created_timestamp = feature_view_proto.meta.created_timestamp diff --git a/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py b/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py index d732119ead5..4b6dec828cf 100644 --- a/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py +++ b/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: tensorflow_metadata/proto/v0/path.proto - +"""Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection diff --git a/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py b/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py index 78fda8003da..d3bfc50616c 100644 --- a/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py +++ b/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: tensorflow_metadata/proto/v0/schema.proto - +"""Generated protocol buffer code.""" from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message diff --git a/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py b/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py index d8e12bd1209..21473adc75c 100644 --- a/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py +++ b/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: tensorflow_metadata/proto/v0/statistics.proto - +"""Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection From 12dbbea3b2a51e200c28c8c5d98e3f705e594b42 Mon Sep 17 00:00:00 2001 From: Matt Delacour Date: Mon, 28 Jun 2021 22:16:16 -0400 Subject: [PATCH 24/43] Add to_table() to RetrievalJob object (#1663) * Add notion of OfflineJob Signed-off-by: Matt Delacour * Use RetrievalJob instead of creating a new OfflineJob object Signed-off-by: Matt Delacour * Add to_table() in integration tests Signed-off-by: Matt Delacour Co-authored-by: Tsotne Tabidze Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/feature_store.py | 26 ++++--- sdk/python/feast/infra/gcp.py | 5 +- sdk/python/feast/infra/local.py | 5 +- .../feast/infra/offline_stores/bigquery.py | 35 ++++----- sdk/python/feast/infra/offline_stores/file.py | 76 +++++++++++-------- .../infra/offline_stores/offline_store.py | 10 ++- .../feast/infra/offline_stores/redshift.py | 4 +- sdk/python/feast/infra/provider.py | 1 + sdk/python/tests/foo_provider.py | 1 + sdk/python/tests/test_historical_retrieval.py | 10 +++ 10 files changed, 102 insertions(+), 71 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index c2f58cd9fe1..70a8dfdb6c8 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -397,12 +397,13 @@ def tqdm_builder(length): end_date = utils.make_tzaware(end_date) provider.materialize_single_feature_view( - feature_view, - start_date, - end_date, - self._registry, - self.project, - tqdm_builder, + config=self.config, + feature_view=feature_view, + start_date=start_date, + end_date=end_date, + registry=self._registry, + project=self.project, + tqdm_builder=tqdm_builder, ) self._registry.apply_materialization( @@ -475,12 +476,13 @@ def tqdm_builder(length): end_date = utils.make_tzaware(end_date) provider.materialize_single_feature_view( - feature_view, - start_date, - end_date, - self._registry, - self.project, - tqdm_builder, + config=self.config, + feature_view=feature_view, + start_date=start_date, + end_date=end_date, + registry=self._registry, + project=self.project, + tqdm_builder=tqdm_builder, ) self._registry.apply_materialization( diff --git a/sdk/python/feast/infra/gcp.py b/sdk/python/feast/infra/gcp.py index 9e307d761b4..a92b9280438 100644 --- a/sdk/python/feast/infra/gcp.py +++ b/sdk/python/feast/infra/gcp.py @@ -81,6 +81,7 @@ def online_read( def materialize_single_feature_view( self, + config: RepoConfig, feature_view: FeatureView, start_date: datetime, end_date: datetime, @@ -99,7 +100,8 @@ def materialize_single_feature_view( created_timestamp_column, ) = _get_column_names(feature_view, entities) - table = self.offline_store.pull_latest_from_table_or_query( + offline_job = self.offline_store.pull_latest_from_table_or_query( + config=config, data_source=feature_view.input, join_key_columns=join_key_columns, feature_name_columns=feature_name_columns, @@ -108,6 +110,7 @@ def materialize_single_feature_view( start_date=start_date, end_date=end_date, ) + table = offline_job.to_table() if feature_view.input.field_mapping is not None: table = _run_field_mapping(table, feature_view.input.field_mapping) diff --git a/sdk/python/feast/infra/local.py b/sdk/python/feast/infra/local.py index 23c813e6083..5e238448a23 100644 --- a/sdk/python/feast/infra/local.py +++ b/sdk/python/feast/infra/local.py @@ -80,6 +80,7 @@ def online_read( def materialize_single_feature_view( self, + config: RepoConfig, feature_view: FeatureView, start_date: datetime, end_date: datetime, @@ -98,7 +99,7 @@ def materialize_single_feature_view( created_timestamp_column, ) = _get_column_names(feature_view, entities) - table = self.offline_store.pull_latest_from_table_or_query( + offline_job = self.offline_store.pull_latest_from_table_or_query( data_source=feature_view.input, join_key_columns=join_key_columns, feature_name_columns=feature_name_columns, @@ -106,7 +107,9 @@ def materialize_single_feature_view( created_timestamp_column=created_timestamp_column, start_date=start_date, end_date=end_date, + config=config, ) + table = offline_job.to_table() if feature_view.input.field_mapping is not None: table = _run_field_mapping(table, feature_view.input.field_mapping) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index bdf30f1db67..8c692d2dc92 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -15,10 +15,9 @@ from feast.data_source import BigQuerySource, DataSource from feast.errors import FeastProviderLoginError from feast.feature_view import FeatureView -from feast.infra.offline_stores.offline_store import OfflineStore +from feast.infra.offline_stores.offline_store import OfflineStore, RetrievalJob from feast.infra.provider import ( DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL, - RetrievalJob, _get_requested_feature_views_to_features_dict, ) from feast.registry import Registry @@ -52,6 +51,7 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel): class BigQueryOfflineStore(OfflineStore): @staticmethod def pull_latest_from_table_or_query( + config: RepoConfig, data_source: DataSource, join_key_columns: List[str], feature_name_columns: List[str], @@ -59,7 +59,7 @@ def pull_latest_from_table_or_query( created_timestamp_column: Optional[str], start_date: datetime, end_date: datetime, - ) -> pyarrow.Table: + ) -> RetrievalJob: assert isinstance(data_source, BigQuerySource) from_expression = data_source.get_table_query_string() @@ -74,6 +74,7 @@ def pull_latest_from_table_or_query( timestamp_desc_string = " DESC, ".join(timestamps) + " DESC" field_string = ", ".join(join_key_columns + feature_name_columns + timestamps) + client = _get_bigquery_client(project=config.offline_store.project_id) query = f""" SELECT {field_string} FROM ( @@ -84,14 +85,7 @@ def pull_latest_from_table_or_query( ) WHERE _feast_row = 1 """ - - return BigQueryOfflineStore._pull_query(query) - - @staticmethod - def _pull_query(query: str) -> pyarrow.Table: - client = _get_bigquery_client() - query_job = client.query(query) - return query_job.to_arrow() + return BigQueryRetrievalJob(query=query, client=client, config=config) @staticmethod def get_historical_features( @@ -104,19 +98,18 @@ def get_historical_features( full_feature_names: bool = False, ) -> RetrievalJob: # TODO: Add entity_df validation in order to fail before interacting with BigQuery + assert isinstance(config.offline_store, BigQueryOfflineStoreConfig) - client = _get_bigquery_client() - + client = _get_bigquery_client(project=config.offline_store.project_id) expected_join_keys = _get_join_keys(project, feature_views, registry) assert isinstance(config.offline_store, BigQueryOfflineStoreConfig) - dataset_project = config.offline_store.project_id or client.project table = _upload_entity_df_into_bigquery( client=client, project=config.project, dataset_name=config.offline_store.dataset, - dataset_project=dataset_project, + dataset_project=client.project, entity_df=entity_df, ) @@ -265,10 +258,7 @@ def _block_until_done(): if not job_config: today = date.today().strftime("%Y%m%d") rand_id = str(uuid.uuid4())[:7] - dataset_project = ( - self.config.offline_store.project_id or self.client.project - ) - path = f"{dataset_project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" + path = f"{self.client.project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" job_config = bigquery.QueryJobConfig(destination=path) bq_job = self.client.query(self.query, job_config=job_config) @@ -287,6 +277,9 @@ def _block_until_done(): print(f"Done writing to '{job_config.destination}'.") return str(job_config.destination) + def to_table(self) -> pyarrow.Table: + return self.client.query(self.query).to_arrow() + @dataclass(frozen=True) class FeatureViewQueryContext: @@ -451,9 +444,9 @@ def build_point_in_time_query( return query -def _get_bigquery_client(): +def _get_bigquery_client(project: Optional[str] = None): try: - client = bigquery.Client() + client = bigquery.Client(project=project) except DefaultCredentialsError as e: raise FeastProviderLoginError( str(e) diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index f2f700cc661..a4ca1141c22 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -38,6 +38,11 @@ def to_df(self): df = self.evaluation_function() return df + def to_table(self): + # Only execute the evaluation function to build the final historical retrieval dataframe at the last moment. + df = self.evaluation_function() + return pyarrow.Table.from_pandas(df) + class FileOfflineStore(OfflineStore): @staticmethod @@ -49,7 +54,7 @@ def get_historical_features( registry: Registry, project: str, full_feature_names: bool = False, - ) -> FileRetrievalJob: + ) -> RetrievalJob: if not isinstance(entity_df, pd.DataFrame): raise ValueError( f"Please provide an entity_df of type {type(pd.DataFrame)} instead of type {type(entity_df)}" @@ -207,6 +212,7 @@ def evaluate_historical_retrieval(): @staticmethod def pull_latest_from_table_or_query( + config: RepoConfig, data_source: DataSource, join_key_columns: List[str], feature_name_columns: List[str], @@ -214,42 +220,48 @@ def pull_latest_from_table_or_query( created_timestamp_column: Optional[str], start_date: datetime, end_date: datetime, - ) -> pyarrow.Table: + ) -> RetrievalJob: assert isinstance(data_source, FileSource) - source_df = pd.read_parquet(data_source.path) - # Make sure all timestamp fields are tz-aware. We default tz-naive fields to UTC - source_df[event_timestamp_column] = source_df[event_timestamp_column].apply( - lambda x: x if x.tzinfo is not None else x.replace(tzinfo=pytz.utc) - ) - if created_timestamp_column: - source_df[created_timestamp_column] = source_df[ - created_timestamp_column - ].apply(lambda x: x if x.tzinfo is not None else x.replace(tzinfo=pytz.utc)) - - source_columns = set(source_df.columns) - if not set(join_key_columns).issubset(source_columns): - raise FeastJoinKeysDuringMaterialization( - data_source.path, set(join_key_columns), source_columns + # Create lazy function that is only called from the RetrievalJob object + def evaluate_offline_job(): + source_df = pd.read_parquet(data_source.path) + # Make sure all timestamp fields are tz-aware. We default tz-naive fields to UTC + source_df[event_timestamp_column] = source_df[event_timestamp_column].apply( + lambda x: x if x.tzinfo is not None else x.replace(tzinfo=pytz.utc) ) + if created_timestamp_column: + source_df[created_timestamp_column] = source_df[ + created_timestamp_column + ].apply( + lambda x: x if x.tzinfo is not None else x.replace(tzinfo=pytz.utc) + ) - ts_columns = ( - [event_timestamp_column, created_timestamp_column] - if created_timestamp_column - else [event_timestamp_column] - ) + source_columns = set(source_df.columns) + if not set(join_key_columns).issubset(source_columns): + raise FeastJoinKeysDuringMaterialization( + data_source.path, set(join_key_columns), source_columns + ) - source_df.sort_values(by=ts_columns, inplace=True) + ts_columns = ( + [event_timestamp_column, created_timestamp_column] + if created_timestamp_column + else [event_timestamp_column] + ) - filtered_df = source_df[ - (source_df[event_timestamp_column] >= start_date) - & (source_df[event_timestamp_column] < end_date) - ] - last_values_df = filtered_df.drop_duplicates( - join_key_columns, keep="last", ignore_index=True - ) + source_df.sort_values(by=ts_columns, inplace=True) - columns_to_extract = set(join_key_columns + feature_name_columns + ts_columns) - table = pyarrow.Table.from_pandas(last_values_df[columns_to_extract]) + filtered_df = source_df[ + (source_df[event_timestamp_column] >= start_date) + & (source_df[event_timestamp_column] < end_date) + ] + last_values_df = filtered_df.drop_duplicates( + join_key_columns, keep="last", ignore_index=True + ) + + columns_to_extract = set( + join_key_columns + feature_name_columns + ts_columns + ) + return last_values_df[columns_to_extract] - return table + return FileRetrievalJob(evaluation_function=evaluate_offline_job) diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index c1c2279dc61..3d477a60358 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -28,10 +28,15 @@ class RetrievalJob(ABC): """RetrievalJob is used to manage the execution of a historical feature retrieval""" @abstractmethod - def to_df(self): + def to_df(self) -> pd.DataFrame: """Return dataset as Pandas DataFrame synchronously""" pass + @abstractmethod + def to_table(self) -> pyarrow.Table: + """Return dataset as pyarrow Table synchronously""" + pass + class OfflineStore(ABC): """ @@ -42,6 +47,7 @@ class OfflineStore(ABC): @staticmethod @abstractmethod def pull_latest_from_table_or_query( + config: RepoConfig, data_source: DataSource, join_key_columns: List[str], feature_name_columns: List[str], @@ -49,7 +55,7 @@ def pull_latest_from_table_or_query( created_timestamp_column: Optional[str], start_date: datetime, end_date: datetime, - ) -> pyarrow.Table: + ) -> RetrievalJob: """ Note that join_key_columns, feature_name_columns, event_timestamp_column, and created_timestamp_column have all already been mapped to column names of the source table and those column names are the values passed diff --git a/sdk/python/feast/infra/offline_stores/redshift.py b/sdk/python/feast/infra/offline_stores/redshift.py index 06a437564a4..f15b9af4513 100644 --- a/sdk/python/feast/infra/offline_stores/redshift.py +++ b/sdk/python/feast/infra/offline_stores/redshift.py @@ -2,7 +2,6 @@ from typing import List, Optional, Union import pandas as pd -import pyarrow from pydantic import StrictStr from pydantic.typing import Literal @@ -38,6 +37,7 @@ class RedshiftOfflineStoreConfig(FeastConfigBaseModel): class RedshiftOfflineStore(OfflineStore): @staticmethod def pull_latest_from_table_or_query( + config: RepoConfig, data_source: DataSource, join_key_columns: List[str], feature_name_columns: List[str], @@ -45,7 +45,7 @@ def pull_latest_from_table_or_query( created_timestamp_column: Optional[str], start_date: datetime, end_date: datetime, - ) -> pyarrow.Table: + ) -> RetrievalJob: pass @staticmethod diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index d82d9c3f2dd..ed038e86aeb 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -97,6 +97,7 @@ def online_write_batch( @abc.abstractmethod def materialize_single_feature_view( self, + config: RepoConfig, feature_view: FeatureView, start_date: datetime, end_date: datetime, diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index 0352645d983..a5d396a458a 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -45,6 +45,7 @@ def online_write_batch( def materialize_single_feature_view( self, + config: RepoConfig, feature_view: FeatureView, start_date: datetime, end_date: datetime, diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index 4a2df9120f5..0abc6eef707 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -508,6 +508,11 @@ def test_historical_features_from_bigquery_sources( check_dtype=False, ) + table_from_sql_entities = job_from_sql.to_table() + assert_frame_equal( + actual_df_from_sql_entities, table_from_sql_entities.to_pandas() + ) + timestamp_column = ( "e_ts" if infer_event_timestamp_col @@ -590,6 +595,11 @@ def test_historical_features_from_bigquery_sources( check_dtype=False, ) + table_from_df_entities = job_from_df.to_table() + assert_frame_equal( + actual_df_from_df_entities, table_from_df_entities.to_pandas() + ) + @pytest.mark.integration def test_feature_name_collision_on_historical_retrieval_from_parquet_sources(): From d0fe0a9a9849f8318146b249d49003c26e4d8a81 Mon Sep 17 00:00:00 2001 From: Matt Delacour Date: Tue, 29 Jun 2021 10:36:16 -0400 Subject: [PATCH 25/43] Rename to_table to to_arrow (#1671) Signed-off-by: Matt Delacour Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/infra/gcp.py | 2 +- sdk/python/feast/infra/local.py | 2 +- sdk/python/feast/infra/offline_stores/bigquery.py | 2 +- sdk/python/feast/infra/offline_stores/file.py | 2 +- sdk/python/feast/infra/offline_stores/offline_store.py | 2 +- sdk/python/tensorflow_metadata/proto/v0/path_pb2.py | 2 +- sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py | 2 +- sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py | 2 +- sdk/python/tests/test_historical_retrieval.py | 5 ++--- 9 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sdk/python/feast/infra/gcp.py b/sdk/python/feast/infra/gcp.py index a92b9280438..9af520d7aae 100644 --- a/sdk/python/feast/infra/gcp.py +++ b/sdk/python/feast/infra/gcp.py @@ -110,7 +110,7 @@ def materialize_single_feature_view( start_date=start_date, end_date=end_date, ) - table = offline_job.to_table() + table = offline_job.to_arrow() if feature_view.input.field_mapping is not None: table = _run_field_mapping(table, feature_view.input.field_mapping) diff --git a/sdk/python/feast/infra/local.py b/sdk/python/feast/infra/local.py index 5e238448a23..d8bc0c91fdc 100644 --- a/sdk/python/feast/infra/local.py +++ b/sdk/python/feast/infra/local.py @@ -109,7 +109,7 @@ def materialize_single_feature_view( end_date=end_date, config=config, ) - table = offline_job.to_table() + table = offline_job.to_arrow() if feature_view.input.field_mapping is not None: table = _run_field_mapping(table, feature_view.input.field_mapping) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 8c692d2dc92..64d3fb072cf 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -277,7 +277,7 @@ def _block_until_done(): print(f"Done writing to '{job_config.destination}'.") return str(job_config.destination) - def to_table(self) -> pyarrow.Table: + def to_arrow(self) -> pyarrow.Table: return self.client.query(self.query).to_arrow() diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index a4ca1141c22..44edc8618dc 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -38,7 +38,7 @@ def to_df(self): df = self.evaluation_function() return df - def to_table(self): + def to_arrow(self): # Only execute the evaluation function to build the final historical retrieval dataframe at the last moment. df = self.evaluation_function() return pyarrow.Table.from_pandas(df) diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index 3d477a60358..e8d32cd3846 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -33,7 +33,7 @@ def to_df(self) -> pd.DataFrame: pass @abstractmethod - def to_table(self) -> pyarrow.Table: + def to_arrow(self) -> pyarrow.Table: """Return dataset as pyarrow Table synchronously""" pass diff --git a/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py b/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py index 4b6dec828cf..d732119ead5 100644 --- a/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py +++ b/sdk/python/tensorflow_metadata/proto/v0/path_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: tensorflow_metadata/proto/v0/path.proto -"""Generated protocol buffer code.""" + from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection diff --git a/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py b/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py index d3bfc50616c..78fda8003da 100644 --- a/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py +++ b/sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: tensorflow_metadata/proto/v0/schema.proto -"""Generated protocol buffer code.""" + from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message diff --git a/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py b/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py index 21473adc75c..d8e12bd1209 100644 --- a/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py +++ b/sdk/python/tensorflow_metadata/proto/v0/statistics_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: tensorflow_metadata/proto/v0/statistics.proto -"""Generated protocol buffer code.""" + from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index 0abc6eef707..a9f4369a597 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -508,7 +508,7 @@ def test_historical_features_from_bigquery_sources( check_dtype=False, ) - table_from_sql_entities = job_from_sql.to_table() + table_from_sql_entities = job_from_sql.to_arrow() assert_frame_equal( actual_df_from_sql_entities, table_from_sql_entities.to_pandas() ) @@ -594,8 +594,7 @@ def test_historical_features_from_bigquery_sources( .reset_index(drop=True), check_dtype=False, ) - - table_from_df_entities = job_from_df.to_table() + table_from_df_entities = job_from_df.to_arrow() assert_frame_equal( actual_df_from_df_entities, table_from_df_entities.to_pandas() ) From 6e8670eadd2b51ecd24b32bab1b4fdb17a24b346 Mon Sep 17 00:00:00 2001 From: Matt Delacour Date: Tue, 29 Jun 2021 11:29:16 -0400 Subject: [PATCH 26/43] Cancel BigQuery job if timeout hits (#1672) * Cancel BigQuery job if timedout hits Signed-off-by: Matt Delacour * Fix typo Signed-off-by: Matt Delacour Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/errors.py | 5 ++++ .../feast/infra/offline_stores/bigquery.py | 27 ++++++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 78d39852557..742e7df5c5e 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -135,6 +135,11 @@ def __init__(self, repo_obj_type: str, specific_issue: str): ) +class BigQueryJobCancelled(Exception): + def __init__(self, job_id): + super().__init__(f"The BigQuery job with ID '{job_id}' was cancelled") + + class RedshiftCredentialsError(Exception): def __init__(self): super().__init__("Redshift API failed due to incorrect credentials") diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 64d3fb072cf..c1166f8cab2 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -13,7 +13,7 @@ from feast import errors from feast.data_source import BigQuerySource, DataSource -from feast.errors import FeastProviderLoginError +from feast.errors import BigQueryJobCancelled, FeastProviderLoginError from feast.feature_view import FeatureView from feast.infra.offline_stores.offline_store import OfflineStore, RetrievalJob from feast.infra.provider import ( @@ -251,10 +251,6 @@ def to_bigquery(self, job_config: bigquery.QueryJobConfig = None) -> Optional[st Returns the destination table name or returns None if job_config.dry_run is True. """ - @retry(wait=wait_fixed(10), stop=stop_after_delay(1800), reraise=True) - def _block_until_done(): - return self.client.get_job(bq_job.job_id).state in ["PENDING", "RUNNING"] - if not job_config: today = date.today().strftime("%Y%m%d") rand_id = str(uuid.uuid4())[:7] @@ -263,7 +259,7 @@ def _block_until_done(): bq_job = self.client.query(self.query, job_config=job_config) - _block_until_done() + block_until_done(client=self.client, bq_job=bq_job) if bq_job.exception(): raise bq_job.exception() @@ -281,6 +277,25 @@ def to_arrow(self) -> pyarrow.Table: return self.client.query(self.query).to_arrow() +def block_until_done(client, bq_job): + def _is_done(job_id): + return client.get_job(job_id).state in ["PENDING", "RUNNING"] + + @retry(wait=wait_fixed(10), stop=stop_after_delay(1800), reraise=True) + def _wait_until_done(job_id): + return _is_done(job_id) + + job_id = bq_job.job_id + _wait_until_done(job_id=job_id) + + if not _is_done(job_id): + client.cancel_job(job_id) + raise BigQueryJobCancelled(job_id=job_id) + + if bq_job.exception(): + raise bq_job.exception() + + @dataclass(frozen=True) class FeatureViewQueryContext: """Context object used to template a BigQuery point-in-time SQL query""" From 531402472c8723f06a40210e4ae646204aa89e6f Mon Sep 17 00:00:00 2001 From: Greg Kuhlmann Date: Wed, 30 Jun 2021 10:30:36 -0500 Subject: [PATCH 27/43] Fix Feature References example (#1674) Fix Feature References example by passing `entity_rows` to `get_online_features()` Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- docs/concepts/data-model-and-concepts.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts/data-model-and-concepts.md b/docs/concepts/data-model-and-concepts.md index 270d98e3d3b..f24dd253858 100644 --- a/docs/concepts/data-model-and-concepts.md +++ b/docs/concepts/data-model-and-concepts.md @@ -20,7 +20,7 @@ online_features = fs.get_online_features( 'driver_locations:lon', 'drivers_activity:trips_today' ], - entities=[{'driver': 'driver_1001'}] + entity_rows=[{'driver': 'driver_1001'}] ) ``` From eb1da5e1568450b33a8d1f6f5789b48804ac9b2f Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 30 Jun 2021 10:41:32 -0700 Subject: [PATCH 28/43] Allow strings for online/offline store instead of dicts (#1673) Signed-off-by: Achal Shah Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/repo_config.py | 5 +++++ sdk/python/tests/test_repo_config.py | 24 ++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 968af8bc9eb..8ef98736f9a 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -83,10 +83,15 @@ def __init__(self, **data: Any): self.online_store = get_online_config_from_type(self.online_store["type"])( **self.online_store ) + elif isinstance(self.online_store, str): + self.online_store = get_online_config_from_type(self.online_store)() + if isinstance(self.offline_store, Dict): self.offline_store = get_offline_config_from_type( self.offline_store["type"] )(**self.offline_store) + elif isinstance(self.offline_store, str): + self.offline_store = get_offline_config_from_type(self.offline_store)() def get_registry_config(self): if isinstance(self.registry, str): diff --git a/sdk/python/tests/test_repo_config.py b/sdk/python/tests/test_repo_config.py index b6a6a330119..f4e15d497f9 100644 --- a/sdk/python/tests/test_repo_config.py +++ b/sdk/python/tests/test_repo_config.py @@ -3,6 +3,7 @@ from textwrap import dedent from typing import Optional +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig from feast.repo_config import FeastConfigError, load_repo_config @@ -18,8 +19,9 @@ def _test_config(config_text, expect_error: Optional[str]): repo_config.write_text(config_text) error = None + rc = None try: - load_repo_config(repo_path) + rc = load_repo_config(repo_path) except FeastConfigError as e: error = e @@ -29,6 +31,8 @@ def _test_config(config_text, expect_error: Optional[str]): print(f"error: {error}") assert error is None + return rc + def test_local_config(): _test_config( @@ -44,7 +48,7 @@ def test_local_config(): def test_local_config_with_full_online_class(): - _test_config( + c = _test_config( dedent( """ project: foo @@ -56,6 +60,22 @@ def test_local_config_with_full_online_class(): ), expect_error=None, ) + assert isinstance(c.online_store, SqliteOnlineStoreConfig) + + +def test_local_config_with_full_online_class_directly(): + c = _test_config( + dedent( + """ + project: foo + registry: "registry.db" + provider: local + online_store: feast.infra.online_stores.sqlite.SqliteOnlineStore + """ + ), + expect_error=None, + ) + assert isinstance(c.online_store, SqliteOnlineStoreConfig) def test_gcp_config(): From 183a0b9dad67e01d939a2871322a4a77aad7757c Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Thu, 1 Jul 2021 16:16:37 -0700 Subject: [PATCH 29/43] Remove default list from the FeatureView constructor (#1679) Signed-off-by: Achal Shah Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/feature_view.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 3d20b9334f4..82b78d1dab8 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -65,14 +65,16 @@ def __init__( input: DataSource, batch_source: Optional[DataSource] = None, stream_source: Optional[DataSource] = None, - features: List[Feature] = [], + features: List[Feature] = None, tags: Optional[Dict[str, str]] = None, online: bool = True, ): _input = input or batch_source assert _input is not None - cols = [entity for entity in entities] + [feat.name for feat in features] + _features = features or [] + + cols = [entity for entity in entities] + [feat.name for feat in _features] for col in cols: if _input.field_mapping is not None and col in _input.field_mapping.keys(): raise ValueError( @@ -83,7 +85,7 @@ def __init__( self.name = name self.entities = entities - self.features = features + self.features = _features self.tags = tags if tags is not None else {} if isinstance(ttl, Duration): From b714a12f0523893ec7a68d89d2cb97a6dc6e3b13 Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Fri, 2 Jul 2021 10:54:03 -0400 Subject: [PATCH 30/43] made changes requested by @tsotnet Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/errors.py | 8 +- sdk/python/feast/feature_store.py | 90 +++++++------------ .../feast/infra/offline_stores/bigquery.py | 2 +- sdk/python/feast/infra/offline_stores/file.py | 2 +- .../feast/infra/offline_stores/redshift.py | 1 + sdk/python/feast/infra/provider.py | 2 +- sdk/python/tests/test_historical_retrieval.py | 85 +++--------------- 7 files changed, 53 insertions(+), 137 deletions(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 742e7df5c5e..f786cfe31b5 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -1,4 +1,4 @@ -from typing import Set +from typing import List, Set from colorama import Fore, Style @@ -79,9 +79,11 @@ def __init__(self, offline_store_name: str, data_source_name: str): class FeatureNameCollisionError(Exception): - def __init__(self, feature_name_collisions: str): + def __init__(self, feature_refs_collisions: List[str]): + feature_name_collisions = [ref.split(":")[1] for ref in feature_refs_collisions] + feature_names = ", ".join(x for x in feature_name_collisions) super().__init__( - f"The following feature name(s) have collisions: {feature_name_collisions}. Set 'full_feature_names' " + f"The following feature name(s) have collisions: {feature_names}. Set 'full_feature_names' " f"argument in the data retrieval function to True to use the full feature name which is prefixed by the feature view name." ) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 70a8dfdb6c8..e32221b7592 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import sys -from collections import OrderedDict, defaultdict +from collections import Counter, OrderedDict, defaultdict from datetime import datetime, timedelta from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union @@ -24,11 +23,7 @@ from feast import utils from feast.entity import Entity -from feast.errors import ( - FeastProviderLoginError, - FeatureNameCollisionError, - FeatureViewNotFoundException, -) +from feast.errors import FeatureNameCollisionError, FeatureViewNotFoundException from feast.feature_view import FeatureView from feast.inference import ( update_data_sources_with_inferred_event_timestamp_col, @@ -300,32 +295,28 @@ def get_historical_features( >>> retrieval_job = fs.get_historical_features( >>> entity_df="SELECT event_timestamp, order_id, customer_id from gcp_project.my_ds.customer_orders", >>> feature_refs=["customer:age", "customer:avg_orders_1d", "customer:avg_orders_7d"], - >>> full_feature_names=True >>> ) >>> feature_data = retrieval_job.to_df() >>> model.fit(feature_data) # insert your modeling framework here. """ all_feature_views = self._registry.list_feature_views(project=self.project) - try: - feature_views = _get_requested_feature_views( - feature_refs, all_feature_views, full_feature_names - ) - except (FeatureNameCollisionError, FeatureViewNotFoundException) as e: - sys.exit(e) + + _validate_feature_refs(feature_refs, full_feature_names) + feature_views = list( + view for view, _ in _group_feature_refs(feature_refs, all_feature_views) + ) provider = self._get_provider() - try: - job = provider.get_historical_features( - self.config, - feature_views, - feature_refs, - entity_df, - self._registry, - self.project, - full_feature_names, - ) - except FeastProviderLoginError as e: - sys.exit(e) + + job = provider.get_historical_features( + self.config, + feature_views, + feature_refs, + entity_df, + self._registry, + self.project, + full_feature_names, + ) return job @@ -562,9 +553,7 @@ def get_online_features( project=self.project, allow_cache=True ) - grouped_refs = _validate_and_group_feature_refs( - feature_refs, all_feature_views, full_feature_names - ) + grouped_refs = _group_feature_refs(feature_refs, all_feature_views) for table, requested_features in grouped_refs: entity_keys = _get_table_entity_keys( table, union_of_entity_keys, entity_name_to_join_key_map @@ -623,10 +612,18 @@ def _entity_row_to_field_values( return result -def _validate_and_group_feature_refs( - feature_refs: List[str], - all_feature_views: List[FeatureView], - full_feature_names: bool = False, +def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = False): + feature_names = [ref.split(":")[1] for ref in feature_refs] + feature_name, count = Counter(feature_names).most_common(1)[0] + if count > 1: + collided_feature_refs = [ + ref for ref in feature_refs if ref.endswith(":" + feature_name) + ] + raise FeatureNameCollisionError(collided_feature_refs) + + +def _group_feature_refs( + feature_refs: List[str], all_feature_views: List[FeatureView] ) -> List[Tuple[FeatureView, List[str]]]: """ Get list of feature views and corresponding feature names based on feature references""" @@ -636,44 +633,19 @@ def _validate_and_group_feature_refs( # view name to feature names views_features = defaultdict(list) - feature_set = set() - feature_collision_set = set() - for ref in feature_refs: view_name, feat_name = ref.split(":") - if feat_name in feature_set: - feature_collision_set.add(feat_name) - else: - feature_set.add(feat_name) + if view_name not in view_index: raise FeatureViewNotFoundException(view_name) views_features[view_name].append(feat_name) - if not full_feature_names and len(feature_collision_set) > 0: - err = ", ".join(x for x in feature_collision_set) - raise FeatureNameCollisionError(err) - result = [] for view_name, feature_names in views_features.items(): result.append((view_index[view_name], feature_names)) return result -def _get_requested_feature_views( - feature_refs: List[str], - all_feature_views: List[FeatureView], - full_feature_names: bool, -) -> List[FeatureView]: - """Get list of feature views based on feature references""" - # TODO: Get rid of this function. We only need _validate_and_group_feature_refs - return list( - view - for view, _ in _validate_and_group_feature_refs( - feature_refs, all_feature_views, full_feature_names - ) - ) - - def _get_table_entity_keys( table: FeatureView, entity_keys: List[EntityKeyProto], join_key_map: Dict[str, str], ) -> List[EntityKeyProto]: diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index c1166f8cab2..55dc2d9cd45 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -380,7 +380,7 @@ def get_feature_view_query_context( """Build a query context containing all information required to template a BigQuery point-in-time SQL query""" feature_views_to_feature_map = _get_requested_feature_views_to_features_dict( - feature_refs, feature_views, full_feature_names + feature_refs, feature_views ) query_context = [] diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index 44edc8618dc..8ff896ba610 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -74,7 +74,7 @@ def get_historical_features( f"Please provide an entity_df with a column named {DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL} representing the time of events." ) feature_views_to_features = _get_requested_feature_views_to_features_dict( - feature_refs, feature_views, full_feature_names + feature_refs, feature_views ) # Create lazy function that is only called from the RetrievalJob object diff --git a/sdk/python/feast/infra/offline_stores/redshift.py b/sdk/python/feast/infra/offline_stores/redshift.py index f15b9af4513..e4507b82624 100644 --- a/sdk/python/feast/infra/offline_stores/redshift.py +++ b/sdk/python/feast/infra/offline_stores/redshift.py @@ -56,5 +56,6 @@ def get_historical_features( entity_df: Union[pd.DataFrame, str], registry: Registry, project: str, + full_feature_names: bool = True, ) -> RetrievalJob: pass diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index ed038e86aeb..b9101cb79e4 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -163,7 +163,7 @@ def get_provider(config: RepoConfig, repo_path: Path) -> Provider: def _get_requested_feature_views_to_features_dict( - feature_refs: List[str], feature_views: List[FeatureView], full_feature_names: bool + feature_refs: List[str], feature_views: List[FeatureView] ) -> Dict[FeatureView, List[str]]: """Create a dict of FeatureView -> List[Feature] for all requested features. Set full_feature_names to True to get feature names prefixed by its featureview.""" diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index a9f4369a597..543a067c622 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -19,7 +19,7 @@ from feast.entity import Entity from feast.errors import FeatureNameCollisionError from feast.feature import Feature -from feast.feature_store import FeatureStore, _validate_and_group_feature_refs +from feast.feature_store import FeatureStore, _validate_feature_refs from feast.feature_view import FeatureView from feast.infra.offline_stores.bigquery import BigQueryOfflineStoreConfig from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig @@ -172,28 +172,21 @@ def get_expected_training_df( filter_key="customer_id", filter_value=order_record["customer_id"], ) + order_record.update( { - f"driver_stats__{k}": driver_record.get(k, None) + (f"driver_stats__{k}" if full_feature_names else k): driver_record.get( + k, None + ) for k in ("conv_rate", "avg_daily_trips") } - if full_feature_names - else { - k: driver_record.get(k, None) for k in ("conv_rate", "avg_daily_trips") - } ) + order_record.update( { - f"customer_profile__{k}": customer_record.get(k, None) - for k in ( - "current_balance", - "avg_passenger_count", - "lifetime_trip_count", - ) - } - if full_feature_names - else { - k: customer_record.get(k, None) + ( + f"customer_profile__{k}" if full_feature_names else k + ): customer_record.get(k, None) for k in ( "current_balance", "avg_passenger_count", @@ -594,6 +587,7 @@ def test_historical_features_from_bigquery_sources( .reset_index(drop=True), check_dtype=False, ) + table_from_df_entities = job_from_df.to_arrow() assert_frame_equal( actual_df_from_df_entities, table_from_df_entities.to_pandas() @@ -601,63 +595,11 @@ def test_historical_features_from_bigquery_sources( @pytest.mark.integration -def test_feature_name_collision_on_historical_retrieval_from_parquet_sources(): - - driver_source = FileSource( - path="driver_stats_path", - event_timestamp_column="datetime", - created_timestamp_column="created", - ) - driver_fv = create_driver_hourly_stats_feature_view(driver_source) - customer_source = FileSource( - path="customer_profile_path", - event_timestamp_column="datetime", - created_timestamp_column="created", - ) - customer_fv = create_customer_daily_profile_feature_view(customer_source) - - # _validate_and_group_feature_refs is the function that checks for colliding feature names - with pytest.raises(FeatureNameCollisionError): - _validate_and_group_feature_refs( - feature_refs=[ - "driver_stats:conv_rate", - "driver_stats:avg_daily_trips", - "customer_profile:current_balance", - "customer_profile:avg_passenger_count", - "customer_profile:lifetime_trip_count", - "customer_profile:avg_daily_trips", - ], - all_feature_views=[driver_fv, customer_fv], - full_feature_names=False, - ) - - -def test_feature_name_collision_on_historical_retrieval_from_bigquery_sources(): - bigquery_dataset = ( - f"test_hist_retrieval_{int(time.time_ns())}_{random.randint(1000, 9999)}" - ) - gcp_project = "project_name" - - # Driver Feature View - driver_table_id = f"{gcp_project}.{bigquery_dataset}.driver_hourly" - driver_source = BigQuerySource( - table_ref=driver_table_id, - event_timestamp_column="datetime", - created_timestamp_column="created", - ) - driver_fv = create_driver_hourly_stats_feature_view(driver_source) - - customer_table_id = f"{gcp_project}.{bigquery_dataset}.customer_profile" - customer_source = BigQuerySource( - table_ref=customer_table_id, - event_timestamp_column="datetime", - created_timestamp_column="", - ) - customer_fv = create_customer_daily_profile_feature_view(customer_source) +def test_feature_name_collision_on_historical_retrieval(): - # _validate_and_group_feature_refs is the function that checks for colliding feature names + # _validate_feature_refs is the function that checks for colliding feature names with pytest.raises(FeatureNameCollisionError): - _validate_and_group_feature_refs( + _validate_feature_refs( feature_refs=[ "driver_stats:conv_rate", "driver_stats:avg_daily_trips", @@ -666,6 +608,5 @@ def test_feature_name_collision_on_historical_retrieval_from_bigquery_sources(): "customer_profile:lifetime_trip_count", "customer_profile:avg_daily_trips", ], - all_feature_views=[driver_fv, customer_fv], full_feature_names=False, ) From c78894fe78478e28c81b00a062248bfbfe393e3d Mon Sep 17 00:00:00 2001 From: Tsotne Tabidze Date: Sat, 3 Jul 2021 12:54:21 -0700 Subject: [PATCH 31/43] Fix unit tests that got broken by Pandas 1.3.0 release (#1683) Signed-off-by: Tsotne Tabidze Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/driver_test_data.py | 4 ++-- sdk/python/tests/test_historical_retrieval.py | 24 +++++++------------ .../test_offline_online_store_consistency.py | 8 +++---- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/sdk/python/feast/driver_test_data.py b/sdk/python/feast/driver_test_data.py index d50128696d0..ea0921bf044 100644 --- a/sdk/python/feast/driver_test_data.py +++ b/sdk/python/feast/driver_test_data.py @@ -140,8 +140,8 @@ def create_driver_hourly_stats_df(drivers, start_date, end_date) -> pd.DataFrame # TODO: These duplicate rows area indirectly being filtered out by the point in time join already. We need to # inject a bad row at a timestamp where we know it will get joined to the entity dataframe, and then test that # we are actually filtering it with the created timestamp - late_row = df_all_drivers.iloc[int(rows / 2)] - df_all_drivers = df_all_drivers.append(late_row).append(late_row) + late_row = df_all_drivers[rows // 2 : rows // 2 + 1] + df_all_drivers = pd.concat([df_all_drivers, late_row, late_row], ignore_index=True) return df_all_drivers diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index 543a067c622..2ad6c4d4e12 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -204,22 +204,14 @@ def get_expected_training_df( expected_df = expected_df[[event_timestamp] + current_cols] # Cast some columns to expected types, since we lose information when converting pandas DFs into Python objects. - expected_df["order_is_success"] = expected_df["order_is_success"].astype("int32") - - if full_feature_names: - expected_df["customer_profile__current_balance"] = expected_df[ - "customer_profile__current_balance" - ].astype("float32") - expected_df["customer_profile__avg_passenger_count"] = expected_df[ - "customer_profile__avg_passenger_count" - ].astype("float32") - else: - expected_df["current_balance"] = expected_df["current_balance"].astype( - "float32" - ) - expected_df["avg_passenger_count"] = expected_df["avg_passenger_count"].astype( - "float32" - ) + expected_column_types = { + "order_is_success": "int32", + "driver_stats__conv_rate": "float32", + "customer_profile__current_balance": "float32", + "customer_profile__avg_passenger_count": "float32", + } + for col, typ in expected_column_types.items(): + expected_df[col] = expected_df[col].astype(typ) return expected_df diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py index b7fb1304947..2cc85a304a9 100644 --- a/sdk/python/tests/test_offline_online_store_consistency.py +++ b/sdk/python/tests/test_offline_online_store_consistency.py @@ -1,4 +1,5 @@ import contextlib +import math import tempfile import time import uuid @@ -223,14 +224,13 @@ def check_offline_and_online_features( if expected_value: assert abs(df.to_dict()[f"{fv.name}__value"][0] - expected_value) < 1e-6 else: - df = df.where(pd.notnull(df), None) - assert df.to_dict()[f"{fv.name}__value"][0] is None + assert math.isnan(df.to_dict()[f"{fv.name}__value"][0]) else: if expected_value: assert abs(df.to_dict()["value"][0] - expected_value) < 1e-6 else: - df = df.where(pd.notnull(df), None) - assert df.to_dict()["value"][0] is None + assert math.isnan(df.to_dict()["value"][0]) + def run_offline_online_store_consistency_test( From 20c94613b677aef66ab39aac277b23bdef3ea06d Mon Sep 17 00:00:00 2001 From: Leonid Date: Sat, 3 Jul 2021 15:41:53 -0500 Subject: [PATCH 32/43] Add support for DynamoDB and S3 registry (#1483) * Add support for DynamoDB and S3 registry Signed-off-by: lblokhin * rcu and wcu as a parameter of dynamodb online store Signed-off-by: lblokhin * fix linter Signed-off-by: lblokhin * aws dependency to extras Signed-off-by: lblokhin * FEAST_S3_ENDPOINT_URL Signed-off-by: lblokhin * tests Signed-off-by: lblokhin * fix signature, after merge Signed-off-by: lblokhin * aws default region name configurable Signed-off-by: lblokhin * add offlinestore config type to test Signed-off-by: lblokhin * review changes Signed-off-by: lblokhin * review requested changes Signed-off-by: lblokhin * integration test for Dynamo Signed-off-by: lblokhin * change the rest of table_name to table_instance (where table_name is actually an instance of DynamoDB Table object) Signed-off-by: lblokhin * fix DynamoDBOnlineStore commit Signed-off-by: lblokhin * move client to _initialize_dynamodb Signed-off-by: lblokhin * rename document_id to entity_id and Row to entity_id Signed-off-by: lblokhin * The default value is None Signed-off-by: lblokhin * Remove Datastore from the docstring. Signed-off-by: lblokhin * get rid of the return call from S3RegistryStore Signed-off-by: lblokhin * merge two exceptions Signed-off-by: lblokhin * For ci requirement Signed-off-by: lblokhin * remove configuration from test Signed-off-by: lblokhin * feast-integration-tests for tests Signed-off-by: lblokhin * change test path Signed-off-by: lblokhin * add fixture feature_store_with_s3_registry to test Signed-off-by: lblokhin * region required Signed-off-by: lblokhin * Address the rest of the comments Signed-off-by: Tsotne Tabidze * Update to_table to to_arrow Signed-off-by: Tsotne Tabidze Co-authored-by: Tsotne Tabidze Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- docs/specs/dynamodb_online_example.monopic | Bin 0 -> 1601 bytes docs/specs/dynamodb_online_example.png | Bin 0 -> 92813 bytes docs/specs/online_store_format.md | 2 +- sdk/python/feast/cli.py | 2 +- sdk/python/feast/errors.py | 10 + sdk/python/feast/infra/aws.py | 141 ++++++++++++++ .../feast/infra/online_stores/datastore.py | 17 +- .../feast/infra/online_stores/dynamodb.py | 182 ++++++++++++++++++ .../feast/infra/online_stores/helpers.py | 10 + sdk/python/feast/infra/provider.py | 4 + sdk/python/feast/registry.py | 75 ++++++++ sdk/python/feast/repo_config.py | 7 +- sdk/python/feast/templates/aws/bootstrap.py | 35 ++++ sdk/python/feast/templates/aws/example.py | 36 ++++ .../feast/templates/aws/feature_store.yaml | 3 + sdk/python/feast/templates/aws/test.py | 38 ++++ sdk/python/setup.py | 7 + sdk/python/tests/test_cli_aws.py | 58 ++++++ sdk/python/tests/test_feature_store.py | 27 ++- .../test_offline_online_store_consistency.py | 46 ++++- 20 files changed, 678 insertions(+), 22 deletions(-) create mode 100644 docs/specs/dynamodb_online_example.monopic create mode 100644 docs/specs/dynamodb_online_example.png create mode 100644 sdk/python/feast/infra/aws.py create mode 100644 sdk/python/feast/infra/online_stores/dynamodb.py create mode 100644 sdk/python/feast/templates/aws/bootstrap.py create mode 100644 sdk/python/feast/templates/aws/example.py create mode 100644 sdk/python/feast/templates/aws/feature_store.yaml create mode 100644 sdk/python/feast/templates/aws/test.py create mode 100644 sdk/python/tests/test_cli_aws.py diff --git a/docs/specs/dynamodb_online_example.monopic b/docs/specs/dynamodb_online_example.monopic new file mode 100644 index 0000000000000000000000000000000000000000..6531749dd94641272f886b7c90060f89bfc3883b GIT binary patch literal 1601 zcmV-H2EO_KO;1iwP)S1pABzY8000000u${T+is&q^j9oi*HM`Z*S93Cs&=>9ULRI< zgvKzo>0)pJJDWz9_G_ws*M3RQ3=9SvvcX=GC|M~Y!+7S*xzD+9t0mLFxSwa5dn<-# zSPY^t=3cjR)MTXK&FVP2anm-(S4kG;VG?^Op89No z4mm(-fVC^2l(WE#lbnei^3)*l#ZG`IR*BWwI-N0}djNRHg93X+R13WspD)EvKy4F~ zCIZ=dT993?x!M8E!w*Zgsv%?G9^fXXA- zr|nQ|phpB&e!fVP^?XsumE}3}7lQVJ^ld?dDHNzfvdbjkk(cuid5J#3aOQHAdDA2f zc-qRe1044cE+P2u=WY8_un88=*AYw0QzRp1sZdoG`->#4@e>C3kS;iLXfZ*UN~xBn>nVwVhTxj!i!qor7#C&k_tYun-XqJ;-Aw*1VrwL zU1xCVWceiCuxQOQcM^a4_}>I>5uldk;Gkara(;gP`qB>8?U;FK5LRYP9wpEP#~=lV{9uJ$R5ne!4=dOv~f?+R63*jRj8c4xc}}v zQU=e(W!2!1t~7}b#}H>Q7fPFU<#aFspcilzvF2d+5H&hPBm%+A)jDd>Lco?|IIxeV0))?<9@Ms{S>2qwhxFc@zfDjuyj?HJ> za%_q?^=%*Plv?&QxVzhO&}#Yq;;r+_Zf~``J%9Diy6$YX{PbzgA*~$7q zh*cxQ8O&>9d$=ljSCLNk?@y|v5o5S(L_lWI3c^YOsmePu{2D`NZr^ldT=p5FR-1AgXXgRON`M(h*U$BVxU<51yP86Ax-1*F+}Ag}Zp{ zsLPw0$APd(tqf~)eH0+SqdlqP>>L@;n}Wn`0bf|ut+vrzvfcgYxN2+F-$C;`k0w<%i;S_>)necJ50=s_XgD_Re_Ff|#ume9&tHvQebQ>DrLZhI z3}?!&R?dbCq`Mirlg=8)L)vfPr~<{-TQv_Z-fpKJ`+%1nkCsus)iY`(EK|cU5|%P; zO{`b-Zg+L@JGFa{p?Z%oF+|1F$J0O)ZFIEGA0CtUEYDokT|b%amnus`e_0BiE#@6~ zy>0^ah|^uE;|C`O=|h?^yg}*`uudgkf7Mw7rH-GDeo{G_9K-%aPm+=oGht3-N-gtbtN zj1QZOPG8EHEvmXPo=h4nkWML~bU;nP4#=m%T}B@~n}~Ou3JQ6Cw=v{RCYxcb1iCr5 z3l8)gy}hqt(@RBm_EAlJxs6T=wYpBvRPJ$2Nm~BSg8s{3q`EnG@7b4|`fXfO81Ew{ zGYkHiF}`1>VPcF&;Tva|AM=E})CacsLg+g>wUpn=%I^SR3xu~ZlBd&kv z!$TpApl114b1L5Ft6oF<6GN}o$v1j^vG3hN>;QB|YX;$#ax7Wwp|)@rPEusQXnTD^*4uC7=oVgD`Z;;XggfS0JHO1cZl+cbRyy9?K0?*Kw|wl|-}%idp8G_3GFI4e zZ2p0=>7|dmd6DxNlBpf#B%3UY?DxPM;@7d8tOOJ}d*tvlTu9Ma_!rX$ zYlEts>`z=D?_kIVp}ZU1K!|29wC*>r`$fK6^@b#B+QA74LNZ|SaN#M! zip0|az346fY+U4B?1xw2cwk#xHN7VCUMk@Qtyk{GF$b)o?M4u#tO&vDU>lG}(BWmS zi;uj}e3^fbpnpqJiO0g zCs&~AGoN6esNGPyjiv{I1YvEWA#3{X9TYwA91#i*nh**ecm@sJg`tW5`CJN`4hr_q z>o8DIp%zeZ|2Rewczpi(2Hc^4j1CpK0Cr2nK zTaILhPW=_algqkz&h~`F^`!ba}C|G{G^kgmpBuSe6c6 zJ}TAzJf$!*S>RYG*TI`b3H^%vKi>RbQ7hNH-5<7$g+luFA8&rtqW*XYF#pAoqOw?- zZACvp(ebqZ@f@IEWwwPI`N7Hj$6Mx4B<0l)<4e5*|IO(tuXeME!chIkoANju-3k5f zk?*};Lvsa|2(eHujnjbu-)RFt}D*|^BIT&huXjV&*Kk;6vqjT+$7fs(QyB7 zCUZt7h$iz&;1PGj@m$N9&r<_r-h0%?tQFKNk?R?iIF8Sd(g9t1-C!G%DnE#j{ppf2Di2AEzYevZbeQy6h%TL+q zdEqrO8wCI7rSAuqjgi^LfXviG{a+*$I(}e9;pilmi~l!^PX)L@oM7CI@BhX1VUze* z!<02Rg6ENaujb4o)$Koi`7?-_XBexH{Id&ZX3c@q?9WH4b)HIYS+wAm^5ADP9U7;_auCr!>E%T3G@|1)P!;jd{$ zhi-Wi}hu*MI&yUl8s_xKPkL zEj+=YX*1yU`Cs2b#|7i{6t5Zm!P=Womp(y6{-25Ziz4~ZJ#|T25pZI)lhY+y(ky9j z{xL)3(!hreVD`ym7T=hZ&n*oymfe>BOsAW%ZPH}k zR4YCZo1NF(OHOKYzE2Mlz35>#`(>@qX^|>MrVn!5$|7)Bt)^Tp+R9K`FK*lV;G%@T zt&!n zi&x9}h(IOe2m}sxmNeB&7Lms3Pcs23Rvfp#lJM=@md86QB}(pPH={X&P?Dea8$AK2 zyjwfVp2tE0PofKGysJ=i#272>(?QfM5v zhcJg+`CX3~-~Hf}4Fg}b5j*rm5S$J3tjxM{yEGnyj$2#~he=otyyM)b>+Lr-6SQnA z&POwRALnpSXIdXWguEnpRa+m>Jvix0=5v!Zlp!Kw|4{Dp;NiXz z$phCa$=bSPQ@8Zfs5h!|BT9fKK4yec4Kn_V>8hM=gHciB{%nxt>+|#z+9sWx$ zp8g7A4UAljj>{Z%o?9h}S^mowGt=Q8CcO+k^6Nb0K3H2;8%A-*zqG4a&`z|yqB}0)J_=+c}>xaRNlhPUOd0(Ofs+}z~-9MjH zQ|ixG$vHr|y4x>f8Q}jIA{Q?w5+^6b(3xaF1jJxiN*jsmB1N>wquY+0A@li^x?XS; z88&7>=~R*W0=fB*xAwy+JaysZGJ#~K`6y>aFORNQeCr&!T0^v$7ak{+MEcLR2H`~% z=!*tTy%LbC2G02CFTBdfw&DlejD%z}wGm&E32T#jowT=YPnKw@KVH?f_P6=N{sOMm z-(K!F7EBAXP9twe@m!UeR(nt@llR?)eURZJtDppL&@ispQ81qFH!MY73Ja#a!lF`S zY;psR0RppB3JlRZof^xjpT19z>_EWxJz%(`U#>hos-y93V&v@rv#edHUY1@b>6({` z&tq_RT)SW!>-%_~LEtEIvl<9bbU$S>f$j(%iEr_S3=Ac6a0!5s+bQ3M~_~7!KD1@ayZW?hV7b~bs`y#+y9j|olY%RLADUZ zCnbuQZyXsE3sZIS<*qsI9c29@2VO=F@Ht1vIpWbASuA^rF`McgN9L zdp2B4S4DXsi;-WbF*eJZp~JTbBOXb$s+3&(rMwIu`m&}oxpAs|{C&+4XZm;_x_FO- za}9$9AmPXbLTxj+J%LDqWLA8inyssUOO2Q62)KT;u3K_R2aAim#Ag*fLAahZik^dy z7N5yV9bE8%93Dom0KrELv9+{Pw3k#9(TaP(h9OEhbYkDIS+iEPV@ zopd-%CO} zrs@~+aWq{BUYv{3y7LT!lcyAP#INL|u^C*B5`K2LP?xCVx-@kbngGLXCqakQKGS@? zyf2$r6bj^+6;pVF_MP83l%MGPK9F>5W92@zEj?+8p7-XuT_6xwgBO{f=VYOxm(;2z z>_iLJDrfAw7jO=n-FSM~);)o>n?Mw}y4w3+8_eS+mBR^kMpe~6_>B_Bai6$LuB`Jc zkN=Y25_!6f7PUInd%8<}hzQ0E327&AO{MtkB`pezU8`!4{g(HLAS3J2@s%7 z+veQ9SiOkEnmPFXPVja!7AeQUqBD{&vuhxpA-`WMQm@<%C8`qPEat?u8ANS)65 zX=x+Me{Vc(M2pZw^b6&cu6SRk(F#sT%(^GAU7HTF5}GGh7LT3KJtP)pOJ$lZSi&hI z8C{KGG1-dX+YskPr~5v-E2ZOU+xeq%^mb5Pf@rhd`y?Uv?zj_XW4=u)>&HbRrR&%Y!+%OfvP%}#&pDf zLF(SATe@Ok7W`@T%#&8AGXi5TWQjB%eSkv`Sv5~F?Pt9Y`Xr`{OFNW+9l3LGP>V{A zRC<27SUAZGyV)A3I_o+1XhN?0aw6!6Kw_5y-_ve9Mdg`Nos!0ejhIfNI4+BEDmE+z zD`Z4X$8yhbpTIB>zxjhjMgW1!mtT&tv`V$0!g9}kUo_%75>SS|BLT>WklWh)oe+g> zsW$T+#gW_B@fPh#4)viV^ydxidm0#n72krSw#oJ&B&>r^hHk!VI;fz7TRZy|>EKi) zeOqJPQ8*eoR@$I96i=n|MWjzQRPItbP ze}JC>Cqv!}mnrMy$2YUL7D`sT5>1DCjFRe)wb{``=Fp&v@jjH*Ckap6p`SW{V?dvD z3^A#LDjXZZ?6biG9Ga$fV?xl?{vG~`-Sc+O-0;?4Ar z)sqL}ivH9@zV1YU zzg~bPW*@^L&DigA?|xZj^n4xQI0Dlh@lJT`aUC3ATlW$Sgr7eYTy9Qbu%@Mt2gf8) z*wdr349~ulk-oE@6rWmXAjHw7T_h&YP%fY<%=0yUz1r0qd8?8f;HU^ zHrn{npGjMqJoJn4d|aO8vTJyoV!`K&fNpjz%Yuh#=NAjvOl7!^mT2S;WC?1EaO+Wc zY3f10Pr}X~)ijSt48_i2sv0N_4!&Al#LeXYcmo7+zG&7q*7kid+&1cwJKT|yz#(U^ zDwK=Eiw53vnN1^Hw(kQ1Pxt46VdXxv!IAtr?pyKx_CvR5+>6#<}Q|OV-UhDUAKlcK8&mCOd#RyGI&*PY$S|CR##m=?b9X?SJtn<^bpuND< zPckx@qioy}ve}t*75B(|WD)1y35WL_mJma+;-IhVEl>0m^xSh8hyuC+oK4u7b6%@W z49tx?uPciLWqcc zgHnb!6g6^*nxwf#GhLli-4h@}CGH41Y9;Mm$=6-(WDRp~4@F!xn|V-9Mm(2NRSK4~ zN_X$tQ+-V#`RZl5vhK-|Tu4TN1|_A;83Ju3NygDjjz|tlKZm{bCk2*Il~e}YB(rwrg76Lb4~hH7zZ4E7 z(l$;aAbep0BMz64r6uX)9)k6r+sF5xDAO#0WE?5_Ag!ahL1uR-%?(TrfMuyIu0^) zuw2mf!NXOf!VKSjf=FCW+k_d2U3^l$CI}fF0A01SwEVoqIuhd;>>ch8%~jMA<^#77 zra5U+_pWK8V*ODffi}A@hoqhd6jI)^N6S4?wCLzbTLpKv?){W+e*jeeyAP5Upc2mO^y2 zla!K%Mf~T*#BPnx!Xd%~ai2>&+0;9{5PqzIQyrtyu2%&Ef$gK9s)_+PrVrcX2#7%L zr$jv;z6~!7@b6`}jYyXk|8uT;v~vvJA#Xw++XJML4)CGN64DNBz1NB9&1BEYTVzalg;|;mxd94_FTBxjT5+OmhVJJ z;GOSCoNg~7-GFa)qdT~L?Flq0*ATT{vAFA)+&6<({JT2%UkS$ZCQjI4c@1!}i2JK= z&lKs$n6NGYSc`{t3CXwQG($fJ4sDLrOElwI zJ9?>}wB6qBhFbT?*B*wej%wQ(kWwkr0WE@ba3T8O8(SFrUU_WPvixHa498=P&fDA` z&Is6XMklHWSsqAKC)$b=bFrx$_c0|?E559-XnrQa08VcY$8Fej8S5I@89v>k`sKz~ za;E6qw_9XzT*6K>vk3<<%Ie$xGU`QSj2-y8&{Dr26RZffwVSoyR@`VAokTyX*XZO* zK_#(zwF67Jqk!)?ePll3y?DB7d;)$qYi}Aym3n^5iq}nGRE`+nA>N6ypDV_!__THA z`NkTueloP}xM_;xlDf5i?HOmEFnpb7>9*_o74l=xd(%`^?i%QYKcM=h@>3R93aFK~ zK6-#Uogs&Hbzq&WZ@~UvXk!k0R(5ZNq+#l#*4+84>g?@KD=znP&?IqE!SYoYNjm*k z9x_rTjYS05@%i!Kt5;MdwqMyth*PNFAW#rp{qH&&a6KK$G&q z{sgwxBV<44Ztcob!XHls&LuwXO_x|TSE%WvrFZ+E^Gzm8@Ra1@y7(4M*hQ8-&Z8YX zo(as(g$Qkixy(f*s+#Lbo}1i0qnpYv;j&wv?wv!XYj4F4-8RxnuI932zoR4$1Nnia z%S#+=#rqk?thKOHfX)2q=bG3I>v>1$c1C_*eV4ofp$4*pUzr@dsK!#xiwdKbIrIYuT` zI|0;U4QX&YYwWz;x^*bpu+hzX4+P!wLaT$d1rnNy7cpX*w=G8 zY)@k!fr$A&?R!9-g;(mXGK&ldRae$I^qsS;=v&HKw9T*KSPO%%o_r_5m-#)`s;cC(Qddc>< z%`+*yfAhT`%vwmHA82Cr+c|&|cKv{Jfb=%^W;xlpor;RWn^9$qOqsjSd3wnbS`>rz zC6dkqI}zrX=&I7)?|JN2Mcef#h-n;>Gz?%O`k@stJAl3I=c_v)gy$F@osSmh99YeY zKV5!E5^5VHL8uSJQGi>^^hjHM-LkO8suWRny3woEP8SsXB~YRxa0tGB0(Vb1^!f+J5r*UO&W<|LrN!{#dq=Iu|i$p~)70>@wPga>k;?x&>5+ScmPaxmA>6>o`i z$yh_1E~m8)?TKfJgB#%QP^5jDPC8H&mwDN{4l6D67BBn6^Bn7VQXs4kt0QE&Fcs=CA)T8wS8EEZOHbB;>{LEV=DW%2`@mLL&w(uNSksz%6pt+?S}0+pNi6C#~h0s ze3E4~U7RcwJ&+$7YUqEHym)_mZWBjtfCqCDNUW%La77XBTb@^pCZ0x)aQK=R$7hWc z3n$eK9syFcwNq}?pX6%)_Th zCTmj$o+rF8wqSEtZGzaZSJpfS#D1Oi1f1sIR)aB24b8E(cJ&rd1NP#?A!k96bI&x4 zs4V7}yT@}LMKHas??VMl77TZ!Xt*E|kksf>9CC7}+|z1T(n!HhhPwVbkp|jH?#4-8 zs@;62U0B`3BeuTyMvnoGpVUjWcdf4+tnw{L8xBIf>Xh~)NCJHF&KY#ROxPW^iVqLg zXkT-!Xxot@Gs@^cA~UftYOHZsq6ODZ@$MH_El%~{wfrNWao7LPeD=;dRm^ti!&kO; zQy!RAMjMTtc2f+a!r@rfzXDDPUR+S0k^>Iid?H?5_l?0!$+!tB1vz)O=qfyq)ih&t zn>^T0@yx4&|?i(#M-b2;;B)dGM{0IxEk zUz@JZ;^V||igjyZM=kngWGBZ6SZ2=H^Mllp*M%q7(mx_f$+h`Hcmg7`5x-GmSxN*S zOZKgBV>A^)K5I{4GM>4&u)Ftb``~@%dUO$?)MFavqnkm-hos`h6JXNj$|;IMl<*QU&oUt=v{#)Fm51XLzIBGB?Q?2~easvO!| zlm2dq5_mTWv0j!|862Q#v5otE1v(GN7};NZEI8^MqE!2Zx)B|iI?PAJy$9q?n=+P` zn{2Kc)Ct#S&uhg2n#6~1$5$3xIU3Ys6@;9*F&S5SlkN?A<+;u%jkp2f$Tj6RevR#K zvAG+k3e=?aRfQ5uzn!;hdu2%4cv39j)!0S=ge^dDaj%QYBesHvKWTk*MA}Y1)iEFW zAS;J|{qSwrLrj&L-L3djRnI41KR@IY$4m48BxeAmuD=YmC}c%5B_-mrG|nlk4bGjA zR7+A0x?-*NvTZt*Lal{i75q6b6Tn*hwJnZ$TULX!C+2m5HZ%BB!GIm4H|=>ZyjA%$ zhKJ#07>HhN(NRzL;CDaNVC}+$;@F#F+bC;S9Lh&Kg&g60-lVk}4}kln(zyIO_#?^d zZx2J?->z}|vTD>waRTvfOr+@^cSdpGv=|CDwuXO0-^+JYip;z1J#5wXV#z`?SV%-2 z&Cj4Ti9O`nsf&8_i9CY{E%Am!9+FFKc9*4h$KyRTM&NaWp|a;yEwyI!a__7JecdL_ zxU?#b`@iSQrm@J^#c?~(u)PxHA_Xa5Dv1QKRmvs5Jw!7C!@7vB-eY<-6spCI(zRL>++}zQg`ensn zrV3f%-T>#aSW_FrEz{O=1Op@*7{uSO6B=e|mE_>uQgZ&;m+*4NjWCL>)M(BIRkYJt z@CF=1n11+SaO8q-+@bs?IMVti)vzUu?7~hXXpAgxnlkhWnT>3y`})%&4PLgl{eF)V z(9$I%={wTn94kixaTWu?U}ICL1=3fUA%-DW*ET14bWGVd&XV_Xd6Nv z?$1Y~^csnpK=0L1h$t7)dim>@0?~zce(#0eYV-OEzT*J6M9*5PAdWG?(2vUe>O1Zq z7u}U=k0}pxcppL^1Cn`pQ#W47zfIPj^clM+M+||&L58VYbO>Kfea|aIC(5}7onMbtH{GGC}W~IdhvANu!Ra< zw~#GFBAepCPC!QcLr2N!;+p-@DLicG*HC2pj_n+a3N5+=bW7J|em)1X>>V^EXWtne z*SRIiZ(bWLmUr)O8z1a>1yX%}H+Gg2!Gs%&Dbkj+dio{u3y)zWv}4045L!N2G-V1d z`sjNd*34+Kq*mo}D=pbqHb;-6&x7rIYs~O=0X;e2-Equznx2 zKa5S?%)UbnMXLQ#@SF9T3lE^bhG;LO=YGn@rk;z`klY!`WQ4C31~3gdk{@&!%-GQ* zp3ku1J**+gA3TWdW)rf%(W5UzlW5Vjiv{PIGb15Rts+@8BH!U7&gOo zywbqfn<`4=O35eUO_gaOI?NCsu38`QTyKhYnDWJ#-qXy`Bs(*>5bW7SPkg_PGI;i2PZ*U>728 zYYcOw4tON9jSD^Q6?x4%y2C+uB_{Q<0%2)jrEsOJ93NFqL~{fa-|X6#h)fZmLSFS>ET)l z|J+>>+XymyN2JDxMETP3UTrCftdzTjkB;+09NYj0SZ>bE^emoLk2k$uK9Tv|O9FN( zgM(Yo`&0cO79_`;jCWQ;i&POhO0-e8-{tU!^Mfjm4{mNh56wYwmn3&l4Job%CC}dE zzzL@;z2+vlgY(TdDov}fw>jrV!9}Vv>gr#fJypbqPEc#Xl;Sh+F#5To$b0wl2p}3= z@0HD5(|;U&_JwG5c=b~a+0tBAZj36(S!+^KUq>L&swVKL+)<>Qez$>SPjbjpDJa6g zQ)66tzaNvUD9@(GaBWJ04LbyNrHaMW+5ll`YIxiP9zvfF!i?kHS|$17Dzxm|^FDop z%*n{*E$zvXq;}P8`WZ>a^Nx@S$Z3br=+(Wmc{ABqQb~!dW*mG&tmw!ND&QC~cYG!i ztthtN#Mr5uW@topr_NEK?r*j`#h4d-08%PoZ|hn@=r_U*2a4_EY7k%BQb;lft)PJn zo~!P1g^_d$ZK>3b$$2-pD=E@xFQCEU+hBtH>}U+g+RHQ^vsC{(x_aNO{vrTOfO7sd zFp=tc)NlnFwSJHK_8frw1y&+W+b6!p4nru8Jp9$%&}@fJQ4~P|(+PlDtu_erj5;DD zIFGZ+|Bb;A=6tm&8@~ZAkI^h0Q^ZGdgCTpRbI_Kp_Km~xQL%Xl zO$I_eU!uihW)DSn@qWZSn0hf*$kENSgV@~S=>$Gw0LRdOAW2>!iG;S+27@)=E8Ayt zKK~fL!Hy7kq$Z~t1m*TU^%(D7=>)}s=w zfhH*ZR=-?Y})m( z{qfi%1z3D{vO-hYDNee<(B6Kxr6XnM)FCm67lYxR1V{trPcPpG*_DO zUw5FR#{KW(rW~W=LmkW*ls>8MO3cgBn(Epdml-`=+f`Un8zX0&W*tSY16JG*nMN-* z;PyQ?GI*oYIi%-6a}v-7t6w2NQ8>Y3xry0rum&pab(YPPLj^wU8VYB$D&I?uoblrVhH)b2?k<3YR->+ZoVDD4J^K7Jjm=1Ez-dO?F*1+w2}WZe2opA*y5Uujj4fA$s$#uo1 z^CX}h?o(PW+a6?Zs{xIemlfI|v2uoiCN%H`D)Q2Cx`Ko-(B<-OP~!op)#$fjGtO7} z$}k4H<~W*SxVrD3>9&LPPEUB4Lhm}I|x31 zdfpJl=xtE}2=Gn_IbxAo1E6DAaxZ(lZhg9Z3&6i$K)S1Mcn3o*CUW^~*tcKCIt2Dq zA(t#3vn?8(N3hi;#PuBd!|O5Df+h?{Vl4wlBYBp0zV0i6ThM=ijDg7Hmmm*x=qSDI zPQKn;_x^b~Jn2h_PG~j2lApnqu3NQ>U|l~Rd7()&-F0`d8{4)WL&t0l`DXS936aqG zBW8i35Ztcj2T%F5;p`9gq=L*MUpcEL4&OxZk0@`4;V>NJ@_%?~H**PSF=X_Q;zzo7 zTl0Zd!YB2rd+Yy%B=^Qkg&ZG%<`W+GVT`v*x($zKO-6s08JITX^Kkc71ruxHlfOOv z{Df1;>X6^FqG4!Lbm;Ym`M~9eORhle2;_z&c#;NyWxQ|qA1&{;p4991y4CP3=vj=y zx5;g*_feYNd#g#z8rJzKrgk#z(?!gv$dk665AF@oC8UWP$r}waz}tmsR+}7*$p*6N z&R^QDS*%Z%CSmzC-BPce6a^?+>4L9S;;vY2eUPv%n>N=ULa)JqfDZnuaoBdWDc~7Y zvNBOMj^cO&=*g%}Ibr~`o_Rc#Su#Nqtb_c3fv36X&L<4J9h!!Z3hh~IwRnaSGory! z4t8C$#xWXT`7DX+A)FL`Tt&xO)z{qhQpl_dw8q=E`_^hpD+un=ZgmBd2#-Ex_E1}q5cp(k{fkWeHhRA)6#ER2fiRd~AU<@;mxg$OE(A875Ii2N9BP^#L*>QInDGx?f((02d1J>p zGEj~f1cFz9c8z!wTPXYw?}68B@>@;R4%@8CKr%up2ZwHZQ3Hb4{L2vjRAnX$<+2$& z(M-}OD*N6iT41C)Bge~k8(3po37s{Y@VYAe(5m)EXuULSG`?N=_g_^GalP6Ai8&@U zLk|-)Oix0I)86D8hj)6u4?t7bH<|Rev{okbHIm(OEZN=cCq9=04G`qVrnhaW}*lXzU)dLbA)ev+tU9@)wM9;@%s$$aBU90@s zYeF_>?3ITXHH)EA&V{R8_ON>Np_^!7>NP!mP>*W<(aLS#v>DCh_)pO`0LTFX@WS1& zkbbm!K!KMR-oyHji|=EA2HK;rx&wBw+ZzzGN#I{+Bo0W-S>ULo>=EsncLowFht{&09wdA|(X83=<`2VYn$AzbEf+J$o3%$-Grr ze+Vj(VDM)^WDH{^YN^?Wr9p*Go96T#U~i?KQM|rqT`N_MA=f`A;7k@UDtL{XEdOgA zKj6hA-{HR)lTe#(`BY`fZrq3AZ&xt8_=C#|&ev+Dl&Q9G?oRM>QjOhmk`B-{2WtC1 zTuOfG9Z)V%-3?VOFf+mDu%-myX8UKmQavQJ=3l#}{E=zLvL#V07V9gDwdxd2M^fJu zsh1^`wLWSBWsI`*@vi9Nc%`*tf2S0RbBy2RfDXu;&YpO~4}U9Me(y?CF_pi2Ls zOn*7#=*s5#JXU=&!1YCZ!T0~)diTFZfSw0B=>;*XzF%;;!EXq$8@*Dud(ge6dcAtAUB6LCx)?|%eayUx-f#q;Fb8z<7m@MjSUzlzq*;{>CNN>8 z#%{X+s`@fN`)`oFqO#3!mKM8!MwdhSsUonMJG#lU5dIkt0~9cN2Mso$gUax!RL4Z5 z5|KkS)}jwZ32LpGDgCjugM}Z8p0kE5xGifXPa*mBIe0WNl6zhT&S{Usbc!$U9e0 z>ZNFWwhqf8F~kiQ;NoI{SkkTQb40I-n$h(vSp)>h0p}x&XfRp8$S92nL#%rt zfROzAjGph3dk-oc`J?0bvOAcB;r-d$`}4Km(quK@Tnl_S^UM92nch?#&&xe(K-G^r zgbvg2J4af}0Q+_3p)`qeoDhf2de_(53>YjzCD#4rW)N2-l7S}BEvM$QhTry4M)Qsy zrO2~1zX$f=+;!>)5NlwE?)G)Q#d@EB2m|X6hlBLIsDcEdKm5~L@{e@zmu^0Od znR5Z>cg}mD4B{Yy$gAhw!D(9n-d+CVUF*{$NUCB6Fk^IJx{&zXtbHs(`5^@mcjZ&u zdrP1?A&H^bKEX^c#L-t@gHQu9tZOS=PxiuydH1C zvh$iUb{$Bx8WU0VB0HIZ#Zvgy-!qfX52{87cvidu$$Qfr2Hr#|+jRsXu{-a*_w1Q1 z$V=brYDXQBD>4sl3C_iJlPE$F=SrP>;g7wC`f(K|5(FfgfoB~Q5c;*E8ll39Tf#T1 zunY$ZIaKuMu<9XGw(ei*rOe&6y;As`9#vU>FiD4Ae&ppiI(oa-2K~ zYY}&=2V8Q4AGK_@3giH}G2FZ!>jom$cc8YGqlV6j;kdcKx*;XU2n=)~zf&N=?-2Wz ztIhS2&;$Q-g9D&0OnDv#`~74F#i7vW1Sl6HI;C37xv%g)tU~{DM3i}9SU>XsNuJ?- zfU+SV)4)r0g?GK%&65XQJmVj?0Tf(`eF?!JY=Lnd&<9y^x$C20h||;shRiAo92z&L zdmiz*Djz^-)7`{BmTFtnBxW{j2+Lc?zT=JoTD#HhNRQhkK=ZBN?6TtR>TfrAY$rVR zEULO5xg$u`sy1J{7gM7rJKD*KR($1>l}ak)A+yxzY!kz(R)A)Tw)Q?V7($!jHEwAs zW~G!CV&!!?<07gxbWDzK1y~MOK&zvFNjKX4D@Ffge^_Wsw=HP;DyqL$#Q*x&-{Ccs zf8vt@ZBi9rO`6yQ95y4flfvG2#t}I%*oZtp9ZmUpGsb+k8LRYmgIe;-&gq<4#vJx@ zvP8^ZouqmV2wLunF11>`8W?q(X8Ed+(^$(`$IBMDiaJuwi41>?6J%4S$=*y7EnT+?Z zyNWmp=+wcV;uPjLoL19A>C!cwXaYf&OzUP2K!UDQ2?)|bG#|_KM;(oCvs!L)=N7pS zRHp4Km=nVBp%?*5n89dOhj%Q^JG(amt|NTlKETV8>LRA`0pwx>X>vK#^mNWB3TCSS z3{0B-RMqu%+cVD`brT@7>I2MD41g2Jwf>f?DT;rO&WipN;B70mU+Z}Sm^AK&=~$*% zfa6H!t$Gaek(*eWe3my(e}c(#D+P44?+;R5tC#2t|LbZi3gb>`cYoSD=m!mJ2yWd< z&@xph`a_EPgxWvtLw2#P2{rbk1zn3Vys3{)Bv@?d!iA&T1y+T)l0x)vmj`P ztCb@adEi)h4zlhNc@m?0rt=t9I|%G~S|mi-6dKf;Cr({uQd<$$7NB64;EjBm;HX~wn_eQHdaU^SZvp(jRMUH0 zz`Ob(A>CBq`O^O1Tjm!eVxSt1mm5fEb^601c)-c3Yug8l)~f-n86V`J%5fYy6(!j5 zmp*+3U*;LxR7ytn!2;Pk2T^j!f z00gE48n^`AW{){`z!4|VaJN@P-Dgd)=d|QJGe^IA@CK1eALazW2EPLqv5(C4t1t{N z_FB7Vsi%+#>B~*A zE&T215;=|zQ~>}B|CP=c*ctJsv4ne$-+;6p`hOF@6o{fZ$fcH3JMuC=q|Oo5RvsT9 zgN~y@X2B!?cNNdG>?Q%U$qZap)73D!Ky`bqz@0T(yEj*r2Ursk07n?!r)G~Dg%s}L z9~%&WRw?=T3Qh=+a~ta?V}yvIAk*ObKHi!ob6R|F0xqo5?bMKgHE@+CKCnTwOXRre z9By8|5C?uZAF$yGeW?8Dqulcbs|#@!=sry@c7A`fL$KjuP3E=Fe)d3;FYeD)$SFs7TE$FR&6Sfu~nXN?q?lPs{3{_Ay?i^J^+$8 z89(DQLvN@|rJmaW>bIF5NK^42{+;cc4g(rdEz=)+Mr;8ACN}@MHYemn@yppI%mB^L zn+5Bt$f=e>U7>sSg!F!(792)&OK71GnqcsI1C2waVK*{^m;cumr+I}wd>D+Rht7tN zGAJeaH1B>Oj+;6#Y1LNeq8mK27J~pJ5YG43Wvn*f+xGdWNS!#Hd8^n4Dsvk7adtTK ztNU9LxWwJXa)ZudT(Ucx_5k<*js^YYS2VPp7Y2<@9Kn+R2VYh=H06D+fpL`nCmg{J z)AZlbD2kM4M>u(0Md-2JGMW6BOpHlFy9wO zG>>rqo0v<+@Iz{2GwPY0ZhQHfGdDB&#sA0GS3qU8bzQ?FkCcQ0A|+rVtsv4NfFreKC=Lh}sBYv>OXiskm-#uol9EHc zqP(81Uoy*)`K#mvdHhZBFsDFgtxWjPlFMfhLQvWrkYjxT#f3NzY-88-onqa z_f+#%4S}C!(a55^vofmNt6*~K=a&H}a6N^?;gXX2)*}NkdV<&gErZ8YrclP5LvBp}(3HwQy%)b0w#`luc#uXtdp<;$Ia#3w@i(*%18!F)M%?_a@@Kl|dCr6*w4cgtn_($Ke(8x9ud zEo3*cw!<_%MN`05LOyx^%L|f^v!b1kpiNord;~84`~y&<%0Xzt1a1d*s7$2bv*M;G z0ku@clye~A3xWE6^(4tiKl|T&V$gHpB|Y*W9Lk7$zZiichGy(TlZ*OW>@PdFw?>-v+xPw z`I8C~i0Sq+lucyD5}NHsr1mOn+9x2mJL?9{tIv-YdOzJY z9Qg!S-T;p1kLC>QH*dAV7Ip_%U6WMM*~%PLC33WIiwf?|oG*g^ZK8C&MWW%^m**!z zG8uxyU7`VZZelhm`ZUfrclD#gACu6l;o`mfivz<=H@pLyXP$&J1x;0h=m<1 z^G4waWY#i>!RQ6*V+Jr?UlE762R2bK_E|e&*75_36t#j3foOcgmpWzGv7J2Ak&nDI zl1ZY>if1nwcBKXhKkos9mTIoS+oUIUq3k;D=NAxeV*rZEHK!eOCRtG`;EA^5IXC`L zC-rPd(-^{L`G-|`eg`8_gKno7Ubm06JzpY>0AfpK%`GfMl&FQQRwb<)5^#PY-xF_GYowu~J;EfD<= z`+cslfklsk%Otc*sTx^4Ce3Wl>&-VTy6!7~1p_ppSseilZ|9Kcoh|S%@*EZx!R^t7 zt?-}qVnzO-SDA#5__7uo%0QJ?Opv+^xuiJE)6?9JUIlx!UF zV>5C}sHyqc{6G7{I4wcJ7`wJXuQ!tZ=v@8emYqNS2 z_`baDo#re+ZmnUD5KjFE!H4Nk8he;I6c8qD&Ly}UcDR<1n@MT) zyX{CsM2MzC3fYci6u&AV%&F7J5~y0-n<(75T52e-KJ+|x);;@so^k&)RL<|lV$qgf z9N#)c7|17LKBAMV*j`z7%H3A&@PjCbm!;4!Z4v3f6UvFml@{1kOwoMbXxiYb{MQd> zz``uE#}^6H{k421f`pMG0Q})A_2W%`Q15+aov`ql6QB^5QO3J?j^Iy=SNTDH9;Vcc<%engi@Byq_k!1eF7a`JR_5 zvPbZ*H*vTjIW3_UbOwhekm3UvTe^{ChHFP4pFZ zB*~DIRoS1q?Ii4skm^!imX#ZRSurMbC;ET3{oMNa*pm{!2b0N72ykxt+?xLv{t-S& z(&j}KO5%=Not)_Ovd}P{^6x8|@B|mOglUJD{(I|SWOmfqsUs^2FQe{f`T=(yBWq#| zq@msmT`h(R$TO`yXkJZ#qL*hZ0QRdlX0+O;5y)+e+7p%;_q%x9mS_u(9i9 z#IOu2F&Nr_OJt5UWXd=#%N7w5p5A?siDNPV+>vTP=v~~|U#Jk8iZ?faSePVW;rKIw zd@dRzi+|(kTrKC{AQ0?ij1SA=zpE45RKADAwr)RdVt?p@Yr=We%bh<9?{q;uaU@jP z-&)iit@7Gh$c(XaE(eT=+fwbmUng`(4-eop&byWEAKSx@AB;w%^nH#g5*a{*F%Sk|YSJ1(@4=}|p(rkH!4 zqYJdZIl^X5LNH3sQ;04)2a@#^lIRwo$>m6_kQUu zEY=c&gy__n(77gKFyj!m#8C=3G)f+ZQ;tx_05m>!_d)uENs05p>g_%lq&)_r@pVi~ z&=421^Bh?@Fe;QvLnHYziWWQfy-JeZj(PdY!=fG!5XB5vdhO;yYXinYvjo4(5 zV*%z+1R_!O(?c&j?8FK>HcW865rp-qeGST>_>!GrL47(kA zCtah&=KC7)nM5fH4xK;G0h0*&LI^GU)7Huwptzf;`CMoJU@G|FjvF}VlmarBZcc6U zw6$^HoSGH#IuY3}Sl(@SW1nKLzi@C?nep`dbAiyOa2ke^r#A)u{doWR9slDqWG6t> z9I4?iE8Cw%^Y0(vJ45RU;YoR3&XDnDel| z(7A>553$-tO%9a}ygokllej>>H@r;02J^uzYI^;joOah98W7F@x)ZW@aaW!x-sKb2 z%i+;{=>F*1&xPyu?o!3JmT5`mfPJvzz6Bp;DAn&`B7BwjuU-(CrYE;y_4vjLq#yhB zi(R}}u`PFTaVNBS%I*d3>#x7A{$4nAszr-s7SpOri;as0+NOW5xJLLNDQ`6J$XUI& zvIu|ub5J03s6n~okGLYnFJ9ul^jAl22#Il%`TGs;^@|GF3gX;~{y=|F0Q}0bduV|5 z*X8|r-2dalERLcKdHrv}!yk0}&kxB9Sg;CN@J}OU@c1O+n;z6aoso=0@1WU60bkLwKS*({KVTY^Z~aa zmPPmLyNk#kjJH-_`x&@Lge^-^%}J!TY7@DVbJG##U`F%(fDZy!2QUrTRIVF!-hnq{ zns7`2hyjR4B=`B#-v3%v|NK+91?Ru@3Opd4;wT0m;Uu&=`8OGHJ{~N+=39CLZBC~H z5kz1S3T(*A1R@@ujG`$@f7-Kj>J$PFRu3?{0n)P(KZoH1E!PhiBGBEYetJt@)7k%K zd}DAUpqmK*nWra6CMq^yD_qB7d60WYnt%0H*frgkb+kmp7a;IUehUPY@mW@9k$Egx zF0xQGa%FXnek{980!v)*6zOt%%hoS@nlHK@BYXpaKvJ)T4tFZJ4>^<1W2FGV&M9SB zStA+az=Q z<&qDLP3Wt2Kva!#w`nF{EW4b6LVJGUW-r{C3I9_&>N8e{_a2X-Fg2mwq}CzN9b_$C zxmg4fipS+YwsQJJ(pr|`MsnZo1;f^5MCvzzB+P`%gG4Hi8m1^p3==ZS$WcIum66MG>w~V&}*O`!Z6U9UnOULx6^-K zxqp73z(wM+EMPX@XIDpU>BkR^YIl6>CC}Qn_qIcZv^pP%z6-e=?GJwxmi5veuYHxH zM|l0!Q7S6qNq%4Lagoq77jC7@SS_t_JQW!LkCV72GO0Qdu8u$^0I~wXuFP+;x z$q}TBf#}t*00Sk&J*t~sB!*_3UE-a*_>|jm8aj6YpGBXHvf}jCcTcnBn z6wp#56~fvYlZ6Bp;^Dxs(Wp80qgMHlk>)E}p&{|xd^*lV)6zv6imcoPIPS*_kw{J^ z$;=etYKKr4_Ui>O8=Z}terf5i%}RU|)rn0h1TA@taNNcBq8jBpWB8X5WG}k;K7ekj z8G7Ien06LaC5aO9+|q9N=3+{Y#Rd&t!9`(`uL6n32`DeHdp*x7qGM2?e^J8A-HNBT zI->o3Ik!VP=Yb$`y3|PBkFzP)7LE+3kkDa4n3%j?jN;GBt(f9 zcm$KTG~etWTpf72Lg7WX2+#VY)1_jo*>~LsI_KVItv;>kdji9y7kqqS&;|Y&LOAxt zq6d`Eui9;VyWNODjG1Xj3>1lJMq|^HW2tZ_Ex;SlbHOl^nDC|}!D0#YC>^!BbhwIo zsIcMhL1Bdb(!Vjmf0bg{d@OBg0=;2(YAg3wgSb;^|LS(@gs%#mZN9NObUs>ZKh@p!4)WV>0D&!v* zZo{CD7=8uvmGEqjmAXYb(zcwOfx{(79A8B{lnc(#Tuv!vpxn$FKL|k!VPhe32 zG?bI)A-N=(<=j@kT2KO*G8^ivT5L@1y8a44-3Is(Mra!17;SZtUi8OID^%vqqz(t4 zNp~*c4&0?Cj@HfzicHg2Z01k-_~V2|=$JS8A`OqPzXh^m6HDYdj+@dOkChYQ_U4fP z7nN%^JVdhS$IPy#Wb)6`cwkaLRs48+}~h5Aob^8d#*uR z$8!@Fwf(zp!Z#w1M5UT(VB=6C!9@`n@Cv{SibQx~y?YhG%AWH)in|}DiWSGeX2@QD z7gU+)S(8xGwzUXF>sK5tvH>D`v4Ko#NOa?3;AmuWCM1s+>!$?Eq z(HzLSvceUJFFZW4OC#ubCWQFTcqEITOu>~~E(a#ju&G}9DQ<#Cc^W4jVpEwNovGGp z^ojM~dOdA>=Q%=2X~<3Dhm_FkfEnd;bZ~xSpMP$V2o9qlan&Q?G(lz`GE6#CCmu+% z^X=49nW@lw@S#q}+>defG9l|Wpk}_;3fnmCd*cBs>Eo(H2ntsk$GUW)OJrt}yrpKe z9cawvcB4k&thUtqOb7x7)6_AyHjuR&cp!OH#@#Z_OjTA?SZ|YQIqNt45@Ve3Cb6HK z^&{xIRjn!so33HQ*!?#_4b0=G)l76cZtmmUq$;?3%Ya5C^f-3qYt7hK+aSvLT57+J z<<42oWW^gu8-$-r&?VNT;sCQbq<>J7#i740n8|&FpI%86eOJiL;B@L~8cq~_j29KR zdCU(den;>)C+s~`p;)bPaP(y96cSt79=SI^-pilgo`J&Nno``rTl zm!1Ukl&=OvLJlPPWR6-c8$+&RSA~GaiZw&aju0WpdH04%26y1pm ztv801fiKPgd#tt40d!6#CtdMYpkfiA?bNr|dvq8Y(D;wyIAHQrdKb5?JbnY9Xt{DH( z8~k~1W7bZ=PT&oxq7`_gF;bnNph&yF0S*aO24LP4PEm*QczS5pv&7teIkWyeq9sIX zp$-^vPiZP2iJ2yild&mS*p?0vHzU2LmM?wa(!u@xT z6zL}XL(=s9-b%0*ebIxv1Ki1{K-IJrsn+1N+gARxq3M=!_@S6jF~B`S^z6yhA1-NxPlkETro^FWtage z=|}d2$Y5heL?&4TW~v?_Vguz@i)S{$aEv(U;XlCwKX85 zeE+s{sRr0ju7wssjBPCNgMCqb$2)w9?_j-`>5dUOX1ufv>=|KtalckGmC-FS1Cbr< z0JphZS|qO7kuX9_;#1qKY-Y0hM$WOyYkSwNx$%%l;1p^XmDTI)i(n z&p}(P;+T1a_g=d3Rf8B>^Ue{#b3-nQOr!6o24p~qCFA7h4}i3L&yfFBjJrvs*_7>y zZ=9jGQyccK#f*L@!LS%e)*6+sHicj{yac4G+x6b$#Q&U^wsnh@bmXfqE`Yp);Zmt_{t3bV`8nt{My|z(mD>P;_CVp&ZpFy{)@PYj zFF-!}z|v$X%WC^+M?xbxRY#K1xc5Nk$85DAtEwc_nlJ;bImoKiqfA z*(dkbpYJ2K@&;%w3<9)l`gHF;so*g)Cr(Dx6h9S|yO_=v^AGUodf>+`FbWB8AMsY)U zH2832q6MSG%nTd`0OF(Q{f29#rduyZx5BM*xc=mO=THlbCL!c@_d=y-Z8d1|We||x zU-JX8VMXP|KLu{&61Tr0M7hn~SL{I)wdei$ALhBzkN(>W21(U^}&_L5`tybF_3`Wukt_Uhr& zry7M$7E@2%Mol8>ev5_s`M>5Xe>R#wALv*xan1mPp-bLGKQTK{l%xH&6qM;o46&ES z-LJV(lmUU>a{M#otD`W9B+s7(q=+K1fvjMWm|(d9D-rAZ5G^kCTxsH3PNU9xJm=Ru zQ)dvGl!hQ3iit{GAJg8*;r=ptV4m$W{<%sLxM2+aFh@i^$)+`MU9_>N(Rn%?TCd%hNNa-qK@JuE-T25Z&N-8DF?II{%^2hUT>fZQ|J&L+BKN``!! zV$-7{6WSCdI_ev=iRaxKko()vJ8#1A@AXU30;RG7OlvNgtO%1asKc&73I;Nf6@K0Y z^L2@O&?r$+c(S(im*Vo zHu*{Q!6qW(GS$zoIAE$Zx<>VR^OEtH&%dz&ZVZvS6w|Lt8M0wa#aJtP! z7G1kFVS4_AyXn$RZfp_qTyiMG2l;BW`$pfCX<5ex%e6?zp$ zR^lj7R)lo>(PMQz&wOB((q2KkD=K`Po{Y0vTmG7#=&*?F@h4XejHjvxfjH?_JXeeM zOq}H0TCeoAS7Jp@))*CR4)Uc^`(KBe$lk>f_Ba=GT1o|rU8Vq|?Cctm+K!d97cnGlk}PV0dK|;tD$wTw?bZ*izwvEZ~G}{F0X? z1$fFOFrmDq{pMBT3H-Xqi$Ubjjy0qS^-=y0l<0PiP8KE&h9JwH2p&0LE^ep8=&c2oeNSEP=5~iW- z#>nx6h2lVhU$72G1T22UZ_eGi@{^{=(P&Qob8{}<{EG{R4I)1!>x#tfzZerTG1uj| zm*=#?CoM_?^~yhW|N4?X5MHtIIUjaTw&V=-PcO^-Erkjc{EG>P-|PV%!j?XxH2Fn& z3V%i9WRFWSZZg7qLEo$TOOYgu21OfN0gdd>Yb~N7XnOT~paBV1o?+$jWZd`l-gd(7-jK!3v(RZ=yg1a%)#z91-yhSW~zQSFI{I&v*19eYmG z0TrOk%=%ylIyPQSVofRJ?4-@T_Yn3ff)gH>Ra(GwHq{z!HvfWa^A{(7GIRl%t`j+K6C}Jr8YuWLq&RlI2U9pOzb8gfNC#yIf~Ev=lL9X9wQbL z^>4GG`Z!coR0|#FKY8fX3$hU`_jdN5$C|iz!ht@jEu?l{v9;U@_h9RkU*zeI_`TTv zfRf)I8lG4DredD0WtL{cms=zt0+RftJN_*jEHlumPY z!#P9xc_D~CDS|xr%aF^_Gym=`&@D`Yv?BeHAWO}UAIiA5~@_a8B1Sf)jjr#TGT=l~ws4)qOy48Zh`=&&sM zDMCXPq{jXry&y+-XA~t@qO-jSBcTW&L@f+Nn~qyjKr5}bcZf|g+o`S zZN-LU_*cKYJEn-&L_l%v%lmCm*gXpg&_9ASPi*~)cPJ`FC6gNI;t z(MKF5AvLi@5#R)+EDD@M1}Us}VoyZJOA_m5fU~%^tAkn&pCwmzg4$5I?8lbcw%2F( z^F|RFarC;m!}pq3mgv9;~HDyU5&s2Rph$ttb8 zHt}^y0~0@bbDXR9$Y7=b=&|8U@gYQw)x*^slCU)+h@4ULs%1Jp-{|aRtw52WPVw?` z^<})y1z^~g$`~8(V7}Smh!egeD(X=(5vCtSVTkDUOgKqDOijthEP%Q9t5G=WnuCoZ z=YPb^j zj(&)#fFoyQEI|k8b)!W0nMkI6@|ft`l-KT5j37<>oR%5$j;-Nz5ezY#fft*&Q8a>- zyD^ke!xlGWy1PGf7QtkowQD~Roy4=dVp-pMF@D|X4p*}qDQnVun{cEN^pFZ4zZ1rJ zR}7P>nTO04=^gN~yyqEIMl%-+m`X13$43rl>;@qAE%Jla{*4U!`GWz~(Zx|+w69Z} zSU;Ja1RkUJd|+{UOwyFKJce9r1Dx~o2s9lT=^aMT1;}(KzU&Wz%R8s?I8|Xxj1a4{ zJK}~sfSDj`Tde-Q)=%yl4MPp&D@GX(M!w$qipV))+8j-1E?WxXM2_E}(8I&yRrj?EP{(aq~OUZWmU zdP}XJnq=B@cX=yz`YPU*35+>SpYx!HNDEiBJoon~_awzCBjU#jn2ebZFS@j^8s-~j z(4$L8W8^TTF{zrdH(>l|Lwf%22c z5jtpUGAl({Nl5R0UCH&NhbhI(LTIJpm`0YwkDk5q!%emv*bAu8vw$GRQPfNEWGl4I zcN0berWh2sqNYY0t}{mwvDz#R@VpzGEcsN0$5bd5O*37vZ!5$ctIjFY1&sQr4Pul& zeDh^{Dm7_=SIa88ofMr37WFB?xQ3DMfY&vYVp|eU6q^IU@KyUU=F2u9>$5t|?qtFf zOlH36)_}Co2M{%}lv|u3NbaCKA}=IlL8i;fIDd6EVc~1-%j^qeg-XFwpe*X|9bLE` za|x?{CG|!Boy-3$(7#LdOR_fdszZEd-J1;DB&>Vy;2VSN)xwQdh_)3@b2}ZjeOq+w zO7A+1unV=tGb%#LPS663dGjiP9`^>gU2Ith-X_xzIrRZYH{c{i87!$P-H^K`CUT9y zKk37Gcq&2>v05@kT5p1lT@QUMXf>hfJbJxK7dW9C4TtS*{&Q(WWiz4XC1l!)+LD+| z=%zI?v?r&3Py-Bzmb^FDtC@^uxCr8swG-T5-WaqB;4@V^oQ7}?i{q5uvSO-;U{|Z< z(ynkY%J;4=(5F@rWEFBN7x*uHpeB=4_+}n>sL^K!T8L}NluV9}MKe4n3iYncfV+(| z0Rll(yB?=XejWPkczbJ>M}b{YH7F_4;^qE;>F3SvCHt;v&ch~g#w2ybdesa^p=n%- zx`w^UV+VYg`lZ;V5a+3h;_~A7D6=EKx`>!jaw@71D=@FN?&6Fe1JyIR5PBY`GaiFN z^SZs-Eb0HLKXk6bk5CP{vZ`=e2?+D3aqC^4{*CGO&6SVIYGLWRI6_}cK z05eK6M5ZbT5&{qH5F3W-8X_@nh#jc^)FAooK!Y)S{)&3%XBk^}3Y&w-@evg|UdnvA2uKA9m?&tOC@Ufp@+O-vIj?LvFe)O%Y^WV^O~KrzYq zFoQj<*Kb&?mAg>R{Wdvtjxqw9f&Cx- z6!B1-=Y%WLG?zGk-1@zY|FN{6Y_0-0DBuZ_yO6!L%goU1Mi!;tgk8aTLuE(eD4is*u#zZErQR?wHgS}O%s?(U z2xa52En2+G6O6~BT=pn1CH@1Q7b>8*_9ausj3yqhCbZ z=+f{UO;DL=sBuOsegD8y5r~`A>PKW{*usI17++tSrKB8^Q}sq>7-3$Z49P9Y{ca9v zp+&p7W0lppPZU0b{-?kBeS%|xE?Up}s7sunCZ#DmzO|%$N{n1%VrxEhhbhISENbY- zfJX83-7xdfN2=m6!B|u22`z2p|AJ>QvG7i|v71v~k`)s@ar%Li-aYv&iRl%$G$+hV z_>P?O77b?<Xb;-y#_+f)@0_OA}BfFG|cxL%xpN1~|X?swhq{W_>FBVrW8A9_f{(D{!EDun_cBP$Q*&V*(#GVFL zFw;VA0*a9NAWp5Wqn1v7iAL6vbbOO5St+|E030B`-o+)ZC}*q&+Xy<6`5o4Rr2;0| ziJZz@(e>cZ;2T&vgYFF+ozy~<-~AaesYJoW1UWHLM0T@#2!b>d|GF$#Eg$w~G{zu< zgo(kfn&3t#%rc8bNl!4xdvdS@yxtZyoDCTdR;W~U(KmID<6hu1WExt?^NCb(qefrnBHH%gYE7xJop2DG^hU10Vv(L5 z8Avd$HEE<(&8M@iPZ()kFC#MtJ$N?jfI_efuCll z$2xwz&~HzPGSlbrc*dLAVhhOB@Ea$WOeEj)BRbhfNXM|?-A?l_Z)txGfgO3E~ik_?Oqhooc;>SS^HcnFO;E*uIl3$j98O!LGW;ZLONhsCKu;bY_GjQSTf?@?LX{8#beNz<;*J;DIT!8&_;Amvd@pIfh zLYiqUzfzefejXJqa6EbFmXAzhceEwy=r1bUghsP{L_7}nREMtn#YIt|*MD#*29sEx z!b-j6>Z%+xz!3i8|H5}VSxq-MPpf2fgmk7C-VxB6IFYCeYvpReekv~V*1Qjf=L7tb z-)+8}ynPF$8_srny8Vq8ax8e^gRhOFvG~|0&Wx0G5)xUftfBAmSAzCvxg}<#8_u?* zjg+Fe2bw&UtSGZsfP|(6s26Zlql{7vow_Yx37bmXXGO#1E-vHKO6zq^p;xO7oa2$$ z4YSl_TY^{iV9};%woKG+1!iF$MPK&TIHM>Lzt&JggfZGaK4uP)vQ0TCa0O3&ToY?u zR*(^xRDxgMvYcnFtmd>@p>=b_<`)u~!pW(OX)5Gk#u72K|Nh48Bdb=Yw=cGg>+96T zL{*GbWYB74uJU@Zt&s|Jh5$}XzV9qg{fT3Ax z5V4o^fi5+Q1@_-d6$)BrwvmVD{;*1}Fljp*NHseN*wQ`T2P}YYbrRj{IZt9G>Aa`OqQ?#3+v{7_(0-b*YJU^wY zT1HGO=ct~2%ES;}9{N03`UsS=R+W@OftrFu=tp2-Uq%V>EtlqelYQDy+HMRe>Gy$?}m{(N4T`|x(#MQ21vNq=?4tGlgG3xy|1zE*z9oKurWe2B6cux zVObkQz^NbQw01WKhOJa+l4nzrcAA+xJV7kyUC<{t(B;<*4-OtQOuQgRTO-2HxY7N< zyg1|6l-C=6_7ASH3t_;3(QC4(Y(Lwe8k=`^ys~^4c_`q_svCPHG~t8()yR~@{g2v# zTTE|~n{MdprOXUz!Jcl~cT|U6MatbkW*$_Youargkj7~%VEZlXNKW?lwq&b+)OB5~ zGHBy7%ICHr_lHc7e>Er0Va^g1woQ;Owdn1$@oXcAtX@ek5@iTO-=zK0IbOzVa;|qU zO=wSSYzCrJM?zmZUt(ZSO8;0i72e1Y*MVLBWA#vDdmrSI)|H&-5zgG$JszDZEZgT( zCmcb5uR+}0v%2i%G&R;5AjkyH(jvYZB$dbc^zQoVYheN0COsEST?=cKadM`MD&=2% z2x-FS+4Fxrz_xhQfuNE6P&gk?t3>mlJg+BG6=hv9%8&4b8OPo{*iPR(Bzu~cL)b;$ z2h;J_``Mp?g=I}UKOfU}{MpCE(O!g{Dl=COmg7&mU}dMlY*h9EM%_IGGMD}1cVC60 zZffn!4;iZ5Aka_pi@cfL5dMKh>ZvqZjkLZ));c^yLRAVch9MwB>7~2Y}^xcx1{Rv6d`cyYVG83LhC$8}7pV7%sbYsb_3+MDdy?1H zU{~9~8|0nRAyG|fl+3St(~v=Ga6X5O}XwbR)!eLBM^!2_C_GH=M z#mF+M3HKROT3CW$_^J=J#hqrX5=OPpYidccLP1YcT{+-wJq>Sbt8H}IzrC%Ufv!{d z?dHZc!L{C2Oj}OW!&WFtd71SfIpP(R7h&d%|1^!;1G0LmmG;LGjQ8Fq{NlRU)3A1) zai&eAtrzpNb=eOaTt@(^up#Q;kg1tXv!q+V__NFEzN*-g4^j4NaKdW+WDc3A%D`RvsnM* zcII=n=GebDAlZv3uY(`&Zj5xnao0WfUlB-%{KS9SO1LZg21;L*r>*TZQ`gJB8U6f5 z?Z`I@*DAI7=-BOWG`1aJ{TF0Pc6PgY==_yxWe%hT3Nw%cvT59Q?B~~{BEM4CP_b&e zc~ppMRg2h3t9{NSol~8chPrQ{lmYTb4Jd>w;!GDa6* zC1X?Ifa}ZDnGo|LqRwds8hI9_wmOcmg_dx5c=+`m^$z6ZsmdiJo@@iU4i$eklcLGs zDX`RYFn!FGcOsvMs_2e^nR#j@`Q^U=N7xO&qLB7k6_OpAPf0Mpb*gCkh$Ovzlb8hM zEhhl{;UjW`IH;%ojo&RSNt)eT=-s7-C)I_0TFExm^PXK=ILvK`$60BBFg~8%qN1n zR3rdtAwzy#he4zzB;6HBW@=x?hkNl3?!`Os+x^SEkZnd$3YtNFfob72g3n1sah&D; z*>|QQIrh53y!$#1L?C4}gsLS2@VTl6axyr8-A4m9&9gaeH}S<@e`9y2T#~GgzQUvn znAf^`*RYJ1pd-;3nEtA_yb0+H_;-60*tKEjy8&yctVHk8G;{4mrwp(%75hc3a1mf9 z>EJi2K4vy1z)f1eOtz_($dsPtJ_{q&c6zj#!ll@vCi++s+_LBv)d7=&~V8CWWG)C4W*Ec z9Ukw%%=&fMlGkBfMu4=aw+WV%9yCd|tp=@80@YHD54t(xb~Bv5<&8ztmuE%|AFcYk zguuEWZ-dJgt}Zq4~fsxVrksRy&O5>?a>eG zrx|Whq_s3!i>vQ3Nlz;@H{}zqNDfsqBvqX6k;}Td&8tqTqPW7S$!QxXq}1v)Yn?RNsYtB8b7N+y!qVZQ4@`fDd!w{s>bhVCJLmimRVHf6nvQ0=8kwhoCm1*cj8XUtoSr0vn z6NwVRSbnYEdu_jQGo!+9>IZT{XcMI78JmOHWugsR!P`4BoD0#w zhI~zkUC-}*=&OAMir-O`CZ6Awh}&VjWd=L^x<)B8h2}^suX|Gx7*)ezXBl2pI_~d| zzZ$z*W!=G?_5@EmnPFWVlPU;Y`c<2C%a&A+j_!_+$;1jrvF&{rCOVof2%6V$tAk{x z?+5U9O~FEC$8+8N5PR8&h;~VdY~e~>mc!_N$~x0Xg6kx5%qE-K%0A4SX0PE_yK$pG zBN->@mVskM@Jogvl-`VnSf6=XdiG2FEDfkcQ&36$5#eWxQ@cIC3p5I6E^9nLf&=Fjj@y)K|R8%kW^%htB zMr<>x{c7qlFGQfPYqGwQ7$V`Sl-9LErqAzpq2lp+()vSI*3EZ>DRPcnklGlVuN zQVxVaEbnJ#pZ76k*9kb>wFpDmCow{tC9{<#EA#!pk zGd&|>VZ{#+S1FZ)JSGhpfqwD_$LK=1)3F^+mb`U%OzCeno4n_Db#;m6#9rW)g3tun5KZH*og(uST*#^(C~+T0l-ju{ zV;s6SniYv7Q_Nc-qP0;x(`g|3<=K{FD^O#l;tyyxue{i!f60)qRpN;d+m?%-#bL`s zY?P*Jxz^6U-!ReFa(wg38lhtcp92q=J740vu|;t5wlE8%j8^*ugCE%CZJ{WKcNw#9-l4T=c#c>W;7GN@XcjC@_p=@U@h z&#lz!D*Un4xbo#kjp;^@0&T!OFzs7tvVD{|;N^0CSJ1jTo@eT#Wo5l!?sm5%Cm$qB z&4@O?uH+flquN2CK`Op^xCc- zk%x&G-Gz!_wu5J<9jM)F9_;d*v0}&u8 z@shl`FYdz$f4g14EPe!nUv7ltnjU+QQTC&OeV+fjo(H*Ve_kwU$QKf>Rq^Jho2@Q~ z9lgS@k-$ogZa?oA9k#){Wj!WidXoar><(C zWlme3eJzIJmMlEv8ghxD(-$zwFASpSi{VnP*!cd#``?(PBAv_*JP+!}NYjhQUNU7< z#?cerKOq0UdWc9~`{^`64CMpiI z2^!Y#`83Nwd=ZJ6DX%fs!)P-|RIV~JAnv%B7s3MvRQWX*bHv^Dw-KeD3#W{dS(omz z{6_qK_1tB$=8NP+cDs)tom+Q9{+sM7lAUD_-0-06bCAkCB_`df+uw$MpuP75N4XIw zU&7ZZD@Dl4Mvm2i$5!H@#E-3q(RuKe+~C&gW$vv

X0)?KrxQD&9}|$$$JewE6MfK}wJ>*a$ahXpuAAG|>jr5HMNoy`S zG!mn@D|P?ixiSF(ft2-&>`3Ll2dhX%e}N%1CBDn|)O#!)=Tef2pK<6lvBNC-9=|jd z*@+xc{NtJTw!ikhZ6o9hj8)r*r6?s&Yy%l0rsYDcir1INU>HWSu)Xexrc!~usuN}i z@*Q%p?f;*REam}vfm$wO7S9wh@r&Hp@H#~1_vzCQaELULbv8MjB5CK?Q(nKnQE$M& zN1&0saGqAy5w`j!5aR?XCd5C1mFxeF-FWYvCWUMn;wUnm5dt{J)^*Xj>|xJhqC#iAm{$G{3!4J5(G5CLJptel+vtgxrmT{Uq=un zo6oA-X};S!S%YjdJoaD|JYQ{WRF9ywW*Q_fr6X1p+hvE@vA{BhLifttqz_KlbgNlM z$@lkZ$ql=s?;#(gGyd%CD0Y0KQLKXY_+Lu?x&8TGo=Ev3eNq=za#91Cpw|L`rEP zSKT-(u^XiH;ELIl;uTgr41*<(BL}c*4sMbefRYEvE1q5m|Gt=$V-@K=q!1&*A<{X9#wL9(sYvcmCLyd{3e}4#A06_09vHLaK<{7T zIs!oD^y3;Yst)4`+SKdq%%xZ{U zonmx2TU}Qhk~d5)pp5wS)$U_F9rYr&iB{FsHC*B*L)VagX}NDj_^dXFq8IX3$z_j# zdQhj6vP{^;Fw598xsBmgg~qWXGp%|1yUzTsKnU%0{ag&jo3e`Tkm&|I#N4RP-E3jf>M+TwIAOZa9u*X%ag5%bNL@TP6s0r`4ju6YKK3Vi8H!-W` zjJk3@mZ44*{f4fwp=u?e@;Symmx$pvlff3pns;7eSRr5zyqok?(`;rII`fa2QY)Re zOZ#Wi+-;^zdI#!fe`2+s|BAkoY zY7Ds(u@+%@UZnCx^7Ni!)Qh&MQ6wt@u6-3Z&>wgRY0Ez-K&`hezRcR5PpBlZ7>?f~ zl9w^|rA&%r^*K(exk7sHV|;F@c6`f{@G**UfsWdU8GrS}+!*8bgQqozR4`1LXfA!Eq+Ui-&=@RJ|X%8VrX-^*odreX_{D?>|9+i zeUskPE2MWeXS!+y>?Olk>-uN|l3Vm?w9C6Z_?>SGAvZqjmAeP+7VOS6Aq4V370bPq zwOQu$F_iU{lU!ohb8mhmRC}N(gv48xjdeS>NL3o@aY*oolBwsP)1jv2CCgsFxVQG2 zHS_L{le!7bN{oerxAX%dUx&*p^zpRV;Quz3Dp7yx>9`i&n0B_&AvZ1y`|fkF2zN?O z=O6ifW`$xCu@G8~cbjg3!o?8uMnFg7CxpWUYV6jkmRQpg)90(9*{^V80T#10B|MeK^%9*hN9&rF(;C}=0Hl{i6&Q~4$845YrBD8*7c2u z8S(y+8nrtL*GZqYL{ox$>JYBxvX+slC%SnvYL#t=*w zIJ-_${FaJ1L&pMqM@k~-kxtE&)xCS)`3E4NQhhj?5TecsJYif&VkbgNAJR4)z$`2k zKMhoxwrDI-&fHVja8>|?)QNwx1pN94XpO2taS@#(e2`2cgT*)Tw!PO$?b-!Jh~V9x z6Vr|_l3=J4IpgV{`)v^HMemUmG7Vs3!nfN|78NRs8hKQS<>CW{#zSV_WbFa+;qQTR z*Z}w?zqd=vdlV2Y)g&A>y{ZPNVg)1#{q>+beo62!zJKD|(VL9H@PUlOY+HXkB~*b2 zsa?FoSq`puz6t&|nwywBwhMFJKxTESPo06GV7N-9L}umZM%cT8WW6!V_onJj{r0DS zq|^U!HH3(8S$vd;hS|dpN~k>%qQNbPy)y~%5+?%h{;bU^wtvADnP)F;Nte|wrAbh# zVKXgX_MV0fJ`Eh1Am<}8Y!Yn2AGHA`p=)11y|gt}1GkI3LVfrndoG*Q{+A@Di6EsW zNGcCa;&TnQK5ovm2+ohzpAG6?(_9=uBm;mcNM?wjG=lpBBUd|y z*Yj5`IOAVY=x2o#s$WEXF5#VDZSQXE3lOu3uTbxI@MlkQu^G_dJM+79amXzU1CZv8 zy{6u*M~)Gs-+lvA9v*>E;p!l&4NdwdH>I z)(dXdV-!}hACF$Kh~9*32`c|_+XZ==z6~dS+})_jN_9DZ3Hv2TSLu5bP#7R#-g+NG zC*#kWh!l9=rIYE%Im30Y3mM6V(i+INp^zq8m;GJakduYT1@5>Ep?d^0CiF)s^hu7% zH>@J6HwYaq8o`XgqF%c`&oi5@1-DM#!wOYA{jCtIdFW?E0pTKDuQj+*; zvtfnmbjDJa`+GZRnY_=JBSLXbUk*izlD&u0UrhmPqm~g>6Ca>LCgJ;7owru#67*k{ z>MuI>JG}BG$$7ozpg&90mR}$DV~dP`5l#QGF+|xjW==4ETMTLN?`dQGeRsreV&QI* zd_aw)epk!s-VPC)2~?0aSEl6bef9S}0g!RtJn#>75m=SDG5L5y@2GjeJv0EuZ>||4 zbiNkbP#Nn(svTd~7zT6^h|K^AWZ(SoB}Hd~EGlS%@7F}x#(5E2C_5t%#aXGKv7kiP zSIzxF4@^qx3e4L?Nvs#wB4#DI*~;4;5;I5^HLlI-FN`dW)gCDVCEhC*C5TCzkAO;U zUUXi)n`mZy#Ui|PU-{@#eviZh=3=u~ct76$6$+0)#<3!Q)g7;=`UX@Y;kb8w;+X!s zlk?|Q)|!tI{NMCGsW1{MdJAQfg{lwG5gtvS8(P)Pj(0=}fVe^1EOc^<3?MoWibU=lZ3)b+ z*l|W02mJ8UZ{{wxtILoWj8s-|0}#TWHQvf}p72d#TwHZQ+~EH0fhsdzg!TwwJuNYc zM_3?w-)q|0DhGGK8Yta)*D)xFVn}++JrO8%mI*pn*pLkMCgd~y@x3Qj7$+?Q*D8}T zNET!Y@)eA`RSl3nL1o=AoxvUFF&EE}j?(xHl1dv6d+!0YqDE`Njd#AkI}t+F;9uZ< zahFkaJ4QTE1!dREtfm3Gr%*Z~;2u$H`5bPXiOaRJYJ$9bXn_p5kB#WlQ63M87;qh`ZRSS+8}07w&p2MDF0V$8 zY2IV-{CJkdN)}M1WIF3bnu{SgbR5-&u(6==FGWPN$_D%zRjunc7Df<8?Ll@Wjq+Y9 zC$T;khh}J2d9tzaup+cYu75kaP`y>WOqOx}Zp3b0!U*~19hH#$)f(fJ%PnSQhUO&! zRT>T3GTd`ASbx@Z|1oPq#@Gn6=_D|iT1LsOb%HV1gUCcl%jcM7A6NLI7EYYhpi z$_jGqVTi73BoK$WST1GG)ANkTOE3}{_1p-b^^(bk9abjk22@iY5GW<2(c5*L=y=79 zPv~*zXJ#U`x_hqRvisp*3I&?1aDb=sIyIzoirEn7fVt}wKwV3GYd^u>(+PG#4^j7; zGV*W{&5z$aaj!6n*}$}H65Mc)8FCzCj)CjVOhs;eLO}e(LGdmGPSnC4Pj`H}M z>$**6wWcO4L=G8<=fTawdtG&YrEX8cj$}y%inRWyx62WYTk3Ex@GkQ85M$v`LNZ6Z z3^+qI6Q!fb`V?<7*hn*>vs`x@Id0cJ+=>l0eMhOt%9lTZL>L%pxS@y|uGEfuZ_mpr zE}M#tWvL4FJNswq`HQDi zxRKntHYBT=F3v-|SYP^L|E-dxa|*4aEU7W{oab^wy53`3=`q#YoLL=?*!TDiMt9#A z282wepc4Z3`W{iB0_hZ*pcB)0PS$BJ(0-(&kgezTkKL3;kzlgel@`M5g`vJ2n=@&K zrWvtRY?`8-cb0k3QlzmMG$_sAQhIDJ!S!t1+m@h_$&uRi3mFpq4w-Un`wE|<$82(U zil8%q59c2T^UrOki~)Hfi-DB{EhtD1<+8@zUD<>Baq)z=_ff4Wbyl)m6p0R-{Vs5~Bp~ZhkVJn7nlUfL$}9medA1_F3_(Uu!uwl?R6y=y& z2;-O_CSPl+&BD19_|=980}*so+Ntv|SX~IJ)azGwa_Y(%p2m+6;W64&a=cqhi=Q8X zYv@s6MG{p~ZW%6JLO$E{jy``!w}gR6N#V3!J9xyi-iPmHJA;nbf<2NeMj^0g)ypH! zOh4}ktM^QIEqH@S(?=D{X=yWEubyNo)q{eF??7NTjd6t;FMS`|pmgv);AraJ{BKqe znb`oESo=VyP4KODG{@EYMCk=KCTte~pn<%$xmz~ezNGj%Nk-SOjPKUeZg27=sJ6st zfn%}iTU~Y{aO3^$?03(JZ3fM6g^d01&=8W0iq0L7R|*3^un)yMxvfk$R0`3bns;!& zC7>U}AWXl)DRuZZ5!R*ECWNg}%rY|r0LUg~-X9f~Ka~V|?w@g}@Jy8|{ajQQx94q>kR>hsmG-TR|-G;%b1CmJhL-!B26#LxW`^k*d^ftiOPuz{nHwz)|VN@PWD z^6ZL0FZOjc#GNsI(4o$3%&&7CQ9l#5blslk3Rz9VUyd1pR&Ih95`Q>2BqaeqT))2Z z((92=l+*@wc@8+jXReHTC_|cy1G0Iy^l%A0;!P{L9D0Hz}w|T(U=t8!85)>lRLNik{ zz~zf+#OyqG7aUZF=E6;82ae7#bs!u*KKVj{2J8LZFP5$){&0GlR%=#q)f9v8ZAff= zoY_o2MG=zd+R)ivE>BPPMb3>(!vz?Ks~*F;^1f^FcjwwF4{C27jklrB(Cc>?Gvu21 z4via+*+WLaC0A>eaC0cpH{lahQYks2UMJ9_9~~m!1ZLWpAN84p34y4PV<-s^L2gR0 zoFVtel_AQc{^3slY(j?8&5oH#Sqyb+%d~0Q-(yDE;wRoA?Su{xZ8Mk4e-qa+PuYV; z9%J^%D9UCtXa>A>m&v7?4rcvxV z6tn!BJ}XLRBcK;r@zl9XI$Lyxmo=0PQW7qd!6*&5Zbgo40E=l^-G;e~YfcY(^||x< z;XcjTx&bw#{V#{zmsWQxAydQI8(A2&hlw0MbdPbSEfy9LZD%71e$G^$3@?PoTzVBG zXzo8uK3AbiKdFhOR-?XJdpBND5;1@6Zd$=a)iK%30`;~7`(IyUBDJ9zhS2~+@meR6 zB{xQA<>ZGw2xw&Nb}gyFt(2a|p5Tvbn{j#Ul**QY7OFDpNzo3ehMAMfXv-c-R=e;G3U?g~-kb^!gDuw|84~S3t-6;Ep+P z#hTT6O!C-Xi+P6}9;pY+gOdjOdc`Pz#w6?8@ULi!ntbNP9xm@oy zg*Fy9z?Z*l3$Tz$gioD3B_>+iROi8?pOu9E zENyA984_(yq}E{yLFnROq5S@vLRVaAo5f-NS)yhkdByk_VPInJ7$@HT5mw?$JMN#i z8$FE3iyjT>-jVB3we;skimHXMoB0=8!-VTFh})4Q1MXw_FVsI`D9wwwy`euWOKbcG z5#&5od=Ts07URXafF!Q|j#WVZ`7rT4@Nl!PZNF^-?6F~ceCQrOlx|3QZ-%2=w;=|8 zle4cWpml_dlvn1}fXP%q!?e|i^LmNacuEg2<_4kf&x>2~^D^Hz!;d%_XzfByXtq7i zcT0JD=O}N8Od^=BYI~zYf8-@{!#MY{b1O(3`FSzp3?D!$ip{1u&M)c+qgE9aem^h7 z4CThWz+~XAWlcG^wwZ){YWDO(LIz!UBu$S>Pj zaFX>t>ayx|(M>pj0D|f}iJz> zkZ_G00(i%qz<`Zi?RyJI^)yRH@o-oL&DCiN-umILR@Qa3OG8~lBs8b*Jt;3=nyE|k zdOnl>?56kKH%X?J8I>fE`XW_dD`Glmf~g(|i8Mi@tD!EWBy4Lt`3`H`wILUH`=OH*J*mpyj@(wXW`fxRXg3iT|2dDyf>u5up!BMj8}A90R|Gh|v1$THo+)AEm<41g0bWc}Vt{O+nSIFC) zAC4^kVd*!;KJ0#(-MhXYjcn!3+3$Nr?~R_Xxw-uwA5bk7y@y=G5TOfaZDVg!Sh&{h z0YNM`B&J!{$#G`}HV;;^!t9hgmKs?pTXLD`83sO7f^MFypSNbmQUIAnE?kror8h+* zZ=rqZVpExGx~f!=9Oa!V4S~>%*bwgq)kj`Lr##+HEZq7L0QRpXA-TJJHs#n7{x=Wg z>>uE5ZtFBR7;-ZHFDC;6>r7@pjHL)T_X1r*MHoZQ3;pOTI%Px;=FB{qPN(%cEPoBk z$^DTschX?YV5pT2^C(a;sBjH(8CkJR0FEova8v#|zi~xAg(TuTJl4waS z)KY5mwpiLhh^}zChv0(iqBwf4Zax2zG=ClNvtBL50C{nJXKvo#ug)hz_}ECEu#3yD(GF&45soFn7A66=e)T{ZLT4ZuI}EK z$A4NKT3-ZQ-lQJPV1DX%Lli5931Z-54sUONHOzthpTTl;+m$~an^B8}uo~O0Yg^&Q z{%`L}b)gj;?V{mA0hLpzx6*D-hcdaMim;Wq680F zQ66A~|9Udn2FeOuF30qL&!PYIeNbGTf|B&i&f(bY|6Ps0J@iDV01t-0#w))4+m!wF zeNac>Ap!LJ)QLkw|2h0rH5h)uh;aYQ0}kL{E`MhU)OdJY4}BB(ZJ+#YWU})x{M227 zc9-9G!LO0UHbbSeYx9WE`hO1J02%(gZNS=pu8}peM$MFmO8;~CcgtY-22~#U|Gh?* zutwHGdVr< zr<39q@W&a%b;~&*FN8GzQ3EKx^O6|+*=19*s6UJ^Y#iLpZ7<}sT{_^jy~UXCFR&yB zaR>4txr>HiuET;g!!KD3|z5X02&f*mO zfFpRqpAlW&xrTndxX8v@K9}V2^E&?9cXav+B}qp$0iOAFUK!v|IU!}xTq4**FhkQg z`eS&C(f@+evtc-#70EvLzYacFR0ElZz^lRLAoueEbW;IvZZMCC#?N0IyqGgGVDxu~ zEFBIkxBk2cIs?IKhL=WJ9XY~f7;eP zOt>0G_>(^UZRY-V!Teb*mO`35QU-48{nuN%Z9z-XJ=gWf^Z$8){5z7sO&$wmrRSut z>9@zr%ZUEkfo?#dKH4SOs`~r9q>4O*f>LH2AlJIu9;*Dx`#i+9TPFPNO%7pDy8$~| zNddSx-i5ipY@zIDm@fLwg;vhrrt7hd8?d~W8QK7)O(?dvAK>+03+RS>5~hWk z%U$Hp&l>(&UF&-|1g9xa84e{<%*u1zUC)s~q00)0JRTMmCWzu<&d;e>Q$%L@S zi7YNYKKfsK{D}|`JT^U;?Dg{m{Tj~Sev+j|)^=R1_4sdV`|BIQf3t$e+Dqi{ey!aB zKgHgF6LYWLKbG{r7itFP|61?PXxL5tWl1Z)H%HtdMAQS)hFV)?Yn1=iL)RJ!%N(u{Vffp27OKO`2U?wi z9Kc@^jVRvpr~dep3xMpGrC2IDbeK6B^Y^VpbqVomin<{LYE>luWhLm9;rP&uQ1pBM z=PqqchF1}LcKzp?AG{-CUn9=&_n|oB|K5K7xP(gvflrSd{3$=bOHvq=uHvA-URML@du zHt^rt|Mzy^gvTbVQC7&W)5!w-{U= z{5A@XT*cYlv{wT+x&C}9svn*2nB_MfUOix?kcbT%>|4fdKfu6A4YTw7vCcwKf5HxV zT*VK_Io!lTV27Mm)9o~+@5H(xrN&_q@$Kj3?1}kDn27Bs`rmUlb2>C{D_qj|mvQ{H z65%R{q!}kjRXiGJ1qt4l%AnMM0-%PDZcLQxEsAP7O+^2j1pytB3c8d-JB4l24Ln9| z5MI&c1mYPsK!%Z+C$9L|00k|Ax$pzME2IhY)oG~1+Wr8IB4x)-r~x#AxaZ^ol9bm1 z{KI)}&hpoV(Sn~4B-~}m(1*|_p#|E4 zL!@2UI*)+BxRw7bbY<8>WhYVuSYVsT4AUPW1gJdpj@OL&qv)jeb05g=1m8Mu|6ylj ze>WzKUH|nabW053ynse|ODGkSxaQu+=f7C0;U5Ix`6i_4T-gU@&;p|EbwjQkM!;Cb z4v+&=MeJ)4uM5?~_cjfKw*WfWa%-#PJqDik z$|gV1luGayq*XyDzG!od)rkUQU&g}v(&6;AU!cr=fK2HXw3%kR)igl`KO)Bx8|%}o zvi=KsN#2h9B<`3Nd4Zyt0YLpkeV}&eDNhG)+k9aiPtFQ z&KR}!x|O&HvMXk#tp8cbP&mkh6N(du0-O&8E%x)ddMjByNSDw$^Bin%JFTj0PN+FJ zTZk6w;vj7-Oyt6goU~t65%#usP02`sU#F~T<}GLMlb6bXn@~&a1cCv50U$|0_~AMPu!CWU=)0y%I;0Cwv}TH^b-_s!QAN7b z<%r+rwm7WK=_{(})2yqeHbbFR7kI`D^qhdBY=eGb7eIVgQyqDHP&*?f@3rYOL?{h^ zw4DK9_FFCq{QsTY&W_vdE6f(lrxXztTqQ&anH{Q|dVQ=yPHzI`lA4O+zeCdj8Tn0% zvOsGRCjCW^W;rGq@-rNWs8g}>v&IHba#ro906;K3wa&Hr@y0+{A;rY^tK(bxf<-** zQ1BkA=xHJ2w{3XZh1NOpWfE{*=3%i=idK1t5}#ETugH%%#!DDqCc%SlM#*3k6> zQoyoOKk#-2Gcx5FUTZBbB~yD7#T1H6*S&V%Ueoe<(2x!Y$LcD2S2&vP zLCet$cfDy4<9r&4hJrrcTHkvE?HJY$MBv;HUsbdaQ0zUVyjJS6Ek{W%ZbO^#`8lK! z>B2UmApw1t3}Hca!G-Gmd!l5D(81{4zM2wxJoFS3Ns7S{qlNy4v-!2>#n!-B{6LYS zj@k^BMbD~|s!LqW@#OtTz#pD-ILME}q$7jtmbcQidA~y>#qEPFdi^aKrFm^)Kz4+2N<9l&^2zxdy-?xxzR2^5FnLXwzZJ04u=+ z$h((SEr88^wyw}8%SojjD4g&e4qzDQ=

+7Y=`TcG|;o4%)J-bBPh$MF+_6y_1Yv zGVd<}%xOtTaP2S9so^bAbU&>GiiH0qeZvPt6mnO0eHi+A-}$kSm$Px6)in}g?2Rc@ zFCMB}VJ^;Aqn)fA$Z}G<>^FQ5P;WEC##F6*fJHS?-fvTN0`hF9?DOihlMZ7J%R0M} zvVB$^QbXmgW4n3n3d*UsD69k9B0WP?EpCqs_zZjt87ML}B>)9(O8`%oajzLsbwloU zW5~ZUY4vJh)R^}-y-a~44tpzX55shV&BNZzNE8Qt-R#`Ng;HBf#^g70fXNz`<`(a? z+~3`k?GAm;s$(pK>V1SYM z?G)WoEDvteW7ka0DQyJHeQoQtf`-e-BkH)c`Wx3Y*w1L#o#m^lIJXGSuC+$^C!d?M zAbW@GPk)rVud0~a1uS_Zn~-T>u+!F{W~ft_nc4wjiO6}R<*?)fLe}k_>}PcKA&pTV zRD3ZEcAUj49YFrn!7Lhl4C?#c?%fL z^?d(%4&jHTz1y;>No1OAd8+ScpzhA%#rFw!CW$e@j(9-2Ku+)?n9jRg(6LbH?d!e| zmwZA_SYY*}Jil*+y4jegms89G{FE7w;w_3%-zbUp-pwPssd#IXxONH4ozDQsNTx$V zw#^slKxxH@&z455T)VDTb&=z4noD7wM2V~5hZe!KZX7!ly?v(dKfKW&S&_RbPb!7b zIw=Xk-+ikQnhiqCvcQCQ=}bBr(;CGLvIPKuD&-EpD@@7(l20@ z1sb2(vtTbaV`HS1H3NeGlC%`lnRgiYHlds$#<9%W997EKBW|E=QfPrj^$DWm1TCXy zdt$%`DI2|$S(#K4owTG5#6uC|qQ(7w zhlCYolyi@s2q}xe_1M_F%HCnE)crt7HA;Cyy10lfs(doo>3XcUwGJDQ@|leH2JX#FW4n z!y}W^gQvk;8M$c5>%3)UTf7(Nnd@~AxWE5+hRHyP*TFy^nlE*ZP=MhitYJ z?tgTlBcIFD;47Qw>|p2(&Nl-XU)fi7pZoIM2ucJ!OU7BtFrEFnFvT_e>+Pwc!?Avo z0n2r|mvrVxUYNw7zI=}uDvPNy^uXt=?;WeVFyC2Xbm%?m#F-RrrFQj2Z~(9~<2GMA zwW+Gwgr+@=ne6<0uk+Zpjy{XttA70>XGzJE8}HQ_GZ#khvI>OSrU6jEB$Fi~Y_$Ke zPg~QsN)i=#p%}~CmU|z3qi?U)5=@bd1ldF9B?bw~nFt^v)90XbWUbO?AbkQ_Q$MeM zK3e&u?859sX|!}|$=(iscI_dt%&$t}0}4F_eW9eN{rGJ`i*fOuGvtH>3-+1Zy2 zhemv~4_G3$-b^ox9haw(*ztWUb>WhG)XOV*z*UEZga!2FWmg-)xyUs1ph#ra$}7!nwg=o#+3wc&YpM`% zkrb)z=A*53q@mlvukuMDIzt;2bLZFSz9krsL5B@!bfohpMOD#j?H9Oi1d{TGD6|GU z9cGTGAUT(0Gn=r+so37xVVSbGJ-e}vFoxzdimojJP

+gW^1k<|}8F<(8^u)4ZVz z+w$z&?_Av)^StwV4|>t-XnBJ*znW6@CD(nU?iZ$a#w$wrt<59sXZj0( z>F}7be(Dn?WcBpf^wjnhfsaa@O5mGOw3lX#OjK&%Y+Xcq?^JiQ>(tidk+!`}ISu1( zQ!~@Ag0|LC2xzWuTd{tsx`%E}!1IXu?ZdqY+jL1^lcp=qy84Te6R%Elc08|Igf6ir zdr@!pd_iZrgYLJ_7-Tn=K$Z6CT&I~fr^0*uduRI|DY09Z` zd|WzEL-T)9SJqpH%2p!M`#wMcEdGM4l{Y_Vu_3tTYpvyR(quRW zKD(=zS$fEvOdbu4XLtl%Ec#;*KsUcibOl&aF{;Qvm+Ydm`d-YR_wXCV%)?c|u8>llz z>`r@s?rKrQA4@v5$DOu{Ea7zcvV8rNuyyx9)e1B z6;sE(at^3NB#Quq1Mqz+7O$gVW%tCyD2ib z7e6*`l^vx~KfegXTk)AKJM!J3-qM||Hfz4@>+P!?d~q5@)$!s(ld8*o_d$Weq-i~z z&)%Yy9p4o=y94iWJbZtsXyhoiJCBOEBR~Fj@+^hf`@^r$n?#Edoy4w^J41Sl)8!MX zB+V|_!uj1VqoxeS05g`UA_l*mj3`k3DUx!sw|wUu0<<2I$y^ z-c5Ve$(K<+pD2mTi=YjWaiHX_yYU#-E}nCc-BJJjo@t~{JxAjPIj`$;xyymiqqv%S zI>jggf2@&N?eopSe(qh>E;@z8v;0~O z!+f*pdF~y~aCw{9Lpu>(VFD7P6`MVdYi$wf-zF>#yuB4Q>$=C_*u8I}7?rrOOK8%V zuDYkew3=>&)g$&Rv-;Oh$+(BHBF{R@kKX2EJ)BtKHH=u0aHN?m?aA^suQ1ugWl)Ap(cksGVeBPqq5vDUvg0prHH- z>GbvGc|+MR8-x%mYO$TXu!@jT7F8m&dxc3A!|f8x^Ud$Zu)m6G5G3K)P4n5jIEMxY zs)YL3*C|v{blNbSJT(s3&N-mC5q0CR63FO;-9?yq-;KU=ziRV>3(>ftp$IOJ(%7Sa z9~Z)md&Un~cQd>_f?TSGeKSJ-Sdt7_lCKV17HYM!slE2OP5XJamq$5+=J%^U2U1ge zHUwM?s`169Mys+1TuM0}C!cjE{!2*m-DGy!RT2jgZ+Y!IckfQp-C;c^!m53K&DD9a zNUy>$qhg)gaS@0$^%l_;S%xtM-E4v}1#9M+tJWh$3gh^N!S|Qh(GjvHll}JW))3a) z5Z|(jR*BYSbi2%DJ#76YOs&8m$VKqTD&Ae5v4-At(PMgMGO`^luz8ia6|SkhLJ;YJeR^mLRqwVX8`jK|+Z^`DO#C23irA9#1EVX0`h$QZT+HT3RK( z=$LG{R_OcDiLeS~TVto^FoiS0)6X|1^?VAd`vopB&^g>I5zb?s+RbG$*vsU*a_Llb zids_GX?3H^ArJ4&zFgV-Q2eUN$K{oiDRnH_i3BjCxB?%SOuaWxU_&2k8l58Z7x4v0 zgGCQzq3a_)orA~NDETJKi6!65__|V-TP6(@B(S&tuWH>-t+qXC!aFaX+IbVr>`4+Z zOW5HS4}oSHdCvD0t%}~ffynFTHG1}J`u5OF>Kr`vTFobt;hTr5MDSVNkciW>u5kle z;X79$0iD;_+Vh=>6}9^UNwuXhes>Wa$gX15Y52w$yfK;QN!D{F)DkL9m5?Z{zV`wT z`Z?C})H%1VzN9HTZ*O$G5hCCIiR;g-1jb+Fc#=~T0Z-UkD6h@j618i1SWATOEO=&U z{Ie|Yi_E`@m7S0dkeLNm!Q5@-u|sOBuhuLVC}#8dTIS4AdCv)^l)=v;rcvUzhvd{P3n-p!{`KU<`WBD-3x_iQt~2(Rj5 z0HhA-&)|9K#hTYl>vfYitW&pdk2Z~YjjW?5QF$T$uDluhhhkf105Ugk5qWcK(Zevn zs+eW}GL_x~77s&;zce3zuINt@I!cVVa@l8k_v03rX{zxqxSdRC89Ci7KjdG<38E{epjSJT;KfAbIZ`1)SV8}@Opb8G{4W^N2=0Eg;|=-Z%U4Bwt0 zs^l7(D%6{*^HlNoH!J5H%KbdD4;{u}UdQNzWKrrob-s`+n+;H-7?u!aHk`rY699iP zb8~e5jpEToZxAv`DI#0b6|nn^{pd3;Gm)8r-DRPhtn;Xo?kCH|-IXt)tJ7*5Jam1{ zLK_<#zt*yu0cX_=S)xqLLij1o1ftX?Q;l!F+xlMqRzu_+-0d#kT$e_LDfzbivh_np zdkc*ptTM5oL%%{h_tdA>zLvNmiO1@6CRp;auMvlv;)mz07` ziW`DC>WT)@bPTGoNb@M&LK`ku;--5*$j9%jry3Qg>jWMXes-!i2k`=W6txY+Vm?+U zvMCuPWA8`f_76M6F?ToKd*;JlaGP~GNW zpSh00^-65frQNK$Fdx#tD{3*KaD-M7oj}sknf)x@G~JG;$J@zjV@TV6{pkcZ-NsO0 zWWw4u$%51844MR?`BSM8S)3K#5!iZ*6g8F;hGs?dFJv``~30 zkDoK}UYIJdN`uyp`^j?P&)@yiAU`K&OeK7NhVFF$%PD799Q%%9x<99-ejJ$%l*M z_(17qv|OZ}8G7(Y#i@uffe{0b+jXzpFx6@EJ5KqCamLJ|v+Xw?zYHfFH=3u9W zY6vHg@qr?c99eK%PWq**mdHy-^U(ntYy@y@-rn2WHU+ipTcufTC5P-{yOQNZK)HD0 zYBS3eIhRRbU!I=cePO2BCwIPjeQ0!k8e*NeJpW}0*>3?C4&$oRUC=yo@cY#dViJ;N zgU}uB7=#?GmXnh9Kt-pszqd=qNvN)F5BE7sz~18i9&>HF7asM`$$5$opPDn)r2H>$ zLyb-4qXg)guDsdmW0aj=LE1J!7ZsklOCjKNKbY~sW#v5GAaDvBVB|V*6SKPRIT;^2 zan)0@={1dS8YPorh6)XXVtN2zDLw-o6+JMGMRjI8Q2J36D5QPg3j;%^l|hSh0xY)6 z#IQSvzI^Kkz2%JAYA1X$_JEyDv}44UnwG!d-p0Ephi}jZ(`o`e zXhToCN}*08G5-P_lYlEIeXe1eoE>r`h%Ngj~Jgx3J8SSo|sai6(ip8T`ZkKaT&_y*>F=Hm=7{U535Es9K{$z3R zKN9ZeXv!x^Qy7y!P8V-@YfgE=S-qxDgT0#+hjI;>ShwQKqwoGr%DNX53G!lAb3f^M zP3JY9n!ePz3#e;p?k9D{=3jRR@4E-8?E|UARu{8!?uSgE)E;pTHq+72l5_$th%YmG zUUya?1Zm8o=pTmiS0%{3mi0&ANr>b2P63}1hm`yI5nzVWSQsqX1%6-Uc0C*`;q55! zq}QR6xOr8$B`b>tdy^%z<7!(=R@B%iAFvdyH=AUlk>mr%(X}XIT^yTC`^ecmOzs}% z_xz1vY)OS0#b%Fk8xONd)VCJ~hfW$G1stp>OK5P-EKDg8pLc$`Vvl3%nxdv>a5je- zv0&OaG2%cBf^#41!qjTJr{4i2!v{ds_&?x|9E*g6qOsu`1-$aHu6yy~CsBQf zY{0kDYFyP(bG>J3@)Zu&2wtV`)a=UnV-NP>CDQFj-{YKj`S|{bRxOJj1YI0AFIy;$ zsXN4+?qGhcWw0O;-y(5oTLQ&%ajZt*$3e%&q_v}|Q5!%1FK5yl6U0a9px#%Y=PC3m zGDEFQak44w9rbZ3-lyu)a@g*n6a4SG5JvQJXaY@4p|a8dl?o24(RAg!yJvD3GTz`Z zm&*d(G-(+&I^mMtqe@70OK-2t4=9PfJa6oyG&fRJh0A|`1YkaNP_eJ%=aeS&CbVyQ z2x^4cqHK)WF88A~$bwr`zdmIN_(;_xymX^j^vW%TnBiMbeO70CLT-5BmITZHDVgpa zeiQpM*bIIC8UnKF#;$71!@|eGEO1olC6XO~MA&{@Nr4MURIAa%jJ_)Z@ssw?v|V3i zvZ+K~CqQ=6hl_6M%f*i~!~X5Uu)(BK1TA$*QJSZx5XvE{5h9>BGLbJf8`CC1R4YPy zy*?5uP&u952G~p2n`4`CRIJ?i=QWkotb>T1f!eh>^=V5+tGLC=H@Q`H_o)|bA@`&C z>_6aUBx`A@sUuPw`N?3W=}qisU&W+CgZ}WByBZ#rKuI!xf-j-H1`)v)%)vvm>$l&; zCBB-hpz?@*KIN6|aa~NtNY|CNTQezGS^Qdw!}(kjx|B=AXtEDYaHrb)YPm#(PtlxT zBe9H)3_#wm{mCKz^7cdYK9_!4=wA*AVp?uqI8=>CnM55q1T1P7p-_>UcYV|zZZV8t zUOfLN80NJN>l5kimUbk@z1hN&yj4dZ5gYaA-*!YC2rAiXN%5QKfXK&`aiGezYOXKu z)^j3;#GVZ|GZz+Y5$QNdypE+?RW7@jG514*-bo=vI$`n8@WT4EH<}#hdap(80e{^Z z>*yF3f#M4e;p;a&bs_ZS zbUMnoW0sGjMS^dRxuV!Y9l6M!UWWUpE&g-PzwGtHI8^JwC%s0;6iTi1QMat2UD`YU zLB7mq0I$VKVLVeyrd7#L?!G0lv<1z)R(v~8l26xpzDrnM))Mp0$#E3ad{lYKQ+$nF{R<465oQ?eeT4zgiK-N4^8^FqEycN z2~WG3{@A?#H6Meiyqntew@Y;+v)MseEhY8!&R6c^>m(b4D9{&O{-x8JG zmKGA zoR%i_B)KmyFMoX~KjSmq#L5U(@>d zxRT|fOhHGtwXQBgXUIWC9apvfq>qz13-;@! zbO{*=6k4-8>eaelcm2g5#)1v@)3Ve0FSq}n=SZ9o<1*hDF!EhxpGp|*oSmV-q@vx= zJ$sS63XEj|`FBaulUF*;B}Wk1k{Sx{enHp0o&GbM)ipT)KMD!{hymLy=* zi|CB@v7~I`e>rZClizH%M$*i41Xah1%yj)#F#S5gyeQ{`J!zq%mseHes*q9^2$!U#>}iyS>B9eZV3L<+74ml zd53S*+x?F2**;5?H)uudVhSMH88K?Nw`0EvKG+IRz^K=@?~a-@Gt&x!+gzhlEj9}u zm%y7sO#1Ely&C1D^~$Ai#LBUrhE)oiHnGe)Ad*7d0%05~t{$*7*4Dlx%!-S#Q0l|x z^YXLngwBC$(VQtmM<>Buj-=x1fcrT?v+gQ*WG6jS)_5RA=Z=v!H6~* zs`WFNIG%?PiMfQE&rY=`$xss2x@Y8o>q<``+=8$TI}n!Iv3@c(ccrf%{W+#c8+;+GTJOrgcFR-F2oyP4^*fVa z3GLjBW**;nM{cC=lPJURTk~g|oyAe;*ZsR*dCkeyV?xMF6Z*hX*NsVief=+~XJX7z zIjqKTjvXcO#FwNO16S9+wTVDdMS+NSZDEK3+~zlo4+_O0U3~B17VsS1AiX?3s0iJL zL7<4H47#ybz+MDG!b$&nC1lT1l}>p^L(&&3^0emJ@>oL6WiZ#d(00EGz)l_Do}v#G zF2>QtCV9FBV3t#M%%oYgc4FW+UbxRywf*qyv3WvaQ81L+z*zR{D=mQVwz)nXDY z6F!YDXA72R-eAi5Gs;LER)V`3k5mb^wKgi8<2#h78leI~E-(008i}(!j-5!|a3^&B zY|qAhOjMm=#Y^e#=hrk?&R>TJb2FkT=li?5{2o`?3vNFZ_#-bjtJ4jYAOSs@Zz5JcUGSmzgJ_ z5v1OO?_&nsN(bNUk?9Pe>&DT9oE$bVOe(jH3-F1R3nWSs|KhoiQUwGCUV_NQCUA#q zsuw)UfXPQ2s&BquxQvNhS8)WjQOE7GAOCT{pGiG&SJE8OSa@-e6GtV9@u1MC?L5d_ z;^yf&vCHJ|ssVo%a!|cqW}73@i6Aj84#$v;lFKvgxUbOimpYE32lu0T|HvXxEkN1_ zau8X?Rj9!Oa}Yx8KT;?@;P$w(`=D5~X!`eX&(KmK^1I$+jb4A<(>G&;AjPEZzn6dR z)j?x@g83yx2n6D+mtk)|811P{q~X_`MBE}!e>eft`SfnGtS|!YPLX@( zt=&jmp(KBQw#EyOV}-i=)fJiZJv@r(_v6q?sdsOKpR$n3MuHtv?6hg2T;Wil{rRjfE&NsS;CY~Wg(I|rFiRT^6iP+geZe6Wa4H_LnQDn)2C^$z zw{O6;V~e8>%*wAJA+@n8Tn%^}T2P+7XHNUjg9Q%>j{V6v+JAU2c6Es$FsB5e^%XD^ zLFyF_oBjeWJ9bDeX<}pfYi8+d8i@v|24P)Lp_*-b=L0xkUbj?jNCdFLsefWk>jda9 zZ78+b?1E;*k>}5Cjzs@;w&KOX3Os+>`#-zj=Ra8~-GE;2Got+fP@+Z}W0}ahZ z!q&^uk+fh}UVZc@pXI5D{(o3|4`{6W|9?Cpmz9f1Br_sK8OffNnNn8SduGOk$heG> zRWeFMLXi<=ZyA{(WbZvLdqw}}yU%@he?FhP`|~~LcmC&eI*wCZ*Y$qCpRechv7Uyc zN6%z`;+fBZQ}D5no!++shgT~i4x@4z`BmG0+*-^XM_vv;ongYmN2=g&{@llhlBZEg zD)SXSR$f8tuDs(f#N5YA4PdOh+U_}U#_BrVWJ{Pqvpjv(BLbi3F9Vx@Dex1k>>oc#1Tf-TS)96#!T!HE7{ zbza5|k|&daro2dll~z{Wg&txYphJ2^bi5vF%!16VLU~Je{;PF;Hz9%n$8SD5I8~Ho4-DR}5B86{jR-|dns0dJAjKM#VI?2iJ*3|pK`V>^A2pDbys8e#FQ}(6!MO8vo;>27E zW52@s`xl92wTJa0^{~D3CJw;?1uWX6N|O+TWtWV)1x*8DT<=Frl7nV=iDz6F``tm5 zZ7H7O8lSH3wf#wtNv=@pklooB%5vrBFlJa=)IyPI1c7}bQ|D72-CKvq=}!XlpKsDw z+kGd1Htdfph$h7+vph!nIqnyz|L0TjuRC)zTq?)tQ!>ETMJuoaWo{X|nIl2!w7Sj_ zd3OeS?KD9?oc$>io;gx1dyo!_DM%Gi!o8hj(@Ge2@ekY8WPsFBC6q}re!lK_5_)t1 zz#sxW8(imGwbu_@eD0Y8kQfcdtCPcVXMSN1@}f|K+e6P@`G@OSaLS;h0O?7|l`N|0 zSXJQ>E&RSA1mN>oKcOD*6$_|-C8X6c$mbut$#L(c?uhXDkt3yp>hWf?3Z}h>R_J9f z%EY)zjC1<2v$M~ff6Rnl0tD(TtigH=W2+MQ7w*Qu3@m>Ratn~O^KaRPetXNAXCHsa zruefyx**5I?vaYU-!doRQhWrDvI!#&8FiPqt>%58tSZqBV-QZdgiLN4NH?cGznqS` zb}dPc`0M~MTQCo?Nd^Bn%HnL`gHO1I&!}+~3iAtsMj98eY*=*wXB|vA66aYhUWb7$ zvg=MgshJ~|kbVkw2u6H<%H78_PW%`6n;Z(PQ0KC--EWHtmAnOhOPfD9wG!Ojgk?{- z3qgPEcGkDx!`(TJVtRaV(1@Gpr#d@=c;}m2cD(0?woedsQ)bDE;yZVs>P(@KxQF5% zpoES!g`20tGLE7ISOl>1z$(UW3pc`NRD&vKIdqtTvwmn|ZBzt~J_wUn=!3>xOEK!q zOm`UJuliwm1xj;o&P4sM@BGIX;b?+KLfz16I7=c+eqzq~4w6I8!zI}hkU*}%?Dh4{ zjVr6(>hEr;LvejwSYG0ZuS?6Tr?YC9xCq$kQ)n)Am`{#44Oif4(vCO#?!j_D{N;4` z$EVFkM1bbY{5Xj}atyXKovNSs;5?)DMk^;N^8=`#@$KF<@)Rq_yU%hpOPlS)uvJ}P zy-(NGlXjgAS#Yc4*uc&Q8$NCMYEGAMW#yMZ zi@5Yi%~zugd^T>rQDct)* zjoEqRpxxS@><;YCdUCc$6j;{Kt{{^P8Qr~vrE6j(zw_`Pr7`c|Ahqy1hk zlu`UNT;>Smv4B%y*lH8XLjB4_qbamUYJl&am;wjps`b?DTI(lhvyq0=9T-x!hNs+` zFU`D6bLL$)=C!rUy-E>UH4H_1yU@|paO!DhB6KWQv-yra|2=#4uC#k92>oV0TFWE0J%54KP=u49cDuW+m)Vb z<2Zo56=2YufPRFcSW+}xD(Q&*j}(Qq0mu9Q<94)E90qGa6y%F3fT&^|GXE7i3XKG} zU_xewBA^jLlJvA&&wYlS?wo=5(arg9H-MEMv04FYq7jJ$K3z?S4SRDVm4n*=Hb6_! z3*9*bpJc~A9Pr_g#yO_n<*{1fHVzF4||YWc4Z_ zb@&lBH9?e);4;RfuZd8_BdstrlDovV$8f+DK81JrR*TNQ3Ps|&G&S58Rmy_7 z3i?*f#99Q$+QY-}e)rV;>t2!SMA`HgTsLcte>bRn z{kXtEiZ#d;|7O$;=fGJQD(=2JI*O$%X2zF*O!js#58Coucb%bmsB=P$)@iaS=M&G( zW{^O3u;NTJcBIFv%IvXIbQ>GvPgLI)+=Ns58=&!05e*qT5cD1nRWfbsAPZ^(+==y| z1#PI)1Gx$20KAvBH7C!qI=h9QPnPLYja6fEWY`LV30feaJvHPVEeZsMDH!qW@3 zy=ms%$V4&cgQ0n=0=Rh_cweF+2ti$NUDP3WI%dPiW{<4>=_N24h5OV`Fu&3G zE5+b-97i#%UYr1|tg9435GxMpsUv9^7i`8rCTv>iOL+P)(jengNFS?t0fz17L5i!a z6U1TFfbq?0-AGZsetiYj6;NUt4L!5L#@TWB_#47ETX+dN@|1Weo1uaTv)u<1WDw5(Rq9v_MkF`IF`m8%lSYq!6ud z7wr-puV3vg>J72s_g1G`yC>`Qu{e+Vgz-H4!ir)dUYsFX*Z_TI_|2FA_CsSdZ$fUp z#89qzSSI=Q`)tPXeBUWm#qGFk7+Lds#{jKu<28@erTcUQB?#@~jb&n%E#^+ls*7}6 z4=`RWBos_j{DJ^MuJg&<;X#?g7`AbH03Nq-c;W4i*nY%O9LbH%g%M)+gAifPY(8`& zd;11@7zMZPNKw~2ZqRsrnp%y0{ThSpp8eWKVn`K{``I#LMJLC||L_d}b|k|@uo?`I zbTFw8kk+$)Sg+Q?r8`x>N7%}LuQlZ5gcYup&@qeK=YaqAZY>pCGX`<+Z3*pQQ2X=->dc5a{8&4;ws zk?l@*d!a!m!M-(^q`B$$L3SHNG84c6^p1m$)m2b1u4=-P;5w>Tui7<9$_IZ1y#H^a zwH{r4F0zK-blGV@(r`G^NAf1zB)(Pj@M$aRBhZ&i0a=ycOs;R0H+j6T%39A?^ z%cYF{ZwD}3Mi(=;-OSX{My3^-+fYbH=ucDe=oUF&KTm#8I`Tld+G|B($6Lb9+y*ZX z$16dSLBWNCPMC^{UDMH;2eW6p3LK0}DY)F4g{Kr6S0RSuGP8vA_cZnNC}s-U_*YV9 zHJ30)g-eT-`(U=66+buVkkZf4%zuan7iHCz5gSk^YaurNp2RF?mS{;t3xbGGs>X{| zMm@3F24|`I0ht1sp!YaxzkGHwM(3-+k>+W>w=W8Rh?`r9mre7p*oMT5>7^%dK%Zv9 za@>RSMBvnM0#&g&F1nJFEIQRnN4AF$M7NUo8jhqTK=7ZAL1N_t@hZ4%uA6D$*3r=Y z9>EB+%n!r$y=h{9p@P(cjk994Do8MeTo3B<^KRCKsS~H#t;f&#UU}XzN0QVHos+KY z<*L_!Z*Ckz=@V6^gEolIdHPzhSQ|0HBSfx{zO%8AZn*d8qEDMGiub{KrC-yy1c|`a zjL+!y!>xy^&{e}Dk#YH9l<$50P2~o;k2xAhlQ{&)5nKtfJcr_#+=7JedU&JbaMx3! zEcJp-C0FN@Yv%hklsszn!m9D^M$n?Nti~L@$1RgwXLot*WDCu)R@s|>F1yGLz1n&=V8dLt`k$F-K z4>N=L#A8zgO@mstf%Jf+*wvCl6y#b!U*Nmz3=?`4yDXFpbmp=>_VPM-NvrK&ng+w{ zAMj!eFF%S5whl-0BDx}SLn7$|@p_l%Wn3|2@WCrx*@3>oFH{TkJ2d#rhlY=E<{7lL z7-}#xyjYfy>AwGCcc>xMfFRPK>QKo?15_~amqyEaa-yYEF_b0f(-b3{2A7?`lGcT3 zt@}yk11R&N5H_TT(gZP=(^V-y3!}8v3E3~Ka3$XynkW4P2kQ+ANxO>`_LyhcLC3NH z0K1r0_cFRKS>Z|EVacYglmkgZa(t^kM4V)a3M#R}?z?(0@8DC?wh9om15VNFwg_My ztr`!{c;D^!CJ$PK0iYINA%e!!9)InB-*Q(CK0T!IWw?fF4v5}Fr5~y$b&XklU3gSW z5A+_)3kGDXBP7Z;H2e-7(dGWpDU zjUiqdmG~e#7JUbCKb>{m4^o%RpxyPunU)Eeoxrv102$T%+C7u-h|L9?9_GvE{LW#$ z5VZ~?t zdizEbp?Lr4Oy?uQD{*E-gc8~%6+(A+{ zWsfMLmE;c+Szxp8)hbJHI@yMT#Ue4Dez`F+wT^Po1m&cip>-aC(I#zVr!S$Xd{&-F z*F*189Y{f>lbPF(6%rs1IYqVd&XCM_QbHy}T3#_L+6X$ojAB?s)7jC0r5CUUX#3sp zF5Y?4c$Gx@vis;jW_y{Y=X|CQk)R z{pqN~OcFYmL+DMrg;gtB`Wp?55_82{;d7Jg|6%Jb_XlMBpE%vmsr3ljTQiXIE_#-t zid4^xN!L`@ePSqE2PZ51GaOT zqs%-6y9A##pSEv($-KhDiMkD)+YQwd3@qNxOM|-N?Io*piM8WFD_zeW$MN_Fl`Q=F zwXC}`zD%NfhEJGCAojp0S-DGBFQk4dN}%*qv5?I@8l(XmpdLe#ig*fV2HT2mC3^M? z*F$>MbJfkFX2RBgmgU`;t0u)AkLby@uuy9am=wa$hC_|G%Qa)d$~PHOi+hao1&_!IYCnY z@#Dv=Gnt;l6XR>=-U4K!mB_*N;+UQYa*|dk5hZnkQj-fe=y2-+75f@h;&GOg$;Rkb z?RWTOB$=ffw=@e2DbUKTP_Bsw zPIt!%iA@}#SbZL+IFPUUJ7_=(KQ_XfSjkLJOq7n9;J93*fmzK1kel?zvsCtqG_oyB zF|fUBy;1!sqZ#YsH>IR#7IQ|wOlLLib*IENaYxHakGgv@u|jPZBO=Ke#d?k=GkVOQ zFBfNSJx(T^S+>7p-Azq&m?0d7W-D#zzI5+iF7Qn~B34?G085drb^&@<1D?fnGvu!1 zc(3m!t7Bv-us9ncY98;Q*Y!Y+ef9T6pldfpF>+UraFeSLGvm2eR_)Zm#mLdY{^>N< z9V;Erc&wq_&kU&J8@2E4UxOECt-Gb|Gb)wEU}rn3UjvIYtW&6MMO}isR(BF*Qx26 zSy6!fZI|MN7VNPV*HV<}JPU_F@B#rQfw{Lz1BRsq0s7Q*yIkm9hsH9F-v=jG4D3%* zAtcW&dnRc879D6EbdzI^smgp1dUM}wEN)Z=C1s3>k2<5B6b%{*!B3gSz^Sg$CvF`* zqxEtsXE9KLvbPmG?*VGT2_Vqy7l!CUuhG&C6Uk_?YBJVz0brO`$1XNyfW{(eJH@mu zA$7OY#8k|Kd~kC70UkJH2p)317p;O)?BHl;TNEX$+D14Aw1BZECB&yy z`WQw;!59fXxZ7DQNu%GA;Q7H`_8y9AyO{HfWBzYg2A|TqBAi>3$CHvg0>pqKzR|4! zyQp6u1w$}N(QWJq5*5|e`b&;7JCz${317@zJ2H4g=3@p{^ zJcZ*_eY;k03#G&|I#9sL4B#wbRy@*qmRf+=CSY6Eq6lQer{{4InF{S&Nr30Nx1J}* zzC}Xd1jk@6$hx*{jEERF%0K9wE3U;?+CY%?aArB^`)Lk1Fe>BTLSw4~RKG=%A-Th& zyOK@B$6>>=H=bBD^bW7*ssyphK?d1?1RuhwJ)HF3iECidYMz^jJJsd}qx09Xw*dJ1yj^$? zI7skvuWWtid(%de_!$_t^_b_YDont=>%!#ZyMa~>rr|y+7{9J>Y3a;T~j=NqOk+}n0+>yB*ax7Za^7}v!_dv14ucfW8`t(TRzb{xM76&@EUFy6ATi8 z@UUFwdrT`_BYMoK?J%aQ%lV+c!9EH*q21p3FVvVU6*sF;Gi9f>P(ogTAnkKv$FHgi z2C<=UUkhuykGtn*ew3`LUWvCfd1^9{vTE4YL`LG7h*OOd@S(<~HLC={cO_qR?)ZfJ zZ7oVo`-cbWn4+$4gEjGu#aD>iDBVmE5B0e?O7|%qo}6U2n%uMK@4PE=-lwihqYjXaW@au19xY z7~cmuW;85W)1qU}V~MvM-}n1q+ow_5%uFUqK19Rs0!9Y)9;dF=IAHtT@U=7Uw|O=Q z8J_)pQ*FYCD1?6aghl7Uf@Rf$kW)MRRi%cQnNpbJ$X^gGv0|wGtx>l>f}7KGJ2V=H zbTgJg^1J>5nA68E`6-=Lsvy0ejpg*QcQJ^K%hx^Wlx)s~N z)o1xt9>sP6buhW|XyB%ks@kW+xys`c=zdZncjO>OXTddODi*C1MN zhv8)%`?-gEa@I)tK;lu#aB^RMAZMBH!4Ix7uVwtFq4H#S3Zg7xG5s7#>~VO!HR%%( z9`?Iz86Q3IF?`%1gmja;3M|(6e9x2LsX&~!dmJQzw{N+tx z{^`CWe)zlp3|ChrS{H3-+CTs%Z@e?q9F>2l`*SpL;sLuM-v0FJX9c z_h%6O*Y7Ve??3X#{cGa#2cM8D$=)ydyPMiOSgO!*Vyy@yLzydbh4O2A?4LbHC4T*; z{VV_SfbE6Rw*LC&|M&t3<&PJ2_@jU8SNoG!p^N_;lE2RL{|}NM&A7YTXYcI`5ViuC zYS@77kA&7u7~x(KeKMkP1PM+!oI2JcP~}WvxRl@W{MaTO7EZjtGx#H#vS(RaAToOOF)*f9A*^aCts}EK6JgvyaI0Lt=IS-ylmP z@#d1$F{3raG-{g*eWx&tT$oG2XX+Y)0CzvfHb|5CPg*Ka)zbNZ!R0fk`oBKs;odrL2KWEM}p*f!D?~qyhHP?36e|QRuE2e+C>+ZRp3W79&;o|=$YYEXv@v}spwIlaT^{5~;MW{4ZY=$$IRF7NFXA(Ku0^Z?V9|XS;&eWP2q~3? zyji9-PM8gmis-RYvy_DVCx&jZg9*DlEYl-)tfodP2?$uF);+h){r4V-+e`? z2!pssvjbrCdiW61!3(!pJ^}k6v*#Sb%D@ESodCUyVjr+i>dek5ucdJe(0g26{sRM# zjOuW(p^|q6T}I4ezBlPF_04~L;WJDCy2`U~7rzN;T$$V;9t;*7p#neh^Y36LeDAgA zj<<^7-a_P#g9QBJpR@rZe#t9GIIjK{WCa!diV_K`h}ON5(iX7^g4#I>Qum}0R|ZSn zY4x|`&q!=370Fo%#qJ@JwYPPj{?x36GF&Glp~hJsQK&?^3YZanqgU#={oT-y?e@QK zi!hXEqP-~GQsXwDyZMadOTdT}4!$UPFmjGVY1dGVw)ys5{}QVz)e2MPN1Ja+S1_+I z)g_fjI6#JQYiZ9%OiAmw9B~HGW+Lo?oBc9sJ6BQ*s?`B2T0u}n8` zd~dA`47)nA!tUjL=u|MOX7NL`U~VR&@0UlDP!<+4N77xdEi3`y*-Wkx;h#{cv%`bJ9Z*q`qpP z)SUpvf#L#IA8Ab9nbwoKfJ$smm_TZEpepydTBhLl#frVGTOZz1pS{&-clgFR)6dk~ z;P?`t(1u1r5j=tVqSS5dU%z5#8-)vh zd)G8?@3j^G^=X@l{AKNt&-bFTU&T2rHP=x|-qYw2F4sPj^V!^Os%JsjgW=sh3B;zi zyIv`l@?wH!G`A}FTQ_IEp8f6_!;Y*W5RSZAK!khcBGg({;a#_7!t9{f;q}R>6__+o zd3Zt?OmQpK+Prlx1$W$YYYXoFc>UuYT~rzPjyW}XFFlp@8P3|#b?Z#xZ=M1{l9#a- zE4G!YQaRWAN@GXu*H)@Q{Etd1BPU$@6P>h-k`EiZnQkKzT~mw(PKV&Vgy)XM@kX{f z^9L+mDrXXv=iY^Tc-`g9uA;K?1a-Nmh8GvpB+(V6Wa6Gd>e{#Z?hCE+9aQ;GiOTf3 zV%Z<T{ReM?ccXZ;xIkosD#W7oHdtrtd<6+I1f@v%^t=8i=b z#Od4^0uDBP0d~pmoAl*8CK&ZWG`HuH6W9Cvf1Sqvd12^YR6reIg~sA3JR4RPXU*@# zKPuaJR3d?~sW^B@?6P3Jk%VAk^p)HHhkvQF`5GRU#j- z|AC5LI0vdbm z*7x^CuNEdd&$O>V#<5j3CEI0Ar|~u45mBEh@tb~E;@QvC166tkv?sa6+D6egv-EVc zC`rx>?}9mpkkK$|`gQ!1%HAp^NBGG*yPqZxIt)ge_mxc;eq8j~+g+RcyGwx!%TK>-&?-d@79z>Z&)=-G_T66aqYa|*YEi%A z-@2;J(wsLWOCdh~b{8n#`Q=aA*sb}d9F|=1Hcrgi;t96!expB}rv}-3bT5)bZqbX|{JSf)g8~b!qzE9Ay z{iN2BVusC15{I+}IP}2Xs#LrPMA(LQQf!wtYmV-s_=DGhnxF<=@bKR?^>10@sk>`S ztX|~BogAOuucPIg)K$&@Wp17Ls0QcNclmNs3;1?ItD-d|CoBUTP8fgV72lZZx4Fou zHw_7c=LGelHI6AJ&|K~iws}8QpvtZo*|uvyJnm)tE<5N}H*nbHBIcE_nmlRYQTNN- z7B@X`OWK+fJGL0bKauMHsUUa9XqV6YLxn;lo*pPSx8WJXQi66JzU7aBYkIt5;Rfn& zZ$?NlqU*!P)~cj#E1kU2%7GcEkZrWYVT)$prstvDx_x75TAH6)lg7()Y<3~i(t+@U zh+0$wC7^kt%U4WWBd8?1L|MlVYmb-3Gd|}?FV@*qq%7=sBC+2(%9|ax;#;|Tnm)jM zALOFXy2;{oTXc*-Pe;i;o_zJ-<;n~YhZ+&rZ4fc?Tbf20ByE0v6!u-Ufo>(Ha#(@3 ze)t2SpLr5HU(qvW+84_}Ey>Q;n=IYw&cPfX2i@-jYf^&^vlVn4ktNs434iT$xAn~Q1$AVxvu)byY ze|VLzM3S&Bn7bW0DVj%FhW>2Z_EOproa_%K+`?^`H1?w_K*G9CAQN)qoI+?Ar48B& zR&!2tcdon*hwkGf_q>){^!FW_v$t-Zv6ILVEB`(VYXUNiuur+=^}M!C$*%`ozI-5e zMfjUhfc_LGbxiidM6wS?pGc&8E^uo`l=WYk7L0j%MKMfoS%#Et%h0XNk~ioGn!_-v z&({-MK~U}bb>+|#a6!#{SpCDjNmrwU3XBR2)zY&=7!2HOyGkbD>sk>UOu} zy+uf%46jYa(NJ*|Q#vZp@ltTtywr_vZ-8z(Pk}jylm?w?aQRpVX=@nbbiBdp63<~5 z$}J$*l7V;dP?~)%X4wc4pLP!go7BzySPHL~qIa1q%F`J;`+BEs`YCYyIn zg$}P02roR6)*&^~m{vG@VN~9tMef z7l)}{OwJexKFG>Eq)tm5t6AW|j=?v?zY_tF~4I40e)Z zUrZ)SS2VtLUtMA-J{Nl==%fbEsaVPTWythI$cq`167x+}0L$Q$nHM{!vxdVm% zYQ1{NQz;BdVI8cxshZva!$IM31-x4?V!O;HZc}6CSg5VQ-S2yqh?3NohJg3p5=b92 z^}KP1;zV6Dk@Il3io2P+Ggew36g%rKa<{*WO0oGg$pNxpR@B$e{k!2@>LI~w*E{Yl z6KYvh0f6iQd2t|p`3#{^O%$SuU2k@Zvt*al|{X>7budbr2fVbl0a z<0M$lhhfVH$R`KKrp9WhHlXjm>^$d7|Z9 zC;M}pJtEki!*8X|5M}3&f08&}&%b$iJLFMJRQuMT`*^!dkK`(`sov7D2tA5_BSnV2 zs)x^S0mxXK0$j2;q8n_S?ao?y%kPVJY^dyL+&w=kT=N8`l5d{I16g1$5OG!9REmuMmDkN?W}*1;P=+j=f3 zmK5EvL!Hmo+N_Qf>bLhhyY+e<^3dFSk@2j9O^i*foebzBM9;ly@>CGQEwlPx}C&ycK@&|x6W&8-(xzRnYyElCiCH~TBSozq;BgMvGJqO^YE6$Je%055F z(~Ue)jwN0O zhy#kE-fomp+}(ob9h|FjPEsr-!vG(xO^6fZQAli6WKk?KNn}gK)21{r>>bJ`!8w+J zKFj4PWyQQNXMD`v7{OEl0cdmI|P*~K7#$J`gQk7niy3| zH+v&fCoO6Pt}JPRra<#N=>+EJi+HPoA+(#Kwh=8>4pSD}_Pk&d;lHgfV!5!=CrLO8 zd?jm+9Ex*JS>p+Usg)!xCwu8TPPj@_HN|@N>C_+u^mR@~hK3NjrMsBIav(IM<)v}nsp@CQc)vR=p}WWv$LyD6mQM$N>P}7_5Vy6*AE^37xDt=Lz+aNUoRNI;5#e0I z(Fv3*3OEUJ##@jLMVkLy-vgUQI;lI}eeoe7=zXH#-oNBf^GJ}os&nMvmtwEI9p8Qn z)awTxwkxTo88m_U^qAMe)wBWK&)=zB{0d$ZwfWJp@HYS{e(uaqc>enn2NEkz;=@4j z7^vQWQg&>xpSMh755aKh23$%d;Fgm?@J!6&VrnzGoFjtW!`5*TnE&6rwKoCPL#a>( z;bidH7%maVWAnC{ua#gNnl+CyrB9qTmL7&JPmewcfQ5@s44yc;{i6;(zin_&qQ!q2 zo-5H*AyF6t<^bl@V}IJys2=?EAFCvjJN>>phM(AQq$l4l;Q|O+@2`++|C>Xb4vYL} z_;8qof*b#1NdI*-`)HvNSHV?3xYB=>gCL?m_y@+tlil*ahs3V4{xmJW0S4kaeMI$N zGm}*B&vA7Ur`YFF@#UXx6zEAGv)C{83nRY1^h0Yk{-u?YY z25?ZgHvt@I>Lr0l%yk|?F|rV;MZ~cmn!r6Vj%2g1YG?u z=AZxZnUQCfJ#x`u6dNaU_w=O^X2Iv)1=%oV0^K41z8V_)%gAYDi?AYqI;L;kD>K^8OarpKCHFPH?zujUKKHQlFG7yrI!MiQ4p4kXbEQv%mqztDq4_k#- z@*t|VT=Ui_AbabaO%qoh%_OxOzK3(5#}_GX1o0XO4+J<_PkIBite%V^Pv9H%(2?jc z-hx3v!NCFpp*OqG1gWF_y1PNf&W#Yq(1<7?Dk%m(a(JaOr@{r-Qf&UoIR6t_Or-RV zZiJP1;}1^7quw37&93Av_9NIF{2nXt!{<_=6TF2QsgE2vX3H`IMVSvw3cLFTp+jD~ z&1HvOeA^yC=F0U-Qr~YNQvRB24Tu`l{vQSdT2~?5g9XnWa|snX@Y_KQzB!t3plS;2 z+47~lXm^UjSGdRYdMCfVFVug*(!{tz^g{B!UiCK`F0~qpia+uGfAgrLctBqO3k5{i z6$>GUTrh43Ts~i|37u%qXtjMkB}j{tsk(=eP7t)#c%zvWZ_pBXhJmLgCwvpdD0Ee} zKnQ#GQHsxbEq$kEL9fE6O=3MlGDGSaTGzwzlqM`EC6YFh=CB>e1BGE(pBKBd(B>2!b zHV#6{HnchV8e(!S1TsQrJwlHGY&pYtV=tECJr)%%T+t}diJSB5en{z{EF6O*FY8>M zKqBjWP%vD(Hh`vKT8N`28ia&15|Y4)+$7Zl-8i7W6VaJ&%w!{aqU2SqHQZS&u923( zQ%TTF1Ij#Zbpe$5<#7fhD`HTM#9KjT#slPy^MG|w7^-fX?8lUj9555(FI)#5j((du zc!;cRc^&c zVKWyJXPcQQ38H5jcV;d6#D=fTO~IhFPm}k*gNv#1BxnWFuTSvtW_Vyoj4Wa3J9XJm$W@ zMbpHQTmMrTpo%#lW(DIYFHmxh3siRAnV9;ncR?UPz=|3J^<$i zg|1sE!-QW|(jgfL0!4Nu^K2f2pO&O*$5NWLZK0_xZ~OoUXn529S&G@i|CTV@ytUWE+=PXX-Hu*X&&KRl=yGl!UZ0q7`-k^rS$X@Hg}c)m3U(yqU{ zdB%yxEQyp2GosB6VxUC^Q#fEB2FZZx=>rq?3Oxuz*H9)QgV+8s6a=bc-~kyT_fI?- zWgv)miDz3PwH)7q+3eY!6?hC>Y>y7!sy`UTcIEI)PtL3KMaduAVS3dEQ7nWZl^$e? zqe-@?5R@!RG*#;f9A0j)V&U-%9x9%O!!d`R;--rrd(C+qj-&!JEs!OjWgLJ5GPCHB zy$?F^Na3k>GWP?Nq3^Q<{rs zICToLBZ2GKk6Dhh9bIRL$JA#9W=CNLao>ggarT?0GU8Rw9|NfcEgyABQr%;=f1<%_ z=zL(Nw6G`vQ^G8%Z4f>tC*-?GwiF}7Wht4^EU-rPiC)0hf~QQku(+)PvsA&>M8`^b z7fgm#EN{)QB9B7Q53kC4y^lOY#lsasx8OeZ^#g>Fkc(T^Ub!fHOl)0^%2GK&i*e+| zQPT7qsi8vx9Qu)N5^?0{Bnq>Yw#p+8p!9Qu2F*P8 z+xyfhY_(LCL0X@5kixMVhksqlf*&7(2Mhkk#pDyf8~^YY0#uOvtrh69EHhcLBh^!2 zXDix}atjU*CS`4hK7n9RE*oTHRxu)R$EC@pY9E~nBsMks>>$VZva>|YHPp1l6(avq z0gn-YBr~_z$Bp9DT(lm($6iR46aus1d|_;YFtHuOD_o6NYoWlR0*Qe$;k~avUVmxc zEJlH;ePMKV(rT!zG+?DhSjJ{l*n5lg3)=8J2=Rj7DqGjSLLXrEdR+^;lk@=}U2OU% z99zU8Bsp^M`9p>WHB4Ke?Hhv z*DN|tCb$J!0;^I=RLR9hw6p|AQKTGkEGzm*WMQ8U0ydfY$TU>mIMX}Gj|6eH1m={w zKtSC~Urnn*%oNFHFIFFGLG$n~BnMPiRT>i;)xev}8bHT`PJu6%?)s;O191lR>{B!) z%A!G~a5qGOinRgE#E1v$mLMpm*KiYjzrywUhcSmWdrLi620n<;D6 zj@K5BKUNZ;X6hA#CC$EjRl^aHkc5A}2ka#sRu9u2k=mWjY6eUr_;b|RxpyR(kSP~f zn0qVVZb;%=p5=`S7tVVSc9{$@p&O@Otr8bS2xNGlnnD_0^g+s6>Ud`u-+ZPi;eh#H z0ZUg=%wlw8Kb6)C>SO->}|6=+B3FOGZ3R2tFY46KHIN z?N)E<4<24yW-?5JP3 zcgc+g|72y$l1Rg%>(34xV0TcElfF3{lFwL@0F{jc)X9o(l!`hbc1Rt9C@fmZDc*tl za`rAXGxLs4zEcDvJ8>g|ly?vksW%y@#K)8}xDyb`Is7^(jF|G_QUX`?m*)vh^ORTC z96qkc6Ojdw2Nr={O z4f{VI?+0=R5^=*3jw|B|KJ`wpOggRZ6SOO-eb|?#B`vM zwH9)kcZ6rtUNr%S@hBlct>KeZlyRtVt+o|)FQA=P&USm$j}`eWBX6)P7RI}Oq)C5# zU$JBhI}8o;v5<$sb#Ef>^P9Igx28M}px;X)_y*U$-%>Gh=fm{qdXdNNR|mf4V`iqO z?Il&chCtEL6hz7Py)MI5Jj{K9+kLVHHjdv5ys;2nc;CzO(w>kETQLcH{9a)Ro41q* z-et_|)lr+@tWftkgl_8q3D8uZ|zct$6{n?;hviBBJ7%Wcp(4IcJj*b z`0)>~KBU3E^kJS?AD%|$rJH>bZ|{7d$a&iG4W+Et!dvjNisx7^#n}yHBr8t1TbXq2 z4sJ?p+5pd2eW$Ynrq2PVaO%LWR5ezOSF>vw(^FCnNYu}B&|XccsO|K4GOc=U>^6F~ z+Mb$mSJ>lz9iKVT+UTijb!tf1t32z~mjtM#7Zf*6KS)bxHq$q5>BEIeu@k!Y{75uE z?IxZ1uif_lxdHWBF+NBe1tB+E;90v3oFE^LTL@iBnVCk`N^P?LfU&pO_Tl*ul8)$= zXJx7ncoT4HXS{AzDpFbqG(mTy`*@+6Y2RMJ)F5%L{5Frj(P`)TAhu9@1eHP85-BF?pvTf>dOB`nXXnZ`Ch+*$oC8B1qmankko4dNnFIp@FX+X)L2fSd z^+;t@|LiqkuqDvy)W|oH0(6r zcdjW0S>F1XcZn@efc2E#$008dr+Sva8dsWu#jDXtmDIx0uc8G;6+dN7LGJ(ln`HC( z_jhyNn9F3TJ$#()vG5_<6tW=p!%3;C^r*!{IX}0>{?}gu85JPxn&~>+jw*%| zriiO~f2&TiXlAr?2B7d~QBg;`@2l*B>prdXD0r&`o;}O_5}ZO=0-R1W9C*zzwoMg@ zLZ}TtBpugej>XnjU4=XTWg>w*MzC`sng4g|7C)THhK}zQJi4$#rWwGUV+$IiN;eu_ zpM9!x@QiKWfLmEg|LUB}8(oz#A&A5fQZZ3uz;L|DrobaWTb#k8(gRP>3_1Sae)hF% zFJHZ?KGD4+5xGdSwU#QmK*b*>wi1N+5&(AzzHK}Mi9SLgwr?dd1L>?8420UdTewi~ zuB2z+(|gY92!TYCF~*?|b1J1ymp3$WF{ekkXrCI*-8Cg$CWc?1X6vKob)RKKy>Byo z`tKLj=OKwrf58{stlZq(r0xl$PlUH_-wsLP?+jLpml`VbgqRDR8-thVvp-Vl6$lK6 zyjDUn8z^q?nI4b(`wzn-cy>kVM)@1gZ5io5@19v{95W8fcLpI|BzAeM&fKgi=KA+>lDIDe3+l|{ zNzs_;#6UhEb(~tG;v=ncTW{>;3EuJqW;*GH%@Q047Im z5_G)vJ%WZw7s^+jW}W<~{@{h>Zgl4)B&fWYf$T8qCF3yhDCb*kW!GBU+@+;!Py6?m zpGvsL=hP-mbjfSk{N|RXCXeSe02Iw>o}*Cs~uN)xsAu0 zvm1fWegC?KzrJcCz~4}Ei3UL9V5x#QGamnyhP+ zbOQr=u_IA8_aIMQ5a7=X;_hqqtzEG1E#$JYIJAeCUs(w}w1W3H2|F4V97}-(dj>$w zH=DtdZy_jA_!k4En2m4d&7gCb%6#)hd0FkM#}M2m%~;|iO|WLvR1elzOuq-#H%%Du zu~)56dr>p`YJI5*UPao)F1F6oP6@klImE@&PQu3%;}h3EKM)+>FNNge$+TCYe2}du zu;kY2@kphD-fa;UKG$+oHTh9USRf)OfI4CvWg6ceM?NQ0!T3>G7%wUQ`BzTwX=?R%YzgC z*Q>dNWcouSm{zncOOJe&aSCais|!U9VRWmlft#Et(UNZJU_`p`nL2%fQeCJ^N;X?T@go zg!lT`-LT4~rxvQOZU~Gjuq6m6UfSyr(IjH9m#Qq7zloWIR_c zIeG`#h0?VYyvbH}VwUO_@1TpgTkY7L-USZ%?_VzcuDs?GDsGd`Tm;%F_k9=nd;O%s zFKA?YagSxiXljvdNc$II2vfWsZQN^kL~*e|B@WoZlty98J`c!y8CdU}1nw{)N1)Npo=*gu`*S1*e?~7rrH5AH=|nbL(76FmKt#^h}jC8nNa-n-!)obO+mmH(x{=G_Xt)DPwzO#CSdn@+?v@z48 z#h}FEL#pz)S_7q?0*{yetyagm?VZ(o@^mkW+N6vIJxdN-9}qnw2Lcb`=) zdaK2?%oxf0?4}Ee7a74GX@b*2-#C@X>6KmHU`SW_mmpI8y#02e@@*BT#6!)NCo;VZ zgn7nCh|Pd(nw-yV4b^|I^hlfN#y!qd@lUCOwKww?pLIR!wM>1{1bl8>uFB5qpyXZE zuNI>E)E*_^n632ewD^GZ#?>sb5)mes%6(OTUcags+YLvF?K~%ad2!OxL>`j3NgMna3)RqE zLuY7j(X|;0VD84(cA{dA=Y7>>;OU;ezID?g{B}o{7|kNBWu% zgp0f4`qbZGHK#o(-q9|pKKuXLyY_gf_AP$SDIL<~r0#KWlhP!VIptMNB08Ss5Kx15wBJ*3o(M<-3BGbZPZ#|%S9)Gd!O9(fH?XNWOP%rF>_yLLr=GPnEB{p)_N zzqb9^v-fYWy?*PrzTdTeYjF{qNIbtO#t&H-Ui#*=%lM)xuJ|@Qo8^lqW(!s<6r9nm zps$lPn0P&>1S?f?)bCb~Z6}|ss)KZNp4J$lXKFp?Hn?1V(^}lAYIMw%;644`26w@G z4g`(xSS9TX=YY)K{ZxzFDyfrYevf^}rcBnN(}`?F1PFkY;lXO=hrO-!?GX%!IKvvaW5Xcrz2*>G?!BV9DUm`?V zPGc9IJRRE)D0g_>=SwK!P6jPMP#Qw&%t5b`-TLCIV$u~Gjfh!B?98ho3<(;*IF4Wdx;c8+!1l764yeQKz3X1>-i^n0CRn3yTyY<@2*< zS-P+=x6;&A1CPFaS`0Z#g#N79?zUixOBPZBTVH|=+sWB^RXN$U^s}=rmXR|o9iHV{ zuZZe>N!Sy08)B|C%s|}l8BUOylCIe0V5zQFe}|I*^U^Bu*L@;E&mD*=SMFVu2XSzV zt1SNb!$q<;AV1BlZY7Q36?4&;7m1_iE(*!M{;K((hfu`8k9ou4_3ZS zT0_L}R_}IgWwlOc3G;s|*M{wu)JOxkrfWX*&`It<`#4RuaO}Z-% zKY)xvJ}8{R-fu(dxAotE@)dZf$wu>}OXP{!>R zj=R-i-#2?upmMhnwj-v7C)iaA*k-?YP`0FEY#moN`n8Fv&%`E|k_Hq#H8+j${4w8$=&-%_ z+89+P-1XUJL^hev|4K@O_;;zg|F!1m!FxF`6Mio;K|I5jutZrzv{uWVn$j&R?|b>_ zuKH;XV$NZ`uNPa&HwgCOjZXmTAvz{z+_QDbGXy-IJuoyCbT%hF!1ATQpIs#HLp!Wr zD09Uv$=t5nE8`Au0}Rx<3L%}x&gizWlb9?#9uI`D$er|bzr_OxW3_Iu$YWY44&>Ns zpLqr(?#*EEm1YW@RqJ_C0PVUw--OSEb&N&PDZHI)2t}@bEC5Xem2<_sOLRe_74uEQ z)@cdEGoA=3a)-i#V^X99Y18H?_hPncCeI zrk&l};{vir6KBL(tyI(l*m7AqX}R?;LgNFQMLP?KXz6Q1Q2X)<#=vX4$*K*-bmM}R zn7*9)#t8Q=TMcGrS^3N;xzxX~9`CkjlX?v5kq)i8BCqA=WXpYPcS*$qlq_Xh#7r06 zDWL!#dofV;D#ajV=zy@XDDin5U#b2NYr4j3l0MGK zF`V{VS^WH;O$E?@Xt(HDyX5FHlB4x+*M379e$hr>0qkT6ZcXH8fJ^mP`TZ_1A~!gw z`moySA``4Jr}~xGZbsOwqAwxdF}url^7KOY-`?m2cr<#@g=RBB|3nA}Z2rq}my_=k``+l8~FdS9n-qq#nA z+D;XkIYU!M7@Gl48Nl;^$Lx8 z^wI-R(PqSwcM_T2mM?N~OLrOQYv4V#^zW_c$jn#6knXYFA|5+5-|a#AMwdpjXI69= zqSRm|=+PpNORh^!Y~F_9QtYUQ@rcP`FaAfwEK+Bb52kqcUZ3e)$i=;hj_Q^6>wz8B z2mJ?YbB{DL_1`w6tdeUjnf%_3gg$hZ7`sg_rPbueaW&PzQaCQ|J^&<0gY*VoVC9| zNa41tOJa|cNF;^QKAXq+an}pnE}IV5L#u{E#~}^oWKjN_(;3|lUQDd{77gC{Z+cS( zjOeV*M@`depv2kZ^<0-|%-W6GN2<>40SMXE50PfAqh?!z@7Y;L0@@-h?jPd znfdHUVKCCKsi1SQZ+X)`o-C+d+V+PB&NPVh!$;hY6$5Kc%kst054&%R8h=|0uFFwU zP}d$&;7SCMPMqVQtgC%m*VMK#A1YuB?#miNt9L7$!D@4Z@W~rWqjd1_f9Um?C0orC zRUeJdcds>WsrNzaio-j{3NBH-28ZP9{76d4mCOy&gsAhT{XryO*16TwY#MsvuD0Nb zNe6U#Frz3tGz!9FvSNk4)jdf1G~~X?1w#8QrR3|4=2;a}Z1fwC72d;kn!jfh(INiQ z(dM4U%%wq>P67mNZ$MQY1;WZLki?*`H*ny`d{%jXjMf2a)oG%u_P`E;iwgk>F~ndHm6yqT^+i3hcBpIsYy(?8&aGJGoOv>VhO`yUv^Ua ziPJ8Fg#nD3>2z|}K1PkBVR=PrcJzHCSr5^}1ymj=GfpNB(pzcjOSzKhqh?F4vAj@m06kxoZ`?oP0bxGVrtkOP z4+@QJ88jp2-zUGNT*t3Eb42Iwgu4mNsaGMVG0%R5?#Wdo^xgQpa!JQoMUqa$toL-x zgnLnnD_$cI(pk=1FfE0u2-K5KHbf><)y<6!U71g9o4dQ;6RDBHllcAOzV&yy(6z{ige@8_6r4J^ICz(1TstMtx*MGl|#z9-0@JD{;1lxWX+^xxPfImI4M|uHc zbjqksGAJmhm;3kn{(gPS<*gGM{_@_r&sWy0~@mSSSwU^B7eo9h-j*ak^)8 zD<6E^mQk*1*xUfbUoYpnW0Cl0%z1gin;^oGe`58Ys&i+=UVr)RO9inbnSXAs55yLN zqeYTpx$pVUb}C+;v2YdrC?b_pMp)^~Z}T{vW^cugWd721Wnp3?Lmxl&pwp>%M{Li| zU(Du5DefIqdaJd+`0^??t)RYxlVi*0wM6{&Qs~V27Zv-+INJ9ry$-F#mtHIvH<$Z@>|9`&e zv17-c;nOe9X;vBL8{*WAaV4jVs!y4_)hBW+)BFZ1Fo$xBii*lLxlJ49ZhU$F`-x0+ zU`H*SS+_1qbaZsM2!O2!W=R`=(5g63IWSN1a;|lP-3^WlfAvujZhZ~2R`*WE{Yy%YGXgvCw=lZ_xjCVV8(#wyB`U)Y_%;1^1a`c2 zPX>|v`G64tI7=Y$H0ba(sPzv>e$&p#$iR-tckZIQuW|9W_};v`hX0Gz??xl8ner|v z5-GyxBK`<8WkH2*EY~*?=UL$t;~-z%eJIo<+zrwRoKAjJ?fAkm6$>x%Wdb-N1d%vhCk}5Wc66{iz5kmlc yBG_gcD1lRjNmas^ROSCS`&9Y=%QAa7`!&ZZ-+pl)#s4e#v-rt+cfRqVU;YI=p|c None: + self.online_store.teardown(self.repo_config, tables, entities) + + def online_write_batch( + self, + config: RepoConfig, + table: Union[FeatureTable, FeatureView], + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + self.online_store.online_write_batch(config, table, data, progress) + + def online_read( + self, + config: RepoConfig, + table: Union[FeatureTable, FeatureView], + entity_keys: List[EntityKeyProto], + requested_features: List[str] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + result = self.online_store.online_read(config, table, entity_keys) + + return result + + def materialize_single_feature_view( + self, + config: RepoConfig, + feature_view: FeatureView, + start_date: datetime, + end_date: datetime, + registry: Registry, + project: str, + tqdm_builder: Callable[[int], tqdm], + ) -> None: + entities = [] + for entity_name in feature_view.entities: + entities.append(registry.get_entity(entity_name, project)) + + ( + join_key_columns, + feature_name_columns, + event_timestamp_column, + created_timestamp_column, + ) = _get_column_names(feature_view, entities) + + offline_job = self.offline_store.pull_latest_from_table_or_query( + config=config, + data_source=feature_view.input, + join_key_columns=join_key_columns, + feature_name_columns=feature_name_columns, + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + start_date=start_date, + end_date=end_date, + ) + + table = offline_job.to_arrow() + + if feature_view.input.field_mapping is not None: + table = _run_field_mapping(table, feature_view.input.field_mapping) + + join_keys = [entity.join_key for entity in entities] + rows_to_write = _convert_arrow_to_proto(table, feature_view, join_keys) + + with tqdm_builder(len(rows_to_write)) as pbar: + self.online_write_batch( + self.repo_config, feature_view, rows_to_write, lambda x: pbar.update(x) + ) + + def get_historical_features( + self, + config: RepoConfig, + feature_views: List[FeatureView], + feature_refs: List[str], + entity_df: Union[pandas.DataFrame, str], + registry: Registry, + project: str, + ) -> RetrievalJob: + job = self.offline_store.get_historical_features( + config=config, + feature_views=feature_views, + feature_refs=feature_refs, + entity_df=entity_df, + registry=registry, + project=project, + ) + return job diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index c623af1c1f8..e5328a27256 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -16,13 +16,12 @@ from multiprocessing.pool import ThreadPool from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple, Union -import mmh3 from pydantic import PositiveInt, StrictStr from pydantic.typing import Literal from feast import Entity, FeatureTable, utils from feast.feature_view import FeatureView -from feast.infra.key_encoding_utils import serialize_entity_key +from feast.infra.online_stores.helpers import compute_entity_id from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto @@ -191,7 +190,7 @@ def _write_minibatch( ): entities = [] for entity_key, features, timestamp, created_ts in data: - document_id = compute_datastore_entity_id(entity_key) + document_id = compute_entity_id(entity_key) key = client.key( "Project", project, "Table", table.name, "Row", document_id, @@ -236,7 +235,7 @@ def online_read( result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] for entity_key in entity_keys: - document_id = compute_datastore_entity_id(entity_key) + document_id = compute_entity_id(entity_key) key = client.key( "Project", feast_project, "Table", table.name, "Row", document_id ) @@ -253,16 +252,6 @@ def online_read( return result -def compute_datastore_entity_id(entity_key: EntityKeyProto) -> str: - """ - Compute Datastore Entity id given Feast Entity Key. - - Remember that Datastore Entity is a concept from the Datastore data model, that has nothing to - do with the Entity concept we have in Feast. - """ - return mmh3.hash_bytes(serialize_entity_key(entity_key)).hex() - - def _delete_all_values(client, key) -> None: """ Delete all data under the key path in datastore. diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py new file mode 100644 index 00000000000..722a081f2ee --- /dev/null +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -0,0 +1,182 @@ +# Copyright 2021 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union + +from pydantic import StrictStr +from pydantic.typing import Literal + +from feast import Entity, FeatureTable, FeatureView, utils +from feast.infra.online_stores.helpers import compute_entity_id +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel, RepoConfig + +try: + import boto3 + from botocore.exceptions import ClientError +except ImportError as e: + from feast.errors import FeastExtrasDependencyImportError + + raise FeastExtrasDependencyImportError("aws", str(e)) + + +class DynamoDBOnlineStoreConfig(FeastConfigBaseModel): + """Online store config for DynamoDB store""" + + type: Literal["dynamodb"] = "dynamodb" + """Online store type selector""" + + region: StrictStr + """ AWS Region Name """ + + +class DynamoDBOnlineStore(OnlineStore): + """ + Online feature store for AWS DynamoDB. + """ + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[Union[FeatureTable, FeatureView]], + tables_to_keep: Sequence[Union[FeatureTable, FeatureView]], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + online_config = config.online_store + assert isinstance(online_config, DynamoDBOnlineStoreConfig) + dynamodb_client, dynamodb_resource = self._initialize_dynamodb(online_config) + + for table_instance in tables_to_keep: + try: + dynamodb_resource.create_table( + TableName=f"{config.project}.{table_instance.name}", + KeySchema=[{"AttributeName": "entity_id", "KeyType": "HASH"}], + AttributeDefinitions=[ + {"AttributeName": "entity_id", "AttributeType": "S"} + ], + BillingMode="PAY_PER_REQUEST", + ) + except ClientError as ce: + # If the table creation fails with ResourceInUseException, + # it means the table already exists or is being created. + # Otherwise, re-raise the exception + if ce.response["Error"]["Code"] != "ResourceInUseException": + raise + + for table_instance in tables_to_keep: + dynamodb_client.get_waiter("table_exists").wait( + TableName=f"{config.project}.{table_instance.name}" + ) + + self._delete_tables_idempotent(dynamodb_resource, config, tables_to_delete) + + def teardown( + self, + config: RepoConfig, + tables: Sequence[Union[FeatureTable, FeatureView]], + entities: Sequence[Entity], + ): + online_config = config.online_store + assert isinstance(online_config, DynamoDBOnlineStoreConfig) + _, dynamodb_resource = self._initialize_dynamodb(online_config) + + self._delete_tables_idempotent(dynamodb_resource, config, tables) + + def online_write_batch( + self, + config: RepoConfig, + table: Union[FeatureTable, FeatureView], + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + online_config = config.online_store + assert isinstance(online_config, DynamoDBOnlineStoreConfig) + _, dynamodb_resource = self._initialize_dynamodb(online_config) + + table_instance = dynamodb_resource.Table(f"{config.project}.{table.name}") + with table_instance.batch_writer() as batch: + for entity_key, features, timestamp, created_ts in data: + entity_id = compute_entity_id(entity_key) + batch.put_item( + Item={ + "entity_id": entity_id, # PartitionKey + "event_ts": str(utils.make_tzaware(timestamp)), + "values": { + k: v.SerializeToString() + for k, v in features.items() # Serialized Features + }, + } + ) + if progress: + progress(1) + + def online_read( + self, + config: RepoConfig, + table: Union[FeatureTable, FeatureView], + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + online_config = config.online_store + assert isinstance(online_config, DynamoDBOnlineStoreConfig) + _, dynamodb_resource = self._initialize_dynamodb(online_config) + + result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + for entity_key in entity_keys: + table_instance = dynamodb_resource.Table(f"{config.project}.{table.name}") + entity_id = compute_entity_id(entity_key) + response = table_instance.get_item(Key={"entity_id": entity_id}) + value = response.get("Item") + + if value is not None: + res = {} + for feature_name, value_bin in value["values"].items(): + val = ValueProto() + val.ParseFromString(value_bin.value) + res[feature_name] = val + result.append((value["event_ts"], res)) + else: + result.append((None, None)) + return result + + def _initialize_dynamodb(self, online_config: DynamoDBOnlineStoreConfig): + return ( + boto3.client("dynamodb", region_name=online_config.region), + boto3.resource("dynamodb", region_name=online_config.region), + ) + + def _delete_tables_idempotent( + self, + dynamodb_resource, + config: RepoConfig, + tables: Sequence[Union[FeatureTable, FeatureView]], + ): + for table_instance in tables: + try: + table = dynamodb_resource.Table( + f"{config.project}.{table_instance.name}" + ) + table.delete() + except ClientError as ce: + # If the table deletion fails with ResourceNotFoundException, + # it means the table has already been deleted. + # Otherwise, re-raise the exception + if ce.response["Error"]["Code"] != "ResourceNotFoundException": + raise diff --git a/sdk/python/feast/infra/online_stores/helpers.py b/sdk/python/feast/infra/online_stores/helpers.py index 9c42c5ea002..788be68b8d5 100644 --- a/sdk/python/feast/infra/online_stores/helpers.py +++ b/sdk/python/feast/infra/online_stores/helpers.py @@ -5,6 +5,7 @@ import mmh3 from feast import errors +from feast.infra.key_encoding_utils import serialize_entity_key from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.storage.Redis_pb2 import RedisKeyV2 as RedisKeyProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto @@ -53,3 +54,12 @@ def _mmh3(key: str): """ key_hash = mmh3.hash(key, signed=False) return bytes.fromhex(struct.pack(" str: + """ + Compute Entity id given Feast Entity Key for online stores. + Remember that Entity here refers to `EntityKeyProto` which is used in some online stores to encode the keys. + It has nothing to do with the Entity concept we have in Feast. + """ + return mmh3.hash_bytes(serialize_entity_key(entity_key)).hex() diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index b9101cb79e4..83089d5a4c0 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -146,6 +146,10 @@ def get_provider(config: RepoConfig, repo_path: Path) -> Provider: from feast.infra.gcp import GcpProvider return GcpProvider(config) + elif config.provider == "aws": + from feast.infra.aws import AwsProvider + + return AwsProvider(config) elif config.provider == "local": from feast.infra.local import LocalProvider diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index 9500194d045..53c3cae1e71 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import uuid from abc import ABC, abstractmethod from datetime import datetime, timedelta @@ -25,6 +26,8 @@ EntityNotFoundException, FeatureTableNotFoundException, FeatureViewNotFoundException, + S3RegistryBucketForbiddenAccess, + S3RegistryBucketNotExist, ) from feast.feature_table import FeatureTable from feast.feature_view import FeatureView @@ -56,6 +59,8 @@ def __init__(self, registry_path: str, repo_path: Path, cache_ttl: timedelta): uri = urlparse(registry_path) if uri.scheme == "gs": self._registry_store: RegistryStore = GCSRegistryStore(registry_path) + elif uri.scheme == "s3": + self._registry_store = S3RegistryStore(registry_path) elif uri.scheme == "file" or uri.scheme == "": self._registry_store = LocalRegistryStore( repo_path=repo_path, registry_path_string=registry_path @@ -537,3 +542,73 @@ def _write_registry(self, registry_proto: RegistryProto): file_obj.seek(0) blob.upload_from_file(file_obj) return + + +class S3RegistryStore(RegistryStore): + def __init__(self, uri: str): + try: + import boto3 + except ImportError as e: + from feast.errors import FeastExtrasDependencyImportError + + raise FeastExtrasDependencyImportError("aws", str(e)) + self._uri = urlparse(uri) + self._bucket = self._uri.hostname + self._key = self._uri.path.lstrip("/") + + self.s3_client = boto3.resource( + "s3", endpoint_url=os.environ.get("FEAST_S3_ENDPOINT_URL") + ) + + def get_registry_proto(self): + file_obj = TemporaryFile() + registry_proto = RegistryProto() + try: + from botocore.exceptions import ClientError + except ImportError as e: + from feast.errors import FeastExtrasDependencyImportError + + raise FeastExtrasDependencyImportError("aws", str(e)) + try: + bucket = self.s3_client.Bucket(self._bucket) + self.s3_client.meta.client.head_bucket(Bucket=bucket.name) + except ClientError as e: + # If a client error is thrown, then check that it was a 404 error. + # If it was a 404 error, then the bucket does not exist. + error_code = int(e.response["Error"]["Code"]) + if error_code == 404: + raise S3RegistryBucketNotExist(self._bucket) + else: + raise S3RegistryBucketForbiddenAccess(self._bucket) from e + + try: + obj = bucket.Object(self._key) + obj.download_fileobj(file_obj) + file_obj.seek(0) + registry_proto.ParseFromString(file_obj.read()) + return registry_proto + except ClientError as e: + raise FileNotFoundError( + f"Error while trying to locate Registry at path {self._uri.geturl()}" + ) from e + + def update_registry_proto( + self, updater: Optional[Callable[[RegistryProto], RegistryProto]] = None + ): + try: + registry_proto = self.get_registry_proto() + except FileNotFoundError: + registry_proto = RegistryProto() + registry_proto.registry_schema_version = REGISTRY_SCHEMA_VERSION + if updater: + registry_proto = updater(registry_proto) + self._write_registry(registry_proto) + + def _write_registry(self, registry_proto: RegistryProto): + registry_proto.version_id = str(uuid.uuid4()) + registry_proto.last_updated.FromDatetime(datetime.utcnow()) + # we have already checked the bucket exists so no need to do it again + file_obj = TemporaryFile() + file_obj.write(registry_proto.SerializeToString()) + file_obj.seek(0) + self.s3_client.Bucket(self._bucket).put_object(Body=file_obj, Key=self._key) diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 8ef98736f9a..5cf17bf7296 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -16,6 +16,7 @@ "sqlite": "feast.infra.online_stores.sqlite.SqliteOnlineStore", "datastore": "feast.infra.online_stores.datastore.DatastoreOnlineStore", "redis": "feast.infra.online_stores.redis.RedisOnlineStore", + "dynamodb": "feast.infra.online_stores.dynamodb.DynamoDBOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { @@ -67,7 +68,7 @@ class RepoConfig(FeastBaseModel): """ provider: StrictStr - """ str: local or gcp """ + """ str: local or gcp or aws """ online_store: Any """ OnlineStoreConfig: Online store configuration (optional depending on provider) """ @@ -127,6 +128,8 @@ def _validate_online_store_config(cls, values): values["online_store"]["type"] = "sqlite" elif values["provider"] == "gcp": values["online_store"]["type"] = "datastore" + elif values["provider"] == "aws": + values["online_store"]["type"] = "dynamodb" online_store_type = values["online_store"]["type"] @@ -161,7 +164,7 @@ def _validate_offline_store_config(cls, values): elif values["provider"] == "gcp": values["offline_store"]["type"] = "bigquery" elif values["provider"] == "aws": - values["offline_store"]["type"] = "redshift" + values["offline_store"]["type"] = "file" offline_store_type = values["offline_store"]["type"] diff --git a/sdk/python/feast/templates/aws/bootstrap.py b/sdk/python/feast/templates/aws/bootstrap.py new file mode 100644 index 00000000000..4013ca5a8d6 --- /dev/null +++ b/sdk/python/feast/templates/aws/bootstrap.py @@ -0,0 +1,35 @@ +def bootstrap(): + # Bootstrap() will automatically be called from the init_repo() during `feast init` + + import pathlib + from datetime import datetime, timedelta + + from feast.driver_test_data import create_driver_hourly_stats_df + + repo_path = pathlib.Path(__file__).parent.absolute() + data_path = repo_path / "data" + data_path.mkdir(exist_ok=True) + + end_date = datetime.now().replace(microsecond=0, second=0, minute=0) + start_date = end_date - timedelta(days=15) + + driver_entities = [1001, 1002, 1003, 1004, 1005] + driver_df = create_driver_hourly_stats_df(driver_entities, start_date, end_date) + + driver_stats_path = data_path / "driver_stats.parquet" + driver_df.to_parquet(path=str(driver_stats_path), allow_truncated_timestamps=True) + + example_py_file = repo_path / "example.py" + replace_str_in_file(example_py_file, "%PARQUET_PATH%", str(driver_stats_path)) + + +def replace_str_in_file(file_path, match_str, sub_str): + with open(file_path, "r") as f: + contents = f.read() + contents = contents.replace(match_str, sub_str) + with open(file_path, "wt") as f: + f.write(contents) + + +if __name__ == "__main__": + bootstrap() diff --git a/sdk/python/feast/templates/aws/example.py b/sdk/python/feast/templates/aws/example.py new file mode 100644 index 00000000000..a66dbba1205 --- /dev/null +++ b/sdk/python/feast/templates/aws/example.py @@ -0,0 +1,36 @@ +# This is an example feature definition file + +from google.protobuf.duration_pb2 import Duration + +from feast import Entity, Feature, FeatureView, ValueType +from feast.data_source import FileSource + +# Read data from parquet files. Parquet is convenient for local development mode. For +# production, you can use your favorite DWH, such as BigQuery. See Feast documentation +# for more info. +driver_hourly_stats = FileSource( + path="%PARQUET_PATH%", + event_timestamp_column="datetime", + created_timestamp_column="created", +) + +# Define an entity for the driver. You can think of entity as a primary key used to +# fetch features. +driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id",) + +# Our parquet files contain sample data that includes a driver_id column, timestamps and +# three feature column. Here we define a Feature View that will allow us to serve this +# data to our model online. +driver_hourly_stats_view = FeatureView( + name="driver_hourly_stats", + entities=["driver_id"], + ttl=Duration(seconds=86400 * 1), + features=[ + Feature(name="conv_rate", dtype=ValueType.FLOAT), + Feature(name="acc_rate", dtype=ValueType.FLOAT), + Feature(name="avg_daily_trips", dtype=ValueType.INT64), + ], + online=True, + input=driver_hourly_stats, + tags={}, +) diff --git a/sdk/python/feast/templates/aws/feature_store.yaml b/sdk/python/feast/templates/aws/feature_store.yaml new file mode 100644 index 00000000000..7f7be8527ef --- /dev/null +++ b/sdk/python/feast/templates/aws/feature_store.yaml @@ -0,0 +1,3 @@ +project: my_project +registry: data/registry.db +provider: aws diff --git a/sdk/python/feast/templates/aws/test.py b/sdk/python/feast/templates/aws/test.py new file mode 100644 index 00000000000..cc2cf7e984a --- /dev/null +++ b/sdk/python/feast/templates/aws/test.py @@ -0,0 +1,38 @@ +from datetime import datetime + +import pandas as pd +from example import driver, driver_hourly_stats_view + +from feast import FeatureStore + + +def main(): + pd.set_option("display.max_columns", None) + pd.set_option("display.width", 1000) + + # Load the feature store from the current path + fs = FeatureStore(repo_path=".") + + # Deploy the feature store to AWS + print("Deploying feature store to AWS...") + fs.apply([driver, driver_hourly_stats_view]) + + # Select features + feature_refs = ["driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate"] + + print("Loading features into the online store...") + fs.materialize_incremental(end_date=datetime.now()) + + print("Retrieving online features...") + + # Retrieve features from the online store (DynamoDB) + online_features = fs.get_online_features( + feature_refs=feature_refs, + entity_rows=[{"driver_id": 1001}, {"driver_id": 1002}], + ).to_dict() + + print(pd.DataFrame.from_dict(online_features)) + + +if __name__ == "__main__": + main() diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 293e6804e77..bd519561602 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -71,6 +71,10 @@ "redis-py-cluster==2.1.2", ] +AWS_REQUIRED = [ + "boto3==1.17.*", +] + CI_REQUIRED = [ "cryptography==3.3.2", "flake8", @@ -104,8 +108,10 @@ "google-cloud-storage>=1.20.*", "google-cloud-core==1.4.*", "redis-py-cluster==2.1.2", + "boto3==1.17.*", ] + # README file from Feast repo root directory repo_root = ( subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE) @@ -198,6 +204,7 @@ def run(self): "dev": ["mypy-protobuf==1.*", "grpcio-testing==1.*"], "ci": CI_REQUIRED, "gcp": GCP_REQUIRED, + "aws": AWS_REQUIRED, "redis": REDIS_REQUIRED, }, include_package_data=True, diff --git a/sdk/python/tests/test_cli_aws.py b/sdk/python/tests/test_cli_aws.py new file mode 100644 index 00000000000..2792858e8d7 --- /dev/null +++ b/sdk/python/tests/test_cli_aws.py @@ -0,0 +1,58 @@ +import random +import string +import tempfile +from pathlib import Path +from textwrap import dedent + +import pytest + +from feast.feature_store import FeatureStore +from tests.cli_utils import CliRunner +from tests.online_read_write_test import basic_rw_test + + +@pytest.mark.integration +def test_basic() -> None: + project_id = "".join( + random.choice(string.ascii_lowercase + string.digits) for _ in range(10) + ) + runner = CliRunner() + with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name: + + repo_path = Path(repo_dir_name) + data_path = Path(data_dir_name) + + repo_config = repo_path / "feature_store.yaml" + + repo_config.write_text( + dedent( + f""" + project: {project_id} + registry: {data_path / "registry.db"} + provider: aws + online_store: + type: dynamodb + region: us-west-2 + """ + ) + ) + + repo_example = repo_path / "example.py" + repo_example.write_text( + (Path(__file__).parent / "example_feature_repo_1.py").read_text() + ) + + result = runner.run(["apply"], cwd=repo_path) + assert result.returncode == 0 + + # Doing another apply should be a no op, and should not cause errors + result = runner.run(["apply"], cwd=repo_path) + assert result.returncode == 0 + + basic_rw_test( + FeatureStore(repo_path=str(repo_path), config=None), + view_name="driver_locations", + ) + + result = runner.run(["teardown"], cwd=repo_path) + assert result.returncode == 0 diff --git a/sdk/python/tests/test_feature_store.py b/sdk/python/tests/test_feature_store.py index 49a3a9a63b0..f169c1336a2 100644 --- a/sdk/python/tests/test_feature_store.py +++ b/sdk/python/tests/test_feature_store.py @@ -24,6 +24,8 @@ from feast.feature import Feature from feast.feature_store import FeatureStore from feast.feature_view import FeatureView +from feast.infra.offline_stores.file import FileOfflineStoreConfig +from feast.infra.online_stores.dynamodb import DynamoDBOnlineStoreConfig from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig from feast.protos.feast.types import Value_pb2 as ValueProto from feast.repo_config import RepoConfig @@ -72,6 +74,19 @@ def feature_store_with_gcs_registry(): ) +@pytest.fixture +def feature_store_with_s3_registry(): + return FeatureStore( + config=RepoConfig( + registry=f"s3://feast-integration-tests/registries/{int(time.time() * 1000)}/registry.db", + project="default", + provider="aws", + online_store=DynamoDBOnlineStoreConfig(region="us-west-2"), + offline_store=FileOfflineStoreConfig(), + ) + ) + + @pytest.mark.parametrize( "test_feature_store", [lazy_fixture("feature_store_with_local_registry")], ) @@ -101,7 +116,11 @@ def test_apply_entity_success(test_feature_store): @pytest.mark.integration @pytest.mark.parametrize( - "test_feature_store", [lazy_fixture("feature_store_with_gcs_registry")], + "test_feature_store", + [ + lazy_fixture("feature_store_with_gcs_registry"), + lazy_fixture("feature_store_with_s3_registry"), + ], ) def test_apply_entity_integration(test_feature_store): entity = Entity( @@ -250,7 +269,11 @@ def test_feature_view_inference_success(test_feature_store, dataframe_source): @pytest.mark.integration @pytest.mark.parametrize( - "test_feature_store", [lazy_fixture("feature_store_with_gcs_registry")], + "test_feature_store", + [ + lazy_fixture("feature_store_with_gcs_registry"), + lazy_fixture("feature_store_with_s3_registry"), + ], ) def test_apply_feature_view_integration(test_feature_store): # Create Feature Views diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py index 2cc85a304a9..3b780337e9e 100644 --- a/sdk/python/tests/test_offline_online_store_consistency.py +++ b/sdk/python/tests/test_offline_online_store_consistency.py @@ -18,7 +18,9 @@ from feast.feature import Feature from feast.feature_store import FeatureStore from feast.feature_view import FeatureView +from feast.infra.offline_stores.file import FileOfflineStoreConfig from feast.infra.online_stores.datastore import DatastoreOnlineStoreConfig +from feast.infra.online_stores.dynamodb import DynamoDBOnlineStoreConfig from feast.infra.online_stores.redis import RedisOnlineStoreConfig, RedisType from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig from feast.repo_config import RepoConfig @@ -167,7 +169,7 @@ def prep_redis_fs_and_fv() -> Iterator[Tuple[FeatureStore, FeatureView]]: join_key="driver_id", value_type=ValueType.INT32, ) - with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory(): + with tempfile.TemporaryDirectory() as repo_dir_name: config = RepoConfig( registry=str(Path(repo_dir_name) / "registry.db"), project=f"test_bq_correctness_{str(uuid.uuid4()).replace('-', '')}", @@ -184,6 +186,41 @@ def prep_redis_fs_and_fv() -> Iterator[Tuple[FeatureStore, FeatureView]]: yield fs, fv +@contextlib.contextmanager +def prep_dynamodb_fs_and_fv() -> Iterator[Tuple[FeatureStore, FeatureView]]: + with tempfile.NamedTemporaryFile(suffix=".parquet") as f: + df = create_dataset() + f.close() + df.to_parquet(f.name) + file_source = FileSource( + file_format=ParquetFormat(), + file_url=f"file://{f.name}", + event_timestamp_column="ts", + created_timestamp_column="created_ts", + date_partition_column="", + field_mapping={"ts_1": "ts", "id": "driver_id"}, + ) + fv = get_feature_view(file_source) + e = Entity( + name="driver", + description="id for driver", + join_key="driver_id", + value_type=ValueType.INT32, + ) + with tempfile.TemporaryDirectory() as repo_dir_name: + config = RepoConfig( + registry=str(Path(repo_dir_name) / "registry.db"), + project=f"test_bq_correctness_{str(uuid.uuid4()).replace('-', '')}", + provider="aws", + online_store=DynamoDBOnlineStoreConfig(region="us-west-2"), + offline_store=FileOfflineStoreConfig(), + ) + fs = FeatureStore(config=config) + fs.apply([fv, e]) + + yield fs, fv + + # Checks that both offline & online store values are as expected def check_offline_and_online_features( fs: FeatureStore, @@ -304,8 +341,13 @@ def test_redis_offline_online_store_consistency(full_feature_names: bool): with prep_redis_fs_and_fv() as (fs, fv): run_offline_online_store_consistency_test(fs, fv, full_feature_names) +@pytest.mark.parametrize("full_feature_names", [True, False]) +@pytest.mark.integration +def test_dynamodb_offline_online_store_consistency(full_feature_names:bool): + with prep_dynamodb_fs_and_fv() as (fs, fv): + run_offline_online_store_consistency_test(fs, fv, full_feature_names) @pytest.mark.parametrize("full_feature_names", [True, False]) -def test_local_offline_online_store_consistency(full_feature_names: bool): +def test_local_offline_online_store_consistency(full_feature_names:bool): with prep_local_fs_and_fv() as (fs, fv): run_offline_online_store_consistency_test(fs, fv, full_feature_names) From d36d1a05fce5137cafd856dcbc9ebd98ae702805 Mon Sep 17 00:00:00 2001 From: Tsotne Tabidze Date: Sun, 4 Jul 2021 15:45:55 -0700 Subject: [PATCH 33/43] Parallelize integration tests (#1684) * Parallelize integration tests Signed-off-by: Tsotne Tabidze * Update the usage flag Signed-off-by: Tsotne Tabidze Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- .github/workflows/integration_tests.yml | 2 +- .github/workflows/pr_integration_tests.yml | 2 +- .github/workflows/unit_tests.yml | 2 +- Makefile | 5 ++++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d82d882ff26..322ee0b6e51 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -52,7 +52,7 @@ jobs: - name: Install dependencies run: make install-python-ci-dependencies - name: Test python - run: FEAST_TELEMETRY=False pytest --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration + run: FEAST_USAGE=False pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 240d87069ec..26ac821ac2b 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -63,7 +63,7 @@ jobs: - name: Install dependencies run: make install-python-ci-dependencies - name: Test python - run: FEAST_TELEMETRY=False pytest --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration + run: FEAST_USAGE=False pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 58d2a7c0a7a..7b480a86094 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -22,7 +22,7 @@ jobs: - name: Install dependencies run: make install-python-ci-dependencies - name: Test Python - run: make test-python + run: FEAST_USAGE=False pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: diff --git a/Makefile b/Makefile index 475d729e9e9..b18c5bca2ef 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,10 @@ install-python: python -m pip install -e sdk/python -U --use-deprecated=legacy-resolver test-python: - FEAST_TELEMETRY=False pytest -n 4 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests + FEAST_USAGE=False pytest -n 8 sdk/python/tests + +test-python-integration: + FEAST_USAGE=False pytest -n 8 --integration sdk/python/tests format-python: # Sort From 651bce3ce688cd9f4a29366a4157606ef4351698 Mon Sep 17 00:00:00 2001 From: Matt Delacour Date: Mon, 5 Jul 2021 11:45:26 -0400 Subject: [PATCH 34/43] BQ exception should be raised first before we check the timedout (#1675) Signed-off-by: Matt Delacour Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/infra/offline_stores/bigquery.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 55dc2d9cd45..e674786864c 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -288,13 +288,13 @@ def _wait_until_done(job_id): job_id = bq_job.job_id _wait_until_done(job_id=job_id) + if bq_job.exception(): + raise bq_job.exception() + if not _is_done(job_id): client.cancel_job(job_id) raise BigQueryJobCancelled(job_id=job_id) - if bq_job.exception(): - raise bq_job.exception() - @dataclass(frozen=True) class FeatureViewQueryContext: From f3b92c309f514e26401ffc93d85b85126befa0d4 Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Mon, 5 Jul 2021 14:44:42 -0400 Subject: [PATCH 35/43] Update sdk/python/feast/infra/provider.py Co-authored-by: Willem Pienaar <6728866+woop@users.noreply.github.com> Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/infra/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 83089d5a4c0..6d75e9741f4 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -170,7 +170,7 @@ def _get_requested_feature_views_to_features_dict( feature_refs: List[str], feature_views: List[FeatureView] ) -> Dict[FeatureView, List[str]]: """Create a dict of FeatureView -> List[Feature] for all requested features. - Set full_feature_names to True to get feature names prefixed by its featureview.""" + Set full_feature_names to True to have feature names prefixed by their feature view name.""" feature_views_to_feature_map = {} # type: Dict[FeatureView, List[str]] From f400d65cdee4b6beae134a44de341a306fd43ce1 Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Mon, 5 Jul 2021 14:44:48 -0400 Subject: [PATCH 36/43] Update sdk/python/feast/feature_store.py Co-authored-by: Willem Pienaar <6728866+woop@users.noreply.github.com> Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/feature_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index e32221b7592..bb350ca5539 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -614,7 +614,7 @@ def _entity_row_to_field_values( def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = False): feature_names = [ref.split(":")[1] for ref in feature_refs] - feature_name, count = Counter(feature_names).most_common(1)[0] + feature_name, occurrences = Counter(feature_names).most_common(1)[0] if count > 1: collided_feature_refs = [ ref for ref in feature_refs if ref.endswith(":" + feature_name) From 082fca7169cf2440d82e6fa7fbe7019916214d83 Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Mon, 5 Jul 2021 18:19:00 -0400 Subject: [PATCH 37/43] made error logic/messages more descriptive Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/errors.py | 11 +++++++--- sdk/python/feast/feature_store.py | 19 +++++++++++----- sdk/python/feast/infra/aws.py | 2 ++ sdk/python/tests/test_historical_retrieval.py | 22 ++++++++++++++----- .../test_offline_online_store_consistency.py | 7 +++--- 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index d259fb342d0..31992fdf832 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -90,11 +90,16 @@ def __init__(self, offline_store_name: str, data_source_name: str): class FeatureNameCollisionError(Exception): def __init__(self, feature_refs_collisions: List[str]): - feature_name_collisions = [ref.split(":")[1] for ref in feature_refs_collisions] - feature_names = ", ".join(x for x in feature_name_collisions) + collisions = [ref.split(":", 1) for ref in feature_refs_collisions] + collision_feature_views = [y for (y, _) in collisions] + collision_feature_names = [x for (_, x) in collisions] + feature_names = ", ".join(set(collision_feature_names)) + feature_views = ", ".join(set(collision_feature_views)) super().__init__( f"The following feature name(s) have collisions: {feature_names}. Set 'full_feature_names' " - f"argument in the data retrieval function to True to use the full feature name which is prefixed by the feature view name." + f"argument in the data retrieval function to True to use the full feature name which is prefixed " + f"by the feature view name, or rename colliding features.\nCollisions occur in the following " + f"feature view(s): {feature_views}." ) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index bb350ca5539..b4ab5e43210 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -553,6 +553,7 @@ def get_online_features( project=self.project, allow_cache=True ) + _validate_feature_refs(feature_refs, full_feature_names) grouped_refs = _group_feature_refs(feature_refs, all_feature_views) for table, requested_features in grouped_refs: entity_keys = _get_table_entity_keys( @@ -613,13 +614,19 @@ def _entity_row_to_field_values( def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = False): - feature_names = [ref.split(":")[1] for ref in feature_refs] - feature_name, occurrences = Counter(feature_names).most_common(1)[0] - if count > 1: + if full_feature_names: collided_feature_refs = [ - ref for ref in feature_refs if ref.endswith(":" + feature_name) - ] - raise FeatureNameCollisionError(collided_feature_refs) + ref for ref, occurrences in Counter(feature_refs).items() if occurrences > 1] + if len(collided_feature_refs) > 0: + raise FeatureNameCollisionError(collided_feature_refs) + else: + feature_names = [ref.split(":")[1] for ref in feature_refs] + feature_name, occurrences = Counter(feature_names).most_common(1)[0] + if occurrences > 1: + collided_feature_refs = [ + ref for ref in feature_refs if ref.endswith(":" + feature_name) + ] + raise FeatureNameCollisionError(collided_feature_refs) def _group_feature_refs( diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py index 272f39840e6..e28d2933642 100644 --- a/sdk/python/feast/infra/aws.py +++ b/sdk/python/feast/infra/aws.py @@ -129,6 +129,7 @@ def get_historical_features( entity_df: Union[pandas.DataFrame, str], registry: Registry, project: str, + full_feature_names: bool = False, ) -> RetrievalJob: job = self.offline_store.get_historical_features( config=config, @@ -137,5 +138,6 @@ def get_historical_features( entity_df=entity_df, registry=registry, project=project, + full_feature_names=full_feature_names, ) return job diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index 2ad6c4d4e12..9cae4dfc9aa 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -204,12 +204,22 @@ def get_expected_training_df( expected_df = expected_df[[event_timestamp] + current_cols] # Cast some columns to expected types, since we lose information when converting pandas DFs into Python objects. - expected_column_types = { - "order_is_success": "int32", - "driver_stats__conv_rate": "float32", - "customer_profile__current_balance": "float32", - "customer_profile__avg_passenger_count": "float32", - } + + if full_feature_names: + expected_column_types = { + "order_is_success": "int32", + "driver_stats__conv_rate": "float32", + "customer_profile__current_balance": "float32", + "customer_profile__avg_passenger_count": "float32", + } + else: + expected_column_types = { + "order_is_success": "int32", + "conv_rate": "float32", + "current_balance": "float32", + "avg_passenger_count": "float32", + } + for col, typ in expected_column_types.items(): expected_df[col] = expected_df[col].astype(typ) diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py index 3b780337e9e..3e1cb1a9c7b 100644 --- a/sdk/python/tests/test_offline_online_store_consistency.py +++ b/sdk/python/tests/test_offline_online_store_consistency.py @@ -267,7 +267,6 @@ def check_offline_and_online_features( assert abs(df.to_dict()["value"][0] - expected_value) < 1e-6 else: assert math.isnan(df.to_dict()["value"][0]) - def run_offline_online_store_consistency_test( @@ -341,13 +340,15 @@ def test_redis_offline_online_store_consistency(full_feature_names: bool): with prep_redis_fs_and_fv() as (fs, fv): run_offline_online_store_consistency_test(fs, fv, full_feature_names) + @pytest.mark.parametrize("full_feature_names", [True, False]) @pytest.mark.integration -def test_dynamodb_offline_online_store_consistency(full_feature_names:bool): +def test_dynamodb_offline_online_store_consistency(full_feature_names: bool): with prep_dynamodb_fs_and_fv() as (fs, fv): run_offline_online_store_consistency_test(fs, fv, full_feature_names) + @pytest.mark.parametrize("full_feature_names", [True, False]) -def test_local_offline_online_store_consistency(full_feature_names:bool): +def test_local_offline_online_store_consistency(full_feature_names: bool): with prep_local_fs_and_fv() as (fs, fv): run_offline_online_store_consistency_test(fs, fv, full_feature_names) From 3aca9760bf8623e53111257ab052836fa9638bc2 Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Mon, 5 Jul 2021 18:23:22 -0400 Subject: [PATCH 38/43] made error logic/messages more descriptive. Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/feature_store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index b4ab5e43210..abef0cc959b 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -616,7 +616,8 @@ def _entity_row_to_field_values( def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = False): if full_feature_names: collided_feature_refs = [ - ref for ref, occurrences in Counter(feature_refs).items() if occurrences > 1] + ref for ref, occurrences in Counter(feature_refs).items() if occurrences > 1 + ] if len(collided_feature_refs) > 0: raise FeatureNameCollisionError(collided_feature_refs) else: From 79aa7364284f3722c225778c959920ebc06fee4c Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Mon, 5 Jul 2021 22:37:26 -0400 Subject: [PATCH 39/43] Simplified error messages Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/errors.py | 26 ++++++++++++++++---------- sdk/python/feast/feature_store.py | 23 +++++++++++++++-------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 31992fdf832..cf2161fff39 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -89,17 +89,23 @@ def __init__(self, offline_store_name: str, data_source_name: str): class FeatureNameCollisionError(Exception): - def __init__(self, feature_refs_collisions: List[str]): - collisions = [ref.split(":", 1) for ref in feature_refs_collisions] - collision_feature_views = [y for (y, _) in collisions] - collision_feature_names = [x for (_, x) in collisions] - feature_names = ", ".join(set(collision_feature_names)) - feature_views = ", ".join(set(collision_feature_views)) + def __init__(self, feature_refs_collisions: List[str], full_feature_names): + if full_feature_names: + collisions = [ref.replace(":", "__") for ref in feature_refs_collisions] + error_message = ( + "To resolve this collision, please ensure that the features in question " + "have different names." + ) + else: + collisions = [ref.split(":")[1] for ref in feature_refs_collisions] + error_message = ( + "To resolve this collision, either use the full feature name by setting " + "'full_feature_names=True', or ensure that the features in question have different names." + ) + + feature_names = ", ".join(set(collisions)) super().__init__( - f"The following feature name(s) have collisions: {feature_names}. Set 'full_feature_names' " - f"argument in the data retrieval function to True to use the full feature name which is prefixed " - f"by the feature view name, or rename colliding features.\nCollisions occur in the following " - f"feature view(s): {feature_views}." + f"Duplicate features named {feature_names} found.\n{error_message}" ) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index abef0cc959b..0d8617f1f11 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -614,20 +614,27 @@ def _entity_row_to_field_values( def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = False): + collided_feature_refs = [] + if full_feature_names: collided_feature_refs = [ ref for ref, occurrences in Counter(feature_refs).items() if occurrences > 1 ] - if len(collided_feature_refs) > 0: - raise FeatureNameCollisionError(collided_feature_refs) else: feature_names = [ref.split(":")[1] for ref in feature_refs] - feature_name, occurrences = Counter(feature_names).most_common(1)[0] - if occurrences > 1: - collided_feature_refs = [ - ref for ref in feature_refs if ref.endswith(":" + feature_name) - ] - raise FeatureNameCollisionError(collided_feature_refs) + collided_feature_names = [ + ref + for ref, occurrences in Counter(feature_names).items() + if occurrences > 1 + ] + + for feature_name in collided_feature_names: + collided_feature_refs.extend( + [ref for ref in feature_refs if ref.endswith(":" + feature_name)] + ) + + if len(collided_feature_refs) > 0: + raise FeatureNameCollisionError(collided_feature_refs, full_feature_names) def _group_feature_refs( From d7d08efcfaa002be77311875c9a4a211eb5b5f9d Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Wed, 7 Jul 2021 17:47:06 -0400 Subject: [PATCH 40/43] ran formatter, issue in errors.py Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/errors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index cf2161fff39..125b309c304 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -109,7 +109,6 @@ def __init__(self, feature_refs_collisions: List[str], full_feature_names): ) - class FeastOnlineStoreInvalidName(Exception): def __init__(self, online_store_class_name: str): super().__init__( From 650340dbc7f6bb290a794db17c96c81a5bba1b29 Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Wed, 7 Jul 2021 18:58:54 -0400 Subject: [PATCH 41/43] python linter issues resolved Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/feature_store.py | 2 +- sdk/python/tests/test_offline_online_store_consistency.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index d4de29205fa..5d8e4604dca 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -229,7 +229,7 @@ def apply( update_entities_with_inferred_types_from_feature_views( entities_to_update, views_to_update, self.config ) - + update_data_sources_with_inferred_event_timestamp_col( [view.input for view in views_to_update], self.config ) diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py index 5af2c9524c5..3e1cb1a9c7b 100644 --- a/sdk/python/tests/test_offline_online_store_consistency.py +++ b/sdk/python/tests/test_offline_online_store_consistency.py @@ -269,7 +269,6 @@ def check_offline_and_online_features( assert math.isnan(df.to_dict()["value"][0]) - def run_offline_online_store_consistency_test( fs: FeatureStore, fv: FeatureView, full_feature_names: bool ) -> None: @@ -348,7 +347,7 @@ def test_dynamodb_offline_online_store_consistency(full_feature_names: bool): with prep_dynamodb_fs_and_fv() as (fs, fv): run_offline_online_store_consistency_test(fs, fv, full_feature_names) - + @pytest.mark.parametrize("full_feature_names", [True, False]) def test_local_offline_online_store_consistency(full_feature_names: bool): with prep_local_fs_and_fv() as (fs, fv): From 5d582a6917658cef66ce4b042de95935dc606d82 Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Wed, 7 Jul 2021 20:08:30 -0400 Subject: [PATCH 42/43] removed unnecessary default assignment in get_historical_features. default now set only in feature_store.py Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/feature_view.py | 2 -- sdk/python/feast/infra/aws.py | 2 +- sdk/python/feast/infra/gcp.py | 2 +- sdk/python/feast/infra/local.py | 2 +- sdk/python/feast/infra/provider.py | 2 +- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index c9cac6a29c6..9728ad20925 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -182,8 +182,6 @@ def to_proto(self) -> FeatureViewProto: ttl_duration = Duration() ttl_duration.FromTimedelta(self.ttl) - print(f"Stream soruce: {self.stream_source}, {type(self.stream_source)}") - spec = FeatureViewSpecProto( name=self.name, entities=self.entities, diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py index e28d2933642..f182bbbcee9 100644 --- a/sdk/python/feast/infra/aws.py +++ b/sdk/python/feast/infra/aws.py @@ -129,7 +129,7 @@ def get_historical_features( entity_df: Union[pandas.DataFrame, str], registry: Registry, project: str, - full_feature_names: bool = False, + full_feature_names: bool, ) -> RetrievalJob: job = self.offline_store.get_historical_features( config=config, diff --git a/sdk/python/feast/infra/gcp.py b/sdk/python/feast/infra/gcp.py index 9af520d7aae..2662a6e54fa 100644 --- a/sdk/python/feast/infra/gcp.py +++ b/sdk/python/feast/infra/gcp.py @@ -131,7 +131,7 @@ def get_historical_features( entity_df: Union[pandas.DataFrame, str], registry: Registry, project: str, - full_feature_names: bool = False, + full_feature_names: bool, ) -> RetrievalJob: job = self.offline_store.get_historical_features( config=config, diff --git a/sdk/python/feast/infra/local.py b/sdk/python/feast/infra/local.py index d8bc0c91fdc..f677c846724 100644 --- a/sdk/python/feast/infra/local.py +++ b/sdk/python/feast/infra/local.py @@ -130,7 +130,7 @@ def get_historical_features( entity_df: Union[pd.DataFrame, str], registry: Registry, project: str, - full_feature_names: bool = False, + full_feature_names: bool, ) -> RetrievalJob: return self.offline_store.get_historical_features( config=config, diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 6d75e9741f4..2775c48173b 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -116,7 +116,7 @@ def get_historical_features( entity_df: Union[pandas.DataFrame, str], registry: Registry, project: str, - full_feature_names: bool = False, + full_feature_names: bool, ) -> RetrievalJob: pass From 8724e0b09419cbd15d875c28a4cedc6dd283368e Mon Sep 17 00:00:00 2001 From: Mwad22 <51929507+Mwad22@users.noreply.github.com> Date: Wed, 7 Jul 2021 20:44:19 -0400 Subject: [PATCH 43/43] added error message assertion for feature name collisions, and other nitpick changes Signed-off-by: Mwad22 <51929507+Mwad22@users.noreply.github.com> --- sdk/python/feast/errors.py | 2 +- .../feast/infra/offline_stores/bigquery.py | 6 +++- sdk/python/tests/test_historical_retrieval.py | 34 +++++++++++++++++-- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 125b309c304..b855dd57ed2 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -89,7 +89,7 @@ def __init__(self, offline_store_name: str, data_source_name: str): class FeatureNameCollisionError(Exception): - def __init__(self, feature_refs_collisions: List[str], full_feature_names): + def __init__(self, feature_refs_collisions: List[str], full_feature_names: bool): if full_feature_names: collisions = [ref.replace(":", "__") for ref in feature_refs_collisions] error_message = ( diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index e674786864c..00b4c06a24b 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -122,7 +122,11 @@ def get_historical_features( # Build a query context containing all information required to template the BigQuery SQL query query_context = get_feature_view_query_context( - feature_refs, feature_views, registry, project, full_feature_names + feature_refs, + feature_views, + registry, + project, + full_feature_names=full_feature_names, ) # TODO: Infer min_timestamp and max_timestamp from entity_df diff --git a/sdk/python/tests/test_historical_retrieval.py b/sdk/python/tests/test_historical_retrieval.py index caebf73f529..3a708c7503a 100644 --- a/sdk/python/tests/test_historical_retrieval.py +++ b/sdk/python/tests/test_historical_retrieval.py @@ -595,11 +595,11 @@ def test_historical_features_from_bigquery_sources( ) -@pytest.mark.integration def test_feature_name_collision_on_historical_retrieval(): # _validate_feature_refs is the function that checks for colliding feature names - with pytest.raises(FeatureNameCollisionError): + # check when feature names collide and 'full_feature_names=False' + with pytest.raises(FeatureNameCollisionError) as error: _validate_feature_refs( feature_refs=[ "driver_stats:conv_rate", @@ -611,3 +611,33 @@ def test_feature_name_collision_on_historical_retrieval(): ], full_feature_names=False, ) + + expected_error_message = ( + "Duplicate features named avg_daily_trips found.\n" + "To resolve this collision, either use the full feature name by setting " + "'full_feature_names=True', or ensure that the features in question have different names." + ) + + assert str(error.value) == expected_error_message + + # check when feature names collide and 'full_feature_names=True' + with pytest.raises(FeatureNameCollisionError) as error: + _validate_feature_refs( + feature_refs=[ + "driver_stats:conv_rate", + "driver_stats:avg_daily_trips", + "driver_stats:avg_daily_trips", + "customer_profile:current_balance", + "customer_profile:avg_passenger_count", + "customer_profile:lifetime_trip_count", + "customer_profile:avg_daily_trips", + ], + full_feature_names=True, + ) + + expected_error_message = ( + "Duplicate features named driver_stats__avg_daily_trips found.\n" + "To resolve this collision, please ensure that the features in question " + "have different names." + ) + assert str(error.value) == expected_error_message