From 1b7c0fcd860355e5d1a35c7109be68b073be79c1 Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Tue, 22 Mar 2022 12:27:46 -0700 Subject: [PATCH 1/5] ci: Pin setup-gcloud actions to v0 instead of master (#2434) Signed-off-by: Felix Wang --- .github/workflows/java_master_only.yml | 2 +- .github/workflows/master_only.yml | 2 +- .github/workflows/publish.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 04313c4120c..a856fbe2cba 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v2 with: submodules: 'true' - - uses: google-github-actions/setup-gcloud@master + - uses: google-github-actions/setup-gcloud@v0 with: version: '290.0.1' export_default_credentials: true diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index e7a89815fe4..3e5c690598e 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -173,7 +173,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0d7476a82ba..93385dcde9b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -64,7 +64,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} @@ -107,7 +107,7 @@ jobs: VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} steps: - uses: actions/checkout@v2 - - uses: google-github-actions/setup-gcloud@master + - uses: google-github-actions/setup-gcloud@v0 with: version: '290.0.1' export_default_credentials: true From 88e01a2545f694534fb60feb0d55f1e6bed1a2f3 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Tue, 29 Mar 2022 16:05:24 -0400 Subject: [PATCH 2/5] fix: Don't prevent apply from running given duplicate empty names in data sources. Also fix repeated apply of Spark data source. (#2415) * fix: Print more warning statements on requirement for data sources to have name in future, but don't prevent apply from running if there are duplicate empty data sources. Also attach class type when applying data sources so repeated feast apply commands properly work for Spark Signed-off-by: Danny Chiao * typo Signed-off-by: Danny Chiao * typo Signed-off-by: Danny Chiao * fix Signed-off-by: Danny Chiao * fix Signed-off-by: Danny Chiao * fix Signed-off-by: Danny Chiao * More tests Signed-off-by: Danny Chiao * fix Signed-off-by: Danny Chiao * fix Signed-off-by: Danny Chiao * fix Signed-off-by: Danny Chiao * fix Signed-off-by: Danny Chiao * revert Signed-off-by: Danny Chiao * fix Signed-off-by: Danny Chiao * fix Signed-off-by: Danny Chiao * fix Signed-off-by: Danny Chiao --- protos/feast/core/SavedDataset.proto | 1 - sdk/python/feast/cli.py | 11 +++++++++++ sdk/python/feast/data_source.py | 5 +++++ sdk/python/feast/diff/registry_diff.py | 10 ++++++++-- sdk/python/feast/feature_store.py | 17 ++++++++++------- .../infra/offline_stores/bigquery_source.py | 2 +- .../infra/offline_stores/redshift_source.py | 16 ++++++++-------- .../infra/offline_stores/snowflake_source.py | 16 ++++++++-------- sdk/python/feast/registry.py | 4 +++- sdk/python/feast/repo_operations.py | 6 ++---- .../example_repos/example_feature_repo_1.py | 12 ++++++++++++ 11 files changed, 68 insertions(+), 32 deletions(-) diff --git a/protos/feast/core/SavedDataset.proto b/protos/feast/core/SavedDataset.proto index e6d103a691b..2e0f3885ed9 100644 --- a/protos/feast/core/SavedDataset.proto +++ b/protos/feast/core/SavedDataset.proto @@ -24,7 +24,6 @@ option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; import "google/protobuf/timestamp.proto"; import "feast/core/DataSource.proto"; -import "feast/core/FeatureService.proto"; message SavedDatasetSpec { // Name of the dataset. Must be unique since it's possible to overwrite dataset by name diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index d2a71bc561b..9d75e14c939 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import warnings from datetime import datetime from pathlib import Path from typing import List, Optional @@ -151,6 +152,11 @@ def data_source_describe(ctx: click.Context, name: str): print(e) exit(1) + warnings.warn( + "Describing data sources will only work properly if all data sources have names or table names specified. " + "Starting Feast 0.21, data source unique names will be required to encourage data source discovery.", + RuntimeWarning, + ) print( yaml.dump( yaml.safe_load(str(data_source)), default_flow_style=False, sort_keys=False @@ -173,6 +179,11 @@ def data_source_list(ctx: click.Context): from tabulate import tabulate + warnings.warn( + "Listing data sources will only work properly if all data sources have names or table names specified. " + "Starting Feast 0.21, data source unique names will be required to encourage data source discovery", + RuntimeWarning, + ) print(tabulate(table, headers=["NAME", "CLASS"], tablefmt="plain")) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 15ce0c23773..f23b1771e13 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -17,6 +17,8 @@ from abc import ABC, abstractmethod from typing import Any, Callable, Dict, Iterable, Optional, Tuple +from google.protobuf.json_format import MessageToJson + from feast import type_map from feast.data_format import StreamFormat from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto @@ -180,6 +182,9 @@ def __init__( def __hash__(self): return hash((id(self), self.name)) + def __str__(self): + return str(MessageToJson(self.to_proto())) + def __eq__(self, other): if not isinstance(other, DataSource): raise TypeError("Comparisons should only involve DataSource class objects.") diff --git a/sdk/python/feast/diff/registry_diff.py b/sdk/python/feast/diff/registry_diff.py index 4558a149a5c..10bd88c56f8 100644 --- a/sdk/python/feast/diff/registry_diff.py +++ b/sdk/python/feast/diff/registry_diff.py @@ -60,6 +60,9 @@ def to_string(self): continue if feast_object_diff.transition_type == TransitionType.UNCHANGED: continue + if feast_object_diff.feast_object_type == FeastObjectType.DATA_SOURCE: + # TODO(adchia): Print statements out starting in Feast 0.21 + continue action, color = message_action_map[feast_object_diff.transition_type] log_string += f"{action} {feast_object_diff.feast_object_type.value} {Style.BRIGHT + color}{feast_object_diff.name}{Style.RESET_ALL}\n" if feast_object_diff.transition_type == TransitionType.UPDATE: @@ -78,8 +81,11 @@ def to_string(self): def tag_objects_for_keep_delete_update_add( existing_objs: Iterable[FeastObject], desired_objs: Iterable[FeastObject] ) -> Tuple[Set[FeastObject], Set[FeastObject], Set[FeastObject], Set[FeastObject]]: - existing_obj_names = {e.name for e in existing_objs} - desired_obj_names = {e.name for e in desired_objs} + # TODO(adchia): Remove the "if X.name" condition when data sources are forced to have names + existing_obj_names = {e.name for e in existing_objs if e.name} + desired_objs = [obj for obj in desired_objs if obj.name] + existing_objs = [obj for obj in existing_objs if obj.name] + desired_obj_names = {e.name for e in desired_objs if e.name} objs_to_add = {e for e in desired_objs if e.name not in existing_obj_names} objs_to_update = {e for e in desired_objs if e.name in existing_obj_names} diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 19741bcf127..c1a4ec7a631 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1971,13 +1971,16 @@ def _validate_feature_views(feature_views: List[BaseFeatureView]): def _validate_data_sources(data_sources: List[DataSource]): """ Verify data sources have case-insensitively unique names""" ds_names = set() - for fv in data_sources: - case_insensitive_ds_name = fv.name.lower() + for ds in data_sources: + case_insensitive_ds_name = ds.name.lower() if case_insensitive_ds_name in ds_names: - raise ValueError( - f"More than one data source with name {case_insensitive_ds_name} found. " - f"Please ensure that all data source names are case-insensitively unique. " - f"It may be necessary to ignore certain files in your feature repository by using a .feastignore file." - ) + if case_insensitive_ds_name.strip(): + warnings.warn( + f"More than one data source with name {case_insensitive_ds_name} found. " + f"Please ensure that all data source names are case-insensitively unique. " + f"It may be necessary to ignore certain files in your feature repository by using a .feastignore " + f"file. Starting in Feast 0.21, unique names (perhaps inferred from the table name) will be " + f"required in data sources to encourage data source discovery" + ) else: ds_names.add(case_insensitive_ds_name) diff --git a/sdk/python/feast/infra/offline_stores/bigquery_source.py b/sdk/python/feast/infra/offline_stores/bigquery_source.py index 92b6939fc3a..1d797077f07 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery_source.py +++ b/sdk/python/feast/infra/offline_stores/bigquery_source.py @@ -64,7 +64,7 @@ def __init__( else: warnings.warn( ( - "Starting in Feast 0.21, Feast will require either a name for a data source (if using query) or `table`." + f"Starting in Feast 0.21, Feast will require either a name for a data source (if using query) or `table`: {self.query}" ), DeprecationWarning, ) diff --git a/sdk/python/feast/infra/offline_stores/redshift_source.py b/sdk/python/feast/infra/offline_stores/redshift_source.py index 8573396aca4..df42e18910c 100644 --- a/sdk/python/feast/infra/offline_stores/redshift_source.py +++ b/sdk/python/feast/infra/offline_stores/redshift_source.py @@ -41,6 +41,12 @@ def __init__( query (optional): The query to be executed to obtain the features. name (optional): Name for the source. Defaults to the table_ref if not specified. """ + # The default Redshift schema is named "public". + _schema = "public" if table and not schema else schema + self.redshift_options = RedshiftOptions( + table=table, schema=_schema, query=query + ) + if table is None and query is None: raise ValueError('No "table" argument provided.') _name = name @@ -50,7 +56,8 @@ def __init__( else: warnings.warn( ( - "Starting in Feast 0.21, Feast will require either a name for a data source (if using query) or `table`." + f"Starting in Feast 0.21, Feast will require either a name for a data source (if using query) " + f"or `table`: {self.query}" ), DeprecationWarning, ) @@ -63,13 +70,6 @@ def __init__( date_partition_column, ) - # The default Redshift schema is named "public". - _schema = "public" if table and not schema else schema - - self.redshift_options = RedshiftOptions( - table=table, schema=_schema, query=query - ) - @staticmethod def from_proto(data_source: DataSourceProto): """ diff --git a/sdk/python/feast/infra/offline_stores/snowflake_source.py b/sdk/python/feast/infra/offline_stores/snowflake_source.py index 6ca5df7d6fd..a972df191b1 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake_source.py +++ b/sdk/python/feast/infra/offline_stores/snowflake_source.py @@ -44,6 +44,12 @@ def __init__( """ if table is None and query is None: raise ValueError('No "table" argument provided.') + # The default Snowflake schema is named "PUBLIC". + _schema = "PUBLIC" if (database and table and not schema) else schema + + self.snowflake_options = SnowflakeOptions( + database=database, schema=_schema, table=table, query=query + ) # If no name, use the table as the default name _name = name @@ -53,7 +59,8 @@ def __init__( else: warnings.warn( ( - "Starting in Feast 0.21, Feast will require either a name for a data source (if using query) or `table`." + f"Starting in Feast 0.21, Feast will require either a name for a data source (if using query) " + f"or `table`: {self.query}" ), DeprecationWarning, ) @@ -66,13 +73,6 @@ def __init__( date_partition_column, ) - # The default Snowflake schema is named "PUBLIC". - _schema = "PUBLIC" if (database and table and not schema) else schema - - self.snowflake_options = SnowflakeOptions( - database=database, schema=_schema, table=table, query=query - ) - @staticmethod def from_proto(data_source: DataSourceProto): """ diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index cb1261d8c93..0f5657fb785 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -314,11 +314,13 @@ def apply_data_source( commit: Whether to immediately commit to the registry """ registry = self._prepare_registry_for_changes() - for idx, existing_data_source_proto in enumerate(registry.data_sources): if existing_data_source_proto.name == data_source.name: del registry.data_sources[idx] data_source_proto = data_source.to_proto() + data_source_proto.data_source_class_type = ( + f"{data_source.__class__.__module__}.{data_source.__class__.__name__}" + ) data_source_proto.project = project registry.data_sources.append(data_source_proto) if commit: diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 4bee79bd60a..fc49c73db18 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -185,10 +185,8 @@ def extract_objects_for_apply_delete(project, registry, repo): return ( all_to_apply, all_to_delete, - set( - objs_to_add[FeastObjectType.FEATURE_VIEW].union( - objs_to_update[FeastObjectType.FEATURE_VIEW] - ) + set(objs_to_add[FeastObjectType.FEATURE_VIEW]).union( + set(objs_to_update[FeastObjectType.FEATURE_VIEW]) ), objs_to_delete[FeastObjectType.FEATURE_VIEW], ) diff --git a/sdk/python/tests/example_repos/example_feature_repo_1.py b/sdk/python/tests/example_repos/example_feature_repo_1.py index 8179906fa45..8f7951854fe 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_1.py +++ b/sdk/python/tests/example_repos/example_feature_repo_1.py @@ -15,6 +15,18 @@ created_timestamp_column="created_timestamp", ) +driver_locations_source_query = BigQuerySource( + query="SELECT * from feast-oss.public.drivers", + event_timestamp_column="event_timestamp", + created_timestamp_column="created_timestamp", +) + +driver_locations_source_query_2 = BigQuerySource( + query="SELECT lat * 2 FROM feast-oss.public.drivers", + event_timestamp_column="event_timestamp", + created_timestamp_column="created_timestamp", +) + customer_profile_source = BigQuerySource( name="customer_profile_source", table_ref="feast-oss.public.customers", From ba22c286b3521e9554d63de8ae72ea5786f67b0e Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 4 Apr 2022 13:01:57 -0700 Subject: [PATCH 3/5] fix: Add spark to lambda dockerfile (#2480) * add spark to lambda dockerfile Signed-off-by: Achal Shah * add *args Signed-off-by: Achal Shah * Add *args Signed-off-by: Felix Wang * Pin protobuf==3.19.4 Signed-off-by: Felix Wang * Remove *args Signed-off-by: Felix Wang * Add a range Signed-off-by: Achal Shah * Add a todo Signed-off-by: Achal Shah * cleanup prints Signed-off-by: Achal Shah * lock deps Signed-off-by: Achal Shah * lock deps correctly Signed-off-by: Achal Shah * fix lint Signed-off-by: Achal Shah * fix lint take 2 Signed-off-by: Achal Shah * Undo general updates Signed-off-by: Achal Shah Co-authored-by: Felix Wang --- .../infra/feature_servers/aws_lambda/Dockerfile | 3 ++- sdk/python/feast/infra/offline_stores/bigquery.py | 8 +++++--- sdk/python/feast/proto_json.py | 8 +++++--- sdk/python/requirements/py3.7-ci-requirements.txt | 4 ++-- sdk/python/setup.py | 14 ++++++++++---- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile index 4d46abd3db2..7e7d188e14a 100644 --- a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile @@ -9,7 +9,8 @@ COPY protos protos COPY README.md README.md # Install Feast for AWS with Lambda dependencies -RUN pip3 install -e 'sdk/python[aws,redis]' +# TODO(achals): The additional spark deps should be removed. Details at https://github.com/feast-dev/feast/pull/2480. +RUN pip3 install -e 'sdk/python[aws,redis,spark]' RUN pip3 install -r sdk/python/feast/infra/feature_servers/aws_lambda/requirements.txt --target "${LAMBDA_TASK_ROOT}" # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 44e62d6ad1a..bc5137c8037 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -338,12 +338,14 @@ def to_bigquery( def _to_arrow_internal(self) -> pyarrow.Table: with self._query_generator() as query: - return self._execute_query(query).to_arrow() + q = self._execute_query(query=query) + assert q + return q.to_arrow() @log_exceptions_and_usage def _execute_query( self, query, job_config=None, timeout: int = 1800 - ) -> bigquery.job.query.QueryJob: + ) -> Optional[bigquery.job.query.QueryJob]: bq_job = self.client.query(query, job_config=job_config) if job_config and job_config.dry_run: @@ -426,7 +428,7 @@ def _get_table_reference_for_new_entity( dataset.location = dataset_location if dataset_location else "US" try: - client.get_dataset(dataset) + client.get_dataset(dataset.reference) except NotFound: # Only create the dataset if it does not exist client.create_dataset(dataset, exists_ok=True) diff --git a/sdk/python/feast/proto_json.py b/sdk/python/feast/proto_json.py index 549d7b6d148..44e004cb036 100644 --- a/sdk/python/feast/proto_json.py +++ b/sdk/python/feast/proto_json.py @@ -15,6 +15,8 @@ JsonObject = Any +# TODO: These methods need to be updated when bumping the version of protobuf. +# https://github.com/feast-dev/feast/issues/2484 def _patch_proto_json_encoding( proto_type: Type[ProtoMessage], to_json_object: Callable[[_Printer, ProtoMessage], JsonObject], @@ -68,7 +70,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject: return value def from_json_object( - parser: _Parser, value: JsonObject, message: ProtoMessage + parser: _Parser, value: JsonObject, message: ProtoMessage, ) -> None: if value is None: message.null_val = 0 @@ -140,7 +142,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject: return [printer._MessageToJsonObject(item) for item in message.val] def from_json_object( - parser: _Parser, value: JsonObject, message: ProtoMessage + parser: _Parser, value: JsonObject, message: ProtoMessage, ) -> None: array = value if isinstance(value, list) else value["val"] for item in array: @@ -181,7 +183,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject: return list(message.val) def from_json_object( - parser: _Parser, value: JsonObject, message: ProtoMessage + parser: _Parser, value: JsonObject, message: ProtoMessage, ) -> None: array = value if isinstance(value, list) else value["val"] message.val.extend(array) diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt index 6cb8c2931b8..5d2539a5373 100644 --- a/sdk/python/requirements/py3.7-ci-requirements.txt +++ b/sdk/python/requirements/py3.7-ci-requirements.txt @@ -400,7 +400,7 @@ mypy==0.931 # via feast (setup.py) mypy-extensions==0.4.3 # via mypy -mypy-protobuf==3.1.0 +mypy-protobuf==3.1 # via feast (setup.py) nbclient==0.5.11 # via nbconvert @@ -850,4 +850,4 @@ zipp==3.7.0 # The following packages are considered to be unsafe in a requirements file: # pip -# setuptools \ No newline at end of file +# setuptools diff --git a/sdk/python/setup.py b/sdk/python/setup.py index f95dd2b806b..96a9556f1e8 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -53,7 +53,7 @@ "mmh3", "pandas>=1.0.0", "pandavro==1.5.*", - "protobuf>=3.10", + "protobuf>=3.10,<3.20", "proto-plus<1.19.7", "pyarrow>=4.0.0", "pydantic>=1.0.0", @@ -112,7 +112,7 @@ "mock==2.0.0", "moto", "mypy==0.931", - "mypy-protobuf==3.1.0", + "mypy-protobuf==3.1", "avro==1.10.0", "gcsfs", "urllib3>=1.25.4", @@ -149,7 +149,7 @@ + GE_REQUIRED ) -DEV_REQUIRED = ["mypy-protobuf>=3.1.0", "grpcio-testing==1.*"] + CI_REQUIRED +DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED # Get git repo root directory repo_root = str(pathlib.Path(__file__).resolve().parent.parent.parent) @@ -264,7 +264,13 @@ def run(self): ], entry_points={"console_scripts": ["feast=feast.cli:cli"]}, use_scm_version=use_scm_version, - setup_requires=["setuptools_scm", "grpcio", "grpcio-tools==1.34.0", "mypy-protobuf==3.1.0", "sphinx!=4.0.0"], + setup_requires=[ + "setuptools_scm", + "grpcio", + "grpcio-tools==1.34.0", + "mypy-protobuf==3.1", + "sphinx!=4.0.0", + ], package_data={ "": [ "protos/feast/**/*.proto", From 2115bd0737c6a06912ef67607dce3dd2b9ce91fd Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Apr 2022 10:48:23 -0700 Subject: [PATCH 4/5] fix: Fix DataSource constructor to unbreak custom data sources (#2492) * fix: Fix DataSource constructor to unbreak custom data sources Signed-off-by: Achal Shah * fix first party refernces to use kwargs only Signed-off-by: Achal Shah * remove force kwargs Signed-off-by: Achal Shah --- sdk/python/feast/data_source.py | 49 +++++++++++++------ .../infra/offline_stores/bigquery_source.py | 10 ++-- .../spark_offline_store/spark_source.py | 10 ++-- .../feast/infra/offline_stores/file_source.py | 10 ++-- .../infra/offline_stores/redshift_source.py | 10 ++-- .../infra/offline_stores/snowflake_source.py | 10 ++-- 6 files changed, 60 insertions(+), 39 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index f23b1771e13..f8a28bf3a26 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -14,6 +14,7 @@ import enum +import warnings from abc import ABC, abstractmethod from typing import Any, Callable, Dict, Iterable, Optional, Tuple @@ -160,14 +161,34 @@ class DataSource(ABC): def __init__( self, - name: str, event_timestamp_column: Optional[str] = None, created_timestamp_column: Optional[str] = None, field_mapping: Optional[Dict[str, str]] = None, date_partition_column: Optional[str] = None, + name: Optional[str] = None, ): - """Creates a DataSource object.""" - self.name = name + """ + Creates a DataSource object. + Args: + name: Name of data source, which should be unique within a project + event_timestamp_column (optional): Event timestamp column used for point in time + joins of feature values. + created_timestamp_column (optional): Timestamp column indicating when the row + was created, used for deduplicating rows. + field_mapping (optional): A dictionary mapping of column names in this data + source to feature names in a feature table or view. Only used for feature + columns, not entity or timestamp columns. + date_partition_column (optional): Timestamp column used for partitioning. + """ + if not name: + warnings.warn( + ( + "Names for data sources need to be supplied. " + "Data sources without names will no tbe supported after Feast 0.23." + ), + UserWarning, + ) + self.name = name or "" self.event_timestamp_column = ( event_timestamp_column if event_timestamp_column else "" ) @@ -321,11 +342,11 @@ def __init__( date_partition_column: Optional[str] = "", ): super().__init__( - name, - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, + name=name, ) self.kafka_options = KafkaOptions( bootstrap_servers=bootstrap_servers, @@ -402,7 +423,7 @@ def __init__( self, name: str, schema: Dict[str, ValueType], ): """Creates a RequestDataSource object.""" - super().__init__(name) + super().__init__(name=name) self.schema = schema def validate(self, config: RepoConfig): @@ -485,11 +506,11 @@ def __init__( date_partition_column: Optional[str] = "", ): super().__init__( - name, - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + name=name, + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) self.kinesis_options = KinesisOptions( record_format=record_format, region=region, stream_name=stream_name diff --git a/sdk/python/feast/infra/offline_stores/bigquery_source.py b/sdk/python/feast/infra/offline_stores/bigquery_source.py index 1d797077f07..ddc594e2ac6 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery_source.py +++ b/sdk/python/feast/infra/offline_stores/bigquery_source.py @@ -70,11 +70,11 @@ def __init__( ) super().__init__( - _name if _name else "", - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + name=_name if _name else "", + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) # Note: Python requires redefining hash in child classes that override __eq__ diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py index 3ffdf6eda0c..b1cbf6ccd88 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py @@ -49,11 +49,11 @@ def __init__( else: raise DataSourceNoNameException() super().__init__( - _name, - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + name=_name if _name else "", + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) warnings.warn( "The spark data source API is an experimental feature in alpha development. " diff --git a/sdk/python/feast/infra/offline_stores/file_source.py b/sdk/python/feast/infra/offline_stores/file_source.py index 756ec2a65e0..0760f4b32b5 100644 --- a/sdk/python/feast/infra/offline_stores/file_source.py +++ b/sdk/python/feast/infra/offline_stores/file_source.py @@ -58,11 +58,11 @@ def __init__( ) super().__init__( - name if name else path, - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + name=name if name else path, + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) # Note: Python requires redefining hash in child classes that override __eq__ diff --git a/sdk/python/feast/infra/offline_stores/redshift_source.py b/sdk/python/feast/infra/offline_stores/redshift_source.py index df42e18910c..7a28c230372 100644 --- a/sdk/python/feast/infra/offline_stores/redshift_source.py +++ b/sdk/python/feast/infra/offline_stores/redshift_source.py @@ -63,11 +63,11 @@ def __init__( ) super().__init__( - _name if _name else "", - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + name=_name if _name else "", + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) @staticmethod diff --git a/sdk/python/feast/infra/offline_stores/snowflake_source.py b/sdk/python/feast/infra/offline_stores/snowflake_source.py index a972df191b1..23c58751dfc 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake_source.py +++ b/sdk/python/feast/infra/offline_stores/snowflake_source.py @@ -66,11 +66,11 @@ def __init__( ) super().__init__( - _name if _name else "", - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + name=_name if _name else "", + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) @staticmethod From 20e7a6fe2fabaf548611d23dd6dd7cbe471cb143 Mon Sep 17 00:00:00 2001 From: feast-ci-bot Date: Wed, 6 Apr 2022 18:47:16 +0000 Subject: [PATCH 5/5] chore(release): release 0.19.4 ## [0.19.4](https://github.com/feast-dev/feast/compare/v0.19.3...v0.19.4) (2022-04-06) ### Bug Fixes * Add spark to lambda dockerfile ([#2480](https://github.com/feast-dev/feast/issues/2480)) ([ba22c28](https://github.com/feast-dev/feast/commit/ba22c286b3521e9554d63de8ae72ea5786f67b0e)) * Don't prevent apply from running given duplicate empty names in data sources. Also fix repeated apply of Spark data source. ([#2415](https://github.com/feast-dev/feast/issues/2415)) ([88e01a2](https://github.com/feast-dev/feast/commit/88e01a2545f694534fb60feb0d55f1e6bed1a2f3)) * Fix DataSource constructor to unbreak custom data sources ([#2492](https://github.com/feast-dev/feast/issues/2492)) ([2115bd0](https://github.com/feast-dev/feast/commit/2115bd0737c6a06912ef67607dce3dd2b9ce91fd)) --- CHANGELOG.md | 9 +++++++++ infra/charts/feast-python-server/Chart.yaml | 2 +- infra/charts/feast-python-server/README.md | 2 +- infra/charts/feast/Chart.yaml | 2 +- infra/charts/feast/README.md | 6 +++--- infra/charts/feast/charts/feature-server/Chart.yaml | 4 ++-- infra/charts/feast/charts/feature-server/README.md | 4 ++-- infra/charts/feast/charts/feature-server/values.yaml | 2 +- .../feast/charts/transformation-service/Chart.yaml | 4 ++-- .../charts/feast/charts/transformation-service/README.md | 4 ++-- .../feast/charts/transformation-service/values.yaml | 2 +- infra/charts/feast/requirements.yaml | 4 ++-- java/pom.xml | 2 +- 13 files changed, 28 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c2dcf53d0e..d2dab154d2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.19.4](https://github.com/feast-dev/feast/compare/v0.19.3...v0.19.4) (2022-04-06) + + +### Bug Fixes + +* Add spark to lambda dockerfile ([#2480](https://github.com/feast-dev/feast/issues/2480)) ([ba22c28](https://github.com/feast-dev/feast/commit/ba22c286b3521e9554d63de8ae72ea5786f67b0e)) +* Don't prevent apply from running given duplicate empty names in data sources. Also fix repeated apply of Spark data source. ([#2415](https://github.com/feast-dev/feast/issues/2415)) ([88e01a2](https://github.com/feast-dev/feast/commit/88e01a2545f694534fb60feb0d55f1e6bed1a2f3)) +* Fix DataSource constructor to unbreak custom data sources ([#2492](https://github.com/feast-dev/feast/issues/2492)) ([2115bd0](https://github.com/feast-dev/feast/commit/2115bd0737c6a06912ef67607dce3dd2b9ce91fd)) + ## [0.19.3](https://github.com/feast-dev/feast/compare/v0.19.2...v0.19.3) (2022-03-09) diff --git a/infra/charts/feast-python-server/Chart.yaml b/infra/charts/feast-python-server/Chart.yaml index 96ba2653b70..67d3c60505f 100644 --- a/infra/charts/feast-python-server/Chart.yaml +++ b/infra/charts/feast-python-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-python-server description: Feast Feature Server in Python type: application -version: 0.19.3 +version: 0.19.4 keywords: - machine learning - big data diff --git a/infra/charts/feast-python-server/README.md b/infra/charts/feast-python-server/README.md index 75f89130732..c34b0cb1e27 100644 --- a/infra/charts/feast-python-server/README.md +++ b/infra/charts/feast-python-server/README.md @@ -1,6 +1,6 @@ # feast-python-server -![Version: 0.19.3](https://img.shields.io/badge/Version-0.19.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.19.4](https://img.shields.io/badge/Version-0.19.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) Feast Feature Server in Python diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index f32c8ce9280..c71de04ffdd 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.19.3 +version: 0.19.4 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 5afd84be134..14f4704b39c 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast components that are being installed on ## Chart: Feast -Feature store for machine learning Current chart version is `0.19.3` +Feature store for machine learning Current chart version is `0.19.4` ## Installation @@ -55,8 +55,8 @@ For more details, please see: https://docs.feast.dev/how-to-guides/running-feast | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.19.3 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.19.3 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.19.4 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.19.4 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index 40d13481c5a..aa457ebf530 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.19.3 -appVersion: v0.19.3 +version: 0.19.4 +appVersion: v0.19.4 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 01a645bc1ee..2ab564eeb47 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.19.3](https://img.shields.io/badge/Version-0.19.3-informational?style=flat-square) ![AppVersion: v0.19.3](https://img.shields.io/badge/AppVersion-v0.19.3-informational?style=flat-square) +![Version: 0.19.4](https://img.shields.io/badge/Version-0.19.4-informational?style=flat-square) ![AppVersion: v0.19.4](https://img.shields.io/badge/AppVersion-v0.19.4-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.19.3"` | Image tag | +| image.tag | string | `"0.19.4"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index e0c18dd7ac6..21cd72a7d50 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.19.3 + tag: 0.19.4 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 934e6ee465c..d571ab66f54 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.19.3 -appVersion: v0.19.3 +version: 0.19.4 +appVersion: v0.19.4 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index b70991e60d6..5672cda4713 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.19.3](https://img.shields.io/badge/Version-0.19.3-informational?style=flat-square) ![AppVersion: v0.19.3](https://img.shields.io/badge/AppVersion-v0.19.3-informational?style=flat-square) +![Version: 0.19.4](https://img.shields.io/badge/Version-0.19.4-informational?style=flat-square) ![AppVersion: v0.19.4](https://img.shields.io/badge/AppVersion-v0.19.4-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.19.3"` | Image tag | +| image.tag | string | `"0.19.4"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index f7d910df650..b8893962f61 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.19.3 + tag: 0.19.4 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index d08eff92e47..279baf104d8 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.19.3 + version: 0.19.4 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.19.3 + version: 0.19.4 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/java/pom.xml b/java/pom.xml index db39e9d9e87..d3be6d80c84 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -38,7 +38,7 @@ - 0.19.3 + 0.19.4 https://github.com/feast-dev/feast UTF-8