diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 04313c4120c..a856fbe2cba 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v2 with: submodules: 'true' - - uses: google-github-actions/setup-gcloud@master + - uses: google-github-actions/setup-gcloud@v0 with: version: '290.0.1' export_default_credentials: true diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index e7a89815fe4..3e5c690598e 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -173,7 +173,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0d7476a82ba..93385dcde9b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -64,7 +64,7 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} @@ -107,7 +107,7 @@ jobs: VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} steps: - uses: actions/checkout@v2 - - uses: google-github-actions/setup-gcloud@master + - uses: google-github-actions/setup-gcloud@v0 with: version: '290.0.1' export_default_credentials: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c2dcf53d0e..d2dab154d2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.19.4](https://github.com/feast-dev/feast/compare/v0.19.3...v0.19.4) (2022-04-06) + + +### Bug Fixes + +* Add spark to lambda dockerfile ([#2480](https://github.com/feast-dev/feast/issues/2480)) ([ba22c28](https://github.com/feast-dev/feast/commit/ba22c286b3521e9554d63de8ae72ea5786f67b0e)) +* Don't prevent apply from running given duplicate empty names in data sources. Also fix repeated apply of Spark data source. ([#2415](https://github.com/feast-dev/feast/issues/2415)) ([88e01a2](https://github.com/feast-dev/feast/commit/88e01a2545f694534fb60feb0d55f1e6bed1a2f3)) +* Fix DataSource constructor to unbreak custom data sources ([#2492](https://github.com/feast-dev/feast/issues/2492)) ([2115bd0](https://github.com/feast-dev/feast/commit/2115bd0737c6a06912ef67607dce3dd2b9ce91fd)) + ## [0.19.3](https://github.com/feast-dev/feast/compare/v0.19.2...v0.19.3) (2022-03-09) diff --git a/infra/charts/feast-python-server/Chart.yaml b/infra/charts/feast-python-server/Chart.yaml index 96ba2653b70..67d3c60505f 100644 --- a/infra/charts/feast-python-server/Chart.yaml +++ b/infra/charts/feast-python-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-python-server description: Feast Feature Server in Python type: application -version: 0.19.3 +version: 0.19.4 keywords: - machine learning - big data diff --git a/infra/charts/feast-python-server/README.md b/infra/charts/feast-python-server/README.md index 75f89130732..c34b0cb1e27 100644 --- a/infra/charts/feast-python-server/README.md +++ b/infra/charts/feast-python-server/README.md @@ -1,6 +1,6 @@ # feast-python-server -![Version: 0.19.3](https://img.shields.io/badge/Version-0.19.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.19.4](https://img.shields.io/badge/Version-0.19.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) Feast Feature Server in Python diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index f32c8ce9280..c71de04ffdd 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.19.3 +version: 0.19.4 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 5afd84be134..14f4704b39c 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast components that are being installed on ## Chart: Feast -Feature store for machine learning Current chart version is `0.19.3` +Feature store for machine learning Current chart version is `0.19.4` ## Installation @@ -55,8 +55,8 @@ For more details, please see: https://docs.feast.dev/how-to-guides/running-feast | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.19.3 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.19.3 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.19.4 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.19.4 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index 40d13481c5a..aa457ebf530 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.19.3 -appVersion: v0.19.3 +version: 0.19.4 +appVersion: v0.19.4 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 01a645bc1ee..2ab564eeb47 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.19.3](https://img.shields.io/badge/Version-0.19.3-informational?style=flat-square) ![AppVersion: v0.19.3](https://img.shields.io/badge/AppVersion-v0.19.3-informational?style=flat-square) +![Version: 0.19.4](https://img.shields.io/badge/Version-0.19.4-informational?style=flat-square) ![AppVersion: v0.19.4](https://img.shields.io/badge/AppVersion-v0.19.4-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.19.3"` | Image tag | +| image.tag | string | `"0.19.4"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index e0c18dd7ac6..21cd72a7d50 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.19.3 + tag: 0.19.4 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 934e6ee465c..d571ab66f54 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.19.3 -appVersion: v0.19.3 +version: 0.19.4 +appVersion: v0.19.4 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index b70991e60d6..5672cda4713 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.19.3](https://img.shields.io/badge/Version-0.19.3-informational?style=flat-square) ![AppVersion: v0.19.3](https://img.shields.io/badge/AppVersion-v0.19.3-informational?style=flat-square) +![Version: 0.19.4](https://img.shields.io/badge/Version-0.19.4-informational?style=flat-square) ![AppVersion: v0.19.4](https://img.shields.io/badge/AppVersion-v0.19.4-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.19.3"` | Image tag | +| image.tag | string | `"0.19.4"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index f7d910df650..b8893962f61 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.19.3 + tag: 0.19.4 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index d08eff92e47..279baf104d8 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.19.3 + version: 0.19.4 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.19.3 + version: 0.19.4 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/java/pom.xml b/java/pom.xml index db39e9d9e87..d3be6d80c84 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -38,7 +38,7 @@ - 0.19.3 + 0.19.4 https://github.com/feast-dev/feast UTF-8 diff --git a/protos/feast/core/SavedDataset.proto b/protos/feast/core/SavedDataset.proto index e6d103a691b..2e0f3885ed9 100644 --- a/protos/feast/core/SavedDataset.proto +++ b/protos/feast/core/SavedDataset.proto @@ -24,7 +24,6 @@ option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; import "google/protobuf/timestamp.proto"; import "feast/core/DataSource.proto"; -import "feast/core/FeatureService.proto"; message SavedDatasetSpec { // Name of the dataset. Must be unique since it's possible to overwrite dataset by name diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index d2a71bc561b..9d75e14c939 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import warnings from datetime import datetime from pathlib import Path from typing import List, Optional @@ -151,6 +152,11 @@ def data_source_describe(ctx: click.Context, name: str): print(e) exit(1) + warnings.warn( + "Describing data sources will only work properly if all data sources have names or table names specified. " + "Starting Feast 0.21, data source unique names will be required to encourage data source discovery.", + RuntimeWarning, + ) print( yaml.dump( yaml.safe_load(str(data_source)), default_flow_style=False, sort_keys=False @@ -173,6 +179,11 @@ def data_source_list(ctx: click.Context): from tabulate import tabulate + warnings.warn( + "Listing data sources will only work properly if all data sources have names or table names specified. " + "Starting Feast 0.21, data source unique names will be required to encourage data source discovery", + RuntimeWarning, + ) print(tabulate(table, headers=["NAME", "CLASS"], tablefmt="plain")) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 15ce0c23773..f8a28bf3a26 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -14,9 +14,12 @@ import enum +import warnings from abc import ABC, abstractmethod from typing import Any, Callable, Dict, Iterable, Optional, Tuple +from google.protobuf.json_format import MessageToJson + from feast import type_map from feast.data_format import StreamFormat from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto @@ -158,14 +161,34 @@ class DataSource(ABC): def __init__( self, - name: str, event_timestamp_column: Optional[str] = None, created_timestamp_column: Optional[str] = None, field_mapping: Optional[Dict[str, str]] = None, date_partition_column: Optional[str] = None, + name: Optional[str] = None, ): - """Creates a DataSource object.""" - self.name = name + """ + Creates a DataSource object. + Args: + name: Name of data source, which should be unique within a project + event_timestamp_column (optional): Event timestamp column used for point in time + joins of feature values. + created_timestamp_column (optional): Timestamp column indicating when the row + was created, used for deduplicating rows. + field_mapping (optional): A dictionary mapping of column names in this data + source to feature names in a feature table or view. Only used for feature + columns, not entity or timestamp columns. + date_partition_column (optional): Timestamp column used for partitioning. + """ + if not name: + warnings.warn( + ( + "Names for data sources need to be supplied. " + "Data sources without names will no tbe supported after Feast 0.23." + ), + UserWarning, + ) + self.name = name or "" self.event_timestamp_column = ( event_timestamp_column if event_timestamp_column else "" ) @@ -180,6 +203,9 @@ def __init__( def __hash__(self): return hash((id(self), self.name)) + def __str__(self): + return str(MessageToJson(self.to_proto())) + def __eq__(self, other): if not isinstance(other, DataSource): raise TypeError("Comparisons should only involve DataSource class objects.") @@ -316,11 +342,11 @@ def __init__( date_partition_column: Optional[str] = "", ): super().__init__( - name, - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, + name=name, ) self.kafka_options = KafkaOptions( bootstrap_servers=bootstrap_servers, @@ -397,7 +423,7 @@ def __init__( self, name: str, schema: Dict[str, ValueType], ): """Creates a RequestDataSource object.""" - super().__init__(name) + super().__init__(name=name) self.schema = schema def validate(self, config: RepoConfig): @@ -480,11 +506,11 @@ def __init__( date_partition_column: Optional[str] = "", ): super().__init__( - name, - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + name=name, + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) self.kinesis_options = KinesisOptions( record_format=record_format, region=region, stream_name=stream_name diff --git a/sdk/python/feast/diff/registry_diff.py b/sdk/python/feast/diff/registry_diff.py index 4558a149a5c..10bd88c56f8 100644 --- a/sdk/python/feast/diff/registry_diff.py +++ b/sdk/python/feast/diff/registry_diff.py @@ -60,6 +60,9 @@ def to_string(self): continue if feast_object_diff.transition_type == TransitionType.UNCHANGED: continue + if feast_object_diff.feast_object_type == FeastObjectType.DATA_SOURCE: + # TODO(adchia): Print statements out starting in Feast 0.21 + continue action, color = message_action_map[feast_object_diff.transition_type] log_string += f"{action} {feast_object_diff.feast_object_type.value} {Style.BRIGHT + color}{feast_object_diff.name}{Style.RESET_ALL}\n" if feast_object_diff.transition_type == TransitionType.UPDATE: @@ -78,8 +81,11 @@ def to_string(self): def tag_objects_for_keep_delete_update_add( existing_objs: Iterable[FeastObject], desired_objs: Iterable[FeastObject] ) -> Tuple[Set[FeastObject], Set[FeastObject], Set[FeastObject], Set[FeastObject]]: - existing_obj_names = {e.name for e in existing_objs} - desired_obj_names = {e.name for e in desired_objs} + # TODO(adchia): Remove the "if X.name" condition when data sources are forced to have names + existing_obj_names = {e.name for e in existing_objs if e.name} + desired_objs = [obj for obj in desired_objs if obj.name] + existing_objs = [obj for obj in existing_objs if obj.name] + desired_obj_names = {e.name for e in desired_objs if e.name} objs_to_add = {e for e in desired_objs if e.name not in existing_obj_names} objs_to_update = {e for e in desired_objs if e.name in existing_obj_names} diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 19741bcf127..c1a4ec7a631 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1971,13 +1971,16 @@ def _validate_feature_views(feature_views: List[BaseFeatureView]): def _validate_data_sources(data_sources: List[DataSource]): """ Verify data sources have case-insensitively unique names""" ds_names = set() - for fv in data_sources: - case_insensitive_ds_name = fv.name.lower() + for ds in data_sources: + case_insensitive_ds_name = ds.name.lower() if case_insensitive_ds_name in ds_names: - raise ValueError( - f"More than one data source with name {case_insensitive_ds_name} found. " - f"Please ensure that all data source names are case-insensitively unique. " - f"It may be necessary to ignore certain files in your feature repository by using a .feastignore file." - ) + if case_insensitive_ds_name.strip(): + warnings.warn( + f"More than one data source with name {case_insensitive_ds_name} found. " + f"Please ensure that all data source names are case-insensitively unique. " + f"It may be necessary to ignore certain files in your feature repository by using a .feastignore " + f"file. Starting in Feast 0.21, unique names (perhaps inferred from the table name) will be " + f"required in data sources to encourage data source discovery" + ) else: ds_names.add(case_insensitive_ds_name) diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile index 4d46abd3db2..7e7d188e14a 100644 --- a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile @@ -9,7 +9,8 @@ COPY protos protos COPY README.md README.md # Install Feast for AWS with Lambda dependencies -RUN pip3 install -e 'sdk/python[aws,redis]' +# TODO(achals): The additional spark deps should be removed. Details at https://github.com/feast-dev/feast/pull/2480. +RUN pip3 install -e 'sdk/python[aws,redis,spark]' RUN pip3 install -r sdk/python/feast/infra/feature_servers/aws_lambda/requirements.txt --target "${LAMBDA_TASK_ROOT}" # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 44e62d6ad1a..bc5137c8037 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -338,12 +338,14 @@ def to_bigquery( def _to_arrow_internal(self) -> pyarrow.Table: with self._query_generator() as query: - return self._execute_query(query).to_arrow() + q = self._execute_query(query=query) + assert q + return q.to_arrow() @log_exceptions_and_usage def _execute_query( self, query, job_config=None, timeout: int = 1800 - ) -> bigquery.job.query.QueryJob: + ) -> Optional[bigquery.job.query.QueryJob]: bq_job = self.client.query(query, job_config=job_config) if job_config and job_config.dry_run: @@ -426,7 +428,7 @@ def _get_table_reference_for_new_entity( dataset.location = dataset_location if dataset_location else "US" try: - client.get_dataset(dataset) + client.get_dataset(dataset.reference) except NotFound: # Only create the dataset if it does not exist client.create_dataset(dataset, exists_ok=True) diff --git a/sdk/python/feast/infra/offline_stores/bigquery_source.py b/sdk/python/feast/infra/offline_stores/bigquery_source.py index 92b6939fc3a..ddc594e2ac6 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery_source.py +++ b/sdk/python/feast/infra/offline_stores/bigquery_source.py @@ -64,17 +64,17 @@ def __init__( else: warnings.warn( ( - "Starting in Feast 0.21, Feast will require either a name for a data source (if using query) or `table`." + f"Starting in Feast 0.21, Feast will require either a name for a data source (if using query) or `table`: {self.query}" ), DeprecationWarning, ) super().__init__( - _name if _name else "", - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + name=_name if _name else "", + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) # Note: Python requires redefining hash in child classes that override __eq__ diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py index 3ffdf6eda0c..b1cbf6ccd88 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py @@ -49,11 +49,11 @@ def __init__( else: raise DataSourceNoNameException() super().__init__( - _name, - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + name=_name if _name else "", + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) warnings.warn( "The spark data source API is an experimental feature in alpha development. " diff --git a/sdk/python/feast/infra/offline_stores/file_source.py b/sdk/python/feast/infra/offline_stores/file_source.py index 756ec2a65e0..0760f4b32b5 100644 --- a/sdk/python/feast/infra/offline_stores/file_source.py +++ b/sdk/python/feast/infra/offline_stores/file_source.py @@ -58,11 +58,11 @@ def __init__( ) super().__init__( - name if name else path, - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, + name=name if name else path, + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) # Note: Python requires redefining hash in child classes that override __eq__ diff --git a/sdk/python/feast/infra/offline_stores/redshift_source.py b/sdk/python/feast/infra/offline_stores/redshift_source.py index 8573396aca4..7a28c230372 100644 --- a/sdk/python/feast/infra/offline_stores/redshift_source.py +++ b/sdk/python/feast/infra/offline_stores/redshift_source.py @@ -41,6 +41,12 @@ def __init__( query (optional): The query to be executed to obtain the features. name (optional): Name for the source. Defaults to the table_ref if not specified. """ + # The default Redshift schema is named "public". + _schema = "public" if table and not schema else schema + self.redshift_options = RedshiftOptions( + table=table, schema=_schema, query=query + ) + if table is None and query is None: raise ValueError('No "table" argument provided.') _name = name @@ -50,24 +56,18 @@ def __init__( else: warnings.warn( ( - "Starting in Feast 0.21, Feast will require either a name for a data source (if using query) or `table`." + f"Starting in Feast 0.21, Feast will require either a name for a data source (if using query) " + f"or `table`: {self.query}" ), DeprecationWarning, ) super().__init__( - _name if _name else "", - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, - ) - - # The default Redshift schema is named "public". - _schema = "public" if table and not schema else schema - - self.redshift_options = RedshiftOptions( - table=table, schema=_schema, query=query + name=_name if _name else "", + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) @staticmethod diff --git a/sdk/python/feast/infra/offline_stores/snowflake_source.py b/sdk/python/feast/infra/offline_stores/snowflake_source.py index 6ca5df7d6fd..23c58751dfc 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake_source.py +++ b/sdk/python/feast/infra/offline_stores/snowflake_source.py @@ -44,6 +44,12 @@ def __init__( """ if table is None and query is None: raise ValueError('No "table" argument provided.') + # The default Snowflake schema is named "PUBLIC". + _schema = "PUBLIC" if (database and table and not schema) else schema + + self.snowflake_options = SnowflakeOptions( + database=database, schema=_schema, table=table, query=query + ) # If no name, use the table as the default name _name = name @@ -53,24 +59,18 @@ def __init__( else: warnings.warn( ( - "Starting in Feast 0.21, Feast will require either a name for a data source (if using query) or `table`." + f"Starting in Feast 0.21, Feast will require either a name for a data source (if using query) " + f"or `table`: {self.query}" ), DeprecationWarning, ) super().__init__( - _name if _name else "", - event_timestamp_column, - created_timestamp_column, - field_mapping, - date_partition_column, - ) - - # The default Snowflake schema is named "PUBLIC". - _schema = "PUBLIC" if (database and table and not schema) else schema - - self.snowflake_options = SnowflakeOptions( - database=database, schema=_schema, table=table, query=query + name=_name if _name else "", + event_timestamp_column=event_timestamp_column, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + date_partition_column=date_partition_column, ) @staticmethod diff --git a/sdk/python/feast/proto_json.py b/sdk/python/feast/proto_json.py index 549d7b6d148..44e004cb036 100644 --- a/sdk/python/feast/proto_json.py +++ b/sdk/python/feast/proto_json.py @@ -15,6 +15,8 @@ JsonObject = Any +# TODO: These methods need to be updated when bumping the version of protobuf. +# https://github.com/feast-dev/feast/issues/2484 def _patch_proto_json_encoding( proto_type: Type[ProtoMessage], to_json_object: Callable[[_Printer, ProtoMessage], JsonObject], @@ -68,7 +70,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject: return value def from_json_object( - parser: _Parser, value: JsonObject, message: ProtoMessage + parser: _Parser, value: JsonObject, message: ProtoMessage, ) -> None: if value is None: message.null_val = 0 @@ -140,7 +142,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject: return [printer._MessageToJsonObject(item) for item in message.val] def from_json_object( - parser: _Parser, value: JsonObject, message: ProtoMessage + parser: _Parser, value: JsonObject, message: ProtoMessage, ) -> None: array = value if isinstance(value, list) else value["val"] for item in array: @@ -181,7 +183,7 @@ def to_json_object(printer: _Printer, message: ProtoMessage) -> JsonObject: return list(message.val) def from_json_object( - parser: _Parser, value: JsonObject, message: ProtoMessage + parser: _Parser, value: JsonObject, message: ProtoMessage, ) -> None: array = value if isinstance(value, list) else value["val"] message.val.extend(array) diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index cb1261d8c93..0f5657fb785 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -314,11 +314,13 @@ def apply_data_source( commit: Whether to immediately commit to the registry """ registry = self._prepare_registry_for_changes() - for idx, existing_data_source_proto in enumerate(registry.data_sources): if existing_data_source_proto.name == data_source.name: del registry.data_sources[idx] data_source_proto = data_source.to_proto() + data_source_proto.data_source_class_type = ( + f"{data_source.__class__.__module__}.{data_source.__class__.__name__}" + ) data_source_proto.project = project registry.data_sources.append(data_source_proto) if commit: diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 4bee79bd60a..fc49c73db18 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -185,10 +185,8 @@ def extract_objects_for_apply_delete(project, registry, repo): return ( all_to_apply, all_to_delete, - set( - objs_to_add[FeastObjectType.FEATURE_VIEW].union( - objs_to_update[FeastObjectType.FEATURE_VIEW] - ) + set(objs_to_add[FeastObjectType.FEATURE_VIEW]).union( + set(objs_to_update[FeastObjectType.FEATURE_VIEW]) ), objs_to_delete[FeastObjectType.FEATURE_VIEW], ) diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt index 6cb8c2931b8..5d2539a5373 100644 --- a/sdk/python/requirements/py3.7-ci-requirements.txt +++ b/sdk/python/requirements/py3.7-ci-requirements.txt @@ -400,7 +400,7 @@ mypy==0.931 # via feast (setup.py) mypy-extensions==0.4.3 # via mypy -mypy-protobuf==3.1.0 +mypy-protobuf==3.1 # via feast (setup.py) nbclient==0.5.11 # via nbconvert @@ -850,4 +850,4 @@ zipp==3.7.0 # The following packages are considered to be unsafe in a requirements file: # pip -# setuptools \ No newline at end of file +# setuptools diff --git a/sdk/python/setup.py b/sdk/python/setup.py index f95dd2b806b..96a9556f1e8 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -53,7 +53,7 @@ "mmh3", "pandas>=1.0.0", "pandavro==1.5.*", - "protobuf>=3.10", + "protobuf>=3.10,<3.20", "proto-plus<1.19.7", "pyarrow>=4.0.0", "pydantic>=1.0.0", @@ -112,7 +112,7 @@ "mock==2.0.0", "moto", "mypy==0.931", - "mypy-protobuf==3.1.0", + "mypy-protobuf==3.1", "avro==1.10.0", "gcsfs", "urllib3>=1.25.4", @@ -149,7 +149,7 @@ + GE_REQUIRED ) -DEV_REQUIRED = ["mypy-protobuf>=3.1.0", "grpcio-testing==1.*"] + CI_REQUIRED +DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED # Get git repo root directory repo_root = str(pathlib.Path(__file__).resolve().parent.parent.parent) @@ -264,7 +264,13 @@ def run(self): ], entry_points={"console_scripts": ["feast=feast.cli:cli"]}, use_scm_version=use_scm_version, - setup_requires=["setuptools_scm", "grpcio", "grpcio-tools==1.34.0", "mypy-protobuf==3.1.0", "sphinx!=4.0.0"], + setup_requires=[ + "setuptools_scm", + "grpcio", + "grpcio-tools==1.34.0", + "mypy-protobuf==3.1", + "sphinx!=4.0.0", + ], package_data={ "": [ "protos/feast/**/*.proto", diff --git a/sdk/python/tests/example_repos/example_feature_repo_1.py b/sdk/python/tests/example_repos/example_feature_repo_1.py index 8179906fa45..8f7951854fe 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_1.py +++ b/sdk/python/tests/example_repos/example_feature_repo_1.py @@ -15,6 +15,18 @@ created_timestamp_column="created_timestamp", ) +driver_locations_source_query = BigQuerySource( + query="SELECT * from feast-oss.public.drivers", + event_timestamp_column="event_timestamp", + created_timestamp_column="created_timestamp", +) + +driver_locations_source_query_2 = BigQuerySource( + query="SELECT lat * 2 FROM feast-oss.public.drivers", + event_timestamp_column="event_timestamp", + created_timestamp_column="created_timestamp", +) + customer_profile_source = BigQuerySource( name="customer_profile_source", table_ref="feast-oss.public.customers",